[vm, gc] Make mutators perform incremental marking based on how much they actually allocated.

TEST=ci
Change-Id: I61013050f0d3cdbf54b8077a6d375ca16841398b
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/319601
Commit-Queue: Ryan Macnak <rmacnak@google.com>
Reviewed-by: Siva Annamalai <asiva@google.com>

runtime/docs/glossary.md

@@ -40,6 +40,10 @@ A point at which all pointers into the Dart heap can be precisely identified.
 
 An indirect pointer to a Dart object.
 
+## Thread-local allocation buffer, TLAB
+
+A contiguous area owned by one thread for allocation, allowing it to bump allocate without locking.
+
 ## [Stack map](../vm/object.h#:~:text=class%20CompressedStackMaps)
 
 Identifies which slots in a stack frame contain objects (to be visited by the GC) and which contain raw bits (to be ignored by the GC) for each return address.
@@ -63,7 +67,7 @@ Having a large number of possible types.
 
 ## [Ahead-of-time compilation, AOT](https://en.wikipedia.org/wiki/Ahead-of-time_compilation)
 
-Compiling an program in a separate process from its execution. Uses conservative optimizations based on a closed-world assumption and whole-program analysis.
+Compiling a program in a separate process from its execution. Uses conservative optimizations based on a closed-world assumption and whole-program analysis.
 
 ## [Just-in-time compilation, JIT](https://en.wikipedia.org/wiki/Just-in-time_compilation)
 

runtime/vm/heap/heap.cc

@@ -612,7 +612,7 @@ void Heap::CheckConcurrentMarking(Thread* thread,
 
   switch (phase) {
     case PageSpace::kMarking:
-      if ((size != 0) && (mode_ != Dart_PerformanceMode_Latency)) {
+      if (mode_ != Dart_PerformanceMode_Latency) {
         old_space_.IncrementalMarkWithSizeBudget(size);
       }
       return;

runtime/vm/heap/marker.cc

@@ -994,6 +994,11 @@ void GCMarker::IncrementalMarkWithUnlimitedBudget(PageSpace* page_space) {
 
 void GCMarker::IncrementalMarkWithSizeBudget(PageSpace* page_space,
                                              intptr_t size) {
+  // Avoid setup overhead for tiny amounts of marking as the last bits of TLABs
+  // get filled in.
+  const intptr_t kMinimumMarkingStep = KB;
+  if (size < kMinimumMarkingStep) return;
+
   TIMELINE_FUNCTION_GC_DURATION(Thread::Current(),
                                 "IncrementalMarkWithSizeBudget");
 

runtime/vm/heap/page.h

@@ -228,16 +228,20 @@ class Page {
     thread->set_end(end_);
     thread->set_true_end(end_);
   }
-  void Release(Thread* thread) {
+  intptr_t Release(Thread* thread) {
     ASSERT(owner_ == thread);
     owner_ = nullptr;
-    top_ = thread->top();
+    uword old_top = top_;
+    uword new_top = thread->top();
+    top_ = new_top;
     thread->set_top(0);
     thread->set_end(0);
     thread->set_true_end(0);
 #if !defined(PRODUCT) || defined(FORCE_INCLUDE_SAMPLING_HEAP_PROFILER)
     thread->heap_sampler().HandleReleasedTLAB(Thread::Current());
 #endif
+    ASSERT(new_top >= old_top);
+    return new_top - old_top;
   }
   void Release() {
     if (owner_ != nullptr) {

runtime/vm/heap/scavenger.cc

@@ -1556,10 +1556,10 @@ void Scavenger::TryAllocateNewTLAB(Thread* thread,
   }
 #endif
 
-  AbandonRemainingTLAB(thread);
+  intptr_t allocated = AbandonRemainingTLAB(thread);
   if (can_safepoint && !thread->force_growth()) {
     ASSERT(thread->no_safepoint_scope_depth() == 0);
-    heap_->CheckConcurrentMarking(thread, GCReason::kNewSpace, kPageSize);
+    heap_->CheckConcurrentMarking(thread, GCReason::kNewSpace, allocated);
   }
 
   MutexLocker ml(&space_lock_);
@@ -1599,14 +1599,17 @@ void Scavenger::AbandonRemainingTLABForDebugging(Thread* thread) {
   AbandonRemainingTLAB(thread);
 }
 
-void Scavenger::AbandonRemainingTLAB(Thread* thread) {
-  if (thread->top() == 0) return;
+intptr_t Scavenger::AbandonRemainingTLAB(Thread* thread) {
+  if (thread->top() == 0) return 0;
+
   Page* page = Page::Of(thread->top() - 1);
+  intptr_t allocated;
   {
     MutexLocker ml(&space_lock_);
-    page->Release(thread);
+    allocated = page->Release(thread);
   }
   ASSERT(thread->top() == 0);
+  return allocated;
 }
 
 template <bool parallel>

runtime/vm/heap/scavenger.h

@@ -146,7 +146,7 @@ class Scavenger {
     TryAllocateNewTLAB(thread, size, false);
     return TryAllocateFromTLAB(thread, size);
   }
-  void AbandonRemainingTLAB(Thread* thread);
+  intptr_t AbandonRemainingTLAB(Thread* thread);
   void AbandonRemainingTLABForDebugging(Thread* thread);
 
   // Collect the garbage in this scavenger.