[vm] Reduce TLS access during runtime allocation.

TEST=ci
Change-Id: Iec212406bc0af93699b08a3c28394551647646b6
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/241202
Reviewed-by: Martin Kustermann <kustermann@google.com>
Commit-Queue: Ryan Macnak <rmacnak@google.com>

runtime/vm/dart_entry.cc

@@ -169,7 +169,7 @@ ObjectPtr DartEntry::InvokeCode(const Code& code,
                                 Thread* thread) {
   ASSERT(!code.IsNull());
   ASSERT(thread->no_callback_scope_depth() == 0);
-  ASSERT(!IsolateGroup::Current()->null_safety_not_set());
+  ASSERT(!thread->isolate_group()->null_safety_not_set());
 
   const uword stub = StubCode::InvokeDartCode().EntryPoint();
   SuspendLongJumpScope suspend_long_jump_scope(thread);

runtime/vm/heap/heap.cc

@@ -67,10 +67,9 @@ Heap::~Heap() {
   }
 }
 
-uword Heap::AllocateNew(intptr_t size) {
-  ASSERT(Thread::Current()->no_safepoint_scope_depth() == 0);
-  CollectForDebugging();
-  Thread* thread = Thread::Current();
+uword Heap::AllocateNew(Thread* thread, intptr_t size) {
+  ASSERT(thread->no_safepoint_scope_depth() == 0);
+  CollectForDebugging(thread);
   uword addr = new_space_.TryAllocate(thread, size);
   if (LIKELY(addr != 0)) {
     return addr;
@@ -89,18 +88,17 @@ uword Heap::AllocateNew(intptr_t size) {
 
   // It is possible a GC doesn't clear enough space.
   // In that case, we must fall through and allocate into old space.
-  return AllocateOld(size, OldPage::kData);
+  return AllocateOld(thread, size, OldPage::kData);
 }
 
-uword Heap::AllocateOld(intptr_t size, OldPage::PageType type) {
-  ASSERT(Thread::Current()->no_safepoint_scope_depth() == 0);
+uword Heap::AllocateOld(Thread* thread, intptr_t size, OldPage::PageType type) {
+  ASSERT(thread->no_safepoint_scope_depth() == 0);
   if (old_space_.GrowthControlState()) {
-    CollectForDebugging();
+    CollectForDebugging(thread);
     uword addr = old_space_.TryAllocate(size, type);
     if (addr != 0) {
       return addr;
     }
-    Thread* thread = Thread::Current();
     // Wait for any GC tasks that are in progress.
     WaitForSweeperTasks(thread);
     addr = old_space_.TryAllocate(size, type);
@@ -135,7 +133,7 @@ uword Heap::AllocateOld(intptr_t size, OldPage::PageType type) {
   }
 
   if (old_space_.GrowthControlState()) {
-    WaitForSweeperTasks(Thread::Current());
+    WaitForSweeperTasks(thread);
     old_space_.TryReleaseReservation();
   } else {
     // We may or may not be a safepoint, so we don't know how to wait for the
@@ -428,7 +426,7 @@ void Heap::NotifyIdle(int64_t deadline) {
 void Heap::CollectNewSpaceGarbage(Thread* thread,
                                   GCType type,
                                   GCReason reason) {
-  NoActiveIsolateScope no_active_isolate_scope;
+  NoActiveIsolateScope no_active_isolate_scope(thread);
   ASSERT(reason != GCReason::kPromotion);
   ASSERT(reason != GCReason::kFinalize);
   if (thread->isolate_group() == Dart::vm_isolate_group()) {
@@ -468,7 +466,7 @@ void Heap::CollectNewSpaceGarbage(Thread* thread,
 void Heap::CollectOldSpaceGarbage(Thread* thread,
                                   GCType type,
                                   GCReason reason) {
-  NoActiveIsolateScope no_active_isolate_scope;
+  NoActiveIsolateScope no_active_isolate_scope(thread);
 
   ASSERT(type != GCType::kScavenge);
   ASSERT(reason != GCReason::kNewSpace);
@@ -704,9 +702,9 @@ void Heap::CollectOnNthAllocation(intptr_t num_allocations) {
   gc_on_nth_allocation_ = num_allocations;
 }
 
-void Heap::CollectForDebugging() {
+void Heap::CollectForDebugging(Thread* thread) {
   if (gc_on_nth_allocation_ == kNoForcedGarbageCollection) return;
-  if (Thread::Current()->IsAtSafepoint()) {
+  if (thread->IsAtSafepoint()) {
     // CollectAllGarbage is not supported when we are at a safepoint.
     // Allocating when at a safepoint is not a common case.
     return;
@@ -717,7 +715,7 @@ void Heap::CollectForDebugging() {
     gc_on_nth_allocation_ = kNoForcedGarbageCollection;
   } else {
     // Prevent generated code from using the TLAB fast path on next allocation.
-    new_space_.AbandonRemainingTLABForDebugging(Thread::Current());
+    new_space_.AbandonRemainingTLABForDebugging(thread);
   }
 }
 

runtime/vm/heap/heap.h

@@ -59,19 +59,19 @@ class Heap {
   Scavenger* new_space() { return &new_space_; }
   PageSpace* old_space() { return &old_space_; }
 
-  uword Allocate(intptr_t size, Space space) {
+  uword Allocate(Thread* thread, intptr_t size, Space space) {
     ASSERT(!read_only_);
     switch (space) {
       case kNew:
         // Do not attempt to allocate very large objects in new space.
         if (!IsAllocatableInNewSpace(size)) {
-          return AllocateOld(size, OldPage::kData);
+          return AllocateOld(thread, size, OldPage::kData);
         }
-        return AllocateNew(size);
+        return AllocateNew(thread, size);
       case kOld:
-        return AllocateOld(size, OldPage::kData);
+        return AllocateOld(thread, size, OldPage::kData);
       case kCode:
-        return AllocateOld(size, OldPage::kExecutable);
+        return AllocateOld(thread, size, OldPage::kExecutable);
       default:
         UNREACHABLE();
     }
@@ -325,8 +325,8 @@ class Heap {
        intptr_t max_new_gen_semi_words,  // Max capacity of new semi-space.
        intptr_t max_old_gen_words);
 
-  uword AllocateNew(intptr_t size);
-  uword AllocateOld(intptr_t size, OldPage::PageType type);
+  uword AllocateNew(Thread* thread, intptr_t size);
+  uword AllocateOld(Thread* thread, intptr_t size, OldPage::PageType type);
 
   // Visit all pointers. Caller must ensure concurrent sweeper is not running,
   // and the visitor must not allocate.
@@ -355,7 +355,7 @@ class Heap {
   void AddRegionsToObjectSet(ObjectSet* set) const;
 
   // Trigger major GC if 'gc_on_nth_allocation_' is set.
-  void CollectForDebugging();
+  void CollectForDebugging(Thread* thread);
 
   IsolateGroup* isolate_group_;
   bool is_vm_isolate_;

runtime/vm/object.cc

@@ -538,6 +538,7 @@ static type SpecialCharacter(type value) {
 void Object::InitNullAndBool(IsolateGroup* isolate_group) {
   // Should only be run by the vm isolate.
   ASSERT(isolate_group == Dart::vm_isolate_group());
+  Thread* thread = Thread::Current();
   auto heap = isolate_group->heap();
 
   // TODO(iposva): NoSafepointScope needs to be added here.
@@ -547,7 +548,8 @@ void Object::InitNullAndBool(IsolateGroup* isolate_group) {
   // 'null_' must be the first object allocated as it is used in allocation to
   // clear the object.
   {
-    uword address = heap->Allocate(Instance::InstanceSize(), Heap::kOld);
+    uword address =
+        heap->Allocate(thread, Instance::InstanceSize(), Heap::kOld);
     null_ = static_cast<InstancePtr>(address + kHeapObjectTag);
     // The call below is using 'null_' to initialize itself.
     InitializeObject(address, kNullCid, Instance::InstanceSize(),
@@ -561,14 +563,14 @@ void Object::InitNullAndBool(IsolateGroup* isolate_group) {
   // otherwise identical.
   {
     // Allocate a dummy bool object to give true the desired alignment.
-    uword address = heap->Allocate(Bool::InstanceSize(), Heap::kOld);
+    uword address = heap->Allocate(thread, Bool::InstanceSize(), Heap::kOld);
     InitializeObject(address, kBoolCid, Bool::InstanceSize(),
                      Bool::ContainsCompressedPointers());
     static_cast<BoolPtr>(address + kHeapObjectTag)->untag()->value_ = false;
   }
   {
     // Allocate true.
-    uword address = heap->Allocate(Bool::InstanceSize(), Heap::kOld);
+    uword address = heap->Allocate(thread, Bool::InstanceSize(), Heap::kOld);
     true_ = static_cast<BoolPtr>(address + kHeapObjectTag);
     InitializeObject(address, kBoolCid, Bool::InstanceSize(),
                      Bool::ContainsCompressedPointers());
@@ -577,7 +579,7 @@ void Object::InitNullAndBool(IsolateGroup* isolate_group) {
   }
   {
     // Allocate false.
-    uword address = heap->Allocate(Bool::InstanceSize(), Heap::kOld);
+    uword address = heap->Allocate(thread, Bool::InstanceSize(), Heap::kOld);
     false_ = static_cast<BoolPtr>(address + kHeapObjectTag);
     InitializeObject(address, kBoolCid, Bool::InstanceSize(),
                      Bool::ContainsCompressedPointers());
@@ -744,7 +746,7 @@ void Object::Init(IsolateGroup* isolate_group) {
   // Allocate and initialize the class class.
   {
     intptr_t size = Class::InstanceSize();
-    uword address = heap->Allocate(size, Heap::kOld);
+    uword address = heap->Allocate(thread, size, Heap::kOld);
     class_class_ = static_cast<ClassPtr>(address + kHeapObjectTag);
     InitializeObject(address, Class::kClassId, size,
                      Class::ContainsCompressedPointers());
@@ -979,7 +981,7 @@ void Object::Init(IsolateGroup* isolate_group) {
 
   // Allocate and initialize the empty_array instance.
   {
-    uword address = heap->Allocate(Array::InstanceSize(0), Heap::kOld);
+    uword address = heap->Allocate(thread, Array::InstanceSize(0), Heap::kOld);
     InitializeObject(address, kImmutableArrayCid, Array::InstanceSize(0),
                      Array::ContainsCompressedPointers());
     Array::initializeHandle(empty_array_,
@@ -991,7 +993,7 @@ void Object::Init(IsolateGroup* isolate_group) {
   Smi& smi = Smi::Handle();
   // Allocate and initialize the zero_array instance.
   {
-    uword address = heap->Allocate(Array::InstanceSize(1), Heap::kOld);
+    uword address = heap->Allocate(thread, Array::InstanceSize(1), Heap::kOld);
     InitializeObject(address, kImmutableArrayCid, Array::InstanceSize(1),
                      Array::ContainsCompressedPointers());
     Array::initializeHandle(zero_array_,
@@ -1004,7 +1006,8 @@ void Object::Init(IsolateGroup* isolate_group) {
 
   // Allocate and initialize the canonical empty context scope object.
   {
-    uword address = heap->Allocate(ContextScope::InstanceSize(0), Heap::kOld);
+    uword address =
+        heap->Allocate(thread, ContextScope::InstanceSize(0), Heap::kOld);
     InitializeObject(address, kContextScopeCid, ContextScope::InstanceSize(0),
                      ContextScope::ContainsCompressedPointers());
     ContextScope::initializeHandle(
@@ -1019,7 +1022,8 @@ void Object::Init(IsolateGroup* isolate_group) {
 
   // Allocate and initialize the canonical empty object pool object.
   {
-    uword address = heap->Allocate(ObjectPool::InstanceSize(0), Heap::kOld);
+    uword address =
+        heap->Allocate(thread, ObjectPool::InstanceSize(0), Heap::kOld);
     InitializeObject(address, kObjectPoolCid, ObjectPool::InstanceSize(0),
                      ObjectPool::ContainsCompressedPointers());
     ObjectPool::initializeHandle(
@@ -1033,7 +1037,7 @@ void Object::Init(IsolateGroup* isolate_group) {
   // Allocate and initialize the empty_compressed_stackmaps instance.
   {
     const intptr_t instance_size = CompressedStackMaps::InstanceSize(0);
-    uword address = heap->Allocate(instance_size, Heap::kOld);
+    uword address = heap->Allocate(thread, instance_size, Heap::kOld);
     InitializeObject(address, kCompressedStackMapsCid, instance_size,
                      CompressedStackMaps::ContainsCompressedPointers());
     CompressedStackMaps::initializeHandle(
@@ -1045,7 +1049,8 @@ void Object::Init(IsolateGroup* isolate_group) {
 
   // Allocate and initialize the empty_descriptors instance.
   {
-    uword address = heap->Allocate(PcDescriptors::InstanceSize(0), Heap::kOld);
+    uword address =
+        heap->Allocate(thread, PcDescriptors::InstanceSize(0), Heap::kOld);
     InitializeObject(address, kPcDescriptorsCid, PcDescriptors::InstanceSize(0),
                      PcDescriptors::ContainsCompressedPointers());
     PcDescriptors::initializeHandle(
@@ -1058,8 +1063,8 @@ void Object::Init(IsolateGroup* isolate_group) {
 
   // Allocate and initialize the canonical empty variable descriptor object.
   {
-    uword address =
-        heap->Allocate(LocalVarDescriptors::InstanceSize(0), Heap::kOld);
+    uword address = heap->Allocate(thread, LocalVarDescriptors::InstanceSize(0),
+                                   Heap::kOld);
     InitializeObject(address, kLocalVarDescriptorsCid,
                      LocalVarDescriptors::InstanceSize(0),
                      LocalVarDescriptors::ContainsCompressedPointers());
@@ -1076,7 +1081,7 @@ void Object::Init(IsolateGroup* isolate_group) {
   // and can share this canonical descriptor.
   {
     uword address =
-        heap->Allocate(ExceptionHandlers::InstanceSize(0), Heap::kOld);
+        heap->Allocate(thread, ExceptionHandlers::InstanceSize(0), Heap::kOld);
     InitializeObject(address, kExceptionHandlersCid,
                      ExceptionHandlers::InstanceSize(0),
                      ExceptionHandlers::ContainsCompressedPointers());
@@ -1090,7 +1095,8 @@ void Object::Init(IsolateGroup* isolate_group) {
 
   // Allocate and initialize the canonical empty type arguments object.
   {
-    uword address = heap->Allocate(TypeArguments::InstanceSize(0), Heap::kOld);
+    uword address =
+        heap->Allocate(thread, TypeArguments::InstanceSize(0), Heap::kOld);
     InitializeObject(address, kTypeArgumentsCid, TypeArguments::InstanceSize(0),
                      TypeArguments::ContainsCompressedPointers());
     TypeArguments::initializeHandle(
@@ -2666,7 +2672,7 @@ ObjectPtr Object::Allocate(intptr_t cls_id,
   ASSERT(thread->no_callback_scope_depth() == 0);
   Heap* heap = thread->heap();
 
-  uword address = heap->Allocate(size, space);
+  uword address = heap->Allocate(thread, size, space);
   if (UNLIKELY(address == 0)) {
     // SuspendLongJumpScope during Dart entry ensures that if a longjmp base is
     // available, it is the innermost error handler, so check for a longjmp base
@@ -2684,7 +2690,7 @@ ObjectPtr Object::Allocate(intptr_t cls_id,
       OUT_OF_MEMORY();
     }
   }
-  NoSafepointScope no_safepoint;
+  NoSafepointScope no_safepoint(thread);
   ObjectPtr raw_obj;
   InitializeObject(address, cls_id, size, compressed);
   raw_obj = static_cast<ObjectPtr>(address + kHeapObjectTag);
@@ -19714,6 +19720,11 @@ InstancePtr Instance::New(const Class& cls, Heap::Space space) {
   if (cls.EnsureIsAllocateFinalized(thread) != Error::null()) {
     return Instance::null();
   }
+  return NewAlreadyFinalized(cls, space);
+}
+
+InstancePtr Instance::NewAlreadyFinalized(const Class& cls, Heap::Space space) {
+  ASSERT(cls.is_allocate_finalized());
   intptr_t instance_size = cls.host_instance_size();
   ASSERT(instance_size > 0);
   ObjectPtr raw = Object::Allocate(cls.id(), instance_size, space,
@@ -24306,7 +24317,8 @@ const char* Array::ToCString() const {
 ArrayPtr Array::Grow(const Array& source,
                      intptr_t new_length,
                      Heap::Space space) {
-  Zone* zone = Thread::Current()->zone();
+  Thread* thread = Thread::Current();
+  Zone* zone = thread->zone();
   const Array& result = Array::Handle(zone, Array::New(new_length, space));
   intptr_t len = 0;
   if (!source.IsNull()) {
@@ -24319,7 +24331,7 @@ ArrayPtr Array::Grow(const Array& source,
   PassiveObject& obj = PassiveObject::Handle(zone);
   for (int i = 0; i < len; i++) {
     obj = source.At(i);
-    result.SetAt(i, obj);
+    result.SetAt(i, obj, thread);
   }
   return result.ptr();
 }

runtime/vm/object.h

@@ -7714,6 +7714,8 @@ class Instance : public Object {
   }
 
   static InstancePtr New(const Class& cls, Heap::Space space = Heap::kNew);
+  static InstancePtr NewAlreadyFinalized(const Class& cls,
+                                         Heap::Space space = Heap::kNew);
 
   // Array/list element address computations.
   static intptr_t DataOffsetFor(intptr_t cid);
@@ -10342,9 +10344,12 @@ class Array : public Instance {
   }
   template <std::memory_order order = std::memory_order_relaxed>
   void SetAt(intptr_t index, const Object& value) const {
-    // TODO(iposva): Add storing NoSafepointScope.
     untag()->set_element<order>(index, value.ptr());
   }
+  template <std::memory_order order = std::memory_order_relaxed>
+  void SetAt(intptr_t index, const Object& value, Thread* thread) const {
+    untag()->set_element<order>(index, value.ptr(), thread);
+  }
 
   // Access to the array with acquire release semantics.
   ObjectPtr AtAcquire(intptr_t index) const {
@@ -10479,9 +10484,11 @@ class Array : public Instance {
       memmove(const_cast<CompressedObjectPtr*>(to), from,
               count * kBytesPerElement);
     } else {
+      Thread* thread = Thread::Current();
       const uword heap_base = ptr()->heap_base();
       for (intptr_t i = 0; i < count; ++i) {
-        StoreArrayPointer(&to[i], from[i].Decompress(heap_base));
+        untag()->StoreArrayPointer(&to[i], from[i].Decompress(heap_base),
+                                   thread);
       }
     }
   }

runtime/vm/raw_object.h

@@ -601,9 +601,7 @@ class UntaggedObject {
     }
   }
 
-  template <typename type,
-            typename compressed_type,
-            std::memory_order order = std::memory_order_relaxed>
+  template <typename type, typename compressed_type, std::memory_order order>
   void StoreCompressedArrayPointer(compressed_type const* addr, type value) {
     reinterpret_cast<std::atomic<compressed_type>*>(
         const_cast<compressed_type*>(addr))
@@ -613,6 +611,18 @@ class UntaggedObject {
     }
   }
 
+  template <typename type, typename compressed_type, std::memory_order order>
+  void StoreCompressedArrayPointer(compressed_type const* addr,
+                                   type value,
+                                   Thread* thread) {
+    reinterpret_cast<std::atomic<compressed_type>*>(
+        const_cast<compressed_type*>(addr))
+        ->store(static_cast<compressed_type>(value), order);
+    if (value->IsHeapObject()) {
+      CheckArrayPointerStore(addr, value, thread);
+    }
+  }
+
   template <typename type, typename compressed_type>
   void StoreCompressedArrayPointer(compressed_type const* addr,
                                    type value,
@@ -862,6 +872,10 @@ inline intptr_t ObjectPtr::GetClassId() const {
   template <std::memory_order order = std::memory_order_relaxed>               \
   void set_##accessor_name(intptr_t index, type value) {                       \
     StoreArrayPointer<type, order>(&array_name()[index], value);               \
+  }                                                                            \
+  template <std::memory_order order = std::memory_order_relaxed>               \
+  void set_##accessor_name(intptr_t index, type value, Thread* thread) {       \
+    StoreArrayPointer<type, order>(&array_name()[index], value, thread);       \
   }                                                                            \
                                                                                \
  protected:                                                                    \
@@ -880,6 +894,11 @@ inline intptr_t ObjectPtr::GetClassId() const {
   void set_##accessor_name(intptr_t index, type value) {                       \
     StoreCompressedArrayPointer<type, Compressed##type, order>(                \
         &array_name()[index], value);                                          \
+  }                                                                            \
+  template <std::memory_order order = std::memory_order_relaxed>               \
+  void set_##accessor_name(intptr_t index, type value, Thread* thread) {       \
+    StoreCompressedArrayPointer<type, Compressed##type, order>(                \
+        &array_name()[index], value, thread);                                  \
   }                                                                            \
                                                                                \
  protected:                                                                    \

runtime/vm/runtime_entry.cc

@@ -460,11 +460,9 @@ static void ThrowIfError(const Object& result) {
 // Return value: newly allocated object.
 DEFINE_RUNTIME_ENTRY(AllocateObject, 2) {
   const Class& cls = Class::CheckedHandle(zone, arguments.ArgAt(0));
-  const Error& error =
-      Error::Handle(zone, cls.EnsureIsAllocateFinalized(thread));
-  ThrowIfError(error);
-  const Instance& instance =
-      Instance::Handle(zone, Instance::New(cls, SpaceForRuntimeAllocation()));
+  ASSERT(cls.is_allocate_finalized());
+  const Instance& instance = Instance::Handle(
+      zone, Instance::NewAlreadyFinalized(cls, SpaceForRuntimeAllocation()));
 
   arguments.SetReturn(instance);
   if (cls.NumTypeArguments() == 0) {
@@ -1393,7 +1391,7 @@ static void TrySwitchInstanceCall(Thread* thread,
 
 #if !defined(PRODUCT)
   // Monomorphic/megamorphic do not check the isolate's stepping flag.
-  if (Isolate::Current()->has_attempted_stepping()) return;
+  if (thread->isolate()->has_attempted_stepping()) return;
 #endif
 
   // Monomorphic/megamorphic calls are only for unoptimized code.