From 8378b8fdbf332c9bf2166abb960354bb92492480 Mon Sep 17 00:00:00 2001 From: Erik Corry Date: Thu, 22 Jun 2017 14:56:20 +0200 Subject: [PATCH] VM: Reland Inline instance object hash code into object header on 64bit. Inline instance object hash code into object header on 64 bit. 64 bit objects have 32 bits of free space in the header word. This is used for the hash code in string objects. We take it for the default hash code on all objects that don't override the hashCode getter. This is both faster and a memory reduction. Eg it makes the MegaHashCode part of the Megamorphic benchmark 6 times faster. This is a reland of https://codereview.chromium.org/2912863006/ It fixes issues with the 32 bit compare-swap instruction on ARM64 and fixes a fragile tree shaking test that is sensitive to which private methods are in the core libraries. R=kustermann@google.com, vegorov@google.com BUG= Review-Url: https://codereview.chromium.org/2954453002 . --- .../shaker/empty_program.dart.shaker | 2 - runtime/lib/object.cc | 10 +- runtime/lib/object_patch.dart | 15 ++- runtime/vm/assembler_arm.cc | 4 +- runtime/vm/assembler_arm64.cc | 8 +- runtime/vm/assembler_arm64.h | 15 ++- runtime/vm/assembler_arm64_test.cc | 66 +++++++++- runtime/vm/assembler_ia32.cc | 4 +- runtime/vm/assembler_mips.cc | 4 +- runtime/vm/assembler_x64.cc | 8 +- runtime/vm/assembler_x64.h | 13 +- runtime/vm/assembler_x64_test.cc | 45 ++++++- runtime/vm/become.cc | 6 +- runtime/vm/become.h | 5 +- runtime/vm/clustered_snapshot.cc | 5 +- runtime/vm/freelist.cc | 7 +- runtime/vm/freelist.h | 5 +- runtime/vm/heap.cc | 3 +- runtime/vm/heap.h | 11 +- runtime/vm/intermediate_language_dbc.cc | 4 +- runtime/vm/intrinsifier_arm64.cc | 17 +++ runtime/vm/intrinsifier_x64.cc | 17 +++ runtime/vm/method_recognizer.h | 15 ++- runtime/vm/object.cc | 116 ++++++++++++++---- runtime/vm/object.h | 20 ++- runtime/vm/profiler.cc | 2 +- runtime/vm/raw_object.cc | 14 +-- runtime/vm/raw_object.h | 87 ++++++++----- runtime/vm/simulator_arm64.cc | 48 ++++++-- runtime/vm/simulator_arm64.h | 3 + runtime/vm/simulator_dbc.cc | 8 +- runtime/vm/snapshot.cc | 30 ++++- runtime/vm/snapshot.h | 3 +- runtime/vm/stub_code_arm.cc | 2 +- runtime/vm/stub_code_arm64.cc | 11 +- runtime/vm/stub_code_ia32.cc | 2 +- runtime/vm/stub_code_mips.cc | 2 +- runtime/vm/stub_code_x64.cc | 15 ++- 38 files changed, 501 insertions(+), 151 deletions(-) diff --git a/pkg/front_end/testcases/shaker/empty_program.dart.shaker b/pkg/front_end/testcases/shaker/empty_program.dart.shaker index bd990e7d782..6dc87ba8136 100644 --- a/pkg/front_end/testcases/shaker/empty_program.dart.shaker +++ b/pkg/front_end/testcases/shaker/empty_program.dart.shaker @@ -53,8 +53,6 @@ library dart:core: - toString - noSuchMethod - runtimeType - - dart.core::_getHash - - dart.core::_setHash - dart.core::_objectHashCode - dart.core::_identityHashCode - dart.core::_toString diff --git a/runtime/lib/object.cc b/runtime/lib/object.cc index b7dcea3a208..f0739adc3ac 100644 --- a/runtime/lib/object.cc +++ b/runtime/lib/object.cc @@ -42,17 +42,25 @@ DEFINE_NATIVE_ENTRY(Object_getHash, 1) { // Please note that no handle is created for the argument. // This is safe since the argument is only used in a tail call. // The performance benefit is more than 5% when using hashCode. +#if defined(HASH_IN_OBJECT_HEADER) + return Smi::New(Object::GetCachedHash(arguments->NativeArgAt(0))); +#else Heap* heap = isolate->heap(); ASSERT(arguments->NativeArgAt(0)->IsDartInstance()); return Smi::New(heap->GetHash(arguments->NativeArgAt(0))); +#endif } DEFINE_NATIVE_ENTRY(Object_setHash, 2) { - const Instance& instance = Instance::CheckedHandle(arguments->NativeArgAt(0)); GET_NON_NULL_NATIVE_ARGUMENT(Smi, hash, arguments->NativeArgAt(1)); +#if defined(HASH_IN_OBJECT_HEADER) + Object::SetCachedHash(arguments->NativeArgAt(0), hash.Value()); +#else + const Instance& instance = Instance::CheckedHandle(arguments->NativeArgAt(0)); Heap* heap = isolate->heap(); heap->SetHash(instance.raw(), hash.Value()); +#endif return Object::null(); } diff --git a/runtime/lib/object_patch.dart b/runtime/lib/object_patch.dart index 6fe989d2683..ed77013fd67 100644 --- a/runtime/lib/object_patch.dart +++ b/runtime/lib/object_patch.dart @@ -2,6 +2,9 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +int _getHash(obj) native "Object_getHash"; +int _setHash(obj, hash) native "Object_setHash"; + @patch class Object { // The VM has its own implementation of equals. @@ -9,22 +12,18 @@ class Object { bool operator ==(other) native "Object_equals"; // Helpers used to implement hashCode. If a hashCode is used, we remember it - // in a weak table in the VM. A new hashCode value is calculated using a - // number generator. + // in a weak table in the VM (32 bit) or in the header of the object (64 + // bit). A new hashCode value is calculated using a random number generator. static final _hashCodeRnd = new Random(); - static _getHash(obj) native "Object_getHash"; - static _setHash(obj, hash) native "Object_setHash"; - // Shared static implentation for hashCode and _identityHashCode. static int _objectHashCode(obj) { var result = _getHash(obj); if (result == 0) { // We want the hash to be a Smi value greater than 0. - result = _hashCodeRnd.nextInt(0x40000000); - while (result == 0) { + do { result = _hashCodeRnd.nextInt(0x40000000); - } + } while (result == 0); _setHash(obj, result); } return result; diff --git a/runtime/vm/assembler_arm.cc b/runtime/vm/assembler_arm.cc index 85bf01a8f20..c51cd60a14b 100644 --- a/runtime/vm/assembler_arm.cc +++ b/runtime/vm/assembler_arm.cc @@ -3471,7 +3471,7 @@ void Assembler::TryAllocate(const Class& cls, ASSERT(instance_size >= kHeapObjectTag); AddImmediate(instance_reg, -instance_size + kHeapObjectTag); - uword tags = 0; + uint32_t tags = 0; tags = RawObject::SizeTag::update(instance_size, tags); ASSERT(cls.id() != kIllegalCid); tags = RawObject::ClassIdTag::update(cls.id(), tags); @@ -3520,7 +3520,7 @@ void Assembler::TryAllocateArray(intptr_t cid, // Initialize the tags. // instance: new object start as a tagged pointer. - uword tags = 0; + uint32_t tags = 0; tags = RawObject::ClassIdTag::update(cid, tags); tags = RawObject::SizeTag::update(instance_size, tags); LoadImmediate(temp1, tags); diff --git a/runtime/vm/assembler_arm64.cc b/runtime/vm/assembler_arm64.cc index efe2f1ca6af..1d12ee89d24 100644 --- a/runtime/vm/assembler_arm64.cc +++ b/runtime/vm/assembler_arm64.cc @@ -1363,10 +1363,12 @@ void Assembler::TryAllocate(const Class& cls, AddImmediate(instance_reg, -instance_size + kHeapObjectTag); NOT_IN_PRODUCT(UpdateAllocationStats(cls.id(), space)); - uword tags = 0; + uint32_t tags = 0; tags = RawObject::SizeTag::update(instance_size, tags); ASSERT(cls.id() != kIllegalCid); tags = RawObject::ClassIdTag::update(cls.id(), tags); + // Extends the 32 bit tags with zeros, which is the uninitialized + // hash code. LoadImmediate(TMP, tags); StoreFieldToOffset(TMP, instance_reg, Object::tags_offset()); } else { @@ -1410,9 +1412,11 @@ void Assembler::TryAllocateArray(intptr_t cid, // Initialize the tags. // instance: new object start as a tagged pointer. - uword tags = 0; + uint32_t tags = 0; tags = RawObject::ClassIdTag::update(cid, tags); tags = RawObject::SizeTag::update(instance_size, tags); + // Extends the 32 bit tags with zeros, which is the uninitialized + // hash code. LoadImmediate(temp2, tags); str(temp2, FieldAddress(instance, Array::tags_offset())); // Store tags. } else { diff --git a/runtime/vm/assembler_arm64.h b/runtime/vm/assembler_arm64.h index cbeadfd80e1..1b25c5bb0e8 100644 --- a/runtime/vm/assembler_arm64.h +++ b/runtime/vm/assembler_arm64.h @@ -742,16 +742,19 @@ class Assembler : public ValueObject { EmitLoadStoreRegPair(STP, rt, rt2, a, sz); } - void ldxr(Register rt, Register rn) { + void ldxr(Register rt, Register rn, OperandSize size = kDoubleWord) { // rt = value // rn = address - EmitLoadStoreExclusive(LDXR, R31, rn, rt, kDoubleWord); + EmitLoadStoreExclusive(LDXR, R31, rn, rt, size); } - void stxr(Register rs, Register rt, Register rn) { + void stxr(Register rs, + Register rt, + Register rn, + OperandSize size = kDoubleWord) { // rs = status (1 = failure, 0 = success) // rt = value // rn = address - EmitLoadStoreExclusive(STXR, rs, rn, rt, kDoubleWord); + EmitLoadStoreExclusive(STXR, rs, rn, rt, size); } void clrex() { const int32_t encoding = static_cast(CLREX); @@ -1701,8 +1704,8 @@ class Assembler : public ValueObject { Register rn, Register rt, OperandSize sz = kDoubleWord) { - ASSERT(sz == kDoubleWord); - const int32_t size = B31 | B30; + ASSERT(sz == kDoubleWord || sz == kWord); + const int32_t size = B31 | (sz == kDoubleWord ? B30 : 0); ASSERT((rs != kNoRegister) && (rs != ZR)); ASSERT((rn != kNoRegister) && (rn != ZR)); diff --git a/runtime/vm/assembler_arm64_test.cc b/runtime/vm/assembler_arm64_test.cc index 566a9a323ae..624a950e717 100644 --- a/runtime/vm/assembler_arm64_test.cc +++ b/runtime/vm/assembler_arm64_test.cc @@ -580,7 +580,7 @@ ASSEMBLER_TEST_GENERATE(Semaphore, assembler) { ASSEMBLER_TEST_RUN(Semaphore, test) { EXPECT(test != NULL); - typedef int (*Semaphore)() DART_UNUSED; + typedef intptr_t (*Semaphore)() DART_UNUSED; EXPECT_EQ(42, EXECUTE_TEST_CODE_INT64(Semaphore, test->entry())); } @@ -602,11 +602,73 @@ ASSEMBLER_TEST_GENERATE(FailedSemaphore, assembler) { ASSEMBLER_TEST_RUN(FailedSemaphore, test) { EXPECT(test != NULL); - typedef int (*FailedSemaphore)() DART_UNUSED; + typedef intptr_t (*FailedSemaphore)() DART_UNUSED; EXPECT_EQ(41, EXECUTE_TEST_CODE_INT64(FailedSemaphore, test->entry())); } +ASSEMBLER_TEST_GENERATE(Semaphore32, assembler) { + __ SetupDartSP(); + __ movz(R0, Immediate(40), 0); + __ add(R0, R0, Operand(R0, LSL, 32)); + __ Push(R0); + + __ movz(R0, Immediate(40), 0); + __ movz(R1, Immediate(42), 0); + + Label retry; + __ Bind(&retry); + __ ldxr(R0, SP, kWord); + // 32 bit operation should ignore the high word of R0 that was pushed on the + // stack. + __ stxr(TMP, R1, SP, kWord); // IP == 0, success + __ cmp(TMP, Operand(0)); + __ b(&retry, NE); // NE if context switch occurred between ldrex and strex. + __ Pop(R0); // 42 + 42 * 2**32 + __ RestoreCSP(); + __ ret(); +} + + +ASSEMBLER_TEST_RUN(Semaphore32, test) { + EXPECT(test != NULL); + typedef intptr_t (*Semaphore32)() DART_UNUSED; + // Lower word has been atomically switched from 40 to 42k, whereas upper word + // is unchanged at 40. + EXPECT_EQ(42 + (40l << 32), + EXECUTE_TEST_CODE_INT64(Semaphore32, test->entry())); +} + + +ASSEMBLER_TEST_GENERATE(FailedSemaphore32, assembler) { + __ SetupDartSP(); + __ movz(R0, Immediate(40), 0); + __ add(R0, R0, Operand(R0, LSL, 32)); + __ Push(R0); + + __ movz(R0, Immediate(40), 0); + __ movz(R1, Immediate(42), 0); + + __ ldxr(R0, SP, kWord); + __ clrex(); // Simulate a context switch. + __ stxr(TMP, R1, SP, kWord); // IP == 1, failure + __ Pop(R0); // 40 + __ add(R0, R0, Operand(TMP)); + __ RestoreCSP(); + __ ret(); +} + + +ASSEMBLER_TEST_RUN(FailedSemaphore32, test) { + EXPECT(test != NULL); + typedef intptr_t (*FailedSemaphore32)() DART_UNUSED; + // Lower word has had the failure code (1) added to it. Upper word is + // unchanged at 40. + EXPECT_EQ(41 + (40l << 32), + EXECUTE_TEST_CODE_INT64(FailedSemaphore32, test->entry())); +} + + // Logical register operations. ASSEMBLER_TEST_GENERATE(AndRegs, assembler) { __ movz(R1, Immediate(43), 0); diff --git a/runtime/vm/assembler_ia32.cc b/runtime/vm/assembler_ia32.cc index 7ef2204d775..a437c29cf82 100644 --- a/runtime/vm/assembler_ia32.cc +++ b/runtime/vm/assembler_ia32.cc @@ -2643,7 +2643,7 @@ void Assembler::TryAllocate(const Class& cls, NOT_IN_PRODUCT(UpdateAllocationStats(cls.id(), temp_reg, space)); ASSERT(instance_size >= kHeapObjectTag); subl(instance_reg, Immediate(instance_size - kHeapObjectTag)); - uword tags = 0; + uint32_t tags = 0; tags = RawObject::SizeTag::update(instance_size, tags); ASSERT(cls.id() != kIllegalCid); tags = RawObject::ClassIdTag::update(cls.id(), tags); @@ -2690,7 +2690,7 @@ void Assembler::TryAllocateArray(intptr_t cid, UpdateAllocationStatsWithSize(cid, instance_size, temp_reg, space)); // Initialize the tags. - uword tags = 0; + uint32_t tags = 0; tags = RawObject::ClassIdTag::update(cid, tags); tags = RawObject::SizeTag::update(instance_size, tags); movl(FieldAddress(instance, Object::tags_offset()), Immediate(tags)); diff --git a/runtime/vm/assembler_mips.cc b/runtime/vm/assembler_mips.cc index 7660e2526dc..846a817f738 100644 --- a/runtime/vm/assembler_mips.cc +++ b/runtime/vm/assembler_mips.cc @@ -999,7 +999,7 @@ void Assembler::TryAllocate(const Class& cls, ASSERT(instance_size >= kHeapObjectTag); AddImmediate(instance_reg, -instance_size + kHeapObjectTag); NOT_IN_PRODUCT(UpdateAllocationStats(cls.id(), temp_reg, space)); - uword tags = 0; + uint32_t tags = 0; tags = RawObject::SizeTag::update(instance_size, tags); ASSERT(cls.id() != kIllegalCid); tags = RawObject::ClassIdTag::update(cls.id(), tags); @@ -1049,7 +1049,7 @@ void Assembler::TryAllocateArray(intptr_t cid, // Initialize the tags. // instance: new object start as a tagged pointer. - uword tags = 0; + uint32_t tags = 0; tags = RawObject::ClassIdTag::update(cid, tags); tags = RawObject::SizeTag::update(instance_size, tags); LoadImmediate(temp1, tags); diff --git a/runtime/vm/assembler_x64.cc b/runtime/vm/assembler_x64.cc index 594f04d8e56..0d854e76772 100644 --- a/runtime/vm/assembler_x64.cc +++ b/runtime/vm/assembler_x64.cc @@ -3433,10 +3433,12 @@ void Assembler::TryAllocate(const Class& cls, NOT_IN_PRODUCT(UpdateAllocationStats(cls.id(), space)); ASSERT(instance_size >= kHeapObjectTag); AddImmediate(instance_reg, Immediate(kHeapObjectTag - instance_size)); - uword tags = 0; + uint32_t tags = 0; tags = RawObject::SizeTag::update(instance_size, tags); ASSERT(cls.id() != kIllegalCid); tags = RawObject::ClassIdTag::update(cls.id(), tags); + // Extends the 32 bit tags with zeros, which is the uninitialized + // hash code. MoveImmediate(FieldAddress(instance_reg, Object::tags_offset()), Immediate(tags)); } else { @@ -3480,9 +3482,11 @@ void Assembler::TryAllocateArray(intptr_t cid, // Initialize the tags. // instance: new object start as a tagged pointer. - uword tags = 0; + uint32_t tags = 0; tags = RawObject::ClassIdTag::update(cid, tags); tags = RawObject::SizeTag::update(instance_size, tags); + // Extends the 32 bit tags with zeros, which is the uninitialized + // hash code. movq(FieldAddress(instance, Array::tags_offset()), Immediate(tags)); } else { jmp(failure); diff --git a/runtime/vm/assembler_x64.h b/runtime/vm/assembler_x64.h index 0c17551a865..ad19e6dc699 100644 --- a/runtime/vm/assembler_x64.h +++ b/runtime/vm/assembler_x64.h @@ -690,16 +690,8 @@ class Assembler : public ValueObject { void lock(); void cmpxchgl(const Address& address, Register reg); - void lock_cmpxchgl(const Address& address, Register reg) { - lock(); - cmpxchgl(address, reg); - } void cmpxchgq(const Address& address, Register reg); - void lock_cmpxchgq(const Address& address, Register reg) { - lock(); - cmpxchgq(address, reg); - } void cpuid(); @@ -801,6 +793,11 @@ class Assembler : public ValueObject { cmpxchgq(address, reg); } + void LockCmpxchgl(const Address& address, Register reg) { + lock(); + cmpxchgl(address, reg); + } + void PushRegisters(intptr_t cpu_register_set, intptr_t xmm_register_set); void PopRegisters(intptr_t cpu_register_set, intptr_t xmm_register_set); diff --git a/runtime/vm/assembler_x64_test.cc b/runtime/vm/assembler_x64_test.cc index 5a9a8128547..8c04d7d4f9a 100644 --- a/runtime/vm/assembler_x64_test.cc +++ b/runtime/vm/assembler_x64_test.cc @@ -1749,7 +1749,7 @@ ASSEMBLER_TEST_GENERATE(CompareSwapEQ, assembler) { __ movq(RAX, Immediate(4)); __ movq(RCX, Immediate(0)); __ movq(Address(RSP, 0), RAX); - __ lock_cmpxchgq(Address(RSP, 0), RCX); + __ LockCmpxchgq(Address(RSP, 0), RCX); __ popq(RAX); __ ret(); } @@ -1767,7 +1767,7 @@ ASSEMBLER_TEST_GENERATE(CompareSwapNEQ, assembler) { __ movq(RAX, Immediate(2)); __ movq(RCX, Immediate(4)); __ movq(Address(RSP, 0), RCX); - __ lock_cmpxchgq(Address(RSP, 0), RCX); + __ LockCmpxchgq(Address(RSP, 0), RCX); __ popq(RAX); __ ret(); } @@ -1779,6 +1779,47 @@ ASSEMBLER_TEST_RUN(CompareSwapNEQ, test) { } +ASSEMBLER_TEST_GENERATE(CompareSwapEQ32, assembler) { + __ movq(RAX, Immediate(0x100000000)); + __ pushq(RAX); + __ movq(RAX, Immediate(4)); + __ movq(RCX, Immediate(0)); + // 32 bit store of 4. + __ movl(Address(RSP, 0), RAX); + // Compare 32 bit memory location with RAX (4) and write 0. + __ LockCmpxchgl(Address(RSP, 0), RCX); + // Pop unchanged high word and zeroed out low word. + __ popq(RAX); + __ ret(); +} + + +ASSEMBLER_TEST_RUN(CompareSwapEQ32, test) { + typedef intptr_t (*CompareSwapEQ32Code)(); + EXPECT_EQ(0x100000000, + reinterpret_cast(test->entry())()); +} + + +ASSEMBLER_TEST_GENERATE(CompareSwapNEQ32, assembler) { + __ movq(RAX, Immediate(0x100000000)); + __ pushq(RAX); + __ movq(RAX, Immediate(2)); + __ movq(RCX, Immediate(4)); + __ movl(Address(RSP, 0), RCX); + __ LockCmpxchgl(Address(RSP, 0), RCX); + __ popq(RAX); + __ ret(); +} + + +ASSEMBLER_TEST_RUN(CompareSwapNEQ32, test) { + typedef intptr_t (*CompareSwapNEQ32Code)(); + EXPECT_EQ(0x100000004l, + reinterpret_cast(test->entry())()); +} + + ASSEMBLER_TEST_GENERATE(Exchange, assembler) { __ movq(RAX, Immediate(kLargeConstant)); __ movq(RDX, Immediate(kAnotherLargeConstant)); diff --git a/runtime/vm/become.cc b/runtime/vm/become.cc index 0fddef218f8..142169aae76 100644 --- a/runtime/vm/become.cc +++ b/runtime/vm/become.cc @@ -23,7 +23,7 @@ ForwardingCorpse* ForwardingCorpse::AsForwarder(uword addr, intptr_t size) { ForwardingCorpse* result = reinterpret_cast(addr); - uword tags = 0; + uint32_t tags = 0; tags = RawObject::SizeTag::update(size, tags); tags = RawObject::ClassIdTag::update(kForwardingCorpse, tags); @@ -262,12 +262,16 @@ void Become::ElementsForwardIdentity(const Array& before, const Array& after) { ForwardObjectTo(before_obj, after_obj); +#if defined(HASH_IN_OBJECT_HEADER) + Object::SetCachedHash(after_obj, Object::GetCachedHash(before_obj)); +#else // Forward the identity hash too if it has one. intptr_t hash = heap->GetHash(before_obj); if (hash != 0) { ASSERT(heap->GetHash(after_obj) == 0); heap->SetHash(after_obj, hash); } +#endif } { diff --git a/runtime/vm/become.h b/runtime/vm/become.h index 1784d22edb3..aaa60e18b9d 100644 --- a/runtime/vm/become.h +++ b/runtime/vm/become.h @@ -52,7 +52,10 @@ class ForwardingCorpse { private: // This layout mirrors the layout of RawObject. - uword tags_; + uint32_t tags_; +#if defined(HASH_IN_OBJECT_HEADER) + uint32_t hash_; +#endif RawObject* target_; // Returns the address of the embedded size. diff --git a/runtime/vm/clustered_snapshot.cc b/runtime/vm/clustered_snapshot.cc index 56f766f12be..a5346975c0b 100644 --- a/runtime/vm/clustered_snapshot.cc +++ b/runtime/vm/clustered_snapshot.cc @@ -40,12 +40,15 @@ void Deserializer::InitializeHeader(RawObject* raw, bool is_vm_isolate, bool is_canonical) { ASSERT(Utils::IsAligned(size, kObjectAlignment)); - uword tags = 0; + uint32_t tags = 0; tags = RawObject::ClassIdTag::update(class_id, tags); tags = RawObject::SizeTag::update(size, tags); tags = RawObject::VMHeapObjectTag::update(is_vm_isolate, tags); tags = RawObject::CanonicalObjectTag::update(is_canonical, tags); raw->ptr()->tags_ = tags; +#if defined(HASH_IN_OBJECT_HEADER) + raw->ptr()->hash_ = 0; +#endif } diff --git a/runtime/vm/freelist.cc b/runtime/vm/freelist.cc index e4a880e8113..221a5ea4bf9 100644 --- a/runtime/vm/freelist.cc +++ b/runtime/vm/freelist.cc @@ -22,13 +22,18 @@ FreeListElement* FreeListElement::AsElement(uword addr, intptr_t size) { FreeListElement* result = reinterpret_cast(addr); - uword tags = 0; + uint32_t tags = 0; tags = RawObject::SizeTag::update(size, tags); tags = RawObject::ClassIdTag::update(kFreeListElement, tags); // All words in a freelist element header should look like Smis. ASSERT(!reinterpret_cast(tags)->IsHeapObject()); result->tags_ = tags; +#if defined(HASH_IN_OBJECT_HEADER) + // Clearing this is mostly for neatness. The identityHashCode + // of free list entries is not used. + result->hash_ = 0; +#endif if (size > RawObject::SizeTag::kMaxSizeTag) { *result->SizeAddress() = size; } diff --git a/runtime/vm/freelist.h b/runtime/vm/freelist.h index f8a8e82f66a..cbd7656181c 100644 --- a/runtime/vm/freelist.h +++ b/runtime/vm/freelist.h @@ -56,7 +56,10 @@ class FreeListElement { private: // This layout mirrors the layout of RawObject. - uword tags_; + uint32_t tags_; +#if defined(HASH_IN_OBJECT_HEADER) + uint32_t hash_; +#endif FreeListElement* next_; // Returns the address of the embedded size. diff --git a/runtime/vm/heap.cc b/runtime/vm/heap.cc index a713a8cf882..94d702fe3b7 100644 --- a/runtime/vm/heap.cc +++ b/runtime/vm/heap.cc @@ -653,11 +653,12 @@ int64_t Heap::PeerCount() const { return new_weak_tables_[kPeers]->count() + old_weak_tables_[kPeers]->count(); } - +#if !defined(HASH_IN_OBJECT_HEADER) int64_t Heap::HashCount() const { return new_weak_tables_[kHashes]->count() + old_weak_tables_[kHashes]->count(); } +#endif int64_t Heap::ObjectIdCount() const { diff --git a/runtime/vm/heap.h b/runtime/vm/heap.h index 1c9968660d3..a2fa51f8c27 100644 --- a/runtime/vm/heap.h +++ b/runtime/vm/heap.h @@ -33,7 +33,14 @@ class Heap { kCode, }; - enum WeakSelector { kPeers = 0, kHashes, kObjectIds, kNumWeakSelectors }; + enum WeakSelector { + kPeers = 0, +#if !defined(HASH_IN_OBJECT_HEADER) + kHashes, +#endif + kObjectIds, + kNumWeakSelectors + }; enum ApiCallbacks { kIgnoreApiCallbacks, kInvokeApiCallbacks }; @@ -173,6 +180,7 @@ class Heap { } int64_t PeerCount() const; +#if !defined(HASH_IN_OBJECT_HEADER) // Associate an identity hashCode with an object. An non-existent hashCode // is equal to 0. void SetHash(RawObject* raw_obj, intptr_t hash) { @@ -181,6 +189,7 @@ class Heap { intptr_t GetHash(RawObject* raw_obj) const { return GetWeakEntry(raw_obj, kHashes); } +#endif int64_t HashCount() const; // Associate an id with an object (used when serializing an object). diff --git a/runtime/vm/intermediate_language_dbc.cc b/runtime/vm/intermediate_language_dbc.cc index 00142af0cb1..1e060e3a80f 100644 --- a/runtime/vm/intermediate_language_dbc.cc +++ b/runtime/vm/intermediate_language_dbc.cc @@ -1012,7 +1012,7 @@ EMIT_NATIVE_CODE(AllocateObject, Isolate* isolate = Isolate::Current(); if (Heap::IsAllocatableInNewSpace(instance_size) && !cls().TraceAllocation(isolate)) { - uword tags = 0; + uint32_t tags = 0; tags = RawObject::SizeTag::update(instance_size, tags); ASSERT(cls().id() != kIllegalCid); tags = RawObject::ClassIdTag::update(cls().id(), tags); @@ -1046,6 +1046,7 @@ EMIT_NATIVE_CODE(AllocateObject, tags = RawObject::SizeTag::update(instance_size, tags); ASSERT(cls().id() != kIllegalCid); tags = RawObject::ClassIdTag::update(cls().id(), tags); + // tags also has the initial zero hash code on 64 bit. if (Smi::IsValid(tags)) { const intptr_t tags_kidx = __ AddConstant(Smi::Handle(Smi::New(tags))); __ AllocateOpt(locs()->out(0).reg(), tags_kidx); @@ -1657,6 +1658,7 @@ EMIT_NATIVE_CODE(Box, 1, Location::RequiresRegister(), LocationSummary::kCall) { uword tags = 0; tags = RawObject::SizeTag::update(instance_size, tags); tags = RawObject::ClassIdTag::update(compiler->double_class().id(), tags); + // tags also has the initial zero hash code on 64 bit. if (Smi::IsValid(tags)) { const intptr_t tags_kidx = __ AddConstant(Smi::Handle(Smi::New(tags))); __ AllocateOpt(out, tags_kidx); diff --git a/runtime/vm/intrinsifier_arm64.cc b/runtime/vm/intrinsifier_arm64.cc index 80d51eb3e4c..3c5f4f6c341 100644 --- a/runtime/vm/intrinsifier_arm64.cc +++ b/runtime/vm/intrinsifier_arm64.cc @@ -1832,6 +1832,23 @@ void Intrinsifier::String_getHashCode(Assembler* assembler) { } +void Intrinsifier::Object_getHash(Assembler* assembler) { + __ ldr(R0, Address(SP, 0 * kWordSize)); + __ ldr(R0, FieldAddress(R0, String::hash_offset()), kUnsignedWord); + __ SmiTag(R0); + __ ret(); +} + + +void Intrinsifier::Object_setHash(Assembler* assembler) { + __ ldr(R0, Address(SP, 1 * kWordSize)); // Object. + __ ldr(R1, Address(SP, 0 * kWordSize)); // Value. + __ SmiUntag(R1); + __ str(R1, FieldAddress(R0, String::hash_offset()), kUnsignedWord); + __ ret(); +} + + void GenerateSubstringMatchesSpecialization(Assembler* assembler, intptr_t receiver_cid, intptr_t other_cid, diff --git a/runtime/vm/intrinsifier_x64.cc b/runtime/vm/intrinsifier_x64.cc index f0ca5913f91..3b97a1bf80b 100644 --- a/runtime/vm/intrinsifier_x64.cc +++ b/runtime/vm/intrinsifier_x64.cc @@ -1754,6 +1754,23 @@ void Intrinsifier::String_getHashCode(Assembler* assembler) { } +void Intrinsifier::Object_getHash(Assembler* assembler) { + __ movq(RAX, Address(RSP, +1 * kWordSize)); // Object. + __ movl(RAX, FieldAddress(RAX, String::hash_offset())); + __ SmiTag(RAX); + __ ret(); +} + + +void Intrinsifier::Object_setHash(Assembler* assembler) { + __ movq(RAX, Address(RSP, +2 * kWordSize)); // Object. + __ movq(RDX, Address(RSP, +1 * kWordSize)); // Value. + __ SmiUntag(RDX); + __ movl(FieldAddress(RAX, String::hash_offset()), RDX); + __ ret(); +} + + void GenerateSubstringMatchesSpecialization(Assembler* assembler, intptr_t receiver_cid, intptr_t other_cid, diff --git a/runtime/vm/method_recognizer.h b/runtime/vm/method_recognizer.h index 33416164c78..d8624443056 100644 --- a/runtime/vm/method_recognizer.h +++ b/runtime/vm/method_recognizer.h @@ -139,7 +139,7 @@ namespace dart { // List of intrinsics: // (class-name, function-name, intrinsification method, fingerprint). -#define CORE_LIB_INTRINSIC_LIST(V) \ +#define CORE_LIB_NON_HASH_INTRINSIC_LIST(V) \ V(_Smi, ~, Smi_bitNegate, Smi, 0x6574c6b0) \ V(_Smi, get:bitLength, Smi_bitLength, Smi, 0x25b356ab) \ V(_Smi, _bitAndFromSmi, Smi_bitAndFromSmi, Smi, 0x490a4da1) \ @@ -190,6 +190,19 @@ namespace dart { V(_TwoByteString, ==, TwoByteString_equality, Bool, 0x4719e83f) \ +#define CORE_LIB_IN_HEADER_HASH_INTRINSIC_LIST(V) \ + V(::, _getHash, Object_getHash, Smi, 0x2827856d) \ + V(::, _setHash, Object_setHash, Object, 0x302d1fe8) \ + +#if defined(HASH_IN_OBJECT_HEADER) +#define CORE_LIB_INTRINSIC_LIST(V) \ + CORE_LIB_NON_HASH_INTRINSIC_LIST(V) \ + CORE_LIB_IN_HEADER_HASH_INTRINSIC_LIST(V) +#else +#define CORE_LIB_INTRINSIC_LIST(V) \ + CORE_LIB_NON_HASH_INTRINSIC_LIST(V) +#endif + #define CORE_INTEGER_LIB_INTRINSIC_LIST(V) \ V(_IntegerImplementation, _addFromInteger, Integer_addFromInteger, \ Dynamic, 0x6a10c54a) \ diff --git a/runtime/vm/object.cc b/runtime/vm/object.cc index c826ac30dd3..d4642901f64 100644 --- a/runtime/vm/object.cc +++ b/runtime/vm/object.cc @@ -992,10 +992,17 @@ void Object::InitOnce(Isolate* isolate) { // An object visitor which will mark all visited objects. This is used to -// premark all objects in the vm_isolate_ heap. -class PremarkingVisitor : public ObjectVisitor { +// premark all objects in the vm_isolate_ heap. Also precalculates hash +// codes so that we can get the identity hash code of objects in the read- +// only VM isolate. +class FinalizeVMIsolateVisitor : public ObjectVisitor { public: - PremarkingVisitor() {} + FinalizeVMIsolateVisitor() +#if defined(HASH_IN_OBJECT_HEADER) + : counter_(1337) +#endif + { + } void VisitObject(RawObject* obj) { // Free list elements should never be marked. @@ -1005,8 +1012,35 @@ class PremarkingVisitor : public ObjectVisitor { if (!obj->IsFreeListElement()) { ASSERT(obj->IsVMHeapObject()); obj->SetMarkBitUnsynchronized(); + if (obj->IsStringInstance()) { + RawString* str = reinterpret_cast(obj); + intptr_t hash = String::Hash(str); + String::SetCachedHash(str, hash); + } +#if defined(HASH_IN_OBJECT_HEADER) + // These objects end up in the read-only VM isolate which is shared + // between isolates, so we have to prepopulate them with identity hash + // codes, since we can't add hash codes later. + if (Object::GetCachedHash(obj) == 0) { + // Some classes have identity hash codes that depend on their contents, + // not per object. + ASSERT(!obj->IsStringInstance()); + if (!obj->IsMint() && !obj->IsDouble() && !obj->IsBigint() && + !obj->IsRawNull() && !obj->IsBool()) { + counter_ += 2011; // The year Dart was announced and a prime. + counter_ &= 0x3fffffff; + if (counter_ == 0) counter_++; + Object::SetCachedHash(obj, counter_); + } + } +#endif } } + + private: +#if defined(HASH_IN_OBJECT_HEADER) + int32_t counter_; +#endif }; @@ -1086,7 +1120,7 @@ void Object::FinalizeVMIsolate(Isolate* isolate) { { ASSERT(isolate == Dart::vm_isolate()); WritableVMIsolateScope scope(Thread::Current()); - PremarkingVisitor premarker; + FinalizeVMIsolateVisitor premarker; ASSERT(isolate->heap()->UsedInWords(Heap::kNew) == 0); isolate->heap()->IterateOldObjectsNoImagePages(&premarker); // Make the VM isolate read-only again after setting all objects as marked. @@ -1121,13 +1155,15 @@ void Object::MakeUnusedSpaceTraversable(const Object& obj, reinterpret_cast(RawObject::FromAddr(addr)); uword new_tags = RawObject::ClassIdTag::update(kTypedDataInt8ArrayCid, 0); new_tags = RawObject::SizeTag::update(leftover_size, new_tags); - uword tags = raw->ptr()->tags_; - uword old_tags; + uint32_t tags = raw->ptr()->tags_; + uint32_t old_tags; // TODO(iposva): Investigate whether CompareAndSwapWord is necessary. do { old_tags = tags; - tags = AtomicOperations::CompareAndSwapWord(&raw->ptr()->tags_, - old_tags, new_tags); + // We can't use obj.CompareAndSwapTags here because we don't have a + // handle for the new object. + tags = AtomicOperations::CompareAndSwapUint32(&raw->ptr()->tags_, + old_tags, new_tags); } while (tags != old_tags); intptr_t leftover_len = (leftover_size - TypedData::InstanceSize(0)); @@ -1139,13 +1175,12 @@ void Object::MakeUnusedSpaceTraversable(const Object& obj, RawObject* raw = reinterpret_cast(RawObject::FromAddr(addr)); uword new_tags = RawObject::ClassIdTag::update(kInstanceCid, 0); new_tags = RawObject::SizeTag::update(leftover_size, new_tags); - uword tags = raw->ptr()->tags_; - uword old_tags; + uint32_t tags = raw->ptr()->tags_; + uint32_t old_tags; // TODO(iposva): Investigate whether CompareAndSwapWord is necessary. do { old_tags = tags; - tags = AtomicOperations::CompareAndSwapWord(&raw->ptr()->tags_, - old_tags, new_tags); + tags = obj.CompareAndSwapTags(old_tags, new_tags); } while (tags != old_tags); } } @@ -1867,12 +1902,15 @@ void Object::InitializeObject(uword address, *reinterpret_cast(cur) = initial_value; cur += kWordSize; } - uword tags = 0; + uint32_t tags = 0; ASSERT(class_id != kIllegalCid); tags = RawObject::ClassIdTag::update(class_id, tags); tags = RawObject::SizeTag::update(size, tags); tags = RawObject::VMHeapObjectTag::update(is_vm_object, tags); reinterpret_cast(address)->tags_ = tags; +#if defined(HASH_IN_OBJECT_HEADER) + reinterpret_cast(address)->hash_ = 0; +#endif ASSERT(is_vm_object == RawObject::IsVMHeapObject(tags)); } @@ -20327,6 +20365,35 @@ static intptr_t HashImpl(const T* characters, intptr_t len) { } +intptr_t String::Hash(RawString* raw) { + StringHasher hasher; + uword length = Smi::Value(raw->ptr()->length_); + if (raw->IsOneByteString() || raw->IsExternalOneByteString()) { + const uint8_t* data; + if (raw->IsOneByteString()) { + data = reinterpret_cast(raw)->ptr()->data(); + } else { + ASSERT(raw->IsExternalOneByteString()); + RawExternalOneByteString* str = + reinterpret_cast(raw); + data = str->ptr()->external_data_->data(); + } + return String::Hash(data, length); + } else { + const uint16_t* data; + if (raw->IsTwoByteString()) { + data = reinterpret_cast(raw)->ptr()->data(); + } else { + ASSERT(raw->IsExternalTwoByteString()); + RawExternalTwoByteString* str = + reinterpret_cast(raw); + data = str->ptr()->external_data_->data(); + } + return String::Hash(data, length); + } +} + + intptr_t String::Hash(const char* characters, intptr_t len) { return HashImpl(characters, len); } @@ -21147,11 +21214,11 @@ RawString* String::MakeExternal(void* array, // Update the class information of the object. const intptr_t class_id = kExternalOneByteStringCid; - uword tags = raw_ptr()->tags_; - uword old_tags; + uint32_t tags = raw_ptr()->tags_; + uint32_t old_tags; do { old_tags = tags; - uword new_tags = RawObject::SizeTag::update(used_size, old_tags); + uint32_t new_tags = RawObject::SizeTag::update(used_size, old_tags); new_tags = RawObject::ClassIdTag::update(class_id, new_tags); tags = CompareAndSwapTags(old_tags, new_tags); } while (tags != old_tags); @@ -21184,11 +21251,11 @@ RawString* String::MakeExternal(void* array, // Update the class information of the object. const intptr_t class_id = kExternalTwoByteStringCid; - uword tags = raw_ptr()->tags_; - uword old_tags; + uint32_t tags = raw_ptr()->tags_; + uint32_t old_tags; do { old_tags = tags; - uword new_tags = RawObject::SizeTag::update(used_size, old_tags); + uint32_t new_tags = RawObject::SizeTag::update(used_size, old_tags); new_tags = RawObject::ClassIdTag::update(class_id, new_tags); tags = CompareAndSwapTags(old_tags, new_tags); } while (tags != old_tags); @@ -22076,11 +22143,11 @@ void Array::MakeImmutable() const { if (IsImmutable()) return; ASSERT(!IsCanonical()); NoSafepointScope no_safepoint; - uword tags = raw_ptr()->tags_; - uword old_tags; + uint32_t tags = raw_ptr()->tags_; + uint32_t old_tags; do { old_tags = tags; - uword new_tags = + uint32_t new_tags = RawObject::ClassIdTag::update(kImmutableArrayCid, old_tags); tags = CompareAndSwapTags(old_tags, new_tags); } while (tags != old_tags); @@ -22145,6 +22212,7 @@ RawArray* Array::MakeFixedLength(const GrowableObjectArray& growable_array, } intptr_t capacity_len = growable_array.Capacity(); const Array& array = Array::Handle(zone, growable_array.data()); + ASSERT(array.IsArray()); array.SetTypeArguments(type_arguments); intptr_t capacity_size = Array::InstanceSize(capacity_len); intptr_t used_size = Array::InstanceSize(used_len); @@ -22158,10 +22226,10 @@ RawArray* Array::MakeFixedLength(const GrowableObjectArray& growable_array, // Update the size in the header field and length of the array object. uword tags = array.raw_ptr()->tags_; ASSERT(kArrayCid == RawObject::ClassIdTag::decode(tags)); - uword old_tags; + uint32_t old_tags; do { old_tags = tags; - uword new_tags = RawObject::SizeTag::update(used_size, old_tags); + uint32_t new_tags = RawObject::SizeTag::update(used_size, old_tags); tags = array.CompareAndSwapTags(old_tags, new_tags); } while (tags != old_tags); // TODO(22501): For the heap to remain walkable by the sweeper, it must diff --git a/runtime/vm/object.h b/runtime/vm/object.h index fae79c2ae23..4514c02705e 100644 --- a/runtime/vm/object.h +++ b/runtime/vm/object.h @@ -245,9 +245,9 @@ class Object { RawObject* raw() const { return raw_; } void operator=(RawObject* value) { initializeHandle(this, value); } - uword CompareAndSwapTags(uword old_tags, uword new_tags) const { - return AtomicOperations::CompareAndSwapWord(&raw()->ptr()->tags_, old_tags, - new_tags); + uint32_t CompareAndSwapTags(uint32_t old_tags, uint32_t new_tags) const { + return AtomicOperations::CompareAndSwapUint32(&raw()->ptr()->tags_, + old_tags, new_tags); } bool IsCanonical() const { return raw()->IsCanonical(); } void SetCanonical() const { raw()->SetCanonical(); } @@ -434,13 +434,11 @@ class Object { #if defined(HASH_IN_OBJECT_HEADER) static uint32_t GetCachedHash(const RawObject* obj) { - uword tags = obj->ptr()->tags_; - return tags >> 32; + return obj->ptr()->hash_; } - static void SetCachedHash(RawObject* obj, uintptr_t hash) { - ASSERT(hash >> 32 == 0); - obj->ptr()->tags_ |= hash << 32; + static void SetCachedHash(RawObject* obj, uint32_t hash) { + obj->ptr()->hash_ = hash; } #endif @@ -6831,16 +6829,14 @@ class String : public Instance { return result; } + static intptr_t Hash(RawString* raw); + bool HasHash() const { ASSERT(Smi::New(0) == NULL); return GetCachedHash(raw()) != 0; } -#if defined(HASH_IN_OBJECT_HEADER) - static intptr_t hash_offset() { return kInt32Size; } // Wrong for big-endian? -#else static intptr_t hash_offset() { return OFFSET_OF(RawString, hash_); } -#endif static intptr_t Hash(const String& str, intptr_t begin_index, intptr_t len); static intptr_t Hash(const char* characters, intptr_t len); static intptr_t Hash(const uint16_t* characters, intptr_t len); diff --git a/runtime/vm/profiler.cc b/runtime/vm/profiler.cc index 6eaca60e708..a7c3f0e9ee7 100644 --- a/runtime/vm/profiler.cc +++ b/runtime/vm/profiler.cc @@ -1322,7 +1322,7 @@ class CodeLookupTableBuilder : public ObjectVisitor { ~CodeLookupTableBuilder() {} void VisitObject(RawObject* raw_obj) { - uword tags = raw_obj->ptr()->tags_; + uint32_t tags = raw_obj->ptr()->tags_; if (RawObject::ClassIdTag::decode(tags) == kCodeCid) { RawCode* raw_code = reinterpret_cast(raw_obj); const Code& code = Code::Handle(raw_code); diff --git a/runtime/vm/raw_object.cc b/runtime/vm/raw_object.cc index 15dfe26aecd..3a0a5a6f864 100644 --- a/runtime/vm/raw_object.cc +++ b/runtime/vm/raw_object.cc @@ -30,10 +30,10 @@ void RawObject::Validate(Isolate* isolate) const { FATAL("RAW_NULL encountered"); } // Validate that the tags_ field is sensible. - uword tags = ptr()->tags_; + uint32_t tags = ptr()->tags_; intptr_t reserved = ReservedBits::decode(tags); if (reserved != 0) { - FATAL1("Invalid tags field encountered %#" Px "\n", tags); + FATAL1("Invalid tags field encountered %x\n", tags); } intptr_t class_id = ClassIdTag::decode(tags); if (!isolate->class_table()->IsValidIndex(class_id)) { @@ -185,7 +185,7 @@ intptr_t RawObject::SizeFromClass() const { ClassTable* class_table = isolate->class_table(); if (!class_table->IsValidIndex(class_id) || !class_table->HasValidClassAt(class_id)) { - FATAL2("Invalid class id: %" Pd " from tags %" Px "\n", class_id, + FATAL2("Invalid class id: %" Pd " from tags %x\n", class_id, ptr()->tags_); } #endif // DEBUG @@ -196,7 +196,7 @@ intptr_t RawObject::SizeFromClass() const { } ASSERT(instance_size != 0); #if defined(DEBUG) - uword tags = ptr()->tags_; + uint32_t tags = ptr()->tags_; intptr_t tags_size = SizeTag::decode(tags); if ((class_id == kArrayCid) && (instance_size > tags_size && tags_size > 0)) { // TODO(22501): Array::MakeFixedLength could be in the process of shrinking @@ -211,7 +211,7 @@ intptr_t RawObject::SizeFromClass() const { } while ((instance_size > tags_size) && (--retries_remaining > 0)); } if ((instance_size != tags_size) && (tags_size != 0)) { - FATAL3("Size mismatch: %" Pd " from class vs %" Pd " from tags %" Px "\n", + FATAL3("Size mismatch: %" Pd " from class vs %" Pd " from tags %x\n", instance_size, tags_size, tags); } #endif // DEBUG @@ -524,7 +524,7 @@ intptr_t RawNamespace::VisitNamespacePointers(RawNamespace* raw_obj, bool RawCode::ContainsPC(RawObject* raw_obj, uword pc) { - uword tags = raw_obj->ptr()->tags_; + uint32_t tags = raw_obj->ptr()->tags_; if (RawObject::ClassIdTag::decode(tags) == kCodeCid) { RawCode* raw_code = reinterpret_cast(raw_obj); return RawInstructions::ContainsPC(raw_code->ptr()->instructions_, pc); @@ -737,7 +737,7 @@ intptr_t RawInstance::VisitInstancePointers(RawInstance* raw_obj, ObjectPointerVisitor* visitor) { // Make sure that we got here with the tagged pointer as this. ASSERT(raw_obj->IsHeapObject()); - uword tags = raw_obj->ptr()->tags_; + uint32_t tags = raw_obj->ptr()->tags_; intptr_t instance_size = SizeTag::decode(tags); if (instance_size == 0) { RawClass* cls = diff --git a/runtime/vm/raw_object.h b/runtime/vm/raw_object.h index db8bce33a71..b1626ea5bf2 100644 --- a/runtime/vm/raw_object.h +++ b/runtime/vm/raw_object.h @@ -271,6 +271,10 @@ class RawObject { kSizeTagSize = 8, kClassIdTagPos = kSizeTagPos + kSizeTagSize, // = 16 kClassIdTagSize = 16, +#if defined(HASH_IN_OBJECT_HEADER) + kHashTagPos = kClassIdTagPos + kClassIdTagSize, // = 32 + kHashTagSize = 16, +#endif }; COMPILE_ASSERT(kClassIdTagSize == (sizeof(classid_t) * kBitsPerByte)); @@ -296,7 +300,7 @@ class RawObject { private: // The actual unscaled bit field used within the tag field. class SizeBits - : public BitField {}; + : public BitField {}; static intptr_t SizeToTagValue(intptr_t size) { ASSERT(Utils::IsAligned(size, kObjectAlignment)); @@ -308,7 +312,7 @@ class RawObject { }; class ClassIdTag - : public BitField {}; + : public BitField {}; bool IsWellFormed() const { uword value = reinterpret_cast(this); @@ -352,7 +356,7 @@ class RawObject { } void SetMarkBitUnsynchronized() { ASSERT(!IsMarked()); - uword tags = ptr()->tags_; + uint32_t tags = ptr()->tags_; ptr()->tags_ = MarkBit::update(true, tags); } void ClearMarkBit() { @@ -378,12 +382,12 @@ class RawObject { } void SetRememberedBitUnsynchronized() { ASSERT(!IsRemembered()); - uword tags = ptr()->tags_; + uint32_t tags = ptr()->tags_; ptr()->tags_ = RememberedBit::update(true, tags); } void ClearRememberedBit() { UpdateTagBit(false); } void ClearRememberedBitUnsynchronized() { - uword tags = ptr()->tags_; + uint32_t tags = ptr()->tags_; ptr()->tags_ = RememberedBit::update(false, tags); } // Returns false if the bit was already set. @@ -409,6 +413,7 @@ class RawObject { #undef DEFINE_IS_CID bool IsStringInstance() const { return IsStringClassId(GetClassId()); } + bool IsRawNull() const { return GetClassId() == kNullCid; } bool IsDartInstance() const { return (!IsHeapObject() || (GetClassId() >= kInstanceCid)); } @@ -423,7 +428,7 @@ class RawObject { } intptr_t Size() const { - uword tags = ptr()->tags_; + uint32_t tags = ptr()->tags_; intptr_t result = SizeTag::decode(tags); if (result != 0) { #if defined(DEBUG) @@ -533,18 +538,25 @@ class RawObject { static intptr_t NumberOfTypedDataClasses(); private: - uword tags_; // Various object tags (bits). + uint32_t tags_; // Various object tags (bits). +#if defined(HASH_IN_OBJECT_HEADER) + // On 64 bit there is a hash field in the header for the identity hash. + uint32_t hash_; +#endif - class MarkBit : public BitField {}; + class MarkBit : public BitField {}; - class RememberedBit : public BitField {}; + class RememberedBit : public BitField {}; - class CanonicalObjectTag : public BitField {}; + class CanonicalObjectTag : public BitField { + }; - class VMHeapObjectTag : public BitField {}; + class VMHeapObjectTag : public BitField { + }; class ReservedBits - : public BitField {}; + : public BitField { + }; // TODO(koda): After handling tags_, return const*, like Object::raw_ptr(). RawObject* ptr() const { @@ -559,37 +571,37 @@ class RawObject { intptr_t SizeFromClass() const; intptr_t GetClassId() const { - uword tags = ptr()->tags_; + uint32_t tags = ptr()->tags_; return ClassIdTag::decode(tags); } void SetClassId(intptr_t new_cid) { - uword tags = ptr()->tags_; + uint32_t tags = ptr()->tags_; ptr()->tags_ = ClassIdTag::update(new_cid, tags); } template void UpdateTagBit(bool value) { - uword tags = ptr()->tags_; - uword old_tags; + uint32_t tags = ptr()->tags_; + uint32_t old_tags; do { old_tags = tags; - uword new_tags = TagBitField::update(value, old_tags); - tags = AtomicOperations::CompareAndSwapWord(&ptr()->tags_, old_tags, - new_tags); + uint32_t new_tags = TagBitField::update(value, old_tags); + tags = AtomicOperations::CompareAndSwapUint32(&ptr()->tags_, old_tags, + new_tags); } while (tags != old_tags); } template bool TryAcquireTagBit() { - uword tags = ptr()->tags_; - uword old_tags; + uint32_t tags = ptr()->tags_; + uint32_t old_tags; do { old_tags = tags; if (TagBitField::decode(tags)) return false; - uword new_tags = TagBitField::update(true, old_tags); - tags = AtomicOperations::CompareAndSwapWord(&ptr()->tags_, old_tags, - new_tags); + uint32_t new_tags = TagBitField::update(true, old_tags); + tags = AtomicOperations::CompareAndSwapUint32(&ptr()->tags_, old_tags, + new_tags); } while (tags != old_tags); return true; } @@ -1318,7 +1330,10 @@ class RawPcDescriptors : public RawObject { private: RAW_HEAP_OBJECT_IMPLEMENTATION(PcDescriptors); - int32_t length_; // Number of descriptors. + // Number of descriptors. This only needs to be an int32_t, but we make it a + // uword so that the variable length data is 64 bit aligned on 64 bit + // platforms. + uword length_; // Variable length data follows here. uint8_t* data() { OPEN_ARRAY_START(uint8_t, intptr_t); } @@ -1334,7 +1349,9 @@ class RawCodeSourceMap : public RawObject { private: RAW_HEAP_OBJECT_IMPLEMENTATION(CodeSourceMap); - int32_t length_; // Length in bytes. + // Length in bytes. This only needs to be an int32_t, but we make it a uword + // so that the variable length data is 64 bit aligned on 64 bit platforms. + uword length_; // Variable length data follows here. uint8_t* data() { OPEN_ARRAY_START(uint8_t, intptr_t); } @@ -1361,8 +1378,9 @@ class RawStackMap : public RawObject { int32_t slow_path_bit_count_; // Slow path live values, included in length_. // Offset from code entry point corresponding to this stack map - // representation. - uint32_t pc_offset_; + // representation. This only needs to be an int32_t, but we make it a uword + // so that the variable length data is 64 bit aligned on 64 bit platforms. + uword pc_offset_; // Variable length data follows here (bitmap of the stack layout). uint8_t* data() { OPEN_ARRAY_START(uint8_t, uint8_t); } @@ -1416,7 +1434,10 @@ class RawLocalVarDescriptors : public RawObject { private: RAW_HEAP_OBJECT_IMPLEMENTATION(LocalVarDescriptors); - int32_t num_entries_; // Number of descriptors. + // Number of descriptors. This only needs to be an int32_t, but we make it a + // uword so that the variable length data is 64 bit aligned on 64 bit + // platforms. + uword num_entries_; RawObject** from() { return reinterpret_cast(&ptr()->names()[0]); @@ -1924,8 +1945,9 @@ class RawOneByteString : public RawString { const uint8_t* data() const { OPEN_ARRAY_START(uint8_t, uint8_t); } friend class ApiMessageReader; - friend class SnapshotReader; friend class RODataSerializationCluster; + friend class SnapshotReader; + friend class String; }; @@ -1936,8 +1958,9 @@ class RawTwoByteString : public RawString { uint16_t* data() { OPEN_ARRAY_START(uint16_t, uint16_t); } const uint16_t* data() const { OPEN_ARRAY_START(uint16_t, uint16_t); } - friend class SnapshotReader; friend class RODataSerializationCluster; + friend class SnapshotReader; + friend class String; }; @@ -1973,6 +1996,7 @@ class RawExternalOneByteString : public RawString { private: ExternalData* external_data_; friend class Api; + friend class String; }; @@ -1985,6 +2009,7 @@ class RawExternalTwoByteString : public RawString { private: ExternalData* external_data_; friend class Api; + friend class String; }; diff --git a/runtime/vm/simulator_arm64.cc b/runtime/vm/simulator_arm64.cc index 6125bae52ac..99912dfa29c 100644 --- a/runtime/vm/simulator_arm64.cc +++ b/runtime/vm/simulator_arm64.cc @@ -1225,6 +1225,13 @@ intptr_t Simulator::ReadExclusiveX(uword addr, Instr* instr) { } +intptr_t Simulator::ReadExclusiveW(uword addr, Instr* instr) { + MutexLocker ml(exclusive_access_lock_); + SetExclusiveAccess(addr); + return ReadWU(addr, instr); +} + + intptr_t Simulator::WriteExclusiveX(uword addr, intptr_t value, Instr* instr) { MutexLocker ml(exclusive_access_lock_); bool write_allowed = HasExclusiveAccessAndOpen(addr); @@ -1236,6 +1243,17 @@ intptr_t Simulator::WriteExclusiveX(uword addr, intptr_t value, Instr* instr) { } +intptr_t Simulator::WriteExclusiveW(uword addr, intptr_t value, Instr* instr) { + MutexLocker ml(exclusive_access_lock_); + bool write_allowed = HasExclusiveAccessAndOpen(addr); + if (write_allowed) { + WriteW(addr, value, instr); + return 0; // Success. + } + return 1; // Failure. +} + + uword Simulator::CompareExchange(uword* address, uword compare_value, uword new_value) { @@ -2199,25 +2217,37 @@ void Simulator::DecodeLoadStoreExclusive(Instr* instr) { UNIMPLEMENTED(); } const int32_t size = instr->Bits(30, 2); - if (size != 3) { + if (size != 3 && size != 2) { UNIMPLEMENTED(); } - const Register rs = instr->RsField(); const Register rn = instr->RnField(); const Register rt = instr->RtField(); const bool is_load = instr->Bit(22) == 1; if (is_load) { // Format(instr, "ldxr 'rt, 'rn"); - const int64_t addr = get_register(rn, R31IsSP); - intptr_t value = ReadExclusiveX(addr, instr); - set_register(instr, rt, value, R31IsSP); + if (size == 3) { + const int64_t addr = get_register(rn, R31IsSP); + intptr_t value = ReadExclusiveX(addr, instr); + set_register(instr, rt, value, R31IsSP); + } else { + const int64_t addr = get_register(rn, R31IsSP); + intptr_t value = ReadExclusiveW(addr, instr); + set_register(instr, rt, value, R31IsSP); + } } else { // Format(instr, "stxr 'rs, 'rt, 'rn"); - uword value = get_register(rt, R31IsSP); - uword addr = get_register(rn, R31IsSP); - intptr_t status = WriteExclusiveX(addr, value, instr); - set_register(instr, rs, status, R31IsSP); + if (size == 3) { + uword value = get_register(rt, R31IsSP); + uword addr = get_register(rn, R31IsSP); + intptr_t status = WriteExclusiveX(addr, value, instr); + set_register(instr, rs, status, R31IsSP); + } else { + uint32_t value = get_register(rt, R31IsSP); + uword addr = get_register(rn, R31IsSP); + intptr_t status = WriteExclusiveW(addr, value, instr); + set_register(instr, rs, status, R31IsSP); + } } } diff --git a/runtime/vm/simulator_arm64.h b/runtime/vm/simulator_arm64.h index 43c057153f4..a1a36699e39 100644 --- a/runtime/vm/simulator_arm64.h +++ b/runtime/vm/simulator_arm64.h @@ -207,6 +207,9 @@ class Simulator { void ClearExclusive(); intptr_t ReadExclusiveX(uword addr, Instr* instr); intptr_t WriteExclusiveX(uword addr, intptr_t value, Instr* instr); + // 32 bit versions. + intptr_t ReadExclusiveW(uword addr, Instr* instr); + intptr_t WriteExclusiveW(uword addr, intptr_t value, Instr* instr); // Set access to given address to 'exclusive state' for current thread. static void SetExclusiveAccess(uword addr); diff --git a/runtime/vm/simulator_dbc.cc b/runtime/vm/simulator_dbc.cc index 3a9b2d8414b..127b6d4f6e4 100644 --- a/runtime/vm/simulator_dbc.cc +++ b/runtime/vm/simulator_dbc.cc @@ -322,6 +322,7 @@ class SimulatorHelpers { uword tags = 0; tags = RawObject::ClassIdTag::update(kDoubleCid, tags); tags = RawObject::SizeTag::update(instance_size, tags); + // Also writes zero in the hash_ field. *reinterpret_cast(start + Double::tags_offset()) = tags; *reinterpret_cast(start + Double::value_offset()) = value; return reinterpret_cast(start + kHeapObjectTag); @@ -2855,9 +2856,10 @@ RawObject* Simulator::Call(const Code& code, const intptr_t instance_size = Context::InstanceSize(num_context_variables); const uword start = thread->heap()->new_space()->TryAllocate(instance_size); if (LIKELY(start != 0)) { - uword tags = 0; + uint32_t tags = 0; tags = RawObject::ClassIdTag::update(kContextCid, tags); tags = RawObject::SizeTag::update(instance_size, tags); + // Also writes 0 in the hash_ field of the header. *reinterpret_cast(start + Array::tags_offset()) = tags; *reinterpret_cast(start + Context::num_variables_offset()) = num_context_variables; @@ -2898,6 +2900,7 @@ RawObject* Simulator::Call(const Code& code, const intptr_t instance_size = RawObject::SizeTag::decode(tags); const uword start = thread->heap()->new_space()->TryAllocate(instance_size); if (LIKELY(start != 0)) { + // Writes both the tags and the initial identity hash on 64 bit platforms. *reinterpret_cast(start + Instance::tags_offset()) = tags; for (intptr_t current_offset = sizeof(RawInstance); current_offset < instance_size; current_offset += kWordSize) { @@ -2929,6 +2932,7 @@ RawObject* Simulator::Call(const Code& code, if (LIKELY(start != 0)) { RawObject* type_args = SP[0]; const intptr_t type_args_offset = Bytecode::DecodeD(*pc); + // Writes both the tags and the initial identity hash on 64 bit platforms. *reinterpret_cast(start + Instance::tags_offset()) = tags; for (intptr_t current_offset = sizeof(RawInstance); current_offset < instance_size; current_offset += kWordSize) { @@ -2972,6 +2976,8 @@ RawObject* Simulator::Call(const Code& code, tags = RawObject::SizeTag::update(instance_size, tags); } tags = RawObject::ClassIdTag::update(cid, tags); + // Writes both the tags and the initial identity hash on 64 bit + // platforms. *reinterpret_cast(start + Instance::tags_offset()) = tags; *reinterpret_cast(start + Array::length_offset()) = FP[rB]; diff --git a/runtime/vm/snapshot.cc b/runtime/vm/snapshot.cc index 609262e3e5c..01f99f0a7b1 100644 --- a/runtime/vm/snapshot.cc +++ b/runtime/vm/snapshot.cc @@ -755,6 +755,9 @@ void ImageWriter::WriteROData(WriteStream* stream) { uword marked_tags = obj.raw()->ptr()->tags_; marked_tags = RawObject::VMHeapObjectTag::update(true, marked_tags); marked_tags = RawObject::MarkBit::update(true, marked_tags); +#if defined(HASH_IN_OBJECT_HEADER) + marked_tags |= static_cast(obj.raw()->ptr()->hash_) << 32; +#endif stream->WriteWord(marked_tags); start += sizeof(uword); for (uword* cursor = reinterpret_cast(start); @@ -841,6 +844,11 @@ void AssemblyImageWriter::WriteText(WriteStream* clustered_stream, bool vm) { uword marked_tags = insns.raw_ptr()->tags_; marked_tags = RawObject::VMHeapObjectTag::update(true, marked_tags); marked_tags = RawObject::MarkBit::update(true, marked_tags); +#if defined(HASH_IN_OBJECT_HEADER) + // Can't use GetObjectTagsAndHash because the update methods discard the + // high bits. + marked_tags |= static_cast(insns.raw_ptr()->hash_) << 32; +#endif WriteWordLiteralText(marked_tags); beginning += sizeof(uword); @@ -1046,6 +1054,11 @@ void BlobImageWriter::WriteText(WriteStream* clustered_stream, bool vm) { uword marked_tags = insns.raw_ptr()->tags_; marked_tags = RawObject::VMHeapObjectTag::update(true, marked_tags); marked_tags = RawObject::MarkBit::update(true, marked_tags); +#if defined(HASH_IN_OBJECT_HEADER) + // Can't use GetObjectTagsAndHash because the update methods discard the + // high bits. + marked_tags |= static_cast(insns.raw_ptr()->hash_) << 32; +#endif instructions_blob_stream_.WriteWord(marked_tags); beginning += sizeof(uword); @@ -1350,11 +1363,20 @@ void SnapshotWriter::WriteObject(RawObject* rawobj) { } -uword SnapshotWriter::GetObjectTags(RawObject* raw) { +uint32_t SnapshotWriter::GetObjectTags(RawObject* raw) { return raw->ptr()->tags_; } +uword SnapshotWriter::GetObjectTagsAndHash(RawObject* raw) { + uword result = raw->ptr()->tags_; +#if defined(HASH_IN_OBJECT_HEADER) + result |= static_cast(raw->ptr()->hash_) << 32; +#endif + return result; +} + + #define VM_OBJECT_CLASS_LIST(V) \ V(OneByteString) \ V(TwoByteString) \ @@ -1578,7 +1600,7 @@ void SnapshotWriter::WriteObjectImpl(RawObject* raw, bool as_reference) { // When we know that we are dealing with leaf or shallow objects we write // these objects inline even when 'as_reference' is true. const bool write_as_reference = as_reference && !raw->IsCanonical(); - intptr_t tags = raw->ptr()->tags_; + uintptr_t tags = GetObjectTagsAndHash(raw); // Add object to the forward ref list and mark it so that future references // to this object in the snapshot will use this object id. Mark the @@ -1655,7 +1677,7 @@ class WriteInlinedObjectVisitor : public ObjectVisitor { virtual void VisitObject(RawObject* obj) { intptr_t object_id = writer_->forward_list_->FindObject(obj); ASSERT(object_id != kInvalidIndex); - intptr_t tags = writer_->GetObjectTags(obj); + intptr_t tags = MessageWriter::GetObjectTagsAndHash(obj); writer_->WriteMarkedObjectImpl(obj, tags, object_id, kAsInlinedObject); } @@ -1820,7 +1842,7 @@ RawFunction* SnapshotWriter::IsSerializableClosure(RawClosure* closure) { RawClass* SnapshotWriter::GetFunctionOwner(RawFunction* func) { RawObject* owner = func->ptr()->owner_; - uword tags = GetObjectTags(owner); + uint32_t tags = GetObjectTags(owner); intptr_t class_id = RawObject::ClassIdTag::decode(tags); if (class_id == kClassCid) { return reinterpret_cast(owner); diff --git a/runtime/vm/snapshot.h b/runtime/vm/snapshot.h index faf5eede693..c0d10166108 100644 --- a/runtime/vm/snapshot.h +++ b/runtime/vm/snapshot.h @@ -842,7 +842,8 @@ class SnapshotWriter : public BaseWriter { // Serialize an object into the buffer. void WriteObject(RawObject* raw); - uword GetObjectTags(RawObject* raw); + static uint32_t GetObjectTags(RawObject* raw); + static uword GetObjectTagsAndHash(RawObject* raw); Exceptions::ExceptionType exception_type() const { return exception_type_; } void set_exception_type(Exceptions::ExceptionType type) { diff --git a/runtime/vm/stub_code_arm.cc b/runtime/vm/stub_code_arm.cc index b09dc0a6de7..c1e09c0010c 100644 --- a/runtime/vm/stub_code_arm.cc +++ b/runtime/vm/stub_code_arm.cc @@ -1152,7 +1152,7 @@ void StubCode::GenerateAllocationStubForClass(Assembler* assembler, // R1: next object start. // R9: allocation stats table. // Set the tags. - uword tags = 0; + uint32_t tags = 0; tags = RawObject::SizeTag::update(instance_size, tags); ASSERT(cls.id() != kIllegalCid); tags = RawObject::ClassIdTag::update(cls.id(), tags); diff --git a/runtime/vm/stub_code_arm64.cc b/runtime/vm/stub_code_arm64.cc index c59e6f217c9..fac4ef6be68 100644 --- a/runtime/vm/stub_code_arm64.cc +++ b/runtime/vm/stub_code_arm64.cc @@ -1090,7 +1090,7 @@ void StubCode::GenerateUpdateStoreBufferStub(Assembler* assembler) { Label add_to_buffer; // Check whether this object has already been remembered. Skip adding to the // store buffer if the object is in the store buffer already. - __ LoadFieldFromOffset(TMP, R0, Object::tags_offset()); + __ LoadFieldFromOffset(TMP, R0, Object::tags_offset(), kWord); __ tsti(TMP, Immediate(1 << RawObject::kRememberedBit)); __ b(&add_to_buffer, EQ); __ ret(); @@ -1105,11 +1105,13 @@ void StubCode::GenerateUpdateStoreBufferStub(Assembler* assembler) { ASSERT(Object::tags_offset() == 0); __ sub(R3, R0, Operand(kHeapObjectTag)); // R3: Untagged address of header word (ldxr/stxr do not support offsets). + // Note that we use 32 bit operations here to match the size of the + // background sweeper which is also manipulating this 32 bit word. Label retry; __ Bind(&retry); - __ ldxr(R2, R3); + __ ldxr(R2, R3, kWord); __ orri(R2, R2, Immediate(1 << RawObject::kRememberedBit)); - __ stxr(R1, R2, R3); + __ stxr(R1, R2, R3, kWord); __ cmp(R1, Operand(1)); __ b(&retry, EQ); @@ -1196,11 +1198,12 @@ void StubCode::GenerateAllocationStubForClass(Assembler* assembler, // R3: next object start. // R1: new object type arguments (if is_cls_parameterized). // Set the tags. - uword tags = 0; + uint32_t tags = 0; tags = RawObject::SizeTag::update(instance_size, tags); ASSERT(cls.id() != kIllegalCid); tags = RawObject::ClassIdTag::update(cls.id(), tags); __ LoadImmediate(R0, tags); + // 64 bit store also zeros the hash_field. __ StoreToOffset(R0, R2, Instance::tags_offset()); // Initialize the remaining words of the object. diff --git a/runtime/vm/stub_code_ia32.cc b/runtime/vm/stub_code_ia32.cc index 513c07d2637..445c0f21dca 100644 --- a/runtime/vm/stub_code_ia32.cc +++ b/runtime/vm/stub_code_ia32.cc @@ -1078,7 +1078,7 @@ void StubCode::GenerateAllocationStubForClass(Assembler* assembler, // EBX: next object start. // EDX: new object type arguments (if is_cls_parameterized). // Set the tags. - uword tags = 0; + uint32_t tags = 0; tags = RawObject::SizeTag::update(instance_size, tags); ASSERT(cls.id() != kIllegalCid); tags = RawObject::ClassIdTag::update(cls.id(), tags); diff --git a/runtime/vm/stub_code_mips.cc b/runtime/vm/stub_code_mips.cc index 410fd139789..294d2858dfa 100644 --- a/runtime/vm/stub_code_mips.cc +++ b/runtime/vm/stub_code_mips.cc @@ -1215,7 +1215,7 @@ void StubCode::GenerateAllocationStubForClass(Assembler* assembler, // T3: next object start. // T1: new object type arguments (if is_cls_parameterized). // Set the tags. - uword tags = 0; + uint32_t tags = 0; tags = RawObject::SizeTag::update(instance_size, tags); ASSERT(cls.id() != kIllegalCid); tags = RawObject::ClassIdTag::update(cls.id(), tags); diff --git a/runtime/vm/stub_code_x64.cc b/runtime/vm/stub_code_x64.cc index 58520becc4e..67a1832ad08 100644 --- a/runtime/vm/stub_code_x64.cc +++ b/runtime/vm/stub_code_x64.cc @@ -1034,21 +1034,23 @@ void StubCode::GenerateUpdateStoreBufferStub(Assembler* assembler) { // RDX: Address being stored Label reload; __ Bind(&reload); - __ movq(RAX, FieldAddress(RDX, Object::tags_offset())); - __ testq(RAX, Immediate(1 << RawObject::kRememberedBit)); + __ movl(RAX, FieldAddress(RDX, Object::tags_offset())); + __ testl(RAX, Immediate(1 << RawObject::kRememberedBit)); __ j(EQUAL, &add_to_buffer, Assembler::kNearJump); __ popq(RCX); __ popq(RAX); __ ret(); // Update the tags that this object has been remembered. + // Note that we use 32 bit operations here to match the size of the + // background sweeper which is also manipulating this 32 bit word. // RDX: Address being stored // RAX: Current tag value __ Bind(&add_to_buffer); - __ movq(RCX, RAX); - __ orq(RCX, Immediate(1 << RawObject::kRememberedBit)); + __ movl(RCX, RAX); + __ orl(RCX, Immediate(1 << RawObject::kRememberedBit)); // Compare the tag word with RAX, update to RCX if unchanged. - __ LockCmpxchgq(FieldAddress(RDX, Object::tags_offset()), RCX); + __ LockCmpxchgl(FieldAddress(RDX, Object::tags_offset()), RCX); __ j(NOT_EQUAL, &reload); // Load the StoreBuffer block out of the thread. Then load top_ out of the @@ -1132,10 +1134,11 @@ void StubCode::GenerateAllocationStubForClass(Assembler* assembler, // RBX: next object start. // RDX: new object type arguments (if is_cls_parameterized). // Set the tags. - uword tags = 0; + uint32_t tags = 0; tags = RawObject::SizeTag::update(instance_size, tags); ASSERT(cls.id() != kIllegalCid); tags = RawObject::ClassIdTag::update(cls.id(), tags); + // 64 bit store also zeros the identity hash field. __ movq(Address(RAX, Instance::tags_offset()), Immediate(tags)); __ addq(RAX, Immediate(kHeapObjectTag));