languages/dart-sdk
languages/dart-sdk
1
0
Fork 0
mirror of https://github.com/dart-lang/sdk synced 2024-09-05 08:24:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

VM: Reland Inline instance object hash code into object header on 64bit.

Browse source 
Inline instance object hash code into object header on 64 bit.

64 bit objects have 32 bits of free space in the header word.
This is used for the hash code in string objects. We take it
for the default hash code on all objects that don't override
the hashCode getter.

This is both faster and a memory reduction. Eg it makes the
MegaHashCode part of the Megamorphic benchmark 6 times faster.

This is a reland of https://codereview.chromium.org/2912863006/
It fixes issues with the 32 bit compare-swap instruction on
ARM64 and fixes a fragile tree shaking test that is sensitive
to which private methods are in the core libraries.

R=kustermann@google.com, vegorov@google.com
BUG=

Review-Url: https://codereview.chromium.org/2954453002 .
This commit is contained in:
Erik Corry 2017-06-22 14:56:20 +02:00
parent 3078734ff7
commit 8378b8fdbf
38 changed files with 501 additions and 151 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
pkg/front_end/testcases/shaker/empty_program.dart.shaker
View file

@ -53,8 +53,6 @@ library dart:core:
- toString
- noSuchMethod
- runtimeType
- dart.core::_getHash
- dart.core::_setHash
- dart.core::_objectHashCode
- dart.core::_identityHashCode
- dart.core::_toString

10
runtime/lib/object.cc
View file

@ -42,17 +42,25 @@ DEFINE_NATIVE_ENTRY(Object_getHash, 1) {
// Please note that no handle is created for the argument.
// This is safe since the argument is only used in a tail call.
// The performance benefit is more than 5% when using hashCode.
#if defined(HASH_IN_OBJECT_HEADER)
return Smi::New(Object::GetCachedHash(arguments->NativeArgAt(0)));
#else
Heap* heap = isolate->heap();
ASSERT(arguments->NativeArgAt(0)->IsDartInstance());
return Smi::New(heap->GetHash(arguments->NativeArgAt(0)));
#endif
}
DEFINE_NATIVE_ENTRY(Object_setHash, 2) {
const Instance& instance = Instance::CheckedHandle(arguments->NativeArgAt(0));
GET_NON_NULL_NATIVE_ARGUMENT(Smi, hash, arguments->NativeArgAt(1));
#if defined(HASH_IN_OBJECT_HEADER)
Object::SetCachedHash(arguments->NativeArgAt(0), hash.Value());
#else
const Instance& instance = Instance::CheckedHandle(arguments->NativeArgAt(0));
Heap* heap = isolate->heap();
heap->SetHash(instance.raw(), hash.Value());
#endif
return Object::null();
}

15
runtime/lib/object_patch.dart
View file

@ -2,6 +2,9 @@
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
int _getHash(obj) native "Object_getHash";
int _setHash(obj, hash) native "Object_setHash";
@patch
class Object {
// The VM has its own implementation of equals.
@ -9,22 +12,18 @@ class Object {
bool operator ==(other) native "Object_equals";
// Helpers used to implement hashCode. If a hashCode is used, we remember it
// in a weak table in the VM. A new hashCode value is calculated using a
// number generator.
// in a weak table in the VM (32 bit) or in the header of the object (64
// bit). A new hashCode value is calculated using a random number generator.
static final _hashCodeRnd = new Random();
static _getHash(obj) native "Object_getHash";
static _setHash(obj, hash) native "Object_setHash";
// Shared static implentation for hashCode and _identityHashCode.
static int _objectHashCode(obj) {
var result = _getHash(obj);
if (result == 0) {
// We want the hash to be a Smi value greater than 0.
result = _hashCodeRnd.nextInt(0x40000000);
while (result == 0) {
do {
result = _hashCodeRnd.nextInt(0x40000000);
}
} while (result == 0);
_setHash(obj, result);
}
return result;

4
runtime/vm/assembler_arm.cc
View file

@ -3471,7 +3471,7 @@ void Assembler::TryAllocate(const Class& cls,
ASSERT(instance_size >= kHeapObjectTag);
AddImmediate(instance_reg, -instance_size + kHeapObjectTag);
uword tags = 0;
uint32_t tags = 0;
tags = RawObject::SizeTag::update(instance_size, tags);
ASSERT(cls.id() != kIllegalCid);
tags = RawObject::ClassIdTag::update(cls.id(), tags);
@ -3520,7 +3520,7 @@ void Assembler::TryAllocateArray(intptr_t cid,
// Initialize the tags.
// instance: new object start as a tagged pointer.
uword tags = 0;
uint32_t tags = 0;
tags = RawObject::ClassIdTag::update(cid, tags);
tags = RawObject::SizeTag::update(instance_size, tags);
LoadImmediate(temp1, tags);

8
runtime/vm/assembler_arm64.cc
View file

@ -1363,10 +1363,12 @@ void Assembler::TryAllocate(const Class& cls,
AddImmediate(instance_reg, -instance_size + kHeapObjectTag);
NOT_IN_PRODUCT(UpdateAllocationStats(cls.id(), space));
uword tags = 0;
uint32_t tags = 0;
tags = RawObject::SizeTag::update(instance_size, tags);
ASSERT(cls.id() != kIllegalCid);
tags = RawObject::ClassIdTag::update(cls.id(), tags);
// Extends the 32 bit tags with zeros, which is the uninitialized
// hash code.
LoadImmediate(TMP, tags);
StoreFieldToOffset(TMP, instance_reg, Object::tags_offset());
} else {
@ -1410,9 +1412,11 @@ void Assembler::TryAllocateArray(intptr_t cid,
// Initialize the tags.
// instance: new object start as a tagged pointer.
uword tags = 0;
uint32_t tags = 0;
tags = RawObject::ClassIdTag::update(cid, tags);
tags = RawObject::SizeTag::update(instance_size, tags);
// Extends the 32 bit tags with zeros, which is the uninitialized
// hash code.
LoadImmediate(temp2, tags);
str(temp2, FieldAddress(instance, Array::tags_offset())); // Store tags.
} else {

15
runtime/vm/assembler_arm64.h
View file

@ -742,16 +742,19 @@ class Assembler : public ValueObject {
EmitLoadStoreRegPair(STP, rt, rt2, a, sz);
}
void ldxr(Register rt, Register rn) {
void ldxr(Register rt, Register rn, OperandSize size = kDoubleWord) {
// rt = value
// rn = address
EmitLoadStoreExclusive(LDXR, R31, rn, rt, kDoubleWord);
EmitLoadStoreExclusive(LDXR, R31, rn, rt, size);
}
void stxr(Register rs, Register rt, Register rn) {
void stxr(Register rs,
Register rt,
Register rn,
OperandSize size = kDoubleWord) {
// rs = status (1 = failure, 0 = success)
// rt = value
// rn = address
EmitLoadStoreExclusive(STXR, rs, rn, rt, kDoubleWord);
EmitLoadStoreExclusive(STXR, rs, rn, rt, size);
}
void clrex() {
const int32_t encoding = static_cast<int32_t>(CLREX);
@ -1701,8 +1704,8 @@ class Assembler : public ValueObject {
Register rn,
Register rt,
OperandSize sz = kDoubleWord) {
ASSERT(sz == kDoubleWord);
const int32_t size = B31 | B30;
ASSERT(sz == kDoubleWord || sz == kWord);
const int32_t size = B31 | (sz == kDoubleWord ? B30 : 0);
ASSERT((rs != kNoRegister) && (rs != ZR));
ASSERT((rn != kNoRegister) && (rn != ZR));

66
runtime/vm/assembler_arm64_test.cc
View file

@ -580,7 +580,7 @@ ASSEMBLER_TEST_GENERATE(Semaphore, assembler) {
ASSEMBLER_TEST_RUN(Semaphore, test) {
EXPECT(test != NULL);
typedef int (*Semaphore)() DART_UNUSED;
typedef intptr_t (*Semaphore)() DART_UNUSED;
EXPECT_EQ(42, EXECUTE_TEST_CODE_INT64(Semaphore, test->entry()));
}
@ -602,11 +602,73 @@ ASSEMBLER_TEST_GENERATE(FailedSemaphore, assembler) {
ASSEMBLER_TEST_RUN(FailedSemaphore, test) {
EXPECT(test != NULL);
typedef int (*FailedSemaphore)() DART_UNUSED;
typedef intptr_t (*FailedSemaphore)() DART_UNUSED;
EXPECT_EQ(41, EXECUTE_TEST_CODE_INT64(FailedSemaphore, test->entry()));
}
ASSEMBLER_TEST_GENERATE(Semaphore32, assembler) {
__ SetupDartSP();
__ movz(R0, Immediate(40), 0);
__ add(R0, R0, Operand(R0, LSL, 32));
__ Push(R0);
__ movz(R0, Immediate(40), 0);
__ movz(R1, Immediate(42), 0);
Label retry;
__ Bind(&retry);
__ ldxr(R0, SP, kWord);
// 32 bit operation should ignore the high word of R0 that was pushed on the
// stack.
__ stxr(TMP, R1, SP, kWord); // IP == 0, success
__ cmp(TMP, Operand(0));
__ b(&retry, NE); // NE if context switch occurred between ldrex and strex.
__ Pop(R0); // 42 + 42 * 2**32
__ RestoreCSP();
__ ret();
}
ASSEMBLER_TEST_RUN(Semaphore32, test) {
EXPECT(test != NULL);
typedef intptr_t (*Semaphore32)() DART_UNUSED;
// Lower word has been atomically switched from 40 to 42k, whereas upper word
// is unchanged at 40.
EXPECT_EQ(42 + (40l << 32),
EXECUTE_TEST_CODE_INT64(Semaphore32, test->entry()));
}
ASSEMBLER_TEST_GENERATE(FailedSemaphore32, assembler) {
__ SetupDartSP();
__ movz(R0, Immediate(40), 0);
__ add(R0, R0, Operand(R0, LSL, 32));
__ Push(R0);
__ movz(R0, Immediate(40), 0);
__ movz(R1, Immediate(42), 0);
__ ldxr(R0, SP, kWord);
__ clrex(); // Simulate a context switch.
__ stxr(TMP, R1, SP, kWord); // IP == 1, failure
__ Pop(R0); // 40
__ add(R0, R0, Operand(TMP));
__ RestoreCSP();
__ ret();
}
ASSEMBLER_TEST_RUN(FailedSemaphore32, test) {
EXPECT(test != NULL);
typedef intptr_t (*FailedSemaphore32)() DART_UNUSED;
// Lower word has had the failure code (1) added to it. Upper word is
// unchanged at 40.
EXPECT_EQ(41 + (40l << 32),
EXECUTE_TEST_CODE_INT64(FailedSemaphore32, test->entry()));
}
// Logical register operations.
ASSEMBLER_TEST_GENERATE(AndRegs, assembler) {
__ movz(R1, Immediate(43), 0);

4
runtime/vm/assembler_ia32.cc
View file

@ -2643,7 +2643,7 @@ void Assembler::TryAllocate(const Class& cls,
NOT_IN_PRODUCT(UpdateAllocationStats(cls.id(), temp_reg, space));
ASSERT(instance_size >= kHeapObjectTag);
subl(instance_reg, Immediate(instance_size - kHeapObjectTag));
uword tags = 0;
uint32_t tags = 0;
tags = RawObject::SizeTag::update(instance_size, tags);
ASSERT(cls.id() != kIllegalCid);
tags = RawObject::ClassIdTag::update(cls.id(), tags);
@ -2690,7 +2690,7 @@ void Assembler::TryAllocateArray(intptr_t cid,
UpdateAllocationStatsWithSize(cid, instance_size, temp_reg, space));
// Initialize the tags.
uword tags = 0;
uint32_t tags = 0;
tags = RawObject::ClassIdTag::update(cid, tags);
tags = RawObject::SizeTag::update(instance_size, tags);
movl(FieldAddress(instance, Object::tags_offset()), Immediate(tags));

4
runtime/vm/assembler_mips.cc
View file

@ -999,7 +999,7 @@ void Assembler::TryAllocate(const Class& cls,
ASSERT(instance_size >= kHeapObjectTag);
AddImmediate(instance_reg, -instance_size + kHeapObjectTag);
NOT_IN_PRODUCT(UpdateAllocationStats(cls.id(), temp_reg, space));
uword tags = 0;
uint32_t tags = 0;
tags = RawObject::SizeTag::update(instance_size, tags);
ASSERT(cls.id() != kIllegalCid);
tags = RawObject::ClassIdTag::update(cls.id(), tags);
@ -1049,7 +1049,7 @@ void Assembler::TryAllocateArray(intptr_t cid,
// Initialize the tags.
// instance: new object start as a tagged pointer.
uword tags = 0;
uint32_t tags = 0;
tags = RawObject::ClassIdTag::update(cid, tags);
tags = RawObject::SizeTag::update(instance_size, tags);
LoadImmediate(temp1, tags);

8
runtime/vm/assembler_x64.cc
View file

@ -3433,10 +3433,12 @@ void Assembler::TryAllocate(const Class& cls,
NOT_IN_PRODUCT(UpdateAllocationStats(cls.id(), space));
ASSERT(instance_size >= kHeapObjectTag);
AddImmediate(instance_reg, Immediate(kHeapObjectTag - instance_size));
uword tags = 0;
uint32_t tags = 0;
tags = RawObject::SizeTag::update(instance_size, tags);
ASSERT(cls.id() != kIllegalCid);
tags = RawObject::ClassIdTag::update(cls.id(), tags);
// Extends the 32 bit tags with zeros, which is the uninitialized
// hash code.
MoveImmediate(FieldAddress(instance_reg, Object::tags_offset()),
Immediate(tags));
} else {
@ -3480,9 +3482,11 @@ void Assembler::TryAllocateArray(intptr_t cid,
// Initialize the tags.
// instance: new object start as a tagged pointer.
uword tags = 0;
uint32_t tags = 0;
tags = RawObject::ClassIdTag::update(cid, tags);
tags = RawObject::SizeTag::update(instance_size, tags);
// Extends the 32 bit tags with zeros, which is the uninitialized
// hash code.
movq(FieldAddress(instance, Array::tags_offset()), Immediate(tags));
} else {
jmp(failure);

13
runtime/vm/assembler_x64.h
View file

@ -690,16 +690,8 @@ class Assembler : public ValueObject {
void lock();
void cmpxchgl(const Address& address, Register reg);
void lock_cmpxchgl(const Address& address, Register reg) {
lock();
cmpxchgl(address, reg);
}
void cmpxchgq(const Address& address, Register reg);
void lock_cmpxchgq(const Address& address, Register reg) {
lock();
cmpxchgq(address, reg);
}
void cpuid();
@ -801,6 +793,11 @@ class Assembler : public ValueObject {
cmpxchgq(address, reg);
}
void LockCmpxchgl(const Address& address, Register reg) {
lock();
cmpxchgl(address, reg);
}
void PushRegisters(intptr_t cpu_register_set, intptr_t xmm_register_set);
void PopRegisters(intptr_t cpu_register_set, intptr_t xmm_register_set);

45
runtime/vm/assembler_x64_test.cc
View file

@ -1749,7 +1749,7 @@ ASSEMBLER_TEST_GENERATE(CompareSwapEQ, assembler) {
__ movq(RAX, Immediate(4));
__ movq(RCX, Immediate(0));
__ movq(Address(RSP, 0), RAX);
__ lock_cmpxchgq(Address(RSP, 0), RCX);
__ LockCmpxchgq(Address(RSP, 0), RCX);
__ popq(RAX);
__ ret();
}
@ -1767,7 +1767,7 @@ ASSEMBLER_TEST_GENERATE(CompareSwapNEQ, assembler) {
__ movq(RAX, Immediate(2));
__ movq(RCX, Immediate(4));
__ movq(Address(RSP, 0), RCX);
__ lock_cmpxchgq(Address(RSP, 0), RCX);
__ LockCmpxchgq(Address(RSP, 0), RCX);
__ popq(RAX);
__ ret();
}
@ -1779,6 +1779,47 @@ ASSEMBLER_TEST_RUN(CompareSwapNEQ, test) {
}
ASSEMBLER_TEST_GENERATE(CompareSwapEQ32, assembler) {
__ movq(RAX, Immediate(0x100000000));
__ pushq(RAX);
__ movq(RAX, Immediate(4));
__ movq(RCX, Immediate(0));
// 32 bit store of 4.
__ movl(Address(RSP, 0), RAX);
// Compare 32 bit memory location with RAX (4) and write 0.
__ LockCmpxchgl(Address(RSP, 0), RCX);
// Pop unchanged high word and zeroed out low word.
__ popq(RAX);
__ ret();
}
ASSEMBLER_TEST_RUN(CompareSwapEQ32, test) {
typedef intptr_t (*CompareSwapEQ32Code)();
EXPECT_EQ(0x100000000,
reinterpret_cast<CompareSwapEQ32Code>(test->entry())());
}
ASSEMBLER_TEST_GENERATE(CompareSwapNEQ32, assembler) {
__ movq(RAX, Immediate(0x100000000));
__ pushq(RAX);
__ movq(RAX, Immediate(2));
__ movq(RCX, Immediate(4));
__ movl(Address(RSP, 0), RCX);
__ LockCmpxchgl(Address(RSP, 0), RCX);
__ popq(RAX);
__ ret();
}
ASSEMBLER_TEST_RUN(CompareSwapNEQ32, test) {
typedef intptr_t (*CompareSwapNEQ32Code)();
EXPECT_EQ(0x100000004l,
reinterpret_cast<CompareSwapNEQ32Code>(test->entry())());
}
ASSEMBLER_TEST_GENERATE(Exchange, assembler) {
__ movq(RAX, Immediate(kLargeConstant));
__ movq(RDX, Immediate(kAnotherLargeConstant));

6
runtime/vm/become.cc
View file

@ -23,7 +23,7 @@ ForwardingCorpse* ForwardingCorpse::AsForwarder(uword addr, intptr_t size) {
ForwardingCorpse* result = reinterpret_cast<ForwardingCorpse*>(addr);
uword tags = 0;
uint32_t tags = 0;
tags = RawObject::SizeTag::update(size, tags);
tags = RawObject::ClassIdTag::update(kForwardingCorpse, tags);
@ -262,12 +262,16 @@ void Become::ElementsForwardIdentity(const Array& before, const Array& after) {
ForwardObjectTo(before_obj, after_obj);
#if defined(HASH_IN_OBJECT_HEADER)
Object::SetCachedHash(after_obj, Object::GetCachedHash(before_obj));
#else
// Forward the identity hash too if it has one.
intptr_t hash = heap->GetHash(before_obj);
if (hash != 0) {
ASSERT(heap->GetHash(after_obj) == 0);
heap->SetHash(after_obj, hash);
}
#endif
}
{

5
runtime/vm/become.h
View file

@ -52,7 +52,10 @@ class ForwardingCorpse {
private:
// This layout mirrors the layout of RawObject.
uword tags_;
uint32_t tags_;
#if defined(HASH_IN_OBJECT_HEADER)
uint32_t hash_;
#endif
RawObject* target_;
// Returns the address of the embedded size.

5
runtime/vm/clustered_snapshot.cc
View file

@ -40,12 +40,15 @@ void Deserializer::InitializeHeader(RawObject* raw,
bool is_vm_isolate,
bool is_canonical) {
ASSERT(Utils::IsAligned(size, kObjectAlignment));
uword tags = 0;
uint32_t tags = 0;
tags = RawObject::ClassIdTag::update(class_id, tags);
tags = RawObject::SizeTag::update(size, tags);
tags = RawObject::VMHeapObjectTag::update(is_vm_isolate, tags);
tags = RawObject::CanonicalObjectTag::update(is_canonical, tags);
raw->ptr()->tags_ = tags;
#if defined(HASH_IN_OBJECT_HEADER)
raw->ptr()->hash_ = 0;
#endif
}

7
runtime/vm/freelist.cc
View file

@ -22,13 +22,18 @@ FreeListElement* FreeListElement::AsElement(uword addr, intptr_t size) {
FreeListElement* result = reinterpret_cast<FreeListElement*>(addr);
uword tags = 0;
uint32_t tags = 0;
tags = RawObject::SizeTag::update(size, tags);
tags = RawObject::ClassIdTag::update(kFreeListElement, tags);
// All words in a freelist element header should look like Smis.
ASSERT(!reinterpret_cast<RawObject*>(tags)->IsHeapObject());
result->tags_ = tags;
#if defined(HASH_IN_OBJECT_HEADER)
// Clearing this is mostly for neatness. The identityHashCode
// of free list entries is not used.
result->hash_ = 0;
#endif
if (size > RawObject::SizeTag::kMaxSizeTag) {
*result->SizeAddress() = size;
}

5
runtime/vm/freelist.h
View file

@ -56,7 +56,10 @@ class FreeListElement {
private:
// This layout mirrors the layout of RawObject.
uword tags_;
uint32_t tags_;
#if defined(HASH_IN_OBJECT_HEADER)
uint32_t hash_;
#endif
FreeListElement* next_;
// Returns the address of the embedded size.

3
runtime/vm/heap.cc
View file

@ -653,11 +653,12 @@ int64_t Heap::PeerCount() const {
return new_weak_tables_[kPeers]->count() + old_weak_tables_[kPeers]->count();
}
#if !defined(HASH_IN_OBJECT_HEADER)
int64_t Heap::HashCount() const {
return new_weak_tables_[kHashes]->count() +
old_weak_tables_[kHashes]->count();
}
#endif
int64_t Heap::ObjectIdCount() const {

11
runtime/vm/heap.h
View file

@ -33,7 +33,14 @@ class Heap {
kCode,
};
enum WeakSelector { kPeers = 0, kHashes, kObjectIds, kNumWeakSelectors };
enum WeakSelector {
kPeers = 0,
#if !defined(HASH_IN_OBJECT_HEADER)
kHashes,
#endif
kObjectIds,
kNumWeakSelectors
};
enum ApiCallbacks { kIgnoreApiCallbacks, kInvokeApiCallbacks };
@ -173,6 +180,7 @@ class Heap {
}
int64_t PeerCount() const;
#if !defined(HASH_IN_OBJECT_HEADER)
// Associate an identity hashCode with an object. An non-existent hashCode
// is equal to 0.
void SetHash(RawObject* raw_obj, intptr_t hash) {
@ -181,6 +189,7 @@ class Heap {
intptr_t GetHash(RawObject* raw_obj) const {
return GetWeakEntry(raw_obj, kHashes);
}
#endif
int64_t HashCount() const;
// Associate an id with an object (used when serializing an object).

4
runtime/vm/intermediate_language_dbc.cc
View file

@ -1012,7 +1012,7 @@ EMIT_NATIVE_CODE(AllocateObject,
Isolate* isolate = Isolate::Current();
if (Heap::IsAllocatableInNewSpace(instance_size) &&
!cls().TraceAllocation(isolate)) {
uword tags = 0;
uint32_t tags = 0;
tags = RawObject::SizeTag::update(instance_size, tags);
ASSERT(cls().id() != kIllegalCid);
tags = RawObject::ClassIdTag::update(cls().id(), tags);
@ -1046,6 +1046,7 @@ EMIT_NATIVE_CODE(AllocateObject,
tags = RawObject::SizeTag::update(instance_size, tags);
ASSERT(cls().id() != kIllegalCid);
tags = RawObject::ClassIdTag::update(cls().id(), tags);
// tags also has the initial zero hash code on 64 bit.
if (Smi::IsValid(tags)) {
const intptr_t tags_kidx = __ AddConstant(Smi::Handle(Smi::New(tags)));
__ AllocateOpt(locs()->out(0).reg(), tags_kidx);
@ -1657,6 +1658,7 @@ EMIT_NATIVE_CODE(Box, 1, Location::RequiresRegister(), LocationSummary::kCall) {
uword tags = 0;
tags = RawObject::SizeTag::update(instance_size, tags);
tags = RawObject::ClassIdTag::update(compiler->double_class().id(), tags);
// tags also has the initial zero hash code on 64 bit.
if (Smi::IsValid(tags)) {
const intptr_t tags_kidx = __ AddConstant(Smi::Handle(Smi::New(tags)));
__ AllocateOpt(out, tags_kidx);

17
runtime/vm/intrinsifier_arm64.cc
View file

@ -1832,6 +1832,23 @@ void Intrinsifier::String_getHashCode(Assembler* assembler) {
}
void Intrinsifier::Object_getHash(Assembler* assembler) {
__ ldr(R0, Address(SP, 0 * kWordSize));
__ ldr(R0, FieldAddress(R0, String::hash_offset()), kUnsignedWord);
__ SmiTag(R0);
__ ret();
}
void Intrinsifier::Object_setHash(Assembler* assembler) {
__ ldr(R0, Address(SP, 1 * kWordSize)); // Object.
__ ldr(R1, Address(SP, 0 * kWordSize)); // Value.
__ SmiUntag(R1);
__ str(R1, FieldAddress(R0, String::hash_offset()), kUnsignedWord);
__ ret();
}
void GenerateSubstringMatchesSpecialization(Assembler* assembler,
intptr_t receiver_cid,
intptr_t other_cid,

17
runtime/vm/intrinsifier_x64.cc
View file

@ -1754,6 +1754,23 @@ void Intrinsifier::String_getHashCode(Assembler* assembler) {
}
void Intrinsifier::Object_getHash(Assembler* assembler) {
__ movq(RAX, Address(RSP, +1 * kWordSize)); // Object.
__ movl(RAX, FieldAddress(RAX, String::hash_offset()));
__ SmiTag(RAX);
__ ret();
}
void Intrinsifier::Object_setHash(Assembler* assembler) {
__ movq(RAX, Address(RSP, +2 * kWordSize)); // Object.
__ movq(RDX, Address(RSP, +1 * kWordSize)); // Value.
__ SmiUntag(RDX);
__ movl(FieldAddress(RAX, String::hash_offset()), RDX);
__ ret();
}
void GenerateSubstringMatchesSpecialization(Assembler* assembler,
intptr_t receiver_cid,
intptr_t other_cid,

15
runtime/vm/method_recognizer.h
View file

@ -139,7 +139,7 @@ namespace dart {
// List of intrinsics:
// (class-name, function-name, intrinsification method, fingerprint).
#define CORE_LIB_INTRINSIC_LIST(V) \
#define CORE_LIB_NON_HASH_INTRINSIC_LIST(V) \
V(_Smi, ~, Smi_bitNegate, Smi, 0x6574c6b0) \
V(_Smi, get:bitLength, Smi_bitLength, Smi, 0x25b356ab) \
V(_Smi, _bitAndFromSmi, Smi_bitAndFromSmi, Smi, 0x490a4da1) \
@ -190,6 +190,19 @@ namespace dart {
V(_TwoByteString, ==, TwoByteString_equality, Bool, 0x4719e83f) \
#define CORE_LIB_IN_HEADER_HASH_INTRINSIC_LIST(V) \
V(::, _getHash, Object_getHash, Smi, 0x2827856d) \
V(::, _setHash, Object_setHash, Object, 0x302d1fe8) \
#if defined(HASH_IN_OBJECT_HEADER)
#define CORE_LIB_INTRINSIC_LIST(V) \
CORE_LIB_NON_HASH_INTRINSIC_LIST(V) \
CORE_LIB_IN_HEADER_HASH_INTRINSIC_LIST(V)
#else
#define CORE_LIB_INTRINSIC_LIST(V) \
CORE_LIB_NON_HASH_INTRINSIC_LIST(V)
#endif
#define CORE_INTEGER_LIB_INTRINSIC_LIST(V) \
V(_IntegerImplementation, _addFromInteger, Integer_addFromInteger, \
Dynamic, 0x6a10c54a) \

116
runtime/vm/object.cc
View file

@ -992,10 +992,17 @@ void Object::InitOnce(Isolate* isolate) {
// An object visitor which will mark all visited objects. This is used to
// premark all objects in the vm_isolate_ heap.
class PremarkingVisitor : public ObjectVisitor {
// premark all objects in the vm_isolate_ heap. Also precalculates hash
// codes so that we can get the identity hash code of objects in the read-
// only VM isolate.
class FinalizeVMIsolateVisitor : public ObjectVisitor {
public:
PremarkingVisitor() {}
FinalizeVMIsolateVisitor()
#if defined(HASH_IN_OBJECT_HEADER)
: counter_(1337)
#endif
{
}
void VisitObject(RawObject* obj) {
// Free list elements should never be marked.
@ -1005,8 +1012,35 @@ class PremarkingVisitor : public ObjectVisitor {
if (!obj->IsFreeListElement()) {
ASSERT(obj->IsVMHeapObject());
obj->SetMarkBitUnsynchronized();
if (obj->IsStringInstance()) {
RawString* str = reinterpret_cast<RawString*>(obj);
intptr_t hash = String::Hash(str);
String::SetCachedHash(str, hash);
}
#if defined(HASH_IN_OBJECT_HEADER)
// These objects end up in the read-only VM isolate which is shared
// between isolates, so we have to prepopulate them with identity hash
// codes, since we can't add hash codes later.
if (Object::GetCachedHash(obj) == 0) {
// Some classes have identity hash codes that depend on their contents,
// not per object.
ASSERT(!obj->IsStringInstance());
if (!obj->IsMint() && !obj->IsDouble() && !obj->IsBigint() &&
!obj->IsRawNull() && !obj->IsBool()) {
counter_ += 2011; // The year Dart was announced and a prime.
counter_ &= 0x3fffffff;
if (counter_ == 0) counter_++;
Object::SetCachedHash(obj, counter_);
}
}
#endif
}
}
private:
#if defined(HASH_IN_OBJECT_HEADER)
int32_t counter_;
#endif
};
@ -1086,7 +1120,7 @@ void Object::FinalizeVMIsolate(Isolate* isolate) {
{
ASSERT(isolate == Dart::vm_isolate());
WritableVMIsolateScope scope(Thread::Current());
PremarkingVisitor premarker;
FinalizeVMIsolateVisitor premarker;
ASSERT(isolate->heap()->UsedInWords(Heap::kNew) == 0);
isolate->heap()->IterateOldObjectsNoImagePages(&premarker);
// Make the VM isolate read-only again after setting all objects as marked.
@ -1121,13 +1155,15 @@ void Object::MakeUnusedSpaceTraversable(const Object& obj,
reinterpret_cast<RawTypedData*>(RawObject::FromAddr(addr));
uword new_tags = RawObject::ClassIdTag::update(kTypedDataInt8ArrayCid, 0);
new_tags = RawObject::SizeTag::update(leftover_size, new_tags);
uword tags = raw->ptr()->tags_;
uword old_tags;
uint32_t tags = raw->ptr()->tags_;
uint32_t old_tags;
// TODO(iposva): Investigate whether CompareAndSwapWord is necessary.
do {
old_tags = tags;
tags = AtomicOperations::CompareAndSwapWord(&raw->ptr()->tags_,
old_tags, new_tags);
// We can't use obj.CompareAndSwapTags here because we don't have a
// handle for the new object.
tags = AtomicOperations::CompareAndSwapUint32(&raw->ptr()->tags_,
old_tags, new_tags);
} while (tags != old_tags);
intptr_t leftover_len = (leftover_size - TypedData::InstanceSize(0));
@ -1139,13 +1175,12 @@ void Object::MakeUnusedSpaceTraversable(const Object& obj,
RawObject* raw = reinterpret_cast<RawObject*>(RawObject::FromAddr(addr));
uword new_tags = RawObject::ClassIdTag::update(kInstanceCid, 0);
new_tags = RawObject::SizeTag::update(leftover_size, new_tags);
uword tags = raw->ptr()->tags_;
uword old_tags;
uint32_t tags = raw->ptr()->tags_;
uint32_t old_tags;
// TODO(iposva): Investigate whether CompareAndSwapWord is necessary.
do {
old_tags = tags;
tags = AtomicOperations::CompareAndSwapWord(&raw->ptr()->tags_,
old_tags, new_tags);
tags = obj.CompareAndSwapTags(old_tags, new_tags);
} while (tags != old_tags);
}
}
@ -1867,12 +1902,15 @@ void Object::InitializeObject(uword address,
*reinterpret_cast<uword*>(cur) = initial_value;
cur += kWordSize;
}
uword tags = 0;
uint32_t tags = 0;
ASSERT(class_id != kIllegalCid);
tags = RawObject::ClassIdTag::update(class_id, tags);
tags = RawObject::SizeTag::update(size, tags);
tags = RawObject::VMHeapObjectTag::update(is_vm_object, tags);
reinterpret_cast<RawObject*>(address)->tags_ = tags;
#if defined(HASH_IN_OBJECT_HEADER)
reinterpret_cast<RawObject*>(address)->hash_ = 0;
#endif
ASSERT(is_vm_object == RawObject::IsVMHeapObject(tags));
}
@ -20327,6 +20365,35 @@ static intptr_t HashImpl(const T* characters, intptr_t len) {
}
intptr_t String::Hash(RawString* raw) {
StringHasher hasher;
uword length = Smi::Value(raw->ptr()->length_);
if (raw->IsOneByteString() || raw->IsExternalOneByteString()) {
const uint8_t* data;
if (raw->IsOneByteString()) {
data = reinterpret_cast<RawOneByteString*>(raw)->ptr()->data();
} else {
ASSERT(raw->IsExternalOneByteString());
RawExternalOneByteString* str =
reinterpret_cast<RawExternalOneByteString*>(raw);
data = str->ptr()->external_data_->data();
}
return String::Hash(data, length);
} else {
const uint16_t* data;
if (raw->IsTwoByteString()) {
data = reinterpret_cast<RawTwoByteString*>(raw)->ptr()->data();
} else {
ASSERT(raw->IsExternalTwoByteString());
RawExternalTwoByteString* str =
reinterpret_cast<RawExternalTwoByteString*>(raw);
data = str->ptr()->external_data_->data();
}
return String::Hash(data, length);
}
}
intptr_t String::Hash(const char* characters, intptr_t len) {
return HashImpl(characters, len);
}
@ -21147,11 +21214,11 @@ RawString* String::MakeExternal(void* array,
// Update the class information of the object.
const intptr_t class_id = kExternalOneByteStringCid;
uword tags = raw_ptr()->tags_;
uword old_tags;
uint32_t tags = raw_ptr()->tags_;
uint32_t old_tags;
do {
old_tags = tags;
uword new_tags = RawObject::SizeTag::update(used_size, old_tags);
uint32_t new_tags = RawObject::SizeTag::update(used_size, old_tags);
new_tags = RawObject::ClassIdTag::update(class_id, new_tags);
tags = CompareAndSwapTags(old_tags, new_tags);
} while (tags != old_tags);
@ -21184,11 +21251,11 @@ RawString* String::MakeExternal(void* array,
// Update the class information of the object.
const intptr_t class_id = kExternalTwoByteStringCid;
uword tags = raw_ptr()->tags_;
uword old_tags;
uint32_t tags = raw_ptr()->tags_;
uint32_t old_tags;
do {
old_tags = tags;
uword new_tags = RawObject::SizeTag::update(used_size, old_tags);
uint32_t new_tags = RawObject::SizeTag::update(used_size, old_tags);
new_tags = RawObject::ClassIdTag::update(class_id, new_tags);
tags = CompareAndSwapTags(old_tags, new_tags);
} while (tags != old_tags);
@ -22076,11 +22143,11 @@ void Array::MakeImmutable() const {
if (IsImmutable()) return;
ASSERT(!IsCanonical());
NoSafepointScope no_safepoint;
uword tags = raw_ptr()->tags_;
uword old_tags;
uint32_t tags = raw_ptr()->tags_;
uint32_t old_tags;
do {
old_tags = tags;
uword new_tags =
uint32_t new_tags =
RawObject::ClassIdTag::update(kImmutableArrayCid, old_tags);
tags = CompareAndSwapTags(old_tags, new_tags);
} while (tags != old_tags);
@ -22145,6 +22212,7 @@ RawArray* Array::MakeFixedLength(const GrowableObjectArray& growable_array,
}
intptr_t capacity_len = growable_array.Capacity();
const Array& array = Array::Handle(zone, growable_array.data());
ASSERT(array.IsArray());
array.SetTypeArguments(type_arguments);
intptr_t capacity_size = Array::InstanceSize(capacity_len);
intptr_t used_size = Array::InstanceSize(used_len);
@ -22158,10 +22226,10 @@ RawArray* Array::MakeFixedLength(const GrowableObjectArray& growable_array,
// Update the size in the header field and length of the array object.
uword tags = array.raw_ptr()->tags_;
ASSERT(kArrayCid == RawObject::ClassIdTag::decode(tags));
uword old_tags;
uint32_t old_tags;
do {
old_tags = tags;
uword new_tags = RawObject::SizeTag::update(used_size, old_tags);
uint32_t new_tags = RawObject::SizeTag::update(used_size, old_tags);
tags = array.CompareAndSwapTags(old_tags, new_tags);
} while (tags != old_tags);
// TODO(22501): For the heap to remain walkable by the sweeper, it must

20
runtime/vm/object.h
View file

@ -245,9 +245,9 @@ class Object {
RawObject* raw() const { return raw_; }
void operator=(RawObject* value) { initializeHandle(this, value); }
uword CompareAndSwapTags(uword old_tags, uword new_tags) const {
return AtomicOperations::CompareAndSwapWord(&raw()->ptr()->tags_, old_tags,
new_tags);
uint32_t CompareAndSwapTags(uint32_t old_tags, uint32_t new_tags) const {
return AtomicOperations::CompareAndSwapUint32(&raw()->ptr()->tags_,
old_tags, new_tags);
}
bool IsCanonical() const { return raw()->IsCanonical(); }
void SetCanonical() const { raw()->SetCanonical(); }
@ -434,13 +434,11 @@ class Object {
#if defined(HASH_IN_OBJECT_HEADER)
static uint32_t GetCachedHash(const RawObject* obj) {
uword tags = obj->ptr()->tags_;
return tags >> 32;
return obj->ptr()->hash_;
}
static void SetCachedHash(RawObject* obj, uintptr_t hash) {
ASSERT(hash >> 32 == 0);
obj->ptr()->tags_ |= hash << 32;
static void SetCachedHash(RawObject* obj, uint32_t hash) {
obj->ptr()->hash_ = hash;
}
#endif
@ -6831,16 +6829,14 @@ class String : public Instance {
return result;
}
static intptr_t Hash(RawString* raw);
bool HasHash() const {
ASSERT(Smi::New(0) == NULL);
return GetCachedHash(raw()) != 0;
}
#if defined(HASH_IN_OBJECT_HEADER)
static intptr_t hash_offset() { return kInt32Size; } // Wrong for big-endian?
#else
static intptr_t hash_offset() { return OFFSET_OF(RawString, hash_); }
#endif
static intptr_t Hash(const String& str, intptr_t begin_index, intptr_t len);
static intptr_t Hash(const char* characters, intptr_t len);
static intptr_t Hash(const uint16_t* characters, intptr_t len);

2
runtime/vm/profiler.cc
View file

@ -1322,7 +1322,7 @@ class CodeLookupTableBuilder : public ObjectVisitor {
~CodeLookupTableBuilder() {}
void VisitObject(RawObject* raw_obj) {
uword tags = raw_obj->ptr()->tags_;
uint32_t tags = raw_obj->ptr()->tags_;
if (RawObject::ClassIdTag::decode(tags) == kCodeCid) {
RawCode* raw_code = reinterpret_cast<RawCode*>(raw_obj);
const Code& code = Code::Handle(raw_code);

14
runtime/vm/raw_object.cc
View file

@ -30,10 +30,10 @@ void RawObject::Validate(Isolate* isolate) const {
FATAL("RAW_NULL encountered");
}
// Validate that the tags_ field is sensible.
uword tags = ptr()->tags_;
uint32_t tags = ptr()->tags_;
intptr_t reserved = ReservedBits::decode(tags);
if (reserved != 0) {
FATAL1("Invalid tags field encountered %#" Px "\n", tags);
FATAL1("Invalid tags field encountered %x\n", tags);
}
intptr_t class_id = ClassIdTag::decode(tags);
if (!isolate->class_table()->IsValidIndex(class_id)) {
@ -185,7 +185,7 @@ intptr_t RawObject::SizeFromClass() const {
ClassTable* class_table = isolate->class_table();
if (!class_table->IsValidIndex(class_id) ||
!class_table->HasValidClassAt(class_id)) {
FATAL2("Invalid class id: %" Pd " from tags %" Px "\n", class_id,
FATAL2("Invalid class id: %" Pd " from tags %x\n", class_id,
ptr()->tags_);
}
#endif // DEBUG
@ -196,7 +196,7 @@ intptr_t RawObject::SizeFromClass() const {
}
ASSERT(instance_size != 0);
#if defined(DEBUG)
uword tags = ptr()->tags_;
uint32_t tags = ptr()->tags_;
intptr_t tags_size = SizeTag::decode(tags);
if ((class_id == kArrayCid) && (instance_size > tags_size && tags_size > 0)) {
// TODO(22501): Array::MakeFixedLength could be in the process of shrinking
@ -211,7 +211,7 @@ intptr_t RawObject::SizeFromClass() const {
} while ((instance_size > tags_size) && (--retries_remaining > 0));
}
if ((instance_size != tags_size) && (tags_size != 0)) {
FATAL3("Size mismatch: %" Pd " from class vs %" Pd " from tags %" Px "\n",
FATAL3("Size mismatch: %" Pd " from class vs %" Pd " from tags %x\n",
instance_size, tags_size, tags);
}
#endif // DEBUG
@ -524,7 +524,7 @@ intptr_t RawNamespace::VisitNamespacePointers(RawNamespace* raw_obj,
bool RawCode::ContainsPC(RawObject* raw_obj, uword pc) {
uword tags = raw_obj->ptr()->tags_;
uint32_t tags = raw_obj->ptr()->tags_;
if (RawObject::ClassIdTag::decode(tags) == kCodeCid) {
RawCode* raw_code = reinterpret_cast<RawCode*>(raw_obj);
return RawInstructions::ContainsPC(raw_code->ptr()->instructions_, pc);
@ -737,7 +737,7 @@ intptr_t RawInstance::VisitInstancePointers(RawInstance* raw_obj,
ObjectPointerVisitor* visitor) {
// Make sure that we got here with the tagged pointer as this.
ASSERT(raw_obj->IsHeapObject());
uword tags = raw_obj->ptr()->tags_;
uint32_t tags = raw_obj->ptr()->tags_;
intptr_t instance_size = SizeTag::decode(tags);
if (instance_size == 0) {
RawClass* cls =

87
runtime/vm/raw_object.h
View file

@ -271,6 +271,10 @@ class RawObject {
kSizeTagSize = 8,
kClassIdTagPos = kSizeTagPos + kSizeTagSize, // = 16
kClassIdTagSize = 16,
#if defined(HASH_IN_OBJECT_HEADER)
kHashTagPos = kClassIdTagPos + kClassIdTagSize, // = 32
kHashTagSize = 16,
#endif
};
COMPILE_ASSERT(kClassIdTagSize == (sizeof(classid_t) * kBitsPerByte));
@ -296,7 +300,7 @@ class RawObject {
private:
// The actual unscaled bit field used within the tag field.
class SizeBits
: public BitField<uword, intptr_t, kSizeTagPos, kSizeTagSize> {};
: public BitField<uint32_t, intptr_t, kSizeTagPos, kSizeTagSize> {};
static intptr_t SizeToTagValue(intptr_t size) {
ASSERT(Utils::IsAligned(size, kObjectAlignment));
@ -308,7 +312,7 @@ class RawObject {
};
class ClassIdTag
: public BitField<uword, intptr_t, kClassIdTagPos, kClassIdTagSize> {};
: public BitField<uint32_t, intptr_t, kClassIdTagPos, kClassIdTagSize> {};
bool IsWellFormed() const {
uword value = reinterpret_cast<uword>(this);
@ -352,7 +356,7 @@ class RawObject {
}
void SetMarkBitUnsynchronized() {
ASSERT(!IsMarked());
uword tags = ptr()->tags_;
uint32_t tags = ptr()->tags_;
ptr()->tags_ = MarkBit::update(true, tags);
}
void ClearMarkBit() {
@ -378,12 +382,12 @@ class RawObject {
}
void SetRememberedBitUnsynchronized() {
ASSERT(!IsRemembered());
uword tags = ptr()->tags_;
uint32_t tags = ptr()->tags_;
ptr()->tags_ = RememberedBit::update(true, tags);
}
void ClearRememberedBit() { UpdateTagBit<RememberedBit>(false); }
void ClearRememberedBitUnsynchronized() {
uword tags = ptr()->tags_;
uint32_t tags = ptr()->tags_;
ptr()->tags_ = RememberedBit::update(false, tags);
}
// Returns false if the bit was already set.
@ -409,6 +413,7 @@ class RawObject {
#undef DEFINE_IS_CID
bool IsStringInstance() const { return IsStringClassId(GetClassId()); }
bool IsRawNull() const { return GetClassId() == kNullCid; }
bool IsDartInstance() const {
return (!IsHeapObject() || (GetClassId() >= kInstanceCid));
}
@ -423,7 +428,7 @@ class RawObject {
}
intptr_t Size() const {
uword tags = ptr()->tags_;
uint32_t tags = ptr()->tags_;
intptr_t result = SizeTag::decode(tags);
if (result != 0) {
#if defined(DEBUG)
@ -533,18 +538,25 @@ class RawObject {
static intptr_t NumberOfTypedDataClasses();
private:
uword tags_; // Various object tags (bits).
uint32_t tags_; // Various object tags (bits).
#if defined(HASH_IN_OBJECT_HEADER)
// On 64 bit there is a hash field in the header for the identity hash.
uint32_t hash_;
#endif
class MarkBit : public BitField<uword, bool, kMarkBit, 1> {};
class MarkBit : public BitField<uint32_t, bool, kMarkBit, 1> {};
class RememberedBit : public BitField<uword, bool, kRememberedBit, 1> {};
class RememberedBit : public BitField<uint32_t, bool, kRememberedBit, 1> {};
class CanonicalObjectTag : public BitField<uword, bool, kCanonicalBit, 1> {};
class CanonicalObjectTag : public BitField<uint32_t, bool, kCanonicalBit, 1> {
};
class VMHeapObjectTag : public BitField<uword, bool, kVMHeapObjectBit, 1> {};
class VMHeapObjectTag : public BitField<uint32_t, bool, kVMHeapObjectBit, 1> {
};
class ReservedBits
: public BitField<uword, intptr_t, kReservedTagPos, kReservedTagSize> {};
: public BitField<uint32_t, intptr_t, kReservedTagPos, kReservedTagSize> {
};
// TODO(koda): After handling tags_, return const*, like Object::raw_ptr().
RawObject* ptr() const {
@ -559,37 +571,37 @@ class RawObject {
intptr_t SizeFromClass() const;
intptr_t GetClassId() const {
uword tags = ptr()->tags_;
uint32_t tags = ptr()->tags_;
return ClassIdTag::decode(tags);
}
void SetClassId(intptr_t new_cid) {
uword tags = ptr()->tags_;
uint32_t tags = ptr()->tags_;
ptr()->tags_ = ClassIdTag::update(new_cid, tags);
}
template <class TagBitField>
void UpdateTagBit(bool value) {
uword tags = ptr()->tags_;
uword old_tags;
uint32_t tags = ptr()->tags_;
uint32_t old_tags;
do {
old_tags = tags;
uword new_tags = TagBitField::update(value, old_tags);
tags = AtomicOperations::CompareAndSwapWord(&ptr()->tags_, old_tags,
new_tags);
uint32_t new_tags = TagBitField::update(value, old_tags);
tags = AtomicOperations::CompareAndSwapUint32(&ptr()->tags_, old_tags,
new_tags);
} while (tags != old_tags);
}
template <class TagBitField>
bool TryAcquireTagBit() {
uword tags = ptr()->tags_;
uword old_tags;
uint32_t tags = ptr()->tags_;
uint32_t old_tags;
do {
old_tags = tags;
if (TagBitField::decode(tags)) return false;
uword new_tags = TagBitField::update(true, old_tags);
tags = AtomicOperations::CompareAndSwapWord(&ptr()->tags_, old_tags,
new_tags);
uint32_t new_tags = TagBitField::update(true, old_tags);
tags = AtomicOperations::CompareAndSwapUint32(&ptr()->tags_, old_tags,
new_tags);
} while (tags != old_tags);
return true;
}
@ -1318,7 +1330,10 @@ class RawPcDescriptors : public RawObject {
private:
RAW_HEAP_OBJECT_IMPLEMENTATION(PcDescriptors);
int32_t length_; // Number of descriptors.
// Number of descriptors. This only needs to be an int32_t, but we make it a
// uword so that the variable length data is 64 bit aligned on 64 bit
// platforms.
uword length_;
// Variable length data follows here.
uint8_t* data() { OPEN_ARRAY_START(uint8_t, intptr_t); }
@ -1334,7 +1349,9 @@ class RawCodeSourceMap : public RawObject {
private:
RAW_HEAP_OBJECT_IMPLEMENTATION(CodeSourceMap);
int32_t length_; // Length in bytes.
// Length in bytes. This only needs to be an int32_t, but we make it a uword
// so that the variable length data is 64 bit aligned on 64 bit platforms.
uword length_;
// Variable length data follows here.
uint8_t* data() { OPEN_ARRAY_START(uint8_t, intptr_t); }
@ -1361,8 +1378,9 @@ class RawStackMap : public RawObject {
int32_t slow_path_bit_count_; // Slow path live values, included in length_.
// Offset from code entry point corresponding to this stack map
// representation.
uint32_t pc_offset_;
// representation. This only needs to be an int32_t, but we make it a uword
// so that the variable length data is 64 bit aligned on 64 bit platforms.
uword pc_offset_;
// Variable length data follows here (bitmap of the stack layout).
uint8_t* data() { OPEN_ARRAY_START(uint8_t, uint8_t); }
@ -1416,7 +1434,10 @@ class RawLocalVarDescriptors : public RawObject {
private:
RAW_HEAP_OBJECT_IMPLEMENTATION(LocalVarDescriptors);
int32_t num_entries_; // Number of descriptors.
// Number of descriptors. This only needs to be an int32_t, but we make it a
// uword so that the variable length data is 64 bit aligned on 64 bit
// platforms.
uword num_entries_;
RawObject** from() {
return reinterpret_cast<RawObject**>(&ptr()->names()[0]);
@ -1924,8 +1945,9 @@ class RawOneByteString : public RawString {
const uint8_t* data() const { OPEN_ARRAY_START(uint8_t, uint8_t); }
friend class ApiMessageReader;
friend class SnapshotReader;
friend class RODataSerializationCluster;
friend class SnapshotReader;
friend class String;
};
@ -1936,8 +1958,9 @@ class RawTwoByteString : public RawString {
uint16_t* data() { OPEN_ARRAY_START(uint16_t, uint16_t); }
const uint16_t* data() const { OPEN_ARRAY_START(uint16_t, uint16_t); }
friend class SnapshotReader;
friend class RODataSerializationCluster;
friend class SnapshotReader;
friend class String;
};
@ -1973,6 +1996,7 @@ class RawExternalOneByteString : public RawString {
private:
ExternalData* external_data_;
friend class Api;
friend class String;
};
@ -1985,6 +2009,7 @@ class RawExternalTwoByteString : public RawString {
private:
ExternalData* external_data_;
friend class Api;
friend class String;
};

48
runtime/vm/simulator_arm64.cc
View file

@ -1225,6 +1225,13 @@ intptr_t Simulator::ReadExclusiveX(uword addr, Instr* instr) {
}
intptr_t Simulator::ReadExclusiveW(uword addr, Instr* instr) {
MutexLocker ml(exclusive_access_lock_);
SetExclusiveAccess(addr);
return ReadWU(addr, instr);
}
intptr_t Simulator::WriteExclusiveX(uword addr, intptr_t value, Instr* instr) {
MutexLocker ml(exclusive_access_lock_);
bool write_allowed = HasExclusiveAccessAndOpen(addr);
@ -1236,6 +1243,17 @@ intptr_t Simulator::WriteExclusiveX(uword addr, intptr_t value, Instr* instr) {
}
intptr_t Simulator::WriteExclusiveW(uword addr, intptr_t value, Instr* instr) {
MutexLocker ml(exclusive_access_lock_);
bool write_allowed = HasExclusiveAccessAndOpen(addr);
if (write_allowed) {
WriteW(addr, value, instr);
return 0; // Success.
}
return 1; // Failure.
}
uword Simulator::CompareExchange(uword* address,
uword compare_value,
uword new_value) {
@ -2199,25 +2217,37 @@ void Simulator::DecodeLoadStoreExclusive(Instr* instr) {
UNIMPLEMENTED();
}
const int32_t size = instr->Bits(30, 2);
if (size != 3) {
if (size != 3 && size != 2) {
UNIMPLEMENTED();
}
const Register rs = instr->RsField();
const Register rn = instr->RnField();
const Register rt = instr->RtField();
const bool is_load = instr->Bit(22) == 1;
if (is_load) {
// Format(instr, "ldxr 'rt, 'rn");
const int64_t addr = get_register(rn, R31IsSP);
intptr_t value = ReadExclusiveX(addr, instr);
set_register(instr, rt, value, R31IsSP);
if (size == 3) {
const int64_t addr = get_register(rn, R31IsSP);
intptr_t value = ReadExclusiveX(addr, instr);
set_register(instr, rt, value, R31IsSP);
} else {
const int64_t addr = get_register(rn, R31IsSP);
intptr_t value = ReadExclusiveW(addr, instr);
set_register(instr, rt, value, R31IsSP);
}
} else {
// Format(instr, "stxr 'rs, 'rt, 'rn");
uword value = get_register(rt, R31IsSP);
uword addr = get_register(rn, R31IsSP);
intptr_t status = WriteExclusiveX(addr, value, instr);
set_register(instr, rs, status, R31IsSP);
if (size == 3) {
uword value = get_register(rt, R31IsSP);
uword addr = get_register(rn, R31IsSP);
intptr_t status = WriteExclusiveX(addr, value, instr);
set_register(instr, rs, status, R31IsSP);
} else {
uint32_t value = get_register(rt, R31IsSP);
uword addr = get_register(rn, R31IsSP);
intptr_t status = WriteExclusiveW(addr, value, instr);
set_register(instr, rs, status, R31IsSP);
}
}
}

3
runtime/vm/simulator_arm64.h
View file

@ -207,6 +207,9 @@ class Simulator {
void ClearExclusive();
intptr_t ReadExclusiveX(uword addr, Instr* instr);
intptr_t WriteExclusiveX(uword addr, intptr_t value, Instr* instr);
// 32 bit versions.
intptr_t ReadExclusiveW(uword addr, Instr* instr);
intptr_t WriteExclusiveW(uword addr, intptr_t value, Instr* instr);
// Set access to given address to 'exclusive state' for current thread.
static void SetExclusiveAccess(uword addr);

8
runtime/vm/simulator_dbc.cc
View file

@ -322,6 +322,7 @@ class SimulatorHelpers {
uword tags = 0;
tags = RawObject::ClassIdTag::update(kDoubleCid, tags);
tags = RawObject::SizeTag::update(instance_size, tags);
// Also writes zero in the hash_ field.
*reinterpret_cast<uword*>(start + Double::tags_offset()) = tags;
*reinterpret_cast<double*>(start + Double::value_offset()) = value;
return reinterpret_cast<RawObject*>(start + kHeapObjectTag);
@ -2855,9 +2856,10 @@ RawObject* Simulator::Call(const Code& code,
const intptr_t instance_size = Context::InstanceSize(num_context_variables);
const uword start = thread->heap()->new_space()->TryAllocate(instance_size);
if (LIKELY(start != 0)) {
uword tags = 0;
uint32_t tags = 0;
tags = RawObject::ClassIdTag::update(kContextCid, tags);
tags = RawObject::SizeTag::update(instance_size, tags);
// Also writes 0 in the hash_ field of the header.
*reinterpret_cast<uword*>(start + Array::tags_offset()) = tags;
*reinterpret_cast<uword*>(start + Context::num_variables_offset()) =
num_context_variables;
@ -2898,6 +2900,7 @@ RawObject* Simulator::Call(const Code& code,
const intptr_t instance_size = RawObject::SizeTag::decode(tags);
const uword start = thread->heap()->new_space()->TryAllocate(instance_size);
if (LIKELY(start != 0)) {
// Writes both the tags and the initial identity hash on 64 bit platforms.
*reinterpret_cast<uword*>(start + Instance::tags_offset()) = tags;
for (intptr_t current_offset = sizeof(RawInstance);
current_offset < instance_size; current_offset += kWordSize) {
@ -2929,6 +2932,7 @@ RawObject* Simulator::Call(const Code& code,
if (LIKELY(start != 0)) {
RawObject* type_args = SP[0];
const intptr_t type_args_offset = Bytecode::DecodeD(*pc);
// Writes both the tags and the initial identity hash on 64 bit platforms.
*reinterpret_cast<uword*>(start + Instance::tags_offset()) = tags;
for (intptr_t current_offset = sizeof(RawInstance);
current_offset < instance_size; current_offset += kWordSize) {
@ -2972,6 +2976,8 @@ RawObject* Simulator::Call(const Code& code,
tags = RawObject::SizeTag::update(instance_size, tags);
}
tags = RawObject::ClassIdTag::update(cid, tags);
// Writes both the tags and the initial identity hash on 64 bit
// platforms.
*reinterpret_cast<uword*>(start + Instance::tags_offset()) = tags;
*reinterpret_cast<RawObject**>(start + Array::length_offset()) =
FP[rB];

30
runtime/vm/snapshot.cc
View file

@ -755,6 +755,9 @@ void ImageWriter::WriteROData(WriteStream* stream) {
uword marked_tags = obj.raw()->ptr()->tags_;
marked_tags = RawObject::VMHeapObjectTag::update(true, marked_tags);
marked_tags = RawObject::MarkBit::update(true, marked_tags);
#if defined(HASH_IN_OBJECT_HEADER)
marked_tags |= static_cast<uword>(obj.raw()->ptr()->hash_) << 32;
#endif
stream->WriteWord(marked_tags);
start += sizeof(uword);
for (uword* cursor = reinterpret_cast<uword*>(start);
@ -841,6 +844,11 @@ void AssemblyImageWriter::WriteText(WriteStream* clustered_stream, bool vm) {
uword marked_tags = insns.raw_ptr()->tags_;
marked_tags = RawObject::VMHeapObjectTag::update(true, marked_tags);
marked_tags = RawObject::MarkBit::update(true, marked_tags);
#if defined(HASH_IN_OBJECT_HEADER)
// Can't use GetObjectTagsAndHash because the update methods discard the
// high bits.
marked_tags |= static_cast<uword>(insns.raw_ptr()->hash_) << 32;
#endif
WriteWordLiteralText(marked_tags);
beginning += sizeof(uword);
@ -1046,6 +1054,11 @@ void BlobImageWriter::WriteText(WriteStream* clustered_stream, bool vm) {
uword marked_tags = insns.raw_ptr()->tags_;
marked_tags = RawObject::VMHeapObjectTag::update(true, marked_tags);
marked_tags = RawObject::MarkBit::update(true, marked_tags);
#if defined(HASH_IN_OBJECT_HEADER)
// Can't use GetObjectTagsAndHash because the update methods discard the
// high bits.
marked_tags |= static_cast<uword>(insns.raw_ptr()->hash_) << 32;
#endif
instructions_blob_stream_.WriteWord(marked_tags);
beginning += sizeof(uword);
@ -1350,11 +1363,20 @@ void SnapshotWriter::WriteObject(RawObject* rawobj) {
}
uword SnapshotWriter::GetObjectTags(RawObject* raw) {
uint32_t SnapshotWriter::GetObjectTags(RawObject* raw) {
return raw->ptr()->tags_;
}
uword SnapshotWriter::GetObjectTagsAndHash(RawObject* raw) {
uword result = raw->ptr()->tags_;
#if defined(HASH_IN_OBJECT_HEADER)
result |= static_cast<uword>(raw->ptr()->hash_) << 32;
#endif
return result;
}
#define VM_OBJECT_CLASS_LIST(V) \
V(OneByteString) \
V(TwoByteString) \
@ -1578,7 +1600,7 @@ void SnapshotWriter::WriteObjectImpl(RawObject* raw, bool as_reference) {
// When we know that we are dealing with leaf or shallow objects we write
// these objects inline even when 'as_reference' is true.
const bool write_as_reference = as_reference && !raw->IsCanonical();
intptr_t tags = raw->ptr()->tags_;
uintptr_t tags = GetObjectTagsAndHash(raw);
// Add object to the forward ref list and mark it so that future references
// to this object in the snapshot will use this object id. Mark the
@ -1655,7 +1677,7 @@ class WriteInlinedObjectVisitor : public ObjectVisitor {
virtual void VisitObject(RawObject* obj) {
intptr_t object_id = writer_->forward_list_->FindObject(obj);
ASSERT(object_id != kInvalidIndex);
intptr_t tags = writer_->GetObjectTags(obj);
intptr_t tags = MessageWriter::GetObjectTagsAndHash(obj);
writer_->WriteMarkedObjectImpl(obj, tags, object_id, kAsInlinedObject);
}
@ -1820,7 +1842,7 @@ RawFunction* SnapshotWriter::IsSerializableClosure(RawClosure* closure) {
RawClass* SnapshotWriter::GetFunctionOwner(RawFunction* func) {
RawObject* owner = func->ptr()->owner_;
uword tags = GetObjectTags(owner);
uint32_t tags = GetObjectTags(owner);
intptr_t class_id = RawObject::ClassIdTag::decode(tags);
if (class_id == kClassCid) {
return reinterpret_cast<RawClass*>(owner);

3
runtime/vm/snapshot.h
View file

@ -842,7 +842,8 @@ class SnapshotWriter : public BaseWriter {
// Serialize an object into the buffer.
void WriteObject(RawObject* raw);
uword GetObjectTags(RawObject* raw);
static uint32_t GetObjectTags(RawObject* raw);
static uword GetObjectTagsAndHash(RawObject* raw);
Exceptions::ExceptionType exception_type() const { return exception_type_; }
void set_exception_type(Exceptions::ExceptionType type) {

2
runtime/vm/stub_code_arm.cc
View file

@ -1152,7 +1152,7 @@ void StubCode::GenerateAllocationStubForClass(Assembler* assembler,
// R1: next object start.
// R9: allocation stats table.
// Set the tags.
uword tags = 0;
uint32_t tags = 0;
tags = RawObject::SizeTag::update(instance_size, tags);
ASSERT(cls.id() != kIllegalCid);
tags = RawObject::ClassIdTag::update(cls.id(), tags);

11
runtime/vm/stub_code_arm64.cc
View file

@ -1090,7 +1090,7 @@ void StubCode::GenerateUpdateStoreBufferStub(Assembler* assembler) {
Label add_to_buffer;
// Check whether this object has already been remembered. Skip adding to the
// store buffer if the object is in the store buffer already.
__ LoadFieldFromOffset(TMP, R0, Object::tags_offset());
__ LoadFieldFromOffset(TMP, R0, Object::tags_offset(), kWord);
__ tsti(TMP, Immediate(1 << RawObject::kRememberedBit));
__ b(&add_to_buffer, EQ);
__ ret();
@ -1105,11 +1105,13 @@ void StubCode::GenerateUpdateStoreBufferStub(Assembler* assembler) {
ASSERT(Object::tags_offset() == 0);
__ sub(R3, R0, Operand(kHeapObjectTag));
// R3: Untagged address of header word (ldxr/stxr do not support offsets).
// Note that we use 32 bit operations here to match the size of the
// background sweeper which is also manipulating this 32 bit word.
Label retry;
__ Bind(&retry);
__ ldxr(R2, R3);
__ ldxr(R2, R3, kWord);
__ orri(R2, R2, Immediate(1 << RawObject::kRememberedBit));
__ stxr(R1, R2, R3);
__ stxr(R1, R2, R3, kWord);
__ cmp(R1, Operand(1));
__ b(&retry, EQ);
@ -1196,11 +1198,12 @@ void StubCode::GenerateAllocationStubForClass(Assembler* assembler,
// R3: next object start.
// R1: new object type arguments (if is_cls_parameterized).
// Set the tags.
uword tags = 0;
uint32_t tags = 0;
tags = RawObject::SizeTag::update(instance_size, tags);
ASSERT(cls.id() != kIllegalCid);
tags = RawObject::ClassIdTag::update(cls.id(), tags);
__ LoadImmediate(R0, tags);
// 64 bit store also zeros the hash_field.
__ StoreToOffset(R0, R2, Instance::tags_offset());
// Initialize the remaining words of the object.

2
runtime/vm/stub_code_ia32.cc
View file

@ -1078,7 +1078,7 @@ void StubCode::GenerateAllocationStubForClass(Assembler* assembler,
// EBX: next object start.
// EDX: new object type arguments (if is_cls_parameterized).
// Set the tags.
uword tags = 0;
uint32_t tags = 0;
tags = RawObject::SizeTag::update(instance_size, tags);
ASSERT(cls.id() != kIllegalCid);
tags = RawObject::ClassIdTag::update(cls.id(), tags);

2
runtime/vm/stub_code_mips.cc
View file

@ -1215,7 +1215,7 @@ void StubCode::GenerateAllocationStubForClass(Assembler* assembler,
// T3: next object start.
// T1: new object type arguments (if is_cls_parameterized).
// Set the tags.
uword tags = 0;
uint32_t tags = 0;
tags = RawObject::SizeTag::update(instance_size, tags);
ASSERT(cls.id() != kIllegalCid);
tags = RawObject::ClassIdTag::update(cls.id(), tags);

15
runtime/vm/stub_code_x64.cc
View file

@ -1034,21 +1034,23 @@ void StubCode::GenerateUpdateStoreBufferStub(Assembler* assembler) {
// RDX: Address being stored
Label reload;
__ Bind(&reload);
__ movq(RAX, FieldAddress(RDX, Object::tags_offset()));
__ testq(RAX, Immediate(1 << RawObject::kRememberedBit));
__ movl(RAX, FieldAddress(RDX, Object::tags_offset()));
__ testl(RAX, Immediate(1 << RawObject::kRememberedBit));
__ j(EQUAL, &add_to_buffer, Assembler::kNearJump);
__ popq(RCX);
__ popq(RAX);
__ ret();
// Update the tags that this object has been remembered.
// Note that we use 32 bit operations here to match the size of the
// background sweeper which is also manipulating this 32 bit word.
// RDX: Address being stored
// RAX: Current tag value
__ Bind(&add_to_buffer);
__ movq(RCX, RAX);
__ orq(RCX, Immediate(1 << RawObject::kRememberedBit));
__ movl(RCX, RAX);
__ orl(RCX, Immediate(1 << RawObject::kRememberedBit));
// Compare the tag word with RAX, update to RCX if unchanged.
__ LockCmpxchgq(FieldAddress(RDX, Object::tags_offset()), RCX);
__ LockCmpxchgl(FieldAddress(RDX, Object::tags_offset()), RCX);
__ j(NOT_EQUAL, &reload);
// Load the StoreBuffer block out of the thread. Then load top_ out of the
@ -1132,10 +1134,11 @@ void StubCode::GenerateAllocationStubForClass(Assembler* assembler,
// RBX: next object start.
// RDX: new object type arguments (if is_cls_parameterized).
// Set the tags.
uword tags = 0;
uint32_t tags = 0;
tags = RawObject::SizeTag::update(instance_size, tags);
ASSERT(cls.id() != kIllegalCid);
tags = RawObject::ClassIdTag::update(cls.id(), tags);
// 64 bit store also zeros the identity hash field.
__ movq(Address(RAX, Instance::tags_offset()), Immediate(tags));
__ addq(RAX, Immediate(kHeapObjectTag));