mirror of
https://github.com/dart-lang/sdk
synced 2024-09-05 00:13:50 +00:00
[vm, tsan] Consistently modify the object header as a full word.
An address containing an object header is sometimes interpreted as a object slot due to array truncation and concurrent marking/sweeping. Always accessing it as a word assuages TSAN that it won't be subject to tearing, and allows for the removal of TSAN suppressions from header access. In turn, this will allow TSAN to verify some acquire/release header accesses. TEST=tsan Bug: https://github.com/dart-lang/sdk/issues/44091 Change-Id: I7bf6449bf7dfeabd9db76e4bb37456e3be2d5124 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/172861 Commit-Queue: Ryan Macnak <rmacnak@google.com> Reviewed-by: Alexander Markov <alexmarkov@google.com> Reviewed-by: Martin Kustermann <kustermann@google.com>
This commit is contained in:
Ryan Macnak
commit-bot@chromium.org
parent
4fc9632b6c
commit
d3d92e316b
|
|
@ -99,7 +99,7 @@ void Deserializer::InitializeHeader(ObjectPtr raw,
|
|||
intptr_t size,
|
||||
bool is_canonical) {
|
||||
ASSERT(Utils::IsAligned(size, kObjectAlignment));
|
||||
uint32_t tags = 0;
|
||||
uword tags = 0;
|
||||
tags = ObjectLayout::ClassIdTag::update(class_id, tags);
|
||||
tags = ObjectLayout::SizeTag::update(size, tags);
|
||||
tags = ObjectLayout::CanonicalBit::update(is_canonical, tags);
|
||||
|
|
@ -108,9 +108,6 @@ void Deserializer::InitializeHeader(ObjectPtr raw,
|
|||
tags = ObjectLayout::OldAndNotRememberedBit::update(true, tags);
|
||||
tags = ObjectLayout::NewBit::update(false, tags);
|
||||
raw->ptr()->tags_ = tags;
|
||||
#if defined(HASH_IN_OBJECT_HEADER)
|
||||
raw->ptr()->hash_ = 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
#if !defined(DART_PRECOMPILED_RUNTIME)
|
||||
|
|
|
|||
|
|
@ -1950,7 +1950,7 @@ static void TryAllocateString(Assembler* assembler,
|
|||
|
||||
// Get the class index and insert it into the tags.
|
||||
// R3: size and bit tags.
|
||||
const uint32_t tags =
|
||||
const uword tags =
|
||||
target::MakeTagWordForNewSpaceObject(cid, /*instance_size=*/0);
|
||||
__ LoadImmediate(TMP, tags);
|
||||
__ orr(R3, R3, Operand(TMP));
|
||||
|
|
|
|||
|
|
@ -1736,9 +1736,16 @@ void AsmIntrinsifier::Object_setHash(Assembler* assembler,
|
|||
Label* normal_ir_body) {
|
||||
__ ldr(R0, Address(SP, 1 * target::kWordSize)); // Object.
|
||||
__ ldr(R1, Address(SP, 0 * target::kWordSize)); // Value.
|
||||
// R0: Untagged address of header word (ldxr/stxr do not support offsets).
|
||||
__ sub(R0, R0, Operand(kHeapObjectTag));
|
||||
__ SmiUntag(R1);
|
||||
__ str(R1, FieldAddress(R0, target::String::hash_offset(), kFourBytes),
|
||||
kUnsignedFourBytes);
|
||||
__ LslImmediate(R1, R1, target::ObjectLayout::kHashTagPos);
|
||||
Label retry;
|
||||
__ Bind(&retry);
|
||||
__ ldxr(R2, R0, kEightBytes);
|
||||
__ orr(R2, R2, Operand(R1));
|
||||
__ stxr(R4, R2, R0, kEightBytes);
|
||||
__ cbnz(&retry, R4);
|
||||
__ ret();
|
||||
}
|
||||
|
||||
|
|
@ -1961,8 +1968,18 @@ void AsmIntrinsifier::OneByteString_getHashCode(Assembler* assembler,
|
|||
// return hash_ == 0 ? 1 : hash_;
|
||||
__ Bind(&done);
|
||||
__ csinc(R0, R0, ZR, NE); // R0 <- (R0 != 0) ? R0 : (ZR + 1).
|
||||
__ str(R0, FieldAddress(R1, target::String::hash_offset()),
|
||||
kUnsignedFourBytes);
|
||||
|
||||
// R1: Untagged address of header word (ldxr/stxr do not support offsets).
|
||||
__ sub(R1, R1, Operand(kHeapObjectTag));
|
||||
__ LslImmediate(R0, R0, target::ObjectLayout::kHashTagPos);
|
||||
Label retry;
|
||||
__ Bind(&retry);
|
||||
__ ldxr(R2, R1, kEightBytes);
|
||||
__ orr(R2, R2, Operand(R0));
|
||||
__ stxr(R4, R2, R1, kEightBytes);
|
||||
__ cbnz(&retry, R4);
|
||||
|
||||
__ LsrImmediate(R0, R0, target::ObjectLayout::kHashTagPos);
|
||||
__ SmiTag(R0);
|
||||
__ ret();
|
||||
}
|
||||
|
|
@ -2031,7 +2048,7 @@ static void TryAllocateString(Assembler* assembler,
|
|||
// Get the class index and insert it into the tags.
|
||||
// R2: size and bit tags.
|
||||
// This also clears the hash, which is in the high word of the tags.
|
||||
const uint32_t tags =
|
||||
const uword tags =
|
||||
target::MakeTagWordForNewSpaceObject(cid, /*instance_size=*/0);
|
||||
__ LoadImmediate(TMP, tags);
|
||||
__ orr(R2, R2, Operand(TMP));
|
||||
|
|
|
|||
|
|
@ -1965,7 +1965,7 @@ static void TryAllocateString(Assembler* assembler,
|
|||
__ Bind(&done);
|
||||
|
||||
// Get the class index and insert it into the tags.
|
||||
const uint32_t tags =
|
||||
const uword tags =
|
||||
target::MakeTagWordForNewSpaceObject(cid, /*instance_size=*/0);
|
||||
__ orl(EDI, Immediate(tags));
|
||||
__ movl(FieldAddress(EAX, target::Object::tags_offset()), EDI); // Tags.
|
||||
|
|
|
|||
|
|
@ -1676,7 +1676,10 @@ void AsmIntrinsifier::Object_setHash(Assembler* assembler,
|
|||
__ movq(RAX, Address(RSP, +2 * target::kWordSize)); // Object.
|
||||
__ movq(RDX, Address(RSP, +1 * target::kWordSize)); // Value.
|
||||
__ SmiUntag(RDX);
|
||||
__ movl(FieldAddress(RAX, target::String::hash_offset()), RDX);
|
||||
__ shlq(RDX, Immediate(target::ObjectLayout::kHashTagPos));
|
||||
// lock+orq is an atomic read-modify-write.
|
||||
__ lock();
|
||||
__ orq(FieldAddress(RAX, target::Object::tags_offset()), RDX);
|
||||
__ ret();
|
||||
}
|
||||
|
||||
|
|
@ -1911,7 +1914,11 @@ void AsmIntrinsifier::OneByteString_getHashCode(Assembler* assembler,
|
|||
__ j(NOT_EQUAL, &set_hash_code, Assembler::kNearJump);
|
||||
__ incq(RAX);
|
||||
__ Bind(&set_hash_code);
|
||||
__ movl(FieldAddress(RBX, target::String::hash_offset()), RAX);
|
||||
__ shlq(RAX, Immediate(target::ObjectLayout::kHashTagPos));
|
||||
// lock+orq is an atomic read-modify-write.
|
||||
__ lock();
|
||||
__ orq(FieldAddress(RBX, target::Object::tags_offset()), RAX);
|
||||
__ sarq(RAX, Immediate(target::ObjectLayout::kHashTagPos));
|
||||
__ SmiTag(RAX);
|
||||
__ ret();
|
||||
}
|
||||
|
|
@ -1986,7 +1993,7 @@ static void TryAllocateString(Assembler* assembler,
|
|||
|
||||
// Get the class index and insert it into the tags.
|
||||
// This also clears the hash, which is in the high bits of the tags.
|
||||
const uint32_t tags =
|
||||
const uword tags =
|
||||
target::MakeTagWordForNewSpaceObject(cid, /*instance_size=*/0);
|
||||
__ orq(RDI, Immediate(tags));
|
||||
__ movq(FieldAddress(RAX, target::Object::tags_offset()), RDI); // Tags.
|
||||
|
|
|
|||
|
|
@ -3420,8 +3420,7 @@ void Assembler::TryAllocate(const Class& cls,
|
|||
ASSERT(instance_size >= kHeapObjectTag);
|
||||
AddImmediate(instance_reg, -instance_size + kHeapObjectTag);
|
||||
|
||||
const uint32_t tags =
|
||||
target::MakeTagWordForNewSpaceObject(cid, instance_size);
|
||||
const uword tags = target::MakeTagWordForNewSpaceObject(cid, instance_size);
|
||||
LoadImmediate(IP, tags);
|
||||
str(IP, FieldAddress(instance_reg, target::Object::tags_offset()));
|
||||
} else {
|
||||
|
|
@ -3463,8 +3462,7 @@ void Assembler::TryAllocateArray(intptr_t cid,
|
|||
|
||||
// Initialize the tags.
|
||||
// instance: new object start as a tagged pointer.
|
||||
const uint32_t tags =
|
||||
target::MakeTagWordForNewSpaceObject(cid, instance_size);
|
||||
const uword tags = target::MakeTagWordForNewSpaceObject(cid, instance_size);
|
||||
LoadImmediate(temp2, tags);
|
||||
str(temp2,
|
||||
FieldAddress(instance, target::Object::tags_offset())); // Store tags.
|
||||
|
|
|
|||
|
|
@ -1758,10 +1758,7 @@ void Assembler::TryAllocate(const Class& cls,
|
|||
// next object start and store the class in the class field of object.
|
||||
str(top_reg, Address(THR, target::Thread::top_offset()));
|
||||
|
||||
const uint32_t tags =
|
||||
target::MakeTagWordForNewSpaceObject(cid, instance_size);
|
||||
// Extends the 32 bit tags with zeros, which is the uninitialized
|
||||
// hash code.
|
||||
const uword tags = target::MakeTagWordForNewSpaceObject(cid, instance_size);
|
||||
LoadImmediate(TMP, tags);
|
||||
StoreToOffset(TMP, instance_reg, target::Object::tags_offset());
|
||||
|
||||
|
|
@ -1806,10 +1803,7 @@ void Assembler::TryAllocateArray(intptr_t cid,
|
|||
|
||||
// Initialize the tags.
|
||||
// instance: new object start as a tagged pointer.
|
||||
const uint32_t tags =
|
||||
target::MakeTagWordForNewSpaceObject(cid, instance_size);
|
||||
// Extends the 32 bit tags with zeros, which is the uninitialized
|
||||
// hash code.
|
||||
const uword tags = target::MakeTagWordForNewSpaceObject(cid, instance_size);
|
||||
LoadImmediate(temp2, tags);
|
||||
str(temp2, FieldAddress(instance, target::Object::tags_offset()));
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -2510,8 +2510,7 @@ void Assembler::TryAllocate(const Class& cls,
|
|||
movl(Address(THR, target::Thread::top_offset()), instance_reg);
|
||||
ASSERT(instance_size >= kHeapObjectTag);
|
||||
subl(instance_reg, Immediate(instance_size - kHeapObjectTag));
|
||||
const uint32_t tags =
|
||||
target::MakeTagWordForNewSpaceObject(cid, instance_size);
|
||||
const uword tags = target::MakeTagWordForNewSpaceObject(cid, instance_size);
|
||||
movl(FieldAddress(instance_reg, target::Object::tags_offset()),
|
||||
Immediate(tags));
|
||||
} else {
|
||||
|
|
@ -2552,8 +2551,7 @@ void Assembler::TryAllocateArray(intptr_t cid,
|
|||
addl(instance, Immediate(kHeapObjectTag));
|
||||
|
||||
// Initialize the tags.
|
||||
const uint32_t tags =
|
||||
target::MakeTagWordForNewSpaceObject(cid, instance_size);
|
||||
const uword tags = target::MakeTagWordForNewSpaceObject(cid, instance_size);
|
||||
movl(FieldAddress(instance, target::Object::tags_offset()),
|
||||
Immediate(tags));
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -1971,10 +1971,7 @@ void Assembler::TryAllocate(const Class& cls,
|
|||
movq(Address(THR, target::Thread::top_offset()), instance_reg);
|
||||
ASSERT(instance_size >= kHeapObjectTag);
|
||||
AddImmediate(instance_reg, Immediate(kHeapObjectTag - instance_size));
|
||||
const uint32_t tags =
|
||||
target::MakeTagWordForNewSpaceObject(cid, instance_size);
|
||||
// Extends the 32 bit tags with zeros, which is the uninitialized
|
||||
// hash code.
|
||||
const uword tags = target::MakeTagWordForNewSpaceObject(cid, instance_size);
|
||||
MoveImmediate(FieldAddress(instance_reg, target::Object::tags_offset()),
|
||||
Immediate(tags));
|
||||
} else {
|
||||
|
|
@ -2015,10 +2012,7 @@ void Assembler::TryAllocateArray(intptr_t cid,
|
|||
|
||||
// Initialize the tags.
|
||||
// instance: new object start as a tagged pointer.
|
||||
const uint32_t tags =
|
||||
target::MakeTagWordForNewSpaceObject(cid, instance_size);
|
||||
// Extends the 32 bit tags with zeros, which is the uninitialized
|
||||
// hash code.
|
||||
const uword tags = target::MakeTagWordForNewSpaceObject(cid, instance_size);
|
||||
movq(FieldAddress(instance, target::Object::tags_offset()),
|
||||
Immediate(tags));
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -298,7 +298,7 @@ bool SizeFitsInSizeTag(uword instance_size) {
|
|||
TranslateOffsetInWordsToHost(instance_size));
|
||||
}
|
||||
|
||||
uint32_t MakeTagWordForNewSpaceObject(classid_t cid, uword instance_size) {
|
||||
uword MakeTagWordForNewSpaceObject(classid_t cid, uword instance_size) {
|
||||
return dart::ObjectLayout::SizeTag::encode(
|
||||
TranslateOffsetInWordsToHost(instance_size)) |
|
||||
dart::ObjectLayout::ClassIdTag::encode(cid) |
|
||||
|
|
@ -326,6 +326,10 @@ const word ObjectLayout::kClassIdTagPos = dart::ObjectLayout::kClassIdTagPos;
|
|||
|
||||
const word ObjectLayout::kClassIdTagSize = dart::ObjectLayout::kClassIdTagSize;
|
||||
|
||||
const word ObjectLayout::kHashTagPos = dart::ObjectLayout::kHashTagPos;
|
||||
|
||||
const word ObjectLayout::kHashTagSize = dart::ObjectLayout::kHashTagSize;
|
||||
|
||||
const word ObjectLayout::kSizeTagMaxSizeTag =
|
||||
dart::ObjectLayout::SizeTag::kMaxSizeTagInUnitsOfAlignment *
|
||||
ObjectAlignment::kObjectAlignment;
|
||||
|
|
|
|||
|
|
@ -354,7 +354,7 @@ bool SizeFitsInSizeTag(uword instance_size);
|
|||
// size.
|
||||
//
|
||||
// Note: even on 64-bit platforms we only use lower 32-bits of the tag word.
|
||||
uint32_t MakeTagWordForNewSpaceObject(classid_t cid, uword instance_size);
|
||||
uword MakeTagWordForNewSpaceObject(classid_t cid, uword instance_size);
|
||||
|
||||
//
|
||||
// Target specific information about objects.
|
||||
|
|
@ -415,6 +415,8 @@ class ObjectLayout : public AllStatic {
|
|||
static const word kSizeTagSize;
|
||||
static const word kClassIdTagPos;
|
||||
static const word kClassIdTagSize;
|
||||
static const word kHashTagPos;
|
||||
static const word kHashTagSize;
|
||||
static const word kSizeTagMaxSizeTag;
|
||||
static const word kTagBitsSizeTagPos;
|
||||
static const word kBarrierOverlapShift;
|
||||
|
|
|
|||
|
|
@ -962,7 +962,7 @@ static constexpr dart::compiler::target::word
|
|||
static constexpr dart::compiler::target::word ApiError_InstanceSize = 16;
|
||||
static constexpr dart::compiler::target::word Array_InstanceSize = 24;
|
||||
static constexpr dart::compiler::target::word Array_header_size = 24;
|
||||
static constexpr dart::compiler::target::word Bool_InstanceSize = 12;
|
||||
static constexpr dart::compiler::target::word Bool_InstanceSize = 16;
|
||||
static constexpr dart::compiler::target::word Capability_InstanceSize = 16;
|
||||
static constexpr dart::compiler::target::word Class_InstanceSize = 208;
|
||||
static constexpr dart::compiler::target::word Closure_InstanceSize = 56;
|
||||
|
|
@ -970,7 +970,7 @@ static constexpr dart::compiler::target::word ClosureData_InstanceSize = 56;
|
|||
static constexpr dart::compiler::target::word Code_InstanceSize = 176;
|
||||
static constexpr dart::compiler::target::word CodeSourceMap_HeaderSize = 16;
|
||||
static constexpr dart::compiler::target::word CompressedStackMaps_HeaderSize =
|
||||
12;
|
||||
16;
|
||||
static constexpr dart::compiler::target::word Context_InstanceSize = 24;
|
||||
static constexpr dart::compiler::target::word Context_header_size = 24;
|
||||
static constexpr dart::compiler::target::word ContextScope_InstanceSize = 16;
|
||||
|
|
@ -996,7 +996,7 @@ static constexpr dart::compiler::target::word GrowableObjectArray_InstanceSize =
|
|||
static constexpr dart::compiler::target::word ICData_InstanceSize = 56;
|
||||
static constexpr dart::compiler::target::word Instance_InstanceSize = 8;
|
||||
static constexpr dart::compiler::target::word Instructions_UnalignedHeaderSize =
|
||||
12;
|
||||
16;
|
||||
static constexpr dart::compiler::target::word
|
||||
InstructionsSection_UnalignedHeaderSize = 40;
|
||||
static constexpr dart::compiler::target::word Int32x4_InstanceSize = 24;
|
||||
|
|
@ -2000,7 +2000,7 @@ static constexpr dart::compiler::target::word
|
|||
static constexpr dart::compiler::target::word ApiError_InstanceSize = 16;
|
||||
static constexpr dart::compiler::target::word Array_InstanceSize = 24;
|
||||
static constexpr dart::compiler::target::word Array_header_size = 24;
|
||||
static constexpr dart::compiler::target::word Bool_InstanceSize = 12;
|
||||
static constexpr dart::compiler::target::word Bool_InstanceSize = 16;
|
||||
static constexpr dart::compiler::target::word Capability_InstanceSize = 16;
|
||||
static constexpr dart::compiler::target::word Class_InstanceSize = 208;
|
||||
static constexpr dart::compiler::target::word Closure_InstanceSize = 56;
|
||||
|
|
@ -2008,7 +2008,7 @@ static constexpr dart::compiler::target::word ClosureData_InstanceSize = 56;
|
|||
static constexpr dart::compiler::target::word Code_InstanceSize = 176;
|
||||
static constexpr dart::compiler::target::word CodeSourceMap_HeaderSize = 16;
|
||||
static constexpr dart::compiler::target::word CompressedStackMaps_HeaderSize =
|
||||
12;
|
||||
16;
|
||||
static constexpr dart::compiler::target::word Context_InstanceSize = 24;
|
||||
static constexpr dart::compiler::target::word Context_header_size = 24;
|
||||
static constexpr dart::compiler::target::word ContextScope_InstanceSize = 16;
|
||||
|
|
@ -2034,7 +2034,7 @@ static constexpr dart::compiler::target::word GrowableObjectArray_InstanceSize =
|
|||
static constexpr dart::compiler::target::word ICData_InstanceSize = 56;
|
||||
static constexpr dart::compiler::target::word Instance_InstanceSize = 8;
|
||||
static constexpr dart::compiler::target::word Instructions_UnalignedHeaderSize =
|
||||
12;
|
||||
16;
|
||||
static constexpr dart::compiler::target::word
|
||||
InstructionsSection_UnalignedHeaderSize = 40;
|
||||
static constexpr dart::compiler::target::word Int32x4_InstanceSize = 24;
|
||||
|
|
@ -3030,7 +3030,7 @@ static constexpr dart::compiler::target::word
|
|||
static constexpr dart::compiler::target::word ApiError_InstanceSize = 16;
|
||||
static constexpr dart::compiler::target::word Array_InstanceSize = 24;
|
||||
static constexpr dart::compiler::target::word Array_header_size = 24;
|
||||
static constexpr dart::compiler::target::word Bool_InstanceSize = 12;
|
||||
static constexpr dart::compiler::target::word Bool_InstanceSize = 16;
|
||||
static constexpr dart::compiler::target::word Capability_InstanceSize = 16;
|
||||
static constexpr dart::compiler::target::word Class_InstanceSize = 208;
|
||||
static constexpr dart::compiler::target::word Closure_InstanceSize = 56;
|
||||
|
|
@ -3038,7 +3038,7 @@ static constexpr dart::compiler::target::word ClosureData_InstanceSize = 56;
|
|||
static constexpr dart::compiler::target::word Code_InstanceSize = 144;
|
||||
static constexpr dart::compiler::target::word CodeSourceMap_HeaderSize = 16;
|
||||
static constexpr dart::compiler::target::word CompressedStackMaps_HeaderSize =
|
||||
12;
|
||||
16;
|
||||
static constexpr dart::compiler::target::word Context_InstanceSize = 24;
|
||||
static constexpr dart::compiler::target::word Context_header_size = 24;
|
||||
static constexpr dart::compiler::target::word ContextScope_InstanceSize = 16;
|
||||
|
|
@ -3064,7 +3064,7 @@ static constexpr dart::compiler::target::word GrowableObjectArray_InstanceSize =
|
|||
static constexpr dart::compiler::target::word ICData_InstanceSize = 56;
|
||||
static constexpr dart::compiler::target::word Instance_InstanceSize = 8;
|
||||
static constexpr dart::compiler::target::word Instructions_UnalignedHeaderSize =
|
||||
12;
|
||||
16;
|
||||
static constexpr dart::compiler::target::word
|
||||
InstructionsSection_UnalignedHeaderSize = 40;
|
||||
static constexpr dart::compiler::target::word Int32x4_InstanceSize = 24;
|
||||
|
|
@ -4056,7 +4056,7 @@ static constexpr dart::compiler::target::word
|
|||
static constexpr dart::compiler::target::word ApiError_InstanceSize = 16;
|
||||
static constexpr dart::compiler::target::word Array_InstanceSize = 24;
|
||||
static constexpr dart::compiler::target::word Array_header_size = 24;
|
||||
static constexpr dart::compiler::target::word Bool_InstanceSize = 12;
|
||||
static constexpr dart::compiler::target::word Bool_InstanceSize = 16;
|
||||
static constexpr dart::compiler::target::word Capability_InstanceSize = 16;
|
||||
static constexpr dart::compiler::target::word Class_InstanceSize = 208;
|
||||
static constexpr dart::compiler::target::word Closure_InstanceSize = 56;
|
||||
|
|
@ -4064,7 +4064,7 @@ static constexpr dart::compiler::target::word ClosureData_InstanceSize = 56;
|
|||
static constexpr dart::compiler::target::word Code_InstanceSize = 144;
|
||||
static constexpr dart::compiler::target::word CodeSourceMap_HeaderSize = 16;
|
||||
static constexpr dart::compiler::target::word CompressedStackMaps_HeaderSize =
|
||||
12;
|
||||
16;
|
||||
static constexpr dart::compiler::target::word Context_InstanceSize = 24;
|
||||
static constexpr dart::compiler::target::word Context_header_size = 24;
|
||||
static constexpr dart::compiler::target::word ContextScope_InstanceSize = 16;
|
||||
|
|
@ -4090,7 +4090,7 @@ static constexpr dart::compiler::target::word GrowableObjectArray_InstanceSize =
|
|||
static constexpr dart::compiler::target::word ICData_InstanceSize = 56;
|
||||
static constexpr dart::compiler::target::word Instance_InstanceSize = 8;
|
||||
static constexpr dart::compiler::target::word Instructions_UnalignedHeaderSize =
|
||||
12;
|
||||
16;
|
||||
static constexpr dart::compiler::target::word
|
||||
InstructionsSection_UnalignedHeaderSize = 40;
|
||||
static constexpr dart::compiler::target::word Int32x4_InstanceSize = 24;
|
||||
|
|
@ -5193,7 +5193,7 @@ static constexpr dart::compiler::target::word
|
|||
static constexpr dart::compiler::target::word AOT_ApiError_InstanceSize = 16;
|
||||
static constexpr dart::compiler::target::word AOT_Array_InstanceSize = 24;
|
||||
static constexpr dart::compiler::target::word AOT_Array_header_size = 24;
|
||||
static constexpr dart::compiler::target::word AOT_Bool_InstanceSize = 12;
|
||||
static constexpr dart::compiler::target::word AOT_Bool_InstanceSize = 16;
|
||||
static constexpr dart::compiler::target::word AOT_Capability_InstanceSize = 16;
|
||||
static constexpr dart::compiler::target::word AOT_Class_InstanceSize = 192;
|
||||
static constexpr dart::compiler::target::word AOT_Closure_InstanceSize = 56;
|
||||
|
|
@ -5201,7 +5201,7 @@ static constexpr dart::compiler::target::word AOT_ClosureData_InstanceSize = 56;
|
|||
static constexpr dart::compiler::target::word AOT_Code_InstanceSize = 152;
|
||||
static constexpr dart::compiler::target::word AOT_CodeSourceMap_HeaderSize = 16;
|
||||
static constexpr dart::compiler::target::word
|
||||
AOT_CompressedStackMaps_HeaderSize = 12;
|
||||
AOT_CompressedStackMaps_HeaderSize = 16;
|
||||
static constexpr dart::compiler::target::word AOT_Context_InstanceSize = 24;
|
||||
static constexpr dart::compiler::target::word AOT_Context_header_size = 24;
|
||||
static constexpr dart::compiler::target::word AOT_ContextScope_InstanceSize =
|
||||
|
|
@ -5229,7 +5229,7 @@ static constexpr dart::compiler::target::word
|
|||
static constexpr dart::compiler::target::word AOT_ICData_InstanceSize = 48;
|
||||
static constexpr dart::compiler::target::word AOT_Instance_InstanceSize = 8;
|
||||
static constexpr dart::compiler::target::word
|
||||
AOT_Instructions_UnalignedHeaderSize = 12;
|
||||
AOT_Instructions_UnalignedHeaderSize = 16;
|
||||
static constexpr dart::compiler::target::word
|
||||
AOT_InstructionsSection_UnalignedHeaderSize = 40;
|
||||
static constexpr dart::compiler::target::word AOT_Int32x4_InstanceSize = 24;
|
||||
|
|
@ -5301,7 +5301,7 @@ static constexpr dart::compiler::target::word AOT_UserTag_InstanceSize = 24;
|
|||
static constexpr dart::compiler::target::word AOT_WeakProperty_InstanceSize =
|
||||
32;
|
||||
static constexpr dart::compiler::target::word
|
||||
AOT_WeakSerializationReference_InstanceSize = 12;
|
||||
AOT_WeakSerializationReference_InstanceSize = 16;
|
||||
#endif // defined(TARGET_ARCH_X64)
|
||||
|
||||
#if defined(TARGET_ARCH_IA32)
|
||||
|
|
@ -5773,7 +5773,7 @@ static constexpr dart::compiler::target::word
|
|||
static constexpr dart::compiler::target::word AOT_ApiError_InstanceSize = 16;
|
||||
static constexpr dart::compiler::target::word AOT_Array_InstanceSize = 24;
|
||||
static constexpr dart::compiler::target::word AOT_Array_header_size = 24;
|
||||
static constexpr dart::compiler::target::word AOT_Bool_InstanceSize = 12;
|
||||
static constexpr dart::compiler::target::word AOT_Bool_InstanceSize = 16;
|
||||
static constexpr dart::compiler::target::word AOT_Capability_InstanceSize = 16;
|
||||
static constexpr dart::compiler::target::word AOT_Class_InstanceSize = 192;
|
||||
static constexpr dart::compiler::target::word AOT_Closure_InstanceSize = 56;
|
||||
|
|
@ -5781,7 +5781,7 @@ static constexpr dart::compiler::target::word AOT_ClosureData_InstanceSize = 56;
|
|||
static constexpr dart::compiler::target::word AOT_Code_InstanceSize = 152;
|
||||
static constexpr dart::compiler::target::word AOT_CodeSourceMap_HeaderSize = 16;
|
||||
static constexpr dart::compiler::target::word
|
||||
AOT_CompressedStackMaps_HeaderSize = 12;
|
||||
AOT_CompressedStackMaps_HeaderSize = 16;
|
||||
static constexpr dart::compiler::target::word AOT_Context_InstanceSize = 24;
|
||||
static constexpr dart::compiler::target::word AOT_Context_header_size = 24;
|
||||
static constexpr dart::compiler::target::word AOT_ContextScope_InstanceSize =
|
||||
|
|
@ -5809,7 +5809,7 @@ static constexpr dart::compiler::target::word
|
|||
static constexpr dart::compiler::target::word AOT_ICData_InstanceSize = 48;
|
||||
static constexpr dart::compiler::target::word AOT_Instance_InstanceSize = 8;
|
||||
static constexpr dart::compiler::target::word
|
||||
AOT_Instructions_UnalignedHeaderSize = 12;
|
||||
AOT_Instructions_UnalignedHeaderSize = 16;
|
||||
static constexpr dart::compiler::target::word
|
||||
AOT_InstructionsSection_UnalignedHeaderSize = 40;
|
||||
static constexpr dart::compiler::target::word AOT_Int32x4_InstanceSize = 24;
|
||||
|
|
@ -5881,7 +5881,7 @@ static constexpr dart::compiler::target::word AOT_UserTag_InstanceSize = 24;
|
|||
static constexpr dart::compiler::target::word AOT_WeakProperty_InstanceSize =
|
||||
32;
|
||||
static constexpr dart::compiler::target::word
|
||||
AOT_WeakSerializationReference_InstanceSize = 12;
|
||||
AOT_WeakSerializationReference_InstanceSize = 16;
|
||||
#endif // defined(TARGET_ARCH_ARM64)
|
||||
|
||||
#else // !defined(PRODUCT)
|
||||
|
|
@ -6910,7 +6910,7 @@ static constexpr dart::compiler::target::word
|
|||
static constexpr dart::compiler::target::word AOT_ApiError_InstanceSize = 16;
|
||||
static constexpr dart::compiler::target::word AOT_Array_InstanceSize = 24;
|
||||
static constexpr dart::compiler::target::word AOT_Array_header_size = 24;
|
||||
static constexpr dart::compiler::target::word AOT_Bool_InstanceSize = 12;
|
||||
static constexpr dart::compiler::target::word AOT_Bool_InstanceSize = 16;
|
||||
static constexpr dart::compiler::target::word AOT_Capability_InstanceSize = 16;
|
||||
static constexpr dart::compiler::target::word AOT_Class_InstanceSize = 192;
|
||||
static constexpr dart::compiler::target::word AOT_Closure_InstanceSize = 56;
|
||||
|
|
@ -6918,7 +6918,7 @@ static constexpr dart::compiler::target::word AOT_ClosureData_InstanceSize = 56;
|
|||
static constexpr dart::compiler::target::word AOT_Code_InstanceSize = 120;
|
||||
static constexpr dart::compiler::target::word AOT_CodeSourceMap_HeaderSize = 16;
|
||||
static constexpr dart::compiler::target::word
|
||||
AOT_CompressedStackMaps_HeaderSize = 12;
|
||||
AOT_CompressedStackMaps_HeaderSize = 16;
|
||||
static constexpr dart::compiler::target::word AOT_Context_InstanceSize = 24;
|
||||
static constexpr dart::compiler::target::word AOT_Context_header_size = 24;
|
||||
static constexpr dart::compiler::target::word AOT_ContextScope_InstanceSize =
|
||||
|
|
@ -6946,7 +6946,7 @@ static constexpr dart::compiler::target::word
|
|||
static constexpr dart::compiler::target::word AOT_ICData_InstanceSize = 48;
|
||||
static constexpr dart::compiler::target::word AOT_Instance_InstanceSize = 8;
|
||||
static constexpr dart::compiler::target::word
|
||||
AOT_Instructions_UnalignedHeaderSize = 12;
|
||||
AOT_Instructions_UnalignedHeaderSize = 16;
|
||||
static constexpr dart::compiler::target::word
|
||||
AOT_InstructionsSection_UnalignedHeaderSize = 40;
|
||||
static constexpr dart::compiler::target::word AOT_Int32x4_InstanceSize = 24;
|
||||
|
|
@ -7018,7 +7018,7 @@ static constexpr dart::compiler::target::word AOT_UserTag_InstanceSize = 24;
|
|||
static constexpr dart::compiler::target::word AOT_WeakProperty_InstanceSize =
|
||||
32;
|
||||
static constexpr dart::compiler::target::word
|
||||
AOT_WeakSerializationReference_InstanceSize = 12;
|
||||
AOT_WeakSerializationReference_InstanceSize = 16;
|
||||
#endif // defined(TARGET_ARCH_X64)
|
||||
|
||||
#if defined(TARGET_ARCH_IA32)
|
||||
|
|
@ -7483,7 +7483,7 @@ static constexpr dart::compiler::target::word
|
|||
static constexpr dart::compiler::target::word AOT_ApiError_InstanceSize = 16;
|
||||
static constexpr dart::compiler::target::word AOT_Array_InstanceSize = 24;
|
||||
static constexpr dart::compiler::target::word AOT_Array_header_size = 24;
|
||||
static constexpr dart::compiler::target::word AOT_Bool_InstanceSize = 12;
|
||||
static constexpr dart::compiler::target::word AOT_Bool_InstanceSize = 16;
|
||||
static constexpr dart::compiler::target::word AOT_Capability_InstanceSize = 16;
|
||||
static constexpr dart::compiler::target::word AOT_Class_InstanceSize = 192;
|
||||
static constexpr dart::compiler::target::word AOT_Closure_InstanceSize = 56;
|
||||
|
|
@ -7491,7 +7491,7 @@ static constexpr dart::compiler::target::word AOT_ClosureData_InstanceSize = 56;
|
|||
static constexpr dart::compiler::target::word AOT_Code_InstanceSize = 120;
|
||||
static constexpr dart::compiler::target::word AOT_CodeSourceMap_HeaderSize = 16;
|
||||
static constexpr dart::compiler::target::word
|
||||
AOT_CompressedStackMaps_HeaderSize = 12;
|
||||
AOT_CompressedStackMaps_HeaderSize = 16;
|
||||
static constexpr dart::compiler::target::word AOT_Context_InstanceSize = 24;
|
||||
static constexpr dart::compiler::target::word AOT_Context_header_size = 24;
|
||||
static constexpr dart::compiler::target::word AOT_ContextScope_InstanceSize =
|
||||
|
|
@ -7519,7 +7519,7 @@ static constexpr dart::compiler::target::word
|
|||
static constexpr dart::compiler::target::word AOT_ICData_InstanceSize = 48;
|
||||
static constexpr dart::compiler::target::word AOT_Instance_InstanceSize = 8;
|
||||
static constexpr dart::compiler::target::word
|
||||
AOT_Instructions_UnalignedHeaderSize = 12;
|
||||
AOT_Instructions_UnalignedHeaderSize = 16;
|
||||
static constexpr dart::compiler::target::word
|
||||
AOT_InstructionsSection_UnalignedHeaderSize = 40;
|
||||
static constexpr dart::compiler::target::word AOT_Int32x4_InstanceSize = 24;
|
||||
|
|
@ -7591,7 +7591,7 @@ static constexpr dart::compiler::target::word AOT_UserTag_InstanceSize = 24;
|
|||
static constexpr dart::compiler::target::word AOT_WeakProperty_InstanceSize =
|
||||
32;
|
||||
static constexpr dart::compiler::target::word
|
||||
AOT_WeakSerializationReference_InstanceSize = 12;
|
||||
AOT_WeakSerializationReference_InstanceSize = 16;
|
||||
#endif // defined(TARGET_ARCH_ARM64)
|
||||
|
||||
#endif // !defined(PRODUCT)
|
||||
|
|
|
|||
|
|
@ -1065,7 +1065,7 @@ void StubCodeCompiler::GenerateAllocateArrayStub(Assembler* assembler) {
|
|||
|
||||
// Get the class index and insert it into the tags.
|
||||
// R8: size and bit tags.
|
||||
const uint32_t tags =
|
||||
const uword tags =
|
||||
target::MakeTagWordForNewSpaceObject(cid, /*instance_size=*/0);
|
||||
__ LoadImmediate(TMP, tags);
|
||||
__ orr(R8, R8, Operand(TMP));
|
||||
|
|
@ -1356,7 +1356,7 @@ static void GenerateAllocateContext(Assembler* assembler, Label* slow_case) {
|
|||
|
||||
// Get the class index and insert it into the tags.
|
||||
// R9: size and bit tags.
|
||||
const uint32_t tags =
|
||||
const uword tags =
|
||||
target::MakeTagWordForNewSpaceObject(kContextCid, /*instance_size=*/0);
|
||||
|
||||
__ LoadImmediate(IP, tags);
|
||||
|
|
@ -1875,7 +1875,7 @@ void StubCodeCompiler::GenerateAllocationStubForClass(
|
|||
ASSERT(instance_size > 0);
|
||||
RELEASE_ASSERT(target::Heap::IsAllocatableInNewSpace(instance_size));
|
||||
|
||||
const uint32_t tags =
|
||||
const uword tags =
|
||||
target::MakeTagWordForNewSpaceObject(cls_id, instance_size);
|
||||
|
||||
// Note: Keep in sync with helper function.
|
||||
|
|
@ -3488,8 +3488,7 @@ void StubCodeCompiler::GenerateAllocateTypedDataArrayStub(Assembler* assembler,
|
|||
__ mov(R3, Operand(0), HI);
|
||||
|
||||
/* Get the class index and insert it into the tags. */
|
||||
uint32_t tags =
|
||||
target::MakeTagWordForNewSpaceObject(cid, /*instance_size=*/0);
|
||||
uword tags = target::MakeTagWordForNewSpaceObject(cid, /*instance_size=*/0);
|
||||
__ LoadImmediate(TMP, tags);
|
||||
__ orr(R3, R3, Operand(TMP));
|
||||
__ str(R3, FieldAddress(R0, target::Object::tags_offset())); /* Tags. */
|
||||
|
|
|
|||
|
|
@ -1195,7 +1195,7 @@ void StubCodeCompiler::GenerateAllocateArrayStub(Assembler* assembler) {
|
|||
__ csel(R1, ZR, R1, HI);
|
||||
|
||||
// Get the class index and insert it into the tags.
|
||||
const uint32_t tags =
|
||||
const uword tags =
|
||||
target::MakeTagWordForNewSpaceObject(cid, /*instance_size=*/0);
|
||||
|
||||
__ LoadImmediate(TMP, tags);
|
||||
|
|
@ -1502,7 +1502,7 @@ static void GenerateAllocateContextSpaceStub(Assembler* assembler,
|
|||
|
||||
// Get the class index and insert it into the tags.
|
||||
// R2: size and bit tags.
|
||||
const uint32_t tags =
|
||||
const uword tags =
|
||||
target::MakeTagWordForNewSpaceObject(kContextCid, /*instance_size=*/0);
|
||||
|
||||
__ LoadImmediate(TMP, tags);
|
||||
|
|
@ -1716,14 +1716,12 @@ static void GenerateWriteBarrierStubHelper(Assembler* assembler,
|
|||
ASSERT(target::Object::tags_offset() == 0);
|
||||
__ sub(R3, R1, Operand(kHeapObjectTag));
|
||||
// R3: Untagged address of header word (ldxr/stxr do not support offsets).
|
||||
// Note that we use 32 bit operations here to match the size of the
|
||||
// background sweeper which is also manipulating this 32 bit word.
|
||||
Label retry;
|
||||
__ Bind(&retry);
|
||||
__ ldxr(R2, R3, kFourBytes);
|
||||
__ ldxr(R2, R3, kEightBytes);
|
||||
__ AndImmediate(R2, R2,
|
||||
~(1 << target::ObjectLayout::kOldAndNotRememberedBit));
|
||||
__ stxr(R4, R2, R3, kFourBytes);
|
||||
__ stxr(R4, R2, R3, kEightBytes);
|
||||
__ cbnz(&retry, R4);
|
||||
|
||||
// Load the StoreBuffer block out of the thread. Then load top_ out of the
|
||||
|
|
@ -1766,17 +1764,15 @@ static void GenerateWriteBarrierStubHelper(Assembler* assembler,
|
|||
__ Push(R4); // Spill.
|
||||
|
||||
// Atomically clear kOldAndNotMarkedBit.
|
||||
// Note that we use 32 bit operations here to match the size of the
|
||||
// background sweeper which is also manipulating this 32 bit word.
|
||||
Label marking_retry, lost_race, marking_overflow;
|
||||
ASSERT(target::Object::tags_offset() == 0);
|
||||
__ sub(R3, R0, Operand(kHeapObjectTag));
|
||||
// R3: Untagged address of header word (ldxr/stxr do not support offsets).
|
||||
__ Bind(&marking_retry);
|
||||
__ ldxr(R2, R3, kFourBytes);
|
||||
__ ldxr(R2, R3, kEightBytes);
|
||||
__ tbz(&lost_race, R2, target::ObjectLayout::kOldAndNotMarkedBit);
|
||||
__ AndImmediate(R2, R2, ~(1 << target::ObjectLayout::kOldAndNotMarkedBit));
|
||||
__ stxr(R4, R2, R3, kFourBytes);
|
||||
__ stxr(R4, R2, R3, kEightBytes);
|
||||
__ cbnz(&marking_retry, R4);
|
||||
|
||||
__ LoadFromOffset(R4, THR, target::Thread::marking_stack_block_offset());
|
||||
|
|
@ -2024,7 +2020,7 @@ void StubCodeCompiler::GenerateAllocationStubForClass(
|
|||
ASSERT(instance_size > 0);
|
||||
RELEASE_ASSERT(target::Heap::IsAllocatableInNewSpace(instance_size));
|
||||
|
||||
const uint32_t tags =
|
||||
const uword tags =
|
||||
target::MakeTagWordForNewSpaceObject(cls_id, instance_size);
|
||||
|
||||
// Note: Keep in sync with helper function.
|
||||
|
|
@ -3629,8 +3625,7 @@ void StubCodeCompiler::GenerateAllocateTypedDataArrayStub(Assembler* assembler,
|
|||
__ csel(R2, ZR, R2, HI);
|
||||
|
||||
/* Get the class index and insert it into the tags. */
|
||||
uint32_t tags =
|
||||
target::MakeTagWordForNewSpaceObject(cid, /*instance_size=*/0);
|
||||
uword tags = target::MakeTagWordForNewSpaceObject(cid, /*instance_size=*/0);
|
||||
__ LoadImmediate(TMP, tags);
|
||||
__ orr(R2, R2, Operand(TMP));
|
||||
__ str(R2, FieldAddress(R0, target::Object::tags_offset())); /* Tags. */
|
||||
|
|
|
|||
|
|
@ -833,7 +833,7 @@ void StubCodeCompiler::GenerateAllocateArrayStub(Assembler* assembler) {
|
|||
__ Bind(&done);
|
||||
|
||||
// Get the class index and insert it into the tags.
|
||||
uint32_t tags = target::MakeTagWordForNewSpaceObject(cid, 0);
|
||||
uword tags = target::MakeTagWordForNewSpaceObject(cid, 0);
|
||||
__ orl(EDI, Immediate(tags));
|
||||
__ movl(FieldAddress(EAX, target::Object::tags_offset()), EDI); // Tags.
|
||||
}
|
||||
|
|
@ -1098,7 +1098,7 @@ static void GenerateAllocateContextSpaceStub(Assembler* assembler,
|
|||
// EAX: new object.
|
||||
// EDX: number of context variables.
|
||||
// EBX: size and bit tags.
|
||||
uint32_t tags = target::MakeTagWordForNewSpaceObject(kContextCid, 0);
|
||||
uword tags = target::MakeTagWordForNewSpaceObject(kContextCid, 0);
|
||||
__ orl(EBX, Immediate(tags));
|
||||
__ movl(FieldAddress(EAX, target::Object::tags_offset()), EBX); // Tags.
|
||||
}
|
||||
|
|
@ -1453,8 +1453,8 @@ void StubCodeCompiler::GenerateAllocationStubForClass(
|
|||
// EDX: new object type arguments (if is_cls_parameterized).
|
||||
// Set the tags.
|
||||
ASSERT(target::Class::GetId(cls) != kIllegalCid);
|
||||
uint32_t tags = target::MakeTagWordForNewSpaceObject(
|
||||
target::Class::GetId(cls), instance_size);
|
||||
uword tags = target::MakeTagWordForNewSpaceObject(target::Class::GetId(cls),
|
||||
instance_size);
|
||||
__ movl(Address(EAX, target::Object::tags_offset()), Immediate(tags));
|
||||
__ addl(EAX, Immediate(kHeapObjectTag));
|
||||
|
||||
|
|
@ -2956,8 +2956,7 @@ void StubCodeCompiler::GenerateAllocateTypedDataArrayStub(Assembler* assembler,
|
|||
__ movl(EDI, Immediate(0));
|
||||
__ Bind(&done);
|
||||
/* Get the class index and insert it into the tags. */
|
||||
uint32_t tags =
|
||||
target::MakeTagWordForNewSpaceObject(cid, /*instance_size=*/0);
|
||||
uword tags = target::MakeTagWordForNewSpaceObject(cid, /*instance_size=*/0);
|
||||
__ orl(EDI, Immediate(tags));
|
||||
__ movl(FieldAddress(EAX, target::Object::tags_offset()), EDI); /* Tags. */
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1111,7 +1111,7 @@ void StubCodeCompiler::GenerateAllocateArrayStub(Assembler* assembler) {
|
|||
__ Bind(&done);
|
||||
|
||||
// Get the class index and insert it into the tags.
|
||||
uint32_t tags = target::MakeTagWordForNewSpaceObject(cid, 0);
|
||||
uword tags = target::MakeTagWordForNewSpaceObject(cid, 0);
|
||||
__ orq(RDI, Immediate(tags));
|
||||
__ movq(FieldAddress(RAX, target::Array::tags_offset()), RDI); // Tags.
|
||||
}
|
||||
|
|
@ -1434,7 +1434,7 @@ static void GenerateAllocateContextSpaceStub(Assembler* assembler,
|
|||
// RAX: new object.
|
||||
// R10: number of context variables.
|
||||
// R13: size and bit tags.
|
||||
uint32_t tags = target::MakeTagWordForNewSpaceObject(kContextCid, 0);
|
||||
uword tags = target::MakeTagWordForNewSpaceObject(kContextCid, 0);
|
||||
__ orq(R13, Immediate(tags));
|
||||
__ movq(FieldAddress(RAX, target::Object::tags_offset()), R13); // Tags.
|
||||
}
|
||||
|
|
@ -1635,13 +1635,11 @@ static void GenerateWriteBarrierStubHelper(Assembler* assembler,
|
|||
}
|
||||
|
||||
// Update the tags that this object has been remembered.
|
||||
// Note that we use 32 bit operations here to match the size of the
|
||||
// background sweeper which is also manipulating this 32 bit word.
|
||||
// RDX: Address being stored
|
||||
// RAX: Current tag value
|
||||
// lock+andl is an atomic read-modify-write.
|
||||
// lock+andq is an atomic read-modify-write.
|
||||
__ lock();
|
||||
__ andl(FieldAddress(RDX, target::Object::tags_offset()),
|
||||
__ andq(FieldAddress(RDX, target::Object::tags_offset()),
|
||||
Immediate(~(1 << target::ObjectLayout::kOldAndNotRememberedBit)));
|
||||
|
||||
// Save registers being destroyed.
|
||||
|
|
@ -1688,16 +1686,14 @@ static void GenerateWriteBarrierStubHelper(Assembler* assembler,
|
|||
__ movq(TMP, RAX); // RAX is fixed implicit operand of CAS.
|
||||
|
||||
// Atomically clear kOldAndNotMarkedBit.
|
||||
// Note that we use 32 bit operations here to match the size of the
|
||||
// background marker which is also manipulating this 32 bit word.
|
||||
Label retry, lost_race, marking_overflow;
|
||||
__ movl(RAX, FieldAddress(TMP, target::Object::tags_offset()));
|
||||
__ movq(RAX, FieldAddress(TMP, target::Object::tags_offset()));
|
||||
__ Bind(&retry);
|
||||
__ movl(RCX, RAX);
|
||||
__ testl(RCX, Immediate(1 << target::ObjectLayout::kOldAndNotMarkedBit));
|
||||
__ movq(RCX, RAX);
|
||||
__ testq(RCX, Immediate(1 << target::ObjectLayout::kOldAndNotMarkedBit));
|
||||
__ j(ZERO, &lost_race); // Marked by another thread.
|
||||
__ andl(RCX, Immediate(~(1 << target::ObjectLayout::kOldAndNotMarkedBit)));
|
||||
__ LockCmpxchgl(FieldAddress(TMP, target::Object::tags_offset()), RCX);
|
||||
__ andq(RCX, Immediate(~(1 << target::ObjectLayout::kOldAndNotMarkedBit)));
|
||||
__ LockCmpxchgq(FieldAddress(TMP, target::Object::tags_offset()), RCX);
|
||||
__ j(NOT_EQUAL, &retry, Assembler::kNearJump);
|
||||
|
||||
__ movq(RAX, Address(THR, target::Thread::marking_stack_block_offset()));
|
||||
|
|
@ -1951,7 +1947,7 @@ void StubCodeCompiler::GenerateAllocationStubForClass(
|
|||
// User-defined classes should always be allocatable in new space.
|
||||
RELEASE_ASSERT(target::Heap::IsAllocatableInNewSpace(instance_size));
|
||||
|
||||
const uint32_t tags =
|
||||
const uword tags =
|
||||
target::MakeTagWordForNewSpaceObject(cls_id, instance_size);
|
||||
|
||||
// Note: Keep in sync with helper function.
|
||||
|
|
@ -3584,8 +3580,7 @@ void StubCodeCompiler::GenerateAllocateTypedDataArrayStub(Assembler* assembler,
|
|||
__ Bind(&done);
|
||||
|
||||
/* Get the class index and insert it into the tags. */
|
||||
uint32_t tags =
|
||||
target::MakeTagWordForNewSpaceObject(cid, /*instance_size=*/0);
|
||||
uword tags = target::MakeTagWordForNewSpaceObject(cid, /*instance_size=*/0);
|
||||
__ orq(RDI, Immediate(tags));
|
||||
__ movq(FieldAddress(RAX, target::Object::tags_offset()), RDI); /* Tags. */
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ ForwardingCorpse* ForwardingCorpse::AsForwarder(uword addr, intptr_t size) {
|
|||
|
||||
ForwardingCorpse* result = reinterpret_cast<ForwardingCorpse*>(addr);
|
||||
|
||||
uint32_t tags = 0;
|
||||
uword tags = result->tags_; // Carry-over any identity hash.
|
||||
tags = ObjectLayout::SizeTag::update(size, tags);
|
||||
tags = ObjectLayout::ClassIdTag::update(kForwardingCorpse, tags);
|
||||
bool is_old = (addr & kNewObjectAlignmentOffset) == kOldObjectAlignmentOffset;
|
||||
|
|
|
|||
|
|
@ -53,11 +53,8 @@ class ForwardingCorpse {
|
|||
|
||||
private:
|
||||
// This layout mirrors the layout of RawObject.
|
||||
RelaxedAtomic<uint32_t> tags_;
|
||||
#if defined(HASH_IN_OBJECT_HEADER)
|
||||
uint32_t hash_;
|
||||
#endif
|
||||
ObjectPtr target_;
|
||||
RelaxedAtomic<uword> tags_;
|
||||
RelaxedAtomic<ObjectPtr> target_;
|
||||
|
||||
// Returns the address of the embedded size.
|
||||
intptr_t* SizeAddress() const {
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ FreeListElement* FreeListElement::AsElement(uword addr, intptr_t size) {
|
|||
|
||||
FreeListElement* result = reinterpret_cast<FreeListElement*>(addr);
|
||||
|
||||
uint32_t tags = 0;
|
||||
uword tags = 0;
|
||||
tags = ObjectLayout::SizeTag::update(size, tags);
|
||||
tags = ObjectLayout::ClassIdTag::update(kFreeListElement, tags);
|
||||
ASSERT((addr & kNewObjectAlignmentOffset) == kOldObjectAlignmentOffset);
|
||||
|
|
@ -30,11 +30,7 @@ FreeListElement* FreeListElement::AsElement(uword addr, intptr_t size) {
|
|||
tags = ObjectLayout::OldAndNotRememberedBit::update(true, tags);
|
||||
tags = ObjectLayout::NewBit::update(false, tags);
|
||||
result->tags_ = tags;
|
||||
#if defined(HASH_IN_OBJECT_HEADER)
|
||||
// Clearing this is mostly for neatness. The identityHashCode
|
||||
// of free list entries is not used.
|
||||
result->hash_ = 0;
|
||||
#endif
|
||||
|
||||
if (size > ObjectLayout::SizeTag::kMaxSizeTag) {
|
||||
*result->SizeAddress() = size;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -57,11 +57,8 @@ class FreeListElement {
|
|||
|
||||
private:
|
||||
// This layout mirrors the layout of RawObject.
|
||||
RelaxedAtomic<uint32_t> tags_;
|
||||
#if defined(HASH_IN_OBJECT_HEADER)
|
||||
uint32_t hash_;
|
||||
#endif
|
||||
FreeListElement* next_;
|
||||
RelaxedAtomic<uword> tags_;
|
||||
RelaxedAtomic<FreeListElement*> next_;
|
||||
|
||||
// Returns the address of the embedded size.
|
||||
intptr_t* SizeAddress() const {
|
||||
|
|
|
|||
|
|
@ -240,7 +240,11 @@ class MarkingVisitorBase : public ObjectPointerVisitor {
|
|||
// change in the value.
|
||||
// Doing this before checking for an Instructions object avoids
|
||||
// unnecessary queueing of pre-marked objects.
|
||||
if (raw_obj->ptr()->IsMarked()) {
|
||||
// Race: The concurrent marker may observe a pointer into a heap page that
|
||||
// was allocated after the concurrent marker started. It can read either a
|
||||
// zero or the header of an object allocated black, both of which appear
|
||||
// marked.
|
||||
if (raw_obj->ptr()->IsMarkedIgnoreRace()) {
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -318,7 +318,7 @@ class ScavengerVisitorBase : public ObjectPointerVisitor {
|
|||
new_obj = ObjectLayout::FromAddr(new_addr);
|
||||
if (new_obj->IsOldObject()) {
|
||||
// Promoted: update age/barrier tags.
|
||||
uint32_t tags = static_cast<uint32_t>(header);
|
||||
uword tags = static_cast<uword>(header);
|
||||
tags = ObjectLayout::OldBit::update(true, tags);
|
||||
tags = ObjectLayout::OldAndNotRememberedBit::update(true, tags);
|
||||
tags = ObjectLayout::NewBit::update(false, tags);
|
||||
|
|
@ -1618,15 +1618,15 @@ void Scavenger::ReverseScavenge(SemiSpace** from) {
|
|||
intptr_t size = to_obj->ptr()->HeapSize();
|
||||
|
||||
// Reset the ages bits in case this was a promotion.
|
||||
uint32_t tags = static_cast<uint32_t>(to_header);
|
||||
tags = ObjectLayout::OldBit::update(false, tags);
|
||||
tags = ObjectLayout::OldAndNotRememberedBit::update(false, tags);
|
||||
tags = ObjectLayout::NewBit::update(true, tags);
|
||||
tags = ObjectLayout::OldAndNotMarkedBit::update(false, tags);
|
||||
uword original_header =
|
||||
(to_header & ~static_cast<uword>(0xFFFFFFFF)) | tags;
|
||||
uword from_header = static_cast<uword>(to_header);
|
||||
from_header = ObjectLayout::OldBit::update(false, from_header);
|
||||
from_header =
|
||||
ObjectLayout::OldAndNotRememberedBit::update(false, from_header);
|
||||
from_header = ObjectLayout::NewBit::update(true, from_header);
|
||||
from_header =
|
||||
ObjectLayout::OldAndNotMarkedBit::update(false, from_header);
|
||||
|
||||
WriteHeader(from_obj, original_header);
|
||||
WriteHeader(from_obj, from_header);
|
||||
|
||||
ForwardingCorpse::AsForwarder(ObjectLayout::ToAddr(to_obj), size)
|
||||
->set_target(from_obj);
|
||||
|
|
|
|||
|
|
@ -514,7 +514,7 @@ void ImageWriter::WriteROData(NonStreamingWriteStream* stream, bool vm) {
|
|||
if (obj.IsCompressedStackMaps()) {
|
||||
const CompressedStackMaps& map = CompressedStackMaps::Cast(obj);
|
||||
const intptr_t payload_size = map.payload_size();
|
||||
stream->WriteFixed<uint32_t>(map.raw()->ptr()->flags_and_size_);
|
||||
stream->WriteTargetWord(map.raw()->ptr()->flags_and_size_);
|
||||
ASSERT_EQUAL(stream->Position() - object_start,
|
||||
compiler::target::CompressedStackMaps::HeaderSize());
|
||||
stream->WriteBytes(map.raw()->ptr()->data(), payload_size);
|
||||
|
|
@ -589,12 +589,12 @@ uword ImageWriter::GetMarkedTags(classid_t cid,
|
|||
}
|
||||
|
||||
uword ImageWriter::GetMarkedTags(const Object& obj) {
|
||||
return
|
||||
uword tags = GetMarkedTags(obj.raw()->GetClassId(), SizeInSnapshot(obj),
|
||||
obj.IsCanonical());
|
||||
#if defined(HASH_IN_OBJECT_HEADER)
|
||||
static_cast<uword>(obj.raw()->ptr()->hash_) << kBitsPerInt32 |
|
||||
tags = ObjectLayout::HashTag::update(obj.raw()->ptr()->GetHeaderHash(), tags);
|
||||
#endif
|
||||
GetMarkedTags(obj.raw()->GetClassId(), SizeInSnapshot(obj),
|
||||
obj.IsCanonical());
|
||||
return tags;
|
||||
}
|
||||
|
||||
const char* ImageWriter::SectionSymbol(ProgramSection section, bool vm) const {
|
||||
|
|
|
|||
|
|
@ -1469,8 +1469,8 @@ void Object::MakeUnusedSpaceTraversable(const Object& obj,
|
|||
// new array length, and so treat it as a pointer. Ensure it is a Smi so
|
||||
// the marker won't dereference it.
|
||||
ASSERT((new_tags & kSmiTagMask) == kSmiTag);
|
||||
uint32_t tags = raw->ptr()->tags_;
|
||||
uint32_t old_tags;
|
||||
uword tags = raw->ptr()->tags_;
|
||||
uword old_tags;
|
||||
// TODO(iposva): Investigate whether CompareAndSwapWord is necessary.
|
||||
do {
|
||||
old_tags = tags;
|
||||
|
|
@ -1498,8 +1498,8 @@ void Object::MakeUnusedSpaceTraversable(const Object& obj,
|
|||
// new array length, and so treat it as a pointer. Ensure it is a Smi so
|
||||
// the marker won't dereference it.
|
||||
ASSERT((new_tags & kSmiTagMask) == kSmiTag);
|
||||
uint32_t tags = raw->ptr()->tags_;
|
||||
uint32_t old_tags;
|
||||
uword tags = raw->ptr()->tags_;
|
||||
uword old_tags;
|
||||
// TODO(iposva): Investigate whether CompareAndSwapWord is necessary.
|
||||
do {
|
||||
old_tags = tags;
|
||||
|
|
@ -2471,10 +2471,9 @@ StringPtr Object::DictionaryName() const {
|
|||
}
|
||||
|
||||
void Object::InitializeObject(uword address, intptr_t class_id, intptr_t size) {
|
||||
// Note: we skip the header word here because it confuses TSAN. TSAN records
|
||||
// an 8-byte write from the this loop, but doesn't overwrite that entry with
|
||||
// the 4-byte relaxed store of the header below, then reports false data races
|
||||
// based on the record of the 8-byte write.
|
||||
// Note: we skip the header word here to avoid a racy read in the concurrent
|
||||
// marker from observing the null object when it reads into a heap page
|
||||
// allocated after marking started.
|
||||
uword cur = address + sizeof(ObjectLayout);
|
||||
uword end = address + size;
|
||||
if (class_id == kInstructionsCid) {
|
||||
|
|
@ -2513,7 +2512,7 @@ void Object::InitializeObject(uword address, intptr_t class_id, intptr_t size) {
|
|||
#endif
|
||||
}
|
||||
}
|
||||
uint32_t tags = 0;
|
||||
uword tags = 0;
|
||||
ASSERT(class_id != kIllegalCid);
|
||||
tags = ObjectLayout::ClassIdTag::update(class_id, tags);
|
||||
tags = ObjectLayout::SizeTag::update(size, tags);
|
||||
|
|
@ -2524,9 +2523,6 @@ void Object::InitializeObject(uword address, intptr_t class_id, intptr_t size) {
|
|||
tags = ObjectLayout::OldAndNotRememberedBit::update(is_old, tags);
|
||||
tags = ObjectLayout::NewBit::update(!is_old, tags);
|
||||
reinterpret_cast<ObjectLayout*>(address)->tags_ = tags;
|
||||
#if defined(HASH_IN_OBJECT_HEADER)
|
||||
reinterpret_cast<ObjectLayout*>(address)->hash_ = 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
void Object::CheckHandle() const {
|
||||
|
|
@ -23107,11 +23103,11 @@ void Array::MakeImmutable() const {
|
|||
if (IsImmutable()) return;
|
||||
ASSERT(!IsCanonical());
|
||||
NoSafepointScope no_safepoint;
|
||||
uint32_t tags = raw_ptr()->tags_;
|
||||
uint32_t old_tags;
|
||||
uword tags = raw_ptr()->tags_;
|
||||
uword old_tags;
|
||||
do {
|
||||
old_tags = tags;
|
||||
uint32_t new_tags =
|
||||
uword new_tags =
|
||||
ObjectLayout::ClassIdTag::update(kImmutableArrayCid, old_tags);
|
||||
tags = CompareAndSwapTags(old_tags, new_tags);
|
||||
} while (tags != old_tags);
|
||||
|
|
@ -23176,12 +23172,12 @@ void Array::Truncate(intptr_t new_len) const {
|
|||
std::atomic_thread_fence(std::memory_order_release);
|
||||
|
||||
// Update the size in the header field and length of the array object.
|
||||
uint32_t tags = array.raw_ptr()->tags_;
|
||||
uword tags = array.raw_ptr()->tags_;
|
||||
ASSERT(kArrayCid == ObjectLayout::ClassIdTag::decode(tags));
|
||||
uint32_t old_tags;
|
||||
uword old_tags;
|
||||
do {
|
||||
old_tags = tags;
|
||||
uint32_t new_tags = ObjectLayout::SizeTag::update(new_size, old_tags);
|
||||
uword new_tags = ObjectLayout::SizeTag::update(new_size, old_tags);
|
||||
tags = CompareAndSwapTags(old_tags, new_tags);
|
||||
} while (tags != old_tags);
|
||||
|
||||
|
|
|
|||
|
|
@ -290,7 +290,7 @@ class Object {
|
|||
ObjectPtr raw() const { return raw_; }
|
||||
void operator=(ObjectPtr value) { initializeHandle(this, value); }
|
||||
|
||||
uint32_t CompareAndSwapTags(uint32_t old_tags, uint32_t new_tags) const {
|
||||
uword CompareAndSwapTags(uword old_tags, uword new_tags) const {
|
||||
raw()->ptr()->tags_.StrongCAS(old_tags, new_tags);
|
||||
return old_tags;
|
||||
}
|
||||
|
|
@ -393,11 +393,11 @@ class Object {
|
|||
|
||||
#if defined(HASH_IN_OBJECT_HEADER)
|
||||
static uint32_t GetCachedHash(const ObjectPtr obj) {
|
||||
return obj->ptr()->hash_;
|
||||
return obj->ptr()->GetHeaderHash();
|
||||
}
|
||||
|
||||
static void SetCachedHash(ObjectPtr obj, uint32_t hash) {
|
||||
obj->ptr()->hash_ = hash;
|
||||
obj->ptr()->SetHeaderHash(hash);
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
@ -8661,7 +8661,15 @@ class String : public Instance {
|
|||
return GetCachedHash(raw()) != 0;
|
||||
}
|
||||
|
||||
static intptr_t hash_offset() { return OFFSET_OF(StringLayout, hash_); }
|
||||
static intptr_t hash_offset() {
|
||||
#if defined(HASH_IN_OBJECT_HEADER)
|
||||
COMPILE_ASSERT(ObjectLayout::kHashTagPos % kBitsPerByte == 0);
|
||||
return OFFSET_OF(ObjectLayout, tags_) +
|
||||
ObjectLayout::kHashTagPos / kBitsPerByte;
|
||||
#else
|
||||
return OFFSET_OF(StringLayout, hash_);
|
||||
#endif
|
||||
}
|
||||
static intptr_t Hash(const String& str, intptr_t begin_index, intptr_t len);
|
||||
static intptr_t Hash(const char* characters, intptr_t len);
|
||||
static intptr_t Hash(const uint16_t* characters, intptr_t len);
|
||||
|
|
@ -8886,7 +8894,7 @@ class String : public Instance {
|
|||
return Smi::Value(obj->ptr()->hash_);
|
||||
}
|
||||
|
||||
static void SetCachedHash(StringPtr obj, uintptr_t hash) {
|
||||
static void SetCachedHash(StringPtr obj, uint32_t hash) {
|
||||
obj->ptr()->hash_ = Smi::New(hash);
|
||||
}
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -45,26 +45,26 @@ void ObjectLayout::Validate(IsolateGroup* isolate_group) const {
|
|||
return;
|
||||
}
|
||||
// Validate that the tags_ field is sensible.
|
||||
uint32_t tags = tags_;
|
||||
uword tags = tags_;
|
||||
if (IsNewObject()) {
|
||||
if (!NewBit::decode(tags)) {
|
||||
FATAL1("New object missing kNewBit: %x\n", tags);
|
||||
FATAL1("New object missing kNewBit: %" Px "\n", tags);
|
||||
}
|
||||
if (OldBit::decode(tags)) {
|
||||
FATAL1("New object has kOldBit: %x\n", tags);
|
||||
FATAL1("New object has kOldBit: %" Px "\n", tags);
|
||||
}
|
||||
if (OldAndNotMarkedBit::decode(tags)) {
|
||||
FATAL1("New object has kOldAndNotMarkedBit: %x\n", tags);
|
||||
FATAL1("New object has kOldAndNotMarkedBit: %" Px "\n", tags);
|
||||
}
|
||||
if (OldAndNotRememberedBit::decode(tags)) {
|
||||
FATAL1("New object has kOldAndNotRememberedBit: %x\n", tags);
|
||||
FATAL1("New object has kOldAndNotRememberedBit: %" Px "\n", tags);
|
||||
}
|
||||
} else {
|
||||
if (NewBit::decode(tags)) {
|
||||
FATAL1("Old object has kNewBit: %x\n", tags);
|
||||
FATAL1("Old object has kNewBit: %" Px "\n", tags);
|
||||
}
|
||||
if (!OldBit::decode(tags)) {
|
||||
FATAL1("Old object missing kOldBit: %x\n", tags);
|
||||
FATAL1("Old object missing kOldBit: %" Px "\n", tags);
|
||||
}
|
||||
}
|
||||
const intptr_t class_id = ClassIdTag::decode(tags);
|
||||
|
|
@ -90,7 +90,7 @@ void ObjectLayout::Validate(IsolateGroup* isolate_group) const {
|
|||
// compaction when the class objects are moving. Can use the class
|
||||
// id in the header and the sizes in the Class Table.
|
||||
// Cannot deference ptr()->tags_. May dereference other parts of the object.
|
||||
intptr_t ObjectLayout::HeapSizeFromClass(uint32_t tags) const {
|
||||
intptr_t ObjectLayout::HeapSizeFromClass(uword tags) const {
|
||||
intptr_t class_id = ClassIdTag::decode(tags);
|
||||
intptr_t instance_size = 0;
|
||||
switch (class_id) {
|
||||
|
|
@ -268,7 +268,7 @@ intptr_t ObjectLayout::HeapSizeFromClass(uint32_t tags) const {
|
|||
} while ((instance_size > tags_size) && (--retries_remaining > 0));
|
||||
}
|
||||
if ((instance_size != tags_size) && (tags_size != 0)) {
|
||||
FATAL3("Size mismatch: %" Pd " from class vs %" Pd " from tags %x\n",
|
||||
FATAL3("Size mismatch: %" Pd " from class vs %" Pd " from tags %" Px "\n",
|
||||
instance_size, tags_size, tags);
|
||||
}
|
||||
#endif // DEBUG
|
||||
|
|
@ -664,7 +664,7 @@ intptr_t InstanceLayout::VisitInstancePointers(InstancePtr raw_obj,
|
|||
ObjectPointerVisitor* visitor) {
|
||||
// Make sure that we got here with the tagged pointer as this.
|
||||
ASSERT(raw_obj->IsHeapObject());
|
||||
uint32_t tags = raw_obj->ptr()->tags_;
|
||||
uword tags = raw_obj->ptr()->tags_;
|
||||
intptr_t instance_size = SizeTag::decode(tags);
|
||||
if (instance_size == 0) {
|
||||
instance_size = visitor->isolate_group()->GetClassSizeForHeapWalkAt(
|
||||
|
|
|
|||
|
|
@ -130,10 +130,8 @@ class ObjectLayout {
|
|||
kSizeTagSize = 8,
|
||||
kClassIdTagPos = kSizeTagPos + kSizeTagSize, // = 16
|
||||
kClassIdTagSize = 16,
|
||||
#if defined(HASH_IN_OBJECT_HEADER)
|
||||
kHashTagPos = kClassIdTagPos + kClassIdTagSize, // = 32
|
||||
kHashTagSize = 16,
|
||||
#endif
|
||||
kHashTagSize = 32,
|
||||
};
|
||||
|
||||
static const intptr_t kGenerationalBarrierMask = 1 << kNewBit;
|
||||
|
|
@ -177,7 +175,7 @@ class ObjectLayout {
|
|||
private:
|
||||
// The actual unscaled bit field used within the tag field.
|
||||
class SizeBits
|
||||
: public BitField<uint32_t, intptr_t, kSizeTagPos, kSizeTagSize> {};
|
||||
: public BitField<uword, intptr_t, kSizeTagPos, kSizeTagSize> {};
|
||||
|
||||
static UNLESS_DEBUG(constexpr) intptr_t SizeToTagValue(intptr_t size) {
|
||||
DEBUG_ASSERT(Utils::IsAligned(size, kObjectAlignment));
|
||||
|
|
@ -188,64 +186,68 @@ class ObjectLayout {
|
|||
}
|
||||
};
|
||||
|
||||
class ClassIdTag : public BitField<uint32_t,
|
||||
class ClassIdTag : public BitField<uword,
|
||||
ClassIdTagType,
|
||||
kClassIdTagPos,
|
||||
kClassIdTagSize> {};
|
||||
COMPILE_ASSERT(kBitsPerByte * sizeof(ClassIdTagType) == kClassIdTagSize);
|
||||
|
||||
#if defined(HASH_IN_OBJECT_HEADER)
|
||||
class HashTag : public BitField<uword, uint32_t, kHashTagPos, kHashTagSize> {
|
||||
};
|
||||
#endif
|
||||
|
||||
class CardRememberedBit
|
||||
: public BitField<uint32_t, bool, kCardRememberedBit, 1> {};
|
||||
: public BitField<uword, bool, kCardRememberedBit, 1> {};
|
||||
|
||||
class OldAndNotMarkedBit
|
||||
: public BitField<uint32_t, bool, kOldAndNotMarkedBit, 1> {};
|
||||
: public BitField<uword, bool, kOldAndNotMarkedBit, 1> {};
|
||||
|
||||
class NewBit : public BitField<uint32_t, bool, kNewBit, 1> {};
|
||||
class NewBit : public BitField<uword, bool, kNewBit, 1> {};
|
||||
|
||||
class CanonicalBit : public BitField<uint32_t, bool, kCanonicalBit, 1> {};
|
||||
class CanonicalBit : public BitField<uword, bool, kCanonicalBit, 1> {};
|
||||
|
||||
class OldBit : public BitField<uint32_t, bool, kOldBit, 1> {};
|
||||
class OldBit : public BitField<uword, bool, kOldBit, 1> {};
|
||||
|
||||
class OldAndNotRememberedBit
|
||||
: public BitField<uint32_t, bool, kOldAndNotRememberedBit, 1> {};
|
||||
: public BitField<uword, bool, kOldAndNotRememberedBit, 1> {};
|
||||
|
||||
class ReservedBits
|
||||
: public BitField<uint32_t, intptr_t, kReservedTagPos, kReservedTagSize> {
|
||||
};
|
||||
: public BitField<uword, intptr_t, kReservedTagPos, kReservedTagSize> {};
|
||||
|
||||
class Tags {
|
||||
public:
|
||||
Tags() : tags_(0) {}
|
||||
|
||||
NO_SANITIZE_THREAD
|
||||
operator uint32_t() const {
|
||||
return *reinterpret_cast<const uint32_t*>(&tags_);
|
||||
operator uword() const { return tags_.load(std::memory_order_relaxed); }
|
||||
|
||||
uword operator=(uword tags) {
|
||||
tags_.store(tags, std::memory_order_relaxed);
|
||||
return tags;
|
||||
}
|
||||
|
||||
NO_SANITIZE_THREAD
|
||||
uint32_t operator=(uint32_t tags) {
|
||||
return *reinterpret_cast<uint32_t*>(&tags_) = tags;
|
||||
}
|
||||
|
||||
NO_SANITIZE_THREAD
|
||||
bool StrongCAS(uint32_t old_tags, uint32_t new_tags) {
|
||||
bool StrongCAS(uword old_tags, uword new_tags) {
|
||||
return tags_.compare_exchange_strong(old_tags, new_tags,
|
||||
std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
NO_SANITIZE_THREAD
|
||||
bool WeakCAS(uint32_t old_tags, uint32_t new_tags) {
|
||||
bool WeakCAS(uword old_tags, uword new_tags) {
|
||||
return tags_.compare_exchange_weak(old_tags, new_tags,
|
||||
std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
template <class TagBitField>
|
||||
NO_SANITIZE_THREAD typename TagBitField::Type Read() const {
|
||||
return TagBitField::decode(*reinterpret_cast<const uint32_t*>(&tags_));
|
||||
typename TagBitField::Type Read() const {
|
||||
return TagBitField::decode(tags_.load(std::memory_order_relaxed));
|
||||
}
|
||||
|
||||
template <class TagBitField>
|
||||
NO_SANITIZE_THREAD void UpdateBool(bool value) {
|
||||
NO_SANITIZE_THREAD typename TagBitField::Type ReadIgnoreRace() const {
|
||||
return TagBitField::decode(*reinterpret_cast<const uword*>(&tags_));
|
||||
}
|
||||
|
||||
template <class TagBitField>
|
||||
void UpdateBool(bool value) {
|
||||
if (value) {
|
||||
tags_.fetch_or(TagBitField::encode(true), std::memory_order_relaxed);
|
||||
} else {
|
||||
|
|
@ -254,29 +256,39 @@ class ObjectLayout {
|
|||
}
|
||||
|
||||
template <class TagBitField>
|
||||
NO_SANITIZE_THREAD void UpdateUnsynchronized(
|
||||
typename TagBitField::Type value) {
|
||||
*reinterpret_cast<uint32_t*>(&tags_) =
|
||||
TagBitField::update(value, *reinterpret_cast<uint32_t*>(&tags_));
|
||||
void Update(typename TagBitField::Type value) {
|
||||
uword old_tags = tags_.load(std::memory_order_relaxed);
|
||||
uword new_tags;
|
||||
do {
|
||||
new_tags = TagBitField::update(value, old_tags);
|
||||
} while (!tags_.compare_exchange_weak(old_tags, new_tags,
|
||||
std::memory_order_relaxed));
|
||||
}
|
||||
|
||||
template <class TagBitField>
|
||||
NO_SANITIZE_THREAD bool TryAcquire() {
|
||||
uint32_t mask = TagBitField::encode(true);
|
||||
uint32_t old_tags = tags_.fetch_or(mask, std::memory_order_relaxed);
|
||||
void UpdateUnsynchronized(typename TagBitField::Type value) {
|
||||
tags_.store(
|
||||
TagBitField::update(value, tags_.load(std::memory_order_relaxed)),
|
||||
std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
template <class TagBitField>
|
||||
bool TryAcquire() {
|
||||
uword mask = TagBitField::encode(true);
|
||||
uword old_tags = tags_.fetch_or(mask, std::memory_order_relaxed);
|
||||
return !TagBitField::decode(old_tags);
|
||||
}
|
||||
|
||||
template <class TagBitField>
|
||||
NO_SANITIZE_THREAD bool TryClear() {
|
||||
uint32_t mask = ~TagBitField::encode(true);
|
||||
uint32_t old_tags = tags_.fetch_and(mask, std::memory_order_relaxed);
|
||||
bool TryClear() {
|
||||
uword mask = ~TagBitField::encode(true);
|
||||
uword old_tags = tags_.fetch_and(mask, std::memory_order_relaxed);
|
||||
return TagBitField::decode(old_tags);
|
||||
}
|
||||
|
||||
private:
|
||||
std::atomic<uint32_t> tags_;
|
||||
COMPILE_ASSERT(sizeof(std::atomic<uint32_t>) == sizeof(uint32_t));
|
||||
std::atomic<uword> tags_;
|
||||
COMPILE_ASSERT(sizeof(std::atomic<uword>) == sizeof(uword));
|
||||
};
|
||||
|
||||
// Assumes this is a heap object.
|
||||
|
|
@ -296,6 +308,10 @@ class ObjectLayout {
|
|||
ASSERT(IsOldObject());
|
||||
return !tags_.Read<OldAndNotMarkedBit>();
|
||||
}
|
||||
bool IsMarkedIgnoreRace() const {
|
||||
ASSERT(IsOldObject());
|
||||
return !tags_.ReadIgnoreRace<OldAndNotMarkedBit>();
|
||||
}
|
||||
void SetMarkBit() {
|
||||
ASSERT(IsOldObject());
|
||||
ASSERT(!IsMarked());
|
||||
|
|
@ -357,8 +373,13 @@ class ObjectLayout {
|
|||
|
||||
intptr_t GetClassId() const { return tags_.Read<ClassIdTag>(); }
|
||||
|
||||
#if defined(HASH_IN_OBJECT_HEADER)
|
||||
uint32_t GetHeaderHash() const { return tags_.Read<HashTag>(); }
|
||||
void SetHeaderHash(uint32_t h) { tags_.Update<HashTag>(h); }
|
||||
#endif
|
||||
|
||||
intptr_t HeapSize() const {
|
||||
uint32_t tags = tags_;
|
||||
uword tags = tags_;
|
||||
intptr_t result = SizeTag::decode(tags);
|
||||
if (result != 0) {
|
||||
#if defined(DEBUG)
|
||||
|
|
@ -382,7 +403,7 @@ class ObjectLayout {
|
|||
}
|
||||
|
||||
// This variant must not deference this->tags_.
|
||||
intptr_t HeapSize(uint32_t tags) const {
|
||||
intptr_t HeapSize(uword tags) const {
|
||||
intptr_t result = SizeTag::decode(tags);
|
||||
if (result != 0) {
|
||||
return result;
|
||||
|
|
@ -502,19 +523,11 @@ class ObjectLayout {
|
|||
|
||||
private:
|
||||
Tags tags_; // Various object tags (bits).
|
||||
#if defined(HASH_IN_OBJECT_HEADER)
|
||||
// On 64 bit there is a hash field in the header for the identity hash.
|
||||
uint32_t hash_;
|
||||
#elif defined(IS_SIMARM_X64)
|
||||
// On simarm_x64 the hash isn't used, but we need the padding anyway so that
|
||||
// the object layout fits assumptions made about X64.
|
||||
uint32_t padding_;
|
||||
#endif
|
||||
|
||||
intptr_t VisitPointersPredefined(ObjectPointerVisitor* visitor,
|
||||
intptr_t class_id);
|
||||
|
||||
intptr_t HeapSizeFromClass(uint32_t tags) const;
|
||||
intptr_t HeapSizeFromClass(uword tags) const;
|
||||
|
||||
void SetClassId(intptr_t new_cid) {
|
||||
tags_.UpdateUnsynchronized<ClassIdTag>(new_cid);
|
||||
|
|
@ -557,8 +570,8 @@ class ObjectLayout {
|
|||
|
||||
DART_FORCE_INLINE
|
||||
void CheckHeapPointerStore(ObjectPtr value, Thread* thread) {
|
||||
uint32_t source_tags = this->tags_;
|
||||
uint32_t target_tags = value->ptr()->tags_;
|
||||
uword source_tags = this->tags_;
|
||||
uword target_tags = value->ptr()->tags_;
|
||||
if (((source_tags >> kBarrierOverlapShift) & target_tags &
|
||||
thread->write_barrier_mask()) != 0) {
|
||||
if (value->IsNewObject()) {
|
||||
|
|
@ -604,8 +617,8 @@ class ObjectLayout {
|
|||
DART_FORCE_INLINE void CheckArrayPointerStore(type const* addr,
|
||||
ObjectPtr value,
|
||||
Thread* thread) {
|
||||
uint32_t source_tags = this->tags_;
|
||||
uint32_t target_tags = value->ptr()->tags_;
|
||||
uword source_tags = this->tags_;
|
||||
uword target_tags = value->ptr()->tags_;
|
||||
if (((source_tags >> kBarrierOverlapShift) & target_tags &
|
||||
thread->write_barrier_mask()) != 0) {
|
||||
if (value->IsNewObject()) {
|
||||
|
|
|
|||
|
|
@ -152,7 +152,6 @@ namespace dart {
|
|||
F(Closure, context_) \
|
||||
F(Closure, hash_) \
|
||||
F(String, length_) \
|
||||
F(String, hash_) \
|
||||
F(Array, type_arguments_) \
|
||||
F(Array, length_) \
|
||||
F(GrowableObjectArray, type_arguments_) \
|
||||
|
|
@ -209,6 +208,8 @@ namespace dart {
|
|||
F(ReceivePort, debug_name_) \
|
||||
F(ReceivePort, allocation_location_)
|
||||
|
||||
#define NON_HEADER_HASH_CLASSES_AND_FIELDS(F) F(String, hash_)
|
||||
|
||||
OffsetsTable::OffsetsTable(Zone* zone) : cached_offsets_(zone) {
|
||||
for (intptr_t i = 0; offsets_table[i].class_id != -1; ++i) {
|
||||
OffsetsTableEntry entry = offsets_table[i];
|
||||
|
|
@ -232,6 +233,10 @@ OffsetsTable::OffsetsTableEntry OffsetsTable::offsets_table[] = {
|
|||
NON_PRODUCT_CLASSES_AND_FIELDS(DEFINE_OFFSETS_TABLE_ENTRY)
|
||||
#endif
|
||||
|
||||
#if !defined(HASH_IN_OBJECT_HEADER)
|
||||
NON_HEADER_HASH_CLASSES_AND_FIELDS(DEFINE_OFFSETS_TABLE_ENTRY)
|
||||
#endif
|
||||
|
||||
#if defined(DART_PRECOMPILED_RUNTIME)
|
||||
AOT_CLASSES_AND_FIELDS(DEFINE_OFFSETS_TABLE_ENTRY)
|
||||
#else
|
||||
|
|
|
|||
|
|
@ -926,19 +926,25 @@ void SnapshotWriter::WriteObject(ObjectPtr rawobj) {
|
|||
}
|
||||
|
||||
uint32_t SnapshotWriter::GetObjectTags(ObjectPtr raw) {
|
||||
return raw->ptr()->tags_;
|
||||
uword tags = raw->ptr()->tags_;
|
||||
#if defined(HASH_IN_OBJECT_HEADER)
|
||||
// Clear hash to make the narrowing cast safe / appease UBSAN.
|
||||
tags = ObjectLayout::HashTag::update(0, tags);
|
||||
#endif
|
||||
return tags;
|
||||
}
|
||||
|
||||
uint32_t SnapshotWriter::GetObjectTags(ObjectLayout* raw) {
|
||||
return raw->tags_;
|
||||
uword tags = raw->tags_;
|
||||
#if defined(HASH_IN_OBJECT_HEADER)
|
||||
// Clear hash to make the narrowing cast safe / appease UBSAN.
|
||||
tags = ObjectLayout::HashTag::update(0, tags);
|
||||
#endif
|
||||
return tags;
|
||||
}
|
||||
|
||||
uword SnapshotWriter::GetObjectTagsAndHash(ObjectPtr raw) {
|
||||
uword result = raw->ptr()->tags_;
|
||||
#if defined(HASH_IN_OBJECT_HEADER)
|
||||
result |= static_cast<uword>(raw->ptr()->hash_) << 32;
|
||||
#endif
|
||||
return result;
|
||||
return raw->ptr()->tags_;
|
||||
}
|
||||
|
||||
#define VM_OBJECT_CLASS_LIST(V) \
|
||||
|
|
@ -1417,7 +1423,7 @@ FunctionPtr SnapshotWriter::IsSerializableClosure(ClosurePtr closure) {
|
|||
|
||||
ClassPtr SnapshotWriter::GetFunctionOwner(FunctionPtr func) {
|
||||
ObjectPtr owner = func->ptr()->owner_;
|
||||
uint32_t tags = GetObjectTags(owner);
|
||||
uword tags = GetObjectTags(owner);
|
||||
intptr_t class_id = ObjectLayout::ClassIdTag::decode(tags);
|
||||
if (class_id == kClassCid) {
|
||||
return static_cast<ClassPtr>(owner);
|
||||
|
|
|
|||
Loading…
Reference in a new issue