languages/dart-sdk
languages/dart-sdk
1
0
Fork 0
mirror of https://github.com/dart-lang/sdk synced 2024-10-14 08:41:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

[vm, compiler] More consistently support large load/store offsets.

Browse source 
TEST=ci
Change-Id: I7691808679c57c6f16f3859641cf52b9e606e304
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/275381
Reviewed-by: Alexander Markov <alexmarkov@google.com>
Commit-Queue: Ryan Macnak <rmacnak@google.com>
This commit is contained in:
Ryan Macnak 2022-12-14 22:17:35 +00:00 committed by Commit Queue
parent faf8027465
commit 8a37b70a53
7 changed files with 154 additions and 143 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
runtime/platform/assert.h
View file

@ -281,6 +281,13 @@ void Expect::Null(const T p) {
} \
} while (false)
#define ASSERT_IMPLIES(antecedent, consequent) \
do { \
if (antecedent) { \
ASSERT(consequent); \
} \
} while (false)
// DEBUG_ASSERT allows identifiers in condition to be undeclared in release
// mode.
#define DEBUG_ASSERT(cond) ASSERT(cond)
@ -300,7 +307,11 @@ void Expect::Null(const T p) {
#define ASSERT_EQUAL(expected, actual) \
do { \
} while (false && (expected) != (actual))
} while (false && ((expected) != (actual)))
#define ASSERT_IMPLIES(antecedent, consequent) \
do { \
} while (false && (!(antecedent) || (consequent)))
#define DEBUG_ASSERT(cond)

155
runtime/vm/compiler/assembler/assembler_arm.cc
View file

@ -1738,7 +1738,7 @@ void Assembler::StoreIntoObject(Register object,
if (memory_order == kRelease) {
StoreRelease(value, dest);
} else {
str(value, dest);
StoreToOffset(value, dest);
}
// In parallel, test whether
@ -1874,7 +1874,7 @@ void Assembler::StoreIntoObjectNoBarrier(Register object,
if (memory_order == kRelease) {
StoreRelease(value, dest);
} else {
str(value, dest);
StoreToOffset(value, dest);
}
#if defined(DEBUG)
// We can't assert the incremental barrier is not needed here, only the
@ -1995,7 +1995,7 @@ void Assembler::StoreIntoSmiField(const Address& dest, Register value) {
Stop("New value must be Smi.");
Bind(&done);
#endif // defined(DEBUG)
str(value, dest);
StoreToOffset(value, dest);
}
void Assembler::ExtractClassIdFromTags(Register result,
@ -2814,38 +2814,15 @@ void Assembler::LoadQImmediate(QRegister qd, simd128_value_t value) {
LoadMultipleDFromOffset(EvenDRegisterOf(qd), 2, PP, offset);
}
void Assembler::LoadFromOffset(Register reg,
const Address& address,
OperandSize size,
Condition cond) {
switch (size) {
case kByte:
ldrsb(reg, address, cond);
break;
case kUnsignedByte:
ldrb(reg, address, cond);
break;
case kTwoBytes:
ldrsh(reg, address, cond);
break;
case kUnsignedTwoBytes:
ldrh(reg, address, cond);
break;
case kUnsignedFourBytes:
case kFourBytes:
ldr(reg, address, cond);
break;
default:
UNREACHABLE();
}
}
void Assembler::LoadFromOffset(Register reg,
Register base,
int32_t offset,
OperandSize size,
Condition cond) {
Address Assembler::PrepareLargeLoadOffset(const Address& address,
OperandSize size,
Condition cond) {
ASSERT(size != kWordPair);
if (address.kind() != Address::Immediate) {
return address;
}
Register base = address.base();
int32_t offset = address.offset();
int32_t offset_mask = 0;
if (!Address::CanHoldLoadOffset(size, offset, &offset_mask)) {
ASSERT(base != IP);
@ -2853,7 +2830,53 @@ void Assembler::LoadFromOffset(Register reg,
base = IP;
offset = offset & offset_mask;
}
LoadFromOffset(reg, Address(base, offset), size, cond);
return Address(base, offset);
}
Address Assembler::PrepareLargeStoreOffset(const Address& address,
OperandSize size,
Condition cond) {
ASSERT(size != kWordPair);
if (address.kind() != Address::Immediate) {
return address;
}
Register base = address.base();
int32_t offset = address.offset();
int32_t offset_mask = 0;
if (!Address::CanHoldStoreOffset(size, offset, &offset_mask)) {
ASSERT(base != IP);
AddImmediate(IP, base, offset & ~offset_mask, cond);
base = IP;
offset = offset & offset_mask;
}
return Address(base, offset);
}
void Assembler::LoadFromOffset(Register reg,
const Address& address,
OperandSize size,
Condition cond) {
const Address& addr = PrepareLargeLoadOffset(address, size, cond);
switch (size) {
case kByte:
ldrsb(reg, addr, cond);
break;
case kUnsignedByte:
ldrb(reg, addr, cond);
break;
case kTwoBytes:
ldrsh(reg, addr, cond);
break;
case kUnsignedTwoBytes:
ldrh(reg, addr, cond);
break;
case kUnsignedFourBytes:
case kFourBytes:
ldr(reg, addr, cond);
break;
default:
UNREACHABLE();
}
}
void Assembler::LoadFromStack(Register dst, intptr_t depth) {
@ -2875,95 +2898,53 @@ void Assembler::StoreToOffset(Register reg,
const Address& address,
OperandSize size,
Condition cond) {
const Address& addr = PrepareLargeStoreOffset(address, size, cond);
switch (size) {
case kUnsignedByte:
case kByte:
strb(reg, address, cond);
strb(reg, addr, cond);
break;
case kUnsignedTwoBytes:
case kTwoBytes:
strh(reg, address, cond);
strh(reg, addr, cond);
break;
case kUnsignedFourBytes:
case kFourBytes:
str(reg, address, cond);
str(reg, addr, cond);
break;
default:
UNREACHABLE();
}
}
void Assembler::StoreToOffset(Register reg,
Register base,
int32_t offset,
OperandSize size,
Condition cond) {
ASSERT(size != kWordPair);
int32_t offset_mask = 0;
if (!Address::CanHoldStoreOffset(size, offset, &offset_mask)) {
ASSERT(reg != IP);
ASSERT(base != IP);
AddImmediate(IP, base, offset & ~offset_mask, cond);
base = IP;
offset = offset & offset_mask;
}
StoreToOffset(reg, Address(base, offset), size, cond);
}
void Assembler::LoadSFromOffset(SRegister reg,
Register base,
int32_t offset,
Condition cond) {
int32_t offset_mask = 0;
if (!Address::CanHoldLoadOffset(kSWord, offset, &offset_mask)) {
ASSERT(base != IP);
AddImmediate(IP, base, offset & ~offset_mask, cond);
base = IP;
offset = offset & offset_mask;
}
vldrs(reg, Address(base, offset), cond);
vldrs(reg, PrepareLargeLoadOffset(Address(base, offset), kSWord, cond), cond);
}
void Assembler::StoreSToOffset(SRegister reg,
Register base,
int32_t offset,
Condition cond) {
int32_t offset_mask = 0;
if (!Address::CanHoldStoreOffset(kSWord, offset, &offset_mask)) {
ASSERT(base != IP);
AddImmediate(IP, base, offset & ~offset_mask, cond);
base = IP;
offset = offset & offset_mask;
}
vstrs(reg, Address(base, offset), cond);
vstrs(reg, PrepareLargeStoreOffset(Address(base, offset), kSWord, cond),
cond);
}
void Assembler::LoadDFromOffset(DRegister reg,
Register base,
int32_t offset,
Condition cond) {
int32_t offset_mask = 0;
if (!Address::CanHoldLoadOffset(kDWord, offset, &offset_mask)) {
ASSERT(base != IP);
AddImmediate(IP, base, offset & ~offset_mask, cond);
base = IP;
offset = offset & offset_mask;
}
vldrd(reg, Address(base, offset), cond);
vldrd(reg, PrepareLargeLoadOffset(Address(base, offset), kDWord, cond), cond);
}
void Assembler::StoreDToOffset(DRegister reg,
Register base,
int32_t offset,
Condition cond) {
int32_t offset_mask = 0;
if (!Address::CanHoldStoreOffset(kDWord, offset, &offset_mask)) {
ASSERT(base != IP);
AddImmediate(IP, base, offset & ~offset_mask, cond);
base = IP;
offset = offset & offset_mask;
}
vstrd(reg, Address(base, offset), cond);
vstrd(reg, PrepareLargeStoreOffset(Address(base, offset), kDWord, cond),
cond);
}
void Assembler::LoadMultipleDFromOffset(DRegister first,

56
runtime/vm/compiler/assembler/assembler_arm.h
View file

@ -235,11 +235,17 @@ class Address : public ValueObject {
};
Address(const Address& other)
: ValueObject(), encoding_(other.encoding_), kind_(other.kind_) {}
: ValueObject(),
encoding_(other.encoding_),
kind_(other.kind_),
base_(other.base_),
offset_(other.offset_) {}
Address& operator=(const Address& other) {
encoding_ = other.encoding_;
kind_ = other.kind_;
base_ = other.base_;
offset_ = other.offset_;
return *this;
}
@ -248,8 +254,9 @@ class Address : public ValueObject {
}
explicit Address(Register rn, int32_t offset = 0, Mode am = Offset) {
ASSERT(Utils::MagnitudeIsUint(12, offset));
kind_ = Immediate;
base_ = rn;
offset_ = offset;
if (offset < 0) {
encoding_ = (am ^ (1 << kUShift)) | -offset; // Flip U to adjust sign.
} else {
@ -328,7 +335,10 @@ class Address : public ValueObject {
}
}
uint32_t encoding() const { return encoding_; }
uint32_t encoding() const {
ASSERT_IMPLIES(kind_ == Immediate, Utils::MagnitudeIsUint(12, offset_));
return encoding_;
}
// Encoding for addressing mode 3.
uint32_t encoding3() const;
@ -337,10 +347,14 @@ class Address : public ValueObject {
uint32_t vencoding() const;
OffsetKind kind() const { return kind_; }
Register base() const { return base_; }
int32_t offset() const { return offset_; }
uint32_t encoding_;
OffsetKind kind_;
Register base_;
int32_t offset_;
friend class Assembler;
};
@ -403,12 +417,14 @@ class Assembler : public AssemblerBase {
void Jump(const Address& address) { Branch(address); }
void LoadField(Register dst, const FieldAddress& address) override {
ldr(dst, address);
LoadFromOffset(dst, address);
}
void LoadMemoryValue(Register dst, Register base, int32_t offset) {
LoadFromOffset(dst, base, offset);
}
void LoadCompressed(Register dest, const Address& slot) { ldr(dest, slot); }
void LoadCompressed(Register dest, const Address& slot) {
LoadFromOffset(dest, slot);
}
void LoadCompressedSmi(Register dest, const Address& slot) override;
void StoreMemoryValue(Register src, Register base, int32_t offset) {
StoreToOffset(src, base, offset);
@ -416,7 +432,7 @@ class Assembler : public AssemblerBase {
void LoadAcquire(Register dst,
Register address,
int32_t offset = 0) override {
ldr(dst, Address(address, offset));
LoadFromOffset(dst, Address(address, offset));
dmb();
}
void StoreRelease(Register src,
@ -426,7 +442,7 @@ class Assembler : public AssemblerBase {
}
void StoreRelease(Register src, Address dest) {
dmb();
str(src, dest);
StoreToOffset(src, dest);
// We don't run TSAN bots on 32 bit.
}
@ -439,7 +455,7 @@ class Assembler : public AssemblerBase {
}
void CompareWithMemoryValue(Register value, Address address) {
ldr(TMP, address);
LoadFromOffset(TMP, address);
cmp(value, Operand(TMP));
}
@ -1062,6 +1078,14 @@ class Assembler : public AssemblerBase {
bool can_be_null = false) override;
bool CanLoadFromObjectPool(const Object& object) const;
Address PrepareLargeLoadOffset(const Address& addr,
OperandSize sz,
Condition cond);
Address PrepareLargeStoreOffset(const Address& addr,
OperandSize sz,
Condition cond);
void LoadFromOffset(Register reg,
const Address& address,
OperandSize type,
@ -1075,19 +1099,21 @@ class Assembler : public AssemblerBase {
Register base,
int32_t offset,
OperandSize type = kFourBytes,
Condition cond = AL);
Condition cond = AL) {
LoadFromOffset(reg, Address(base, offset), type, cond);
}
void LoadFieldFromOffset(Register reg,
Register base,
int32_t offset,
OperandSize sz = kFourBytes) override {
LoadFieldFromOffset(reg, base, offset, sz, AL);
LoadFromOffset(reg, FieldAddress(base, offset), sz, AL);
}
void LoadFieldFromOffset(Register reg,
Register base,
int32_t offset,
OperandSize type,
Condition cond) {
LoadFromOffset(reg, base, offset - kHeapObjectTag, type, cond);
LoadFromOffset(reg, FieldAddress(base, offset), type, cond);
}
void LoadCompressedFieldFromOffset(Register reg,
Register base,
@ -1137,17 +1163,19 @@ class Assembler : public AssemblerBase {
Register base,
int32_t offset,
OperandSize type = kFourBytes,
Condition cond = AL);
Condition cond = AL) {
StoreToOffset(reg, Address(base, offset), type, cond);
}
void StoreFieldToOffset(Register reg,
Register base,
int32_t offset,
OperandSize type = kFourBytes,
Condition cond = AL) {
StoreToOffset(reg, base, offset - kHeapObjectTag, type, cond);
StoreToOffset(reg, FieldAddress(base, offset), type, cond);
}
void StoreZero(const Address& address, Register temp) {
mov(temp, Operand(0));
str(temp, address);
StoreToOffset(temp, address);
}
void LoadSFromOffset(SRegister reg,
Register base,

10
runtime/vm/compiler/assembler/assembler_arm64.cc
View file

@ -877,10 +877,9 @@ Address Assembler::PrepareLargeOffset(Register base,
}
void Assembler::LoadFromOffset(Register dest,
Register base,
int32_t offset,
const Address& addr,
OperandSize sz) {
LoadFromOffset(dest, PrepareLargeOffset(base, offset, sz), sz);
ldr(dest, PrepareLargeOffset(addr.base(), addr.offset(), sz), sz);
}
void Assembler::LoadSFromOffset(VRegister dest, Register base, int32_t offset) {
@ -896,10 +895,9 @@ void Assembler::LoadQFromOffset(VRegister dest, Register base, int32_t offset) {
}
void Assembler::StoreToOffset(Register src,
Register base,
int32_t offset,
const Address& addr,
OperandSize sz) {
StoreToOffset(src, PrepareLargeOffset(base, offset, sz), sz);
str(src, PrepareLargeOffset(addr.base(), addr.offset(), sz), sz);
}
void Assembler::StoreSToOffset(VRegister src, Register base, int32_t offset) {

27
runtime/vm/compiler/assembler/assembler_arm64.h
View file

@ -357,6 +357,7 @@ class Address : public ValueObject {
AddressType type() const { return type_; }
Register base() const { return base_; }
int32_t offset() const { return offset_; }
Address() : type_(Unknown), base_(kNoRegister), offset_(0) {}
@ -557,7 +558,7 @@ class Assembler : public AssemblerBase {
}
void LoadField(Register dst, const FieldAddress& address) override {
ldr(dst, address);
LoadFromOffset(dst, address);
}
void LoadCompressedField(Register dst, const FieldAddress& address) override {
LoadCompressed(dst, address);
@ -646,7 +647,7 @@ class Assembler : public AssemblerBase {
void CompareWithMemoryValue(Register value,
Address address,
OperandSize sz = kEightBytes) {
ldr(TMP, address, sz);
LoadFromOffset(TMP, address, sz);
cmp(value, Operand(TMP), sz);
}
@ -1887,18 +1888,18 @@ class Assembler : public AssemblerBase {
Address PrepareLargeOffset(Register base, int32_t offset, OperandSize sz);
void LoadFromOffset(Register dest,
const Address& address,
OperandSize sz = kEightBytes) override {
ldr(dest, address, sz);
}
OperandSize sz = kEightBytes) override;
void LoadFromOffset(Register dest,
Register base,
int32_t offset,
OperandSize sz = kEightBytes);
OperandSize sz = kEightBytes) {
LoadFromOffset(dest, Address(base, offset), sz);
}
void LoadFieldFromOffset(Register dest,
Register base,
int32_t offset,
OperandSize sz = kEightBytes) override {
LoadFromOffset(dest, base, offset - kHeapObjectTag, sz);
LoadFromOffset(dest, FieldAddress(base, offset), sz);
}
void LoadCompressedFieldFromOffset(Register dest,
Register base,
@ -1945,21 +1946,21 @@ class Assembler : public AssemblerBase {
void StoreToOffset(Register src,
const Address& address,
OperandSize sz = kEightBytes) override {
str(src, address, sz);
}
OperandSize sz = kEightBytes) override;
void StoreToOffset(Register src,
Register base,
int32_t offset,
OperandSize sz = kEightBytes);
OperandSize sz = kEightBytes) {
StoreToOffset(src, Address(base, offset), sz);
}
void StoreFieldToOffset(Register src,
Register base,
int32_t offset,
OperandSize sz = kEightBytes) {
StoreToOffset(src, base, offset - kHeapObjectTag, sz);
StoreToOffset(src, FieldAddress(base, offset), sz);
}
void StoreZero(const Address& address, Register temp = kNoRegister) {
str(ZR, address);
StoreToOffset(ZR, address);
}
void StoreSToOffset(VRegister src, Register base, int32_t offset);

20
runtime/vm/compiler/assembler/assembler_riscv.cc
View file

@ -3177,13 +3177,7 @@ Address Assembler::PrepareLargeOffset(Register base, int32_t offset) {
void Assembler::LoadFromOffset(Register dest,
const Address& address,
OperandSize sz) {
LoadFromOffset(dest, address.base(), address.offset(), sz);
}
void Assembler::LoadFromOffset(Register dest,
Register base,
int32_t offset,
OperandSize sz) {
Address addr = PrepareLargeOffset(base, offset);
Address addr = PrepareLargeOffset(address.base(), address.offset());
switch (sz) {
#if XLEN == 64
case kEightBytes:
@ -3248,13 +3242,7 @@ void Assembler::CompareToStack(Register src, intptr_t depth) {
void Assembler::StoreToOffset(Register src,
const Address& address,
OperandSize sz) {
StoreToOffset(src, address.base(), address.offset(), sz);
}
void Assembler::StoreToOffset(Register src,
Register base,
int32_t offset,
OperandSize sz) {
Address addr = PrepareLargeOffset(base, offset);
Address addr = PrepareLargeOffset(address.base(), address.offset());
switch (sz) {
#if XLEN == 64
case kEightBytes:
@ -3294,7 +3282,7 @@ void Assembler::StoreIntoObject(Register object,
MemoryOrder memory_order) {
// stlr does not feature an address operand.
ASSERT(memory_order == kRelaxedNonAtomic);
sx(value, dest);
StoreToOffset(value, dest);
StoreBarrier(object, value, can_value_be_smi);
}
void Assembler::StoreCompressedIntoObject(Register object,
@ -3446,7 +3434,7 @@ void Assembler::StoreIntoObjectNoBarrier(Register object,
Register value,
MemoryOrder memory_order) {
ASSERT(memory_order == kRelaxedNonAtomic);
sx(value, dest);
StoreToOffset(value, dest);
#if defined(DEBUG)
// We can't assert the incremental barrier is not needed here, only the
// generational barrier. We sometimes omit the write barrier when 'value' is

16
runtime/vm/compiler/assembler/assembler_riscv.h
View file

@ -1086,7 +1086,9 @@ class Assembler : public MicroAssembler {
void LoadFromOffset(Register dest,
Register base,
int32_t offset,
OperandSize sz = kWordBytes);
OperandSize sz = kWordBytes) {
LoadFromOffset(dest, Address(base, offset), sz);
}
void LoadFieldFromOffset(Register dest,
Register base,
int32_t offset,
@ -1132,15 +1134,17 @@ class Assembler : public MicroAssembler {
void StoreToOffset(Register src,
Register base,
int32_t offset,
OperandSize sz = kWordBytes);
OperandSize sz = kWordBytes) {
StoreToOffset(src, Address(base, offset), sz);
}
void StoreFieldToOffset(Register src,
Register base,
int32_t offset,
OperandSize sz = kWordBytes) {
StoreToOffset(src, base, offset - kHeapObjectTag, sz);
StoreToOffset(src, FieldAddress(base, offset), sz);
}
void StoreZero(const Address& address, Register temp = kNoRegister) {
sx(ZR, address);
StoreToOffset(ZR, address);
}
void StoreSToOffset(FRegister src, Register base, int32_t offset);
void StoreDToOffset(FRegister src, Register base, int32_t offset);
@ -1172,13 +1176,13 @@ class Assembler : public MicroAssembler {
}
void LoadCompressed(Register dest, const Address& slot) {
lx(dest, slot);
LoadFromOffset(dest, slot);
}
void LoadCompressedFromOffset(Register dest, Register base, int32_t offset) {
LoadFromOffset(dest, base, offset);
}
void LoadCompressedSmi(Register dest, const Address& slot) override {
lx(dest, slot);
LoadFromOffset(dest, slot);
#if defined(DEBUG)
Label done;
BranchIfSmi(dest, &done, kNearJump);