languages/dart-sdk
languages/dart-sdk
1
0
Fork 0
mirror of https://github.com/dart-lang/sdk synced 2024-10-14 15:47:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

[vm/compiler] Handle hash caches in the InstantiateTypeArguments stubs.

Browse source 
Lower the threshold for converting from a linear to a hash-based cache
on most architectures from 500 to 10.

Due to register pressure, the InstantiateTypeArguments stub on IA32
continues to go to the runtime for hash caches, and so we do not
lower the threshold for converting to a hash-based cache there.

The following are benchmark results for those benchmark that use enough
Instantiations to trigger the use of hash-based caches. In the following
tables, "Results 1" denotes the benchmark results from only this change,
whereas "Results 2" include from comparing to the results prior to
4f925105cf, when only linear caches were used.

Dart AOT:

* InstantiateTypeArguments.Instantiate100

Arch   | CPU            | Results 1 | Results 2
-------|----------------|-----------------------
ARM    | Odroid-C2      |   382.8%  |   381.5%
ARM    | Raspberry Pi 4 |   486.7%  |   449.2%
ARM64  | Odroid-C2      |   328.1%  |   372.8%
ARM64  | Raspberry Pi 4 |  1283%    |  1281%
ARM64C | Raspberry Pi 4 |  2353%    |  2811%
X64    | Intel Xeon     |   568.7%  |   584.9%

* InstantiateTypeArguments.Instantiate1000

Arch   | CPU            | Results 1  | Results 2
-------|----------------|------------------------
ARM    | Odroid-C2      |   743.7%   |  3821%
ARM    | Raspberry Pi 4 |   486.7%   |  3218%
ARM64  | Odroid-C2      |   584.7%   |  3222%
ARM64  | Raspberry Pi 4 |   430.7%   |  8172%
ARM64C | Raspberry Pi 4 |   491.4%   | 16699%
X64    | Intel Xeon     |   954.1%   |  5528%


Dart JIT:

* InstantiateTypeArguments.Instantiate100

Arch   | CPU            | Results 1 | Results 2
-------|----------------|-----------------------
ARM    | Raspberry Pi 4 |   315.7%  |   295.1%
ARM64  | Raspberry Pi 4 |  1070%    |  1058%
ARM64C | Raspberry Pi 4 |  1769%    |  2095%
X64    | Intel Xeon     |   507.4%  |   496.2%

* InstantiateTypeArguments.Instantiate1000

Arch   | CPU            | Results 1 | Results 2
-------|----------------|-----------------------
ARM    | Raspberry Pi 4 |   565.2%  |  2550%
ARM64  | Raspberry Pi 4 |   406.8%  |  7375%
ARM64C | Raspberry Pi 4 |   379.6%  | 12996%
X64    | Intel Xeon     |   807.9%  |  4459%

During work on this change, an issue was found where cache lookups
in the stub on ARM64C always failed and went to runtime, even with
the old linear-only caches, hence the much larger performance gains
in those rows above.

TEST=vm/cc/TypeArguments_Cache_{Some,Many}Instantiations

Fixes: https://github.com/dart-lang/sdk/issues/48344
Change-Id: I3d29566ba0582502954c9fc59626ceb8fd40317a
Cq-Include-Trybots: luci.dart.try:vm-kernel-precomp-dwarf-linux-product-x64-try,vm-kernel-precomp-linux-product-x64-try,vm-kernel-precomp-linux-release-x64-try,vm-kernel-precomp-nnbd-mac-release-arm64-try,vm-kernel-precomp-nnbd-linux-release-simarm_x64-try,vm-kernel-precomp-linux-release-simarm-try,vm-kernel-precomp-nnbd-linux-release-x64-try,vm-kernel-precomp-nnbd-linux-release-simarm64-try,vm-kernel-precomp-nnbd-linux-debug-simriscv64-try,vm-kernel-precomp-tsan-linux-release-x64-try,vm-kernel-tsan-linux-release-x64-try,vm-kernel-precomp-linux-debug-x64c-try,vm-kernel-nnbd-linux-debug-simriscv64-try,vm-kernel-linux-release-simarm-try,vm-kernel-linux-release-simarm64-try,vm-kernel-linux-release-ia32-try,vm-kernel-nnbd-linux-release-simarm-try,vm-kernel-nnbd-linux-release-simarm64-try,vm-kernel-nnbd-linux-release-ia32-try,vm-kernel-nnbd-mac-release-arm64-try,vm-kernel-linux-debug-x64c-try
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/270702
Reviewed-by: Ryan Macnak <rmacnak@google.com>
Commit-Queue: Tess Strickland <sstrickl@google.com>
Reviewed-by: Martin Kustermann <kustermann@google.com>
This commit is contained in:
Tess Strickland 2022-12-01 10:15:58 +00:00 committed by Commit Queue
parent 5994c04ef4
commit 49f998dc31
31 changed files with 915 additions and 85 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
runtime/vm/compiler/assembler/assembler_arm.cc
View file

@ -3457,6 +3457,26 @@ void Assembler::BranchOnMonomorphicCheckedEntryJIT(Label* label) {
}
}
void Assembler::CombineHashes(Register hash, Register other) {
// hash += other_hash
add(hash, hash, Operand(other));
// hash += hash << 10
add(hash, hash, Operand(hash, LSL, 10));
// hash ^= hash >> 6
eor(hash, hash, Operand(hash, LSR, 6));
}
void Assembler::FinalizeHash(Register hash, Register scratch) {
// hash += hash << 3;
add(hash, hash, Operand(hash, LSL, 3));
// hash ^= hash >> 11; // Logical shift, unsigned hash.
eor(hash, hash, Operand(hash, LSR, 11));
// hash += hash << 15;
adds(hash, hash, Operand(hash, LSL, 15));
// return (hash == 0) ? 1 : hash;
LoadImmediate(hash, 1, ZERO);
}
#ifndef PRODUCT
void Assembler::MaybeTraceAllocation(Register stats_addr_reg, Label* trace) {
ASSERT(stats_addr_reg != kNoRegister);

17
runtime/vm/compiler/assembler/assembler_arm.h
View file

@ -856,6 +856,17 @@ class Assembler : public AssemblerBase {
void SubRegisters(Register dest, Register src) {
sub(dest, dest, Operand(src));
}
void MulImmediate(Register reg,
int32_t imm,
OperandSize width = kFourBytes) override {
ASSERT(width == kFourBytes);
if (Utils::IsPowerOfTwo(imm)) {
LslImmediate(reg, Utils::ShiftForPowerOfTwo(imm));
} else {
LoadImmediate(TMP, imm);
mul(reg, reg, TMP);
}
}
void AndImmediate(Register rd, Register rs, int32_t imm, Condition cond = AL);
void AndImmediate(Register rd, int32_t imm, Condition cond = AL) {
AndImmediate(rd, rd, imm, cond);
@ -880,6 +891,9 @@ class Assembler : public AssemblerBase {
void LslImmediate(Register rd, int32_t shift) {
LslImmediate(rd, rd, shift);
}
void LslRegister(Register dst, Register shift) override {
Lsl(dst, dst, shift);
}
void LsrImmediate(Register rd, Register rn, int32_t shift) {
ASSERT((shift >= 0) && (shift < kBitsPerInt32));
Lsr(rd, rn, Operand(shift));
@ -1397,6 +1411,9 @@ class Assembler : public AssemblerBase {
void MonomorphicCheckedEntryAOT();
void BranchOnMonomorphicCheckedEntryJIT(Label* label);
void CombineHashes(Register dst, Register other) override;
void FinalizeHash(Register dst, Register scratch = TMP) override;
// The register into which the allocation tracing state table is loaded with
// LoadAllocationTracingStateAddress should be passed to MaybeTraceAllocation.
//

20
runtime/vm/compiler/assembler/assembler_arm64.cc
View file

@ -1910,6 +1910,26 @@ void Assembler::BranchOnMonomorphicCheckedEntryJIT(Label* label) {
}
}
void Assembler::CombineHashes(Register hash, Register other) {
// hash += other_hash
add(hash, hash, Operand(other), kFourBytes);
// hash += hash << 10
add(hash, hash, Operand(hash, LSL, 10), kFourBytes);
// hash ^= hash >> 6
eor(hash, hash, Operand(hash, LSR, 6), kFourBytes);
}
void Assembler::FinalizeHash(Register hash, Register scratch) {
// hash += hash << 3;
add(hash, hash, Operand(hash, LSL, 3), kFourBytes);
// hash ^= hash >> 11; // Logical shift, unsigned hash.
eor(hash, hash, Operand(hash, LSR, 11), kFourBytes);
// hash += hash << 15;
adds(hash, hash, Operand(hash, LSL, 15), kFourBytes);
// return (hash == 0) ? 1 : hash;
cinc(hash, hash, ZERO);
}
#ifndef PRODUCT
void Assembler::MaybeTraceAllocation(intptr_t cid,
Label* trace,

41
runtime/vm/compiler/assembler/assembler_arm64.h
View file

@ -594,18 +594,20 @@ class Assembler : public AssemblerBase {
void LoadAcquireCompressed(Register dst,
Register address,
int32_t offset = 0) override {
Register src = address;
if (offset != 0) {
AddImmediate(TMP2, address, offset);
ldar(dst, TMP2, kObjectBytes);
#if defined(USING_THREAD_SANITIZER)
TsanLoadAcquire(TMP2);
#endif
} else {
ldar(dst, address, kObjectBytes);
#if defined(USING_THREAD_SANITIZER)
TsanLoadAcquire(address);
#endif
src = TMP2;
}
#if !defined(DART_COMPRESSED_POINTERS)
ldar(dst, src);
#else
ldar(dst, src, kUnsignedFourBytes); // Zero-extension.
add(dst, dst, Operand(HEAP_BITS, LSL, 32));
#endif
#if defined(USING_THREAD_SANITIZER)
TsanLoadAcquire(src);
#endif
}
void StoreRelease(Register src,
@ -1697,6 +1699,9 @@ class Assembler : public AssemblerBase {
void LslImmediate(Register rd, int32_t shift, OperandSize sz = kEightBytes) {
LslImmediate(rd, rd, shift, sz);
}
void LslRegister(Register dst, Register shift) override {
lslv(dst, dst, shift);
}
void LsrImmediate(Register rd,
Register rn,
int shift,
@ -1834,6 +1839,21 @@ class Assembler : public AssemblerBase {
void SubRegisters(Register dest, Register src) {
sub(dest, dest, Operand(src));
}
void MulImmediate(Register reg,
int64_t imm,
OperandSize width = kEightBytes) override {
ASSERT(width == kFourBytes || width == kEightBytes);
if (Utils::IsPowerOfTwo(imm)) {
LslImmediate(reg, Utils::ShiftForPowerOfTwo(imm), width);
} else {
LoadImmediate(TMP, imm);
if (width == kFourBytes) {
mulw(reg, reg, TMP);
} else {
mul(reg, reg, TMP);
}
}
}
void AndImmediate(Register rd,
Register rn,
int64_t imm,
@ -2210,6 +2230,9 @@ class Assembler : public AssemblerBase {
void MonomorphicCheckedEntryAOT();
void BranchOnMonomorphicCheckedEntryJIT(Label* label);
void CombineHashes(Register hash, Register other) override;
void FinalizeHash(Register hash, Register scratch = TMP) override;
// If allocation tracing for |cid| is enabled, will jump to |trace| label,
// which will allocate in the runtime where tracing occurs.
void MaybeTraceAllocation(intptr_t cid,

6
runtime/vm/compiler/assembler/assembler_base.cc
View file

@ -37,7 +37,11 @@ void AssemblerBase::LoadFromSlot(Register dst,
return LoadFromOffset(dst, address, sz);
}
if (slot.is_compressed()) {
return LoadCompressedField(dst, address);
if (slot.ComputeCompileType().ToCid() == kSmiCid) {
return LoadCompressedSmi(dst, address);
} else {
return LoadCompressedField(dst, address);
}
}
return LoadField(dst, address);
}

21
runtime/vm/compiler/assembler/assembler_base.h
View file

@ -811,13 +811,34 @@ class AssemblerBase : public StackResource {
virtual void CompareImmediate(Register reg,
target::word imm,
OperandSize width = kWordBytes) = 0;
virtual void LsrImmediate(Register dst, int32_t shift) = 0;
virtual void MulImmediate(Register dst,
target::word imm,
OperandSize = kWordBytes) = 0;
// If src2 == kNoRegister, dst = dst & src1, otherwise dst = src1 & src2.
virtual void AndRegisters(Register dst,
Register src1,
Register src2 = kNoRegister) = 0;
// dst = dst << shift. On some architectures, we must use a specific register
// for the shift, so either the shift register must be that specific register
// or the architecture must define a TMP register, which is clobbered.
virtual void LslRegister(Register dst, Register shift) = 0;
// Performs CombineHashes from runtime/vm/hash.h on the hashes contained in
// dst and other. Puts the result in dst. Clobbers other.
//
// Note: Only uses the lower 32 bits of the hashes and returns a 32 bit hash.
virtual void CombineHashes(Register dst, Register other) = 0;
// Performs FinalizeHash from runtime/vm/hash.h on the hash contained in
// dst. May clobber scratch if provided, otherwise may clobber TMP.
//
// Note: Only uses the lower 32 bits of the hash and returns a 32 bit hash.
virtual void FinalizeHash(Register hash, Register scratch = TMP) = 0;
void LoadTypeClassId(Register dst, Register src) {
#if !defined(TARGET_ARCH_IA32)
EnsureHasClassIdInDEBUG(kTypeCid, src, TMP);

34
runtime/vm/compiler/assembler/assembler_ia32.cc
View file

@ -2383,6 +2383,40 @@ void Assembler::BranchOnMonomorphicCheckedEntryJIT(Label* label) {
}
}
void Assembler::CombineHashes(Register dst, Register other) {
// hash += other_hash
addl(dst, other);
// hash += hash << 10
movl(other, dst);
shll(other, Immediate(10));
addl(dst, other);
// hash ^= hash >> 6
movl(other, dst);
shrl(other, Immediate(6));
xorl(dst, other);
}
void Assembler::FinalizeHash(Register dst, Register scratch) {
ASSERT(scratch != kNoRegister);
// hash += hash << 3;
movl(scratch, dst);
shll(scratch, Immediate(3));
addl(dst, scratch);
// hash ^= hash >> 11; // Logical shift, unsigned hash.
movl(scratch, dst);
shrl(scratch, Immediate(11));
xorl(dst, scratch);
// hash += hash << 15;
movl(scratch, dst);
shll(scratch, Immediate(15));
addl(dst, scratch);
// return (hash == 0) ? 1 : hash;
Label done;
j(NOT_ZERO, &done, kNearJump);
incl(dst);
Bind(&done);
}
void Assembler::EnterFullSafepoint(Register scratch) {
// We generate the same number of instructions whether or not the slow-path is
// forced. This simplifies GenerateJitCallbackTrampolines.

20
runtime/vm/compiler/assembler/assembler_ia32.h
View file

@ -759,8 +759,17 @@ class Assembler : public AssemblerBase {
}
void SubImmediate(Register reg, const Immediate& imm);
void SubRegisters(Register dest, Register src) {
subl(dest, src);
void SubRegisters(Register dest, Register src) { subl(dest, src); }
void MulImmediate(Register reg,
int32_t imm,
OperandSize width = kFourBytes) override {
ASSERT(width == kFourBytes);
if (Utils::IsPowerOfTwo(imm)) {
const intptr_t shift = Utils::ShiftForPowerOfTwo(imm);
shll(reg, Immediate(shift));
} else {
imull(reg, Immediate(imm));
}
}
void AndImmediate(Register dst, int32_t value) {
andl(dst, Immediate(value));
@ -774,6 +783,10 @@ class Assembler : public AssemblerBase {
void LslImmediate(Register dst, int32_t shift) {
shll(dst, Immediate(shift));
}
void LslRegister(Register dst, Register shift) override {
ASSERT_EQUAL(shift, ECX); // IA32 does not have a TMP.
shll(dst, shift);
}
void LsrImmediate(Register dst, int32_t shift) override {
shrl(dst, Immediate(shift));
}
@ -925,6 +938,9 @@ class Assembler : public AssemblerBase {
void MonomorphicCheckedEntryAOT();
void BranchOnMonomorphicCheckedEntryJIT(Label* label);
void CombineHashes(Register dst, Register other) override;
void FinalizeHash(Register dst, Register scratch = kNoRegister) override;
// In debug mode, this generates code to check that:
// FP + kExitLinkSlotFromEntryFp == SP
// or triggers breakpoint otherwise.

84
runtime/vm/compiler/assembler/assembler_riscv.cc
View file

@ -2975,6 +2975,40 @@ void Assembler::AddImmediate(Register rd,
add(rd, rs1, TMP2);
}
}
void Assembler::MulImmediate(Register rd,
Register rs1,
intx_t imm,
OperandSize sz) {
if (Utils::IsPowerOfTwo(imm)) {
const intx_t shift = Utils::ShiftForPowerOfTwo(imm);
#if XLEN >= 64
ASSERT(sz == kFourBytes || sz == kEightBytes);
if (sz == kFourBytes) {
slliw(rd, rs1, shift);
} else {
slli(rd, rs1, shift);
}
#else
ASSERT(sz == kFourBytes);
slli(rd, rs1, shift);
#endif
} else {
LoadImmediate(TMP, imm);
#if XLEN >= 64
ASSERT(sz == kFourBytes || sz == kEightBytes);
if (sz == kFourBytes) {
mulw(rd, rs1, TMP);
} else {
mul(rd, rs1, TMP);
}
#else
ASSERT(sz == kFourBytes);
mul(rd, rs1, TMP);
#endif
}
}
void Assembler::AndImmediate(Register rd,
Register rs1,
intx_t imm,
@ -4176,6 +4210,56 @@ void Assembler::BranchOnMonomorphicCheckedEntryJIT(Label* label) {
}
}
void Assembler::CombineHashes(Register hash, Register other) {
#if XLEN >= 64
// hash += other_hash
addw(hash, hash, other);
// hash += hash << 10
slliw(other, hash, 10);
addw(hash, hash, other);
// hash ^= hash >> 6
srliw(other, hash, 6);
xor_(hash, hash, other);
#else
// hash += other_hash
add(hash, hash, other);
// hash += hash << 10
slli(other, hash, 10);
add(hash, hash, other);
// hash ^= hash >> 6
srli(other, hash, 6);
xor_(hash, hash, other);
#endif
}
void Assembler::FinalizeHash(Register hash, Register scratch) {
ASSERT(scratch != kNoRegister);
#if XLEN >= 64
// hash += hash << 3;
slliw(scratch, hash, 3);
addw(hash, hash, scratch);
// hash ^= hash >> 11; // Logical shift, unsigned hash.
srliw(scratch, hash, 11);
xor_(hash, hash, scratch);
// hash += hash << 15;
slliw(scratch, hash, 15);
addw(hash, hash, scratch);
#else
// hash += hash << 3;
slli(scratch, hash, 3);
add(hash, hash, scratch);
// hash ^= hash >> 11; // Logical shift, unsigned hash.
srli(scratch, hash, 11);
xor_(hash, hash, scratch);
// hash += hash << 15;
slli(scratch, hash, 15);
add(hash, hash, scratch);
#endif
// return (hash == 0) ? 1 : hash;
seqz(scratch, hash);
add(hash, hash, scratch);
}
#ifndef PRODUCT
void Assembler::MaybeTraceAllocation(intptr_t cid,
Label* trace,

17
runtime/vm/compiler/assembler/assembler_riscv.h
View file

@ -995,9 +995,12 @@ class Assembler : public MicroAssembler {
void AddImmediate(Register dest, intx_t imm) {
AddImmediate(dest, dest, imm);
}
void AddRegisters(Register dest, Register src) {
add(dest, dest, src);
void MulImmediate(Register dest,
intx_t imm,
OperandSize width = kWordBytes) override {
MulImmediate(dest, dest, imm, width);
}
void AddRegisters(Register dest, Register src) { add(dest, dest, src); }
void AddScaled(Register dest,
Register src,
ScaleFactor scale,
@ -1018,6 +1021,10 @@ class Assembler : public MicroAssembler {
Register rn,
intx_t imm,
OperandSize sz = kWordBytes);
void MulImmediate(Register dest,
Register rn,
intx_t imm,
OperandSize width = kWordBytes);
void AndImmediate(Register rd,
Register rn,
intx_t imm,
@ -1048,6 +1055,9 @@ class Assembler : public MicroAssembler {
void LslImmediate(Register rd, int32_t shift) {
slli(rd, rd, shift);
}
void LslRegister(Register dst, Register shift) override {
sll(dst, dst, shift);
}
void LsrImmediate(Register rd, int32_t shift) override {
srli(rd, rd, shift);
}
@ -1390,6 +1400,9 @@ class Assembler : public MicroAssembler {
void MonomorphicCheckedEntryAOT();
void BranchOnMonomorphicCheckedEntryJIT(Label* label);
void CombineHashes(Register dst, Register other) override;
void FinalizeHash(Register dst, Register scratch = TMP) override;
// If allocation tracing for |cid| is enabled, will jump to |trace| label,
// which will allocate in the runtime where tracing occurs.
void MaybeTraceAllocation(intptr_t cid,

72
runtime/vm/compiler/assembler/assembler_test.cc
View file

@ -4,7 +4,9 @@
#include "vm/compiler/assembler/assembler.h"
#include "vm/globals.h"
#include "vm/hash.h"
#include "vm/os.h"
#include "vm/random.h"
#include "vm/simulator.h"
#include "vm/unit_test.h"
#include "vm/virtual_memory.h"
@ -64,4 +66,74 @@ ASSEMBLER_TEST_RUN(StoreIntoObject, test) {
EXPECT(!thread->StoreBufferContains(grow_new_array.ptr()));
}
namespace compiler {
#define __ assembler->
ASSEMBLER_TEST_GENERATE(InstantiateTypeArgumentsHashKeys, assembler) {
#if defined(TARGET_ARCH_IA32)
const Register kArg1Reg = EAX;
const Register kArg2Reg = ECX;
__ movl(kArg1Reg, Address(ESP, 2 * target::kWordSize));
__ movl(kArg2Reg, Address(ESP, 1 * target::kWordSize));
#else
const Register kArg1Reg = CallingConventions::ArgumentRegisters[0];
const Register kArg2Reg = CallingConventions::ArgumentRegisters[1];
#endif
__ CombineHashes(kArg1Reg, kArg2Reg);
__ FinalizeHash(kArg1Reg, kArg2Reg);
__ MoveRegister(CallingConventions::kReturnReg, kArg1Reg);
__ Ret();
}
#undef __
} // namespace compiler
ASSEMBLER_TEST_RUN(InstantiateTypeArgumentsHashKeys, test) {
typedef uint32_t (*HashKeysCode)(uword hash, uword other) DART_UNUSED;
auto hash_test = [&](const Expect& expect, uword hash1, uword hash2) {
const uint32_t expected = FinalizeHash(CombineHashes(hash1, hash2));
const uint32_t got = EXECUTE_TEST_CODE_UWORD_UWORD_UINT32(
HashKeysCode, test->entry(), hash1, hash2);
if (got == expected) return;
TextBuffer buffer(128);
buffer.Printf("For hash1 = %" Pu " and hash2 = %" Pu
": expected result %u, got result %u",
hash1, hash2, expected, got);
expect.Fail("%s", buffer.buffer());
};
#define HASH_TEST(hash1, hash2) \
hash_test(Expect(__FILE__, __LINE__), hash1, hash2)
const intptr_t kNumRandomTests = 500;
Random random;
// First, fixed and random 32 bit tests for all architectures.
HASH_TEST(1, 1);
HASH_TEST(10, 20);
HASH_TEST(20, 10);
HASH_TEST(kMaxUint16, kMaxUint32 - kMaxUint16);
HASH_TEST(kMaxUint32 - kMaxUint16, kMaxUint16);
for (intptr_t i = 0; i < kNumRandomTests; i++) {
const uword hash1 = random.NextUInt32();
const uword hash2 = random.NextUInt32();
HASH_TEST(hash1, hash2);
}
#if defined(TARGET_ARCH_IS_64_BIT)
// Now 64-bit tests on 64-bit architectures.
HASH_TEST(kMaxUint16, kMaxUint64 - kMaxUint16);
HASH_TEST(kMaxUint64 - kMaxUint16, kMaxUint16);
for (intptr_t i = 0; i < kNumRandomTests; i++) {
const uword hash1 = random.NextUInt64();
const uword hash2 = random.NextUInt64();
HASH_TEST(hash1, hash2);
}
#endif
#undef HASH_TEST
}
} // namespace dart

54
runtime/vm/compiler/assembler/assembler_x64.cc
View file

@ -766,6 +766,17 @@ void Assembler::AndRegisters(Register dst, Register src1, Register src2) {
}
}
void Assembler::LslRegister(Register dst, Register shift) {
if (shift != RCX) {
movq(TMP, RCX);
movq(RCX, shift);
shlq(dst == RCX ? TMP : dst, RCX);
movq(RCX, TMP);
} else {
shlq(dst, shift);
}
}
void Assembler::OrImmediate(Register dst, const Immediate& imm) {
if (imm.is_int32()) {
orq(dst, imm);
@ -850,7 +861,14 @@ void Assembler::MulImmediate(Register reg,
const Immediate& imm,
OperandSize width) {
ASSERT(width == kFourBytes || width == kEightBytes);
if (imm.is_int32()) {
if (Utils::IsPowerOfTwo(imm.value())) {
const intptr_t shift = Utils::ShiftForPowerOfTwo(imm.value());
if (width == kFourBytes) {
shll(reg, Immediate(shift));
} else {
shlq(reg, Immediate(shift));
}
} else if (imm.is_int32()) {
if (width == kFourBytes) {
imull(reg, imm);
} else {
@ -2173,6 +2191,40 @@ void Assembler::BranchOnMonomorphicCheckedEntryJIT(Label* label) {
}
}
void Assembler::CombineHashes(Register dst, Register other) {
// hash += other_hash
addl(dst, other);
// hash += hash << 10
movl(other, dst);
shll(other, Immediate(10));
addl(dst, other);
// hash ^= hash >> 6
movl(other, dst);
shrl(other, Immediate(6));
xorl(dst, other);
}
void Assembler::FinalizeHash(Register dst, Register scratch) {
ASSERT(scratch != kNoRegister);
// hash += hash << 3;
movl(scratch, dst);
shll(scratch, Immediate(3));
addl(dst, scratch);
// hash ^= hash >> 11; // Logical shift, unsigned hash.
movl(scratch, dst);
shrl(scratch, Immediate(11));
xorl(dst, scratch);
// hash += hash << 15;
movl(scratch, dst);
shll(scratch, Immediate(15));
addl(dst, scratch);
// return (hash == 0) ? 1 : hash;
Label done;
j(NOT_ZERO, &done, kNearJump);
incl(dst);
Bind(&done);
}
#ifndef PRODUCT
void Assembler::MaybeTraceAllocation(intptr_t cid,
Label* trace,

9
runtime/vm/compiler/assembler/assembler_x64.h
View file

@ -597,6 +597,7 @@ class Assembler : public AssemblerBase {
void LslImmediate(Register dst, int32_t shift) {
shlq(dst, Immediate(shift));
}
void LslRegister(Register dst, Register shift) override;
void LsrImmediate(Register dst, int32_t shift) override {
shrq(dst, Immediate(shift));
}
@ -659,6 +660,11 @@ class Assembler : public AssemblerBase {
void MulImmediate(Register reg,
const Immediate& imm,
OperandSize width = kEightBytes);
void MulImmediate(Register reg,
int64_t imm,
OperandSize width = kEightBytes) override {
MulImmediate(reg, Immediate(imm), width);
}
void shll(Register reg, const Immediate& imm);
void shll(Register operand, Register shifter);
@ -1304,6 +1310,9 @@ class Assembler : public AssemblerBase {
void MonomorphicCheckedEntryAOT();
void BranchOnMonomorphicCheckedEntryJIT(Label* label);
void CombineHashes(Register dst, Register other) override;
void FinalizeHash(Register dst, Register scratch = TMP) override;
// If allocation tracing for |cid| is enabled, will jump to |trace| label,
// which will allocate in the runtime where tracing occurs.
void MaybeTraceAllocation(intptr_t cid,

4
runtime/vm/compiler/backend/locations.h
View file

@ -704,6 +704,10 @@ class RegisterSet : public ValueObject {
intptr_t CpuRegisterCount() const { return RegisterCount(cpu_registers()); }
intptr_t FpuRegisterCount() const { return RegisterCount(fpu_registers()); }
bool IsEmpty() const {
return CpuRegisterCount() == 0 && FpuRegisterCount() == 0;
}
static intptr_t RegisterCount(intptr_t registers);
static bool Contains(uintptr_t register_set, intptr_t reg) {
return (register_set & (static_cast<uintptr_t>(1) << reg)) != 0;

4
runtime/vm/compiler/backend/range_analysis.cc
View file

@ -2766,6 +2766,10 @@ void LoadFieldInstr::InferRange(RangeAnalysis* analysis, Range* range) {
*range = Range(RangeBoundary::FromConstant(0), RangeBoundary::MaxSmi());
break;
case Slot::Kind::kTypeArguments_hash:
*range = Range(RangeBoundary::MinSmi(), RangeBoundary::MaxSmi());
break;
case Slot::Kind::kTypeArguments_length:
*range = Range(RangeBoundary::FromConstant(0),
RangeBoundary::FromConstant(

1
runtime/vm/compiler/backend/slot.cc
View file

@ -160,6 +160,7 @@ bool Slot::IsImmutableLengthSlot() const {
case Slot::Kind::kArrayElement:
case Slot::Kind::kInstance_native_fields_array:
case Slot::Kind::kTypeArguments:
case Slot::Kind::kTypeArguments_hash:
case Slot::Kind::kTypedDataView_offset_in_bytes:
case Slot::Kind::kTypedDataView_typed_data:
case Slot::Kind::kGrowableObjectArray_data:

1
runtime/vm/compiler/backend/slot.h
View file

@ -127,6 +127,7 @@ class ParsedFunction;
V(ArgumentsDescriptor, UntaggedArray, size, Smi, FINAL) \
V(Record, UntaggedRecord, field_names, ImmutableArray, FINAL) \
V(Record, UntaggedRecord, num_fields, Smi, FINAL) \
V(TypeArguments, UntaggedTypeArguments, hash, Smi, VAR) \
V(TypeArguments, UntaggedTypeArguments, length, Smi, FINAL) \
V(TypeParameters, UntaggedTypeParameters, names, Array, FINAL) \
V(TypeParameter, UntaggedTypeParameter, bound, Dynamic, FINAL) \

1
runtime/vm/compiler/runtime_api.h
View file

@ -1544,6 +1544,7 @@ class TypeParameters : public AllStatic {
class TypeArguments : public AllStatic {
public:
static word hash_offset();
static word instantiations_offset();
static word length_offset();
static word nullability_offset();

44
runtime/vm/compiler/runtime_offsets_extracted.h
View file

@ -540,6 +540,7 @@ static constexpr dart::compiler::target::word
static constexpr dart::compiler::target::word
TypeParameter_parameterized_class_id_offset = 24;
static constexpr dart::compiler::target::word TypeParameter_index_offset = 29;
static constexpr dart::compiler::target::word TypeArguments_hash_offset = 12;
static constexpr dart::compiler::target::word
TypeArguments_instantiations_offset = 4;
static constexpr dart::compiler::target::word TypeArguments_length_offset = 8;
@ -1209,6 +1210,7 @@ static constexpr dart::compiler::target::word
static constexpr dart::compiler::target::word
TypeParameter_parameterized_class_id_offset = 48;
static constexpr dart::compiler::target::word TypeParameter_index_offset = 53;
static constexpr dart::compiler::target::word TypeArguments_hash_offset = 24;
static constexpr dart::compiler::target::word
TypeArguments_instantiations_offset = 8;
static constexpr dart::compiler::target::word TypeArguments_length_offset = 16;
@ -1870,6 +1872,7 @@ static constexpr dart::compiler::target::word
static constexpr dart::compiler::target::word
TypeParameter_parameterized_class_id_offset = 24;
static constexpr dart::compiler::target::word TypeParameter_index_offset = 29;
static constexpr dart::compiler::target::word TypeArguments_hash_offset = 12;
static constexpr dart::compiler::target::word
TypeArguments_instantiations_offset = 4;
static constexpr dart::compiler::target::word TypeArguments_length_offset = 8;
@ -2539,6 +2542,7 @@ static constexpr dart::compiler::target::word
static constexpr dart::compiler::target::word
TypeParameter_parameterized_class_id_offset = 48;
static constexpr dart::compiler::target::word TypeParameter_index_offset = 53;
static constexpr dart::compiler::target::word TypeArguments_hash_offset = 24;
static constexpr dart::compiler::target::word
TypeArguments_instantiations_offset = 8;
static constexpr dart::compiler::target::word TypeArguments_length_offset = 16;
@ -3207,6 +3211,7 @@ static constexpr dart::compiler::target::word
static constexpr dart::compiler::target::word
TypeParameter_parameterized_class_id_offset = 32;
static constexpr dart::compiler::target::word TypeParameter_index_offset = 37;
static constexpr dart::compiler::target::word TypeArguments_hash_offset = 16;
static constexpr dart::compiler::target::word
TypeArguments_instantiations_offset = 8;
static constexpr dart::compiler::target::word TypeArguments_length_offset = 12;
@ -3874,6 +3879,7 @@ static constexpr dart::compiler::target::word
static constexpr dart::compiler::target::word
TypeParameter_parameterized_class_id_offset = 32;
static constexpr dart::compiler::target::word TypeParameter_index_offset = 37;
static constexpr dart::compiler::target::word TypeArguments_hash_offset = 16;
static constexpr dart::compiler::target::word
TypeArguments_instantiations_offset = 8;
static constexpr dart::compiler::target::word TypeArguments_length_offset = 12;
@ -4537,6 +4543,7 @@ static constexpr dart::compiler::target::word
static constexpr dart::compiler::target::word
TypeParameter_parameterized_class_id_offset = 24;
static constexpr dart::compiler::target::word TypeParameter_index_offset = 29;
static constexpr dart::compiler::target::word TypeArguments_hash_offset = 12;
static constexpr dart::compiler::target::word
TypeArguments_instantiations_offset = 4;
static constexpr dart::compiler::target::word TypeArguments_length_offset = 8;
@ -5208,6 +5215,7 @@ static constexpr dart::compiler::target::word
static constexpr dart::compiler::target::word
TypeParameter_parameterized_class_id_offset = 48;
static constexpr dart::compiler::target::word TypeParameter_index_offset = 53;
static constexpr dart::compiler::target::word TypeArguments_hash_offset = 24;
static constexpr dart::compiler::target::word
TypeArguments_instantiations_offset = 8;
static constexpr dart::compiler::target::word TypeArguments_length_offset = 16;
@ -5864,6 +5872,7 @@ static constexpr dart::compiler::target::word
static constexpr dart::compiler::target::word
TypeParameter_parameterized_class_id_offset = 24;
static constexpr dart::compiler::target::word TypeParameter_index_offset = 29;
static constexpr dart::compiler::target::word TypeArguments_hash_offset = 12;
static constexpr dart::compiler::target::word
TypeArguments_instantiations_offset = 4;
static constexpr dart::compiler::target::word TypeArguments_length_offset = 8;
@ -6525,6 +6534,7 @@ static constexpr dart::compiler::target::word
static constexpr dart::compiler::target::word
TypeParameter_parameterized_class_id_offset = 48;
static constexpr dart::compiler::target::word TypeParameter_index_offset = 53;
static constexpr dart::compiler::target::word TypeArguments_hash_offset = 24;
static constexpr dart::compiler::target::word
TypeArguments_instantiations_offset = 8;
static constexpr dart::compiler::target::word TypeArguments_length_offset = 16;
@ -7178,6 +7188,7 @@ static constexpr dart::compiler::target::word
static constexpr dart::compiler::target::word
TypeParameter_parameterized_class_id_offset = 24;
static constexpr dart::compiler::target::word TypeParameter_index_offset = 29;
static constexpr dart::compiler::target::word TypeArguments_hash_offset = 12;
static constexpr dart::compiler::target::word
TypeArguments_instantiations_offset = 4;
static constexpr dart::compiler::target::word TypeArguments_length_offset = 8;
@ -7839,6 +7850,7 @@ static constexpr dart::compiler::target::word
static constexpr dart::compiler::target::word
TypeParameter_parameterized_class_id_offset = 48;
static constexpr dart::compiler::target::word TypeParameter_index_offset = 53;
static constexpr dart::compiler::target::word TypeArguments_hash_offset = 24;
static constexpr dart::compiler::target::word
TypeArguments_instantiations_offset = 8;
static constexpr dart::compiler::target::word TypeArguments_length_offset = 16;
@ -8499,6 +8511,7 @@ static constexpr dart::compiler::target::word
static constexpr dart::compiler::target::word
TypeParameter_parameterized_class_id_offset = 32;
static constexpr dart::compiler::target::word TypeParameter_index_offset = 37;
static constexpr dart::compiler::target::word TypeArguments_hash_offset = 16;
static constexpr dart::compiler::target::word
TypeArguments_instantiations_offset = 8;
static constexpr dart::compiler::target::word TypeArguments_length_offset = 12;
@ -9158,6 +9171,7 @@ static constexpr dart::compiler::target::word
static constexpr dart::compiler::target::word
TypeParameter_parameterized_class_id_offset = 32;
static constexpr dart::compiler::target::word TypeParameter_index_offset = 37;
static constexpr dart::compiler::target::word TypeArguments_hash_offset = 16;
static constexpr dart::compiler::target::word
TypeArguments_instantiations_offset = 8;
static constexpr dart::compiler::target::word TypeArguments_length_offset = 12;
@ -9813,6 +9827,7 @@ static constexpr dart::compiler::target::word
static constexpr dart::compiler::target::word
TypeParameter_parameterized_class_id_offset = 24;
static constexpr dart::compiler::target::word TypeParameter_index_offset = 29;
static constexpr dart::compiler::target::word TypeArguments_hash_offset = 12;
static constexpr dart::compiler::target::word
TypeArguments_instantiations_offset = 4;
static constexpr dart::compiler::target::word TypeArguments_length_offset = 8;
@ -10476,6 +10491,7 @@ static constexpr dart::compiler::target::word
static constexpr dart::compiler::target::word
TypeParameter_parameterized_class_id_offset = 48;
static constexpr dart::compiler::target::word TypeParameter_index_offset = 53;
static constexpr dart::compiler::target::word TypeArguments_hash_offset = 24;
static constexpr dart::compiler::target::word
TypeArguments_instantiations_offset = 8;
static constexpr dart::compiler::target::word TypeArguments_length_offset = 16;
@ -11192,6 +11208,8 @@ static constexpr dart::compiler::target::word
AOT_TypeParameter_parameterized_class_id_offset = 24;
static constexpr dart::compiler::target::word AOT_TypeParameter_index_offset =
29;
static constexpr dart::compiler::target::word AOT_TypeArguments_hash_offset =
12;
static constexpr dart::compiler::target::word
AOT_TypeArguments_instantiations_offset = 4;
static constexpr dart::compiler::target::word AOT_TypeArguments_length_offset =
@ -11930,6 +11948,8 @@ static constexpr dart::compiler::target::word
AOT_TypeParameter_parameterized_class_id_offset = 48;
static constexpr dart::compiler::target::word AOT_TypeParameter_index_offset =
53;
static constexpr dart::compiler::target::word AOT_TypeArguments_hash_offset =
24;
static constexpr dart::compiler::target::word
AOT_TypeArguments_instantiations_offset = 8;
static constexpr dart::compiler::target::word AOT_TypeArguments_length_offset =
@ -12673,6 +12693,8 @@ static constexpr dart::compiler::target::word
AOT_TypeParameter_parameterized_class_id_offset = 48;
static constexpr dart::compiler::target::word AOT_TypeParameter_index_offset =
53;
static constexpr dart::compiler::target::word AOT_TypeArguments_hash_offset =
24;
static constexpr dart::compiler::target::word
AOT_TypeArguments_instantiations_offset = 8;
static constexpr dart::compiler::target::word AOT_TypeArguments_length_offset =
@ -13414,6 +13436,8 @@ static constexpr dart::compiler::target::word
AOT_TypeParameter_parameterized_class_id_offset = 32;
static constexpr dart::compiler::target::word AOT_TypeParameter_index_offset =
37;
static constexpr dart::compiler::target::word AOT_TypeArguments_hash_offset =
16;
static constexpr dart::compiler::target::word
AOT_TypeArguments_instantiations_offset = 8;
static constexpr dart::compiler::target::word AOT_TypeArguments_length_offset =
@ -14154,6 +14178,8 @@ static constexpr dart::compiler::target::word
AOT_TypeParameter_parameterized_class_id_offset = 32;
static constexpr dart::compiler::target::word AOT_TypeParameter_index_offset =
37;
static constexpr dart::compiler::target::word AOT_TypeArguments_hash_offset =
16;
static constexpr dart::compiler::target::word
AOT_TypeArguments_instantiations_offset = 8;
static constexpr dart::compiler::target::word AOT_TypeArguments_length_offset =
@ -14892,6 +14918,8 @@ static constexpr dart::compiler::target::word
AOT_TypeParameter_parameterized_class_id_offset = 24;
static constexpr dart::compiler::target::word AOT_TypeParameter_index_offset =
29;
static constexpr dart::compiler::target::word AOT_TypeArguments_hash_offset =
12;
static constexpr dart::compiler::target::word
AOT_TypeArguments_instantiations_offset = 4;
static constexpr dart::compiler::target::word AOT_TypeArguments_length_offset =
@ -15632,6 +15660,8 @@ static constexpr dart::compiler::target::word
AOT_TypeParameter_parameterized_class_id_offset = 48;
static constexpr dart::compiler::target::word AOT_TypeParameter_index_offset =
53;
static constexpr dart::compiler::target::word AOT_TypeArguments_hash_offset =
24;
static constexpr dart::compiler::target::word
AOT_TypeArguments_instantiations_offset = 8;
static constexpr dart::compiler::target::word AOT_TypeArguments_length_offset =
@ -16363,6 +16393,8 @@ static constexpr dart::compiler::target::word
AOT_TypeParameter_parameterized_class_id_offset = 24;
static constexpr dart::compiler::target::word AOT_TypeParameter_index_offset =
29;
static constexpr dart::compiler::target::word AOT_TypeArguments_hash_offset =
12;
static constexpr dart::compiler::target::word
AOT_TypeArguments_instantiations_offset = 4;
static constexpr dart::compiler::target::word AOT_TypeArguments_length_offset =
@ -17092,6 +17124,8 @@ static constexpr dart::compiler::target::word
AOT_TypeParameter_parameterized_class_id_offset = 48;
static constexpr dart::compiler::target::word AOT_TypeParameter_index_offset =
53;
static constexpr dart::compiler::target::word AOT_TypeArguments_hash_offset =
24;
static constexpr dart::compiler::target::word
AOT_TypeArguments_instantiations_offset = 8;
static constexpr dart::compiler::target::word AOT_TypeArguments_length_offset =
@ -17826,6 +17860,8 @@ static constexpr dart::compiler::target::word
AOT_TypeParameter_parameterized_class_id_offset = 48;
static constexpr dart::compiler::target::word AOT_TypeParameter_index_offset =
53;
static constexpr dart::compiler::target::word AOT_TypeArguments_hash_offset =
24;
static constexpr dart::compiler::target::word
AOT_TypeArguments_instantiations_offset = 8;
static constexpr dart::compiler::target::word AOT_TypeArguments_length_offset =
@ -18558,6 +18594,8 @@ static constexpr dart::compiler::target::word
AOT_TypeParameter_parameterized_class_id_offset = 32;
static constexpr dart::compiler::target::word AOT_TypeParameter_index_offset =
37;
static constexpr dart::compiler::target::word AOT_TypeArguments_hash_offset =
16;
static constexpr dart::compiler::target::word
AOT_TypeArguments_instantiations_offset = 8;
static constexpr dart::compiler::target::word AOT_TypeArguments_length_offset =
@ -19289,6 +19327,8 @@ static constexpr dart::compiler::target::word
AOT_TypeParameter_parameterized_class_id_offset = 32;
static constexpr dart::compiler::target::word AOT_TypeParameter_index_offset =
37;
static constexpr dart::compiler::target::word AOT_TypeArguments_hash_offset =
16;
static constexpr dart::compiler::target::word
AOT_TypeArguments_instantiations_offset = 8;
static constexpr dart::compiler::target::word AOT_TypeArguments_length_offset =
@ -20018,6 +20058,8 @@ static constexpr dart::compiler::target::word
AOT_TypeParameter_parameterized_class_id_offset = 24;
static constexpr dart::compiler::target::word AOT_TypeParameter_index_offset =
29;
static constexpr dart::compiler::target::word AOT_TypeArguments_hash_offset =
12;
static constexpr dart::compiler::target::word
AOT_TypeArguments_instantiations_offset = 4;
static constexpr dart::compiler::target::word AOT_TypeArguments_length_offset =
@ -20749,6 +20791,8 @@ static constexpr dart::compiler::target::word
AOT_TypeParameter_parameterized_class_id_offset = 48;
static constexpr dart::compiler::target::word AOT_TypeParameter_index_offset =
53;
static constexpr dart::compiler::target::word AOT_TypeArguments_hash_offset =
24;
static constexpr dart::compiler::target::word
AOT_TypeArguments_instantiations_offset = 8;
static constexpr dart::compiler::target::word AOT_TypeArguments_length_offset =

1
runtime/vm/compiler/runtime_offsets_list.h
View file

@ -359,6 +359,7 @@
FIELD(FunctionType, type_parameters_offset) \
FIELD(TypeParameter, parameterized_class_id_offset) \
FIELD(TypeParameter, index_offset) \
FIELD(TypeArguments, hash_offset) \
FIELD(TypeArguments, instantiations_offset) \
FIELD(TypeArguments, length_offset) \
FIELD(TypeArguments, nullability_offset) \

190
runtime/vm/compiler/stub_code_compiler.cc
View file

@ -15,6 +15,7 @@
#include "vm/compiler/api/type_check_mode.h"
#include "vm/compiler/assembler/assembler.h"
#include "vm/compiler/backend/locations.h"
#include "vm/stack_frame.h"
#define __ assembler->
@ -282,64 +283,174 @@ void StubCodeCompiler::GenerateInstantiateTypeArgumentsStub(
// the runtime or until we retrieve the instantiated type arguments out of it
// to put in the result register, so we use the result register to store it.
const Register kEntryReg = InstantiationABI::kResultTypeArgumentsReg;
// The registers that need spilling prior to traversing a hash-based cache.
const RegisterSet saved_registers(InstantiateTAVInternalRegs::kSavedRegisters,
/*fpu_register_mask=*/0);
static_assert(((1 << InstantiationABI::kInstantiatorTypeArgumentsReg) &
InstantiateTAVInternalRegs::kSavedRegisters) == 0,
"Must handle possibility of inst tav reg being spilled");
static_assert(((1 << InstantiationABI::kFunctionTypeArgumentsReg) &
InstantiateTAVInternalRegs::kSavedRegisters) == 0,
"Must handle possibility of inst tav reg being spilled");
// Takes labels for the cache hit/miss cases (to allow for restoring spilled
// registers).
auto check_entry = [&](compiler::Label* found, compiler::Label* not_found) {
__ Comment("Check cache entry");
// Use load-acquire to get the entry.
static_assert(TypeArguments::Cache::kSentinelIndex ==
TypeArguments::Cache::kInstantiatorTypeArgsIndex,
"sentinel is not same index as instantiator type args");
__ LoadAcquireCompressed(InstantiationABI::kScratchReg, kEntryReg,
TypeArguments::Cache::kInstantiatorTypeArgsIndex *
target::kCompressedWordSize);
// Test for an unoccupied entry by checking for the Smi sentinel.
__ BranchIfSmi(InstantiationABI::kScratchReg, not_found);
// Otherwise it must be occupied and contain TypeArguments objects.
compiler::Label next;
__ CompareRegisters(InstantiationABI::kScratchReg,
InstantiationABI::kInstantiatorTypeArgumentsReg);
__ BranchIf(NOT_EQUAL, &next, compiler::Assembler::kNearJump);
__ LoadCompressed(
InstantiationABI::kScratchReg,
compiler::Address(kEntryReg,
TypeArguments::Cache::kFunctionTypeArgsIndex *
target::kCompressedWordSize));
__ CompareRegisters(InstantiationABI::kScratchReg,
InstantiationABI::kFunctionTypeArgumentsReg);
__ BranchIf(EQUAL, found);
__ Bind(&next);
};
// Lookup cache before calling runtime.
__ LoadCompressed(
InstantiationABI::kScratchReg,
compiler::FieldAddress(InstantiationABI::kUninstantiatedTypeArgumentsReg,
target::TypeArguments::instantiations_offset()));
// Both the linear and hash-based cache access loops assume kEntryReg is
// the address of the first cache entry, so set it before branching.
// Go ahead and load the backing array data address into kEntryReg.
__ LoadFieldAddressForOffset(kEntryReg, InstantiationABI::kScratchReg,
Array::data_offset());
__ AddImmediate(kEntryReg, TypeArguments::Cache::kHeaderSize *
target::kCompressedWordSize);
target::Array::data_offset());
compiler::Label linear_cache_loop, hash_cache_loop, found, call_runtime;
compiler::Label linear_cache_loop, hash_cache_search, cache_hit, call_runtime;
// There is a maximum size for linear caches that is smaller than the size of
// any hash-based cache, so we check the size of the backing array to
// There is a maximum size for linear caches that is smaller than the size
// of any hash-based cache, so we check the size of the backing array to
// determine if this is a linear or hash-based cache.
__ LoadFromSlot(InstantiationABI::kScratchReg, InstantiationABI::kScratchReg,
Slot::Array_length());
__ CompareImmediate(
InstantiationABI::kScratchReg,
target::ToRawSmi(TypeArguments::Cache::kMaxLinearCacheSize));
#if defined(TARGET_ARCH_IA32)
// We just don't have enough registers to do hash-based cache searching in a
// way that doesn't overly complicate the generation code, so just go to
// runtime.
__ BranchIf(GREATER, &call_runtime);
#else
__ BranchIf(GREATER, &hash_cache_search);
#endif
__ Comment("Check linear cache");
// Move kEntryReg to the start of the first entry.
__ AddImmediate(kEntryReg, TypeArguments::Cache::kHeaderSize *
target::kCompressedWordSize);
__ Bind(&linear_cache_loop);
// Use load-acquire to get the entry.
static_assert(TypeArguments::Cache::kSentinelIndex ==
TypeArguments::Cache::kInstantiatorTypeArgsIndex,
"sentinel is not same index as instantiator type args");
__ LoadAcquireCompressed(InstantiationABI::kScratchReg, kEntryReg,
TypeArguments::Cache::kInstantiatorTypeArgsIndex *
target::kCompressedWordSize);
// Must either be the sentinel (a Smi) or a TypeArguments object, so test for
// a Smi and go to the runtime if found.
__ BranchIfSmi(InstantiationABI::kScratchReg, &call_runtime,
compiler::Assembler::kNearJump);
// We have a TypeArguments object, so this is an array cache and we can
// safely access the other entries.
compiler::Label next;
__ CompareRegisters(InstantiationABI::kScratchReg,
InstantiationABI::kInstantiatorTypeArgumentsReg);
__ BranchIf(NOT_EQUAL, &next, compiler::Assembler::kNearJump);
__ LoadCompressed(
InstantiationABI::kScratchReg,
compiler::Address(kEntryReg,
TypeArguments::Cache::kFunctionTypeArgsIndex *
target::kCompressedWordSize));
__ CompareRegisters(InstantiationABI::kScratchReg,
InstantiationABI::kFunctionTypeArgumentsReg);
__ BranchIf(EQUAL, &found, compiler::Assembler::kNearJump);
__ Bind(&next);
check_entry(&cache_hit, &call_runtime);
__ AddImmediate(kEntryReg, TypeArguments::Cache::kEntrySize *
target::kCompressedWordSize);
__ Jump(&linear_cache_loop, compiler::Assembler::kNearJump);
#if !defined(TARGET_ARCH_IA32)
__ Bind(&hash_cache_search);
__ Comment("Check hash-based cache");
compiler::Label pop_before_success, pop_before_failure;
if (!saved_registers.IsEmpty()) {
__ Comment("Spills due to register pressure");
__ PushRegisters(saved_registers);
}
__ Comment("Calculate address of first entry");
__ AddImmediate(
InstantiateTAVInternalRegs::kEntryStartReg, kEntryReg,
TypeArguments::Cache::kHeaderSize * target::kCompressedWordSize);
__ Comment("Calculate probe mask");
__ LoadAcquireCompressed(
InstantiationABI::kScratchReg, kEntryReg,
TypeArguments::Cache::kMetadataIndex * target::kCompressedWordSize);
__ LsrImmediate(
InstantiationABI::kScratchReg,
TypeArguments::Cache::EntryCountLog2Bits::shift() + kSmiTagShift);
__ LoadImmediate(InstantiateTAVInternalRegs::kProbeMaskReg, 1);
__ LslRegister(InstantiateTAVInternalRegs::kProbeMaskReg,
InstantiationABI::kScratchReg);
__ AddImmediate(InstantiateTAVInternalRegs::kProbeMaskReg, -1);
// Can use kEntryReg as scratch now until we're entering the loop.
// Retrieve the hash from the TAV. If the retrieved hash is 0, jumps to
// not_found, otherwise falls through.
auto retrieve_hash = [&](Register dst, Register src) {
Label is_not_null, done;
__ CompareObject(src, NullObject());
__ BranchIf(NOT_EQUAL, &is_not_null, compiler::Assembler::kNearJump);
__ LoadImmediate(dst, TypeArguments::kAllDynamicHash);
__ Jump(&done, compiler::Assembler::kNearJump);
__ Bind(&is_not_null);
__ LoadFromSlot(dst, src, Slot::TypeArguments_hash());
__ SmiUntag(dst);
// If the retrieved hash is 0, then it hasn't been computed yet.
__ BranchIfZero(dst, &pop_before_failure);
__ Bind(&done);
};
__ Comment("Calculate initial probe from type argument vector hashes");
retrieve_hash(InstantiateTAVInternalRegs::kCurrentEntryIndexReg,
InstantiationABI::kInstantiatorTypeArgumentsReg);
retrieve_hash(InstantiationABI::kScratchReg,
InstantiationABI::kFunctionTypeArgumentsReg);
__ CombineHashes(InstantiateTAVInternalRegs::kCurrentEntryIndexReg,
InstantiationABI::kScratchReg);
__ FinalizeHash(InstantiateTAVInternalRegs::kCurrentEntryIndexReg,
InstantiationABI::kScratchReg);
// Use the probe mask to get a valid entry index.
__ AndRegisters(InstantiateTAVInternalRegs::kCurrentEntryIndexReg,
InstantiateTAVInternalRegs::kProbeMaskReg);
// Start off the probing distance at zero (will increment prior to use).
__ LoadImmediate(InstantiateTAVInternalRegs::kProbeDistanceReg, 0);
compiler::Label loop;
__ Bind(&loop);
__ Comment("Loop over hash cache entries");
// Convert the current entry index into the entry address.
__ MoveRegister(kEntryReg, InstantiateTAVInternalRegs::kCurrentEntryIndexReg);
__ MulImmediate(kEntryReg, TypeArguments::Cache::kEntrySize *
target::kCompressedWordSize);
__ AddRegisters(kEntryReg, InstantiateTAVInternalRegs::kEntryStartReg);
check_entry(&pop_before_success, &pop_before_failure);
// Increment the probing distance and then add it to the current entry
// index, then mask the result with the probe mask.
__ AddImmediate(InstantiateTAVInternalRegs::kProbeDistanceReg, 1);
__ AddRegisters(InstantiateTAVInternalRegs::kCurrentEntryIndexReg,
InstantiateTAVInternalRegs::kProbeDistanceReg);
__ AndRegisters(InstantiateTAVInternalRegs::kCurrentEntryIndexReg,
InstantiateTAVInternalRegs::kProbeMaskReg);
__ Jump(&loop);
__ Bind(&pop_before_failure);
if (!saved_registers.IsEmpty()) {
__ Comment("Restore spilled registers on cache miss");
__ PopRegisters(saved_registers);
}
#endif
// Instantiate non-null type arguments.
// A runtime call to instantiate the type arguments is required.
__ Bind(&call_runtime);
__ Comment("Cache miss");
__ EnterStubFrame();
#if !defined(DART_ASSEMBLER_HAS_NULL_REG)
__ PushObject(Object::null_object()); // Make room for the result.
@ -365,7 +476,16 @@ void StubCodeCompiler::GenerateInstantiateTypeArgumentsStub(
__ LeaveStubFrame();
__ Ret();
__ Bind(&found);
#if !defined(TARGET_ARCH_IA32)
__ Bind(&pop_before_success);
if (!saved_registers.IsEmpty()) {
__ Comment("Restore spilled registers on cache hit");
__ PopRegisters(saved_registers);
}
#endif
__ Bind(&cache_hit);
__ Comment("Cache hit");
__ LoadCompressed(
InstantiationABI::kResultTypeArgumentsReg,
compiler::Address(kEntryReg,

19
runtime/vm/constants_arm.h
View file

@ -352,6 +352,25 @@ struct InstantiationABI {
static const Register kScratchReg = R8;
};
// Registers in addition to those listed in InstantiationABI used inside the
// implementation of the InstantiateTypeArguments stubs.
struct InstantiateTAVInternalRegs {
// The set of registers that must be pushed/popped when probing a hash-based
// cache due to overlap with the registers in InstantiationABI.
static const intptr_t kSavedRegisters =
#if defined(DART_PRECOMPILER)
(1 << DISPATCH_TABLE_REG) |
#endif
(1 << InstantiationABI::kUninstantiatedTypeArgumentsReg);
// Additional registers used to probe hash-based caches.
static const Register kEntryStartReg = R9;
static const Register kProbeMaskReg = R4;
static const Register kProbeDistanceReg = DISPATCH_TABLE_REG;
static const Register kCurrentEntryIndexReg =
InstantiationABI::kUninstantiatedTypeArgumentsReg;
};
// Registers in addition to those listed in TypeTestABI used inside the
// implementation of type testing stubs that are _not_ preserved.
struct TTSInternalRegs {

14
runtime/vm/constants_arm64.h
View file

@ -181,6 +181,20 @@ struct InstantiationABI {
static const Register kScratchReg = R8;
};
// Registers in addition to those listed in InstantiationABI used inside the
// implementation of the InstantiateTypeArguments stubs.
struct InstantiateTAVInternalRegs {
// The set of registers that must be pushed/popped when probing a hash-based
// cache due to overlap with the registers in InstantiationABI.
static const intptr_t kSavedRegisters = 0;
// Additional registers used to probe hash-based caches.
static const Register kEntryStartReg = R9;
static const Register kProbeMaskReg = R7;
static const Register kProbeDistanceReg = R6;
static const Register kCurrentEntryIndexReg = R10;
};
// Registers in addition to those listed in TypeTestABI used inside the
// implementation of type testing stubs that are _not_ preserved.
struct TTSInternalRegs {

8
runtime/vm/constants_ia32.h
View file

@ -120,6 +120,14 @@ struct InstantiationABI {
static const Register kScratchReg = EDI; // On ia32 we don't use CODE_REG.
};
// Registers in addition to those listed in InstantiationABI used inside the
// implementation of the InstantiateTypeArguments stubs.
struct InstantiateTAVInternalRegs {
// On IA32, we don't do hash cache checks in the stub. We only define
// kSavedRegisters to avoid needing to #ifdef uses of it.
static const intptr_t kSavedRegisters = 0;
};
// Calling convention when calling SubtypeTestCacheStub.
// Although ia32 uses a stack-based calling convention, we keep the same
// 'TypeTestABI' name for symmetry with other architectures with a proper ABI.

14
runtime/vm/constants_riscv.h
View file

@ -190,6 +190,20 @@ struct InstantiationABI {
static constexpr Register kScratchReg = T4;
};
// Registers in addition to those listed in InstantiationABI used inside the
// implementation of the InstantiateTypeArguments stubs.
struct InstantiateTAVInternalRegs {
// The set of registers that must be pushed/popped when probing a hash-based
// cache due to overlap with the registers in InstantiationABI.
static const intptr_t kSavedRegisters = 0;
// Additional registers used to probe hash-based caches.
static const Register kEntryStartReg = S3;
static const Register kProbeMaskReg = S4;
static const Register kProbeDistanceReg = S5;
static const Register kCurrentEntryIndexReg = S6;
};
// Registers in addition to those listed in TypeTestABI used inside the
// implementation of type testing stubs that are _not_ preserved.
struct TTSInternalRegs {

18
runtime/vm/constants_x64.h
View file

@ -31,8 +31,8 @@ enum Register {
R8 = 8,
R9 = 9,
R10 = 10,
R11 = 11,
R12 = 12,
R11 = 11, // TMP
R12 = 12, // CODE_REG
R13 = 13,
R14 = 14, // THR
R15 = 15, // PP
@ -155,6 +155,20 @@ struct InstantiationABI {
static const Register kScratchReg = R9;
};
// Registers in addition to those listed in InstantiationABI used inside the
// implementation of the InstantiateTypeArguments stubs.
struct InstantiateTAVInternalRegs {
// The set of registers that must be pushed/popped when probing a hash-based
// cache due to overlap with the registers in InstantiationABI.
static const intptr_t kSavedRegisters = 0;
// Additional registers used to probe hash-based caches.
static const Register kEntryStartReg = R10;
static const Register kProbeMaskReg = R13;
static const Register kProbeDistanceReg = R8;
static const Register kCurrentEntryIndexReg = RSI;
};
// Registers in addition to those listed in TypeTestABI used inside the
// implementation of type testing stubs that are _not_ preserved.
struct TTSInternalRegs {

2
runtime/vm/hash.h
View file

@ -10,6 +10,7 @@
namespace dart {
inline uint32_t CombineHashes(uint32_t hash, uint32_t other_hash) {
// Keep in sync with AssemblerBase::CombineHashes.
hash += other_hash;
hash += hash << 10;
hash ^= hash >> 6; // Logical shift, unsigned hash.
@ -17,6 +18,7 @@ inline uint32_t CombineHashes(uint32_t hash, uint32_t other_hash) {
}
inline uint32_t FinalizeHash(uint32_t hash, intptr_t hashbits = kBitsPerInt32) {
// Keep in sync with AssemblerBase::FinalizeHash.
hash += hash << 3;
hash ^= hash >> 11; // Logical shift, unsigned hash.
hash += hash << 15;

71
runtime/vm/object.cc
View file

@ -1016,10 +1016,10 @@ void Object::Init(IsolateGroup* isolate_group) {
static_cast<ArrayPtr>(address + kHeapObjectTag));
empty_instantiations_cache_array_->untag()->set_length(
Smi::New(array_size));
// The empty cache has no occupied entries.
// The empty cache has no occupied entries and is not a hash-based cache.
smi = Smi::New(0);
empty_instantiations_cache_array_->SetAt(
TypeArguments::Cache::kOccupiedEntriesIndex, smi);
TypeArguments::Cache::kMetadataIndex, smi);
// Make the first (and only) entry unoccupied by setting its first element
// to the sentinel value.
smi = TypeArguments::Cache::Sentinel();
@ -6779,7 +6779,8 @@ bool TypeArguments::Cache::IsHash(const Array& array) {
}
intptr_t TypeArguments::Cache::NumOccupied(const Array& array) {
return RawSmiValue(Smi::RawCast(array.AtAcquire(kOccupiedEntriesIndex)));
return NumOccupiedBits::decode(
RawSmiValue(Smi::RawCast(array.AtAcquire(kMetadataIndex))));
}
#if defined(DEBUG)
@ -6806,7 +6807,14 @@ bool TypeArguments::Cache::IsValidStorageLocked(const Array& array) {
const bool is_linear_cache = IsLinear(array);
// The capacity of a hash-based cache must be a power of two (see
// EnsureCapacityLocked as to why).
if (!is_linear_cache && !Utils::IsPowerOfTwo(num_entries)) return false;
if (!is_linear_cache) {
if (!Utils::IsPowerOfTwo(num_entries)) return false;
const intptr_t metadata =
RawSmiValue(Smi::RawCast(array.AtAcquire(kMetadataIndex)));
if ((1 << EntryCountLog2Bits::decode(metadata)) != num_entries) {
return false;
}
}
for (intptr_t i = 0; i < num_entries; i++) {
const intptr_t index = kHeaderSize + i * kEntrySize;
if (array.At(index + kSentinelIndex) == Sentinel()) {
@ -6910,12 +6918,11 @@ TypeArguments::Cache::KeyLocation TypeArguments::Cache::AddEntry(
entry = loc.entry;
}
// Increment the number of occupied entries prior to adding the entry.
// Only the Cache class uses the information, and Cache objects are only
// created when holding the type arguments canonicalization mutex, so we
// don't need a store-release barrier for this.
smi_handle_ = Smi::New(new_occupied);
data_.SetAt(kOccupiedEntriesIndex, smi_handle_);
// Go ahead and increment the number of occupied entries prior to adding the
// entry. Use a store-release barrier in case of concurrent readers.
const intptr_t metadata = RawSmiValue(Smi::RawCast(data_.At(kMetadataIndex)));
smi_handle_ = Smi::New(NumOccupiedBits::update(new_occupied, metadata));
data_.SetAtRelease(kMetadataIndex, smi_handle_);
InstantiationsCacheTable table(data_);
const auto& tuple = table.At(entry);
@ -7004,12 +7011,11 @@ bool TypeArguments::Cache::EnsureCapacity(intptr_t new_occupied) const {
const auto& new_data =
Array::Handle(zone_, Array::NewUninitialized(new_size, Heap::kOld));
ASSERT(!new_data.IsNull());
// First copy over the metadata.
auto& object = Object::Handle(zone_);
for (intptr_t i = 0; i < kHeaderSize; i++) {
object = data_.At(i);
new_data.SetAt(i, object);
}
// First set up the metadata in new_data.
const intptr_t metadata = RawSmiValue(Smi::RawCast(data_.At(kMetadataIndex)));
smi_handle_ = Smi::New(EntryCountLog2Bits::update(
Utils::ShiftForPowerOfTwo(new_capacity), metadata));
new_data.SetAt(kMetadataIndex, smi_handle_);
// Then mark all the entries in new_data as unoccupied.
smi_handle_ = Sentinel();
InstantiationsCacheTable to_table(new_data);
@ -7017,14 +7023,14 @@ bool TypeArguments::Cache::EnsureCapacity(intptr_t new_occupied) const {
tuple.Set<kSentinelIndex>(smi_handle_);
}
// Finally, copy over the entries.
auto& instantiator_tav = TypeArguments::Handle(zone_);
auto& function_tav = TypeArguments::Handle(zone_);
auto& result_tav = TypeArguments::Handle(zone_);
const InstantiationsCacheTable from_table(data_);
for (const auto& from_tuple : from_table) {
// Skip unoccupied entries.
if (from_tuple.Get<kSentinelIndex>() == Sentinel()) continue;
object = from_tuple.Get<kInstantiatorTypeArgsIndex>();
const auto& instantiator_tav = TypeArguments::Cast(object);
instantiator_tav ^= from_tuple.Get<kInstantiatorTypeArgsIndex>();
function_tav = from_tuple.Get<kFunctionTypeArgsIndex>();
result_tav = from_tuple.Get<kInstantiatedTypeArgsIndex>();
// Since new_data has a different total capacity, we can't use the old
@ -7350,6 +7356,14 @@ TypeArgumentsPtr TypeArguments::InstantiateFrom(
return instantiated_array.ptr();
}
#if !defined(PRODUCT) && !defined(DART_PRECOMPILED_RUNTIME)
// A local flag used only in object_test.cc that, when true, causes a failure
// when a cache entry for the given instantiator and function type arguments
// already exists. Used to check that the InstantiateTypeArguments stub found
// the cache entry instead of calling the runtime.
bool TESTING_runtime_fail_on_existing_cache_entry = false;
#endif
TypeArgumentsPtr TypeArguments::InstantiateAndCanonicalizeFrom(
const TypeArguments& instantiator_type_arguments,
const TypeArguments& function_type_arguments) const {
@ -7368,6 +7382,27 @@ TypeArgumentsPtr TypeArguments::InstantiateAndCanonicalizeFrom(
auto const loc = cache.FindKeyOrUnused(instantiator_type_arguments,
function_type_arguments);
if (loc.present) {
#if !defined(PRODUCT) && !defined(DART_PRECOMPILED_RUNTIME)
if (TESTING_runtime_fail_on_existing_cache_entry) {
TextBuffer buffer(1024);
buffer.Printf("for\n");
buffer.Printf(" * uninstantiated type arguments %s\n", ToCString());
buffer.Printf(" * instantiation type arguments: %s (hash: %" Pu ")\n",
instantiator_type_arguments.ToCString(),
instantiator_type_arguments.Hash());
buffer.Printf(" * function type arguments: %s (hash: %" Pu ")\n",
function_type_arguments.ToCString(),
function_type_arguments.Hash());
buffer.Printf(" * number of occupied entries in cache: %" Pd "\n",
cache.NumOccupied());
buffer.Printf(" * number of total entries in cache: %" Pd "\n",
cache.NumEntries());
buffer.Printf("expected to find entry %" Pd
" of cache in stub, but reached runtime",
loc.entry);
FATAL("%s", buffer.buffer());
}
#endif
return cache.Retrieve(loc.entry);
}
// Cache lookup failed. Instantiate the type arguments.

42
runtime/vm/object.h
View file

@ -8140,11 +8140,29 @@ class TypeArguments : public Instance {
// type arguments.
enum Header {
// The number of occupied entries in the cache.
kOccupiedEntriesIndex = 0,
// A single Smi that is a bitfield containing two values:
// - The number of occupied entries in the cache for all caches.
// - For hash-based caches, the upper bits contain log2(N) where N
// is the number of total entries in the cache, so this information can
// be quickly retrieved by stubs.
//
// Note: accesses outside of the type arguments canonicalization mutex
// must have acquire semantics. In C++ code, use NumOccupied to retrieve
// the number of occupied entries.
kMetadataIndex = 0,
kHeaderSize,
};
using NumOccupiedBits = BitField<intptr_t,
intptr_t,
0,
compiler::target::kSmiBits -
compiler::target::kBitsPerWordLog2>;
using EntryCountLog2Bits = BitField<intptr_t,
intptr_t,
NumOccupiedBits::kNextBit,
compiler::target::kBitsPerWordLog2>;
// The tuple of values stored in a given entry.
//
// Note: accesses of the first component outside of the type arguments
@ -8239,9 +8257,20 @@ class TypeArguments : public Instance {
// Returns whether the backing store changed.
bool EnsureCapacity(intptr_t occupied) const;
public: // For testing purposes only.
// Retrieves the number of entries (occupied or unoccupied) in the cache.
intptr_t NumEntries() const { return NumEntries(data_); }
// The maximum number of occupied entries for a linear cache of
// instantiations before swapping to a hash table-based cache.
#if defined(TARGET_ARCH_IA32)
// We don't generate hash cache probing in the stub on IA32.
static constexpr intptr_t kMaxLinearCacheEntries = 500;
#else
static constexpr intptr_t kMaxLinearCacheEntries = 10;
#endif
private:
// Retrieves the number of entries (occupied or unoccupied) in a cache
// backed by the given array.
static intptr_t NumEntries(const Array& array);
@ -8258,11 +8287,7 @@ class TypeArguments : public Instance {
// The sentinel value in the Smi returned from Sentinel().
static constexpr intptr_t kSentinelValue = 0;
public:
// The maximum number of occupied entries for a linear cache of
// instantiations before swapping to a hash table-based cache.
static constexpr intptr_t kMaxLinearCacheEntries = 500;
public: // Used in the StubCodeCompiler.
// The maximum size of the array backing a linear cache. All hash based
// caches are guaranteed to have sizes larger than this.
static constexpr intptr_t kMaxLinearCacheSize =
@ -8319,6 +8344,9 @@ class TypeArguments : public Instance {
}
uword Hash() const;
uword HashForRange(intptr_t from_index, intptr_t len) const;
static intptr_t hash_offset() {
return OFFSET_OF(UntaggedTypeArguments, hash_);
}
static TypeArgumentsPtr New(intptr_t len, Heap::Space space = Heap::kOld);

126
runtime/vm/object_test.cc
View file

@ -8077,6 +8077,71 @@ FutureOr<T?> bar<T>() { return null; }
}
}
#define __ assembler->
static void GenerateInvokeInstantiateTAVStub(compiler::Assembler* assembler) {
__ EnterDartFrame(0);
// Load the arguments into the right stub calling convention registers.
const intptr_t uninstantiated_offset =
(kCallerSpSlotFromFp + 2) * compiler::target::kWordSize;
const intptr_t inst_type_args_offset =
(kCallerSpSlotFromFp + 1) * compiler::target::kWordSize;
const intptr_t fun_type_args_offset =
(kCallerSpSlotFromFp + 0) * compiler::target::kWordSize;
__ LoadMemoryValue(InstantiationABI::kUninstantiatedTypeArgumentsReg, FPREG,
uninstantiated_offset);
__ LoadMemoryValue(InstantiationABI::kInstantiatorTypeArgumentsReg, FPREG,
inst_type_args_offset);
__ LoadMemoryValue(InstantiationABI::kFunctionTypeArgumentsReg, FPREG,
fun_type_args_offset);
__ Call(StubCode::InstantiateTypeArguments());
// Set the return from the stub.
__ MoveRegister(CallingConventions::kReturnReg,
InstantiationABI::kResultTypeArgumentsReg);
__ LeaveDartFrame();
__ Ret();
}
#undef __
static CodePtr CreateInvokeInstantiateTypeArgumentsStub(Thread* thread) {
Zone* const zone = thread->zone();
const auto& klass = Class::Handle(
zone, thread->isolate_group()->class_table()->At(kInstanceCid));
const auto& symbol = String::Handle(
zone, Symbols::New(thread, OS::SCreate(zone, "InstantiateTAVTest")));
const auto& signature = FunctionType::Handle(zone, FunctionType::New());
const auto& function = Function::Handle(
zone, Function::New(signature, symbol, UntaggedFunction::kRegularFunction,
false, false, false, false, false, klass,
TokenPosition::kNoSource));
compiler::ObjectPoolBuilder pool_builder;
const auto& invoke_instantiate_tav =
Code::Handle(zone, StubCode::Generate("InstantiateTAV", &pool_builder,
&GenerateInvokeInstantiateTAVStub));
const auto& pool =
ObjectPool::Handle(zone, ObjectPool::NewFromBuilder(pool_builder));
invoke_instantiate_tav.set_object_pool(pool.ptr());
invoke_instantiate_tav.set_owner(function);
invoke_instantiate_tav.set_exception_handlers(
ExceptionHandlers::Handle(zone, ExceptionHandlers::New(0)));
#if defined(TARGET_ARCH_IA32)
EXPECT_EQ(0, pool.Length());
#else
EXPECT_EQ(1, pool.Length()); // The InstantiateTypeArguments stub.
#endif
return invoke_instantiate_tav.ptr();
}
#if !defined(PRODUCT)
// Defined before TypeArguments::InstantiateAndCanonicalizeFrom in object.cc.
extern bool TESTING_runtime_fail_on_existing_cache_entry;
#endif
static void TypeArgumentsHashCacheTest(Thread* thread, intptr_t num_classes) {
TextBuffer buffer(MB);
buffer.AddString("class D<T> {}\n");
@ -8086,7 +8151,7 @@ static void TypeArgumentsHashCacheTest(Thread* thread, intptr_t num_classes) {
}
buffer.AddString("main() {\n");
for (intptr_t i = 0; i < num_classes; i++) {
buffer.Printf(" new C%" Pd "().toString();\n", i);
buffer.Printf(" C%" Pd "().toString();\n", i);
}
buffer.AddString("}\n");
@ -8122,7 +8187,16 @@ static void TypeArgumentsHashCacheTest(Thread* thread, intptr_t num_classes) {
// Cache the first computed set of instantiator type arguments to check that
// no entries from the cache have been lost when the cache grows.
auto& first_instantiator_type_args = TypeArguments::Handle(zone);
// Used for the cache hit in stub check.
const auto& invoke_instantiate_tav =
Code::Handle(zone, CreateInvokeInstantiateTypeArgumentsStub(thread));
const auto& invoke_instantiate_tav_arguments =
Array::Handle(zone, Array::New(3));
const auto& invoke_instantiate_tav_args_descriptor =
Array::Handle(zone, ArgumentsDescriptor::NewBoxed(0, 3));
for (intptr_t i = 0; i < num_classes; ++i) {
const bool updated_cache_is_linear =
i < TypeArguments::Cache::kMaxLinearCacheEntries;
auto const name = OS::SCreate(zone, "C%" Pd "", i);
class_c = GetClass(root_lib, name);
ASSERT(!class_c.IsNull());
@ -8131,7 +8205,16 @@ static void TypeArgumentsHashCacheTest(Thread* thread, intptr_t num_classes) {
instantiator_type_args.SetTypeAt(0, decl_type_c);
instantiator_type_args = instantiator_type_args.Canonicalize(thread);
#if !defined(PRODUCT)
// The first call to InstantiateAndCanonicalizeFrom shouldn't have a cache
// hit since the instantiator type arguments should be unique for each
// iteration, and after that we do a check that the InstantiateTypeArguments
// stub finds the entry (unless the cache is hash-based on IA32).
TESTING_runtime_fail_on_existing_cache_entry = true;
#endif
// Check that the key does not currently exist in the cache.
intptr_t old_capacity;
{
SafepointMutexLocker ml(
thread->isolate_group()->type_arguments_canonicalization_mutex());
@ -8140,6 +8223,7 @@ static void TypeArgumentsHashCacheTest(Thread* thread, intptr_t num_classes) {
auto loc =
cache.FindKeyOrUnused(instantiator_type_args, function_type_args);
EXPECT(!loc.present);
old_capacity = cache.NumEntries();
}
decl_type_d_type_args.InstantiateAndCanonicalizeFrom(instantiator_type_args,
@ -8147,18 +8231,45 @@ static void TypeArgumentsHashCacheTest(Thread* thread, intptr_t num_classes) {
// Check that the key now does exist in the cache.
TypeArguments::Cache::KeyLocation loc;
bool storage_changed;
{
SafepointMutexLocker ml(
thread->isolate_group()->type_arguments_canonicalization_mutex());
TypeArguments::Cache cache(zone, decl_type_d_type_args);
EXPECT_EQ(i + 1, cache.NumOccupied());
// Double-check that we got the expected type of cache.
EXPECT(i < TypeArguments::Cache::kMaxLinearCacheEntries ? cache.IsLinear()
: cache.IsHash());
EXPECT(updated_cache_is_linear ? cache.IsLinear() : cache.IsHash());
loc = cache.FindKeyOrUnused(instantiator_type_args, function_type_args);
EXPECT(loc.present);
storage_changed = cache.NumEntries() != old_capacity;
}
#if defined(TARGET_ARCH_IA32)
const bool stub_checks_hash_caches = false;
#else
const bool stub_checks_hash_caches = true;
#endif
// Now check that we get the expected result from calling the stub if it
// checks the cache (e.g., in all cases but hash-based caches on IA32).
if (updated_cache_is_linear || stub_checks_hash_caches) {
invoke_instantiate_tav_arguments.SetAt(0, decl_type_d_type_args);
invoke_instantiate_tav_arguments.SetAt(1, instantiator_type_args);
invoke_instantiate_tav_arguments.SetAt(2, function_type_args);
result_type_args ^= DartEntry::InvokeCode(
invoke_instantiate_tav, invoke_instantiate_tav.EntryPoint(),
invoke_instantiate_tav_args_descriptor,
invoke_instantiate_tav_arguments, thread);
EXPECT_EQ(1, result_type_args.Length());
result_type = result_type_args.TypeAt(0);
EXPECT_TYPES_SYNTACTICALLY_EQUIVALENT(decl_type_c, result_type);
}
#if !defined(PRODUCT)
// Setting to false prior to re-calling InstantiateAndCanonicalizeFrom with
// the same keys, as now we want a runtime check of an existing cache entry.
TESTING_runtime_fail_on_existing_cache_entry = false;
#endif
result_type_args = decl_type_d_type_args.InstantiateAndCanonicalizeFrom(
instantiator_type_args, function_type_args);
result_type = result_type_args.TypeAt(0);
@ -8178,8 +8289,8 @@ static void TypeArgumentsHashCacheTest(Thread* thread, intptr_t num_classes) {
if (i == 0) {
first_instantiator_type_args = instantiator_type_args.ptr();
} else {
// Check that the first instantiator TAV still exists in the cache.
} else if (storage_changed) {
// Check that the first instantiator TAV still exists in the new cache.
SafepointMutexLocker ml(
thread->isolate_group()->type_arguments_canonicalization_mutex());
TypeArguments::Cache cache(zone, decl_type_d_type_args);
@ -8201,10 +8312,11 @@ TEST_CASE(TypeArguments_Cache_SomeInstantiations) {
2 * TypeArguments::Cache::kMaxLinearCacheEntries);
}
// Too slow in debug mode. Also avoid the sanitizers for similar reasons.
// Too slow in debug mode. Also avoid the sanitizers and simulators for similar
// reasons. Any core issues will likely be found by SomeInstantiations.
#if !defined(DEBUG) && !defined(USING_MEMORY_SANITIZER) && \
!defined(USING_THREAD_SANITIZER) && !defined(USING_LEAK_SANITIZER) && \
!defined(USING_UNDEFINED_BEHAVIOR_SANITIZER)
!defined(USING_UNDEFINED_BEHAVIOR_SANITIZER) && !defined(USING_SIMULATOR)
TEST_CASE(TypeArguments_Cache_ManyInstantiations) {
const intptr_t kNumClasses = 100000;
static_assert(kNumClasses > TypeArguments::Cache::kMaxLinearCacheEntries,

25
runtime/vm/unit_test.h
View file

@ -128,7 +128,26 @@
#define ASSEMBLER_TEST_RUN(name, test) \
ASSEMBLER_TEST_RUN_WITH_EXPECTATION(name, test, "Pass")
#if defined(TARGET_ARCH_ARM) || defined(TARGET_ARCH_ARM64) || \
#if defined(TARGET_ARCH_IA32) || defined(TARGET_ARCH_X64)
// We don't have simulators for these architectures, but define the macros so
// they can be used in architecture-independent tests.
#define EXECUTE_TEST_CODE_INT32(name, entry) reinterpret_cast<name>(entry)()
#define EXECUTE_TEST_CODE_INT64(name, entry) reinterpret_cast<name>(entry)()
#define EXECUTE_TEST_CODE_INT64_LL(name, entry, long_arg0, long_arg1) \
reinterpret_cast<name>(entry)(long_arg0, long_arg1)
#define EXECUTE_TEST_CODE_FLOAT(name, entry) reinterpret_cast<name>(entry)()
#define EXECUTE_TEST_CODE_DOUBLE(name, entry) reinterpret_cast<name>(entry)()
#define EXECUTE_TEST_CODE_INT32_F(name, entry, float_arg) \
reinterpret_cast<name>(entry)(float_arg)
#define EXECUTE_TEST_CODE_INT32_D(name, entry, double_arg) \
reinterpret_cast<name>(entry)(double_arg)
#define EXECUTE_TEST_CODE_INTPTR_INTPTR(name, entry, pointer_arg) \
reinterpret_cast<name>(entry)(pointer_arg)
#define EXECUTE_TEST_CODE_INT32_INTPTR(name, entry, pointer_arg) \
reinterpret_cast<name>(entry)(pointer_arg)
#define EXECUTE_TEST_CODE_UWORD_UWORD_UINT32(name, entry, arg0, arg1) \
reinterpret_cast<name>(entry)(arg0, arg1)
#elif defined(TARGET_ARCH_ARM) || defined(TARGET_ARCH_ARM64) || \
defined(TARGET_ARCH_RISCV32) || defined(TARGET_ARCH_RISCV64)
#if defined(HOST_ARCH_ARM) || defined(HOST_ARCH_ARM64) || \
defined(HOST_ARCH_RISCV32) || defined(HOST_ARCH_RISCV64)
@ -147,6 +166,8 @@
reinterpret_cast<name>(entry)(pointer_arg)
#define EXECUTE_TEST_CODE_INT32_INTPTR(name, entry, pointer_arg) \
reinterpret_cast<name>(entry)(pointer_arg)
#define EXECUTE_TEST_CODE_UWORD_UWORD_UINT32(name, entry, arg0, arg1) \
reinterpret_cast<name>(entry)(arg0, arg1)
#else
// Not running on ARM hardware, call simulator to execute code.
#if defined(ARCH_IS_64_BIT)
@ -198,6 +219,8 @@
Utils::Low32Bits(bit_cast<int64_t, double>(double_arg)), \
Utils::High32Bits(bit_cast<int64_t, double>(double_arg)), 0, 0, false, \
true))
#define EXECUTE_TEST_CODE_UWORD_UWORD_UINT32(name, entry, arg0, arg1) \
static_cast<uint32_t>(Simulator::Current()->Call(entry, arg0, arg1, 0, 0))
#endif // defined(HOST_ARCH_ARM)
#endif // defined(TARGET_ARCH_{ARM, ARM64})