languages/dart-sdk
languages/dart-sdk
1
0
Fork 0
mirror of https://github.com/dart-lang/sdk synced 2024-09-05 00:13:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

[vm, compiler] Specialize unoptimized monomorphic and megamorphic calls.

Browse source 
dart-bytecode, arm64:            +4.742% geomean
dart-bytecode-jit-unopt, arm64: +12.73% geomean
dart2js-compile, x64:            +3.635% geomean

In the polymorphic and unlinked cases, call to a stub the does a linear scan against an ICData.

In the monomorphic case, call to a prologue of the expected target function that checks the expected receiver class. There is additional indirection in the JIT version compared to the AOT version to also tick a usage counter so the inliner can make good decisions.

In the megamorphic case, call to a stub that does a hash table lookup against a MegamorphicCache.

Megamorphic call sites face a loss of precision in usage counts. The call site count is not recorded and the usage counter of the target function is used as an approximation.

Monomorphic and megamorphic calls sites are reset to the polymorphic/unlinked state on hot reload.

Monomorphic and megamorphic calls sites do not check the stepping state, so they are reset to the polymorphic/unlinked state when stepping begins and disabled.

Back-edges now increment the usage counter in addition to checking it. This ensures function with loops containing monomorphic calls will eventually cross the optimization threshold.

Fixed backwards use of kMonomorphicEntryOffset and kPolymorphicEntryOffset.

Fixed C stack overflow when bouncing between the KBC interpreter and a simulator.

Bug: https://github.com/dart-lang/sdk/issues/26780
Bug: https://github.com/dart-lang/sdk/issues/36409
Bug: https://github.com/dart-lang/sdk/issues/36731
Change-Id: I78a49cccd962703a459288e71ce246ed845df474
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/102820
Commit-Queue: Ryan Macnak <rmacnak@google.com>
Reviewed-by: Alexander Markov <alexmarkov@google.com>
This commit is contained in:
Ryan Macnak 2019-06-12 21:56:53 +00:00 committed by commit-bot@chromium.org
parent f005bd5a3e
commit 1e24fe7d69
63 changed files with 1635 additions and 557 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
runtime/tests/vm/vm.status
View file

@ -13,8 +13,9 @@ cc/IsolateReload_PendingUnqualifiedCall_InstanceToStatic: Fail # Issue 32981
cc/IsolateReload_PendingUnqualifiedCall_StaticToInstance: Fail # Issue 32981
cc/IsolateReload_RunNewFieldInitializersWithGenerics: Fail # Issue 32299
dart/data_uri_import_test/none: SkipByDesign
dart/snapshot_version_test: Skip # This test is a Dart1 test (script snapshot)
dart/entrypoints/jit: Skip # Tests with brittle dependencies on usage counters - Issue 37144
dart/slow_path_shared_stub_test: Pass, Slow # Uses --shared-slow-path-triggers-gc flag.
dart/snapshot_version_test: Skip # This test is a Dart1 test (script snapshot)
dart/stack_overflow_shared_test: Pass, Slow # Uses --shared-slow-path-triggers-gc flag.
dart/use_bare_instructions_flag_test: Pass, Slow # Spawns several subprocesses

24
runtime/vm/code_patcher.h
View file

@ -49,12 +49,18 @@ class CodePatcher : public AllStatic {
// in given code.
static RawCode* GetStaticCallTargetAt(uword return_address, const Code& code);
// Get instance call information. Returns the call target and sets each
// of the output parameters ic_data and arguments_descriptor if they are
// non-NULL.
// Get instance call information. Returns the call target and sets the output
// parameter data if non-NULL.
static RawCode* GetInstanceCallAt(uword return_address,
const Code& code,
ICData* ic_data);
const Code& caller_code,
Object* data);
// Change the state of an instance call by patching the corresponding object
// pool entries (non-IA32) or instructions (IA32).
static void PatchInstanceCallAt(uword return_address,
const Code& caller_code,
const Object& data,
const Code& target);
// Return target of an unoptimized static call and its ICData object
// (calls target via a stub).
@ -79,22 +85,22 @@ class CodePatcher : public AllStatic {
#if defined(TARGET_ARCH_DBC)
static NativeFunctionWrapper GetNativeCallAt(uword return_address,
const Code& code,
const Code& caller_code,
NativeFunction* target);
#else
static RawCode* GetNativeCallAt(uword return_address,
const Code& code,
const Code& caller_code,
NativeFunction* target);
#endif
#if defined(TARGET_ARCH_DBC)
static void PatchNativeCallAt(uword return_address,
const Code& code,
const Code& caller_code,
NativeFunction target,
NativeFunctionWrapper trampoline);
#else
static void PatchNativeCallAt(uword return_address,
const Code& code,
const Code& caller_code,
NativeFunction target,
const Code& trampoline);
#endif

30
runtime/vm/code_patcher_arm.cc
View file

@ -33,23 +33,33 @@ void CodePatcher::InsertDeoptimizationCallAt(uword start) {
}
RawCode* CodePatcher::GetInstanceCallAt(uword return_address,
const Code& code,
ICData* ic_data) {
ASSERT(code.ContainsInstructionAt(return_address));
CallPattern call(return_address, code);
if (ic_data != NULL) {
*ic_data = call.IcData();
const Code& caller_code,
Object* data) {
ASSERT(caller_code.ContainsInstructionAt(return_address));
ICCallPattern call(return_address, caller_code);
if (data != NULL) {
*data = call.Data();
}
return call.TargetCode();
}
void CodePatcher::PatchInstanceCallAt(uword return_address,
const Code& caller_code,
const Object& data,
const Code& target) {
ASSERT(caller_code.ContainsInstructionAt(return_address));
ICCallPattern call(return_address, caller_code);
call.SetData(data);
call.SetTargetCode(target);
}
RawFunction* CodePatcher::GetUnoptimizedStaticCallAt(uword return_address,
const Code& code,
const Code& caller_code,
ICData* ic_data_result) {
ASSERT(code.ContainsInstructionAt(return_address));
CallPattern static_call(return_address, code);
ASSERT(caller_code.ContainsInstructionAt(return_address));
ICCallPattern static_call(return_address, caller_code);
ICData& ic_data = ICData::Handle();
ic_data = static_call.IcData();
ic_data ^= static_call.Data();
if (ic_data_result != NULL) {
*ic_data_result = ic_data.raw();
}

38
runtime/vm/code_patcher_arm64.cc
View file

@ -68,23 +68,33 @@ void CodePatcher::InsertDeoptimizationCallAt(uword start) {
}
RawCode* CodePatcher::GetInstanceCallAt(uword return_address,
const Code& code,
ICData* ic_data) {
ASSERT(code.ContainsInstructionAt(return_address));
CallPattern call(return_address, code);
if (ic_data != NULL) {
*ic_data = call.IcData();
const Code& caller_code,
Object* data) {
ASSERT(caller_code.ContainsInstructionAt(return_address));
ICCallPattern call(return_address, caller_code);
if (data != NULL) {
*data = call.Data();
}
return call.TargetCode();
}
void CodePatcher::PatchInstanceCallAt(uword return_address,
const Code& caller_code,
const Object& data,
const Code& target) {
ASSERT(caller_code.ContainsInstructionAt(return_address));
ICCallPattern call(return_address, caller_code);
call.SetData(data);
call.SetTargetCode(target);
}
RawFunction* CodePatcher::GetUnoptimizedStaticCallAt(uword return_address,
const Code& code,
ICData* ic_data_result) {
ASSERT(code.ContainsInstructionAt(return_address));
CallPattern static_call(return_address, code);
ICCallPattern static_call(return_address, code);
ICData& ic_data = ICData::Handle();
ic_data ^= static_call.IcData();
ic_data ^= static_call.Data();
if (ic_data_result != NULL) {
*ic_data_result = ic_data.raw();
}
@ -132,20 +142,20 @@ RawObject* CodePatcher::GetSwitchableCallDataAt(uword return_address,
}
void CodePatcher::PatchNativeCallAt(uword return_address,
const Code& code,
const Code& caller_code,
NativeFunction target,
const Code& trampoline) {
ASSERT(code.ContainsInstructionAt(return_address));
NativeCallPattern call(return_address, code);
ASSERT(caller_code.ContainsInstructionAt(return_address));
NativeCallPattern call(return_address, caller_code);
call.set_target(trampoline);
call.set_native_function(target);
}
RawCode* CodePatcher::GetNativeCallAt(uword return_address,
const Code& code,
const Code& caller_code,
NativeFunction* target) {
ASSERT(code.ContainsInstructionAt(return_address));
NativeCallPattern call(return_address, code);
ASSERT(caller_code.ContainsInstructionAt(return_address));
NativeCallPattern call(return_address, caller_code);
*target = call.native_function();
return call.target();
}

14
runtime/vm/code_patcher_arm64_test.cc
View file

@ -39,12 +39,22 @@ ASSEMBLER_TEST_GENERATE(IcDataAccess, assembler) {
ArgumentsDescriptor::New(kTypeArgsLen, kNumArgs, Object::null_array()));
const ICData& ic_data = ICData::ZoneHandle(ICData::New(
function, target_name, args_descriptor, 15, 1, ICData::kInstance));
const Code& stub = StubCode::OneArgCheckInlineCache();
// Code accessing pp is generated, but not executed. Uninitialized pp is OK.
__ set_constant_pool_allowed(true);
__ LoadObject(R5, ic_data);
__ BranchLinkPatchable(StubCode::OneArgCheckInlineCache());
ObjectPoolBuilder& op = __ object_pool_builder();
const intptr_t ic_data_index =
op.AddObject(ic_data, ObjectPool::Patchability::kPatchable);
const intptr_t stub_index =
op.AddObject(stub, ObjectPool::Patchability::kPatchable);
ASSERT((ic_data_index + 1) == stub_index);
__ LoadDoubleWordFromPoolOffset(R5, CODE_REG,
ObjectPool::element_offset(ic_data_index));
__ ldr(LR, FieldAddress(CODE_REG, Code::entry_point_offset(
Code::EntryKind::kMonomorphic)));
__ blr(LR);
__ ret();
}

36
runtime/vm/code_patcher_dbc.cc
View file

@ -33,23 +33,33 @@ void CodePatcher::InsertDeoptimizationCallAt(uword start) {
}
RawCode* CodePatcher::GetInstanceCallAt(uword return_address,
const Code& code,
ICData* ic_data) {
ASSERT(code.ContainsInstructionAt(return_address));
CallPattern call(return_address, code);
if (ic_data != NULL) {
*ic_data = call.IcData();
const Code& caller_code,
Object* cache) {
ASSERT(caller_code.ContainsInstructionAt(return_address));
CallPattern call(return_address, caller_code);
if (cache != NULL) {
*cache = call.Data();
}
return call.TargetCode();
}
void CodePatcher::PatchInstanceCallAt(uword return_address,
const Code& caller_code,
const Object& data,
const Code& target) {
ASSERT(caller_code.ContainsInstructionAt(return_address));
CallPattern call(return_address, caller_code);
call.SetData(data);
call.SetTargetCode(target);
}
RawFunction* CodePatcher::GetUnoptimizedStaticCallAt(uword return_address,
const Code& code,
const Code& caller_code,
ICData* ic_data_result) {
ASSERT(code.ContainsInstructionAt(return_address));
CallPattern static_call(return_address, code);
ASSERT(caller_code.ContainsInstructionAt(return_address));
CallPattern static_call(return_address, caller_code);
ICData& ic_data = ICData::Handle();
ic_data ^= static_call.IcData();
ic_data ^= static_call.Data();
if (ic_data_result != NULL) {
*ic_data_result = ic_data.raw();
}
@ -91,10 +101,10 @@ void CodePatcher::PatchNativeCallAt(uword return_address,
}
NativeFunctionWrapper CodePatcher::GetNativeCallAt(uword return_address,
const Code& code,
const Code& caller_code,
NativeFunction* target) {
ASSERT(code.ContainsInstructionAt(return_address));
NativeCallPattern call(return_address, code);
ASSERT(caller_code.ContainsInstructionAt(return_address));
NativeCallPattern call(return_address, caller_code);
*target = call.native_function();
return call.target();
}

61
runtime/vm/code_patcher_ia32.cc
View file

@ -72,17 +72,40 @@ class NativeCall : public UnoptimizedCall {
DISALLOW_IMPLICIT_CONSTRUCTORS(NativeCall);
};
// b9xxxxxxxx mov ecx,<data>
// bfyyyyyyyy mov edi,<target>
// ff5707 call [edi+<monomorphic-entry-offset>]
class InstanceCall : public UnoptimizedCall {
public:
explicit InstanceCall(uword return_address)
: UnoptimizedCall(return_address) {
#if defined(DEBUG)
ICData& test_ic_data = ICData::Handle();
test_ic_data ^= ic_data();
ASSERT(test_ic_data.NumArgsTested() > 0);
Object& test_data = Object::Handle(data());
ASSERT(test_data.IsArray() || test_data.IsICData() ||
test_data.IsMegamorphicCache());
if (test_data.IsICData()) {
ASSERT(ICData::Cast(test_data).NumArgsTested() > 0);
}
#endif // DEBUG
}
RawObject* data() const { return *reinterpret_cast<RawObject**>(start_ + 1); }
void set_data(const Object& data) const {
uword* cache_addr = reinterpret_cast<uword*>(start_ + 1);
uword imm = reinterpret_cast<uword>(data.raw());
*cache_addr = imm;
}
RawCode* target() const {
const uword imm = *reinterpret_cast<uword*>(start_ + 6);
return reinterpret_cast<RawCode*>(imm);
}
void set_target(const Code& target) const {
uword* target_addr = reinterpret_cast<uword*>(start_ + 6);
uword imm = reinterpret_cast<uword>(target.raw());
*target_addr = imm;
}
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(InstanceCall);
};
@ -168,20 +191,32 @@ void CodePatcher::InsertDeoptimizationCallAt(uword start) {
}
RawCode* CodePatcher::GetInstanceCallAt(uword return_address,
const Code& code,
ICData* ic_data) {
ASSERT(code.ContainsInstructionAt(return_address));
const Code& caller_code,
Object* data) {
ASSERT(caller_code.ContainsInstructionAt(return_address));
InstanceCall call(return_address);
if (ic_data != NULL) {
*ic_data ^= call.ic_data();
if (data != NULL) {
*data = call.data();
}
return Code::null();
return call.target();
}
void CodePatcher::PatchInstanceCallAt(uword return_address,
const Code& caller_code,
const Object& data,
const Code& target) {
ASSERT(caller_code.ContainsInstructionAt(return_address));
const Instructions& instrs = Instructions::Handle(caller_code.instructions());
WritableInstructionsScope writable(instrs.PayloadStart(), instrs.Size());
InstanceCall call(return_address);
call.set_data(data);
call.set_target(target);
}
RawFunction* CodePatcher::GetUnoptimizedStaticCallAt(uword return_address,
const Code& code,
const Code& caller_code,
ICData* ic_data_result) {
ASSERT(code.ContainsInstructionAt(return_address));
ASSERT(caller_code.ContainsInstructionAt(return_address));
UnoptimizedStaticCall static_call(return_address);
ICData& ic_data = ICData::Handle();
ic_data ^= static_call.ic_data();
@ -214,14 +249,14 @@ RawObject* CodePatcher::GetSwitchableCallDataAt(uword return_address,
}
void CodePatcher::PatchNativeCallAt(uword return_address,
const Code& code,
const Code& caller_code,
NativeFunction target,
const Code& trampoline) {
UNREACHABLE();
}
RawCode* CodePatcher::GetNativeCallAt(uword return_address,
const Code& code,
const Code& caller_code,
NativeFunction* target) {
UNREACHABLE();
return NULL;

67
runtime/vm/code_patcher_x64.cc
View file

@ -77,8 +77,6 @@ class UnoptimizedCall : public ValueObject {
intptr_t argument_index() const { return argument_index_; }
RawObject* ic_data() const { return object_pool_.ObjectAt(argument_index()); }
RawCode* target() const {
Code& code = Code::Handle();
code ^= object_pool_.ObjectAt(code_index_);
@ -123,20 +121,29 @@ class InstanceCall : public UnoptimizedCall {
InstanceCall(uword return_address, const Code& code)
: UnoptimizedCall(return_address, code) {
#if defined(DEBUG)
ICData& test_ic_data = ICData::Handle();
test_ic_data ^= ic_data();
ASSERT(test_ic_data.NumArgsTested() > 0);
Object& test_data = Object::Handle(data());
ASSERT(test_data.IsArray() || test_data.IsICData() ||
test_data.IsMegamorphicCache());
if (test_data.IsICData()) {
ASSERT(ICData::Cast(test_data).NumArgsTested() > 0);
}
#endif // DEBUG
}
RawObject* data() const { return object_pool_.ObjectAt(argument_index()); }
void set_data(const Object& data) const {
ASSERT(data.IsArray() || data.IsICData() || data.IsMegamorphicCache());
object_pool_.SetObjectAt(argument_index(), data);
}
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(InstanceCall);
};
class UnoptimizedStaticCall : public UnoptimizedCall {
public:
UnoptimizedStaticCall(uword return_address, const Code& code)
: UnoptimizedCall(return_address, code) {
UnoptimizedStaticCall(uword return_address, const Code& caller_code)
: UnoptimizedCall(return_address, caller_code) {
#if defined(DEBUG)
ICData& test_ic_data = ICData::Handle();
test_ic_data ^= ic_data();
@ -144,6 +151,8 @@ class UnoptimizedStaticCall : public UnoptimizedCall {
#endif // DEBUG
}
RawObject* ic_data() const { return object_pool_.ObjectAt(argument_index()); }
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(UnoptimizedStaticCall);
};
@ -152,8 +161,8 @@ class UnoptimizedStaticCall : public UnoptimizedCall {
// the object pool.
class PoolPointerCall : public ValueObject {
public:
explicit PoolPointerCall(uword return_address, const Code& code)
: object_pool_(ObjectPool::Handle(code.GetObjectPool())),
explicit PoolPointerCall(uword return_address, const Code& caller_code)
: object_pool_(ObjectPool::Handle(caller_code.GetObjectPool())),
code_index_(-1) {
uword pc = return_address;
@ -424,25 +433,35 @@ void CodePatcher::PatchPoolPointerCallAt(uword return_address,
}
RawCode* CodePatcher::GetInstanceCallAt(uword return_address,
const Code& code,
ICData* ic_data) {
ASSERT(code.ContainsInstructionAt(return_address));
InstanceCall call(return_address, code);
if (ic_data != NULL) {
*ic_data ^= call.ic_data();
const Code& caller_code,
Object* data) {
ASSERT(caller_code.ContainsInstructionAt(return_address));
InstanceCall call(return_address, caller_code);
if (data != NULL) {
*data = call.data();
}
return call.target();
}
void CodePatcher::PatchInstanceCallAt(uword return_address,
const Code& caller_code,
const Object& data,
const Code& target) {
ASSERT(caller_code.ContainsInstructionAt(return_address));
InstanceCall call(return_address, caller_code);
call.set_data(data);
call.set_target(target);
}
void CodePatcher::InsertDeoptimizationCallAt(uword start) {
UNREACHABLE();
}
RawFunction* CodePatcher::GetUnoptimizedStaticCallAt(uword return_address,
const Code& code,
const Code& caller_code,
ICData* ic_data_result) {
ASSERT(code.ContainsInstructionAt(return_address));
UnoptimizedStaticCall static_call(return_address, code);
ASSERT(caller_code.ContainsInstructionAt(return_address));
UnoptimizedStaticCall static_call(return_address, caller_code);
ICData& ic_data = ICData::Handle();
ic_data ^= static_call.ic_data();
if (ic_data_result != NULL) {
@ -492,20 +511,20 @@ RawObject* CodePatcher::GetSwitchableCallDataAt(uword return_address,
}
void CodePatcher::PatchNativeCallAt(uword return_address,
const Code& code,
const Code& caller_code,
NativeFunction target,
const Code& trampoline) {
ASSERT(code.ContainsInstructionAt(return_address));
NativeCall call(return_address, code);
ASSERT(caller_code.ContainsInstructionAt(return_address));
NativeCall call(return_address, caller_code);
call.set_target(trampoline);
call.set_native_function(target);
}
RawCode* CodePatcher::GetNativeCallAt(uword return_address,
const Code& code,
const Code& caller_code,
NativeFunction* target) {
ASSERT(code.ContainsInstructionAt(return_address));
NativeCall call(return_address, code);
ASSERT(caller_code.ContainsInstructionAt(return_address));
NativeCall call(return_address, caller_code);
*target = call.native_function();
return call.target();
}

15
runtime/vm/compiler/aot/aot_call_specializer.cc
View file

@ -161,7 +161,8 @@ bool AotCallSpecializer::RecognizeRuntimeTypeGetter(InstanceCallInstr* call) {
Function::Handle(Z, call->ResolveForReceiverClass(cls));
ASSERT(!function.IsNull());
const Function& target = Function::ZoneHandle(Z, function.raw());
StaticCallInstr* static_call = StaticCallInstr::FromCall(Z, call, target);
StaticCallInstr* static_call =
StaticCallInstr::FromCall(Z, call, target, call->CallCount());
static_call->SetResultType(Z, CompileType::FromCid(kTypeCid));
call->ReplaceWith(static_call, current_iterator());
return true;
@ -849,7 +850,8 @@ void AotCallSpecializer::VisitInstanceCall(InstanceCallInstr* instr) {
CallTargets* targets = CallTargets::Create(Z, unary_checks);
ASSERT(targets->HasSingleTarget());
const Function& target = targets->FirstTarget();
StaticCallInstr* call = StaticCallInstr::FromCall(Z, instr, target);
StaticCallInstr* call = StaticCallInstr::FromCall(
Z, instr, target, targets->AggregateCallCount());
instr->ReplaceWith(call, current_iterator());
return;
}
@ -911,7 +913,8 @@ void AotCallSpecializer::VisitInstanceCall(InstanceCallInstr* instr) {
Function::Handle(Z, instr->ResolveForReceiverClass(receiver_class));
if (!function.IsNull()) {
const Function& target = Function::ZoneHandle(Z, function.raw());
StaticCallInstr* call = StaticCallInstr::FromCall(Z, instr, target);
StaticCallInstr* call =
StaticCallInstr::FromCall(Z, instr, target, instr->CallCount());
instr->ReplaceWith(call, current_iterator());
return;
}
@ -1024,7 +1027,8 @@ void AotCallSpecializer::VisitInstanceCall(InstanceCallInstr* instr) {
// We have computed that there is only a single target for this call
// within the whole hierarchy. Replace InstanceCall with StaticCall.
const Function& target = Function::ZoneHandle(Z, single_target.raw());
StaticCallInstr* call = StaticCallInstr::FromCall(Z, instr, target);
StaticCallInstr* call =
StaticCallInstr::FromCall(Z, instr, target, instr->CallCount());
instr->ReplaceWith(call, current_iterator());
return;
} else if ((ic_data.raw() != ICData::null()) &&
@ -1181,7 +1185,8 @@ void AotCallSpecializer::VisitPolymorphicInstanceCall(
Z, call->instance_call()->ResolveForReceiverClass(receiver_class));
if (!function.IsNull()) {
// Only one target. Replace by static call.
StaticCallInstr* new_call = StaticCallInstr::FromCall(Z, call, function);
StaticCallInstr* new_call =
StaticCallInstr::FromCall(Z, call, function, call->CallCount());
call->ReplaceWith(new_call, current_iterator());
}
}

54
runtime/vm/compiler/assembler/assembler_arm.cc
View file

@ -3393,9 +3393,9 @@ void Assembler::LeaveStubFrame() {
LeaveDartFrame();
}
// R0 receiver, R9 guarded cid as Smi.
// R0 receiver, R9 ICData entries array
// Preserve R4 (ARGS_DESC_REG), not required today, but maybe later.
void Assembler::MonomorphicCheckedEntry() {
void Assembler::MonomorphicCheckedEntryJIT() {
has_single_entry_point_ = false;
#if defined(TESTING) || defined(DEBUG)
bool saved_use_far_branches = use_far_branches();
@ -3404,19 +3404,65 @@ void Assembler::MonomorphicCheckedEntry() {
intptr_t start = CodeSize();
Comment("MonomorphicCheckedEntry");
ASSERT(CodeSize() - start == Instructions::kPolymorphicEntryOffset);
ASSERT(CodeSize() - start == Instructions::kMonomorphicEntryOffsetJIT);
const intptr_t cid_offset = target::Array::element_offset(0);
const intptr_t count_offset = target::Array::element_offset(1);
// Sadly this cannot use ldm because ldm takes no offset.
ldr(R1, FieldAddress(R9, cid_offset));
ldr(R2, FieldAddress(R9, count_offset));
LoadClassIdMayBeSmi(IP, R0);
add(R2, R2, Operand(target::ToRawSmi(1)));
cmp(R1, Operand(IP, LSL, 1));
Branch(Address(THR, Thread::monomorphic_miss_entry_offset()), NE);
str(R2, FieldAddress(R9, count_offset));
LoadImmediate(R4, 0); // GC-safe for OptimizeInvokedFunction.
// Fall through to unchecked entry.
ASSERT(CodeSize() - start == Instructions::kPolymorphicEntryOffsetJIT);
#if defined(TESTING) || defined(DEBUG)
set_use_far_branches(saved_use_far_branches);
#endif
}
// R0 receiver, R9 guarded cid as Smi.
// Preserve R4 (ARGS_DESC_REG), not required today, but maybe later.
void Assembler::MonomorphicCheckedEntryAOT() {
has_single_entry_point_ = false;
#if defined(TESTING) || defined(DEBUG)
bool saved_use_far_branches = use_far_branches();
set_use_far_branches(false);
#endif
intptr_t start = CodeSize();
Comment("MonomorphicCheckedEntry");
ASSERT(CodeSize() - start == Instructions::kMonomorphicEntryOffsetAOT);
LoadClassIdMayBeSmi(IP, R0);
cmp(R9, Operand(IP, LSL, 1));
Branch(Address(THR, Thread::monomorphic_miss_entry_offset()), NE);
// Fall through to unchecked entry.
ASSERT(CodeSize() - start == Instructions::kMonomorphicEntryOffset);
ASSERT(CodeSize() - start == Instructions::kPolymorphicEntryOffsetAOT);
#if defined(TESTING) || defined(DEBUG)
set_use_far_branches(saved_use_far_branches);
#endif
}
void Assembler::BranchOnMonomorphicCheckedEntryJIT(Label* label) {
has_single_entry_point_ = false;
while (CodeSize() < Instructions::kMonomorphicEntryOffsetJIT) {
bkpt(0);
}
b(label);
while (CodeSize() < Instructions::kPolymorphicEntryOffsetJIT) {
bkpt(0);
}
}
#ifndef PRODUCT
void Assembler::MaybeTraceAllocation(Register stats_addr_reg, Label* trace) {
ASSERT(stats_addr_reg != kNoRegister);

4
runtime/vm/compiler/assembler/assembler_arm.h
View file

@ -1058,7 +1058,9 @@ class Assembler : public AssemblerBase {
void EnterStubFrame();
void LeaveStubFrame();
void MonomorphicCheckedEntry();
void MonomorphicCheckedEntryJIT();
void MonomorphicCheckedEntryAOT();
void BranchOnMonomorphicCheckedEntryJIT(Label* label);
// The register into which the allocation stats table is loaded with
// LoadAllocationStatsAddress should be passed to MaybeTraceAllocation and

52
runtime/vm/compiler/assembler/assembler_arm64.cc
View file

@ -1461,9 +1461,44 @@ void Assembler::LeaveStubFrame() {
LeaveDartFrame();
}
// R0 receiver, R5 ICData entries array
// Preserve R4 (ARGS_DESC_REG), not required today, but maybe later.
void Assembler::MonomorphicCheckedEntryJIT() {
ASSERT(has_single_entry_point_);
has_single_entry_point_ = false;
const bool saved_use_far_branches = use_far_branches();
set_use_far_branches(false);
Label immediate, miss;
Bind(&miss);
ldr(IP0, Address(THR, Thread::monomorphic_miss_entry_offset()));
br(IP0);
Comment("MonomorphicCheckedEntry");
ASSERT(CodeSize() == Instructions::kMonomorphicEntryOffsetJIT);
const intptr_t cid_offset = target::Array::element_offset(0);
const intptr_t count_offset = target::Array::element_offset(1);
// Sadly this cannot use ldp because ldp requires aligned offsets.
ldr(R1, FieldAddress(R5, cid_offset));
ldr(R2, FieldAddress(R5, count_offset));
LoadClassIdMayBeSmi(IP0, R0);
add(R2, R2, Operand(target::ToRawSmi(1)));
cmp(R1, Operand(IP0, LSL, 1));
b(&miss, NE);
str(R2, FieldAddress(R5, count_offset));
LoadImmediate(R4, 0); // GC-safe for OptimizeInvokedFunction.
// Fall through to unchecked entry.
ASSERT(CodeSize() == Instructions::kPolymorphicEntryOffsetJIT);
set_use_far_branches(saved_use_far_branches);
}
// R0 receiver, R5 guarded cid as Smi.
// Preserve R4 (ARGS_DESC_REG), not required today, but maybe later.
void Assembler::MonomorphicCheckedEntry() {
void Assembler::MonomorphicCheckedEntryAOT() {
ASSERT(has_single_entry_point_);
has_single_entry_point_ = false;
bool saved_use_far_branches = use_far_branches();
@ -1477,17 +1512,28 @@ void Assembler::MonomorphicCheckedEntry() {
br(IP0);
Comment("MonomorphicCheckedEntry");
ASSERT(CodeSize() - start == Instructions::kPolymorphicEntryOffset);
ASSERT(CodeSize() - start == Instructions::kMonomorphicEntryOffsetAOT);
LoadClassIdMayBeSmi(IP0, R0);
cmp(R5, Operand(IP0, LSL, 1));
b(&miss, NE);
// Fall through to unchecked entry.
ASSERT(CodeSize() - start == Instructions::kMonomorphicEntryOffset);
ASSERT(CodeSize() - start == Instructions::kPolymorphicEntryOffsetAOT);
set_use_far_branches(saved_use_far_branches);
}
void Assembler::BranchOnMonomorphicCheckedEntryJIT(Label* label) {
has_single_entry_point_ = false;
while (CodeSize() < Instructions::kMonomorphicEntryOffsetJIT) {
brk(0);
}
b(label);
while (CodeSize() < Instructions::kPolymorphicEntryOffsetJIT) {
brk(0);
}
}
#ifndef PRODUCT
void Assembler::MaybeTraceAllocation(intptr_t cid,
Register temp_reg,

4
runtime/vm/compiler/assembler/assembler_arm64.h
View file

@ -1561,7 +1561,9 @@ class Assembler : public AssemblerBase {
void EnterStubFrame();
void LeaveStubFrame();
void MonomorphicCheckedEntry();
void MonomorphicCheckedEntryJIT();
void MonomorphicCheckedEntryAOT();
void BranchOnMonomorphicCheckedEntryJIT(Label* label);
void UpdateAllocationStats(intptr_t cid);

3
runtime/vm/compiler/assembler/assembler_dbc.h
View file

@ -39,7 +39,8 @@ class Assembler : public AssemblerBase {
// Misc. functionality
intptr_t prologue_offset() const { return 0; }
void MonomorphicCheckedEntry() {}
void MonomorphicCheckedEntryJIT() {}
void MonomorphicCheckedEntryAOT() {}
// Debugging and bringup support.
void Stop(const char* message) override;

50
runtime/vm/compiler/assembler/assembler_ia32.cc
View file

@ -2097,6 +2097,56 @@ void Assembler::EmitEntryFrameVerification() {
#endif
}
// EBX receiver, ECX ICData entries array
// Preserve EDX (ARGS_DESC_REG), not required today, but maybe later.
void Assembler::MonomorphicCheckedEntryJIT() {
has_single_entry_point_ = false;
intptr_t start = CodeSize();
Label have_cid, miss;
Bind(&miss);
jmp(Address(THR, Thread::monomorphic_miss_entry_offset()));
Comment("MonomorphicCheckedEntry");
ASSERT(CodeSize() - start == Instructions::kMonomorphicEntryOffsetJIT);
const intptr_t cid_offset = target::Array::element_offset(0);
const intptr_t count_offset = target::Array::element_offset(1);
movl(EAX, Immediate(kSmiCid << 1));
testl(EBX, Immediate(kSmiTagMask));
j(ZERO, &have_cid, kNearJump);
LoadClassId(EAX, EBX);
SmiTag(EAX);
Bind(&have_cid);
// EAX: cid as Smi
cmpl(EAX, FieldAddress(ECX, cid_offset));
j(NOT_EQUAL, &miss, Assembler::kNearJump);
addl(FieldAddress(ECX, count_offset), Immediate(target::ToRawSmi(1)));
xorl(EDX, EDX); // GC-safe for OptimizeInvokedFunction.
nop(1);
// Fall through to unchecked entry.
ASSERT(CodeSize() - start == Instructions::kPolymorphicEntryOffsetJIT);
}
// EBX receiver, ECX guarded cid as Smi.
// Preserve EDX (ARGS_DESC_REG), not required today, but maybe later.
void Assembler::MonomorphicCheckedEntryAOT() {
UNIMPLEMENTED();
}
void Assembler::BranchOnMonomorphicCheckedEntryJIT(Label* label) {
has_single_entry_point_ = false;
while (CodeSize() < Instructions::kMonomorphicEntryOffsetJIT) {
int3();
}
jmp(label);
while (CodeSize() < Instructions::kPolymorphicEntryOffsetJIT) {
int3();
}
}
void Assembler::TransitionGeneratedToNative(Register destination_address,
Register new_exit_frame,
Register scratch) {

6
runtime/vm/compiler/assembler/assembler_ia32.h
View file

@ -646,7 +646,9 @@ class Assembler : public AssemblerBase {
void LeaveFrame();
void ReserveAlignedFrameSpace(intptr_t frame_space);
void MonomorphicCheckedEntry() {}
void MonomorphicCheckedEntryJIT();
void MonomorphicCheckedEntryAOT();
void BranchOnMonomorphicCheckedEntryJIT(Label* label);
// In debug mode, this generates code to check that:
// FP + kExitLinkSlotFromEntryFp == SP
@ -744,8 +746,6 @@ class Assembler : public AssemblerBase {
// Needs a temporary register.
void MoveMemoryToMemory(Address to, Address from, Register tmp);
bool has_single_entry_point() const { return true; }
// Set up a Dart frame on entry with a frame pointer and PC information to
// enable easy access to the RawInstruction object of code corresponding
// to this frame.

50
runtime/vm/compiler/assembler/assembler_x64.cc
View file

@ -1742,9 +1742,9 @@ void Assembler::LeaveStubFrame() {
LeaveDartFrame();
}
// RDX receiver, RBX guarded cid as Smi.
// RDX receiver, RBX ICData entries array
// Preserve R10 (ARGS_DESC_REG), not required today, but maybe later.
void Assembler::MonomorphicCheckedEntry() {
void Assembler::MonomorphicCheckedEntryJIT() {
has_single_entry_point_ = false;
intptr_t start = CodeSize();
Label have_cid, miss;
@ -1756,7 +1756,38 @@ void Assembler::MonomorphicCheckedEntry() {
nop(1);
Comment("MonomorphicCheckedEntry");
ASSERT(CodeSize() - start == Instructions::kPolymorphicEntryOffset);
ASSERT(CodeSize() - start == Instructions::kMonomorphicEntryOffsetJIT);
ASSERT((CodeSize() & kSmiTagMask) == kSmiTag);
const intptr_t cid_offset = target::Array::element_offset(0);
const intptr_t count_offset = target::Array::element_offset(1);
LoadTaggedClassIdMayBeSmi(RAX, RDX);
cmpq(RAX, FieldAddress(RBX, cid_offset));
j(NOT_EQUAL, &miss, Assembler::kNearJump);
addl(FieldAddress(RBX, count_offset), Immediate(target::ToRawSmi(1)));
xorq(R10, R10); // GC-safe for OptimizeInvokedFunction.
nop(1);
// Fall through to unchecked entry.
ASSERT(CodeSize() - start == Instructions::kPolymorphicEntryOffsetJIT);
ASSERT(((CodeSize() - start) & kSmiTagMask) == kSmiTag);
}
void Assembler::MonomorphicCheckedEntryAOT() {
has_single_entry_point_ = false;
intptr_t start = CodeSize();
Label have_cid, miss;
Bind(&miss);
jmp(Address(THR, Thread::monomorphic_miss_entry_offset()));
// Ensure the monomorphic entry is 2-byte aligned (so GC can see them if we
// store them in ICData / MegamorphicCache arrays)
nop(1);
Comment("MonomorphicCheckedEntry");
ASSERT(CodeSize() - start == Instructions::kMonomorphicEntryOffsetAOT);
ASSERT((CodeSize() & kSmiTagMask) == kSmiTag);
movq(RAX, Immediate(kSmiCid));
@ -1774,10 +1805,21 @@ void Assembler::MonomorphicCheckedEntry() {
nop(1);
// Fall through to unchecked entry.
ASSERT(CodeSize() - start == Instructions::kMonomorphicEntryOffset);
ASSERT(CodeSize() - start == Instructions::kPolymorphicEntryOffsetAOT);
ASSERT(((CodeSize() - start) & kSmiTagMask) == kSmiTag);
}
void Assembler::BranchOnMonomorphicCheckedEntryJIT(Label* label) {
has_single_entry_point_ = false;
while (CodeSize() < Instructions::kMonomorphicEntryOffsetJIT) {
int3();
}
jmp(label);
while (CodeSize() < Instructions::kPolymorphicEntryOffsetJIT) {
int3();
}
}
#ifndef PRODUCT
void Assembler::MaybeTraceAllocation(intptr_t cid,
Label* trace,

4
runtime/vm/compiler/assembler/assembler_x64.h
View file

@ -885,7 +885,9 @@ class Assembler : public AssemblerBase {
void EnterStubFrame();
void LeaveStubFrame();
void MonomorphicCheckedEntry();
void MonomorphicCheckedEntryJIT();
void MonomorphicCheckedEntryAOT();
void BranchOnMonomorphicCheckedEntryJIT(Label* label);
void UpdateAllocationStats(intptr_t cid);

11
runtime/vm/compiler/backend/flow_graph_compiler.cc
View file

@ -339,7 +339,8 @@ intptr_t FlowGraphCompiler::UncheckedEntryOffset() const {
// Intrinsification happened.
if (parsed_function().function().IsDynamicFunction()) {
return Instructions::kMonomorphicEntryOffset;
return FLAG_precompiled_mode ? Instructions::kPolymorphicEntryOffsetAOT
: Instructions::kPolymorphicEntryOffsetJIT;
}
return 0;
@ -1360,7 +1361,7 @@ void FlowGraphCompiler::GenerateInstanceCall(intptr_t deopt_id,
if (FLAG_precompiled_mode) {
// TODO(#34162): Support unchecked entry-points in precompiled mode.
ic_data = ic_data.AsUnaryClassChecks();
EmitSwitchableInstanceCall(ic_data, deopt_id, token_pos, locs, entry_kind);
EmitInstanceCallAOT(ic_data, deopt_id, token_pos, locs, entry_kind);
return;
}
ASSERT(!ic_data.IsNull());
@ -1382,8 +1383,8 @@ void FlowGraphCompiler::GenerateInstanceCall(intptr_t deopt_id,
return;
}
EmitInstanceCall(StubEntryFor(ic_data, /*optimized=*/false), ic_data,
deopt_id, token_pos, locs);
EmitInstanceCallJIT(StubEntryFor(ic_data, /*optimized=*/false), ic_data,
deopt_id, token_pos, locs, entry_kind);
}
void FlowGraphCompiler::GenerateStaticCall(intptr_t deopt_id,
@ -2087,7 +2088,7 @@ void FlowGraphCompiler::EmitPolymorphicInstanceCall(
zone(), original_call.ic_data()->AsUnaryClassChecks());
// TODO(sjindel/entrypoints): Support skiping type checks on switchable
// calls.
EmitSwitchableInstanceCall(unary_checks, deopt_id, token_pos, locs);
EmitInstanceCallAOT(unary_checks, deopt_id, token_pos, locs);
}
}
}

13
runtime/vm/compiler/backend/flow_graph_compiler.h
View file

@ -616,11 +616,12 @@ class FlowGraphCompiler : public ValueObject {
LocationSummary* locs,
Code::EntryKind entry_kind = Code::EntryKind::kNormal);
void EmitInstanceCall(const Code& stub,
const ICData& ic_data,
intptr_t deopt_id,
TokenPosition token_pos,
LocationSummary* locs);
void EmitInstanceCallJIT(const Code& stub,
const ICData& ic_data,
intptr_t deopt_id,
TokenPosition token_pos,
LocationSummary* locs,
Code::EntryKind entry_kind);
void EmitPolymorphicInstanceCall(
const CallTargets& targets,
@ -641,7 +642,7 @@ class FlowGraphCompiler : public ValueObject {
intptr_t try_index,
intptr_t slow_path_argument_count = 0);
void EmitSwitchableInstanceCall(
void EmitInstanceCallAOT(
const ICData& ic_data,
intptr_t deopt_id,
TokenPosition token_pos,

35
runtime/vm/compiler/backend/flow_graph_compiler_arm.cc
View file

@ -1066,17 +1066,26 @@ void FlowGraphCompiler::EmitOptimizedInstanceCall(const Code& stub,
__ Drop(ic_data.CountWithTypeArgs());
}
void FlowGraphCompiler::EmitInstanceCall(const Code& stub,
const ICData& ic_data,
intptr_t deopt_id,
TokenPosition token_pos,
LocationSummary* locs) {
void FlowGraphCompiler::EmitInstanceCallJIT(const Code& stub,
const ICData& ic_data,
intptr_t deopt_id,
TokenPosition token_pos,
LocationSummary* locs,
Code::EntryKind entry_kind) {
ASSERT(entry_kind == Code::EntryKind::kNormal ||
entry_kind == Code::EntryKind::kUnchecked);
ASSERT(Array::Handle(zone(), ic_data.arguments_descriptor()).Length() > 0);
__ LoadFromOffset(kWord, R0, SP,
(ic_data.CountWithoutTypeArgs() - 1) * kWordSize);
__ LoadUniqueObject(R9, ic_data);
GenerateDartCall(deopt_id, token_pos, stub, RawPcDescriptors::kIcCall, locs,
Code::EntryKind::kMonomorphic);
__ LoadUniqueObject(CODE_REG, stub);
const intptr_t entry_point_offset =
entry_kind == Code::EntryKind::kNormal
? Code::entry_point_offset(Code::EntryKind::kMonomorphic)
: Code::entry_point_offset(Code::EntryKind::kMonomorphicUnchecked);
__ ldr(LR, FieldAddress(CODE_REG, entry_point_offset));
__ blx(LR);
EmitCallsiteMetadata(token_pos, deopt_id, RawPcDescriptors::kIcCall, locs);
__ Drop(ic_data.CountWithTypeArgs());
}
@ -1129,17 +1138,17 @@ void FlowGraphCompiler::EmitMegamorphicInstanceCall(
__ Drop(args_desc.CountWithTypeArgs());
}
void FlowGraphCompiler::EmitSwitchableInstanceCall(const ICData& ic_data,
intptr_t deopt_id,
TokenPosition token_pos,
LocationSummary* locs,
Code::EntryKind entry_kind) {
void FlowGraphCompiler::EmitInstanceCallAOT(const ICData& ic_data,
intptr_t deopt_id,
TokenPosition token_pos,
LocationSummary* locs,
Code::EntryKind entry_kind) {
ASSERT(entry_kind == Code::EntryKind::kNormal ||
entry_kind == Code::EntryKind::kUnchecked);
ASSERT(ic_data.NumArgsTested() == 1);
const Code& initial_stub = StubCode::ICCallThroughFunction();
__ Comment("SwitchableCall");
__ Comment("InstanceCallAOT");
__ LoadFromOffset(
kWord, R0, SP,
(ic_data.CountWithoutTypeArgs() - 1) * compiler::target::kWordSize);

46
runtime/vm/compiler/backend/flow_graph_compiler_arm64.cc
View file

@ -1032,16 +1032,32 @@ void FlowGraphCompiler::EmitOptimizedInstanceCall(const Code& stub,
__ Drop(ic_data.CountWithTypeArgs());
}
void FlowGraphCompiler::EmitInstanceCall(const Code& stub,
const ICData& ic_data,
intptr_t deopt_id,
TokenPosition token_pos,
LocationSummary* locs) {
void FlowGraphCompiler::EmitInstanceCallJIT(const Code& stub,
const ICData& ic_data,
intptr_t deopt_id,
TokenPosition token_pos,
LocationSummary* locs,
Code::EntryKind entry_kind) {
ASSERT(entry_kind == Code::EntryKind::kNormal ||
entry_kind == Code::EntryKind::kUnchecked);
ASSERT(Array::Handle(zone(), ic_data.arguments_descriptor()).Length() > 0);
__ LoadFromOffset(R0, SP, (ic_data.CountWithoutTypeArgs() - 1) * kWordSize);
__ LoadUniqueObject(R5, ic_data);
GenerateDartCall(deopt_id, token_pos, stub, RawPcDescriptors::kIcCall, locs,
Code::EntryKind::kMonomorphic);
ObjectPoolBuilder& op = __ object_pool_builder();
const intptr_t ic_data_index =
op.AddObject(ic_data, ObjectPool::Patchability::kPatchable);
const intptr_t stub_index =
op.AddObject(stub, ObjectPool::Patchability::kPatchable);
ASSERT((ic_data_index + 1) == stub_index);
__ LoadDoubleWordFromPoolOffset(R5, CODE_REG,
ObjectPool::element_offset(ic_data_index));
const intptr_t entry_point_offset =
entry_kind == Code::EntryKind::kNormal
? Code::entry_point_offset(Code::EntryKind::kMonomorphic)
: Code::entry_point_offset(Code::EntryKind::kMonomorphicUnchecked);
__ ldr(LR, FieldAddress(CODE_REG, entry_point_offset));
__ blr(LR);
EmitCallsiteMetadata(token_pos, deopt_id, RawPcDescriptors::kIcCall, locs);
__ Drop(ic_data.CountWithTypeArgs());
}
@ -1090,18 +1106,18 @@ void FlowGraphCompiler::EmitMegamorphicInstanceCall(
__ Drop(args_desc.CountWithTypeArgs());
}
void FlowGraphCompiler::EmitSwitchableInstanceCall(const ICData& ic_data,
intptr_t deopt_id,
TokenPosition token_pos,
LocationSummary* locs,
Code::EntryKind entry_kind) {
void FlowGraphCompiler::EmitInstanceCallAOT(const ICData& ic_data,
intptr_t deopt_id,
TokenPosition token_pos,
LocationSummary* locs,
Code::EntryKind entry_kind) {
// TODO(34162): Support multiple entry-points on ARM64.
ASSERT(ic_data.NumArgsTested() == 1);
const Code& initial_stub = StubCode::ICCallThroughFunction();
auto& op = __ object_pool_builder();
ObjectPoolBuilder& op = __ object_pool_builder();
__ Comment("SwitchableCall");
__ Comment("InstanceCallAOT");
__ LoadFromOffset(R0, SP, (ic_data.CountWithoutTypeArgs() - 1) * kWordSize);
const intptr_t ic_data_index =

34
runtime/vm/compiler/backend/flow_graph_compiler_ia32.cc
View file

@ -922,17 +922,25 @@ void FlowGraphCompiler::EmitOptimizedInstanceCall(const Code& stub,
__ Drop(ic_data.CountWithTypeArgs());
}
void FlowGraphCompiler::EmitInstanceCall(const Code& stub,
const ICData& ic_data,
intptr_t deopt_id,
TokenPosition token_pos,
LocationSummary* locs) {
void FlowGraphCompiler::EmitInstanceCallJIT(const Code& stub,
const ICData& ic_data,
intptr_t deopt_id,
TokenPosition token_pos,
LocationSummary* locs,
Code::EntryKind entry_kind) {
ASSERT(entry_kind == Code::EntryKind::kNormal ||
entry_kind == Code::EntryKind::kUnchecked);
ASSERT(Array::Handle(ic_data.arguments_descriptor()).Length() > 0);
// Load receiver into EBX.
__ movl(EBX, Address(ESP, (ic_data.CountWithoutTypeArgs() - 1) * kWordSize));
__ LoadObject(ECX, ic_data);
GenerateDartCall(deopt_id, token_pos, stub, RawPcDescriptors::kIcCall, locs,
Code::EntryKind::kMonomorphic);
__ LoadObject(ECX, ic_data, true);
__ LoadObject(CODE_REG, stub, true);
const intptr_t entry_point_offset =
entry_kind == Code::EntryKind::kNormal
? Code::entry_point_offset(Code::EntryKind::kMonomorphic)
: Code::entry_point_offset(Code::EntryKind::kMonomorphicUnchecked);
__ call(FieldAddress(CODE_REG, entry_point_offset));
EmitCallsiteMetadata(token_pos, deopt_id, RawPcDescriptors::kIcCall, locs);
__ Drop(ic_data.CountWithTypeArgs());
}
@ -972,11 +980,11 @@ void FlowGraphCompiler::EmitMegamorphicInstanceCall(
__ Drop(args_desc.CountWithTypeArgs());
}
void FlowGraphCompiler::EmitSwitchableInstanceCall(const ICData& ic_data,
intptr_t deopt_id,
TokenPosition token_pos,
LocationSummary* locs,
Code::EntryKind entry_kind) {
void FlowGraphCompiler::EmitInstanceCallAOT(const ICData& ic_data,
intptr_t deopt_id,
TokenPosition token_pos,
LocationSummary* locs,
Code::EntryKind entry_kind) {
// Only generated with precompilation.
UNREACHABLE();
}

36
runtime/vm/compiler/backend/flow_graph_compiler_x64.cc
View file

@ -1051,17 +1051,25 @@ void FlowGraphCompiler::EmitOptimizedInstanceCall(const Code& stub,
__ Drop(ic_data.CountWithTypeArgs(), RCX);
}
void FlowGraphCompiler::EmitInstanceCall(const Code& stub,
const ICData& ic_data,
intptr_t deopt_id,
TokenPosition token_pos,
LocationSummary* locs) {
void FlowGraphCompiler::EmitInstanceCallJIT(const Code& stub,
const ICData& ic_data,
intptr_t deopt_id,
TokenPosition token_pos,
LocationSummary* locs,
Code::EntryKind entry_kind) {
ASSERT(entry_kind == Code::EntryKind::kNormal ||
entry_kind == Code::EntryKind::kUnchecked);
ASSERT(Array::Handle(zone(), ic_data.arguments_descriptor()).Length() > 0);
// Load receiver into RDX.
__ movq(RDX, Address(RSP, (ic_data.CountWithoutTypeArgs() - 1) * kWordSize));
__ LoadUniqueObject(RBX, ic_data);
GenerateDartCall(deopt_id, token_pos, stub, RawPcDescriptors::kIcCall, locs,
Code::EntryKind::kMonomorphic);
__ LoadUniqueObject(CODE_REG, stub);
const intptr_t entry_point_offset =
entry_kind == Code::EntryKind::kNormal
? Code::entry_point_offset(Code::EntryKind::kMonomorphic)
: Code::entry_point_offset(Code::EntryKind::kMonomorphicUnchecked);
__ call(FieldAddress(CODE_REG, entry_point_offset));
EmitCallsiteMetadata(token_pos, deopt_id, RawPcDescriptors::kIcCall, locs);
__ Drop(ic_data.CountWithTypeArgs(), RCX);
}
@ -1107,24 +1115,24 @@ void FlowGraphCompiler::EmitMegamorphicInstanceCall(
__ Drop(args_desc.CountWithTypeArgs(), RCX);
}
void FlowGraphCompiler::EmitSwitchableInstanceCall(const ICData& ic_data,
intptr_t deopt_id,
TokenPosition token_pos,
LocationSummary* locs,
Code::EntryKind entry_kind) {
void FlowGraphCompiler::EmitInstanceCallAOT(const ICData& ic_data,
intptr_t deopt_id,
TokenPosition token_pos,
LocationSummary* locs,
Code::EntryKind entry_kind) {
ASSERT(entry_kind == Code::EntryKind::kNormal ||
entry_kind == Code::EntryKind::kUnchecked);
ASSERT(ic_data.NumArgsTested() == 1);
const Code& initial_stub = StubCode::ICCallThroughFunction();
__ Comment("SwitchableCall");
__ Comment("InstanceCallAOT");
__ movq(RDX, Address(RSP, (ic_data.CountWithoutTypeArgs() - 1) * kWordSize));
if (FLAG_precompiled_mode && FLAG_use_bare_instructions) {
// The AOT runtime will replace the slot in the object pool with the
// entrypoint address - see clustered_snapshot.cc.
__ LoadUniqueObject(RCX, initial_stub);
} else {
intptr_t entry_point_offset =
const intptr_t entry_point_offset =
entry_kind == Code::EntryKind::kNormal
? Code::entry_point_offset(Code::EntryKind::kMonomorphic)
: Code::entry_point_offset(Code::EntryKind::kMonomorphicUnchecked);

67
runtime/vm/compiler/backend/il.cc
View file

@ -723,6 +723,23 @@ Cids* Cids::CreateAndExpand(Zone* zone,
return cids;
}
static intptr_t Usage(const Function& function) {
intptr_t count = function.usage_counter();
if (count < 0) {
if (function.HasCode()) {
// 'function' is queued for optimized compilation
count = FLAG_optimization_counter_threshold;
} else {
// 'function' is queued for unoptimized compilation
count = FLAG_compilation_counter_threshold;
}
} else if (Code::IsOptimized(function.CurrentCode())) {
// 'function' was optimized and stopped counting
count = FLAG_optimization_counter_threshold;
}
return count;
}
void Cids::CreateHelper(Zone* zone,
const ICData& ic_data,
int argument_number,
@ -748,12 +765,36 @@ void Cids::CreateHelper(Zone* zone,
}
if (include_targets) {
Function& function = Function::ZoneHandle(zone, ic_data.GetTargetAt(i));
cid_ranges_.Add(new (zone) TargetInfo(
id, id, &function, ic_data.GetCountAt(i), ic_data.GetExactnessAt(i)));
intptr_t count = ic_data.GetCountAt(i);
cid_ranges_.Add(new (zone) TargetInfo(id, id, &function, count,
ic_data.GetExactnessAt(i)));
} else {
cid_ranges_.Add(new (zone) CidRange(id, id));
}
}
if (ic_data.is_megamorphic()) {
const MegamorphicCache& cache =
MegamorphicCache::Handle(zone, ic_data.AsMegamorphicCache());
SafepointMutexLocker ml(Isolate::Current()->megamorphic_lookup_mutex());
MegamorphicCacheEntries entries(Array::Handle(zone, cache.buckets()));
for (intptr_t i = 0; i < entries.Length(); i++) {
const intptr_t id =
Smi::Value(entries[i].Get<MegamorphicCache::kClassIdIndex>());
if (id == kIllegalCid) {
continue;
}
if (include_targets) {
Function& function = Function::ZoneHandle(zone);
function ^= entries[i].Get<MegamorphicCache::kTargetFunctionIndex>();
cid_ranges_.Add(new (zone) TargetInfo(
id, id, &function, Usage(function) / cache.filled_entry_count(),
StaticTypeExactnessState::NotTracking()));
} else {
cid_ranges_.Add(new (zone) CidRange(id, id));
}
}
}
}
bool Cids::IsMonomorphic() const {
@ -3824,6 +3865,14 @@ void CallTargets::MergeIntoRanges() {
Sort(OrderByFrequency);
}
void CallTargets::Print() const {
for (intptr_t i = 0; i < length(); i++) {
THR_Print("cid = [%" Pd ", %" Pd "], count = %" Pd ", target = %s\n",
TargetAt(i)->cid_start, TargetAt(i)->cid_end, TargetAt(i)->count,
TargetAt(i)->target->ToQualifiedCString());
}
}
// Shared code generation methods (EmitNativeCode and
// MakeLocationSummary). Only assembly code that can be shared across all
// architectures can be used. Machine specific register allocation and code
@ -3914,7 +3963,11 @@ void FunctionEntryInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
const Function& function = compiler->parsed_function().function();
if (function.IsDynamicFunction()) {
compiler->SpecialStatsBegin(CombinedCodeStatistics::kTagCheckedEntry);
__ MonomorphicCheckedEntry();
if (!FLAG_precompiled_mode) {
__ MonomorphicCheckedEntryJIT();
} else {
__ MonomorphicCheckedEntryAOT();
}
compiler->SpecialStatsEnd(CombinedCodeStatistics::kTagCheckedEntry);
}
@ -4349,8 +4402,8 @@ void InstanceCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
}
if (is_smi_two_args_op) {
ASSERT(ArgumentCount() == 2);
compiler->EmitInstanceCall(stub, *call_ic_data, deopt_id(), token_pos(),
locs());
compiler->EmitInstanceCallJIT(stub, *call_ic_data, deopt_id(),
token_pos(), locs(), entry_kind());
} else {
compiler->GenerateInstanceCall(deopt_id(), token_pos(), locs(),
*call_ic_data);
@ -4532,8 +4585,8 @@ Definition* InstanceCallInstr::Canonicalize(FlowGraph* flow_graph) {
ASSERT(new_target->HasSingleTarget());
const Function& target = new_target->FirstTarget();
StaticCallInstr* specialized =
StaticCallInstr::FromCall(flow_graph->zone(), this, target);
StaticCallInstr* specialized = StaticCallInstr::FromCall(
flow_graph->zone(), this, target, new_target->AggregateCallCount());
flow_graph->InsertBefore(this, specialized, env(), FlowGraph::kValue);
return specialized;
}

7
runtime/vm/compiler/backend/il.h
View file

@ -622,6 +622,8 @@ class CallTargets : public Cids {
const Function& FirstTarget() const;
const Function& MostPopularTarget() const;
void Print() const;
private:
void MergeIntoRanges();
};
@ -3982,7 +3984,8 @@ class StaticCallInstr : public TemplateDartCall<0> {
template <class C>
static StaticCallInstr* FromCall(Zone* zone,
const C* call,
const Function& target) {
const Function& target,
intptr_t call_count) {
PushArgumentsArray* args =
new (zone) PushArgumentsArray(call->ArgumentCount());
for (intptr_t i = 0; i < call->ArgumentCount(); i++) {
@ -3991,7 +3994,7 @@ class StaticCallInstr : public TemplateDartCall<0> {
StaticCallInstr* new_call = new (zone)
StaticCallInstr(call->token_pos(), target, call->type_args_len(),
call->argument_names(), args, call->deopt_id(),
call->CallCount(), ICData::kNoRebind);
call_count, ICData::kNoRebind);
if (call->result_type() != NULL) {
new_call->result_type_ = call->result_type();
}

19
runtime/vm/compiler/backend/il_arm.cc
View file

@ -3262,7 +3262,7 @@ void CatchBlockEntryInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
LocationSummary* CheckStackOverflowInstr::MakeLocationSummary(Zone* zone,
bool opt) const {
const intptr_t kNumInputs = 0;
const intptr_t kNumTemps = 1;
const intptr_t kNumTemps = 2;
const bool using_shared_stub = UseSharedSlowPathStub(opt);
ASSERT((kReservedCpuRegisters & (1 << LR)) != 0);
LocationSummary* summary = new (zone)
@ -3270,6 +3270,7 @@ LocationSummary* CheckStackOverflowInstr::MakeLocationSummary(Zone* zone,
using_shared_stub ? LocationSummary::kCallOnSharedSlowPath
: LocationSummary::kCallOnSlowPath);
summary->set_temp(0, Location::RequiresRegister());
summary->set_temp(1, Location::RequiresRegister());
return summary;
}
@ -3377,16 +3378,22 @@ void CheckStackOverflowInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
compiler->AddSlowPathCode(slow_path);
__ b(slow_path->entry_label(), LS);
if (compiler->CanOSRFunction() && in_loop()) {
const Register temp = locs()->temp(0).reg();
const Register function = locs()->temp(0).reg();
const Register count = locs()->temp(1).reg();
// In unoptimized code check the usage counter to trigger OSR at loop
// stack checks. Use progressively higher thresholds for more deeply
// nested loops to attempt to hit outer loops with OSR when possible.
__ LoadObject(temp, compiler->parsed_function().function());
__ LoadObject(function, compiler->parsed_function().function());
intptr_t threshold =
FLAG_optimization_counter_threshold * (loop_depth() + 1);
__ ldr(temp, FieldAddress(
temp, compiler::target::Function::usage_counter_offset()));
__ CompareImmediate(temp, threshold);
__ ldr(count,
FieldAddress(function,
compiler::target::Function::usage_counter_offset()));
__ add(count, count, Operand(1));
__ str(count,
FieldAddress(function,
compiler::target::Function::usage_counter_offset()));
__ CompareImmediate(count, threshold);
__ b(slow_path->osr_entry_label(), GE);
}
if (compiler->ForceSlowPathForStackOverflow()) {

12
runtime/vm/compiler/backend/il_arm64.cc
View file

@ -3004,15 +3004,19 @@ void CheckStackOverflowInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
__ CompareRegisters(SP, TMP);
__ b(slow_path->entry_label(), LS);
if (compiler->CanOSRFunction() && in_loop()) {
const Register temp = locs()->temp(0).reg();
const Register function = locs()->temp(0).reg();
// In unoptimized code check the usage counter to trigger OSR at loop
// stack checks. Use progressively higher thresholds for more deeply
// nested loops to attempt to hit outer loops with OSR when possible.
__ LoadObject(temp, compiler->parsed_function().function());
__ LoadObject(function, compiler->parsed_function().function());
intptr_t threshold =
FLAG_optimization_counter_threshold * (loop_depth() + 1);
__ LoadFieldFromOffset(temp, temp, Function::usage_counter_offset(), kWord);
__ CompareImmediate(temp, threshold);
__ LoadFieldFromOffset(TMP, function, Function::usage_counter_offset(),
kWord);
__ add(TMP, TMP, Operand(1));
__ StoreFieldToOffset(TMP, function, Function::usage_counter_offset(),
kWord);
__ CompareImmediate(TMP, threshold);
__ b(slow_path->osr_entry_label(), GE);
}
if (compiler->ForceSlowPathForStackOverflow()) {

1
runtime/vm/compiler/backend/il_ia32.cc
View file

@ -2858,6 +2858,7 @@ void CheckStackOverflowInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
__ LoadObject(EDI, compiler->parsed_function().function());
intptr_t threshold =
FLAG_optimization_counter_threshold * (loop_depth() + 1);
__ incl(FieldAddress(EDI, Function::usage_counter_offset()));
__ cmpl(FieldAddress(EDI, Function::usage_counter_offset()),
Immediate(threshold));
__ j(GREATER_EQUAL, slow_path->osr_entry_label());

1
runtime/vm/compiler/backend/il_x64.cc
View file

@ -2989,6 +2989,7 @@ void CheckStackOverflowInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
__ LoadObject(temp, compiler->parsed_function().function());
int32_t threshold =
FLAG_optimization_counter_threshold * (loop_depth() + 1);
__ incl(FieldAddress(temp, Function::usage_counter_offset()));
__ cmpl(FieldAddress(temp, Function::usage_counter_offset()),
Immediate(threshold));
__ j(GREATER_EQUAL, slow_path->osr_entry_label());

3
runtime/vm/compiler/call_specializer.cc
View file

@ -314,7 +314,8 @@ void CallSpecializer::SpecializePolymorphicInstanceCall(
ASSERT(targets->HasSingleTarget());
const Function& target = targets->FirstTarget();
StaticCallInstr* specialized = StaticCallInstr::FromCall(Z, call, target);
StaticCallInstr* specialized =
StaticCallInstr::FromCall(Z, call, target, targets->AggregateCallCount());
call->ReplaceWith(specialized, current_iterator());
}

12
runtime/vm/compiler/jit/jit_call_specializer.cc
View file

@ -52,8 +52,10 @@ bool JitCallSpecializer::TryOptimizeStaticCallUsingStaticTypes(
void JitCallSpecializer::ReplaceWithStaticCall(InstanceCallInstr* instr,
const ICData& unary_checks,
const Function& target) {
StaticCallInstr* call = StaticCallInstr::FromCall(Z, instr, target);
const Function& target,
intptr_t call_count) {
StaticCallInstr* call =
StaticCallInstr::FromCall(Z, instr, target, call_count);
if (unary_checks.NumberOfChecks() == 1 &&
unary_checks.GetExactnessAt(0).IsExact()) {
if (unary_checks.GetExactnessAt(0).IsTriviallyExact()) {
@ -143,7 +145,8 @@ void JitCallSpecializer::VisitInstanceCall(InstanceCallInstr* instr) {
Function::ZoneHandle(Z, unary_checks.GetTargetAt(0));
if (flow_graph()->CheckForInstanceCall(instr, target.kind()) ==
FlowGraph::ToCheck::kNoCheck) {
ReplaceWithStaticCall(instr, unary_checks, target);
ReplaceWithStaticCall(instr, unary_checks, target,
targets.AggregateCallCount());
return;
}
}
@ -168,7 +171,8 @@ void JitCallSpecializer::VisitInstanceCall(InstanceCallInstr* instr) {
// Call can still deoptimize, do not detach environment from instr.
const Function& target =
Function::ZoneHandle(Z, unary_checks.GetTargetAt(0));
ReplaceWithStaticCall(instr, unary_checks, target);
ReplaceWithStaticCall(instr, unary_checks, target,
targets.AggregateCallCount());
} else {
PolymorphicInstanceCallInstr* call =
new (Z) PolymorphicInstanceCallInstr(instr, targets,

3
runtime/vm/compiler/jit/jit_call_specializer.h
View file

@ -37,7 +37,8 @@ class JitCallSpecializer : public CallSpecializer {
void ReplaceWithStaticCall(InstanceCallInstr* instr,
const ICData& unary_checks,
const Function& target);
const Function& target,
intptr_t call_count);
DISALLOW_COPY_AND_ASSIGN(JitCallSpecializer);
};

7
runtime/vm/compiler/runtime_api.h
View file

@ -676,7 +676,6 @@ class Thread : public AllStatic {
static word fix_allocation_stub_code_offset();
static word monomorphic_miss_stub_offset();
static word ic_lookup_through_code_stub_offset();
static word lazy_specialize_type_test_stub_offset();
static word slow_type_test_stub_offset();
static word call_to_runtime_stub_offset();
@ -775,8 +774,10 @@ class ClassHeapStats : public AllStatic {
class Instructions : public AllStatic {
public:
static const word kPolymorphicEntryOffset;
static const word kMonomorphicEntryOffset;
static const word kMonomorphicEntryOffsetJIT;
static const word kPolymorphicEntryOffsetJIT;
static const word kMonomorphicEntryOffsetAOT;
static const word kPolymorphicEntryOffsetAOT;
static word HeaderSize();
static word UnalignedHeaderSize();
};

320
runtime/vm/compiler/runtime_offsets_extracted.h
View file

@ -25,9 +25,13 @@ static constexpr dart::compiler::target::word Array_kMaxNewSpaceElements =
static constexpr dart::compiler::target::word ClassTable_kSizeOfClassPairLog2 =
3;
static constexpr dart::compiler::target::word
Instructions_kMonomorphicEntryOffset = 20;
Instructions_kMonomorphicEntryOffsetJIT = 0;
static constexpr dart::compiler::target::word
Instructions_kPolymorphicEntryOffset = 0;
Instructions_kPolymorphicEntryOffsetJIT = 40;
static constexpr dart::compiler::target::word
Instructions_kMonomorphicEntryOffsetAOT = 0;
static constexpr dart::compiler::target::word
Instructions_kPolymorphicEntryOffsetAOT = 20;
static constexpr dart::compiler::target::word HeapPage_kBytesPerCardLog2 = 9;
static constexpr dart::compiler::target::word
NativeEntry_kNumCallWrapperArguments = 2;
@ -187,7 +191,7 @@ static constexpr dart::compiler::target::word String_hash_offset = 8;
static constexpr dart::compiler::target::word String_length_offset = 4;
static constexpr dart::compiler::target::word SubtypeTestCache_cache_offset = 4;
static constexpr dart::compiler::target::word
Thread_AllocateArray_entry_point_offset = 284;
Thread_AllocateArray_entry_point_offset = 280;
static constexpr dart::compiler::target::word Thread_active_exception_offset =
612;
static constexpr dart::compiler::target::word Thread_active_stacktrace_offset =
@ -195,97 +199,95 @@ static constexpr dart::compiler::target::word Thread_active_stacktrace_offset =
static constexpr dart::compiler::target::word
Thread_array_write_barrier_code_offset = 112;
static constexpr dart::compiler::target::word
Thread_array_write_barrier_entry_point_offset = 196;
Thread_array_write_barrier_entry_point_offset = 192;
static constexpr dart::compiler::target::word Thread_async_stack_trace_offset =
84;
static constexpr dart::compiler::target::word
Thread_auto_scope_native_wrapper_entry_point_offset = 244;
Thread_auto_scope_native_wrapper_entry_point_offset = 240;
static constexpr dart::compiler::target::word Thread_bool_false_offset = 104;
static constexpr dart::compiler::target::word Thread_bool_true_offset = 100;
static constexpr dart::compiler::target::word
Thread_call_to_runtime_entry_point_offset = 200;
Thread_call_to_runtime_entry_point_offset = 196;
static constexpr dart::compiler::target::word
Thread_call_to_runtime_stub_offset = 132;
static constexpr dart::compiler::target::word Thread_dart_stream_offset = 644;
static constexpr dart::compiler::target::word Thread_deoptimize_entry_offset =
232;
228;
static constexpr dart::compiler::target::word Thread_deoptimize_stub_offset =
164;
160;
static constexpr dart::compiler::target::word Thread_double_abs_address_offset =
264;
260;
static constexpr dart::compiler::target::word
Thread_double_negate_address_offset = 260;
Thread_double_negate_address_offset = 256;
static constexpr dart::compiler::target::word Thread_end_offset = 60;
static constexpr dart::compiler::target::word
Thread_enter_safepoint_stub_offset = 184;
Thread_enter_safepoint_stub_offset = 180;
static constexpr dart::compiler::target::word Thread_execution_state_offset =
628;
static constexpr dart::compiler::target::word
Thread_exit_safepoint_stub_offset = 188;
Thread_exit_safepoint_stub_offset = 184;
static constexpr dart::compiler::target::word
Thread_fix_allocation_stub_code_offset = 120;
static constexpr dart::compiler::target::word
Thread_fix_callers_target_code_offset = 116;
static constexpr dart::compiler::target::word
Thread_float_absolute_address_offset = 276;
Thread_float_absolute_address_offset = 272;
static constexpr dart::compiler::target::word
Thread_float_negate_address_offset = 272;
Thread_float_negate_address_offset = 268;
static constexpr dart::compiler::target::word Thread_float_not_address_offset =
268;
264;
static constexpr dart::compiler::target::word
Thread_float_zerow_address_offset = 280;
Thread_float_zerow_address_offset = 276;
static constexpr dart::compiler::target::word Thread_global_object_pool_offset =
620;
static constexpr dart::compiler::target::word
Thread_ic_lookup_through_code_stub_offset = 156;
static constexpr dart::compiler::target::word
Thread_interpret_call_entry_point_offset = 248;
Thread_interpret_call_entry_point_offset = 244;
static constexpr dart::compiler::target::word
Thread_invoke_dart_code_from_bytecode_stub_offset = 128;
static constexpr dart::compiler::target::word
Thread_invoke_dart_code_stub_offset = 124;
static constexpr dart::compiler::target::word Thread_isolate_offset = 48;
static constexpr dart::compiler::target::word
Thread_lazy_deopt_from_return_stub_offset = 168;
Thread_lazy_deopt_from_return_stub_offset = 164;
static constexpr dart::compiler::target::word
Thread_lazy_deopt_from_throw_stub_offset = 172;
Thread_lazy_deopt_from_throw_stub_offset = 168;
static constexpr dart::compiler::target::word
Thread_lazy_specialize_type_test_stub_offset = 180;
Thread_lazy_specialize_type_test_stub_offset = 176;
static constexpr dart::compiler::target::word
Thread_marking_stack_block_offset = 72;
static constexpr dart::compiler::target::word
Thread_megamorphic_call_checked_entry_offset = 220;
Thread_megamorphic_call_checked_entry_offset = 216;
static constexpr dart::compiler::target::word
Thread_monomorphic_miss_entry_offset = 224;
Thread_monomorphic_miss_entry_offset = 220;
static constexpr dart::compiler::target::word
Thread_monomorphic_miss_stub_offset = 152;
static constexpr dart::compiler::target::word
Thread_no_scope_native_wrapper_entry_point_offset = 240;
Thread_no_scope_native_wrapper_entry_point_offset = 236;
static constexpr dart::compiler::target::word
Thread_null_error_shared_with_fpu_regs_entry_point_offset = 208;
Thread_null_error_shared_with_fpu_regs_entry_point_offset = 204;
static constexpr dart::compiler::target::word
Thread_null_error_shared_with_fpu_regs_stub_offset = 140;
static constexpr dart::compiler::target::word
Thread_null_error_shared_without_fpu_regs_entry_point_offset = 204;
Thread_null_error_shared_without_fpu_regs_entry_point_offset = 200;
static constexpr dart::compiler::target::word
Thread_null_error_shared_without_fpu_regs_stub_offset = 136;
static constexpr dart::compiler::target::word Thread_object_null_offset = 96;
static constexpr dart::compiler::target::word
Thread_predefined_symbols_address_offset = 252;
Thread_predefined_symbols_address_offset = 248;
static constexpr dart::compiler::target::word Thread_resume_pc_offset = 624;
static constexpr dart::compiler::target::word Thread_safepoint_state_offset =
632;
static constexpr dart::compiler::target::word
Thread_slow_type_test_stub_offset = 176;
Thread_slow_type_test_stub_offset = 172;
static constexpr dart::compiler::target::word Thread_stack_limit_offset = 36;
static constexpr dart::compiler::target::word
Thread_stack_overflow_flags_offset = 40;
static constexpr dart::compiler::target::word
Thread_stack_overflow_shared_with_fpu_regs_entry_point_offset = 216;
Thread_stack_overflow_shared_with_fpu_regs_entry_point_offset = 212;
static constexpr dart::compiler::target::word
Thread_stack_overflow_shared_with_fpu_regs_stub_offset = 148;
static constexpr dart::compiler::target::word
Thread_stack_overflow_shared_without_fpu_regs_entry_point_offset = 212;
Thread_stack_overflow_shared_without_fpu_regs_entry_point_offset = 208;
static constexpr dart::compiler::target::word
Thread_stack_overflow_shared_without_fpu_regs_stub_offset = 144;
static constexpr dart::compiler::target::word Thread_store_buffer_block_offset =
@ -300,11 +302,11 @@ static constexpr dart::compiler::target::word Thread_vm_tag_offset = 80;
static constexpr dart::compiler::target::word Thread_write_barrier_code_offset =
108;
static constexpr dart::compiler::target::word
Thread_write_barrier_entry_point_offset = 192;
Thread_write_barrier_entry_point_offset = 188;
static constexpr dart::compiler::target::word Thread_write_barrier_mask_offset =
44;
static constexpr dart::compiler::target::word
Thread_verify_callback_entry_offset = 236;
Thread_verify_callback_entry_offset = 232;
static constexpr dart::compiler::target::word Thread_callback_code_offset = 636;
static constexpr dart::compiler::target::word TimelineStream_enabled_offset = 8;
static constexpr dart::compiler::target::word TwoByteString_data_offset = 12;
@ -372,9 +374,13 @@ static constexpr dart::compiler::target::word Array_kMaxNewSpaceElements =
static constexpr dart::compiler::target::word ClassTable_kSizeOfClassPairLog2 =
4;
static constexpr dart::compiler::target::word
Instructions_kMonomorphicEntryOffset = 32;
Instructions_kMonomorphicEntryOffsetJIT = 8;
static constexpr dart::compiler::target::word
Instructions_kPolymorphicEntryOffset = 8;
Instructions_kPolymorphicEntryOffsetJIT = 40;
static constexpr dart::compiler::target::word
Instructions_kMonomorphicEntryOffsetAOT = 8;
static constexpr dart::compiler::target::word
Instructions_kPolymorphicEntryOffsetAOT = 32;
static constexpr dart::compiler::target::word HeapPage_kBytesPerCardLog2 = 10;
static constexpr dart::compiler::target::word
NativeEntry_kNumCallWrapperArguments = 2;
@ -535,7 +541,7 @@ static constexpr dart::compiler::target::word String_hash_offset = 4;
static constexpr dart::compiler::target::word String_length_offset = 8;
static constexpr dart::compiler::target::word SubtypeTestCache_cache_offset = 8;
static constexpr dart::compiler::target::word
Thread_AllocateArray_entry_point_offset = 560;
Thread_AllocateArray_entry_point_offset = 552;
static constexpr dart::compiler::target::word Thread_active_exception_offset =
1232;
static constexpr dart::compiler::target::word Thread_active_stacktrace_offset =
@ -543,97 +549,95 @@ static constexpr dart::compiler::target::word Thread_active_stacktrace_offset =
static constexpr dart::compiler::target::word
Thread_array_write_barrier_code_offset = 216;
static constexpr dart::compiler::target::word
Thread_array_write_barrier_entry_point_offset = 384;
Thread_array_write_barrier_entry_point_offset = 376;
static constexpr dart::compiler::target::word Thread_async_stack_trace_offset =
168;
static constexpr dart::compiler::target::word
Thread_auto_scope_native_wrapper_entry_point_offset = 480;
Thread_auto_scope_native_wrapper_entry_point_offset = 472;
static constexpr dart::compiler::target::word Thread_bool_false_offset = 200;
static constexpr dart::compiler::target::word Thread_bool_true_offset = 192;
static constexpr dart::compiler::target::word
Thread_call_to_runtime_entry_point_offset = 392;
Thread_call_to_runtime_entry_point_offset = 384;
static constexpr dart::compiler::target::word
Thread_call_to_runtime_stub_offset = 256;
static constexpr dart::compiler::target::word Thread_dart_stream_offset = 1296;
static constexpr dart::compiler::target::word Thread_deoptimize_entry_offset =
456;
448;
static constexpr dart::compiler::target::word Thread_deoptimize_stub_offset =
320;
312;
static constexpr dart::compiler::target::word Thread_double_abs_address_offset =
520;
512;
static constexpr dart::compiler::target::word
Thread_double_negate_address_offset = 512;
Thread_double_negate_address_offset = 504;
static constexpr dart::compiler::target::word Thread_end_offset = 120;
static constexpr dart::compiler::target::word
Thread_enter_safepoint_stub_offset = 360;
Thread_enter_safepoint_stub_offset = 352;
static constexpr dart::compiler::target::word Thread_execution_state_offset =
1264;
static constexpr dart::compiler::target::word
Thread_exit_safepoint_stub_offset = 368;
Thread_exit_safepoint_stub_offset = 360;
static constexpr dart::compiler::target::word
Thread_fix_allocation_stub_code_offset = 232;
static constexpr dart::compiler::target::word
Thread_fix_callers_target_code_offset = 224;
static constexpr dart::compiler::target::word
Thread_float_absolute_address_offset = 544;
Thread_float_absolute_address_offset = 536;
static constexpr dart::compiler::target::word
Thread_float_negate_address_offset = 536;
Thread_float_negate_address_offset = 528;
static constexpr dart::compiler::target::word Thread_float_not_address_offset =
528;
520;
static constexpr dart::compiler::target::word
Thread_float_zerow_address_offset = 552;
Thread_float_zerow_address_offset = 544;
static constexpr dart::compiler::target::word Thread_global_object_pool_offset =
1248;
static constexpr dart::compiler::target::word
Thread_ic_lookup_through_code_stub_offset = 304;
static constexpr dart::compiler::target::word
Thread_interpret_call_entry_point_offset = 488;
Thread_interpret_call_entry_point_offset = 480;
static constexpr dart::compiler::target::word
Thread_invoke_dart_code_from_bytecode_stub_offset = 248;
static constexpr dart::compiler::target::word
Thread_invoke_dart_code_stub_offset = 240;
static constexpr dart::compiler::target::word Thread_isolate_offset = 96;
static constexpr dart::compiler::target::word
Thread_lazy_deopt_from_return_stub_offset = 328;
Thread_lazy_deopt_from_return_stub_offset = 320;
static constexpr dart::compiler::target::word
Thread_lazy_deopt_from_throw_stub_offset = 336;
Thread_lazy_deopt_from_throw_stub_offset = 328;
static constexpr dart::compiler::target::word
Thread_lazy_specialize_type_test_stub_offset = 352;
Thread_lazy_specialize_type_test_stub_offset = 344;
static constexpr dart::compiler::target::word
Thread_marking_stack_block_offset = 144;
static constexpr dart::compiler::target::word
Thread_megamorphic_call_checked_entry_offset = 432;
Thread_megamorphic_call_checked_entry_offset = 424;
static constexpr dart::compiler::target::word
Thread_monomorphic_miss_entry_offset = 440;
Thread_monomorphic_miss_entry_offset = 432;
static constexpr dart::compiler::target::word
Thread_monomorphic_miss_stub_offset = 296;
static constexpr dart::compiler::target::word
Thread_no_scope_native_wrapper_entry_point_offset = 472;
Thread_no_scope_native_wrapper_entry_point_offset = 464;
static constexpr dart::compiler::target::word
Thread_null_error_shared_with_fpu_regs_entry_point_offset = 408;
Thread_null_error_shared_with_fpu_regs_entry_point_offset = 400;
static constexpr dart::compiler::target::word
Thread_null_error_shared_with_fpu_regs_stub_offset = 272;
static constexpr dart::compiler::target::word
Thread_null_error_shared_without_fpu_regs_entry_point_offset = 400;
Thread_null_error_shared_without_fpu_regs_entry_point_offset = 392;
static constexpr dart::compiler::target::word
Thread_null_error_shared_without_fpu_regs_stub_offset = 264;
static constexpr dart::compiler::target::word Thread_object_null_offset = 184;
static constexpr dart::compiler::target::word
Thread_predefined_symbols_address_offset = 496;
Thread_predefined_symbols_address_offset = 488;
static constexpr dart::compiler::target::word Thread_resume_pc_offset = 1256;
static constexpr dart::compiler::target::word Thread_safepoint_state_offset =
1272;
static constexpr dart::compiler::target::word
Thread_slow_type_test_stub_offset = 344;
Thread_slow_type_test_stub_offset = 336;
static constexpr dart::compiler::target::word Thread_stack_limit_offset = 72;
static constexpr dart::compiler::target::word
Thread_stack_overflow_flags_offset = 80;
static constexpr dart::compiler::target::word
Thread_stack_overflow_shared_with_fpu_regs_entry_point_offset = 424;
Thread_stack_overflow_shared_with_fpu_regs_entry_point_offset = 416;
static constexpr dart::compiler::target::word
Thread_stack_overflow_shared_with_fpu_regs_stub_offset = 288;
static constexpr dart::compiler::target::word
Thread_stack_overflow_shared_without_fpu_regs_entry_point_offset = 416;
Thread_stack_overflow_shared_without_fpu_regs_entry_point_offset = 408;
static constexpr dart::compiler::target::word
Thread_stack_overflow_shared_without_fpu_regs_stub_offset = 280;
static constexpr dart::compiler::target::word Thread_store_buffer_block_offset =
@ -648,11 +652,11 @@ static constexpr dart::compiler::target::word Thread_vm_tag_offset = 160;
static constexpr dart::compiler::target::word Thread_write_barrier_code_offset =
208;
static constexpr dart::compiler::target::word
Thread_write_barrier_entry_point_offset = 376;
Thread_write_barrier_entry_point_offset = 368;
static constexpr dart::compiler::target::word Thread_write_barrier_mask_offset =
88;
static constexpr dart::compiler::target::word
Thread_verify_callback_entry_offset = 464;
Thread_verify_callback_entry_offset = 456;
static constexpr dart::compiler::target::word Thread_callback_code_offset =
1280;
static constexpr dart::compiler::target::word TimelineStream_enabled_offset =
@ -722,9 +726,13 @@ static constexpr dart::compiler::target::word Array_kMaxNewSpaceElements =
static constexpr dart::compiler::target::word ClassTable_kSizeOfClassPairLog2 =
3;
static constexpr dart::compiler::target::word
Instructions_kMonomorphicEntryOffset = 0;
Instructions_kMonomorphicEntryOffsetJIT = 6;
static constexpr dart::compiler::target::word
Instructions_kPolymorphicEntryOffset = 0;
Instructions_kPolymorphicEntryOffsetJIT = 34;
static constexpr dart::compiler::target::word
Instructions_kMonomorphicEntryOffsetAOT = 0;
static constexpr dart::compiler::target::word
Instructions_kPolymorphicEntryOffsetAOT = 0;
static constexpr dart::compiler::target::word HeapPage_kBytesPerCardLog2 = 9;
static constexpr dart::compiler::target::word
NativeEntry_kNumCallWrapperArguments = 2;
@ -884,7 +892,7 @@ static constexpr dart::compiler::target::word String_hash_offset = 8;
static constexpr dart::compiler::target::word String_length_offset = 4;
static constexpr dart::compiler::target::word SubtypeTestCache_cache_offset = 4;
static constexpr dart::compiler::target::word
Thread_AllocateArray_entry_point_offset = 284;
Thread_AllocateArray_entry_point_offset = 280;
static constexpr dart::compiler::target::word Thread_active_exception_offset =
576;
static constexpr dart::compiler::target::word Thread_active_stacktrace_offset =
@ -892,97 +900,95 @@ static constexpr dart::compiler::target::word Thread_active_stacktrace_offset =
static constexpr dart::compiler::target::word
Thread_array_write_barrier_code_offset = 112;
static constexpr dart::compiler::target::word
Thread_array_write_barrier_entry_point_offset = 196;
Thread_array_write_barrier_entry_point_offset = 192;
static constexpr dart::compiler::target::word Thread_async_stack_trace_offset =
84;
static constexpr dart::compiler::target::word
Thread_auto_scope_native_wrapper_entry_point_offset = 244;
Thread_auto_scope_native_wrapper_entry_point_offset = 240;
static constexpr dart::compiler::target::word Thread_bool_false_offset = 104;
static constexpr dart::compiler::target::word Thread_bool_true_offset = 100;
static constexpr dart::compiler::target::word
Thread_call_to_runtime_entry_point_offset = 200;
Thread_call_to_runtime_entry_point_offset = 196;
static constexpr dart::compiler::target::word
Thread_call_to_runtime_stub_offset = 132;
static constexpr dart::compiler::target::word Thread_dart_stream_offset = 608;
static constexpr dart::compiler::target::word Thread_deoptimize_entry_offset =
232;
228;
static constexpr dart::compiler::target::word Thread_deoptimize_stub_offset =
164;
160;
static constexpr dart::compiler::target::word Thread_double_abs_address_offset =
264;
260;
static constexpr dart::compiler::target::word
Thread_double_negate_address_offset = 260;
Thread_double_negate_address_offset = 256;
static constexpr dart::compiler::target::word Thread_end_offset = 60;
static constexpr dart::compiler::target::word
Thread_enter_safepoint_stub_offset = 184;
Thread_enter_safepoint_stub_offset = 180;
static constexpr dart::compiler::target::word Thread_execution_state_offset =
592;
static constexpr dart::compiler::target::word
Thread_exit_safepoint_stub_offset = 188;
Thread_exit_safepoint_stub_offset = 184;
static constexpr dart::compiler::target::word
Thread_fix_allocation_stub_code_offset = 120;
static constexpr dart::compiler::target::word
Thread_fix_callers_target_code_offset = 116;
static constexpr dart::compiler::target::word
Thread_float_absolute_address_offset = 276;
Thread_float_absolute_address_offset = 272;
static constexpr dart::compiler::target::word
Thread_float_negate_address_offset = 272;
Thread_float_negate_address_offset = 268;
static constexpr dart::compiler::target::word Thread_float_not_address_offset =
268;
264;
static constexpr dart::compiler::target::word
Thread_float_zerow_address_offset = 280;
Thread_float_zerow_address_offset = 276;
static constexpr dart::compiler::target::word Thread_global_object_pool_offset =
584;
static constexpr dart::compiler::target::word
Thread_ic_lookup_through_code_stub_offset = 156;
static constexpr dart::compiler::target::word
Thread_interpret_call_entry_point_offset = 248;
Thread_interpret_call_entry_point_offset = 244;
static constexpr dart::compiler::target::word
Thread_invoke_dart_code_from_bytecode_stub_offset = 128;
static constexpr dart::compiler::target::word
Thread_invoke_dart_code_stub_offset = 124;
static constexpr dart::compiler::target::word Thread_isolate_offset = 48;
static constexpr dart::compiler::target::word
Thread_lazy_deopt_from_return_stub_offset = 168;
Thread_lazy_deopt_from_return_stub_offset = 164;
static constexpr dart::compiler::target::word
Thread_lazy_deopt_from_throw_stub_offset = 172;
Thread_lazy_deopt_from_throw_stub_offset = 168;
static constexpr dart::compiler::target::word
Thread_lazy_specialize_type_test_stub_offset = 180;
Thread_lazy_specialize_type_test_stub_offset = 176;
static constexpr dart::compiler::target::word
Thread_marking_stack_block_offset = 72;
static constexpr dart::compiler::target::word
Thread_megamorphic_call_checked_entry_offset = 220;
Thread_megamorphic_call_checked_entry_offset = 216;
static constexpr dart::compiler::target::word
Thread_monomorphic_miss_entry_offset = 224;
Thread_monomorphic_miss_entry_offset = 220;
static constexpr dart::compiler::target::word
Thread_monomorphic_miss_stub_offset = 152;
static constexpr dart::compiler::target::word
Thread_no_scope_native_wrapper_entry_point_offset = 240;
Thread_no_scope_native_wrapper_entry_point_offset = 236;
static constexpr dart::compiler::target::word
Thread_null_error_shared_with_fpu_regs_entry_point_offset = 208;
Thread_null_error_shared_with_fpu_regs_entry_point_offset = 204;
static constexpr dart::compiler::target::word
Thread_null_error_shared_with_fpu_regs_stub_offset = 140;
static constexpr dart::compiler::target::word
Thread_null_error_shared_without_fpu_regs_entry_point_offset = 204;
Thread_null_error_shared_without_fpu_regs_entry_point_offset = 200;
static constexpr dart::compiler::target::word
Thread_null_error_shared_without_fpu_regs_stub_offset = 136;
static constexpr dart::compiler::target::word Thread_object_null_offset = 96;
static constexpr dart::compiler::target::word
Thread_predefined_symbols_address_offset = 252;
Thread_predefined_symbols_address_offset = 248;
static constexpr dart::compiler::target::word Thread_resume_pc_offset = 588;
static constexpr dart::compiler::target::word Thread_safepoint_state_offset =
596;
static constexpr dart::compiler::target::word
Thread_slow_type_test_stub_offset = 176;
Thread_slow_type_test_stub_offset = 172;
static constexpr dart::compiler::target::word Thread_stack_limit_offset = 36;
static constexpr dart::compiler::target::word
Thread_stack_overflow_flags_offset = 40;
static constexpr dart::compiler::target::word
Thread_stack_overflow_shared_with_fpu_regs_entry_point_offset = 216;
Thread_stack_overflow_shared_with_fpu_regs_entry_point_offset = 212;
static constexpr dart::compiler::target::word
Thread_stack_overflow_shared_with_fpu_regs_stub_offset = 148;
static constexpr dart::compiler::target::word
Thread_stack_overflow_shared_without_fpu_regs_entry_point_offset = 212;
Thread_stack_overflow_shared_without_fpu_regs_entry_point_offset = 208;
static constexpr dart::compiler::target::word
Thread_stack_overflow_shared_without_fpu_regs_stub_offset = 144;
static constexpr dart::compiler::target::word Thread_store_buffer_block_offset =
@ -997,11 +1003,11 @@ static constexpr dart::compiler::target::word Thread_vm_tag_offset = 80;
static constexpr dart::compiler::target::word Thread_write_barrier_code_offset =
108;
static constexpr dart::compiler::target::word
Thread_write_barrier_entry_point_offset = 192;
Thread_write_barrier_entry_point_offset = 188;
static constexpr dart::compiler::target::word Thread_write_barrier_mask_offset =
44;
static constexpr dart::compiler::target::word
Thread_verify_callback_entry_offset = 236;
Thread_verify_callback_entry_offset = 232;
static constexpr dart::compiler::target::word Thread_callback_code_offset = 600;
static constexpr dart::compiler::target::word TimelineStream_enabled_offset = 8;
static constexpr dart::compiler::target::word TwoByteString_data_offset = 12;
@ -1065,9 +1071,13 @@ static constexpr dart::compiler::target::word Array_kMaxNewSpaceElements =
static constexpr dart::compiler::target::word ClassTable_kSizeOfClassPairLog2 =
4;
static constexpr dart::compiler::target::word
Instructions_kMonomorphicEntryOffset = 28;
Instructions_kMonomorphicEntryOffsetJIT = 8;
static constexpr dart::compiler::target::word
Instructions_kPolymorphicEntryOffset = 8;
Instructions_kPolymorphicEntryOffsetJIT = 48;
static constexpr dart::compiler::target::word
Instructions_kMonomorphicEntryOffsetAOT = 8;
static constexpr dart::compiler::target::word
Instructions_kPolymorphicEntryOffsetAOT = 28;
static constexpr dart::compiler::target::word HeapPage_kBytesPerCardLog2 = 10;
static constexpr dart::compiler::target::word
NativeEntry_kNumCallWrapperArguments = 2;
@ -1228,7 +1238,7 @@ static constexpr dart::compiler::target::word String_hash_offset = 4;
static constexpr dart::compiler::target::word String_length_offset = 8;
static constexpr dart::compiler::target::word SubtypeTestCache_cache_offset = 8;
static constexpr dart::compiler::target::word
Thread_AllocateArray_entry_point_offset = 560;
Thread_AllocateArray_entry_point_offset = 552;
static constexpr dart::compiler::target::word Thread_active_exception_offset =
1320;
static constexpr dart::compiler::target::word Thread_active_stacktrace_offset =
@ -1236,97 +1246,95 @@ static constexpr dart::compiler::target::word Thread_active_stacktrace_offset =
static constexpr dart::compiler::target::word
Thread_array_write_barrier_code_offset = 216;
static constexpr dart::compiler::target::word
Thread_array_write_barrier_entry_point_offset = 384;
Thread_array_write_barrier_entry_point_offset = 376;
static constexpr dart::compiler::target::word Thread_async_stack_trace_offset =
168;
static constexpr dart::compiler::target::word
Thread_auto_scope_native_wrapper_entry_point_offset = 480;
Thread_auto_scope_native_wrapper_entry_point_offset = 472;
static constexpr dart::compiler::target::word Thread_bool_false_offset = 200;
static constexpr dart::compiler::target::word Thread_bool_true_offset = 192;
static constexpr dart::compiler::target::word
Thread_call_to_runtime_entry_point_offset = 392;
Thread_call_to_runtime_entry_point_offset = 384;
static constexpr dart::compiler::target::word
Thread_call_to_runtime_stub_offset = 256;
static constexpr dart::compiler::target::word Thread_dart_stream_offset = 1384;
static constexpr dart::compiler::target::word Thread_deoptimize_entry_offset =
456;
448;
static constexpr dart::compiler::target::word Thread_deoptimize_stub_offset =
320;
312;
static constexpr dart::compiler::target::word Thread_double_abs_address_offset =
520;
512;
static constexpr dart::compiler::target::word
Thread_double_negate_address_offset = 512;
Thread_double_negate_address_offset = 504;
static constexpr dart::compiler::target::word Thread_end_offset = 120;
static constexpr dart::compiler::target::word
Thread_enter_safepoint_stub_offset = 360;
Thread_enter_safepoint_stub_offset = 352;
static constexpr dart::compiler::target::word Thread_execution_state_offset =
1352;
static constexpr dart::compiler::target::word
Thread_exit_safepoint_stub_offset = 368;
Thread_exit_safepoint_stub_offset = 360;
static constexpr dart::compiler::target::word
Thread_fix_allocation_stub_code_offset = 232;
static constexpr dart::compiler::target::word
Thread_fix_callers_target_code_offset = 224;
static constexpr dart::compiler::target::word
Thread_float_absolute_address_offset = 544;
Thread_float_absolute_address_offset = 536;
static constexpr dart::compiler::target::word
Thread_float_negate_address_offset = 536;
Thread_float_negate_address_offset = 528;
static constexpr dart::compiler::target::word Thread_float_not_address_offset =
528;
520;
static constexpr dart::compiler::target::word
Thread_float_zerow_address_offset = 552;
Thread_float_zerow_address_offset = 544;
static constexpr dart::compiler::target::word Thread_global_object_pool_offset =
1336;
static constexpr dart::compiler::target::word
Thread_ic_lookup_through_code_stub_offset = 304;
static constexpr dart::compiler::target::word
Thread_interpret_call_entry_point_offset = 488;
Thread_interpret_call_entry_point_offset = 480;
static constexpr dart::compiler::target::word
Thread_invoke_dart_code_from_bytecode_stub_offset = 248;
static constexpr dart::compiler::target::word
Thread_invoke_dart_code_stub_offset = 240;
static constexpr dart::compiler::target::word Thread_isolate_offset = 96;
static constexpr dart::compiler::target::word
Thread_lazy_deopt_from_return_stub_offset = 328;
Thread_lazy_deopt_from_return_stub_offset = 320;
static constexpr dart::compiler::target::word
Thread_lazy_deopt_from_throw_stub_offset = 336;
Thread_lazy_deopt_from_throw_stub_offset = 328;
static constexpr dart::compiler::target::word
Thread_lazy_specialize_type_test_stub_offset = 352;
Thread_lazy_specialize_type_test_stub_offset = 344;
static constexpr dart::compiler::target::word
Thread_marking_stack_block_offset = 144;
static constexpr dart::compiler::target::word
Thread_megamorphic_call_checked_entry_offset = 432;
Thread_megamorphic_call_checked_entry_offset = 424;
static constexpr dart::compiler::target::word
Thread_monomorphic_miss_entry_offset = 440;
Thread_monomorphic_miss_entry_offset = 432;
static constexpr dart::compiler::target::word
Thread_monomorphic_miss_stub_offset = 296;
static constexpr dart::compiler::target::word
Thread_no_scope_native_wrapper_entry_point_offset = 472;
Thread_no_scope_native_wrapper_entry_point_offset = 464;
static constexpr dart::compiler::target::word
Thread_null_error_shared_with_fpu_regs_entry_point_offset = 408;
Thread_null_error_shared_with_fpu_regs_entry_point_offset = 400;
static constexpr dart::compiler::target::word
Thread_null_error_shared_with_fpu_regs_stub_offset = 272;
static constexpr dart::compiler::target::word
Thread_null_error_shared_without_fpu_regs_entry_point_offset = 400;
Thread_null_error_shared_without_fpu_regs_entry_point_offset = 392;
static constexpr dart::compiler::target::word
Thread_null_error_shared_without_fpu_regs_stub_offset = 264;
static constexpr dart::compiler::target::word Thread_object_null_offset = 184;
static constexpr dart::compiler::target::word
Thread_predefined_symbols_address_offset = 496;
Thread_predefined_symbols_address_offset = 488;
static constexpr dart::compiler::target::word Thread_resume_pc_offset = 1344;
static constexpr dart::compiler::target::word Thread_safepoint_state_offset =
1360;
static constexpr dart::compiler::target::word
Thread_slow_type_test_stub_offset = 344;
Thread_slow_type_test_stub_offset = 336;
static constexpr dart::compiler::target::word Thread_stack_limit_offset = 72;
static constexpr dart::compiler::target::word
Thread_stack_overflow_flags_offset = 80;
static constexpr dart::compiler::target::word
Thread_stack_overflow_shared_with_fpu_regs_entry_point_offset = 424;
Thread_stack_overflow_shared_with_fpu_regs_entry_point_offset = 416;
static constexpr dart::compiler::target::word
Thread_stack_overflow_shared_with_fpu_regs_stub_offset = 288;
static constexpr dart::compiler::target::word
Thread_stack_overflow_shared_without_fpu_regs_entry_point_offset = 416;
Thread_stack_overflow_shared_without_fpu_regs_entry_point_offset = 408;
static constexpr dart::compiler::target::word
Thread_stack_overflow_shared_without_fpu_regs_stub_offset = 280;
static constexpr dart::compiler::target::word Thread_store_buffer_block_offset =
@ -1341,11 +1349,11 @@ static constexpr dart::compiler::target::word Thread_vm_tag_offset = 160;
static constexpr dart::compiler::target::word Thread_write_barrier_code_offset =
208;
static constexpr dart::compiler::target::word
Thread_write_barrier_entry_point_offset = 376;
Thread_write_barrier_entry_point_offset = 368;
static constexpr dart::compiler::target::word Thread_write_barrier_mask_offset =
88;
static constexpr dart::compiler::target::word
Thread_verify_callback_entry_offset = 464;
Thread_verify_callback_entry_offset = 456;
static constexpr dart::compiler::target::word Thread_callback_code_offset =
1368;
static constexpr dart::compiler::target::word TimelineStream_enabled_offset =
@ -1417,9 +1425,13 @@ static constexpr dart::compiler::target::word Array_kMaxNewSpaceElements =
static constexpr dart::compiler::target::word ClassTable_kSizeOfClassPairLog2 =
4;
static constexpr dart::compiler::target::word
Instructions_kMonomorphicEntryOffset = 0;
Instructions_kMonomorphicEntryOffsetJIT = 0;
static constexpr dart::compiler::target::word
Instructions_kPolymorphicEntryOffset = 0;
Instructions_kPolymorphicEntryOffsetJIT = 0;
static constexpr dart::compiler::target::word
Instructions_kMonomorphicEntryOffsetAOT = 0;
static constexpr dart::compiler::target::word
Instructions_kPolymorphicEntryOffsetAOT = 0;
static constexpr dart::compiler::target::word HeapPage_kBytesPerCardLog2 = 10;
static constexpr dart::compiler::target::word
NativeEntry_kNumCallWrapperArguments = 2;
@ -1582,23 +1594,23 @@ static constexpr dart::compiler::target::word SubtypeTestCache_cache_offset = 8;
static constexpr dart::compiler::target::word
Thread_AllocateArray_entry_point_offset = 296;
static constexpr dart::compiler::target::word Thread_active_exception_offset =
880;
static constexpr dart::compiler::target::word Thread_active_stacktrace_offset =
888;
static constexpr dart::compiler::target::word Thread_active_stacktrace_offset =
896;
static constexpr dart::compiler::target::word Thread_async_stack_trace_offset =
168;
static constexpr dart::compiler::target::word
Thread_auto_scope_native_wrapper_entry_point_offset = 216;
static constexpr dart::compiler::target::word Thread_bool_false_offset = 200;
static constexpr dart::compiler::target::word Thread_bool_true_offset = 192;
static constexpr dart::compiler::target::word Thread_dart_stream_offset = 944;
static constexpr dart::compiler::target::word Thread_dart_stream_offset = 952;
static constexpr dart::compiler::target::word Thread_double_abs_address_offset =
256;
static constexpr dart::compiler::target::word
Thread_double_negate_address_offset = 248;
static constexpr dart::compiler::target::word Thread_end_offset = 120;
static constexpr dart::compiler::target::word Thread_execution_state_offset =
912;
920;
static constexpr dart::compiler::target::word
Thread_float_absolute_address_offset = 280;
static constexpr dart::compiler::target::word
@ -1608,7 +1620,7 @@ static constexpr dart::compiler::target::word Thread_float_not_address_offset =
static constexpr dart::compiler::target::word
Thread_float_zerow_address_offset = 288;
static constexpr dart::compiler::target::word Thread_global_object_pool_offset =
896;
904;
static constexpr dart::compiler::target::word Thread_isolate_offset = 96;
static constexpr dart::compiler::target::word
Thread_marking_stack_block_offset = 144;
@ -1617,9 +1629,9 @@ static constexpr dart::compiler::target::word
static constexpr dart::compiler::target::word Thread_object_null_offset = 184;
static constexpr dart::compiler::target::word
Thread_predefined_symbols_address_offset = 232;
static constexpr dart::compiler::target::word Thread_resume_pc_offset = 904;
static constexpr dart::compiler::target::word Thread_resume_pc_offset = 912;
static constexpr dart::compiler::target::word Thread_safepoint_state_offset =
920;
928;
static constexpr dart::compiler::target::word Thread_stack_limit_offset = 72;
static constexpr dart::compiler::target::word
Thread_stack_overflow_flags_offset = 80;
@ -1634,7 +1646,7 @@ static constexpr dart::compiler::target::word
static constexpr dart::compiler::target::word Thread_vm_tag_offset = 160;
static constexpr dart::compiler::target::word Thread_write_barrier_mask_offset =
88;
static constexpr dart::compiler::target::word Thread_callback_code_offset = 928;
static constexpr dart::compiler::target::word Thread_callback_code_offset = 936;
static constexpr dart::compiler::target::word TimelineStream_enabled_offset =
16;
static constexpr dart::compiler::target::word TwoByteString_data_offset = 16;
@ -1698,9 +1710,13 @@ static constexpr dart::compiler::target::word Array_kMaxNewSpaceElements =
static constexpr dart::compiler::target::word ClassTable_kSizeOfClassPairLog2 =
3;
static constexpr dart::compiler::target::word
Instructions_kMonomorphicEntryOffset = 0;
Instructions_kMonomorphicEntryOffsetJIT = 0;
static constexpr dart::compiler::target::word
Instructions_kPolymorphicEntryOffset = 0;
Instructions_kPolymorphicEntryOffsetJIT = 0;
static constexpr dart::compiler::target::word
Instructions_kMonomorphicEntryOffsetAOT = 0;
static constexpr dart::compiler::target::word
Instructions_kPolymorphicEntryOffsetAOT = 0;
static constexpr dart::compiler::target::word HeapPage_kBytesPerCardLog2 = 9;
static constexpr dart::compiler::target::word
NativeEntry_kNumCallWrapperArguments = 2;
@ -1862,23 +1878,23 @@ static constexpr dart::compiler::target::word SubtypeTestCache_cache_offset = 4;
static constexpr dart::compiler::target::word
Thread_AllocateArray_entry_point_offset = 152;
static constexpr dart::compiler::target::word Thread_active_exception_offset =
444;
static constexpr dart::compiler::target::word Thread_active_stacktrace_offset =
448;
static constexpr dart::compiler::target::word Thread_active_stacktrace_offset =
452;
static constexpr dart::compiler::target::word Thread_async_stack_trace_offset =
84;
static constexpr dart::compiler::target::word
Thread_auto_scope_native_wrapper_entry_point_offset = 112;
static constexpr dart::compiler::target::word Thread_bool_false_offset = 104;
static constexpr dart::compiler::target::word Thread_bool_true_offset = 100;
static constexpr dart::compiler::target::word Thread_dart_stream_offset = 476;
static constexpr dart::compiler::target::word Thread_dart_stream_offset = 480;
static constexpr dart::compiler::target::word Thread_double_abs_address_offset =
132;
static constexpr dart::compiler::target::word
Thread_double_negate_address_offset = 128;
static constexpr dart::compiler::target::word Thread_end_offset = 60;
static constexpr dart::compiler::target::word Thread_execution_state_offset =
460;
464;
static constexpr dart::compiler::target::word
Thread_float_absolute_address_offset = 144;
static constexpr dart::compiler::target::word
@ -1888,7 +1904,7 @@ static constexpr dart::compiler::target::word Thread_float_not_address_offset =
static constexpr dart::compiler::target::word
Thread_float_zerow_address_offset = 148;
static constexpr dart::compiler::target::word Thread_global_object_pool_offset =
452;
456;
static constexpr dart::compiler::target::word Thread_isolate_offset = 48;
static constexpr dart::compiler::target::word
Thread_marking_stack_block_offset = 72;
@ -1897,9 +1913,9 @@ static constexpr dart::compiler::target::word
static constexpr dart::compiler::target::word Thread_object_null_offset = 96;
static constexpr dart::compiler::target::word
Thread_predefined_symbols_address_offset = 120;
static constexpr dart::compiler::target::word Thread_resume_pc_offset = 456;
static constexpr dart::compiler::target::word Thread_resume_pc_offset = 460;
static constexpr dart::compiler::target::word Thread_safepoint_state_offset =
464;
468;
static constexpr dart::compiler::target::word Thread_stack_limit_offset = 36;
static constexpr dart::compiler::target::word
Thread_stack_overflow_flags_offset = 40;
@ -1914,7 +1930,7 @@ static constexpr dart::compiler::target::word
static constexpr dart::compiler::target::word Thread_vm_tag_offset = 80;
static constexpr dart::compiler::target::word Thread_write_barrier_mask_offset =
44;
static constexpr dart::compiler::target::word Thread_callback_code_offset = 468;
static constexpr dart::compiler::target::word Thread_callback_code_offset = 472;
static constexpr dart::compiler::target::word TimelineStream_enabled_offset = 8;
static constexpr dart::compiler::target::word TwoByteString_data_offset = 12;
static constexpr dart::compiler::target::word Type_arguments_offset = 16;

7
runtime/vm/compiler/runtime_offsets_list.h
View file

@ -35,8 +35,10 @@
CONSTANT(Array, kMaxElements) \
CONSTANT(Array, kMaxNewSpaceElements) \
CONSTANT(ClassTable, kSizeOfClassPairLog2) \
CONSTANT(Instructions, kMonomorphicEntryOffset) \
CONSTANT(Instructions, kPolymorphicEntryOffset) \
CONSTANT(Instructions, kMonomorphicEntryOffsetJIT) \
CONSTANT(Instructions, kPolymorphicEntryOffsetJIT) \
CONSTANT(Instructions, kMonomorphicEntryOffsetAOT) \
CONSTANT(Instructions, kPolymorphicEntryOffsetAOT) \
CONSTANT(HeapPage, kBytesPerCardLog2) \
CONSTANT(NativeEntry, kNumCallWrapperArguments) \
CONSTANT(String, kMaxElements) \
@ -170,7 +172,6 @@
FIELD(Thread, float_not_address_offset) \
FIELD(Thread, float_zerow_address_offset) \
FIELD(Thread, global_object_pool_offset) \
NOT_IN_DBC(FIELD(Thread, ic_lookup_through_code_stub_offset)) \
NOT_IN_DBC(FIELD(Thread, interpret_call_entry_point_offset)) \
NOT_IN_DBC(FIELD(Thread, invoke_dart_code_from_bytecode_stub_offset)) \
NOT_IN_DBC(FIELD(Thread, invoke_dart_code_stub_offset)) \

68
runtime/vm/compiler/stub_code_compiler_arm.cc
View file

@ -574,6 +574,9 @@ void StubCodeCompiler::GenerateCallStaticFunctionStub(Assembler* assembler) {
// (invalid because its function was optimized or deoptimized).
// R4: arguments descriptor array.
void StubCodeCompiler::GenerateFixCallersTargetStub(Assembler* assembler) {
Label monomorphic;
__ BranchOnMonomorphicCheckedEntryJIT(&monomorphic);
// Load code pointer to this stub from the thread:
// The one that is passed in, is not correct - it points to the code object
// that needs to be replaced.
@ -593,6 +596,29 @@ void StubCodeCompiler::GenerateFixCallersTargetStub(Assembler* assembler) {
// Jump to the dart function.
__ mov(CODE_REG, Operand(R0));
__ Branch(FieldAddress(R0, target::Code::entry_point_offset()));
__ Bind(&monomorphic);
// Load code pointer to this stub from the thread:
// The one that is passed in, is not correct - it points to the code object
// that needs to be replaced.
__ ldr(CODE_REG,
Address(THR, target::Thread::fix_callers_target_code_offset()));
// Create a stub frame as we are pushing some objects on the stack before
// calling into the runtime.
__ EnterStubFrame();
__ LoadImmediate(R1, 0);
__ Push(R9); // Preserve cache (guarded CID as Smi).
__ Push(R0); // Preserve receiver.
__ Push(R1);
__ CallRuntime(kFixCallersTargetMonomorphicRuntimeEntry, 0);
__ Pop(CODE_REG);
__ Pop(R0); // Restore receiver.
__ Pop(R9); // Restore cache (guarded CID as Smi).
// Remove the stub frame.
__ LeaveStubFrame();
// Jump to the dart function.
__ Branch(FieldAddress(
CODE_REG, target::Code::entry_point_offset(CodeEntryKind::kMonomorphic)));
}
// Called from object allocate instruction when the allocation stub has been
@ -2414,6 +2440,22 @@ void StubCodeCompiler::GenerateICCallBreakpointStub(Assembler* assembler) {
#endif // defined(PRODUCT)
}
void StubCodeCompiler::GenerateUnoptStaticCallBreakpointStub(
Assembler* assembler) {
#if defined(PRODUCT)
__ Stop("No debugging in PRODUCT mode");
#else
__ EnterStubFrame();
__ Push(R9); // Preserve IC data.
__ PushImmediate(0); // Space for result.
__ CallRuntime(kBreakpointRuntimeHandlerRuntimeEntry, 0);
__ Pop(CODE_REG); // Original stub.
__ Pop(R9); // Restore IC data.
__ LeaveStubFrame();
__ Branch(FieldAddress(CODE_REG, target::Code::entry_point_offset()));
#endif // defined(PRODUCT)
}
void StubCodeCompiler::GenerateRuntimeCallBreakpointStub(Assembler* assembler) {
#if defined(PRODUCT)
__ Stop("No debugging in PRODUCT mode");
@ -3196,17 +3238,17 @@ void StubCodeCompiler::GenerateUnlinkedCallStub(Assembler* assembler) {
__ LoadImmediate(IP, 0);
__ Push(IP); // Result slot
__ Push(R0); // Arg0: Receiver
__ Push(R9); // Arg1: UnlinkedCall
__ CallRuntime(kUnlinkedCallRuntimeEntry, 2);
__ Push(IP); // Arg0: stub out
__ Push(R0); // Arg1: Receiver
__ Push(R9); // Arg2: UnlinkedCall
__ CallRuntime(kUnlinkedCallRuntimeEntry, 3);
__ Drop(2);
__ Pop(CODE_REG); // result = stub
__ Pop(R9); // result = IC
__ Pop(R0); // Restore receiver.
__ LeaveStubFrame();
__ ldr(CODE_REG,
Address(THR, target::Thread::ic_lookup_through_code_stub_offset()));
__ Branch(FieldAddress(
CODE_REG, target::Code::entry_point_offset(CodeEntryKind::kMonomorphic)));
}
@ -3239,16 +3281,16 @@ void StubCodeCompiler::GenerateSingleTargetCallStub(Assembler* assembler) {
__ LoadImmediate(IP, 0);
__ Push(IP); // Result slot
__ Push(R0); // Arg0: Receiver
__ CallRuntime(kSingleTargetMissRuntimeEntry, 1);
__ Push(IP); // Arg0: stub out
__ Push(R0); // Arg1: Receiver
__ CallRuntime(kSingleTargetMissRuntimeEntry, 2);
__ Drop(1);
__ Pop(CODE_REG); // result = stub
__ Pop(R9); // result = IC
__ Pop(R0); // Restore receiver.
__ LeaveStubFrame();
__ ldr(CODE_REG,
Address(THR, target::Thread::ic_lookup_through_code_stub_offset()));
__ Branch(FieldAddress(
CODE_REG, target::Code::entry_point_offset(CodeEntryKind::kMonomorphic)));
}
@ -3263,16 +3305,16 @@ void StubCodeCompiler::GenerateMonomorphicMissStub(Assembler* assembler) {
__ LoadImmediate(IP, 0);
__ Push(IP); // Result slot
__ Push(R0); // Arg0: Receiver
__ CallRuntime(kMonomorphicMissRuntimeEntry, 1);
__ Push(IP); // Arg0: stub out
__ Push(R0); // Arg1: Receiver
__ CallRuntime(kMonomorphicMissRuntimeEntry, 2);
__ Drop(1);
__ Pop(CODE_REG); // result = stub
__ Pop(R9); // result = IC
__ Pop(R0); // Restore receiver.
__ LeaveStubFrame();
__ ldr(CODE_REG,
Address(THR, target::Thread::ic_lookup_through_code_stub_offset()));
__ Branch(FieldAddress(
CODE_REG, target::Code::entry_point_offset(CodeEntryKind::kMonomorphic)));
}

70
runtime/vm/compiler/stub_code_compiler_arm64.cc
View file

@ -618,6 +618,9 @@ void StubCodeCompiler::GenerateCallStaticFunctionStub(Assembler* assembler) {
// (invalid because its function was optimized or deoptimized).
// R4: arguments descriptor array.
void StubCodeCompiler::GenerateFixCallersTargetStub(Assembler* assembler) {
Label monomorphic;
__ BranchOnMonomorphicCheckedEntryJIT(&monomorphic);
// Load code pointer to this stub from the thread:
// The one that is passed in, is not correct - it points to the code object
// that needs to be replaced.
@ -638,6 +641,30 @@ void StubCodeCompiler::GenerateFixCallersTargetStub(Assembler* assembler) {
// Jump to the dart function.
__ LoadFieldFromOffset(R0, CODE_REG, target::Code::entry_point_offset());
__ br(R0);
__ Bind(&monomorphic);
// Load code pointer to this stub from the thread:
// The one that is passed in, is not correct - it points to the code object
// that needs to be replaced.
__ ldr(CODE_REG,
Address(THR, target::Thread::fix_callers_target_code_offset()));
// Create a stub frame as we are pushing some objects on the stack before
// calling into the runtime.
__ EnterStubFrame();
__ Push(R5); // Preserve cache (guarded CID as Smi).
__ Push(R0); // Preserve receiver.
__ Push(ZR);
__ CallRuntime(kFixCallersTargetMonomorphicRuntimeEntry, 0);
__ Pop(CODE_REG);
__ Pop(R0); // Restore receiver.
__ Pop(R5); // Restore cache (guarded CID as Smi).
// Remove the stub frame.
__ LeaveStubFrame();
// Jump to the dart function.
__ LoadFieldFromOffset(
R1, CODE_REG,
target::Code::entry_point_offset(CodeEntryKind::kMonomorphic));
__ br(R1);
}
// Called from object allocate instruction when the allocation stub has been
@ -2489,6 +2516,23 @@ void StubCodeCompiler::GenerateICCallBreakpointStub(Assembler* assembler) {
#endif // defined(PRODUCT)
}
void StubCodeCompiler::GenerateUnoptStaticCallBreakpointStub(
Assembler* assembler) {
#if defined(PRODUCT)
__ Stop("No debugging in PRODUCT mode");
#else
__ EnterStubFrame();
__ Push(R5); // Preserve IC data.
__ Push(ZR); // Space for result.
__ CallRuntime(kBreakpointRuntimeHandlerRuntimeEntry, 0);
__ Pop(CODE_REG); // Original stub.
__ Pop(R5); // Restore IC data.
__ LeaveStubFrame();
__ LoadFieldFromOffset(TMP, CODE_REG, target::Code::entry_point_offset());
__ br(TMP);
#endif // defined(PRODUCT)
}
void StubCodeCompiler::GenerateRuntimeCallBreakpointStub(Assembler* assembler) {
#if defined(PRODUCT)
__ Stop("No debugging in PRODUCT mode");
@ -3265,17 +3309,17 @@ void StubCodeCompiler::GenerateUnlinkedCallStub(Assembler* assembler) {
__ Push(R0); // Preserve receiver.
__ Push(ZR); // Result slot.
__ Push(R0); // Arg0: Receiver
__ Push(R5); // Arg1: UnlinkedCall
__ CallRuntime(kUnlinkedCallRuntimeEntry, 2);
__ Push(ZR); // Arg0: stub out.
__ Push(R0); // Arg1: Receiver
__ Push(R5); // Arg2: UnlinkedCall
__ CallRuntime(kUnlinkedCallRuntimeEntry, 3);
__ Drop(2);
__ Pop(CODE_REG); // result = stub
__ Pop(R5); // result = IC
__ Pop(R0); // Restore receiver.
__ LeaveStubFrame();
__ ldr(CODE_REG,
Address(THR, target::Thread::ic_lookup_through_code_stub_offset()));
__ ldr(R1, FieldAddress(CODE_REG, target::Code::entry_point_offset(
CodeEntryKind::kMonomorphic)));
__ br(R1);
@ -3309,16 +3353,16 @@ void StubCodeCompiler::GenerateSingleTargetCallStub(Assembler* assembler) {
__ Push(R0); // Preserve receiver.
__ Push(ZR); // Result slot.
__ Push(R0); // Arg0: Receiver
__ CallRuntime(kSingleTargetMissRuntimeEntry, 1);
__ Push(ZR); // Arg0: Stub out.
__ Push(R0); // Arg1: Receiver
__ CallRuntime(kSingleTargetMissRuntimeEntry, 2);
__ Drop(1);
__ Pop(CODE_REG); // result = stub
__ Pop(R5); // result = IC
__ Pop(R0); // Restore receiver.
__ LeaveStubFrame();
__ ldr(CODE_REG,
Address(THR, target::Thread::ic_lookup_through_code_stub_offset()));
__ ldr(R1, FieldAddress(CODE_REG, target::Code::entry_point_offset(
CodeEntryKind::kMonomorphic)));
__ br(R1);
@ -3333,16 +3377,16 @@ void StubCodeCompiler::GenerateMonomorphicMissStub(Assembler* assembler) {
__ Push(R0); // Preserve receiver.
__ Push(ZR); // Result slot.
__ Push(R0); // Arg0: Receiver
__ CallRuntime(kMonomorphicMissRuntimeEntry, 1);
__ Push(ZR); // Arg0: stub out
__ Push(R0); // Arg1: Receiver
__ CallRuntime(kMonomorphicMissRuntimeEntry, 2);
__ Drop(1);
__ Pop(CODE_REG); // result = stub
__ Pop(R5); // result = IC
__ Pop(R0); // Restore receiver.
__ LeaveStubFrame();
__ ldr(CODE_REG,
Address(THR, target::Thread::ic_lookup_through_code_stub_offset()));
__ ldr(R1, FieldAddress(CODE_REG, target::Code::entry_point_offset(
CodeEntryKind::kMonomorphic)));
__ br(R1);

57
runtime/vm/compiler/stub_code_compiler_ia32.cc
View file

@ -407,8 +407,10 @@ void StubCodeCompiler::GenerateCallStaticFunctionStub(Assembler* assembler) {
// (invalid because its function was optimized or deoptimized).
// EDX: arguments descriptor array.
void StubCodeCompiler::GenerateFixCallersTargetStub(Assembler* assembler) {
// Create a stub frame as we are pushing some objects on the stack before
// calling into the runtime.
Label monomorphic;
__ BranchOnMonomorphicCheckedEntryJIT(&monomorphic);
// This was a static call.
__ EnterStubFrame();
__ pushl(EDX); // Preserve arguments descriptor array.
__ pushl(Immediate(0)); // Setup space on stack for return value.
@ -419,6 +421,22 @@ void StubCodeCompiler::GenerateFixCallersTargetStub(Assembler* assembler) {
__ LeaveFrame();
__ jmp(EAX);
__ int3();
__ Bind(&monomorphic);
// This was a switchable call.
__ EnterStubFrame();
__ pushl(ECX); // Preserve cache (guarded CID as Smi).
__ pushl(EBX); // Preserve receiver.
__ pushl(Immediate(0)); // Result slot.
__ CallRuntime(kFixCallersTargetMonomorphicRuntimeEntry, 0);
__ popl(CODE_REG); // Get Code object.
__ popl(EBX); // Restore receiver.
__ popl(ECX); // Restore cache (guarded CID as Smi).
__ movl(EAX, FieldAddress(CODE_REG, target::Code::entry_point_offset(
CodeEntryKind::kMonomorphic)));
__ LeaveFrame();
__ jmp(EAX);
__ int3();
}
// Called from object allocate instruction when the allocation stub has been
@ -2119,6 +2137,23 @@ void StubCodeCompiler::GenerateICCallBreakpointStub(Assembler* assembler) {
#endif // defined(PRODUCT)
}
void StubCodeCompiler::GenerateUnoptStaticCallBreakpointStub(
Assembler* assembler) {
#if defined(PRODUCT)
__ Stop("No debugging in PRODUCT mode");
#else
__ EnterStubFrame();
__ pushl(ECX); // Preserve ICData.
__ pushl(Immediate(0)); // Room for result.
__ CallRuntime(kBreakpointRuntimeHandlerRuntimeEntry, 0);
__ popl(EAX); // Code of original stub.
__ popl(ECX); // Restore ICData.
__ LeaveFrame();
// Jump to original stub.
__ jmp(FieldAddress(EAX, target::Code::entry_point_offset()));
#endif // defined(PRODUCT)
}
void StubCodeCompiler::GenerateRuntimeCallBreakpointStub(Assembler* assembler) {
#if defined(PRODUCT)
__ Stop("No debugging in PRODUCT mode");
@ -2661,7 +2696,23 @@ void StubCodeCompiler::GenerateSingleTargetCallStub(Assembler* assembler) {
}
void StubCodeCompiler::GenerateMonomorphicMissStub(Assembler* assembler) {
__ int3(); // AOT only.
__ EnterStubFrame();
__ pushl(EBX); // Preserve receiver.
__ pushl(Immediate(0)); // Result slot.
__ pushl(Immediate(0)); // Arg0: stub out
__ pushl(EBX); // Arg1: Receiver
__ CallRuntime(kMonomorphicMissRuntimeEntry, 2);
__ popl(ECX);
__ popl(CODE_REG); // result = stub
__ popl(ECX); // result = IC
__ popl(EBX); // Restore receiver.
__ LeaveFrame();
__ movl(EAX, FieldAddress(CODE_REG, target::Code::entry_point_offset(
CodeEntryKind::kMonomorphic)));
__ jmp(EAX);
}
void StubCodeCompiler::GenerateFrameAwaitingMaterializationStub(

71
runtime/vm/compiler/stub_code_compiler_x64.cc
View file

@ -571,6 +571,10 @@ void StubCodeCompiler::GenerateCallStaticFunctionStub(Assembler* assembler) {
// (invalid because its function was optimized or deoptimized).
// R10: arguments descriptor array.
void StubCodeCompiler::GenerateFixCallersTargetStub(Assembler* assembler) {
Label monomorphic;
__ BranchOnMonomorphicCheckedEntryJIT(&monomorphic);
// This was a static call.
// Load code pointer to this stub from the thread:
// The one that is passed in, is not correct - it points to the code object
// that needs to be replaced.
@ -587,6 +591,27 @@ void StubCodeCompiler::GenerateFixCallersTargetStub(Assembler* assembler) {
__ LeaveStubFrame();
__ jmp(RAX);
__ int3();
__ Bind(&monomorphic);
// This was a switchable call.
// Load code pointer to this stub from the thread:
// The one that is passed in, is not correct - it points to the code object
// that needs to be replaced.
__ movq(CODE_REG,
Address(THR, target::Thread::fix_callers_target_code_offset()));
__ EnterStubFrame();
__ pushq(RBX); // Preserve cache (guarded CID as Smi).
__ pushq(RDX); // Preserve receiver.
__ pushq(Immediate(0)); // Result slot.
__ CallRuntime(kFixCallersTargetMonomorphicRuntimeEntry, 0);
__ popq(CODE_REG); // Get Code object.
__ popq(RDX); // Restore receiver.
__ popq(RBX); // Restore cache (guarded CID as Smi).
__ movq(RAX, FieldAddress(CODE_REG, target::Code::entry_point_offset(
CodeEntryKind::kMonomorphic)));
__ LeaveStubFrame();
__ jmp(RAX);
__ int3();
}
// Called from object allocate instruction when the allocation stub has been
@ -2500,6 +2525,26 @@ void StubCodeCompiler::GenerateICCallBreakpointStub(Assembler* assembler) {
#endif // defined(PRODUCT)
}
void StubCodeCompiler::GenerateUnoptStaticCallBreakpointStub(
Assembler* assembler) {
#if defined(PRODUCT)
__ Stop("No debugging in PRODUCT mode");
#else
__ EnterStubFrame();
__ pushq(RDX); // Preserve receiver.
__ pushq(RBX); // Preserve IC data.
__ pushq(Immediate(0)); // Result slot.
__ CallRuntime(kBreakpointRuntimeHandlerRuntimeEntry, 0);
__ popq(CODE_REG); // Original stub.
__ popq(RBX); // Restore IC data.
__ popq(RDX); // Restore receiver.
__ LeaveStubFrame();
__ movq(RAX, FieldAddress(CODE_REG, target::Code::entry_point_offset()));
__ jmp(RAX); // Jump to original stub.
#endif // defined(PRODUCT)
}
// TOS(0): return address (Dart code).
void StubCodeCompiler::GenerateRuntimeCallBreakpointStub(Assembler* assembler) {
#if defined(PRODUCT)
@ -3284,18 +3329,18 @@ void StubCodeCompiler::GenerateUnlinkedCallStub(Assembler* assembler) {
__ pushq(RDX); // Preserve receiver.
__ pushq(Immediate(0)); // Result slot.
__ pushq(RDX); // Arg0: Receiver
__ pushq(RBX); // Arg1: UnlinkedCall
__ CallRuntime(kUnlinkedCallRuntimeEntry, 2);
__ pushq(Immediate(0)); // Arg0: stub out.
__ pushq(RDX); // Arg1: Receiver
__ pushq(RBX); // Arg2: UnlinkedCall
__ CallRuntime(kUnlinkedCallRuntimeEntry, 3);
__ popq(RBX);
__ popq(RBX);
__ popq(CODE_REG); // result = stub
__ popq(RBX); // result = IC
__ popq(RDX); // Restore receiver.
__ LeaveStubFrame();
__ movq(CODE_REG,
Address(THR, target::Thread::ic_lookup_through_code_stub_offset()));
__ movq(RCX, FieldAddress(CODE_REG, target::Code::entry_point_offset(
CodeEntryKind::kMonomorphic)));
__ jmp(RCX);
@ -3328,16 +3373,16 @@ void StubCodeCompiler::GenerateSingleTargetCallStub(Assembler* assembler) {
__ pushq(RDX); // Preserve receiver.
__ pushq(Immediate(0)); // Result slot.
__ pushq(RDX); // Arg0: Receiver
__ CallRuntime(kSingleTargetMissRuntimeEntry, 1);
__ pushq(Immediate(0)); // Arg0: stub out
__ pushq(RDX); // Arg1: Receiver
__ CallRuntime(kSingleTargetMissRuntimeEntry, 2);
__ popq(RBX);
__ popq(CODE_REG); // result = stub
__ popq(RBX); // result = IC
__ popq(RDX); // Restore receiver.
__ LeaveStubFrame();
__ movq(CODE_REG,
Address(THR, target::Thread::ic_lookup_through_code_stub_offset()));
__ movq(RCX, FieldAddress(CODE_REG, target::Code::entry_point_offset(
CodeEntryKind::kMonomorphic)));
__ jmp(RCX);
@ -3352,16 +3397,16 @@ void StubCodeCompiler::GenerateMonomorphicMissStub(Assembler* assembler) {
__ pushq(RDX); // Preserve receiver.
__ pushq(Immediate(0)); // Result slot.
__ pushq(RDX); // Arg0: Receiver
__ CallRuntime(kMonomorphicMissRuntimeEntry, 1);
__ pushq(Immediate(0)); // Arg0: stub out.
__ pushq(RDX); // Arg1: Receiver
__ CallRuntime(kMonomorphicMissRuntimeEntry, 2);
__ popq(RBX);
__ popq(CODE_REG); // result = stub
__ popq(RBX); // result = IC
__ popq(RDX); // Restore receiver.
__ LeaveStubFrame();
__ movq(CODE_REG,
Address(THR, target::Thread::ic_lookup_through_code_stub_offset()));
__ movq(RCX, FieldAddress(CODE_REG, target::Code::entry_point_offset(
CodeEntryKind::kMonomorphic)));
__ jmp(RCX);

29
runtime/vm/debugger.cc
View file

@ -1860,18 +1860,26 @@ RawFunction* Debugger::ResolveFunction(const Library& library,
// that inline the function that contains the newly created breakpoint.
// We currently don't have this info so we deoptimize all functions.
void Debugger::DeoptimizeWorld() {
#if defined(DART_PRECOMPILED_RUNTIME)
UNREACHABLE();
#else
BackgroundCompiler::Stop(isolate_);
if (FLAG_trace_deoptimization) {
THR_Print("Deopt for debugger\n");
}
isolate_->set_has_attempted_stepping(true);
DeoptimizeFunctionsOnStack();
// Iterate over all classes, deoptimize functions.
// TODO(hausner): Could possibly be combined with RemoveOptimizedCode()
const ClassTable& class_table = *isolate_->class_table();
Class& cls = Class::Handle();
Array& functions = Array::Handle();
GrowableObjectArray& closures = GrowableObjectArray::Handle();
Function& function = Function::Handle();
Zone* zone = Thread::Current()->zone();
Class& cls = Class::Handle(zone);
Array& functions = Array::Handle(zone);
GrowableObjectArray& closures = GrowableObjectArray::Handle(zone);
Function& function = Function::Handle(zone);
Code& code = Code::Handle(zone);
intptr_t num_classes = class_table.NumCids();
for (intptr_t i = 1; i < num_classes; i++) {
if (class_table.HasValidClassAt(i)) {
@ -1887,12 +1895,20 @@ void Debugger::DeoptimizeWorld() {
if (function.HasOptimizedCode()) {
function.SwitchToUnoptimizedCode();
}
code = function.unoptimized_code();
if (!code.IsNull()) {
code.ResetSwitchableCalls(zone);
}
// Also disable any optimized implicit closure functions.
if (function.HasImplicitClosureFunction()) {
function = function.ImplicitClosureFunction();
if (function.HasOptimizedCode()) {
function.SwitchToUnoptimizedCode();
}
code = function.unoptimized_code();
if (!code.IsNull()) {
code.ResetSwitchableCalls(zone);
}
}
}
}
@ -1908,7 +1924,12 @@ void Debugger::DeoptimizeWorld() {
if (function.HasOptimizedCode()) {
function.SwitchToUnoptimizedCode();
}
code = function.unoptimized_code();
if (!code.IsNull()) {
code.ResetSwitchableCalls(zone);
}
}
#endif // defined(DART_PRECOMPILED_RUNTIME)
}
void Debugger::NotifySingleStepping(bool value) const {

4
runtime/vm/debugger_arm.cc
View file

@ -24,9 +24,11 @@ void CodeBreakpoint::PatchCode() {
Code& stub_target = Code::Handle();
switch (breakpoint_kind_) {
case RawPcDescriptors::kIcCall:
case RawPcDescriptors::kUnoptStaticCall:
stub_target = StubCode::ICCallBreakpoint().raw();
break;
case RawPcDescriptors::kUnoptStaticCall:
stub_target = StubCode::UnoptStaticCallBreakpoint().raw();
break;
case RawPcDescriptors::kRuntimeCall:
stub_target = StubCode::RuntimeCallBreakpoint().raw();
break;

32
runtime/vm/debugger_arm64.cc
View file

@ -21,22 +21,30 @@ RawCode* CodeBreakpoint::OrigStubAddress() const {
void CodeBreakpoint::PatchCode() {
ASSERT(!is_enabled_);
Code& stub_target = Code::Handle();
const Code& code = Code::Handle(code_);
switch (breakpoint_kind_) {
case RawPcDescriptors::kIcCall:
case RawPcDescriptors::kUnoptStaticCall:
stub_target = StubCode::ICCallBreakpoint().raw();
case RawPcDescriptors::kIcCall: {
Object& data = Object::Handle();
saved_value_ = CodePatcher::GetInstanceCallAt(pc_, code, &data);
CodePatcher::PatchInstanceCallAt(pc_, code, data,
StubCode::ICCallBreakpoint());
break;
}
case RawPcDescriptors::kUnoptStaticCall: {
saved_value_ = CodePatcher::GetStaticCallTargetAt(pc_, code);
CodePatcher::PatchPoolPointerCallAt(
pc_, code, StubCode::UnoptStaticCallBreakpoint());
break;
}
case RawPcDescriptors::kRuntimeCall: {
stub_target = StubCode::RuntimeCallBreakpoint().raw();
saved_value_ = CodePatcher::GetStaticCallTargetAt(pc_, code);
CodePatcher::PatchPoolPointerCallAt(pc_, code,
StubCode::RuntimeCallBreakpoint());
break;
}
default:
UNREACHABLE();
}
const Code& code = Code::Handle(code_);
saved_value_ = CodePatcher::GetStaticCallTargetAt(pc_, code);
CodePatcher::PatchPoolPointerCallAt(pc_, code, stub_target);
is_enabled_ = true;
}
@ -44,7 +52,13 @@ void CodeBreakpoint::RestoreCode() {
ASSERT(is_enabled_);
const Code& code = Code::Handle(code_);
switch (breakpoint_kind_) {
case RawPcDescriptors::kIcCall:
case RawPcDescriptors::kIcCall: {
Object& data = Object::Handle();
CodePatcher::GetInstanceCallAt(pc_, code, &data);
CodePatcher::PatchInstanceCallAt(pc_, code, data,
Code::Handle(saved_value_));
break;
}
case RawPcDescriptors::kUnoptStaticCall:
case RawPcDescriptors::kRuntimeCall: {
CodePatcher::PatchPoolPointerCallAt(pc_, code,

7
runtime/vm/debugger_ia32.cc
View file

@ -31,11 +31,14 @@ void CodeBreakpoint::PatchCode() {
{
WritableInstructionsScope writable(instrs.PayloadStart(), instrs.Size());
switch (breakpoint_kind_) {
case RawPcDescriptors::kIcCall:
case RawPcDescriptors::kUnoptStaticCall: {
case RawPcDescriptors::kIcCall: {
stub_target = StubCode::ICCallBreakpoint().raw();
break;
}
case RawPcDescriptors::kUnoptStaticCall: {
stub_target = StubCode::UnoptStaticCallBreakpoint().raw();
break;
}
case RawPcDescriptors::kRuntimeCall: {
saved_value_ = CodePatcher::GetStaticCallTargetAt(pc_, code);
stub_target = StubCode::RuntimeCallBreakpoint().raw();

4
runtime/vm/debugger_x64.cc
View file

@ -26,9 +26,11 @@ void CodeBreakpoint::PatchCode() {
Code& stub_target = Code::Handle();
switch (breakpoint_kind_) {
case RawPcDescriptors::kIcCall:
case RawPcDescriptors::kUnoptStaticCall:
stub_target = StubCode::ICCallBreakpoint().raw();
break;
case RawPcDescriptors::kUnoptStaticCall:
stub_target = StubCode::UnoptStaticCallBreakpoint().raw();
break;
case RawPcDescriptors::kRuntimeCall:
stub_target = StubCode::RuntimeCallBreakpoint().raw();
break;

18
runtime/vm/deferred_objects.cc
View file

@ -124,15 +124,15 @@ void DeferredRetAddr::Materialize(DeoptContext* deopt_context) {
if (pc != 0) {
// If the deoptimization happened at an IC call, update the IC data
// to avoid repeated deoptimization at the same site next time around.
ICData& ic_data = ICData::Handle(zone);
CodePatcher::GetInstanceCallAt(pc, code, &ic_data);
if (!ic_data.IsNull()) {
ic_data.AddDeoptReason(deopt_context->deopt_reason());
// Propagate the reason to all ICData-s with same deopt_id since
// only unoptimized-code ICData (IC calls) are propagated.
function.SetDeoptReasonForAll(ic_data.deopt_id(),
deopt_context->deopt_reason());
}
// We cannot use CodePatcher::GetInstanceCallAt because the call site
// may have switched to from referencing an ICData to a target Code or
// MegamorphicCache.
ICData& ic_data = ICData::Handle(zone, function.FindICData(deopt_id_));
ic_data.AddDeoptReason(deopt_context->deopt_reason());
// Propagate the reason to all ICData-s with same deopt_id since
// only unoptimized-code ICData (IC calls) are propagated.
function.SetDeoptReasonForAll(ic_data.deopt_id(),
deopt_context->deopt_reason());
} else {
if (deopt_context->HasDeoptFlag(ICData::kHoisted)) {
// Prevent excessive deoptimization.

56
runtime/vm/instructions_arm.cc
View file

@ -18,20 +18,35 @@ namespace dart {
CallPattern::CallPattern(uword pc, const Code& code)
: object_pool_(ObjectPool::Handle(code.GetObjectPool())),
end_(pc),
ic_data_load_end_(0),
target_code_pool_index_(-1),
ic_data_(ICData::Handle()) {
target_code_pool_index_(-1) {
ASSERT(code.ContainsInstructionAt(pc));
// Last instruction: blx lr.
ASSERT(*(reinterpret_cast<uword*>(end_) - 1) == 0xe12fff3e);
ASSERT(*(reinterpret_cast<uword*>(pc) - 1) == 0xe12fff3e);
Register reg;
ic_data_load_end_ = InstructionPattern::DecodeLoadWordFromPool(
end_ - 2 * Instr::kInstrSize, &reg, &target_code_pool_index_);
InstructionPattern::DecodeLoadWordFromPool(pc - 2 * Instr::kInstrSize, &reg,
&target_code_pool_index_);
ASSERT(reg == CODE_REG);
}
ICCallPattern::ICCallPattern(uword pc, const Code& code)
: object_pool_(ObjectPool::Handle(code.GetObjectPool())),
target_pool_index_(-1),
data_pool_index_(-1) {
ASSERT(code.ContainsInstructionAt(pc));
// Last instruction: blx lr.
ASSERT(*(reinterpret_cast<uword*>(pc) - 1) == 0xe12fff3e);
Register reg;
uword data_load_end = InstructionPattern::DecodeLoadWordFromPool(
pc - 2 * Instr::kInstrSize, &reg, &target_pool_index_);
ASSERT(reg == CODE_REG);
InstructionPattern::DecodeLoadWordFromPool(data_load_end, &reg,
&data_pool_index_);
ASSERT(reg == R9);
}
NativeCallPattern::NativeCallPattern(uword pc, const Code& code)
: object_pool_(ObjectPool::Handle(code.GetObjectPool())),
end_(pc),
@ -245,16 +260,6 @@ bool DecodeLoadObjectFromPoolOrThread(uword pc, const Code& code, Object* obj) {
return false;
}
RawICData* CallPattern::IcData() {
if (ic_data_.IsNull()) {
Register reg;
InstructionPattern::DecodeLoadObject(ic_data_load_end_, object_pool_, &reg,
&ic_data_);
ASSERT(reg == R9);
}
return ic_data_.raw();
}
RawCode* CallPattern::TargetCode() const {
return reinterpret_cast<RawCode*>(
object_pool_.ObjectAt(target_code_pool_index_));
@ -264,6 +269,23 @@ void CallPattern::SetTargetCode(const Code& target_code) const {
object_pool_.SetObjectAt(target_code_pool_index_, target_code);
}
RawObject* ICCallPattern::Data() const {
return object_pool_.ObjectAt(data_pool_index_);
}
void ICCallPattern::SetData(const Object& data) const {
ASSERT(data.IsArray() || data.IsICData() || data.IsMegamorphicCache());
object_pool_.SetObjectAt(data_pool_index_, data);
}
RawCode* ICCallPattern::TargetCode() const {
return reinterpret_cast<RawCode*>(object_pool_.ObjectAt(target_pool_index_));
}
void ICCallPattern::SetTargetCode(const Code& target_code) const {
object_pool_.SetObjectAt(target_pool_index_, target_code);
}
SwitchableCallPatternBase::SwitchableCallPatternBase(const Code& code)
: object_pool_(ObjectPool::Handle(code.GetObjectPool())),
data_pool_index_(-1),

27
runtime/vm/instructions_arm.h
View file

@ -72,7 +72,23 @@ class CallPattern : public ValueObject {
public:
CallPattern(uword pc, const Code& code);
RawICData* IcData();
RawCode* TargetCode() const;
void SetTargetCode(const Code& code) const;
private:
const ObjectPool& object_pool_;
intptr_t target_code_pool_index_;
DISALLOW_COPY_AND_ASSIGN(CallPattern);
};
class ICCallPattern : public ValueObject {
public:
ICCallPattern(uword pc, const Code& code);
RawObject* Data() const;
void SetData(const Object& data) const;
RawCode* TargetCode() const;
void SetTargetCode(const Code& code) const;
@ -80,13 +96,10 @@ class CallPattern : public ValueObject {
private:
const ObjectPool& object_pool_;
uword end_;
uword ic_data_load_end_;
intptr_t target_pool_index_;
intptr_t data_pool_index_;
intptr_t target_code_pool_index_;
ICData& ic_data_;
DISALLOW_COPY_AND_ASSIGN(CallPattern);
DISALLOW_COPY_AND_ASSIGN(ICCallPattern);
};
class NativeCallPattern : public ValueObject {

58
runtime/vm/instructions_arm64.cc
View file

@ -18,20 +18,36 @@ namespace dart {
CallPattern::CallPattern(uword pc, const Code& code)
: object_pool_(ObjectPool::Handle(code.GetObjectPool())),
end_(pc),
ic_data_load_end_(0),
target_code_pool_index_(-1),
ic_data_(ICData::Handle()) {
target_code_pool_index_(-1) {
ASSERT(code.ContainsInstructionAt(pc));
// Last instruction: blr ip0.
ASSERT(*(reinterpret_cast<uint32_t*>(end_) - 1) == 0xd63f0200);
ASSERT(*(reinterpret_cast<uint32_t*>(pc) - 1) == 0xd63f0200);
Register reg;
ic_data_load_end_ = InstructionPattern::DecodeLoadWordFromPool(
end_ - 2 * Instr::kInstrSize, &reg, &target_code_pool_index_);
InstructionPattern::DecodeLoadWordFromPool(pc - 2 * Instr::kInstrSize, &reg,
&target_code_pool_index_);
ASSERT(reg == CODE_REG);
}
ICCallPattern::ICCallPattern(uword pc, const Code& code)
: object_pool_(ObjectPool::Handle(code.GetObjectPool())),
target_pool_index_(-1),
data_pool_index_(-1) {
ASSERT(code.ContainsInstructionAt(pc));
// Last instruction: blr lr.
ASSERT(*(reinterpret_cast<uint32_t*>(pc) - 1) == 0xd63f03c0);
Register data_reg, code_reg;
intptr_t pool_index;
InstructionPattern::DecodeLoadDoubleWordFromPool(
pc - 2 * Instr::kInstrSize, &data_reg, &code_reg, &pool_index);
ASSERT(data_reg == R5);
ASSERT(code_reg == CODE_REG);
data_pool_index_ = pool_index;
target_pool_index_ = pool_index + 1;
}
NativeCallPattern::NativeCallPattern(uword pc, const Code& code)
: object_pool_(ObjectPool::Handle(code.GetObjectPool())),
end_(pc),
@ -353,16 +369,6 @@ void InstructionPattern::EncodeLoadWordFromPoolFixed(uword end,
instr->SetInstructionBits(instr->InstructionBits() | B22);
}
RawICData* CallPattern::IcData() {
if (ic_data_.IsNull()) {
Register reg;
InstructionPattern::DecodeLoadObject(ic_data_load_end_, object_pool_, &reg,
&ic_data_);
ASSERT(reg == R5);
}
return ic_data_.raw();
}
RawCode* CallPattern::TargetCode() const {
return reinterpret_cast<RawCode*>(
object_pool_.ObjectAt(target_code_pool_index_));
@ -373,6 +379,24 @@ void CallPattern::SetTargetCode(const Code& target) const {
// No need to flush the instruction cache, since the code is not modified.
}
RawObject* ICCallPattern::Data() const {
return object_pool_.ObjectAt(data_pool_index_);
}
void ICCallPattern::SetData(const Object& data) const {
ASSERT(data.IsArray() || data.IsICData() || data.IsMegamorphicCache());
object_pool_.SetObjectAt(data_pool_index_, data);
}
RawCode* ICCallPattern::TargetCode() const {
return reinterpret_cast<RawCode*>(object_pool_.ObjectAt(target_pool_index_));
}
void ICCallPattern::SetTargetCode(const Code& target) const {
object_pool_.SetObjectAt(target_pool_index_, target);
// No need to flush the instruction cache, since the code is not modified.
}
SwitchableCallPatternBase::SwitchableCallPatternBase(const Code& code)
: object_pool_(ObjectPool::Handle(code.GetObjectPool())),
data_pool_index_(-1),

27
runtime/vm/instructions_arm64.h
View file

@ -82,7 +82,23 @@ class CallPattern : public ValueObject {
public:
CallPattern(uword pc, const Code& code);
RawICData* IcData();
RawCode* TargetCode() const;
void SetTargetCode(const Code& target) const;
private:
const ObjectPool& object_pool_;
intptr_t target_code_pool_index_;
DISALLOW_COPY_AND_ASSIGN(CallPattern);
};
class ICCallPattern : public ValueObject {
public:
ICCallPattern(uword pc, const Code& caller_code);
RawObject* Data() const;
void SetData(const Object& data) const;
RawCode* TargetCode() const;
void SetTargetCode(const Code& target) const;
@ -90,13 +106,10 @@ class CallPattern : public ValueObject {
private:
const ObjectPool& object_pool_;
uword end_;
uword ic_data_load_end_;
intptr_t target_pool_index_;
intptr_t data_pool_index_;
intptr_t target_code_pool_index_;
ICData& ic_data_;
DISALLOW_COPY_AND_ASSIGN(CallPattern);
DISALLOW_COPY_AND_ASSIGN(ICCallPattern);
};
class NativeCallPattern : public ValueObject {

28
runtime/vm/instructions_dbc.cc
View file

@ -54,15 +54,14 @@ static bool GetLoadedObjectAt(uword pc,
CallPattern::CallPattern(uword pc, const Code& code)
: object_pool_(ObjectPool::Handle(code.GetObjectPool())),
end_(pc),
ic_data_load_end_(0),
target_code_pool_index_(-1),
ic_data_(ICData::Handle()) {
data_pool_index_(-1),
target_pool_index_(-1) {
ASSERT(code.ContainsInstructionAt(end_));
const uword call_pc = end_ - sizeof(Instr);
Instr call_instr = SimulatorBytecode::At(call_pc);
ASSERT(SimulatorBytecode::IsCallOpcode(call_instr));
ic_data_load_end_ = call_pc;
target_code_pool_index_ = SimulatorBytecode::DecodeD(call_instr);
data_pool_index_ = SimulatorBytecode::DecodeD(call_instr);
target_pool_index_ = SimulatorBytecode::DecodeD(call_instr);
}
NativeCallPattern::NativeCallPattern(uword pc, const Code& code)
@ -143,21 +142,20 @@ bool DecodeLoadObjectFromPoolOrThread(uword pc, const Code& code, Object* obj) {
return GetLoadedObjectAt(pc, pool, obj);
}
RawICData* CallPattern::IcData() {
if (ic_data_.IsNull()) {
bool found = GetLoadedObjectAt(ic_data_load_end_, object_pool_, &ic_data_);
ASSERT(found);
}
return ic_data_.raw();
RawObject* CallPattern::Data() const {
return object_pool_.ObjectAt(data_pool_index_);
}
void CallPattern::SetData(const Object& data) const {
object_pool_.SetObjectAt(data_pool_index_, data);
}
RawCode* CallPattern::TargetCode() const {
return reinterpret_cast<RawCode*>(
object_pool_.ObjectAt(target_code_pool_index_));
return reinterpret_cast<RawCode*>(object_pool_.ObjectAt(target_pool_index_));
}
void CallPattern::SetTargetCode(const Code& target_code) const {
object_pool_.SetObjectAt(target_code_pool_index_, target_code);
void CallPattern::SetTargetCode(const Code& target) const {
object_pool_.SetObjectAt(target_pool_index_, target);
}
void CallPattern::InsertDeoptCallAt(uword pc) {

10
runtime/vm/instructions_dbc.h
View file

@ -49,9 +49,10 @@ class InstructionPattern : public AllStatic {
class CallPattern : public ValueObject {
public:
CallPattern(uword pc, const Code& code);
CallPattern(uword pc, const Code& caller_code);
RawICData* IcData();
RawObject* Data() const;
void SetData(const Object& data) const;
RawCode* TargetCode() const;
void SetTargetCode(const Code& code) const;
@ -62,10 +63,9 @@ class CallPattern : public ValueObject {
const ObjectPool& object_pool_;
uword end_;
uword ic_data_load_end_;
intptr_t target_code_pool_index_;
ICData& ic_data_;
intptr_t data_pool_index_;
intptr_t target_pool_index_;
DISALLOW_COPY_AND_ASSIGN(CallPattern);
};

6
runtime/vm/interpreter.cc
View file

@ -565,6 +565,12 @@ DART_NOINLINE bool Interpreter::InvokeCompiled(Thread* thread,
USE(entrypoint);
UNIMPLEMENTED();
#elif defined(USING_SIMULATOR)
// We need to beware that bouncing between the interpreter and the
// simulator may exhaust the C stack before exhausting either the
// interpreter or simulator stacks.
if (!thread->os_thread()->HasStackHeadroom()) {
thread->SetStackLimit(-1);
}
result = bit_copy<RawObject*, int64_t>(
Simulator::Current()->Call(reinterpret_cast<intptr_t>(entrypoint),
reinterpret_cast<intptr_t>(code),

8
runtime/vm/isolate.h
View file

@ -795,6 +795,13 @@ class Isolate : public BaseIsolate {
UsingNewBytecodeInstructionsBit::update(value, isolate_flags_);
}
bool has_attempted_stepping() const {
return HasAttemptedSteppingBit::decode(isolate_flags_);
}
void set_has_attempted_stepping(bool value) {
isolate_flags_ = HasAttemptedSteppingBit::update(value, isolate_flags_);
}
static void KillAllIsolates(LibMsgId msg_id);
static void KillIfExists(Isolate* isolate, LibMsgId msg_id);
@ -909,6 +916,7 @@ class Isolate : public BaseIsolate {
V(RemappingCids) \
V(ResumeRequest) \
V(HasAttemptedReload) \
V(HasAttemptedStepping) \
V(ShouldPausePostServiceRequest) \
V(EnableTypeChecks) \
V(EnableAsserts) \

3
runtime/vm/isolate_reload.cc
View file

@ -1907,8 +1907,10 @@ void IsolateReloadContext::ResetUnoptimizedICsOnStack() {
function = code.function();
code = function.unoptimized_code();
ASSERT(!code.IsNull());
code.ResetSwitchableCalls(zone);
code.ResetICDatas(zone);
} else {
code.ResetSwitchableCalls(zone);
code.ResetICDatas(zone);
}
}
@ -2029,6 +2031,7 @@ void IsolateReloadContext::RunInvalidationVisitors() {
if (!stub_code) {
// We are preserving the unoptimized code, fill all ICData arrays with
// the sentinel values so that we have no stale type feedback.
code.ResetSwitchableCalls(zone);
code.ResetICDatas(zone);
}
if (!bytecode.IsNull()) {

10
runtime/vm/native_arguments.h
View file

@ -101,6 +101,14 @@ class NativeArguments {
return *arg_ptr;
}
void SetArgAt(int index, const Object& value) const {
ASSERT(thread_->execution_state() == Thread::kThreadInVM);
ASSERT((index >= 0) && (index < ArgCount()));
RawObject** arg_ptr =
&(argv_[ReverseArgOrderBit::decode(argc_tag_) ? index : -index]);
*arg_ptr = value.raw();
}
// Does not include hidden type arguments vector.
int NativeArgCount() const {
int function_bits = FunctionBits::decode(argc_tag_);
@ -158,8 +166,6 @@ class NativeArguments {
return type_args.TypeAt(index);
}
RawObject** ReturnValueAddress() const { return retval_; }
void SetReturn(const Object& value) const {
ASSERT(thread_->execution_state() == Thread::kThreadInVM);
*retval_ = value.raw();

55
runtime/vm/object.cc
View file

@ -8006,6 +8006,18 @@ void Function::ClearICDataArray() const {
set_ic_data_array(Array::null_array());
}
RawICData* Function::FindICData(intptr_t deopt_id) const {
const Array& array = Array::Handle(ic_data_array());
ICData& ic_data = ICData::Handle();
for (intptr_t i = 1; i < array.Length(); i++) {
ic_data ^= array.At(i);
if (ic_data.deopt_id() == deopt_id) {
return ic_data.raw();
}
}
return ICData::null();
}
void Function::SetDeoptReasonForAll(intptr_t deopt_id,
ICData::DeoptReasonId reason) {
const Array& array = Array::Handle(ic_data_array());
@ -8049,6 +8061,10 @@ RawCode* Function::EnsureHasCode() const {
const Object& result =
Object::Handle(zone, Compiler::CompileFunction(thread, *this));
if (result.IsError()) {
if (result.IsLanguageError()) {
Exceptions::ThrowCompileTimeError(LanguageError::Cast(result));
UNREACHABLE();
}
Exceptions::PropagateError(Error::Cast(result));
UNREACHABLE();
}
@ -14121,19 +14137,10 @@ RawICData* ICData::AsUnaryClassChecksSortedByCount() const {
return result.raw();
}
bool ICData::AllTargetsHaveSameOwner(intptr_t owner_cid) const {
if (NumberOfChecksIs(0)) return false;
Class& cls = Class::Handle();
const intptr_t len = NumberOfChecks();
for (intptr_t i = 0; i < len; i++) {
if (IsUsedAt(i)) {
cls = Function::Handle(GetTargetAt(i)).Owner();
if (cls.id() != owner_cid) {
return false;
}
}
}
return true;
RawMegamorphicCache* ICData::AsMegamorphicCache() const {
const String& name = String::Handle(target_name());
const Array& descriptor = Array::Handle(arguments_descriptor());
return MegamorphicCacheTable::Lookup(Isolate::Current(), name, descriptor);
}
bool ICData::HasReceiverClassId(intptr_t class_id) const {
@ -14152,6 +14159,8 @@ bool ICData::HasReceiverClassId(intptr_t class_id) const {
// Returns true if all targets are the same.
// TODO(srdjan): if targets are native use their C_function to compare.
// TODO(rmacnak): this question should only be asked against a CallTargets,
// not an ICData.
bool ICData::HasOneTarget() const {
ASSERT(!NumberOfChecksIs(0));
const Function& first_target = Function::Handle(GetTargetAt(0));
@ -14161,6 +14170,23 @@ bool ICData::HasOneTarget() const {
return false;
}
}
if (is_megamorphic()) {
const MegamorphicCache& cache =
MegamorphicCache::Handle(AsMegamorphicCache());
SafepointMutexLocker ml(Isolate::Current()->megamorphic_lookup_mutex());
MegamorphicCacheEntries entries(Array::Handle(cache.buckets()));
for (intptr_t i = 0; i < entries.Length(); i++) {
const intptr_t id =
Smi::Value(entries[i].Get<MegamorphicCache::kClassIdIndex>());
if (id == kIllegalCid) {
continue;
}
if (entries[i].Get<MegamorphicCache::kTargetFunctionIndex>() !=
first_target.raw()) {
return false;
}
}
}
return true;
}
@ -14319,6 +14345,7 @@ RawICData* ICData::NewFrom(const ICData& from, intptr_t num_args_tested) {
AbstractType::Handle(from.receivers_static_type())));
// Copy deoptimization reasons.
result.SetDeoptReasons(from.DeoptReasons());
result.set_is_megamorphic(from.is_megamorphic());
return result.raw();
}
@ -14343,6 +14370,7 @@ RawICData* ICData::Clone(const ICData& from) {
result.set_entries(cloned_array);
// Copy deoptimization reasons.
result.SetDeoptReasons(from.DeoptReasons());
result.set_is_megamorphic(from.is_megamorphic());
return result.raw();
}
#endif
@ -15675,6 +15703,7 @@ void MegamorphicCache::EnsureCapacity() const {
void MegamorphicCache::Insert(const Smi& class_id, const Object& target) const {
ASSERT(static_cast<double>(filled_entry_count() + 1) <=
(kLoadFactor * static_cast<double>(mask() + 1)));
SafepointMutexLocker ml(Isolate::Current()->megamorphic_lookup_mutex());
const Array& backing_array = Array::Handle(buckets());
intptr_t id_mask = mask();
intptr_t index = (class_id.Value() * kSpreadFactor) & id_mask;

96
runtime/vm/object.h
View file

@ -1697,6 +1697,18 @@ class ICData : public Object {
RebindRule rebind_rule() const;
void set_rebind_rule(uint32_t rebind_rule) const;
// This bit is set when a call site becomes megamorphic and starts using a
// MegamorphicCache instead of ICData. It means that the entries in the
// ICData are incomplete and the MegamorphicCache needs to also be consulted
// to list the call site's observed receiver classes and targets.
bool is_megamorphic() const {
return MegamorphicBit::decode(raw_ptr()->state_bits_);
}
void set_is_megamorphic(bool value) const {
StoreNonPointer(&raw_ptr()->state_bits_,
MegamorphicBit::update(value, raw_ptr()->state_bits_));
}
// The length of the array. This includes all sentinel entries including
// the final one.
intptr_t Length() const;
@ -1826,9 +1838,9 @@ class ICData : public Object {
// Used for printing and optimizations.
RawICData* AsUnaryClassChecksSortedByCount() const;
RawMegamorphicCache* AsMegamorphicCache() const;
// Consider only used entries.
bool AllTargetsHaveSameOwner(intptr_t owner_cid) const;
bool AllReceiversAreNumbers() const;
bool HasOneTarget() const;
bool HasReceiverClassId(intptr_t class_id) const;
@ -1889,13 +1901,13 @@ class ICData : public Object {
intptr_t FindCheck(const GrowableArray<intptr_t>& cids) const;
private:
static RawICData* New();
RawArray* entries() const {
return AtomicOperations::LoadAcquire(&raw_ptr()->entries_);
}
private:
static RawICData* New();
// Grows the array and also sets the argument to the index that should be used
// for the new entry.
RawArray* Grow(intptr_t* index) const;
@ -1918,7 +1930,9 @@ class ICData : public Object {
kDeoptReasonPos = kTrackingExactnessPos + kTrackingExactnessSize,
kDeoptReasonSize = kLastRecordedDeoptReason + 1,
kRebindRulePos = kDeoptReasonPos + kDeoptReasonSize,
kRebindRuleSize = 3
kRebindRuleSize = 3,
kMegamorphicPos = kRebindRulePos + kRebindRuleSize,
kMegamorphicSize = 1,
};
COMPILE_ASSERT(kNumRebindRules <= (1 << kRebindRuleSize));
@ -1939,6 +1953,9 @@ class ICData : public Object {
uint32_t,
ICData::kRebindRulePos,
ICData::kRebindRuleSize> {};
class MegamorphicBit
: public BitField<uint32_t, bool, kMegamorphicPos, kMegamorphicSize> {};
#if defined(DEBUG)
// Used in asserts to verify that a check is not added twice.
bool HasCheck(const GrowableArray<intptr_t>& cids) const;
@ -2863,6 +2880,7 @@ class Function : public Object {
RawArray* ic_data_array() const;
void ClearICDataArray() const;
RawICData* FindICData(intptr_t deopt_id) const;
// Sets deopt reason in all ICData-s with given deopt_id.
void SetDeoptReasonForAll(intptr_t deopt_id, ICData::DeoptReasonId reason);
@ -4488,20 +4506,30 @@ class Instructions : public Object {
// Note: We keep the checked entrypoint offsets even (emitting NOPs if
// necessary) to allow them to be seen as Smis by the GC.
#if defined(TARGET_ARCH_IA32)
static const intptr_t kPolymorphicEntryOffset = 0;
static const intptr_t kMonomorphicEntryOffset = 0;
static const intptr_t kMonomorphicEntryOffsetJIT = 6;
static const intptr_t kPolymorphicEntryOffsetJIT = 34;
static const intptr_t kMonomorphicEntryOffsetAOT = 0;
static const intptr_t kPolymorphicEntryOffsetAOT = 0;
#elif defined(TARGET_ARCH_X64)
static const intptr_t kPolymorphicEntryOffset = 8;
static const intptr_t kMonomorphicEntryOffset = 32;
static const intptr_t kMonomorphicEntryOffsetJIT = 8;
static const intptr_t kPolymorphicEntryOffsetJIT = 40;
static const intptr_t kMonomorphicEntryOffsetAOT = 8;
static const intptr_t kPolymorphicEntryOffsetAOT = 32;
#elif defined(TARGET_ARCH_ARM)
static const intptr_t kPolymorphicEntryOffset = 0;
static const intptr_t kMonomorphicEntryOffset = 20;
static const intptr_t kMonomorphicEntryOffsetJIT = 0;
static const intptr_t kPolymorphicEntryOffsetJIT = 40;
static const intptr_t kMonomorphicEntryOffsetAOT = 0;
static const intptr_t kPolymorphicEntryOffsetAOT = 20;
#elif defined(TARGET_ARCH_ARM64)
static const intptr_t kPolymorphicEntryOffset = 8;
static const intptr_t kMonomorphicEntryOffset = 28;
static const intptr_t kMonomorphicEntryOffsetJIT = 8;
static const intptr_t kPolymorphicEntryOffsetJIT = 48;
static const intptr_t kMonomorphicEntryOffsetAOT = 8;
static const intptr_t kPolymorphicEntryOffsetAOT = 28;
#elif defined(TARGET_ARCH_DBC)
static const intptr_t kPolymorphicEntryOffset = 0;
static const intptr_t kMonomorphicEntryOffset = 0;
static const intptr_t kMonomorphicEntryOffsetJIT = 0;
static const intptr_t kPolymorphicEntryOffsetJIT = 0;
static const intptr_t kMonomorphicEntryOffsetAOT = 0;
static const intptr_t kPolymorphicEntryOffsetAOT = 0;
#else
#error Missing entry offsets for current architecture
#endif
@ -4509,7 +4537,8 @@ class Instructions : public Object {
static uword MonomorphicEntryPoint(const RawInstructions* instr) {
uword entry = PayloadStart(instr);
if (!HasSingleEntryPoint(instr)) {
entry += kPolymorphicEntryOffset;
entry += !FLAG_precompiled_mode ? kMonomorphicEntryOffsetJIT
: kMonomorphicEntryOffsetAOT;
}
return entry;
}
@ -4517,7 +4546,8 @@ class Instructions : public Object {
static uword EntryPoint(const RawInstructions* instr) {
uword entry = PayloadStart(instr);
if (!HasSingleEntryPoint(instr)) {
entry += kMonomorphicEntryOffset;
entry += !FLAG_precompiled_mode ? kPolymorphicEntryOffsetJIT
: kPolymorphicEntryOffsetAOT;
}
return entry;
}
@ -4526,7 +4556,8 @@ class Instructions : public Object {
uword entry =
PayloadStart(instr) + instr->ptr()->unchecked_entrypoint_pc_offset_;
if (!HasSingleEntryPoint(instr)) {
entry += kMonomorphicEntryOffset;
entry += !FLAG_precompiled_mode ? kPolymorphicEntryOffsetJIT
: kPolymorphicEntryOffsetAOT;
}
return entry;
}
@ -4535,7 +4566,8 @@ class Instructions : public Object {
uword entry =
PayloadStart(instr) + instr->ptr()->unchecked_entrypoint_pc_offset_;
if (!HasSingleEntryPoint(instr)) {
entry += kPolymorphicEntryOffset;
entry += !FLAG_precompiled_mode ? kMonomorphicEntryOffsetJIT
: kMonomorphicEntryOffsetAOT;
}
return entry;
}
@ -5038,6 +5070,10 @@ class Code : public Object {
return OptimizedBit::decode(raw_ptr()->state_bits_);
}
void set_is_optimized(bool value) const;
static bool IsOptimized(RawCode* code) {
return Code::OptimizedBit::decode(code->ptr()->state_bits_);
}
bool is_alive() const { return AliveBit::decode(raw_ptr()->state_bits_); }
void set_is_alive(bool value) const;
@ -5090,6 +5126,7 @@ class Code : public Object {
// Used during reloading (see object_reload.cc). Calls Reset on all ICDatas
// that are embedded inside the Code or ObjecPool objects.
void ResetICDatas(Zone* zone) const;
void ResetSwitchableCalls(Zone* zone) const;
// Array of DeoptInfo objects.
RawArray* deopt_info_array() const {
@ -5417,10 +5454,6 @@ class Code : public Object {
DISALLOW_COPY_AND_ASSIGN(SlowFindRawCodeVisitor);
};
static bool IsOptimized(RawCode* code) {
return Code::OptimizedBit::decode(code->ptr()->state_bits_);
}
static const intptr_t kEntrySize = sizeof(int32_t); // NOLINT
void set_compile_timestamp(int64_t timestamp) const {
@ -5771,6 +5804,12 @@ class MegamorphicCache : public Object {
static const intptr_t kSpreadFactor = 7;
static const double kLoadFactor;
enum EntryType {
kClassIdIndex,
kTargetFunctionIndex,
kEntryLength,
};
RawArray* buckets() const;
void set_buckets(const Array& buckets) const;
@ -5817,12 +5856,6 @@ class MegamorphicCache : public Object {
void set_target_name(const String& value) const;
void set_arguments_descriptor(const Array& value) const;
enum {
kClassIdIndex,
kTargetFunctionIndex,
kEntryLength,
};
static inline void SetEntry(const Array& array,
intptr_t index,
const Smi& class_id,
@ -9960,6 +9993,9 @@ using SubtypeTestCacheTable = ArrayOfTuplesView<SubtypeTestCache::Entries,
TypeArguments,
TypeArguments>>;
using MegamorphicCacheEntries =
ArrayOfTuplesView<MegamorphicCache::EntryType, std::tuple<Smi, Object>>;
void DumpTypeTable(Isolate* isolate);
void DumpTypeArgumentsTable(Isolate* isolate);

60
runtime/vm/object_reload.cc
View file

@ -4,10 +4,13 @@
#include "vm/object.h"
#include "vm/code_patcher.h"
#include "vm/hash_table.h"
#include "vm/isolate_reload.h"
#include "vm/log.h"
#include "vm/object_store.h"
#include "vm/resolver.h"
#include "vm/stub_code.h"
#include "vm/symbols.h"
namespace dart {
@ -89,6 +92,63 @@ void Code::ResetICDatas(Zone* zone) const {
#endif
}
void Code::ResetSwitchableCalls(Zone* zone) const {
#if !defined(TARGET_ARCH_DBC)
if (is_optimized()) {
return; // No switchable calls in optimized code.
}
const Object& owner = Object::Handle(zone, this->owner());
if (!owner.IsFunction()) {
return; // No switchable calls in stub code.
}
const Array& ic_data_array =
Array::Handle(zone, Function::Cast(owner).ic_data_array());
if (ic_data_array.IsNull()) {
// The megamorphic miss stub and some recognized function doesn't populate
// their ic_data_array. Check this only happens for functions without IC
// calls.
#if defined(DEBUG)
const PcDescriptors& descriptors =
PcDescriptors::Handle(zone, pc_descriptors());
PcDescriptors::Iterator iter(descriptors, RawPcDescriptors::kIcCall);
while (iter.MoveNext()) {
FATAL1("%s has IC calls but no ic_data_array\n", owner.ToCString());
}
#endif
return;
}
ICData& ic_data = ICData::Handle(zone);
Object& data = Object::Handle(zone);
for (intptr_t i = 1; i < ic_data_array.Length(); i++) {
ic_data ^= ic_data_array.At(i);
if (ic_data.rebind_rule() != ICData::kInstance) {
continue;
}
if (ic_data.NumArgsTested() != 1) {
continue;
}
uword pc = GetPcForDeoptId(ic_data.deopt_id(), RawPcDescriptors::kIcCall);
CodePatcher::GetInstanceCallAt(pc, *this, &data);
// This check both avoids unnecessary patching to reduce log spam and
// prevents patching over breakpoint stubs.
if (!data.IsICData()) {
const Code& stub =
ic_data.is_tracking_exactness()
? StubCode::OneArgCheckInlineCacheWithExactnessCheck()
: StubCode::OneArgCheckInlineCache();
CodePatcher::PatchInstanceCallAt(pc, *this, ic_data, stub);
if (FLAG_trace_ic) {
OS::PrintErr("Instance call at %" Px
" resetting to polymorphic dispatch, %s\n",
pc, ic_data.ToCString());
}
}
}
#endif
}
void Bytecode::ResetICDatas(Zone* zone) const {
// Iterate over the Bytecode's object pool and reset all ICDatas.
const ObjectPool& pool = ObjectPool::Handle(zone, object_pool());

235
runtime/vm/runtime_entry.cc
View file

@ -1075,6 +1075,87 @@ RawFunction* InlineCacheMissHelper(const Instance& receiver,
return result.raw();
}
static void TrySwitchInstanceCall(const ICData& ic_data,
const Function& target_function) {
#if !defined(TARGET_ARCH_DBC) && !defined(DART_PRECOMPILED_RUNTIME)
// Monomorphic/megamorphic calls only check the receiver CID.
if (ic_data.NumArgsTested() != 1) return;
ASSERT(ic_data.rebind_rule() == ICData::kInstance);
// Monomorphic/megamorphic calls don't record exactness.
if (ic_data.is_tracking_exactness()) return;
#if !defined(PRODUCT)
// Monomorphic/megamorphic do not check the isolate's stepping flag.
if (Isolate::Current()->has_attempted_stepping()) return;
#endif
Thread* thread = Thread::Current();
DartFrameIterator iterator(thread,
StackFrameIterator::kNoCrossThreadIteration);
StackFrame* caller_frame = iterator.NextFrame();
ASSERT(caller_frame->IsDartFrame());
// Monomorphic/megamorphic calls are only for unoptimized code.
if (caller_frame->is_interpreted()) return;
Zone* zone = thread->zone();
const Code& caller_code = Code::Handle(zone, caller_frame->LookupDartCode());
if (caller_code.is_optimized()) return;
// Code is detached from its function. This will prevent us from resetting
// the switchable call later because resets are function based and because
// the ic_data_array belongs to the function instead of the code. This should
// only happen because of reload, but it sometimes happens with KBC mixed mode
// probably through a race between foreground and background compilation.
const Function& caller_function =
Function::Handle(zone, caller_code.function());
if (caller_function.unoptimized_code() != caller_code.raw()) {
return;
}
intptr_t num_checks = ic_data.NumberOfChecks();
// Monomorphic call.
if (num_checks == 1) {
// A call site in the monomorphic state does not load the arguments
// descriptor, so do not allow transition to this state if the callee
// needs it.
if (target_function.HasOptionalParameters() ||
target_function.IsGeneric()) {
return;
}
const Array& data = Array::Handle(zone, ic_data.entries());
const Code& target = Code::Handle(zone, target_function.EnsureHasCode());
CodePatcher::PatchInstanceCallAt(caller_frame->pc(), caller_code, data,
target);
if (FLAG_trace_ic) {
OS::PrintErr("Instance call at %" Px
" switching to monomorphic dispatch, %s\n",
caller_frame->pc(), ic_data.ToCString());
}
return; // Success.
}
// Megamorphic call.
if (num_checks > FLAG_max_polymorphic_checks) {
const MegamorphicCache& cache =
MegamorphicCache::Handle(zone, ic_data.AsMegamorphicCache());
ic_data.set_is_megamorphic(true);
CodePatcher::PatchInstanceCallAt(caller_frame->pc(), caller_code, cache,
StubCode::MegamorphicCall());
if (FLAG_trace_ic) {
OS::PrintErr("Instance call at %" Px
" switching to megamorphic dispatch, %s\n",
caller_frame->pc(), ic_data.ToCString());
}
return; // Success.
}
#endif // !defined(TARGET_ARCH_DBC) && !defined(DART_PRECOMPILED_RUNTIME)
}
// Perform the subtype and return constant function based on the result.
static RawFunction* ComputeTypeCheckTarget(const Instance& receiver,
const AbstractType& type,
@ -1091,7 +1172,8 @@ static RawFunction* ComputeTypeCheckTarget(const Instance& receiver,
static RawFunction* InlineCacheMissHandler(
const GrowableArray<const Instance*>& args, // Checked arguments only.
const ICData& ic_data) {
const ICData& ic_data,
intptr_t count = 1) {
const Instance& receiver = *args[0];
ArgumentsDescriptor arguments_descriptor(
Array::Handle(ic_data.arguments_descriptor()));
@ -1136,13 +1218,13 @@ static RawFunction* InlineCacheMissHandler(
ic_data.receivers_static_type())),
receiver);
ic_data.AddReceiverCheck(
receiver.GetClassId(), target_function,
/*count=*/1, /*exactness=*/state.CollapseSuperTypeExactness());
receiver.GetClassId(), target_function, count,
/*exactness=*/state.CollapseSuperTypeExactness());
#else
UNREACHABLE();
#endif
} else {
ic_data.AddReceiverCheck(args[0]->GetClassId(), target_function);
ic_data.AddReceiverCheck(args[0]->GetClassId(), target_function, count);
}
} else {
GrowableArray<intptr_t> class_ids(args.length());
@ -1150,7 +1232,7 @@ static RawFunction* InlineCacheMissHandler(
for (intptr_t i = 0; i < args.length(); i++) {
class_ids.Add(args[i]->GetClassId());
}
ic_data.AddCheck(class_ids, target_function);
ic_data.AddCheck(class_ids, target_function, count);
}
if (FLAG_trace_ic_miss_in_optimized || FLAG_trace_ic) {
DartFrameIterator iterator(Thread::Current(),
@ -1174,6 +1256,9 @@ static RawFunction* InlineCacheMissHandler(
receiver.GetClassId(), target_function.ToCString());
}
}
TrySwitchInstanceCall(ic_data, target_function);
return target_function.raw();
}
@ -1289,9 +1374,10 @@ static bool IsSingleTarget(Isolate* isolate,
#endif
// Handle a miss of a single target cache.
// Arg0: Receiver.
// Arg1: Receiver.
// Arg0: Stub out.
// Returns: the ICData used to continue with a polymorphic call.
DEFINE_RUNTIME_ENTRY(SingleTargetMiss, 1) {
DEFINE_RUNTIME_ENTRY(SingleTargetMiss, 2) {
#if defined(TARGET_ARCH_DBC)
// DBC does not use switchable calls.
UNREACHABLE();
@ -1360,6 +1446,7 @@ DEFINE_RUNTIME_ENTRY(SingleTargetMiss, 1) {
cache.set_upper_limit(upper);
// Return the ICData. The single target stub will jump to continue in the
// IC call stub.
arguments.SetArgAt(0, StubCode::ICCallThroughCode());
arguments.SetReturn(ic_data);
return;
}
@ -1373,18 +1460,24 @@ DEFINE_RUNTIME_ENTRY(SingleTargetMiss, 1) {
// Return the ICData. The single target stub will jump to continue in the
// IC call stub.
arguments.SetArgAt(0, stub);
arguments.SetReturn(ic_data);
#endif
}
DEFINE_RUNTIME_ENTRY(UnlinkedCall, 2) {
// Handle the first use of an instance call
// Arg2: UnlinkedCall.
// Arg1: Receiver.
// Arg0: Stub out.
// Returns: the ICData used to continue with a polymorphic call.
DEFINE_RUNTIME_ENTRY(UnlinkedCall, 3) {
#if defined(TARGET_ARCH_DBC)
// DBC does not use switchable calls.
UNREACHABLE();
#else
const Instance& receiver = Instance::CheckedHandle(zone, arguments.ArgAt(0));
const Instance& receiver = Instance::CheckedHandle(zone, arguments.ArgAt(1));
const UnlinkedCall& unlinked =
UnlinkedCall::CheckedHandle(zone, arguments.ArgAt(1));
UnlinkedCall::CheckedHandle(zone, arguments.ArgAt(2));
DartFrameIterator iterator(thread,
StackFrameIterator::kNoCrossThreadIteration);
@ -1426,6 +1519,7 @@ DEFINE_RUNTIME_ENTRY(UnlinkedCall, 2) {
// Return the ICData. The miss stub will jump to continue in the IC call
// stub.
arguments.SetArgAt(0, StubCode::ICCallThroughCode());
arguments.SetReturn(ic_data);
return;
}
@ -1438,19 +1532,41 @@ DEFINE_RUNTIME_ENTRY(UnlinkedCall, 2) {
// Return the ICData. The miss stub will jump to continue in the IC lookup
// stub.
arguments.SetArgAt(0, stub);
arguments.SetReturn(ic_data);
#endif // !DBC
}
#if !defined(DART_PRECOMPILED_RUNTIME) && !defined(TARGET_ARCH_DBC)
static RawICData* FindICDataForInstanceCall(Zone* zone,
const Code& code,
uword pc) {
uword pc_offset = pc - code.PayloadStart();
const PcDescriptors& descriptors =
PcDescriptors::Handle(zone, code.pc_descriptors());
PcDescriptors::Iterator iter(descriptors, RawPcDescriptors::kIcCall);
intptr_t deopt_id = -1;
while (iter.MoveNext()) {
if (iter.PcOffset() == pc_offset) {
deopt_id = iter.DeoptId();
break;
}
}
ASSERT(deopt_id != -1);
return Function::Handle(zone, code.function()).FindICData(deopt_id);
}
#endif // !defined(DART_PRECOMPILED_RUNTIME) && !defined(TARGET_ARCH_DBC)
// Handle a miss of a megamorphic cache.
// Arg0: Receiver.
// Arg1: Receiver.
// Arg0: continuation Code (out parameter).
// Returns: the ICData used to continue with a polymorphic call.
DEFINE_RUNTIME_ENTRY(MonomorphicMiss, 1) {
DEFINE_RUNTIME_ENTRY(MonomorphicMiss, 2) {
#if defined(TARGET_ARCH_DBC)
// DBC does not use switchable calls.
UNREACHABLE();
#else
const Instance& receiver = Instance::CheckedHandle(zone, arguments.ArgAt(0));
#elif defined(DART_PRECOMPILED_RUNTIME)
const Instance& receiver = Instance::CheckedHandle(zone, arguments.ArgAt(1));
DartFrameIterator iterator(thread,
StackFrameIterator::kNoCrossThreadIteration);
@ -1520,6 +1636,7 @@ DEFINE_RUNTIME_ENTRY(MonomorphicMiss, 1) {
stub);
// Return the ICData. The miss stub will jump to continue in the IC call
// stub.
arguments.SetArgAt(0, StubCode::ICCallThroughCode());
arguments.SetReturn(ic_data);
return;
}
@ -1533,6 +1650,46 @@ DEFINE_RUNTIME_ENTRY(MonomorphicMiss, 1) {
// Return the ICData. The miss stub will jump to continue in the IC lookup
// stub.
arguments.SetArgAt(0, stub);
arguments.SetReturn(ic_data);
#else // JIT
DartFrameIterator iterator(thread,
StackFrameIterator::kNoCrossThreadIteration);
StackFrame* caller_frame = iterator.NextFrame();
ASSERT(caller_frame->IsDartFrame());
ASSERT(!caller_frame->is_interpreted());
const Code& caller_code = Code::Handle(zone, caller_frame->LookupDartCode());
ASSERT(!caller_code.is_optimized());
const ICData& ic_data = ICData::Handle(
zone, FindICDataForInstanceCall(zone, caller_code, caller_frame->pc()));
RELEASE_ASSERT(!ic_data.IsNull());
ASSERT(ic_data.NumArgsTested() == 1);
const Code& stub = ic_data.is_tracking_exactness()
? StubCode::OneArgCheckInlineCacheWithExactnessCheck()
: StubCode::OneArgCheckInlineCache();
CodePatcher::PatchInstanceCallAt(caller_frame->pc(), caller_code, ic_data,
stub);
if (FLAG_trace_ic) {
OS::PrintErr("Instance call at %" Px
" switching to polymorphic dispatch, %s\n",
caller_frame->pc(), ic_data.ToCString());
}
const Instance& receiver = Instance::CheckedHandle(zone, arguments.ArgAt(1));
// ICData can be shared between unoptimized and optimized code, so beware that
// the new receiver class may have already been added through the optimized
// code.
if (!ic_data.HasReceiverClassId(receiver.GetClassId())) {
GrowableArray<const Instance*> args(1);
args.Add(&receiver);
// Don't count during insertion because the IC stub we continue through will
// do an increment.
intptr_t count = 0;
InlineCacheMissHandler(args, ic_data, count);
}
arguments.SetArgAt(0, stub);
arguments.SetReturn(ic_data);
#endif // !defined(TARGET_ARCH_DBC)
}
@ -2077,7 +2234,7 @@ DEFINE_RUNTIME_ENTRY(StackOverflow, 0) {
// process the stack overflow now and leave the interrupt for next
// time.
// TODO(regis): Warning: IsCalleeFrameOf is overridden in stack_frame_dbc.h.
if (interpreter_stack_overflow ||
if (interpreter_stack_overflow || !thread->os_thread()->HasStackHeadroom() ||
IsCalleeFrameOf(thread->saved_stack_limit(), stack_pos)) {
// Use the preallocated stack overflow exception to avoid calling
// into dart code.
@ -2217,7 +2374,7 @@ DEFINE_RUNTIME_ENTRY(FixCallersTarget, 0) {
}
ASSERT(frame->IsDartFrame());
const Code& caller_code = Code::Handle(zone, frame->LookupDartCode());
ASSERT(caller_code.is_optimized());
RELEASE_ASSERT(caller_code.is_optimized());
const Function& target_function = Function::Handle(
zone, caller_code.GetStaticCallTargetFunctionAt(frame->pc()));
@ -2241,6 +2398,52 @@ DEFINE_RUNTIME_ENTRY(FixCallersTarget, 0) {
#endif
}
// The caller must be a monomorphic call from unoptimized code.
// Patch call to point to new target.
DEFINE_RUNTIME_ENTRY(FixCallersTargetMonomorphic, 0) {
#if !defined(DART_PRECOMPILED_RUNTIME)
StackFrameIterator iterator(ValidationPolicy::kDontValidateFrames, thread,
StackFrameIterator::kNoCrossThreadIteration);
StackFrame* frame = iterator.NextFrame();
ASSERT(frame != NULL);
while (frame->IsStubFrame() || frame->IsExitFrame()) {
frame = iterator.NextFrame();
ASSERT(frame != NULL);
}
if (frame->IsEntryFrame()) {
// Since function's current code is always unpatched, the entry frame always
// calls to unpatched code.
UNREACHABLE();
}
ASSERT(frame->IsDartFrame());
const Code& caller_code = Code::Handle(zone, frame->LookupDartCode());
RELEASE_ASSERT(!caller_code.is_optimized());
Object& cache = Object::Handle(zone);
const Code& old_target_code = Code::Handle(
zone, CodePatcher::GetInstanceCallAt(frame->pc(), caller_code, &cache));
const Function& target_function =
Function::Handle(zone, old_target_code.function());
const Code& current_target_code =
Code::Handle(zone, target_function.EnsureHasCode());
CodePatcher::PatchInstanceCallAt(frame->pc(), caller_code, cache,
current_target_code);
if (FLAG_trace_patching) {
OS::PrintErr(
"FixCallersTargetMonomorphic: caller %#" Px
" "
"target '%s' -> %#" Px " (%s)\n",
frame->pc(), target_function.ToFullyQualifiedCString(),
current_target_code.EntryPoint(),
current_target_code.is_optimized() ? "optimized" : "unoptimized");
}
ASSERT(!current_target_code.IsDisabled());
arguments.SetReturn(current_target_code);
#else
UNREACHABLE();
#endif
}
// The caller tried to allocate an instance via an invalidated allocation
// stub.
DEFINE_RUNTIME_ENTRY(FixAllocationStubTarget, 0) {

1
runtime/vm/runtime_entry_list.h
View file

@ -18,6 +18,7 @@ namespace dart {
V(GetFieldForDispatch) \
V(ResolveCallFunction) \
V(FixCallersTarget) \
V(FixCallersTargetMonomorphic) \
V(FixAllocationStubTarget) \
V(InlineCacheMissHandlerOneArg) \
V(InlineCacheMissHandlerTwoArgs) \

1
runtime/vm/stub_code_list.h
View file

@ -46,6 +46,7 @@ namespace dart {
V(UnoptimizedIdenticalWithNumberCheck) \
V(OptimizedIdenticalWithNumberCheck) \
V(ICCallBreakpoint) \
V(UnoptStaticCallBreakpoint) \
V(RuntimeCallBreakpoint) \
V(OneArgCheckInlineCache) \
V(TwoArgsCheckInlineCache) \

2
runtime/vm/thread.h
View file

@ -112,8 +112,6 @@ class Zone;
V(RawCode*, stack_overflow_shared_with_fpu_regs_stub_, \
StubCode::StackOverflowSharedWithFPURegs().raw(), NULL) \
V(RawCode*, monomorphic_miss_stub_, StubCode::MonomorphicMiss().raw(), NULL) \
V(RawCode*, ic_lookup_through_code_stub_, \
StubCode::ICCallThroughCode().raw(), NULL) \
V(RawCode*, optimize_stub_, StubCode::OptimizeFunction().raw(), NULL) \
V(RawCode*, deoptimize_stub_, StubCode::Deoptimize().raw(), NULL) \
V(RawCode*, lazy_deopt_from_return_stub_, \