[vm, compiler] Shorter invocation counter check.

Remove the optimization function stub and self function from the constant pool of every unoptimized function.

Remove one PP move from unoptimized frame entry.

Bug: https://github.com/dart-lang/sdk/issues/36409
Change-Id: Idbe37ce36b57dc316a131e6c83742ee2b3df4a0d
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/102660
Commit-Queue: Ryan Macnak <rmacnak@google.com>
Reviewed-by: Aart Bik <ajcbik@google.com>
This commit is contained in:
Ryan Macnak 2019-05-16 23:04:22 +00:00 committed by commit-bot@chromium.org
parent 5ffff98440
commit 0e8656e045
21 changed files with 134 additions and 99 deletions

View file

@ -2206,8 +2206,7 @@ void AsmIntrinsifier::IntrinsifyRegExpExecuteMatch(Assembler* assembler,
__ xorl(ECX, ECX);
// Tail-call the function.
__ movl(EDI, FieldAddress(EAX, target::Function::entry_point_offset()));
__ jmp(EDI);
__ jmp(FieldAddress(EAX, target::Function::entry_point_offset()));
}
// On stack: user tag (+1), return-address (+0).

View file

@ -1616,14 +1616,6 @@ void Assembler::LoadUniqueObject(Register rd,
LoadObjectHelper(rd, object, cond, /* is_unique = */ true, PP);
}
void Assembler::LoadFunctionFromCalleePool(Register dst,
const Function& function,
Register new_pp) {
const int32_t offset = ObjectPool::element_offset(
object_pool_builder().FindObject(ToObject(function)));
LoadWordFromPoolOffset(dst, offset - kHeapObjectTag, new_pp, AL);
}
void Assembler::LoadNativeEntry(Register rd,
const ExternalLabel* label,
ObjectPoolBuilderEntry::Patchability patchable,

View file

@ -758,9 +758,6 @@ class Assembler : public AssemblerBase {
void LoadObject(Register rd, const Object& object, Condition cond = AL);
void LoadUniqueObject(Register rd, const Object& object, Condition cond = AL);
void LoadFunctionFromCalleePool(Register dst,
const Function& function,
Register new_pp);
void LoadNativeEntry(Register dst,
const ExternalLabel* label,
ObjectPoolBuilderEntry::Patchability patchable,

View file

@ -495,17 +495,6 @@ void Assembler::LoadObjectHelper(Register dst,
}
}
void Assembler::LoadFunctionFromCalleePool(Register dst,
const Function& function,
Register new_pp) {
ASSERT(!constant_pool_allowed());
ASSERT(new_pp != PP);
const int32_t offset = ObjectPool::element_offset(
object_pool_builder().FindObject(ToObject(function)));
ASSERT(Address::CanHoldOffset(offset));
ldr(dst, Address(new_pp, offset));
}
void Assembler::LoadObject(Register dst, const Object& object) {
LoadObjectHelper(dst, object, false);
}

View file

@ -1485,9 +1485,6 @@ class Assembler : public AssemblerBase {
void LoadNativeEntry(Register dst,
const ExternalLabel* label,
ObjectPoolBuilderEntry::Patchability patchable);
void LoadFunctionFromCalleePool(Register dst,
const Function& function,
Register new_pp);
void LoadIsolate(Register dst);
void LoadObject(Register dst, const Object& obj);
void LoadUniqueObject(Register dst, const Object& obj);
@ -1537,7 +1534,7 @@ class Assembler : public AssemblerBase {
void RestoreCodePointer();
void EnterDartFrame(intptr_t frame_size, Register new_pp = kNoRegister);
void EnterOsrFrame(intptr_t extra_size, Register new_pp);
void EnterOsrFrame(intptr_t extra_size, Register new_pp = kNoRegister);
void LeaveDartFrame(RestorePP restore_pp = kRestoreCallerPP);
void EnterCallRuntimeFrame(intptr_t frame_size);

View file

@ -1667,6 +1667,12 @@ void Assembler::jmp(Register reg) {
EmitRegisterOperand(4, reg);
}
void Assembler::jmp(const Address& address) {
AssemblerBuffer::EnsureCapacity ensured(&buffer_);
EmitUint8(0xFF);
EmitOperand(4, address);
}
void Assembler::jmp(Label* label, bool near) {
AssemblerBuffer::EnsureCapacity ensured(&buffer_);
if (label->IsBound()) {

View file

@ -549,6 +549,7 @@ class Assembler : public AssemblerBase {
void j(Condition condition, const ExternalLabel* label);
void jmp(Register reg);
void jmp(const Address& address);
void jmp(Label* label, bool near = kFarJump);
void jmp(const ExternalLabel* label);

View file

@ -326,6 +326,51 @@ ASSEMBLER_TEST_RUN(Testb, test) {
"ret\n");
}
struct JumpAddress {
uword filler1;
uword filler2;
uword filler3;
uword filler4;
uword filler5;
uword target;
uword filler6;
uword filler7;
uword filler8;
};
static JumpAddress jump_address;
static uword jump_address_offset;
ASSEMBLER_TEST_GENERATE(JumpAddress, assembler) {
__ movl(EAX, Address(ESP, 4));
__ jmp(Address(EAX, OFFSET_OF(JumpAddress, target)));
__ int3();
__ int3();
__ int3();
__ int3();
__ int3();
jump_address_offset = __ CodeSize();
__ movl(EAX, Immediate(42));
__ ret();
}
ASSEMBLER_TEST_RUN(JumpAddress, test) {
memset(&jump_address, 0, sizeof(jump_address));
jump_address.target = test->entry() + jump_address_offset;
typedef int (*TestCode)(void*);
EXPECT_EQ(42, reinterpret_cast<TestCode>(test->entry())(&jump_address));
EXPECT_DISASSEMBLY(
"mov eax,[esp+0x4]\n"
"jmp [eax+0x14]\n"
"int3\n"
"int3\n"
"int3\n"
"int3\n"
"int3\n"
"mov eax,0x2a\n"
"ret\n");
}
ASSEMBLER_TEST_GENERATE(Increment, assembler) {
__ movl(EAX, Immediate(0));
__ pushl(EAX);

View file

@ -996,7 +996,7 @@ void Assembler::JmpPatchable(const Code& target, Register pp) {
const intptr_t idx = object_pool_builder().AddObject(
ToObject(target), ObjectPoolBuilderEntry::kPatchable);
const int32_t offset = target::ObjectPool::element_offset(idx);
movq(CODE_REG, Address::AddressBaseImm32(pp, offset - kHeapObjectTag));
movq(CODE_REG, Address(pp, offset - kHeapObjectTag));
movq(TMP, FieldAddress(CODE_REG, target::Code::entry_point_offset()));
jmp(TMP);
}
@ -1007,8 +1007,7 @@ void Assembler::Jmp(const Code& target, Register pp) {
ToObject(target), ObjectPoolBuilderEntry::kNotPatchable);
const int32_t offset = target::ObjectPool::element_offset(idx);
movq(CODE_REG, FieldAddress(pp, offset));
movq(TMP, FieldAddress(CODE_REG, target::Code::entry_point_offset()));
jmp(TMP);
jmp(FieldAddress(CODE_REG, target::Code::entry_point_offset()));
}
void Assembler::CompareRegisters(Register a, Register b) {
@ -1196,17 +1195,6 @@ void Assembler::LoadObjectHelper(Register dst,
}
}
void Assembler::LoadFunctionFromCalleePool(Register dst,
const Function& function,
Register new_pp) {
ASSERT(!constant_pool_allowed());
ASSERT(new_pp != PP);
const intptr_t idx = object_pool_builder().FindObject(
ToObject(function), ObjectPoolBuilderEntry::kNotPatchable);
const int32_t offset = target::ObjectPool::element_offset(idx);
movq(dst, Address::AddressBaseImm32(new_pp, offset - kHeapObjectTag));
}
void Assembler::LoadObject(Register dst, const Object& object) {
LoadObjectHelper(dst, object, false);
}

View file

@ -696,9 +696,6 @@ class Assembler : public AssemblerBase {
void LoadNativeEntry(Register dst,
const ExternalLabel* label,
ObjectPoolBuilderEntry::Patchability patchable);
void LoadFunctionFromCalleePool(Register dst,
const Function& function,
Register new_pp);
void JmpPatchable(const Code& code, Register pp);
void Jmp(const Code& code, Register pp = PP);
void J(Condition condition, const Code& code, Register pp);
@ -856,7 +853,7 @@ class Assembler : public AssemblerBase {
// ...
// pushq r15
// .....
void EnterDartFrame(intptr_t frame_size, Register new_pp);
void EnterDartFrame(intptr_t frame_size, Register new_pp = kNoRegister);
void LeaveDartFrame(RestorePP restore_pp = kRestoreCallerPP);
// Set up a Dart frame for a function compiled for on-stack replacement.

View file

@ -670,6 +670,49 @@ ASSEMBLER_TEST_RUN(Testb3, test) {
"ret\n");
}
struct JumpAddress {
uword filler1;
uword filler2;
uword filler3;
uword filler4;
uword filler5;
uword target;
uword filler6;
uword filler7;
uword filler8;
};
static JumpAddress jump_address;
static uword jump_address_offset;
ASSEMBLER_TEST_GENERATE(JumpAddress, assembler) {
__ jmp(Address(CallingConventions::kArg1Reg, OFFSET_OF(JumpAddress, target)));
__ int3();
__ int3();
__ int3();
__ int3();
__ int3();
jump_address_offset = __ CodeSize();
__ movl(RAX, Immediate(42));
__ ret();
}
ASSEMBLER_TEST_RUN(JumpAddress, test) {
memset(&jump_address, 0, sizeof(jump_address));
jump_address.target = test->entry() + jump_address_offset;
typedef int (*TestCode)(void*);
EXPECT_EQ(42, reinterpret_cast<TestCode>(test->entry())(&jump_address));
EXPECT_DISASSEMBLY_NOT_WINDOWS(
"jmp [rdi+0x28]\n"
"int3\n"
"int3\n"
"int3\n"
"int3\n"
"int3\n"
"movl rax,0x2a\n"
"ret\n");
}
ASSEMBLER_TEST_GENERATE(Increment, assembler) {
__ movq(RAX, Immediate(0));
__ pushq(RAX);

View file

@ -856,22 +856,13 @@ void FlowGraphCompiler::GenerateSetterIntrinsic(intptr_t offset) {
__ Ret();
}
static const Register new_pp = NOTFP;
void FlowGraphCompiler::EmitFrameEntry() {
const Function& function = parsed_function().function();
if (CanOptimizeFunction() && function.IsOptimizable() &&
(!is_optimizing() || may_reoptimize())) {
__ Comment("Invocation Count Check");
const Register function_reg = R8;
if (!FLAG_precompiled_mode || !FLAG_use_bare_instructions) {
// The pool pointer is not setup before entering the Dart frame.
// Temporarily setup pool pointer for this dart function.
__ LoadPoolPointer(new_pp);
}
// Load function object from object pool.
__ LoadFunctionFromCalleePool(function_reg, function, new_pp);
__ ldr(function_reg, FieldAddress(CODE_REG, Code::owner_offset()));
__ ldr(R3, FieldAddress(function_reg, Function::usage_counter_offset()));
// Reoptimization of an optimized function is triggered by counting in
// IC stubs, but not at the entry of the function.
@ -881,8 +872,7 @@ void FlowGraphCompiler::EmitFrameEntry() {
}
__ CompareImmediate(R3, GetOptimizationThreshold());
ASSERT(function_reg == R8);
__ Branch(StubCode::OptimizeFunction(),
compiler::ObjectPoolBuilderEntry::kNotPatchable, new_pp, GE);
__ Branch(Address(THR, Thread::optimize_entry_offset()), GE);
}
__ Comment("Enter frame");
if (flow_graph().IsCompiledForOsr()) {

View file

@ -837,20 +837,11 @@ void FlowGraphCompiler::GenerateSetterIntrinsic(intptr_t offset) {
void FlowGraphCompiler::EmitFrameEntry() {
const Function& function = parsed_function().function();
Register new_pp = kNoRegister;
if (CanOptimizeFunction() && function.IsOptimizable() &&
(!is_optimizing() || may_reoptimize())) {
__ Comment("Invocation Count Check");
const Register function_reg = R6;
new_pp = R13;
if (!FLAG_precompiled_mode || !FLAG_use_bare_instructions) {
// The pool pointer is not setup before entering the Dart frame.
// Temporarily setup pool pointer for this dart function.
__ LoadPoolPointer(new_pp);
}
// Load function object using the callee's pool pointer.
__ LoadFunctionFromCalleePool(function_reg, function, new_pp);
__ ldr(function_reg, FieldAddress(CODE_REG, Code::owner_offset()));
__ LoadFieldFromOffset(R7, function_reg, Function::usage_counter_offset(),
kWord);
@ -865,17 +856,18 @@ void FlowGraphCompiler::EmitFrameEntry() {
ASSERT(function_reg == R6);
Label dont_optimize;
__ b(&dont_optimize, LT);
__ Branch(StubCode::OptimizeFunction(), new_pp);
__ ldr(TMP, Address(THR, Thread::optimize_entry_offset()));
__ br(TMP);
__ Bind(&dont_optimize);
}
__ Comment("Enter frame");
if (flow_graph().IsCompiledForOsr()) {
const intptr_t extra_slots = ExtraStackSlotsOnOsrEntry();
ASSERT(extra_slots >= 0);
__ EnterOsrFrame(extra_slots * kWordSize, new_pp);
__ EnterOsrFrame(extra_slots * kWordSize);
} else {
ASSERT(StackSize() >= 0);
__ EnterDartFrame(StackSize() * kWordSize, new_pp);
__ EnterDartFrame(StackSize() * kWordSize);
}
}

View file

@ -775,7 +775,10 @@ void FlowGraphCompiler::EmitFrameEntry() {
__ cmpl(FieldAddress(function_reg, Function::usage_counter_offset()),
Immediate(GetOptimizationThreshold()));
ASSERT(function_reg == EBX);
__ J(GREATER_EQUAL, StubCode::OptimizeFunction());
Label dont_optimize;
__ j(LESS, &dont_optimize, Assembler::kNearJump);
__ jmp(Address(THR, Thread::optimize_entry_offset()));
__ Bind(&dont_optimize);
}
__ Comment("Enter frame");
if (flow_graph().IsCompiledForOsr()) {

View file

@ -858,18 +858,12 @@ void FlowGraphCompiler::EmitFrameEntry() {
ASSERT(extra_slots >= 0);
__ EnterOsrFrame(extra_slots * kWordSize);
} else {
const Register new_pp = R13;
if (!FLAG_precompiled_mode || !FLAG_use_bare_instructions) {
__ LoadPoolPointer(new_pp);
}
const Function& function = parsed_function().function();
if (CanOptimizeFunction() && function.IsOptimizable() &&
(!is_optimizing() || may_reoptimize())) {
__ Comment("Invocation Count Check");
const Register function_reg = RDI;
// Load function object using the callee's pool pointer.
__ LoadFunctionFromCalleePool(function_reg, function, new_pp);
__ movq(function_reg, FieldAddress(CODE_REG, Code::owner_offset()));
// Reoptimization of an optimized function is triggered by counting in
// IC stubs, but not at the entry of the function.
@ -879,11 +873,14 @@ void FlowGraphCompiler::EmitFrameEntry() {
__ cmpl(FieldAddress(function_reg, Function::usage_counter_offset()),
Immediate(GetOptimizationThreshold()));
ASSERT(function_reg == RDI);
__ J(GREATER_EQUAL, StubCode::OptimizeFunction(), new_pp);
Label dont_optimize;
__ j(LESS, &dont_optimize, Assembler::kNearJump);
__ jmp(Address(THR, Thread::optimize_entry_offset()));
__ Bind(&dont_optimize);
}
ASSERT(StackSize() >= 0);
__ Comment("Enter frame");
__ EnterDartFrame(StackSize() * kWordSize, new_pp);
__ EnterDartFrame(StackSize() * kWordSize);
}
}

View file

@ -2905,6 +2905,7 @@ void StubCodeCompiler::GenerateDeoptForRewindStub(Assembler* assembler) {
// R8: function to be reoptimized.
// R4: argument descriptor (preserved).
void StubCodeCompiler::GenerateOptimizeFunctionStub(Assembler* assembler) {
__ ldr(CODE_REG, Address(THR, Thread::optimize_stub_offset()));
__ EnterStubFrame();
__ Push(R4);
__ LoadImmediate(IP, 0);

View file

@ -2993,6 +2993,7 @@ void StubCodeCompiler::GenerateDeoptForRewindStub(Assembler* assembler) {
// R6: function to be re-optimized.
// R4: argument descriptor (preserved).
void StubCodeCompiler::GenerateOptimizeFunctionStub(Assembler* assembler) {
__ LoadFromOffset(CODE_REG, THR, Thread::optimize_stub_offset());
__ EnterStubFrame();
__ Push(R4);
// Setup space on stack for the return value.

View file

@ -359,8 +359,7 @@ void StubCodeCompiler::GenerateCallStaticFunctionStub(Assembler* assembler) {
// Remove the stub frame as we are about to jump to the dart function.
__ LeaveFrame();
__ movl(ECX, FieldAddress(EAX, target::Code::entry_point_offset()));
__ jmp(ECX);
__ jmp(FieldAddress(EAX, target::Code::entry_point_offset()));
}
// Called from a static call only when an invalid code has been entered
@ -666,8 +665,7 @@ void StubCodeCompiler::GenerateMegamorphicMissStub(Assembler* assembler) {
__ Bind(&call_target_function);
}
__ movl(EBX, FieldAddress(EAX, target::Function::entry_point_offset()));
__ jmp(EBX);
__ jmp(FieldAddress(EAX, target::Function::entry_point_offset()));
}
// Called for inline allocation of arrays.
@ -1779,8 +1777,7 @@ void StubCodeCompiler::GenerateNArgsCheckInlineCacheStub(
__ Bind(&call_target_function);
__ Comment("Call target");
// EAX: Target function.
__ movl(EBX, FieldAddress(EAX, target::Function::entry_point_offset()));
__ jmp(EBX);
__ jmp(FieldAddress(EAX, target::Function::entry_point_offset()));
#if !defined(PRODUCT)
if (optimized == kUnoptimized) {
@ -1942,8 +1939,7 @@ void StubCodeCompiler::GenerateZeroArgsUnoptimizedStaticCallStub(
// Get function and call it, if possible.
__ movl(EAX, Address(EBX, target_offset));
__ movl(EBX, FieldAddress(EAX, target::Function::entry_point_offset()));
__ jmp(EBX);
__ jmp(FieldAddress(EAX, target::Function::entry_point_offset()));
#if !defined(PRODUCT)
__ Bind(&stepping);
@ -1990,8 +1986,7 @@ void StubCodeCompiler::GenerateLazyCompileStub(Assembler* assembler) {
// When using the interpreter, the function's code may now point to the
// InterpretCall stub. Make sure EAX, ECX, and EDX are preserved.
__ movl(EBX, FieldAddress(EAX, target::Function::entry_point_offset()));
__ jmp(EBX);
__ jmp(FieldAddress(EAX, target::Function::entry_point_offset()));
}
// Stub for interpreting a function call.
@ -2075,8 +2070,7 @@ void StubCodeCompiler::GenerateICCallBreakpointStub(Assembler* assembler) {
__ popl(EBX); // Restore receiver.
__ LeaveFrame();
// Jump to original stub.
__ movl(EAX, FieldAddress(EAX, target::Code::entry_point_offset()));
__ jmp(EAX);
__ jmp(FieldAddress(EAX, target::Code::entry_point_offset()));
#endif // defined(PRODUCT)
}
@ -2092,8 +2086,7 @@ void StubCodeCompiler::GenerateRuntimeCallBreakpointStub(Assembler* assembler) {
__ popl(EAX); // Code of the original stub
__ LeaveFrame();
// Jump to original stub.
__ movl(EAX, FieldAddress(EAX, target::Code::entry_point_offset()));
__ jmp(EAX);
__ jmp(FieldAddress(EAX, target::Code::entry_point_offset()));
#endif // defined(PRODUCT)
}
@ -2411,6 +2404,7 @@ void StubCodeCompiler::GenerateDeoptForRewindStub(Assembler* assembler) {
// EBX: function to be reoptimized.
// EDX: argument descriptor (preserved).
void StubCodeCompiler::GenerateOptimizeFunctionStub(Assembler* assembler) {
__ movl(CODE_REG, Address(THR, Thread::optimize_stub_offset()));
__ EnterStubFrame();
__ pushl(EDX);
__ pushl(Immediate(0)); // Setup space on stack for return value.
@ -2421,8 +2415,7 @@ void StubCodeCompiler::GenerateOptimizeFunctionStub(Assembler* assembler) {
__ popl(EDX); // Restore argument descriptor.
__ LeaveFrame();
__ movl(CODE_REG, FieldAddress(EAX, target::Function::code_offset()));
__ movl(EAX, FieldAddress(EAX, target::Function::entry_point_offset()));
__ jmp(EAX);
__ jmp(FieldAddress(EAX, target::Function::entry_point_offset()));
__ int3();
}
@ -2583,8 +2576,7 @@ void StubCodeCompiler::GenerateMegamorphicCallStub(Assembler* assembler) {
__ movl(EDX,
FieldAddress(
ECX, target::MegamorphicCache::arguments_descriptor_offset()));
__ movl(EBX, FieldAddress(EAX, target::Function::entry_point_offset()));
__ jmp(EBX);
__ jmp(FieldAddress(EAX, target::Function::entry_point_offset()));
__ Bind(&probe_failed);
// Probe failed, check if it is a miss.

View file

@ -2988,6 +2988,7 @@ void StubCodeCompiler::GenerateDeoptForRewindStub(Assembler* assembler) {
// RDI: function to be reoptimized.
// R10: argument descriptor (preserved).
void StubCodeCompiler::GenerateOptimizeFunctionStub(Assembler* assembler) {
__ movq(CODE_REG, Address(THR, Thread::optimize_stub_offset()));
__ EnterStubFrame();
__ pushq(R10); // Preserve args descriptor.
__ pushq(Immediate(0)); // Result slot.

View file

@ -5218,6 +5218,8 @@ class Code : public Object {
StorePointer(&raw_ptr()->owner_, owner.raw());
}
static intptr_t owner_offset() { return OFFSET_OF(RawCode, owner_); }
// We would have a VisitPointers function here to traverse all the
// embedded objects in the instructions using pointer_offsets.

View file

@ -114,6 +114,7 @@ class Zone;
V(RawCode*, monomorphic_miss_stub_, StubCode::MonomorphicMiss().raw(), NULL) \
V(RawCode*, ic_lookup_through_code_stub_, \
StubCode::ICCallThroughCode().raw(), NULL) \
V(RawCode*, optimize_stub_, StubCode::OptimizeFunction().raw(), NULL) \
V(RawCode*, deoptimize_stub_, StubCode::Deoptimize().raw(), NULL) \
V(RawCode*, lazy_deopt_from_return_stub_, \
StubCode::DeoptimizeLazyFromReturn().raw(), NULL) \
@ -166,6 +167,7 @@ class Zone;
StubCode::MegamorphicCall().EntryPoint(), 0) \
V(uword, monomorphic_miss_entry_, StubCode::MonomorphicMiss().EntryPoint(), \
0) \
V(uword, optimize_entry_, StubCode::OptimizeFunction().EntryPoint(), 0) \
V(uword, deoptimize_entry_, StubCode::Deoptimize().EntryPoint(), 0)
#endif