AOT: When a call goes polymorphic but has a single target, transition to a stub that does a cid range check.

Expand the range on a miss if all classes in the range have the same lookup result, otherwise transition to the stub that does a linear scan of ICData.

Because we do a depth-first sort of classes during AOT, if a call site sees only objects all inheriting the same function as the call target, the call site will be handled by this new stub.

Adjust LoadClassIdMayBeSmi on x64 to preserve the object.

R=fschneider@google.com

Review URL: https://codereview.chromium.org/2279563002 .
This commit is contained in:
Ryan Macnak 2016-08-31 11:12:22 -07:00
parent 63046da4be
commit ba94427eb1
17 changed files with 446 additions and 3 deletions

View file

@ -3702,9 +3702,9 @@ void Assembler::LoadClassIdMayBeSmi(Register result, Register object) {
// if it is a Smi, which will be ignored.
LoadClassId(result, object);
movq(object, Immediate(kSmiCid));
movq(TMP, Immediate(kSmiCid));
// If object is a Smi, move the Smi cid into result. o/w leave alone.
cmoveq(result, object);
cmoveq(result, TMP);
}

View file

@ -1026,6 +1026,121 @@ DEFINE_RUNTIME_ENTRY(StaticCallMissHandlerTwoArgs, 3) {
}
#if !defined(TARGET_ARCH_DBC)
static bool IsSingleTarget(Isolate* isolate,
Zone* zone,
intptr_t lower_cid,
intptr_t upper_cid,
const Function& target,
const String& name) {
Class& cls = Class::Handle(zone);
ClassTable* table = isolate->class_table();
Function& other_target = Function::Handle(zone);
for (intptr_t cid = lower_cid; cid <= upper_cid; cid++) {
if (!table->HasValidClassAt(cid)) continue;
cls = table->At(cid);
if (cls.is_abstract()) continue;
if (!cls.is_allocated()) continue;
other_target = Resolver::ResolveDynamicAnyArgs(zone, cls, name,
false /* allow_add */);
if (other_target.raw() != target.raw()) {
return false;
}
}
return true;
}
#endif
// Handle a miss of a single target cache.
// Arg0: Receiver.
// Returns: the ICData used to continue with a polymorphic call.
DEFINE_RUNTIME_ENTRY(SingleTargetMiss, 1) {
#if defined(TARGET_ARCH_DBC)
// DBC does not use switchable calls.
UNREACHABLE();
#else
const Instance& receiver = Instance::CheckedHandle(zone, arguments.ArgAt(0));
DartFrameIterator iterator;
StackFrame* caller_frame = iterator.NextFrame();
ASSERT(caller_frame->IsDartFrame());
const Code& caller_code = Code::Handle(zone, caller_frame->LookupDartCode());
const Function& caller_function =
Function::Handle(zone, caller_frame->LookupDartFunction());
SingleTargetCache& cache = SingleTargetCache::Handle(zone);
cache ^= CodePatcher::GetSwitchableCallDataAt(caller_frame->pc(),
caller_code);
Code& old_target_code = Code::Handle(zone, cache.target());
Function& old_target = Function::Handle(zone);
old_target ^= old_target_code.owner();
// We lost the original ICData when we patched to the monomorphic case.
const String& name = String::Handle(zone, old_target.name());
ASSERT(!old_target.HasOptionalParameters());
const Array& descriptor = Array::Handle(zone,
ArgumentsDescriptor::New(old_target.num_fixed_parameters()));
const ICData& ic_data =
ICData::Handle(zone, ICData::New(caller_function,
name,
descriptor,
Thread::kNoDeoptId,
1, /* args_tested */
false /* static_call */));
// Maybe add the new target.
Class& cls = Class::Handle(zone, receiver.clazz());
ArgumentsDescriptor args_desc(descriptor);
Function& target_function = Function::Handle(zone,
Resolver::ResolveDynamicForReceiverClass(cls,
name,
args_desc));
if (target_function.IsNull()) {
target_function = InlineCacheMissHelper(receiver, descriptor, name);
}
if (target_function.IsNull()) {
ASSERT(!FLAG_lazy_dispatchers);
} else {
ic_data.AddReceiverCheck(receiver.GetClassId(), target_function);
}
if (old_target.raw() == target_function.raw()) {
intptr_t lower, upper;
if (receiver.GetClassId() < cache.lower_limit()) {
lower = receiver.GetClassId();
upper = cache.upper_limit();
} else {
lower = cache.lower_limit();
upper = receiver.GetClassId();
}
if (IsSingleTarget(isolate, zone, lower, upper, target_function, name)) {
cache.set_lower_limit(lower);
cache.set_upper_limit(upper);
// Return the ICData. The single target stub will jump to continue in the
// IC call stub.
arguments.SetReturn(ic_data);
return;
}
}
// Call site is not single target, switch to call using ICData.
const Code& stub =
Code::Handle(zone, StubCode::ICCallThroughCode_entry()->code());
ASSERT(!Isolate::Current()->compilation_allowed());
CodePatcher::PatchSwitchableCallAt(caller_frame->pc(),
caller_code,
ic_data,
stub);
// Return the ICData. The single target stub will jump to continue in the
// IC call stub.
arguments.SetReturn(ic_data);
#endif
}
// Handle a miss of a megamorphic cache.
// Arg0: Receiver.
// Returns: the ICData used to continue with a polymorphic call.
@ -1084,6 +1199,37 @@ DEFINE_RUNTIME_ENTRY(MonomorphicMiss, 1) {
ic_data.AddReceiverCheck(receiver.GetClassId(), target_function);
}
if (old_target.raw() == target_function.raw()) {
intptr_t lower, upper;
if (old_expected_cid.Value() < receiver.GetClassId()) {
lower = old_expected_cid.Value();
upper = receiver.GetClassId();
} else {
lower = receiver.GetClassId();
upper = old_expected_cid.Value();
}
if (IsSingleTarget(isolate, zone, lower, upper, target_function, name)) {
const SingleTargetCache& cache =
SingleTargetCache::Handle(SingleTargetCache::New());
const Code& code = Code::Handle(target_function.CurrentCode());
cache.set_target(code);
cache.set_entry_point(code.UncheckedEntryPoint());
cache.set_lower_limit(lower);
cache.set_upper_limit(upper);
const Code& stub = Code::Handle(zone,
StubCode::SingleTargetCall_entry()->code());
CodePatcher::PatchSwitchableCallAt(caller_frame->pc(),
caller_code,
cache,
stub);
// Return the ICData. The miss stub will jump to continue in the IC call
// stub.
arguments.SetReturn(ic_data);
return;
}
}
// Patch to call through stub.
const Code& stub =
Code::Handle(zone, StubCode::ICCallThroughCode_entry()->code());

View file

@ -95,6 +95,7 @@ static void CheckOffsets() {
CHECK_OFFSET(Isolate::heap_offset(), 8);
CHECK_OFFSET(Thread::stack_limit_offset(), 4);
CHECK_OFFSET(Thread::object_null_offset(), 36);
CHECK_OFFSET(SingleTargetCache::upper_limit_offset(), 14);
NOT_IN_PRODUCT(CHECK_OFFSET(sizeof(ClassHeapStats), 120));
#endif
#if defined(TARGET_ARCH_MIPS)
@ -104,6 +105,7 @@ static void CheckOffsets() {
CHECK_OFFSET(Isolate::heap_offset(), 8);
CHECK_OFFSET(Thread::stack_limit_offset(), 4);
CHECK_OFFSET(Thread::object_null_offset(), 36);
CHECK_OFFSET(SingleTargetCache::upper_limit_offset(), 14);
NOT_IN_PRODUCT(CHECK_OFFSET(sizeof(ClassHeapStats), 120));
#endif
#if defined(TARGET_ARCH_ARM64)
@ -113,6 +115,7 @@ static void CheckOffsets() {
CHECK_OFFSET(Isolate::heap_offset(), 16);
CHECK_OFFSET(Thread::stack_limit_offset(), 8);
CHECK_OFFSET(Thread::object_null_offset(), 72);
CHECK_OFFSET(SingleTargetCache::upper_limit_offset(), 28);
NOT_IN_PRODUCT(CHECK_OFFSET(sizeof(ClassHeapStats), 208));
#endif
#undef CHECK_OFFSET

View file

@ -151,6 +151,8 @@ RawClass* Object::exception_handlers_class_ =
reinterpret_cast<RawClass*>(RAW_NULL);
RawClass* Object::context_class_ = reinterpret_cast<RawClass*>(RAW_NULL);
RawClass* Object::context_scope_class_ = reinterpret_cast<RawClass*>(RAW_NULL);
RawClass* Object::singletargetcache_class_ =
reinterpret_cast<RawClass*>(RAW_NULL);
RawClass* Object::icdata_class_ = reinterpret_cast<RawClass*>(RAW_NULL);
RawClass* Object::megamorphic_cache_class_ =
reinterpret_cast<RawClass*>(RAW_NULL);
@ -651,6 +653,9 @@ void Object::InitOnce(Isolate* isolate) {
cls = Class::New<ContextScope>();
context_scope_class_ = cls.raw();
cls = Class::New<SingleTargetCache>();
singletargetcache_class_ = cls.raw();
cls = Class::New<ICData>();
icdata_class_ = cls.raw();
@ -992,6 +997,7 @@ void Object::FinalizeVMIsolate(Isolate* isolate) {
SET_CLASS_NAME(exception_handlers, ExceptionHandlers);
SET_CLASS_NAME(context, Context);
SET_CLASS_NAME(context_scope, ContextScope);
SET_CLASS_NAME(singletargetcache, SingleTargetCache);
SET_CLASS_NAME(icdata, ICData);
SET_CLASS_NAME(megamorphic_cache, MegamorphicCache);
SET_CLASS_NAME(subtypetestcache, SubtypeTestCache);
@ -3427,6 +3433,8 @@ NOT_IN_PRODUCT(
return Symbols::Context().raw();
case kContextScopeCid:
return Symbols::ContextScope().raw();
case kSingleTargetCacheCid:
return Symbols::SingleTargetCache().raw();
case kICDataCid:
return Symbols::ICData().raw();
case kMegamorphicCacheCid:
@ -12491,6 +12499,34 @@ bool DeoptInfo::VerifyDecompression(const GrowableArray<DeoptInstr*>& original,
}
void SingleTargetCache::set_target(const Code& value) const {
StorePointer(&raw_ptr()->target_, value.raw());
}
const char* SingleTargetCache::ToCString() const {
return "SingleTargetCache";
}
RawSingleTargetCache* SingleTargetCache::New() {
SingleTargetCache& result = SingleTargetCache::Handle();
{
// IC data objects are long living objects, allocate them in old generation.
RawObject* raw = Object::Allocate(SingleTargetCache::kClassId,
SingleTargetCache::InstanceSize(),
Heap::kOld);
NoSafepointScope no_safepoint;
result ^= raw;
}
result.set_target(Code::Handle());
result.set_entry_point(0);
result.set_lower_limit(kIllegalCid);
result.set_upper_limit(kIllegalCid);
return result.raw();
}
void ICData::ResetSwitchable(Zone* zone) const {
ASSERT(NumArgsTested() == 1);
set_ic_data_array(Array::Handle(zone, CachedEmptyICDataArray(1)));

View file

@ -539,6 +539,9 @@ class Object {
return unhandled_exception_class_;
}
static RawClass* unwind_error_class() { return unwind_error_class_; }
static RawClass* singletargetcache_class() {
return singletargetcache_class_;
}
static RawClass* icdata_class() { return icdata_class_; }
static RawClass* megamorphic_cache_class() {
return megamorphic_cache_class_;
@ -790,6 +793,7 @@ class Object {
static RawClass* deopt_info_class_; // Class of DeoptInfo.
static RawClass* context_class_; // Class of the Context vm object.
static RawClass* context_scope_class_; // Class of ContextScope vm object.
static RawClass* singletargetcache_class_; // Class of SingleTargetCache.
static RawClass* icdata_class_; // Class of ICData.
static RawClass* megamorphic_cache_class_; // Class of MegamorphiCache.
static RawClass* subtypetestcache_class_; // Class of SubtypeTestCache.
@ -1820,6 +1824,40 @@ class PatchClass : public Object {
};
class SingleTargetCache : public Object {
public:
RawCode* target() const { return raw_ptr()->target_; }
void set_target(const Code& target) const;
static intptr_t target_offset() {
return OFFSET_OF(RawSingleTargetCache, target_);
}
#define DEFINE_NON_POINTER_FIELD_ACCESSORS(type, name) \
type name() const { return raw_ptr()->name##_; } \
void set_##name(type value) const { \
StoreNonPointer(&raw_ptr()->name##_, value); \
} \
static intptr_t name##_offset() { \
return OFFSET_OF(RawSingleTargetCache, name##_); \
} \
DEFINE_NON_POINTER_FIELD_ACCESSORS(uword, entry_point);
DEFINE_NON_POINTER_FIELD_ACCESSORS(intptr_t, lower_limit);
DEFINE_NON_POINTER_FIELD_ACCESSORS(intptr_t, upper_limit);
#undef DEFINE_NON_POINTER_FIELD_ACCESSORS
static intptr_t InstanceSize() {
return RoundedAllocationSize(sizeof(RawSingleTargetCache));
}
static RawSingleTargetCache* New();
private:
FINAL_HEAP_OBJECT_IMPLEMENTATION(SingleTargetCache, Object);
friend class Class;
};
// Object holding information about an IC: test classes and their
// corresponding targets. The owner of the ICData can be either the function
// or the original ICData object. In case of background compilation we

View file

@ -749,6 +749,11 @@ void ExceptionHandlers::PrintJSONImpl(JSONStream* stream,
}
void SingleTargetCache::PrintJSONImpl(JSONStream* stream, bool ref) const {
Object::PrintJSONImpl(stream, ref);
}
void ICData::PrintJSONImpl(JSONStream* stream, bool ref) const {
JSONObject jsobj(stream);
AddCommonObjectProperties(&jsobj, "Object", ref);

View file

@ -635,6 +635,13 @@ intptr_t RawContextScope::VisitContextScopePointers(
}
intptr_t RawSingleTargetCache::VisitSingleTargetCachePointers(
RawSingleTargetCache* raw_obj, ObjectPointerVisitor* visitor) {
visitor->VisitPointers(raw_obj->from(), raw_obj->to());
return SingleTargetCache::InstanceSize();
}
intptr_t RawICData::VisitICDataPointers(RawICData* raw_obj,
ObjectPointerVisitor* visitor) {
visitor->VisitPointers(raw_obj->from(), raw_obj->to());

View file

@ -39,6 +39,7 @@ namespace dart {
V(ExceptionHandlers) \
V(Context) \
V(ContextScope) \
V(SingleTargetCache) \
V(ICData) \
V(MegamorphicCache) \
V(SubtypeTestCache) \
@ -1480,6 +1481,21 @@ class RawContextScope : public RawObject {
};
class RawSingleTargetCache : public RawObject {
RAW_HEAP_OBJECT_IMPLEMENTATION(SingleTargetCache);
RawObject** from() {
return reinterpret_cast<RawObject**>(&ptr()->target_);
}
RawCode* target_;
RawObject** to() {
return reinterpret_cast<RawObject**>(&ptr()->target_);
}
uword entry_point_;
classid_t lower_limit_;
classid_t upper_limit_;
};
class RawICData : public RawObject {
RAW_HEAP_OBJECT_IMPLEMENTATION(ICData);

View file

@ -1514,6 +1514,24 @@ void RawContextScope::WriteTo(SnapshotWriter* writer,
}
RawSingleTargetCache* SingleTargetCache::ReadFrom(SnapshotReader* reader,
intptr_t object_id,
intptr_t tags,
Snapshot::Kind kind,
bool as_reference) {
UNREACHABLE();
return SingleTargetCache::null();
}
void RawSingleTargetCache::WriteTo(SnapshotWriter* writer,
intptr_t object_id,
Snapshot::Kind kind,
bool as_reference) {
UNREACHABLE();
}
RawICData* ICData::ReadFrom(SnapshotReader* reader,
intptr_t object_id,
intptr_t tags,

View file

@ -45,6 +45,7 @@ namespace dart {
V(GrowRegExpStack) \
V(CompileFunction) \
V(MonomorphicMiss) \
V(SingleTargetMiss) \
#define LEAF_RUNTIME_ENTRY_LIST(V) \
V(void, PrintStopMessage, const char*) \

View file

@ -37,10 +37,11 @@ class Deserializer;
V(OptimizeFunction) \
V(InvokeDartCode) \
V(DebugStepCheck) \
V(MonomorphicMiss) \
V(SingleTargetCall) \
V(ICCallThroughFunction) \
V(ICCallThroughCode) \
V(MegamorphicCall) \
V(MonomorphicMiss) \
V(FixAllocationStubTarget) \
V(Deoptimize) \
V(DeoptimizeLazy) \

View file

@ -2123,6 +2123,48 @@ void StubCode::GenerateICCallThroughCodeStub(Assembler* assembler) {
}
// Called from switchable IC calls.
// R0: receiver
// R9: SingleTargetCache
// Passed to target:
// CODE_REG: target Code object
void StubCode::GenerateSingleTargetCallStub(Assembler* assembler) {
__ NoMonomorphicCheckedEntry();
Label miss;
__ LoadClassIdMayBeSmi(R1, R0);
__ ldrh(R2, FieldAddress(R9, SingleTargetCache::lower_limit_offset()));
__ ldrh(R3, FieldAddress(R9, SingleTargetCache::upper_limit_offset()));
__ cmp(R1, Operand(R2));
__ b(&miss, LT);
__ cmp(R1, Operand(R3));
__ b(&miss, GT);
__ ldr(R1, FieldAddress(R9, SingleTargetCache::entry_point_offset()));
__ ldr(CODE_REG, FieldAddress(R9, SingleTargetCache::target_offset()));
__ bx(R1);
__ Bind(&miss);
__ EnterStubFrame();
__ Push(R0); // Preserve receiver.
__ PushObject(Object::null_object()); // Result.
__ Push(R0); // Arg0: Receiver
__ CallRuntime(kSingleTargetMissRuntimeEntry, 1);
__ Drop(1);
__ Pop(R9); // result = IC
__ Pop(R0); // Restore receiver.
__ LeaveStubFrame();
__ ldr(CODE_REG, Address(THR, Thread::ic_lookup_through_code_stub_offset()));
__ ldr(R1, FieldAddress(CODE_REG, Code::checked_entry_point_offset()));
__ bx(R1);
}
// Called from the monomorphic checked entry.
// R0: receiver
void StubCode::GenerateMonomorphicMissStub(Assembler* assembler) {

View file

@ -2183,6 +2183,50 @@ void StubCode::GenerateICCallThroughCodeStub(Assembler* assembler) {
}
// Called from switchable IC calls.
// R0: receiver
// R5: SingleTargetCache
// Passed to target:
// CODE_REG: target Code object
void StubCode::GenerateSingleTargetCallStub(Assembler* assembler) {
__ NoMonomorphicCheckedEntry();
Label miss;
__ LoadClassIdMayBeSmi(R1, R0);
__ ldr(R2, FieldAddress(R5, SingleTargetCache::lower_limit_offset()),
kUnsignedWord);
__ ldr(R3, FieldAddress(R5, SingleTargetCache::upper_limit_offset()),
kUnsignedWord);
__ cmp(R1, Operand(R2));
__ b(&miss, LT);
__ cmp(R1, Operand(R3));
__ b(&miss, GT);
__ ldr(R1, FieldAddress(R5, SingleTargetCache::entry_point_offset()));
__ ldr(CODE_REG, FieldAddress(R5, SingleTargetCache::target_offset()));
__ br(R1);
__ Bind(&miss);
__ EnterStubFrame();
__ Push(R0); // Preserve receiver.
__ PushObject(Object::null_object()); // Result.
__ Push(R0); // Arg0: Receiver
__ CallRuntime(kSingleTargetMissRuntimeEntry, 1);
__ Drop(1);
__ Pop(R5); // result = IC
__ Pop(R0); // Restore receiver.
__ LeaveStubFrame();
__ ldr(CODE_REG, Address(THR, Thread::ic_lookup_through_code_stub_offset()));
__ ldr(R1, FieldAddress(CODE_REG, Code::checked_entry_point_offset()));
__ br(R1);
}
// Called from the monomorphic checked entry.
// R0: receiver
void StubCode::GenerateMonomorphicMissStub(Assembler* assembler) {

View file

@ -2044,6 +2044,11 @@ void StubCode::GenerateICCallThroughCodeStub(Assembler* assembler) {
}
void StubCode::GenerateSingleTargetCallStub(Assembler* assembler) {
__ int3();
}
void StubCode::GenerateMonomorphicMissStub(Assembler* assembler) {
__ int3();
}

View file

@ -2277,6 +2277,47 @@ void StubCode::GenerateICCallThroughCodeStub(Assembler* assembler) {
}
// Called from switchable IC calls.
// T0: receiver
// S5: SingleTargetCache
// Passed to target:
// CODE_REG: target Code object
void StubCode::GenerateSingleTargetCallStub(Assembler* assembler) {
__ NoMonomorphicCheckedEntry();
Label miss;
__ LoadClassIdMayBeSmi(T1, T0);
__ lhu(T2, FieldAddress(S5, SingleTargetCache::lower_limit_offset()));
__ lhu(T3, FieldAddress(S5, SingleTargetCache::upper_limit_offset()));
__ BranchUnsignedLess(T1, T2, &miss);
__ BranchUnsignedGreater(T1, T3, &miss);
__ lw(T1, FieldAddress(S5, SingleTargetCache::entry_point_offset()));
__ lw(CODE_REG, FieldAddress(S5, SingleTargetCache::target_offset()));
__ jr(T1);
__ Bind(&miss);
__ EnterStubFrame();
__ Push(T0); // Preserve receiver.
__ PushObject(Object::null_object()); // Result.
__ Push(T0); // Arg0: Receiver
__ CallRuntime(kSingleTargetMissRuntimeEntry, 1);
__ Drop(1);
__ Pop(S5); // result = IC
__ Pop(T0); // Restore receiver.
__ LeaveStubFrame();
__ lw(CODE_REG, Address(THR, Thread::ic_lookup_through_code_stub_offset()));
__ lw(T1, FieldAddress(CODE_REG, Code::checked_entry_point_offset()));
__ jr(T1);
}
// Called from the monomorphic checked entry.
// T0: receiver
void StubCode::GenerateMonomorphicMissStub(Assembler* assembler) {

View file

@ -2167,6 +2167,45 @@ void StubCode::GenerateICCallThroughCodeStub(Assembler* assembler) {
}
// Called from switchable IC calls.
// RDI: receiver
// RBX: SingleTargetCache
// Passed to target::
// CODE_REG: target Code object
void StubCode::GenerateSingleTargetCallStub(Assembler* assembler) {
__ NoMonomorphicCheckedEntry();
Label miss;
__ LoadClassIdMayBeSmi(RAX, RDI);
__ movl(R9, FieldAddress(RBX, SingleTargetCache::lower_limit_offset()));
__ movl(R10, FieldAddress(RBX, SingleTargetCache::upper_limit_offset()));
__ cmpq(RAX, R9);
__ j(LESS, &miss, Assembler::kNearJump);
__ cmpq(RAX, R10);
__ j(GREATER, &miss, Assembler::kNearJump);
__ movq(RCX, FieldAddress(RBX, SingleTargetCache::entry_point_offset()));
__ movq(CODE_REG, FieldAddress(RBX, SingleTargetCache::target_offset()));
__ jmp(RCX);
__ Bind(&miss);
__ EnterStubFrame();
__ pushq(RDI); // Preserve receiver.
__ PushObject(Object::null_object()); // Result.
__ pushq(RDI); // Arg0: Receiver
__ CallRuntime(kSingleTargetMissRuntimeEntry, 1);
__ popq(RBX);
__ popq(RBX); // result = IC
__ popq(RDI); // Restore receiver.
__ LeaveStubFrame();
__ movq(CODE_REG, Address(THR, Thread::ic_lookup_through_code_stub_offset()));
__ movq(RCX, FieldAddress(CODE_REG, Code::checked_entry_point_offset()));
__ jmp(RCX);
}
// Called from the monomorphic checked entry.
// RDI: receiver
void StubCode::GenerateMonomorphicMissStub(Assembler* assembler) {

View file

@ -172,6 +172,7 @@ class ObjectPointerVisitor;
V(DeoptInfo, "DeoptInfo") \
V(Context, "Context") \
V(ContextScope, "ContextScope") \
V(SingleTargetCache, "SingleTargetCache") \
V(ICData, "ICData") \
V(MegamorphicCache, "MegamorphicCache") \
V(SubtypeTestCache, "SubtypeTestCache") \