diff --git a/runtime/lib/typed_data.cc b/runtime/lib/typed_data.cc index e28724a093e..f06f90c796c 100644 --- a/runtime/lib/typed_data.cc +++ b/runtime/lib/typed_data.cc @@ -15,20 +15,6 @@ namespace dart { // TypedData. -// Checks to see if offsetInBytes + num_bytes is in the range. -static void RangeCheck(intptr_t offset_in_bytes, - intptr_t access_size, - intptr_t length_in_bytes, - intptr_t element_size_in_bytes) { - if (!Utils::RangeCheck(offset_in_bytes, access_size, length_in_bytes)) { - const intptr_t index = - (offset_in_bytes + access_size) / element_size_in_bytes; - const intptr_t length = length_in_bytes / element_size_in_bytes; - Exceptions::ThrowRangeError("index", Integer::Handle(Integer::New(index)), - 0, length); - } -} - DEFINE_NATIVE_ENTRY(TypedDataBase_length, 0, 1) { GET_NON_NULL_NATIVE_ARGUMENT(TypedDataBase, array, arguments->NativeArgAt(0)); return Smi::New(array.Length()); @@ -130,55 +116,40 @@ DEFINE_NATIVE_ENTRY(TypedDataBase_setClampedRange, 0, 5) { return Object::null(); } -#define TYPED_DATA_GETTER(getter, object, ctor, access_size) \ +// The native getter and setter functions defined here are only called if +// unboxing doubles or SIMD values is not supported by the flow graph compiler, +// and the provided offsets have already been range checked by the calling code. + +#define TYPED_DATA_GETTER(getter, object, ctor) \ DEFINE_NATIVE_ENTRY(TypedData_##getter, 0, 2) { \ GET_NON_NULL_NATIVE_ARGUMENT(TypedDataBase, array, \ arguments->NativeArgAt(0)); \ GET_NON_NULL_NATIVE_ARGUMENT(Smi, offsetInBytes, \ arguments->NativeArgAt(1)); \ - RangeCheck(offsetInBytes.Value(), access_size, array.LengthInBytes(), \ - access_size); \ return object::ctor(array.getter(offsetInBytes.Value())); \ } -#define TYPED_DATA_SETTER(setter, object, get_object_value, access_size, \ - access_type) \ +#define TYPED_DATA_SETTER(setter, object, get_object_value, access_type) \ DEFINE_NATIVE_ENTRY(TypedData_##setter, 0, 3) { \ GET_NON_NULL_NATIVE_ARGUMENT(TypedDataBase, array, \ arguments->NativeArgAt(0)); \ GET_NON_NULL_NATIVE_ARGUMENT(Smi, offsetInBytes, \ arguments->NativeArgAt(1)); \ GET_NON_NULL_NATIVE_ARGUMENT(object, value, arguments->NativeArgAt(2)); \ - RangeCheck(offsetInBytes.Value(), access_size, array.LengthInBytes(), \ - access_size); \ array.setter(offsetInBytes.Value(), \ static_cast(value.get_object_value())); \ return Object::null(); \ } #define TYPED_DATA_NATIVES(type_name, object, ctor, get_object_value, \ - access_size, access_type) \ - TYPED_DATA_GETTER(Get##type_name, object, ctor, access_size) \ - TYPED_DATA_SETTER(Set##type_name, object, get_object_value, access_size, \ - access_type) + access_type) \ + TYPED_DATA_GETTER(Get##type_name, object, ctor) \ + TYPED_DATA_SETTER(Set##type_name, object, get_object_value, access_type) -TYPED_DATA_NATIVES(Int8, Integer, New, AsTruncatedUint32Value, 1, int8_t) -TYPED_DATA_NATIVES(Uint8, Integer, New, AsTruncatedUint32Value, 1, uint8_t) -TYPED_DATA_NATIVES(Int16, Integer, New, AsTruncatedUint32Value, 2, int16_t) -TYPED_DATA_NATIVES(Uint16, Integer, New, AsTruncatedUint32Value, 2, uint16_t) -TYPED_DATA_NATIVES(Int32, Integer, New, AsTruncatedUint32Value, 4, int32_t) -TYPED_DATA_NATIVES(Uint32, Integer, New, AsTruncatedUint32Value, 4, uint32_t) -TYPED_DATA_NATIVES(Int64, Integer, New, AsTruncatedInt64Value, 8, int64_t) -TYPED_DATA_NATIVES(Uint64, - Integer, - NewFromUint64, - AsTruncatedInt64Value, - 8, - uint64_t) -TYPED_DATA_NATIVES(Float32, Double, New, value, 4, float) -TYPED_DATA_NATIVES(Float64, Double, New, value, 8, double) -TYPED_DATA_NATIVES(Float32x4, Float32x4, New, value, 16, simd128_value_t) -TYPED_DATA_NATIVES(Int32x4, Int32x4, New, value, 16, simd128_value_t) -TYPED_DATA_NATIVES(Float64x2, Float64x2, New, value, 16, simd128_value_t) +TYPED_DATA_NATIVES(Float32, Double, New, value, float) +TYPED_DATA_NATIVES(Float64, Double, New, value, double) +TYPED_DATA_NATIVES(Float32x4, Float32x4, New, value, simd128_value_t) +TYPED_DATA_NATIVES(Int32x4, Int32x4, New, value, simd128_value_t) +TYPED_DATA_NATIVES(Float64x2, Float64x2, New, value, simd128_value_t) } // namespace dart diff --git a/runtime/tests/vm/dart/typed_list_index_checkbound_il_test.dart b/runtime/tests/vm/dart/typed_list_index_checkbound_il_test.dart index 794dabf0a46..05e1bab3047 100644 --- a/runtime/tests/vm/dart/typed_list_index_checkbound_il_test.dart +++ b/runtime/tests/vm/dart/typed_list_index_checkbound_il_test.dart @@ -24,10 +24,6 @@ int retrieveFromBase(Int8List src, int n) => src[n]; int retrieveFromExternal(Int8List src, int n) => src[n]; void matchIL$retrieveFromView(FlowGraph graph) { - // TODO(https://github.com/flutter/flutter/issues/138689): Once the regression - // for doing the replacement with the GenericCheckBound instruction is fixed - // on 32-bit archs, remove this. - if (is32BitConfiguration) return; graph.match([ match.block('Graph'), match.block('Function', [ @@ -60,10 +56,6 @@ void matchIL$retrieveFromView(FlowGraph graph) { } void matchIL$retrieveFromBase(FlowGraph graph) { - // TODO(https://github.com/flutter/flutter/issues/138689): Once the regression - // for doing the replacement with the GenericCheckBound instruction is fixed - // on 32-bit archs, remove this. - if (is32BitConfiguration) return; graph.match([ match.block('Graph'), match.block('Function', [ @@ -86,10 +78,6 @@ void matchIL$retrieveFromBase(FlowGraph graph) { } void matchIL$retrieveFromExternal(FlowGraph graph) { - // TODO(https://github.com/flutter/flutter/issues/138689): Once the regression - // for doing the replacement with the GenericCheckBound instruction is fixed - // on 32-bit archs, remove this. - if (is32BitConfiguration) return; graph.match([ match.block('Graph'), match.block('Function', [ diff --git a/runtime/vm/bootstrap_natives.h b/runtime/vm/bootstrap_natives.h index 11be9beb2a0..2e4dba9809d 100644 --- a/runtime/vm/bootstrap_natives.h +++ b/runtime/vm/bootstrap_natives.h @@ -160,22 +160,6 @@ namespace dart { V(Timeline_reportTaskEvent, 5) \ V(TypedDataBase_length, 1) \ V(TypedDataBase_setClampedRange, 5) \ - V(TypedData_GetInt8, 2) \ - V(TypedData_SetInt8, 3) \ - V(TypedData_GetUint8, 2) \ - V(TypedData_SetUint8, 3) \ - V(TypedData_GetInt16, 2) \ - V(TypedData_SetInt16, 3) \ - V(TypedData_GetUint16, 2) \ - V(TypedData_SetUint16, 3) \ - V(TypedData_GetInt32, 2) \ - V(TypedData_SetInt32, 3) \ - V(TypedData_GetUint32, 2) \ - V(TypedData_SetUint32, 3) \ - V(TypedData_GetInt64, 2) \ - V(TypedData_SetInt64, 3) \ - V(TypedData_GetUint64, 2) \ - V(TypedData_SetUint64, 3) \ V(TypedData_GetFloat32, 2) \ V(TypedData_SetFloat32, 3) \ V(TypedData_GetFloat64, 2) \ diff --git a/runtime/vm/compiler/backend/il.cc b/runtime/vm/compiler/backend/il.cc index 92a356bdc9d..8f5030ca248 100644 --- a/runtime/vm/compiler/backend/il.cc +++ b/runtime/vm/compiler/backend/il.cc @@ -6777,6 +6777,16 @@ LoadIndexedInstr::LoadIndexedInstr(Value* array, Definition* LoadIndexedInstr::Canonicalize(FlowGraph* flow_graph) { auto Z = flow_graph->zone(); + if (auto* const untag_payload = array()->definition()->AsLoadField()) { + // If loading from an internal typed data object, remove the load of + // PointerBase.data, as LoadIndexed knows how to load from a tagged + // internal typed data object directly and the LoadField may interfere with + // possible allocation sinking. + if (untag_payload->slot().IsIdentical(Slot::PointerBase_data()) && + IsTypedDataClassId(untag_payload->instance()->Type()->ToCid())) { + array()->BindTo(untag_payload->instance()->definition()); + } + } if (auto box = index()->definition()->AsBoxInt64()) { // TODO(dartbug.com/39432): Make LoadIndexed fully suport unboxed indices. if (!box->ComputeCanDeoptimize() && compiler::target::kWordSize == 8) { @@ -6832,6 +6842,16 @@ StoreIndexedInstr::StoreIndexedInstr(Value* array, Instruction* StoreIndexedInstr::Canonicalize(FlowGraph* flow_graph) { auto Z = flow_graph->zone(); + if (auto* const untag_payload = array()->definition()->AsLoadField()) { + // If loading from an internal typed data object, remove the load of + // PointerBase.data, as LoadIndexed knows how to load from a tagged + // internal typed data object directly and the LoadField may interfere with + // possible allocation sinking. + if (untag_payload->slot().IsIdentical(Slot::PointerBase_data()) && + IsTypedDataClassId(untag_payload->instance()->Type()->ToCid())) { + array()->BindTo(untag_payload->instance()->definition()); + } + } if (auto box = index()->definition()->AsBoxInt64()) { // TODO(dartbug.com/39432): Make StoreIndexed fully suport unboxed indices. if (!box->ComputeCanDeoptimize() && compiler::target::kWordSize == 8) { diff --git a/runtime/vm/compiler/backend/inliner.cc b/runtime/vm/compiler/backend/inliner.cc index 0816a144ee4..dc1a56fe0b7 100644 --- a/runtime/vm/compiler/backend/inliner.cc +++ b/runtime/vm/compiler/backend/inliner.cc @@ -2757,13 +2757,6 @@ static bool InlineTypedDataIndexCheck(FlowGraph* flow_graph, Instruction** last, Definition** result, const String& symbol) { -#if defined(TARGET_ARCH_IS_32_BIT) - // TODO(https://github.com/flutter/flutter/issues/138689): We only convert - // the index check to a GenericCheckBound instruction on 64-bit architectures, - // where the inputs are always unboxed. Once the regressions on 32-bit - // architectures has been identified and fixed, remove the #ifdef. - return false; -#else *entry = new (Z) FunctionEntryInstr(graph_entry, flow_graph->allocate_block_id(), call->GetBlock()->try_index(), DeoptId::kNone); @@ -2790,7 +2783,6 @@ static bool InlineTypedDataIndexCheck(FlowGraph* flow_graph, *last = cursor; *result = index; return true; -#endif } static intptr_t PrepareInlineIndexedOp(FlowGraph* flow_graph, @@ -3214,236 +3206,6 @@ static bool InlineLoadClassId(FlowGraph* flow_graph, return true; } -// Emits preparatory code for a typed getter/setter. -// Handles three cases: -// (1) dynamic: generates load untagged (internal or external) -// (2) external: generates load untagged -// (3) internal: no code required. -static void PrepareInlineByteArrayBaseOp(FlowGraph* flow_graph, - Instruction* call, - Definition* receiver, - intptr_t array_cid, - Definition** array, - Instruction** cursor) { - if (array_cid == kDynamicCid || IsExternalTypedDataClassId(array_cid)) { - // Internal or External typed data: load the untagged base address. - auto const loads_inner_pointer = - IsExternalTypedDataClassId(array_cid) - ? InnerPointerAccess::kCannotBeInnerPointer - : InnerPointerAccess::kMayBeInnerPointer; - auto* const elements = - new (Z) LoadFieldInstr(new (Z) Value(*array), Slot::PointerBase_data(), - loads_inner_pointer, call->source()); - *cursor = - flow_graph->AppendTo(*cursor, elements, nullptr, FlowGraph::kValue); - *array = elements; - } else { - // Internal typed data: no action. - ASSERT(IsTypedDataClassId(array_cid)); - } -} - -static bool InlineByteArrayBaseLoad(FlowGraph* flow_graph, - Definition* call, - Definition* receiver, - intptr_t array_cid, - intptr_t view_cid, - GraphEntryInstr* graph_entry, - FunctionEntryInstr** entry, - Instruction** last, - Definition** result) { - ASSERT(array_cid != kIllegalCid); - - // Dynamic calls are polymorphic due to: - // (A) extra bounds check computations (length stored in receiver), - // (B) external/internal typed data in receiver. - // Both issues are resolved in the inlined code. - // All getters that go through InlineByteArrayBaseLoad() have explicit - // bounds checks in all their clients in the library, so we can omit yet - // another inlined bounds check. - if (array_cid == kDynamicCid) { - ASSERT(call->IsStaticCall()); - } - - Definition* array = receiver; - Definition* index = call->ArgumentAt(1); - *entry = - new (Z) FunctionEntryInstr(graph_entry, flow_graph->allocate_block_id(), - call->GetBlock()->try_index(), DeoptId::kNone); - (*entry)->InheritDeoptTarget(Z, call); - Instruction* cursor = *entry; - - // Generates a template for the load, either a dynamic conditional - // that dispatches on external and internal storage, or a single - // case that deals with either external or internal storage. - PrepareInlineByteArrayBaseOp(flow_graph, call, receiver, array_cid, &array, - &cursor); - - // Fill out the generated template with loads. - // Load from either external or internal. - LoadIndexedInstr* load = new (Z) LoadIndexedInstr( - new (Z) Value(array), new (Z) Value(index), - /*index_unboxed=*/false, /*index_scale=*/1, view_cid, kUnalignedAccess, - DeoptId::kNone, call->source(), ResultType(call)); - flow_graph->AppendTo( - cursor, load, call->deopt_id() != DeoptId::kNone ? call->env() : nullptr, - FlowGraph::kValue); - cursor = *last = load; - - if (view_cid == kTypedDataFloat32ArrayCid) { - *last = new (Z) FloatToDoubleInstr(new (Z) Value((*last)->AsDefinition()), - DeoptId::kNone); - flow_graph->AppendTo(cursor, *last, nullptr, FlowGraph::kValue); - } - *result = (*last)->AsDefinition(); - return true; -} - -static StoreIndexedInstr* NewStore(FlowGraph* flow_graph, - Instruction* call, - Definition* array, - Definition* index, - Definition* stored_value, - intptr_t view_cid) { - return new (Z) StoreIndexedInstr( - new (Z) Value(array), new (Z) Value(index), new (Z) Value(stored_value), - kNoStoreBarrier, /*index_unboxed=*/false, - /*index_scale=*/1, view_cid, kUnalignedAccess, call->deopt_id(), - call->source()); -} - -static bool InlineByteArrayBaseStore(FlowGraph* flow_graph, - const Function& target, - Instruction* call, - Definition* receiver, - intptr_t array_cid, - intptr_t view_cid, - GraphEntryInstr* graph_entry, - FunctionEntryInstr** entry, - Instruction** last, - Definition** result) { - ASSERT(array_cid != kIllegalCid); - - // Dynamic calls are polymorphic due to: - // (A) extra bounds check computations (length stored in receiver), - // (B) external/internal typed data in receiver. - // Both issues are resolved in the inlined code. - // All setters that go through InlineByteArrayBaseLoad() have explicit - // bounds checks in all their clients in the library, so we can omit yet - // another inlined bounds check. - if (array_cid == kDynamicCid) { - ASSERT(call->IsStaticCall()); - } - - Definition* array = receiver; - Definition* index = call->ArgumentAt(1); - *entry = - new (Z) FunctionEntryInstr(graph_entry, flow_graph->allocate_block_id(), - call->GetBlock()->try_index(), DeoptId::kNone); - (*entry)->InheritDeoptTarget(Z, call); - Instruction* cursor = *entry; - - Definition* stored_value = call->ArgumentAt(2); - - // We know that the incomming type matches, but we still need to handle the - // null check. - if (!IsCompilingForSoundNullSafety()) { - String& name = String::ZoneHandle(Z, target.name()); - Instruction* check = new (Z) CheckNullInstr( - new (Z) Value(stored_value), name, call->deopt_id(), call->source()); - cursor = - flow_graph->AppendTo(cursor, check, call->env(), FlowGraph::kEffect); - } - - // Handle conversions and special unboxing (to ensure unboxing instructions - // are marked as truncating, since [SelectRepresentations] does not take care - // of that). - switch (view_cid) { - case kTypedDataInt8ArrayCid: - case kTypedDataInt16ArrayCid: - case kTypedDataUint8ArrayCid: - case kTypedDataUint8ClampedArrayCid: - case kTypedDataUint16ArrayCid: - case kExternalTypedDataUint8ArrayCid: - case kExternalTypedDataUint8ClampedArrayCid: { - stored_value = - UnboxInstr::Create(kUnboxedIntPtr, new (Z) Value(stored_value), - call->deopt_id(), Instruction::kNotSpeculative); - stored_value->AsUnboxInteger()->mark_truncating(); - cursor = flow_graph->AppendTo(cursor, stored_value, call->env(), - FlowGraph::kValue); - break; - } - - case kTypedDataInt64ArrayCid: - case kTypedDataUint64ArrayCid: { - stored_value = - new (Z) UnboxInt64Instr(new (Z) Value(stored_value), call->deopt_id(), - Instruction::kNotSpeculative); - cursor = flow_graph->AppendTo(cursor, stored_value, call->env(), - FlowGraph::kValue); - break; - } - - case kTypedDataFloat32ArrayCid: - case kTypedDataFloat64ArrayCid: { - stored_value = - UnboxInstr::Create(kUnboxedDouble, new (Z) Value(stored_value), - call->deopt_id(), Instruction::kNotSpeculative); - cursor = flow_graph->AppendTo(cursor, stored_value, call->env(), - FlowGraph::kValue); - if (view_cid == kTypedDataFloat32ArrayCid) { - stored_value = new (Z) - DoubleToFloatInstr(new (Z) Value(stored_value), call->deopt_id()); - cursor = flow_graph->AppendTo(cursor, stored_value, call->env(), - FlowGraph::kValue); - } - break; - } - - case kTypedDataInt32ArrayCid: { - stored_value = new (Z) - UnboxInt32Instr(UnboxInt32Instr::kTruncate, - new (Z) Value(stored_value), call->deopt_id()); - cursor = flow_graph->AppendTo(cursor, stored_value, call->env(), - FlowGraph::kValue); - break; - } - case kTypedDataUint32ArrayCid: { - stored_value = new (Z) - UnboxUint32Instr(new (Z) Value(stored_value), call->deopt_id()); - ASSERT(stored_value->AsUnboxInteger()->is_truncating()); - cursor = flow_graph->AppendTo(cursor, stored_value, call->env(), - FlowGraph::kValue); - break; - } - default: - break; - } - - // Generates a template for the store, either a dynamic conditional - // that dispatches on external and internal storage, or a single - // case that deals with either external or internal storage. - PrepareInlineByteArrayBaseOp(flow_graph, call, receiver, array_cid, &array, - &cursor); - - // Fill out the generated template with stores. - { - // Store on either external or internal. - StoreIndexedInstr* store = - NewStore(flow_graph, call, array, index, stored_value, view_cid); - flow_graph->AppendTo( - cursor, store, - call->deopt_id() != DeoptId::kNone ? call->env() : nullptr, - FlowGraph::kEffect); - *last = store; - } - // We need a return value to replace uses of the original definition. However, - // the final instruction is a use of 'void operator[]=()', so we use null. - *result = flow_graph->constant_null(); - return true; -} - // Returns the LoadIndexedInstr. static Definition* PrepareInlineStringIndexOp(FlowGraph* flow_graph, Instruction* call, @@ -4579,73 +4341,6 @@ bool FlowGraphInliner::TryInlineRecognizedMethod( case MethodRecognizer::kUint64ArrayGetIndexed: return InlineGetIndexed(flow_graph, can_speculate, is_dynamic_call, kind, call, receiver, graph_entry, entry, last, result); - case MethodRecognizer::kByteArrayBaseGetInt8: - return InlineByteArrayBaseLoad(flow_graph, call, receiver, receiver_cid, - kTypedDataInt8ArrayCid, graph_entry, entry, - last, result); - case MethodRecognizer::kByteArrayBaseGetUint8: - return InlineByteArrayBaseLoad(flow_graph, call, receiver, receiver_cid, - kTypedDataUint8ArrayCid, graph_entry, - entry, last, result); - case MethodRecognizer::kByteArrayBaseGetInt16: - return InlineByteArrayBaseLoad(flow_graph, call, receiver, receiver_cid, - kTypedDataInt16ArrayCid, graph_entry, - entry, last, result); - case MethodRecognizer::kByteArrayBaseGetUint16: - return InlineByteArrayBaseLoad(flow_graph, call, receiver, receiver_cid, - kTypedDataUint16ArrayCid, graph_entry, - entry, last, result); - case MethodRecognizer::kByteArrayBaseGetInt32: - return InlineByteArrayBaseLoad(flow_graph, call, receiver, receiver_cid, - kTypedDataInt32ArrayCid, graph_entry, - entry, last, result); - case MethodRecognizer::kByteArrayBaseGetUint32: - return InlineByteArrayBaseLoad(flow_graph, call, receiver, receiver_cid, - kTypedDataUint32ArrayCid, graph_entry, - entry, last, result); - case MethodRecognizer::kByteArrayBaseGetInt64: - return InlineByteArrayBaseLoad(flow_graph, call, receiver, receiver_cid, - kTypedDataInt64ArrayCid, graph_entry, - entry, last, result); - case MethodRecognizer::kByteArrayBaseGetUint64: - return InlineByteArrayBaseLoad(flow_graph, call, receiver, receiver_cid, - kTypedDataUint64ArrayCid, graph_entry, - entry, last, result); - case MethodRecognizer::kByteArrayBaseGetFloat32: - if (!CanUnboxDouble()) { - return false; - } - return InlineByteArrayBaseLoad(flow_graph, call, receiver, receiver_cid, - kTypedDataFloat32ArrayCid, graph_entry, - entry, last, result); - case MethodRecognizer::kByteArrayBaseGetFloat64: - if (!CanUnboxDouble()) { - return false; - } - return InlineByteArrayBaseLoad(flow_graph, call, receiver, receiver_cid, - kTypedDataFloat64ArrayCid, graph_entry, - entry, last, result); - case MethodRecognizer::kByteArrayBaseGetFloat32x4: - if (!ShouldInlineSimd()) { - return false; - } - return InlineByteArrayBaseLoad(flow_graph, call, receiver, receiver_cid, - kTypedDataFloat32x4ArrayCid, graph_entry, - entry, last, result); - case MethodRecognizer::kByteArrayBaseGetFloat64x2: - if (!ShouldInlineSimd()) { - return false; - } - return InlineByteArrayBaseLoad(flow_graph, call, receiver, receiver_cid, - kTypedDataFloat64x2ArrayCid, graph_entry, - entry, last, result); - case MethodRecognizer::kByteArrayBaseGetInt32x4: - if (!ShouldInlineSimd()) { - return false; - } - return InlineByteArrayBaseLoad(flow_graph, call, receiver, receiver_cid, - kTypedDataInt32x4ArrayCid, graph_entry, - entry, last, result); case MethodRecognizer::kClassIDgetID: return InlineLoadClassId(flow_graph, call, graph_entry, entry, last, result); @@ -4704,73 +4399,6 @@ bool FlowGraphInliner::TryInlineRecognizedMethod( return InlineSetIndexed(flow_graph, kind, target, call, receiver, source, exactness, graph_entry, entry, last, result); } - case MethodRecognizer::kByteArrayBaseSetInt8: - return InlineByteArrayBaseStore(flow_graph, target, call, receiver, - receiver_cid, kTypedDataInt8ArrayCid, - graph_entry, entry, last, result); - case MethodRecognizer::kByteArrayBaseSetUint8: - return InlineByteArrayBaseStore(flow_graph, target, call, receiver, - receiver_cid, kTypedDataUint8ArrayCid, - graph_entry, entry, last, result); - case MethodRecognizer::kByteArrayBaseSetInt16: - return InlineByteArrayBaseStore(flow_graph, target, call, receiver, - receiver_cid, kTypedDataInt16ArrayCid, - graph_entry, entry, last, result); - case MethodRecognizer::kByteArrayBaseSetUint16: - return InlineByteArrayBaseStore(flow_graph, target, call, receiver, - receiver_cid, kTypedDataUint16ArrayCid, - graph_entry, entry, last, result); - case MethodRecognizer::kByteArrayBaseSetInt32: - return InlineByteArrayBaseStore(flow_graph, target, call, receiver, - receiver_cid, kTypedDataInt32ArrayCid, - graph_entry, entry, last, result); - case MethodRecognizer::kByteArrayBaseSetUint32: - return InlineByteArrayBaseStore(flow_graph, target, call, receiver, - receiver_cid, kTypedDataUint32ArrayCid, - graph_entry, entry, last, result); - case MethodRecognizer::kByteArrayBaseSetInt64: - return InlineByteArrayBaseStore(flow_graph, target, call, receiver, - receiver_cid, kTypedDataInt64ArrayCid, - graph_entry, entry, last, result); - case MethodRecognizer::kByteArrayBaseSetUint64: - return InlineByteArrayBaseStore(flow_graph, target, call, receiver, - receiver_cid, kTypedDataUint64ArrayCid, - graph_entry, entry, last, result); - case MethodRecognizer::kByteArrayBaseSetFloat32: - if (!CanUnboxDouble()) { - return false; - } - return InlineByteArrayBaseStore(flow_graph, target, call, receiver, - receiver_cid, kTypedDataFloat32ArrayCid, - graph_entry, entry, last, result); - case MethodRecognizer::kByteArrayBaseSetFloat64: - if (!CanUnboxDouble()) { - return false; - } - return InlineByteArrayBaseStore(flow_graph, target, call, receiver, - receiver_cid, kTypedDataFloat64ArrayCid, - graph_entry, entry, last, result); - case MethodRecognizer::kByteArrayBaseSetFloat32x4: - if (!ShouldInlineSimd()) { - return false; - } - return InlineByteArrayBaseStore(flow_graph, target, call, receiver, - receiver_cid, kTypedDataFloat32x4ArrayCid, - graph_entry, entry, last, result); - case MethodRecognizer::kByteArrayBaseSetFloat64x2: - if (!ShouldInlineSimd()) { - return false; - } - return InlineByteArrayBaseStore(flow_graph, target, call, receiver, - receiver_cid, kTypedDataFloat64x2ArrayCid, - graph_entry, entry, last, result); - case MethodRecognizer::kByteArrayBaseSetInt32x4: - if (!ShouldInlineSimd()) { - return false; - } - return InlineByteArrayBaseStore(flow_graph, target, call, receiver, - receiver_cid, kTypedDataInt32x4ArrayCid, - graph_entry, entry, last, result); case MethodRecognizer::kOneByteStringCodeUnitAt: case MethodRecognizer::kTwoByteStringCodeUnitAt: case MethodRecognizer::kExternalOneByteStringCodeUnitAt: diff --git a/runtime/vm/compiler/compiler_state.cc b/runtime/vm/compiler/compiler_state.cc index eb9e57ea10d..8085d2dd264 100644 --- a/runtime/vm/compiler/compiler_state.cc +++ b/runtime/vm/compiler/compiler_state.cc @@ -129,6 +129,54 @@ const Function& CompilerState::StringBaseInterpolate() { return *interpolate_; } +const Class& CompilerState::TypedListClass() { + if (typed_list_class_ == nullptr) { + Thread* thread = Thread::Current(); + Zone* zone = thread->zone(); + + const Library& lib = Library::Handle(zone, Library::TypedDataLibrary()); + const Class& cls = Class::ZoneHandle( + zone, lib.LookupClassAllowPrivate(Symbols::_TypedList())); + ASSERT(!cls.IsNull()); + const Error& error = Error::Handle(zone, cls.EnsureIsFinalized(thread)); + ASSERT(!error.IsNull()); + typed_list_class_ = &cls; + } + return *typed_list_class_; +} + +#define DEFINE_TYPED_LIST_NATIVE_FUNCTION_GETTER(Upper, Lower) \ + const Function& CompilerState::TypedListGet##Upper() { \ + if (typed_list_get_##Lower##_ == nullptr) { \ + Thread* thread = Thread::Current(); \ + Zone* zone = thread->zone(); \ + const auto& cls = CompilerState::TypedListClass(); \ + typed_list_get_##Lower##_ = &Function::ZoneHandle( \ + zone, cls.LookupFunctionAllowPrivate(Symbols::_nativeGet##Upper())); \ + ASSERT(!typed_list_get_##Lower##_->IsNull()); \ + } \ + return *typed_list_get_##Lower##_; \ + } \ + const Function& CompilerState::TypedListSet##Upper() { \ + if (typed_list_set_##Lower##_ == nullptr) { \ + Thread* thread = Thread::Current(); \ + Zone* zone = thread->zone(); \ + const auto& cls = CompilerState::TypedListClass(); \ + typed_list_set_##Lower##_ = &Function::ZoneHandle( \ + zone, cls.LookupFunctionAllowPrivate(Symbols::_nativeSet##Upper())); \ + ASSERT(!typed_list_set_##Lower##_->IsNull()); \ + } \ + return *typed_list_set_##Lower##_; \ + } + +DEFINE_TYPED_LIST_NATIVE_FUNCTION_GETTER(Float32, float32) +DEFINE_TYPED_LIST_NATIVE_FUNCTION_GETTER(Float64, float64) +DEFINE_TYPED_LIST_NATIVE_FUNCTION_GETTER(Float32x4, float32x4) +DEFINE_TYPED_LIST_NATIVE_FUNCTION_GETTER(Int32x4, int32x4) +DEFINE_TYPED_LIST_NATIVE_FUNCTION_GETTER(Float64x2, float64x2) + +#undef DEFINE_TYPED_LIST_NATIVE_FUNCTION_GETTER + void CompilerState::ReportCrash() { OS::PrintErr("=== Crash occurred when compiling %s in %s mode in %s pass\n", function() != nullptr ? function()->ToFullyQualifiedCString() diff --git a/runtime/vm/compiler/compiler_state.h b/runtime/vm/compiler/compiler_state.h index ff3ac87afb7..56c5e7d9bf1 100644 --- a/runtime/vm/compiler/compiler_state.h +++ b/runtime/vm/compiler/compiler_state.h @@ -107,6 +107,17 @@ class CompilerState : public ThreadStackResource { // Returns _StringBase._interpolateSingle const Function& StringBaseInterpolateSingle(); + const Function& TypedListGetFloat32(); + const Function& TypedListSetFloat32(); + const Function& TypedListGetFloat64(); + const Function& TypedListSetFloat64(); + const Function& TypedListGetFloat32x4(); + const Function& TypedListSetFloat32x4(); + const Function& TypedListGetInt32x4(); + const Function& TypedListSetInt32x4(); + const Function& TypedListGetFloat64x2(); + const Function& TypedListSetFloat64x2(); + const Function* function() const { return function_; } void set_function(const Function& function) { function_ = &function; } @@ -122,6 +133,8 @@ class CompilerState : public ThreadStackResource { void ReportCrash(); private: + const Class& TypedListClass(); + CHA cha_; intptr_t deopt_id_ = 0; @@ -142,6 +155,17 @@ class CompilerState : public ThreadStackResource { const Class* comparable_class_ = nullptr; const Function* interpolate_ = nullptr; const Function* interpolate_single_ = nullptr; + const Class* typed_list_class_ = nullptr; + const Function* typed_list_get_float32_ = nullptr; + const Function* typed_list_set_float32_ = nullptr; + const Function* typed_list_get_float64_ = nullptr; + const Function* typed_list_set_float64_ = nullptr; + const Function* typed_list_get_float32x4_ = nullptr; + const Function* typed_list_set_float32x4_ = nullptr; + const Function* typed_list_get_int32x4_ = nullptr; + const Function* typed_list_set_int32x4_ = nullptr; + const Function* typed_list_get_float64x2_ = nullptr; + const Function* typed_list_set_float64x2_ = nullptr; const Function* function_ = nullptr; const CompilerPass* pass_ = nullptr; diff --git a/runtime/vm/compiler/frontend/kernel_to_il.cc b/runtime/vm/compiler/frontend/kernel_to_il.cc index fb11748a730..e6e36520fd6 100644 --- a/runtime/vm/compiler/frontend/kernel_to_il.cc +++ b/runtime/vm/compiler/frontend/kernel_to_il.cc @@ -942,6 +942,32 @@ bool FlowGraphBuilder::IsRecognizedMethodForFlowGraph( case MethodRecognizer::kRecord_numFields: case MethodRecognizer::kSuspendState_clone: case MethodRecognizer::kSuspendState_resume: + case MethodRecognizer::kTypedList_GetInt8: + case MethodRecognizer::kTypedList_SetInt8: + case MethodRecognizer::kTypedList_GetUint8: + case MethodRecognizer::kTypedList_SetUint8: + case MethodRecognizer::kTypedList_GetInt16: + case MethodRecognizer::kTypedList_SetInt16: + case MethodRecognizer::kTypedList_GetUint16: + case MethodRecognizer::kTypedList_SetUint16: + case MethodRecognizer::kTypedList_GetInt32: + case MethodRecognizer::kTypedList_SetInt32: + case MethodRecognizer::kTypedList_GetUint32: + case MethodRecognizer::kTypedList_SetUint32: + case MethodRecognizer::kTypedList_GetInt64: + case MethodRecognizer::kTypedList_SetInt64: + case MethodRecognizer::kTypedList_GetUint64: + case MethodRecognizer::kTypedList_SetUint64: + case MethodRecognizer::kTypedList_GetFloat32: + case MethodRecognizer::kTypedList_SetFloat32: + case MethodRecognizer::kTypedList_GetFloat64: + case MethodRecognizer::kTypedList_SetFloat64: + case MethodRecognizer::kTypedList_GetInt32x4: + case MethodRecognizer::kTypedList_SetInt32x4: + case MethodRecognizer::kTypedList_GetFloat32x4: + case MethodRecognizer::kTypedList_SetFloat32x4: + case MethodRecognizer::kTypedList_GetFloat64x2: + case MethodRecognizer::kTypedList_SetFloat64x2: case MethodRecognizer::kTypedData_memMove1: case MethodRecognizer::kTypedData_memMove2: case MethodRecognizer::kTypedData_memMove4: @@ -1155,8 +1181,85 @@ FlowGraph* FlowGraphBuilder::BuildGraphOfRecognizedMethod( body += TailCall(resume_stub); break; } + case MethodRecognizer::kTypedList_GetInt8: + body += BuildTypedListGet(function, kTypedDataInt8ArrayCid); + break; + case MethodRecognizer::kTypedList_SetInt8: + body += BuildTypedListSet(function, kTypedDataInt8ArrayCid); + break; + case MethodRecognizer::kTypedList_GetUint8: + body += BuildTypedListGet(function, kTypedDataUint8ArrayCid); + break; + case MethodRecognizer::kTypedList_SetUint8: + body += BuildTypedListSet(function, kTypedDataUint8ArrayCid); + break; + case MethodRecognizer::kTypedList_GetInt16: + body += BuildTypedListGet(function, kTypedDataInt16ArrayCid); + break; + case MethodRecognizer::kTypedList_SetInt16: + body += BuildTypedListSet(function, kTypedDataInt16ArrayCid); + break; + case MethodRecognizer::kTypedList_GetUint16: + body += BuildTypedListGet(function, kTypedDataUint16ArrayCid); + break; + case MethodRecognizer::kTypedList_SetUint16: + body += BuildTypedListSet(function, kTypedDataUint16ArrayCid); + break; + case MethodRecognizer::kTypedList_GetInt32: + body += BuildTypedListGet(function, kTypedDataInt32ArrayCid); + break; + case MethodRecognizer::kTypedList_SetInt32: + body += BuildTypedListSet(function, kTypedDataInt32ArrayCid); + break; + case MethodRecognizer::kTypedList_GetUint32: + body += BuildTypedListGet(function, kTypedDataUint32ArrayCid); + break; + case MethodRecognizer::kTypedList_SetUint32: + body += BuildTypedListSet(function, kTypedDataUint32ArrayCid); + break; + case MethodRecognizer::kTypedList_GetInt64: + body += BuildTypedListGet(function, kTypedDataInt64ArrayCid); + break; + case MethodRecognizer::kTypedList_SetInt64: + body += BuildTypedListSet(function, kTypedDataInt64ArrayCid); + break; + case MethodRecognizer::kTypedList_GetUint64: + body += BuildTypedListGet(function, kTypedDataUint64ArrayCid); + break; + case MethodRecognizer::kTypedList_SetUint64: + body += BuildTypedListSet(function, kTypedDataUint64ArrayCid); + break; + case MethodRecognizer::kTypedList_GetFloat32: + body += BuildTypedListGet(function, kTypedDataFloat32ArrayCid); + break; + case MethodRecognizer::kTypedList_SetFloat32: + body += BuildTypedListSet(function, kTypedDataFloat32ArrayCid); + break; + case MethodRecognizer::kTypedList_GetFloat64: + body += BuildTypedListGet(function, kTypedDataFloat64ArrayCid); + break; + case MethodRecognizer::kTypedList_SetFloat64: + body += BuildTypedListSet(function, kTypedDataFloat64ArrayCid); + break; + case MethodRecognizer::kTypedList_GetInt32x4: + body += BuildTypedListGet(function, kTypedDataInt32x4ArrayCid); + break; + case MethodRecognizer::kTypedList_SetInt32x4: + body += BuildTypedListSet(function, kTypedDataInt32x4ArrayCid); + break; + case MethodRecognizer::kTypedList_GetFloat32x4: + body += BuildTypedListGet(function, kTypedDataFloat32x4ArrayCid); + break; + case MethodRecognizer::kTypedList_SetFloat32x4: + body += BuildTypedListSet(function, kTypedDataFloat32x4ArrayCid); + break; + case MethodRecognizer::kTypedList_GetFloat64x2: + body += BuildTypedListGet(function, kTypedDataFloat64x2ArrayCid); + break; + case MethodRecognizer::kTypedList_SetFloat64x2: + body += BuildTypedListSet(function, kTypedDataFloat64x2ArrayCid); + break; case MethodRecognizer::kTypedData_memMove1: - // Pick an appropriate typed data cid based on the element size. body += BuildTypedDataMemMove(function, 1); break; case MethodRecognizer::kTypedData_memMove2: @@ -1815,6 +1918,127 @@ Fragment FlowGraphBuilder::BuildTypedDataViewFactoryConstructor( return body; } +static bool CanUnboxElements(intptr_t view_cid) { + switch (view_cid) { + case kTypedDataFloat32ArrayCid: + case kTypedDataFloat64ArrayCid: + return FlowGraphCompiler::SupportsUnboxedDoubles(); + case kTypedDataInt32x4ArrayCid: + case kTypedDataFloat32x4ArrayCid: + case kTypedDataFloat64x2ArrayCid: + return FlowGraphCompiler::SupportsUnboxedSimd128(); + default: + return true; + } +} + +static const Function& TypedListGetNativeFunction(Thread* thread, + intptr_t view_cid) { + auto& state = thread->compiler_state(); + switch (view_cid) { + case kTypedDataFloat32ArrayCid: + return state.TypedListGetFloat32(); + case kTypedDataFloat64ArrayCid: + return state.TypedListGetFloat64(); + case kTypedDataInt32x4ArrayCid: + return state.TypedListGetInt32x4(); + case kTypedDataFloat32x4ArrayCid: + return state.TypedListGetFloat32x4(); + case kTypedDataFloat64x2ArrayCid: + return state.TypedListGetFloat64x2(); + default: + UNREACHABLE(); + return Object::null_function(); + } +} + +Fragment FlowGraphBuilder::BuildTypedListGet(const Function& function, + intptr_t view_cid) { + const intptr_t kNumParameters = 2; + ASSERT_EQUAL(parsed_function_->function().NumParameters(), kNumParameters); + // Guaranteed to be non-null since it's only called internally from other + // instance methods. + LocalVariable* arg_receiver = parsed_function_->RawParameterVariable(0); + // Guaranteed to be a non-null Smi due to bounds checks prior to call. + LocalVariable* arg_offset_in_bytes = + parsed_function_->RawParameterVariable(1); + + Fragment body; + if (CanUnboxElements(view_cid)) { + body += LoadLocal(arg_receiver); + body += LoadNativeField(Slot::PointerBase_data(), + InnerPointerAccess::kMayBeInnerPointer); + body += LoadLocal(arg_offset_in_bytes); + body += LoadIndexed(view_cid, /*index_scale=*/1, + /*index_unboxed=*/false, kUnalignedAccess); + body += Box(LoadIndexedInstr::RepresentationOfArrayElement(view_cid)); + } else { + const auto& native_function = TypedListGetNativeFunction(thread_, view_cid); + body += LoadLocal(arg_receiver); + body += LoadLocal(arg_offset_in_bytes); + body += StaticCall(TokenPosition::kNoSource, native_function, + kNumParameters, ICData::kNoRebind); + } + return body; +} + +static const Function& TypedListSetNativeFunction(Thread* thread, + intptr_t view_cid) { + auto& state = thread->compiler_state(); + switch (view_cid) { + case kTypedDataFloat32ArrayCid: + return state.TypedListSetFloat32(); + case kTypedDataFloat64ArrayCid: + return state.TypedListSetFloat64(); + case kTypedDataInt32x4ArrayCid: + return state.TypedListSetInt32x4(); + case kTypedDataFloat32x4ArrayCid: + return state.TypedListSetFloat32x4(); + case kTypedDataFloat64x2ArrayCid: + return state.TypedListSetFloat64x2(); + default: + UNREACHABLE(); + return Object::null_function(); + } +} + +Fragment FlowGraphBuilder::BuildTypedListSet(const Function& function, + intptr_t view_cid) { + const intptr_t kNumParameters = 3; + ASSERT_EQUAL(parsed_function_->function().NumParameters(), kNumParameters); + // Guaranteed to be non-null since it's only called internally from other + // instance methods. + LocalVariable* arg_receiver = parsed_function_->RawParameterVariable(0); + // Guaranteed to be a non-null Smi due to bounds checks prior to call. + LocalVariable* arg_offset_in_bytes = + parsed_function_->RawParameterVariable(1); + LocalVariable* arg_value = parsed_function_->RawParameterVariable(2); + + Fragment body; + if (CanUnboxElements(view_cid)) { + body += LoadLocal(arg_receiver); + body += LoadNativeField(Slot::PointerBase_data(), + InnerPointerAccess::kMayBeInnerPointer); + body += LoadLocal(arg_offset_in_bytes); + body += LoadLocal(arg_value); + body += + CheckNullOptimized(Symbols::Value(), CheckNullInstr::kArgumentError); + body += UnboxTruncate( + StoreIndexedInstr::RepresentationOfArrayElement(view_cid)); + body += StoreIndexedTypedData(view_cid, /*index_scale=*/1, + /*index_unboxed=*/false, kUnalignedAccess); + body += NullConstant(); + } else { + const auto& native_function = TypedListSetNativeFunction(thread_, view_cid); + body += LoadLocal(arg_receiver); + body += LoadLocal(arg_offset_in_bytes); + body += LoadLocal(arg_value); + body += StaticCall(TokenPosition::kNoSource, native_function, + kNumParameters, ICData::kNoRebind); + } + return body; +} + Fragment FlowGraphBuilder::BuildTypedDataMemMove(const Function& function, intptr_t element_size) { ASSERT_EQUAL(parsed_function_->function().NumParameters(), 5); diff --git a/runtime/vm/compiler/frontend/kernel_to_il.h b/runtime/vm/compiler/frontend/kernel_to_il.h index c112a32f8fc..208ca7d754f 100644 --- a/runtime/vm/compiler/frontend/kernel_to_il.h +++ b/runtime/vm/compiler/frontend/kernel_to_il.h @@ -158,6 +158,8 @@ class FlowGraphBuilder : public BaseFlowGraphBuilder { FlowGraph* BuildGraphOfRecognizedMethod(const Function& function); + Fragment BuildTypedListGet(const Function& function, intptr_t view_cid); + Fragment BuildTypedListSet(const Function& function, intptr_t view_cid); Fragment BuildTypedDataMemMove(const Function& function, intptr_t element_size); Fragment BuildTypedDataViewFactoryConstructor(const Function& function, diff --git a/runtime/vm/compiler/recognized_methods_list.h b/runtime/vm/compiler/recognized_methods_list.h index 8b47dc30117..b6528feaade 100644 --- a/runtime/vm/compiler/recognized_methods_list.h +++ b/runtime/vm/compiler/recognized_methods_list.h @@ -23,32 +23,32 @@ namespace dart { V(_Record, get:_numFields, Record_numFields, 0x7bb37f73) \ V(_Record, get:_shape, Record_shape, 0x70d29513) \ V(_Record, _fieldAt, Record_fieldAt, 0xb48e2c93) \ - V(_TypedList, _getInt8, ByteArrayBaseGetInt8, 0x16155054) \ - V(_TypedList, _getUint8, ByteArrayBaseGetUint8, 0x1771724a) \ - V(_TypedList, _getInt16, ByteArrayBaseGetInt16, 0x2e320a6f) \ - V(_TypedList, _getUint16, ByteArrayBaseGetUint16, 0x2fb36ad9) \ - V(_TypedList, _getInt32, ByteArrayBaseGetInt32, 0x1909a12a) \ - V(_TypedList, _getUint32, ByteArrayBaseGetUint32, 0x194ee29b) \ - V(_TypedList, _getInt64, ByteArrayBaseGetInt64, 0xf652341f) \ - V(_TypedList, _getUint64, ByteArrayBaseGetUint64, 0x2c4ced79) \ - V(_TypedList, _getFloat32, ByteArrayBaseGetFloat32, 0xe8e81527) \ - V(_TypedList, _getFloat64, ByteArrayBaseGetFloat64, 0xf81baa54) \ - V(_TypedList, _getFloat32x4, ByteArrayBaseGetFloat32x4, 0xaf1e8105) \ - V(_TypedList, _getFloat64x2, ByteArrayBaseGetFloat64x2, 0x544ea0e0) \ - V(_TypedList, _getInt32x4, ByteArrayBaseGetInt32x4, 0x5564e82b) \ - V(_TypedList, _setInt8, ByteArrayBaseSetInt8, 0xe17ab7c2) \ - V(_TypedList, _setUint8, ByteArrayBaseSetUint8, 0xaf4b2b68) \ - V(_TypedList, _setInt16, ByteArrayBaseSetInt16, 0xbad7b447) \ - V(_TypedList, _setUint16, ByteArrayBaseSetUint16, 0xce13bc6f) \ - V(_TypedList, _setInt32, ByteArrayBaseSetInt32, 0xbdcc1f60) \ - V(_TypedList, _setUint32, ByteArrayBaseSetUint32, 0xb95817d2) \ - V(_TypedList, _setInt64, ByteArrayBaseSetInt64, 0xc8bec39a) \ - V(_TypedList, _setUint64, ByteArrayBaseSetUint64, 0xda38a625) \ - V(_TypedList, _setFloat32, ByteArrayBaseSetFloat32, 0x2f27a200) \ - V(_TypedList, _setFloat64, ByteArrayBaseSetFloat64, 0x234b6cf2) \ - V(_TypedList, _setFloat32x4, ByteArrayBaseSetFloat32x4, 0x38b79d7a) \ - V(_TypedList, _setFloat64x2, ByteArrayBaseSetFloat64x2, 0xbadc4b5f) \ - V(_TypedList, _setInt32x4, ByteArrayBaseSetInt32x4, 0x5cda767b) \ + V(_TypedList, _getInt8, TypedList_GetInt8, 0x16155415) \ + V(_TypedList, _getUint8, TypedList_GetUint8, 0x1771760b) \ + V(_TypedList, _getInt16, TypedList_GetInt16, 0x2e320e30) \ + V(_TypedList, _getUint16, TypedList_GetUint16, 0x2fb36e9a) \ + V(_TypedList, _getInt32, TypedList_GetInt32, 0x1909a4eb) \ + V(_TypedList, _getUint32, TypedList_GetUint32, 0x194ee65c) \ + V(_TypedList, _getInt64, TypedList_GetInt64, 0xf65237e0) \ + V(_TypedList, _getUint64, TypedList_GetUint64, 0x2c4cf13a) \ + V(_TypedList, _getFloat32, TypedList_GetFloat32, 0xe8e818e8) \ + V(_TypedList, _getFloat64, TypedList_GetFloat64, 0xf81bae15) \ + V(_TypedList, _getFloat32x4, TypedList_GetFloat32x4, 0xaf1e84c6) \ + V(_TypedList, _getFloat64x2, TypedList_GetFloat64x2, 0x544ea4a1) \ + V(_TypedList, _getInt32x4, TypedList_GetInt32x4, 0x5564ebec) \ + V(_TypedList, _setInt8, TypedList_SetInt8, 0xe17abb83) \ + V(_TypedList, _setUint8, TypedList_SetUint8, 0xaf4b2f29) \ + V(_TypedList, _setInt16, TypedList_SetInt16, 0xbad7b808) \ + V(_TypedList, _setUint16, TypedList_SetUint16, 0xce13c030) \ + V(_TypedList, _setInt32, TypedList_SetInt32, 0xbdcc2321) \ + V(_TypedList, _setUint32, TypedList_SetUint32, 0xb9581b93) \ + V(_TypedList, _setInt64, TypedList_SetInt64, 0xc8bec75b) \ + V(_TypedList, _setUint64, TypedList_SetUint64, 0xda38a9e6) \ + V(_TypedList, _setFloat32, TypedList_SetFloat32, 0x2f27a5c1) \ + V(_TypedList, _setFloat64, TypedList_SetFloat64, 0x234b70b3) \ + V(_TypedList, _setFloat32x4, TypedList_SetFloat32x4, 0x38b7a13b) \ + V(_TypedList, _setFloat64x2, TypedList_SetFloat64x2, 0xbadc4f20) \ + V(_TypedList, _setInt32x4, TypedList_SetInt32x4, 0x5cda7a3c) \ V(ByteData, ., ByteDataFactory, 0x45e89423) \ V(_ByteDataView, get:offsetInBytes, ByteDataViewOffsetInBytes, 0x60c0664c) \ V(_ByteDataView, get:_typedData, ByteDataViewTypedData, 0xb9c2d41a) \ @@ -505,30 +505,30 @@ namespace dart { // A list of core functions that internally dispatch based on received id. #define POLYMORPHIC_TARGET_LIST(V) \ V(_StringBase, [], StringBaseCharAt, 0xd0613adf) \ - V(_TypedList, _getInt8, ByteArrayBaseGetInt8, 0x16155054) \ - V(_TypedList, _getUint8, ByteArrayBaseGetUint8, 0x1771724a) \ - V(_TypedList, _getInt16, ByteArrayBaseGetInt16, 0x2e320a6f) \ - V(_TypedList, _getUint16, ByteArrayBaseGetUint16, 0x2fb36ad9) \ - V(_TypedList, _getInt32, ByteArrayBaseGetInt32, 0x1909a12a) \ - V(_TypedList, _getUint32, ByteArrayBaseGetUint32, 0x194ee29b) \ - V(_TypedList, _getInt64, ByteArrayBaseGetInt64, 0xf652341f) \ - V(_TypedList, _getUint64, ByteArrayBaseGetUint64, 0x2c4ced79) \ - V(_TypedList, _getFloat32, ByteArrayBaseGetFloat32, 0xe8e81527) \ - V(_TypedList, _getFloat64, ByteArrayBaseGetFloat64, 0xf81baa54) \ - V(_TypedList, _getFloat32x4, ByteArrayBaseGetFloat32x4, 0xaf1e8105) \ - V(_TypedList, _getInt32x4, ByteArrayBaseGetInt32x4, 0x5564e82b) \ - V(_TypedList, _setInt8, ByteArrayBaseSetInt8, 0xe17ab7c2) \ - V(_TypedList, _setUint8, ByteArrayBaseSetInt8, 0xaf4b2b68) \ - V(_TypedList, _setInt16, ByteArrayBaseSetInt16, 0xbad7b447) \ - V(_TypedList, _setUint16, ByteArrayBaseSetInt16, 0xce13bc6f) \ - V(_TypedList, _setInt32, ByteArrayBaseSetInt32, 0xbdcc1f60) \ - V(_TypedList, _setUint32, ByteArrayBaseSetUint32, 0xb95817d2) \ - V(_TypedList, _setInt64, ByteArrayBaseSetInt64, 0xc8bec39a) \ - V(_TypedList, _setUint64, ByteArrayBaseSetUint64, 0xda38a625) \ - V(_TypedList, _setFloat32, ByteArrayBaseSetFloat32, 0x2f27a200) \ - V(_TypedList, _setFloat64, ByteArrayBaseSetFloat64, 0x234b6cf2) \ - V(_TypedList, _setFloat32x4, ByteArrayBaseSetFloat32x4, 0x38b79d7a) \ - V(_TypedList, _setInt32x4, ByteArrayBaseSetInt32x4, 0x5cda767b) \ + V(_TypedList, _getInt8, TypedList_GetInt8, 0x16155415) \ + V(_TypedList, _getUint8, TypedList_GetUint8, 0x1771760b) \ + V(_TypedList, _getInt16, TypedList_GetInt16, 0x2e320e30) \ + V(_TypedList, _getUint16, TypedList_GetUint16, 0x2fb36e9a) \ + V(_TypedList, _getInt32, TypedList_GetInt32, 0x1909a4eb) \ + V(_TypedList, _getUint32, TypedList_GetUint32, 0x194ee65c) \ + V(_TypedList, _getInt64, TypedList_GetInt64, 0xf65237e0) \ + V(_TypedList, _getUint64, TypedList_GetUint64, 0x2c4cf13a) \ + V(_TypedList, _getFloat32, TypedList_GetFloat32, 0xe8e818e8) \ + V(_TypedList, _getFloat64, TypedList_GetFloat64, 0xf81bae15) \ + V(_TypedList, _getFloat32x4, TypedList_GetFloat32x4, 0xaf1e84c6) \ + V(_TypedList, _getInt32x4, TypedList_GetInt32x4, 0x5564ebec) \ + V(_TypedList, _setInt8, TypedList_SetInt8, 0xe17abb83) \ + V(_TypedList, _setUint8, TypedList_SetInt8, 0xaf4b2f29) \ + V(_TypedList, _setInt16, TypedList_SetInt16, 0xbad7b808) \ + V(_TypedList, _setUint16, TypedList_SetInt16, 0xce13c030) \ + V(_TypedList, _setInt32, TypedList_SetInt32, 0xbdcc2321) \ + V(_TypedList, _setUint32, TypedList_SetUint32, 0xb9581b93) \ + V(_TypedList, _setInt64, TypedList_SetInt64, 0xc8bec75b) \ + V(_TypedList, _setUint64, TypedList_SetUint64, 0xda38a9e6) \ + V(_TypedList, _setFloat32, TypedList_SetFloat32, 0x2f27a5c1) \ + V(_TypedList, _setFloat64, TypedList_SetFloat64, 0x234b70b3) \ + V(_TypedList, _setFloat32x4, TypedList_SetFloat32x4, 0x38b7a13b) \ + V(_TypedList, _setInt32x4, TypedList_SetInt32x4, 0x5cda7a3c) \ V(Object, get:runtimeType, ObjectRuntimeType, 0x03733c71) // List of recognized list factories: diff --git a/runtime/vm/object.cc b/runtime/vm/object.cc index eddb10d8241..0a1e9584b2d 100644 --- a/runtime/vm/object.cc +++ b/runtime/vm/object.cc @@ -9160,6 +9160,13 @@ bool Function::ForceOptimize() const { return InVmTests(*this); } +bool Function::IsPreferInline() const { + if (!has_pragma()) return false; + + return Library::FindPragma(Thread::Current(), /*only_core=*/false, *this, + Symbols::vm_prefer_inline()); +} + bool Function::IsIdempotent() const { if (!has_pragma()) return false; @@ -9255,6 +9262,32 @@ bool Function::RecognizedKindForceOptimize() const { case MethodRecognizer::kRecord_numFields: case MethodRecognizer::kUtf8DecoderScan: case MethodRecognizer::kDouble_hashCode: + case MethodRecognizer::kTypedList_GetInt8: + case MethodRecognizer::kTypedList_SetInt8: + case MethodRecognizer::kTypedList_GetUint8: + case MethodRecognizer::kTypedList_SetUint8: + case MethodRecognizer::kTypedList_GetInt16: + case MethodRecognizer::kTypedList_SetInt16: + case MethodRecognizer::kTypedList_GetUint16: + case MethodRecognizer::kTypedList_SetUint16: + case MethodRecognizer::kTypedList_GetInt32: + case MethodRecognizer::kTypedList_SetInt32: + case MethodRecognizer::kTypedList_GetUint32: + case MethodRecognizer::kTypedList_SetUint32: + case MethodRecognizer::kTypedList_GetInt64: + case MethodRecognizer::kTypedList_SetInt64: + case MethodRecognizer::kTypedList_GetUint64: + case MethodRecognizer::kTypedList_SetUint64: + case MethodRecognizer::kTypedList_GetFloat32: + case MethodRecognizer::kTypedList_SetFloat32: + case MethodRecognizer::kTypedList_GetFloat64: + case MethodRecognizer::kTypedList_SetFloat64: + case MethodRecognizer::kTypedList_GetInt32x4: + case MethodRecognizer::kTypedList_SetInt32x4: + case MethodRecognizer::kTypedList_GetFloat32x4: + case MethodRecognizer::kTypedList_SetFloat32x4: + case MethodRecognizer::kTypedList_GetFloat64x2: + case MethodRecognizer::kTypedList_SetFloat64x2: case MethodRecognizer::kTypedData_memMove1: case MethodRecognizer::kTypedData_memMove2: case MethodRecognizer::kTypedData_memMove4: @@ -9293,6 +9326,7 @@ bool Function::CanBeInlined() const { // idempotent becase if deoptimization is needed in inlined body, the // execution of the force-optimized will be restarted at the beginning of // the function. + ASSERT(!IsPreferInline() || IsIdempotent()); return IsIdempotent(); } diff --git a/runtime/vm/object.h b/runtime/vm/object.h index 9d898ef44c5..2818bdd6a67 100644 --- a/runtime/vm/object.h +++ b/runtime/vm/object.h @@ -3566,6 +3566,9 @@ class Function : public Object { // run. bool ForceOptimize() const; + // Whether this function should be inlined if at all possible. + bool IsPreferInline() const; + // Whether this function is idempotent (i.e. calling it twice has the same // effect as calling it once - no visible side effects). // diff --git a/runtime/vm/object_store.h b/runtime/vm/object_store.h index e9d00634663..cf30fbc1774 100644 --- a/runtime/vm/object_store.h +++ b/runtime/vm/object_store.h @@ -527,6 +527,7 @@ class ObjectStore { #undef DECLARE_LAZY_INIT_ASYNC_GETTER #undef DECLARE_LAZY_INIT_ISOLATE_GETTER #undef DECLARE_LAZY_INIT_INTERNAL_GETTER +#undef DECLARE_LAZY_INIT_TYPED_DATA_GETTER LibraryPtr bootstrap_library(BootstrapLibraryId index) { switch (index) { diff --git a/runtime/vm/symbols.h b/runtime/vm/symbols.h index c937627c092..79ffa6563b5 100644 --- a/runtime/vm/symbols.h +++ b/runtime/vm/symbols.h @@ -386,6 +386,7 @@ class ObjectPointerVisitor; V(_Type, "_Type") \ V(_TypeParameter, "_TypeParameter") \ V(_TypeVariableMirror, "_TypeVariableMirror") \ + V(_TypedList, "_TypedList") \ V(_TypedListBase, "_TypedListBase") \ V(_Uint16ArrayFactory, "Uint16List.") \ V(_Uint16ArrayView, "_Uint16ArrayView") \ @@ -451,6 +452,16 @@ class ObjectPointerVisitor; V(_mapGet, "_mapGet") \ V(_mapKeys, "_mapKeys") \ V(_name, "_name") \ + V(_nativeGetFloat32, "_nativeGetFloat32") \ + V(_nativeSetFloat32, "_nativeSetFloat32") \ + V(_nativeGetFloat64, "_nativeGetFloat64") \ + V(_nativeSetFloat64, "_nativeSetFloat64") \ + V(_nativeGetFloat32x4, "_nativeGetFloat32x4") \ + V(_nativeSetFloat32x4, "_nativeSetFloat32x4") \ + V(_nativeGetInt32x4, "_nativeGetInt32x4") \ + V(_nativeSetInt32x4, "_nativeSetInt32x4") \ + V(_nativeGetFloat64x2, "_nativeGetFloat64x2") \ + V(_nativeSetFloat64x2, "_nativeSetFloat64x2") \ V(_nativeSetRange, "_nativeSetRange") \ V(_objectEquals, "_objectEquals") \ V(_objectHashCode, "_objectHashCode") \ diff --git a/sdk/lib/_internal/vm/lib/typed_data_patch.dart b/sdk/lib/_internal/vm/lib/typed_data_patch.dart index ef8159e78d9..50a12571331 100644 --- a/sdk/lib/_internal/vm/lib/typed_data_patch.dart +++ b/sdk/lib/_internal/vm/lib/typed_data_patch.dart @@ -2055,104 +2055,173 @@ abstract final class _TypedList extends _TypedListBase { @pragma("vm:recognized", "other") @pragma("vm:exact-result-type", "dart:core#_Smi") - @pragma("vm:external-name", "TypedData_GetInt8") + @pragma("vm:prefer-inline") + @pragma("vm:idempotent") external int _getInt8(int offsetInBytes); @pragma("vm:recognized", "other") - @pragma("vm:external-name", "TypedData_SetInt8") + @pragma("vm:prefer-inline") + @pragma("vm:idempotent") external void _setInt8(int offsetInBytes, int value); @pragma("vm:recognized", "other") @pragma("vm:exact-result-type", "dart:core#_Smi") - @pragma("vm:external-name", "TypedData_GetUint8") + @pragma("vm:prefer-inline") + @pragma("vm:idempotent") external int _getUint8(int offsetInBytes); @pragma("vm:recognized", "other") - @pragma("vm:external-name", "TypedData_SetUint8") + @pragma("vm:prefer-inline") + @pragma("vm:idempotent") external void _setUint8(int offsetInBytes, int value); @pragma("vm:recognized", "other") @pragma("vm:exact-result-type", "dart:core#_Smi") - @pragma("vm:external-name", "TypedData_GetInt16") + @pragma("vm:prefer-inline") + @pragma("vm:idempotent") external int _getInt16(int offsetInBytes); @pragma("vm:recognized", "other") - @pragma("vm:external-name", "TypedData_SetInt16") + @pragma("vm:prefer-inline") + @pragma("vm:idempotent") external void _setInt16(int offsetInBytes, int value); @pragma("vm:recognized", "other") @pragma("vm:exact-result-type", "dart:core#_Smi") - @pragma("vm:external-name", "TypedData_GetUint16") + @pragma("vm:prefer-inline") + @pragma("vm:idempotent") external int _getUint16(int offsetInBytes); @pragma("vm:recognized", "other") - @pragma("vm:external-name", "TypedData_SetUint16") + @pragma("vm:prefer-inline") + @pragma("vm:idempotent") external void _setUint16(int offsetInBytes, int value); @pragma("vm:recognized", "other") - @pragma("vm:external-name", "TypedData_GetInt32") + @pragma("vm:prefer-inline") + @pragma("vm:idempotent") external int _getInt32(int offsetInBytes); @pragma("vm:recognized", "other") - @pragma("vm:external-name", "TypedData_SetInt32") + @pragma("vm:prefer-inline") + @pragma("vm:idempotent") external void _setInt32(int offsetInBytes, int value); @pragma("vm:recognized", "other") - @pragma("vm:external-name", "TypedData_GetUint32") + @pragma("vm:prefer-inline") + @pragma("vm:idempotent") external int _getUint32(int offsetInBytes); @pragma("vm:recognized", "other") - @pragma("vm:external-name", "TypedData_SetUint32") + @pragma("vm:prefer-inline") + @pragma("vm:idempotent") external void _setUint32(int offsetInBytes, int value); @pragma("vm:recognized", "other") - @pragma("vm:external-name", "TypedData_GetInt64") + @pragma("vm:prefer-inline") + @pragma("vm:idempotent") external int _getInt64(int offsetInBytes); @pragma("vm:recognized", "other") - @pragma("vm:external-name", "TypedData_SetInt64") + @pragma("vm:prefer-inline") + @pragma("vm:idempotent") external void _setInt64(int offsetInBytes, int value); @pragma("vm:recognized", "other") - @pragma("vm:external-name", "TypedData_GetUint64") + @pragma("vm:prefer-inline") + @pragma("vm:idempotent") external int _getUint64(int offsetInBytes); @pragma("vm:recognized", "other") - @pragma("vm:external-name", "TypedData_SetUint64") + @pragma("vm:prefer-inline") + @pragma("vm:idempotent") external void _setUint64(int offsetInBytes, int value); @pragma("vm:recognized", "other") @pragma("vm:exact-result-type", "dart:core#_Double") - @pragma("vm:external-name", "TypedData_GetFloat32") + @pragma("vm:prefer-inline") + @pragma("vm:idempotent") external double _getFloat32(int offsetInBytes); @pragma("vm:recognized", "other") - @pragma("vm:external-name", "TypedData_SetFloat32") + @pragma("vm:prefer-inline") + @pragma("vm:idempotent") external void _setFloat32(int offsetInBytes, double value); @pragma("vm:recognized", "other") @pragma("vm:exact-result-type", "dart:core#_Double") - @pragma("vm:external-name", "TypedData_GetFloat64") + @pragma("vm:prefer-inline") + @pragma("vm:idempotent") external double _getFloat64(int offsetInBytes); @pragma("vm:recognized", "other") - @pragma("vm:external-name", "TypedData_SetFloat64") + @pragma("vm:prefer-inline") + @pragma("vm:idempotent") external void _setFloat64(int offsetInBytes, double value); @pragma("vm:recognized", "other") @pragma("vm:exact-result-type", _Float32x4) - @pragma("vm:external-name", "TypedData_GetFloat32x4") + @pragma("vm:prefer-inline") + @pragma("vm:idempotent") external Float32x4 _getFloat32x4(int offsetInBytes); @pragma("vm:recognized", "other") - @pragma("vm:external-name", "TypedData_SetFloat32x4") + @pragma("vm:prefer-inline") + @pragma("vm:idempotent") external void _setFloat32x4(int offsetInBytes, Float32x4 value); @pragma("vm:recognized", "other") @pragma("vm:exact-result-type", _Int32x4) - @pragma("vm:external-name", "TypedData_GetInt32x4") + @pragma("vm:prefer-inline") + @pragma("vm:idempotent") external Int32x4 _getInt32x4(int offsetInBytes); @pragma("vm:recognized", "other") - @pragma("vm:external-name", "TypedData_SetInt32x4") + @pragma("vm:prefer-inline") + @pragma("vm:idempotent") external void _setInt32x4(int offsetInBytes, Int32x4 value); @pragma("vm:recognized", "other") @pragma("vm:exact-result-type", _Float64x2) - @pragma("vm:external-name", "TypedData_GetFloat64x2") + @pragma("vm:prefer-inline") + @pragma("vm:idempotent") external Float64x2 _getFloat64x2(int offsetInBytes); @pragma("vm:recognized", "other") - @pragma("vm:external-name", "TypedData_SetFloat64x2") + @pragma("vm:prefer-inline") + @pragma("vm:idempotent") external void _setFloat64x2(int offsetInBytes, Float64x2 value); + // The _nativeGetX and _nativeSetX methods are only used as a fallback when + // unboxed double or SIMD values are unsupported by the flow graph compiler. + + @pragma("vm:entry-point", "call") + @pragma("vm:exact-result-type", "dart:core#_Double") + @pragma("vm:external-name", "TypedData_GetFloat32") + external double _nativeGetFloat32(int offsetInBytes); + @pragma("vm:entry-point", "call") + @pragma("vm:external-name", "TypedData_SetFloat32") + external void _nativeSetFloat32(int offsetInBytes, double value); + + @pragma("vm:entry-point", "call") + @pragma("vm:exact-result-type", "dart:core#_Double") + @pragma("vm:external-name", "TypedData_GetFloat64") + external double _nativeGetFloat64(int offsetInBytes); + @pragma("vm:entry-point", "call") + @pragma("vm:external-name", "TypedData_SetFloat64") + external void _nativeSetFloat64(int offsetInBytes, double value); + + @pragma("vm:entry-point", "call") + @pragma("vm:exact-result-type", _Float32x4) + @pragma("vm:external-name", "TypedData_GetFloat32x4") + external Float32x4 _nativeGetFloat32x4(int offsetInBytes); + @pragma("vm:entry-point", "call") + @pragma("vm:external-name", "TypedData_SetFloat32x4") + external void _nativeSetFloat32x4(int offsetInBytes, Float32x4 value); + + @pragma("vm:entry-point", "call") + @pragma("vm:exact-result-type", _Int32x4) + @pragma("vm:external-name", "TypedData_GetInt32x4") + external Int32x4 _nativeGetInt32x4(int offsetInBytes); + @pragma("vm:entry-point", "call") + @pragma("vm:external-name", "TypedData_SetInt32x4") + external void _nativeSetInt32x4(int offsetInBytes, Int32x4 value); + + @pragma("vm:entry-point", "call") + @pragma("vm:exact-result-type", _Float64x2) + @pragma("vm:external-name", "TypedData_GetFloat64x2") + external Float64x2 _nativeGetFloat64x2(int offsetInBytes); + @pragma("vm:entry-point", "call") + @pragma("vm:external-name", "TypedData_SetFloat64x2") + external void _nativeSetFloat64x2(int offsetInBytes, Float64x2 value); + /** * Stores the [CodeUnits] as UTF-16 units into this TypedData at * positions [start]..[end] (uint16 indices).