mirror of
https://github.com/dart-lang/sdk
synced 2024-10-05 01:24:54 +00:00
b9ad67691c
Even though all canonical objects can be shared, they might cause re-hashing on the receiver side due to user-defined get:hashCode implementation. This is why we do not consider the canonical bit (of a key in a map) when determining whether rehashing is needed. Though some well-known classes will have a properly behaved get:hashCode, the most important being for kNullCid. => So we'll allow kNullCid, kBoolCid. This makes receiving json data much faster due to not needing to re-hash anymore (was done due to `null` values in the map backing store). TEST=Existing test suite. Change-Id: Ie6d39da7fe27ce8925644cf2c17a3944a9e936b2 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/207080 Reviewed-by: Alexander Aprelev <aam@google.com> Commit-Queue: Martin Kustermann <kustermann@google.com>
1680 lines
64 KiB
C++
1680 lines
64 KiB
C++
// Copyright (c) 2021, the Dart project authors. Please see the AUTHORS file
|
|
// for details. All rights reserved. Use of this source code is governed by a
|
|
// BSD-style license that can be found in the LICENSE file.
|
|
|
|
#include "vm/object_graph_copy.h"
|
|
#include "vm/dart_api_state.h"
|
|
#include "vm/flags.h"
|
|
#include "vm/heap/weak_table.h"
|
|
#include "vm/longjump.h"
|
|
#include "vm/object.h"
|
|
#include "vm/snapshot.h"
|
|
#include "vm/symbols.h"
|
|
|
|
#define Z zone_
|
|
|
|
// The list here contains two kinds of classes of objects
|
|
// * objects that will be shared and we will therefore never need to copy
|
|
// * objects that user object graphs should never reference
|
|
#define FOR_UNSUPPORTED_CLASSES(V) \
|
|
V(AbstractType) \
|
|
V(ApiError) \
|
|
V(Bool) \
|
|
V(CallSiteData) \
|
|
V(Capability) \
|
|
V(Class) \
|
|
V(ClosureData) \
|
|
V(Code) \
|
|
V(CodeSourceMap) \
|
|
V(CompressedStackMaps) \
|
|
V(Context) \
|
|
V(ContextScope) \
|
|
V(DynamicLibrary) \
|
|
V(Error) \
|
|
V(ExceptionHandlers) \
|
|
V(FfiTrampolineData) \
|
|
V(Field) \
|
|
V(Function) \
|
|
V(FunctionType) \
|
|
V(FutureOr) \
|
|
V(ICData) \
|
|
V(Instance) \
|
|
V(Instructions) \
|
|
V(InstructionsSection) \
|
|
V(InstructionsTable) \
|
|
V(Int32x4) \
|
|
V(Integer) \
|
|
V(KernelProgramInfo) \
|
|
V(LanguageError) \
|
|
V(Library) \
|
|
V(LibraryPrefix) \
|
|
V(LoadingUnit) \
|
|
V(LocalVarDescriptors) \
|
|
V(MegamorphicCache) \
|
|
V(Mint) \
|
|
V(MirrorReference) \
|
|
V(MonomorphicSmiableCall) \
|
|
V(Namespace) \
|
|
V(Number) \
|
|
V(ObjectPool) \
|
|
V(PatchClass) \
|
|
V(PcDescriptors) \
|
|
V(Pointer) \
|
|
V(ReceivePort) \
|
|
V(RegExp) \
|
|
V(Script) \
|
|
V(Sentinel) \
|
|
V(SendPort) \
|
|
V(SingleTargetCache) \
|
|
V(Smi) \
|
|
V(StackTrace) \
|
|
V(SubtypeTestCache) \
|
|
V(Type) \
|
|
V(TypeArguments) \
|
|
V(TypeParameter) \
|
|
V(TypeParameters) \
|
|
V(TypeRef) \
|
|
V(TypedDataBase) \
|
|
V(UnhandledException) \
|
|
V(UnlinkedCall) \
|
|
V(UnwindError) \
|
|
V(UserTag) \
|
|
V(WeakProperty) \
|
|
V(WeakSerializationReference)
|
|
|
|
namespace dart {
|
|
|
|
DEFINE_FLAG(bool,
|
|
enable_fast_object_copy,
|
|
true,
|
|
"Enable fast path for fast object copy.");
|
|
DEFINE_FLAG(bool,
|
|
gc_on_foc_slow_path,
|
|
false,
|
|
"Cause a GC when falling off the fast path for fast object copy.");
|
|
|
|
const char* kFastAllocationFailed = "fast allocation failed";
|
|
|
|
struct PtrTypes {
|
|
using Object = ObjectPtr;
|
|
static const dart::UntaggedObject* UntagObject(Object arg) {
|
|
return arg.untag();
|
|
}
|
|
static const dart::ObjectPtr GetObjectPtr(Object arg) { return arg; }
|
|
static const dart::Object& HandlifyObject(ObjectPtr arg) {
|
|
return dart::Object::Handle(arg);
|
|
}
|
|
|
|
#define DO(V) \
|
|
using V = V##Ptr; \
|
|
static Untagged##V* Untag##V(V##Ptr arg) { return arg.untag(); } \
|
|
static V##Ptr Get##V##Ptr(V##Ptr arg) { return arg; } \
|
|
static V##Ptr Cast##V(ObjectPtr arg) { return dart::V::RawCast(arg); }
|
|
CLASS_LIST_FOR_HANDLES(DO)
|
|
#undef DO
|
|
};
|
|
|
|
struct HandleTypes {
|
|
using Object = const Object&;
|
|
static const dart::UntaggedObject* UntagObject(Object arg) {
|
|
return arg.ptr().untag();
|
|
}
|
|
static dart::ObjectPtr GetObjectPtr(Object arg) { return arg.ptr(); }
|
|
static Object HandlifyObject(Object arg) { return arg; }
|
|
|
|
#define DO(V) \
|
|
using V = const V&; \
|
|
static Untagged##V* Untag##V(V arg) { return arg.ptr().untag(); } \
|
|
static V##Ptr Get##V##Ptr(V arg) { return arg.ptr(); } \
|
|
static V Cast##V(const dart::Object& arg) { return dart::V::Cast(arg); }
|
|
CLASS_LIST_FOR_HANDLES(DO)
|
|
#undef DO
|
|
};
|
|
|
|
DART_FORCE_INLINE
|
|
static ObjectPtr Marker() {
|
|
return Object::unknown_constant().ptr();
|
|
}
|
|
|
|
DART_FORCE_INLINE
|
|
static bool CanShareObject(uword tags) {
|
|
if ((tags & UntaggedObject::CanonicalBit::mask_in_place()) != 0) {
|
|
return true;
|
|
}
|
|
const auto cid = UntaggedObject::ClassIdTag::decode(tags);
|
|
if (cid == kOneByteStringCid) return true;
|
|
if (cid == kTwoByteStringCid) return true;
|
|
if (cid == kExternalOneByteStringCid) return true;
|
|
if (cid == kExternalTwoByteStringCid) return true;
|
|
if (cid == kMintCid) return true;
|
|
if (cid == kImmutableArrayCid) return true;
|
|
if (cid == kNeverCid) return true;
|
|
if (cid == kSentinelCid) return true;
|
|
#if defined(DART_PRECOMPILED_RUNTIME)
|
|
// In JIT mode we have field guards enabled which means
|
|
// double/float32x4/float64x2 boxes can be mutable and we therefore cannot
|
|
// share them.
|
|
if (cid == kDoubleCid || cid == kFloat32x4Cid || cid == kFloat64x2Cid) {
|
|
return true;
|
|
}
|
|
#endif
|
|
if (cid == kInt32x4Cid) return true; // No field guards here.
|
|
if (cid == kSendPortCid) return true;
|
|
if (cid == kCapabilityCid) return true;
|
|
if (cid == kRegExpCid) return true;
|
|
|
|
return false;
|
|
}
|
|
|
|
// Whether executing `get:hashCode` (possibly in a different isolate) on an
|
|
// object with the given [tags] might return a different answer than the source
|
|
// object (if copying is needed) or on the same object (if the object is
|
|
// shared).
|
|
DART_FORCE_INLINE
|
|
static bool MightNeedReHashing(ObjectPtr object) {
|
|
const uword tags = TagsFromUntaggedObject(object.untag());
|
|
const auto cid = UntaggedObject::ClassIdTag::decode(tags);
|
|
// These use structural hash codes and will therefore always result in the
|
|
// same hash codes.
|
|
if (cid == kOneByteStringCid) return false;
|
|
if (cid == kTwoByteStringCid) return false;
|
|
if (cid == kExternalOneByteStringCid) return false;
|
|
if (cid == kExternalTwoByteStringCid) return false;
|
|
if (cid == kMintCid) return false;
|
|
if (cid == kDoubleCid) return false;
|
|
if (cid == kBoolCid) return false;
|
|
if (cid == kSendPortCid) return false;
|
|
if (cid == kCapabilityCid) return false;
|
|
if (cid == kNullCid) return false;
|
|
|
|
// These are shared and use identity hash codes. If they are used as a key in
|
|
// a map or a value in a set, they will already have the identity hash code
|
|
// set.
|
|
if (cid == kImmutableArrayCid) return false;
|
|
if (cid == kRegExpCid) return false;
|
|
if (cid == kInt32x4Cid) return false;
|
|
|
|
// We copy those (instead of sharing them) - see [CanShareObjct]. They rely
|
|
// on the default hashCode implementation which uses identity hash codes
|
|
// (instead of structural hash code).
|
|
if (cid == kFloat32x4Cid || cid == kFloat64x2Cid) {
|
|
return !kDartPrecompiledRuntime;
|
|
}
|
|
|
|
// If the [tags] indicates this is a canonical object we'll share it instead
|
|
// of copying it. That would suggest we don't have to re-hash maps/sets
|
|
// containing this object on the receiver side.
|
|
//
|
|
// Though the object can be a constant of a user-defined class with a
|
|
// custom hash code that is misbehaving (e.g one that depends on global field
|
|
// state, ...). To be on the safe side we'll force re-hashing if such objects
|
|
// are encountered in maps/sets.
|
|
//
|
|
// => We might want to consider changing the implementation to avoid rehashing
|
|
// in such cases in the future and disambiguate the documentation.
|
|
return true;
|
|
}
|
|
|
|
DART_FORCE_INLINE
|
|
uword TagsFromUntaggedObject(UntaggedObject* obj) {
|
|
return obj->tags_;
|
|
}
|
|
|
|
DART_FORCE_INLINE
|
|
void SetNewSpaceTaggingWord(ObjectPtr to, classid_t cid, uint32_t size) {
|
|
uword tags = 0;
|
|
|
|
tags = UntaggedObject::SizeTag::update(size, tags);
|
|
tags = UntaggedObject::ClassIdTag::update(cid, tags);
|
|
tags = UntaggedObject::OldBit::update(false, tags);
|
|
tags = UntaggedObject::OldAndNotMarkedBit::update(false, tags);
|
|
tags = UntaggedObject::OldAndNotRememberedBit::update(false, tags);
|
|
tags = UntaggedObject::CanonicalBit::update(false, tags);
|
|
tags = UntaggedObject::NewBit::update(true, tags);
|
|
#if defined(HASH_IN_OBJECT_HEADER)
|
|
tags = UntaggedObject::HashTag::update(0, tags);
|
|
#endif
|
|
to.untag()->tags_ = tags;
|
|
}
|
|
|
|
DART_FORCE_INLINE
|
|
ObjectPtr AllocateObject(intptr_t cid, intptr_t size) {
|
|
#if defined(DART_COMPRESSED_POINTERS)
|
|
const bool compressed = true;
|
|
#else
|
|
const bool compressed = false;
|
|
#endif
|
|
return Object::Allocate(cid, size, Heap::kNew, compressed);
|
|
}
|
|
|
|
DART_FORCE_INLINE
|
|
void UpdateLengthField(intptr_t cid, ObjectPtr from, ObjectPtr to) {
|
|
// We share these objects - never copy them.
|
|
ASSERT(!IsStringClassId(cid));
|
|
ASSERT(cid != kImmutableArrayCid);
|
|
|
|
// We update any in-heap variable sized object with the length to keep the
|
|
// length and the size in the object header in-sync for the GC.
|
|
if (cid == kArrayCid) {
|
|
static_cast<UntaggedArray*>(to.untag())->length_ =
|
|
static_cast<UntaggedArray*>(from.untag())->length_;
|
|
} else if (IsTypedDataClassId(cid)) {
|
|
static_cast<UntaggedTypedDataBase*>(to.untag())->length_ =
|
|
static_cast<UntaggedTypedDataBase*>(from.untag())->length_;
|
|
}
|
|
}
|
|
|
|
void InitializeExternalTypedData(intptr_t cid,
|
|
ExternalTypedDataPtr from,
|
|
ExternalTypedDataPtr to) {
|
|
auto raw_from = from.untag();
|
|
auto raw_to = to.untag();
|
|
const intptr_t length =
|
|
TypedData::ElementSizeInBytes(cid) * Smi::Value(raw_from->length_);
|
|
|
|
auto buffer = static_cast<uint8_t*>(malloc(length));
|
|
memmove(buffer, raw_from->data_, length);
|
|
raw_to->length_ = raw_from->length_;
|
|
raw_to->data_ = buffer;
|
|
}
|
|
|
|
void InitializeTypedDataView(TypedDataViewPtr obj) {
|
|
obj.untag()->typed_data_ = TypedDataBase::null();
|
|
obj.untag()->offset_in_bytes_ = 0;
|
|
obj.untag()->length_ = 0;
|
|
}
|
|
|
|
void FreeExternalTypedData(void* isolate_callback_data, void* buffer) {
|
|
free(buffer);
|
|
}
|
|
|
|
void FreeTransferablePeer(void* isolate_callback_data, void* peer) {
|
|
delete static_cast<TransferableTypedDataPeer*>(peer);
|
|
}
|
|
|
|
class ForwardMapBase {
|
|
public:
|
|
explicit ForwardMapBase(Thread* thread)
|
|
: thread_(thread), zone_(thread->zone()), isolate_(thread->isolate()) {}
|
|
|
|
protected:
|
|
friend class ObjectGraphCopier;
|
|
|
|
intptr_t GetObjectId(ObjectPtr object) {
|
|
if (object->IsNewObject()) {
|
|
return isolate_->forward_table_new()->GetValueExclusive(object);
|
|
} else {
|
|
return isolate_->forward_table_old()->GetValueExclusive(object);
|
|
}
|
|
}
|
|
void SetObjectId(ObjectPtr object, intptr_t id) {
|
|
if (object->IsNewObject()) {
|
|
isolate_->forward_table_new()->SetValueExclusive(object, id);
|
|
} else {
|
|
isolate_->forward_table_old()->SetValueExclusive(object, id);
|
|
}
|
|
}
|
|
|
|
void FinalizeTransferable(const TransferableTypedData& from,
|
|
const TransferableTypedData& to) {
|
|
// Get the old peer.
|
|
auto fpeer = static_cast<TransferableTypedDataPeer*>(
|
|
thread_->heap()->GetPeer(from.ptr()));
|
|
ASSERT(fpeer != nullptr && fpeer->data() != nullptr);
|
|
const intptr_t length = fpeer->length();
|
|
|
|
// Allocate new peer object with (data, length).
|
|
auto tpeer = new TransferableTypedDataPeer(fpeer->data(), length);
|
|
thread_->heap()->SetPeer(to.ptr(), tpeer);
|
|
|
|
// Move the handle itself to the new object.
|
|
fpeer->handle()->EnsureFreedExternal(thread_->isolate_group());
|
|
tpeer->set_handle(FinalizablePersistentHandle::New(
|
|
thread_->isolate_group(), to, tpeer, FreeTransferablePeer, length,
|
|
/*auto_delete=*/true));
|
|
fpeer->ClearData();
|
|
}
|
|
|
|
void FinalizeExternalTypedData(const ExternalTypedData& to) {
|
|
to.AddFinalizer(to.DataAddr(0), &FreeExternalTypedData, to.LengthInBytes());
|
|
}
|
|
|
|
Thread* thread_;
|
|
Zone* zone_;
|
|
Isolate* isolate_;
|
|
|
|
private:
|
|
DISALLOW_COPY_AND_ASSIGN(ForwardMapBase);
|
|
};
|
|
|
|
class FastForwardMap : public ForwardMapBase {
|
|
public:
|
|
explicit FastForwardMap(Thread* thread)
|
|
: ForwardMapBase(thread),
|
|
raw_from_to_(thread->zone(), 20),
|
|
raw_transferables_from_to_(thread->zone(), 0),
|
|
raw_objects_to_rehash_(thread->zone(), 0) {
|
|
raw_from_to_.Resize(2);
|
|
raw_from_to_[0] = Object::null();
|
|
raw_from_to_[1] = Object::null();
|
|
fill_cursor_ = 2;
|
|
}
|
|
|
|
ObjectPtr ForwardedObject(ObjectPtr object) {
|
|
const intptr_t id = GetObjectId(object);
|
|
if (id == 0) return Marker();
|
|
return raw_from_to_[id + 1];
|
|
}
|
|
|
|
void Insert(ObjectPtr from, ObjectPtr to) {
|
|
ASSERT(ForwardedObject(from) == Marker());
|
|
ASSERT(raw_from_to_.length() == raw_from_to_.length());
|
|
const auto id = raw_from_to_.length();
|
|
SetObjectId(from, id);
|
|
raw_from_to_.Resize(id + 2);
|
|
raw_from_to_[id] = from;
|
|
raw_from_to_[id + 1] = to;
|
|
}
|
|
|
|
void AddTransferable(TransferableTypedDataPtr from,
|
|
TransferableTypedDataPtr to) {
|
|
raw_transferables_from_to_.Add(from);
|
|
raw_transferables_from_to_.Add(to);
|
|
}
|
|
void AddExternalTypedData(ExternalTypedDataPtr to) {
|
|
raw_external_typed_data_to_.Add(to);
|
|
}
|
|
|
|
void AddObjectToRehash(ObjectPtr to) { raw_objects_to_rehash_.Add(to); }
|
|
|
|
private:
|
|
friend class FastObjectCopy;
|
|
friend class ObjectGraphCopier;
|
|
|
|
GrowableArray<ObjectPtr> raw_from_to_;
|
|
GrowableArray<TransferableTypedDataPtr> raw_transferables_from_to_;
|
|
GrowableArray<ExternalTypedDataPtr> raw_external_typed_data_to_;
|
|
GrowableArray<ObjectPtr> raw_objects_to_rehash_;
|
|
intptr_t fill_cursor_ = 0;
|
|
|
|
DISALLOW_COPY_AND_ASSIGN(FastForwardMap);
|
|
};
|
|
|
|
class SlowForwardMap : public ForwardMapBase {
|
|
public:
|
|
explicit SlowForwardMap(Thread* thread)
|
|
: ForwardMapBase(thread),
|
|
from_to_(thread->zone(), 20),
|
|
transferables_from_to_(thread->zone(), 0) {
|
|
from_to_.Resize(2);
|
|
from_to_[0] = &Object::null_object();
|
|
from_to_[1] = &Object::null_object();
|
|
fill_cursor_ = 2;
|
|
}
|
|
|
|
ObjectPtr ForwardedObject(ObjectPtr object) {
|
|
const intptr_t id = GetObjectId(object);
|
|
if (id == 0) return Marker();
|
|
return from_to_[id + 1]->ptr();
|
|
}
|
|
|
|
void Insert(ObjectPtr from, ObjectPtr to) {
|
|
ASSERT(ForwardedObject(from) == Marker());
|
|
const auto id = from_to_.length();
|
|
SetObjectId(from, id);
|
|
from_to_.Resize(id + 2);
|
|
from_to_[id] = &Object::Handle(Z, from);
|
|
from_to_[id + 1] = &Object::Handle(Z, to);
|
|
}
|
|
|
|
void AddTransferable(const TransferableTypedData& from,
|
|
const TransferableTypedData& to) {
|
|
transferables_from_to_.Add(&TransferableTypedData::Handle(from.ptr()));
|
|
transferables_from_to_.Add(&TransferableTypedData::Handle(to.ptr()));
|
|
}
|
|
|
|
void AddExternalTypedData(ExternalTypedDataPtr to) {
|
|
external_typed_data_.Add(&ExternalTypedData::Handle(to));
|
|
}
|
|
|
|
void AddObjectToRehash(const Object& to) {
|
|
objects_to_rehash_.Add(&Object::Handle(to.ptr()));
|
|
}
|
|
|
|
void FinalizeTransferables() {
|
|
for (intptr_t i = 0; i < transferables_from_to_.length(); i += 2) {
|
|
auto from = transferables_from_to_[i];
|
|
auto to = transferables_from_to_[i + 1];
|
|
FinalizeTransferable(*from, *to);
|
|
}
|
|
}
|
|
|
|
void FinalizeExternalTypedData() {
|
|
for (intptr_t i = 0; i < external_typed_data_.length(); i++) {
|
|
auto to = external_typed_data_[i];
|
|
ForwardMapBase::FinalizeExternalTypedData(*to);
|
|
}
|
|
}
|
|
|
|
private:
|
|
friend class SlowObjectCopy;
|
|
friend class ObjectGraphCopier;
|
|
|
|
GrowableArray<const Object*> from_to_;
|
|
GrowableArray<const TransferableTypedData*> transferables_from_to_;
|
|
GrowableArray<const ExternalTypedData*> external_typed_data_;
|
|
GrowableArray<const Object*> objects_to_rehash_;
|
|
intptr_t fill_cursor_ = 0;
|
|
|
|
DISALLOW_COPY_AND_ASSIGN(SlowForwardMap);
|
|
};
|
|
|
|
class ObjectCopyBase {
|
|
public:
|
|
explicit ObjectCopyBase(Thread* thread)
|
|
: thread_(thread),
|
|
heap_base_(thread->heap_base()),
|
|
zone_(thread->zone()),
|
|
heap_(thread->isolate_group()->heap()),
|
|
class_table_(thread->isolate_group()->class_table()),
|
|
new_space_(heap_->new_space()),
|
|
tmp_(Object::Handle(thread->zone())) {}
|
|
~ObjectCopyBase() {}
|
|
|
|
protected:
|
|
static ObjectPtr LoadPointer(ObjectPtr src, intptr_t offset) {
|
|
return src.untag()->LoadPointer(reinterpret_cast<ObjectPtr*>(
|
|
reinterpret_cast<uint8_t*>(src.untag()) + offset));
|
|
}
|
|
static CompressedObjectPtr LoadCompressedPointer(ObjectPtr src,
|
|
intptr_t offset) {
|
|
return src.untag()->LoadPointer(reinterpret_cast<CompressedObjectPtr*>(
|
|
reinterpret_cast<uint8_t*>(src.untag()) + offset));
|
|
}
|
|
static compressed_uword LoadCompressedNonPointerWord(ObjectPtr src,
|
|
intptr_t offset) {
|
|
return *reinterpret_cast<compressed_uword*>(
|
|
reinterpret_cast<uint8_t*>(src.untag()) + offset);
|
|
}
|
|
static void StorePointerBarrier(ObjectPtr obj,
|
|
intptr_t offset,
|
|
ObjectPtr value) {
|
|
obj.untag()->StorePointer(
|
|
reinterpret_cast<ObjectPtr*>(reinterpret_cast<uint8_t*>(obj.untag()) +
|
|
offset),
|
|
value);
|
|
}
|
|
static void StoreCompressedPointerBarrier(ObjectPtr obj,
|
|
intptr_t offset,
|
|
ObjectPtr value) {
|
|
obj.untag()->StoreCompressedPointer(
|
|
reinterpret_cast<CompressedObjectPtr*>(
|
|
reinterpret_cast<uint8_t*>(obj.untag()) + offset),
|
|
value);
|
|
}
|
|
void StoreCompressedLargeArrayPointerBarrier(ObjectPtr obj,
|
|
intptr_t offset,
|
|
ObjectPtr value) {
|
|
obj.untag()->StoreCompressedArrayPointer(
|
|
reinterpret_cast<CompressedObjectPtr*>(
|
|
reinterpret_cast<uint8_t*>(obj.untag()) + offset),
|
|
value, thread_);
|
|
}
|
|
static void StorePointerNoBarrier(ObjectPtr obj,
|
|
intptr_t offset,
|
|
ObjectPtr value) {
|
|
*reinterpret_cast<ObjectPtr*>(reinterpret_cast<uint8_t*>(obj.untag()) +
|
|
offset) = value;
|
|
}
|
|
template <typename T = ObjectPtr>
|
|
static void StoreCompressedPointerNoBarrier(ObjectPtr obj,
|
|
intptr_t offset,
|
|
T value) {
|
|
*reinterpret_cast<CompressedObjectPtr*>(
|
|
reinterpret_cast<uint8_t*>(obj.untag()) + offset) = value;
|
|
}
|
|
static void StoreCompressedNonPointerWord(ObjectPtr obj,
|
|
intptr_t offset,
|
|
compressed_uword value) {
|
|
*reinterpret_cast<compressed_uword*>(
|
|
reinterpret_cast<uint8_t*>(obj.untag()) + offset) = value;
|
|
}
|
|
|
|
DART_FORCE_INLINE
|
|
bool CanCopyObject(uword tags, ObjectPtr object) {
|
|
const auto cid = UntaggedObject::ClassIdTag::decode(tags);
|
|
if (cid > kNumPredefinedCids) {
|
|
const bool has_native_fields =
|
|
Class::NumNativeFieldsOf(class_table_->At(cid)) != 0;
|
|
if (has_native_fields) {
|
|
exception_msg_ =
|
|
"Illegal argument in isolate message: (object has native fields)";
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
#define HANDLE_ILLEGAL_CASE(Type) \
|
|
case k##Type##Cid: { \
|
|
exception_msg_ = \
|
|
"Illegal argument in isolate message: " \
|
|
"(object is a" #Type ")"; \
|
|
return false; \
|
|
}
|
|
|
|
switch (cid) {
|
|
HANDLE_ILLEGAL_CASE(MirrorReference)
|
|
HANDLE_ILLEGAL_CASE(ReceivePort)
|
|
HANDLE_ILLEGAL_CASE(StackTrace)
|
|
HANDLE_ILLEGAL_CASE(UserTag)
|
|
HANDLE_ILLEGAL_CASE(DynamicLibrary)
|
|
HANDLE_ILLEGAL_CASE(Pointer)
|
|
case kClosureCid: {
|
|
if (!Function::IsImplicitStaticClosureFunction(
|
|
Closure::FunctionOf(Closure::RawCast(object)))) {
|
|
exception_msg_ =
|
|
"Illegal argument in isolate message: (object is a closure)";
|
|
return false;
|
|
}
|
|
ASSERT(Closure::ContextOf(Closure::RawCast(object)) == Object::null());
|
|
return true;
|
|
}
|
|
default:
|
|
return true;
|
|
}
|
|
}
|
|
|
|
Thread* thread_;
|
|
uword heap_base_;
|
|
Zone* zone_;
|
|
Heap* heap_;
|
|
ClassTable* class_table_;
|
|
Scavenger* new_space_;
|
|
Object& tmp_;
|
|
|
|
const char* exception_msg_ = nullptr;
|
|
};
|
|
|
|
class FastObjectCopyBase : public ObjectCopyBase {
|
|
public:
|
|
using Types = PtrTypes;
|
|
|
|
explicit FastObjectCopyBase(Thread* thread)
|
|
: ObjectCopyBase(thread), fast_forward_map_(thread) {}
|
|
|
|
protected:
|
|
DART_FORCE_INLINE
|
|
void ForwardCompressedPointers(ObjectPtr src,
|
|
ObjectPtr dst,
|
|
intptr_t offset,
|
|
intptr_t end_offset) {
|
|
for (; offset < end_offset; offset += kCompressedWordSize) {
|
|
ForwardCompressedPointer(src, dst, offset);
|
|
}
|
|
}
|
|
|
|
DART_FORCE_INLINE
|
|
void ForwardCompressedPointers(ObjectPtr src,
|
|
ObjectPtr dst,
|
|
intptr_t offset,
|
|
intptr_t end_offset,
|
|
UnboxedFieldBitmap bitmap) {
|
|
if (bitmap.IsEmpty()) {
|
|
ForwardCompressedPointers(src, dst, offset, end_offset);
|
|
return;
|
|
}
|
|
intptr_t bit = offset >> kCompressedWordSizeLog2;
|
|
for (; offset < end_offset; offset += kCompressedWordSize) {
|
|
if (bitmap.Get(bit++)) {
|
|
StoreCompressedNonPointerWord(
|
|
dst, offset, LoadCompressedNonPointerWord(src, offset));
|
|
} else {
|
|
ForwardCompressedPointer(src, dst, offset);
|
|
}
|
|
}
|
|
}
|
|
|
|
void ForwardCompressedArrayPointers(intptr_t array_length,
|
|
ObjectPtr src,
|
|
ObjectPtr dst,
|
|
intptr_t offset,
|
|
intptr_t end_offset) {
|
|
for (; offset < end_offset; offset += kCompressedWordSize) {
|
|
ForwardCompressedPointer(src, dst, offset);
|
|
}
|
|
}
|
|
|
|
DART_FORCE_INLINE
|
|
void ForwardCompressedPointer(ObjectPtr src, ObjectPtr dst, intptr_t offset) {
|
|
auto value = LoadCompressedPointer(src, offset);
|
|
if (!value.IsHeapObject()) {
|
|
StoreCompressedPointerNoBarrier(dst, offset, value);
|
|
return;
|
|
}
|
|
auto value_decompressed = value.Decompress(heap_base_);
|
|
const uword tags = TagsFromUntaggedObject(value_decompressed.untag());
|
|
if (CanShareObject(tags)) {
|
|
StoreCompressedPointerNoBarrier(dst, offset, value);
|
|
return;
|
|
}
|
|
|
|
ObjectPtr existing_to =
|
|
fast_forward_map_.ForwardedObject(value_decompressed);
|
|
if (existing_to != Marker()) {
|
|
StoreCompressedPointerNoBarrier(dst, offset, existing_to);
|
|
return;
|
|
}
|
|
|
|
if (UNLIKELY(!CanCopyObject(tags, value_decompressed))) {
|
|
ASSERT(exception_msg_ != nullptr);
|
|
StoreCompressedPointerNoBarrier(dst, offset, Object::null());
|
|
return;
|
|
}
|
|
|
|
auto to = Forward(tags, value.Decompress(heap_base_));
|
|
StoreCompressedPointerNoBarrier(dst, offset, to);
|
|
}
|
|
|
|
ObjectPtr Forward(uword tags, ObjectPtr from) {
|
|
const intptr_t header_size = UntaggedObject::SizeTag::decode(tags);
|
|
const auto cid = UntaggedObject::ClassIdTag::decode(tags);
|
|
const uword size =
|
|
header_size != 0 ? header_size : from.untag()->HeapSize();
|
|
if (Heap::IsAllocatableInNewSpace(size)) {
|
|
const uword alloc = new_space_->TryAllocate(thread_, size);
|
|
if (alloc != 0) {
|
|
ObjectPtr to(reinterpret_cast<UntaggedObject*>(alloc));
|
|
fast_forward_map_.Insert(from, to);
|
|
|
|
if (IsExternalTypedDataClassId(cid)) {
|
|
SetNewSpaceTaggingWord(to, cid, header_size);
|
|
InitializeExternalTypedData(cid, ExternalTypedData::RawCast(from),
|
|
ExternalTypedData::RawCast(to));
|
|
fast_forward_map_.AddExternalTypedData(
|
|
ExternalTypedData::RawCast(to));
|
|
} else if (IsTypedDataViewClassId(cid)) {
|
|
// We set the views backing store to `null` to satisfy an assertion in
|
|
// GCCompactor::VisitTypedDataViewPointers().
|
|
SetNewSpaceTaggingWord(to, cid, header_size);
|
|
InitializeTypedDataView(TypedDataView::RawCast(to));
|
|
}
|
|
return to;
|
|
}
|
|
}
|
|
exception_msg_ = kFastAllocationFailed;
|
|
return Marker();
|
|
}
|
|
|
|
void EnqueueTransferable(TransferableTypedDataPtr from,
|
|
TransferableTypedDataPtr to) {
|
|
fast_forward_map_.AddTransferable(from, to);
|
|
}
|
|
void EnqueueObjectToRehash(ObjectPtr to) {
|
|
fast_forward_map_.AddObjectToRehash(to);
|
|
}
|
|
|
|
static void StoreCompressedArrayPointers(intptr_t array_length,
|
|
ObjectPtr src,
|
|
ObjectPtr dst,
|
|
intptr_t offset,
|
|
intptr_t end_offset) {
|
|
StoreCompressedPointers(src, dst, offset, end_offset);
|
|
}
|
|
static void StoreCompressedPointers(ObjectPtr src,
|
|
ObjectPtr dst,
|
|
intptr_t offset,
|
|
intptr_t end_offset) {
|
|
StoreCompressedPointersNoBarrier(src, dst, offset, end_offset);
|
|
}
|
|
static void StoreCompressedPointersNoBarrier(ObjectPtr src,
|
|
ObjectPtr dst,
|
|
intptr_t offset,
|
|
intptr_t end_offset) {
|
|
for (; offset <= end_offset; offset += kCompressedWordSize) {
|
|
StoreCompressedPointerNoBarrier(dst, offset,
|
|
LoadCompressedPointer(src, offset));
|
|
}
|
|
}
|
|
|
|
protected:
|
|
friend class ObjectGraphCopier;
|
|
|
|
FastForwardMap fast_forward_map_;
|
|
};
|
|
|
|
class SlowObjectCopyBase : public ObjectCopyBase {
|
|
public:
|
|
using Types = HandleTypes;
|
|
|
|
explicit SlowObjectCopyBase(Thread* thread)
|
|
: ObjectCopyBase(thread), slow_forward_map_(thread) {}
|
|
|
|
protected:
|
|
DART_FORCE_INLINE
|
|
void ForwardCompressedPointers(const Object& src,
|
|
const Object& dst,
|
|
intptr_t offset,
|
|
intptr_t end_offset) {
|
|
for (; offset < end_offset; offset += kCompressedWordSize) {
|
|
ForwardCompressedPointer(src, dst, offset);
|
|
}
|
|
}
|
|
|
|
DART_FORCE_INLINE
|
|
void ForwardCompressedPointers(const Object& src,
|
|
const Object& dst,
|
|
intptr_t offset,
|
|
intptr_t end_offset,
|
|
UnboxedFieldBitmap bitmap) {
|
|
intptr_t bit = offset >> kCompressedWordSizeLog2;
|
|
for (; offset < end_offset; offset += kCompressedWordSize) {
|
|
if (bitmap.Get(bit++)) {
|
|
StoreCompressedNonPointerWord(
|
|
dst.ptr(), offset, LoadCompressedNonPointerWord(src.ptr(), offset));
|
|
} else {
|
|
ForwardCompressedPointer(src, dst, offset);
|
|
}
|
|
}
|
|
}
|
|
|
|
void ForwardCompressedArrayPointers(intptr_t array_length,
|
|
const Object& src,
|
|
const Object& dst,
|
|
intptr_t offset,
|
|
intptr_t end_offset) {
|
|
if (Array::UseCardMarkingForAllocation(array_length)) {
|
|
for (; offset < end_offset; offset += kCompressedWordSize) {
|
|
ForwardCompressedLargeArrayPointer(src, dst, offset);
|
|
}
|
|
} else {
|
|
for (; offset < end_offset; offset += kCompressedWordSize) {
|
|
ForwardCompressedPointer(src, dst, offset);
|
|
}
|
|
}
|
|
}
|
|
|
|
DART_FORCE_INLINE
|
|
void ForwardCompressedLargeArrayPointer(const Object& src,
|
|
const Object& dst,
|
|
intptr_t offset) {
|
|
auto value = LoadCompressedPointer(src.ptr(), offset);
|
|
if (!value.IsHeapObject()) {
|
|
StoreCompressedPointerNoBarrier(dst.ptr(), offset, value);
|
|
return;
|
|
}
|
|
|
|
auto value_decompressed = value.Decompress(heap_base_);
|
|
const uword tags = TagsFromUntaggedObject(value_decompressed.untag());
|
|
if (CanShareObject(tags)) {
|
|
StoreCompressedLargeArrayPointerBarrier(dst.ptr(), offset,
|
|
value_decompressed);
|
|
return;
|
|
}
|
|
|
|
ObjectPtr existing_to =
|
|
slow_forward_map_.ForwardedObject(value_decompressed);
|
|
if (existing_to != Marker()) {
|
|
StoreCompressedLargeArrayPointerBarrier(dst.ptr(), offset, existing_to);
|
|
return;
|
|
}
|
|
|
|
if (UNLIKELY(!CanCopyObject(tags, value_decompressed))) {
|
|
ASSERT(exception_msg_ != nullptr);
|
|
StoreCompressedLargeArrayPointerBarrier(dst.ptr(), offset,
|
|
Object::null());
|
|
return;
|
|
}
|
|
|
|
tmp_ = value_decompressed;
|
|
tmp_ = Forward(tags, tmp_); // Only this can cause allocation.
|
|
StoreCompressedLargeArrayPointerBarrier(dst.ptr(), offset, tmp_.ptr());
|
|
}
|
|
DART_FORCE_INLINE
|
|
void ForwardCompressedPointer(const Object& src,
|
|
const Object& dst,
|
|
intptr_t offset) {
|
|
auto value = LoadCompressedPointer(src.ptr(), offset);
|
|
if (!value.IsHeapObject()) {
|
|
StoreCompressedPointerNoBarrier(dst.ptr(), offset, value);
|
|
return;
|
|
}
|
|
auto value_decompressed = value.Decompress(heap_base_);
|
|
const uword tags = TagsFromUntaggedObject(value_decompressed.untag());
|
|
if (CanShareObject(tags)) {
|
|
StoreCompressedPointerBarrier(dst.ptr(), offset, value_decompressed);
|
|
return;
|
|
}
|
|
|
|
ObjectPtr existing_to =
|
|
slow_forward_map_.ForwardedObject(value_decompressed);
|
|
if (existing_to != Marker()) {
|
|
StoreCompressedPointerBarrier(dst.ptr(), offset, existing_to);
|
|
return;
|
|
}
|
|
|
|
if (UNLIKELY(!CanCopyObject(tags, value_decompressed))) {
|
|
ASSERT(exception_msg_ != nullptr);
|
|
StoreCompressedPointerNoBarrier(dst.ptr(), offset, Object::null());
|
|
return;
|
|
}
|
|
|
|
tmp_ = value_decompressed;
|
|
tmp_ = Forward(tags, tmp_); // Only this can cause allocation.
|
|
StoreCompressedPointerBarrier(dst.ptr(), offset, tmp_.ptr());
|
|
}
|
|
ObjectPtr Forward(uword tags, const Object& from) {
|
|
const intptr_t cid = UntaggedObject::ClassIdTag::decode(tags);
|
|
intptr_t size = UntaggedObject::SizeTag::decode(tags);
|
|
if (size == 0) {
|
|
size = from.ptr().untag()->HeapSize();
|
|
}
|
|
ObjectPtr to = AllocateObject(cid, size);
|
|
slow_forward_map_.Insert(from.ptr(), to);
|
|
UpdateLengthField(cid, from.ptr(), to);
|
|
if (cid == kArrayCid && !Heap::IsAllocatableInNewSpace(size)) {
|
|
to.untag()->SetCardRememberedBitUnsynchronized();
|
|
}
|
|
if (IsExternalTypedDataClassId(cid)) {
|
|
InitializeExternalTypedData(cid, ExternalTypedData::RawCast(from.ptr()),
|
|
ExternalTypedData::RawCast(to));
|
|
slow_forward_map_.AddExternalTypedData(ExternalTypedData::RawCast(to));
|
|
} else if (IsTypedDataViewClassId(cid)) {
|
|
// We set the views backing store to `null` to satisfy an assertion in
|
|
// GCCompactor::VisitTypedDataViewPointers().
|
|
InitializeTypedDataView(TypedDataView::RawCast(to));
|
|
}
|
|
return to;
|
|
}
|
|
void EnqueueTransferable(const TransferableTypedData& from,
|
|
const TransferableTypedData& to) {
|
|
slow_forward_map_.AddTransferable(from, to);
|
|
}
|
|
void EnqueueObjectToRehash(const Object& to) {
|
|
slow_forward_map_.AddObjectToRehash(to);
|
|
}
|
|
|
|
void StoreCompressedArrayPointers(intptr_t array_length,
|
|
const Object& src,
|
|
const Object& dst,
|
|
intptr_t offset,
|
|
intptr_t end_offset) {
|
|
auto src_ptr = src.ptr();
|
|
auto dst_ptr = dst.ptr();
|
|
if (Array::UseCardMarkingForAllocation(array_length)) {
|
|
for (; offset <= end_offset; offset += kCompressedWordSize) {
|
|
StoreCompressedLargeArrayPointerBarrier(
|
|
dst_ptr, offset,
|
|
LoadCompressedPointer(src_ptr, offset).Decompress(heap_base_));
|
|
}
|
|
} else {
|
|
for (; offset <= end_offset; offset += kCompressedWordSize) {
|
|
StoreCompressedPointerBarrier(
|
|
dst_ptr, offset,
|
|
LoadCompressedPointer(src_ptr, offset).Decompress(heap_base_));
|
|
}
|
|
}
|
|
}
|
|
void StoreCompressedPointers(const Object& src,
|
|
const Object& dst,
|
|
intptr_t offset,
|
|
intptr_t end_offset) {
|
|
auto src_ptr = src.ptr();
|
|
auto dst_ptr = dst.ptr();
|
|
for (; offset <= end_offset; offset += kCompressedWordSize) {
|
|
StoreCompressedPointerBarrier(
|
|
dst_ptr, offset,
|
|
LoadCompressedPointer(src_ptr, offset).Decompress(heap_base_));
|
|
}
|
|
}
|
|
static void StoreCompressedPointersNoBarrier(const Object& src,
|
|
const Object& dst,
|
|
intptr_t offset,
|
|
intptr_t end_offset) {
|
|
auto src_ptr = src.ptr();
|
|
auto dst_ptr = dst.ptr();
|
|
for (; offset <= end_offset; offset += kCompressedWordSize) {
|
|
StoreCompressedPointerNoBarrier(dst_ptr, offset,
|
|
LoadCompressedPointer(src_ptr, offset));
|
|
}
|
|
}
|
|
|
|
protected:
|
|
friend class ObjectGraphCopier;
|
|
|
|
SlowForwardMap slow_forward_map_;
|
|
};
|
|
|
|
template <typename Base>
|
|
class ObjectCopy : public Base {
|
|
public:
|
|
using Types = typename Base::Types;
|
|
|
|
explicit ObjectCopy(Thread* thread) : Base(thread) {}
|
|
|
|
void CopyPredefinedInstance(typename Types::Object from,
|
|
typename Types::Object to,
|
|
intptr_t cid) {
|
|
if (IsImplicitFieldClassId(cid)) {
|
|
CopyUserdefinedInstance(from, to);
|
|
return;
|
|
}
|
|
switch (cid) {
|
|
#define COPY_TO(clazz) \
|
|
case clazz::kClassId: { \
|
|
typename Types::clazz casted_from = Types::Cast##clazz(from); \
|
|
typename Types::clazz casted_to = Types::Cast##clazz(to); \
|
|
Copy##clazz(casted_from, casted_to); \
|
|
return; \
|
|
}
|
|
|
|
CLASS_LIST_NO_OBJECT_NOR_STRING_NOR_ARRAY_NOR_MAP(COPY_TO)
|
|
COPY_TO(Array)
|
|
COPY_TO(LinkedHashMap)
|
|
COPY_TO(LinkedHashSet)
|
|
#undef COPY_TO
|
|
|
|
#define COPY_TO(clazz) case kTypedData##clazz##Cid:
|
|
|
|
CLASS_LIST_TYPED_DATA(COPY_TO) {
|
|
typename Types::TypedData casted_from = Types::CastTypedData(from);
|
|
typename Types::TypedData casted_to = Types::CastTypedData(to);
|
|
CopyTypedData(casted_from, casted_to);
|
|
return;
|
|
}
|
|
#undef COPY_TO
|
|
|
|
case kByteDataViewCid:
|
|
#define COPY_TO(clazz) case kTypedData##clazz##ViewCid:
|
|
CLASS_LIST_TYPED_DATA(COPY_TO) {
|
|
typename Types::TypedDataView casted_from =
|
|
Types::CastTypedDataView(from);
|
|
typename Types::TypedDataView casted_to =
|
|
Types::CastTypedDataView(to);
|
|
CopyTypedDataView(casted_from, casted_to);
|
|
return;
|
|
}
|
|
#undef COPY_TO
|
|
|
|
#define COPY_TO(clazz) case kExternalTypedData##clazz##Cid:
|
|
|
|
CLASS_LIST_TYPED_DATA(COPY_TO) {
|
|
typename Types::ExternalTypedData casted_from =
|
|
Types::CastExternalTypedData(from);
|
|
typename Types::ExternalTypedData casted_to =
|
|
Types::CastExternalTypedData(to);
|
|
CopyExternalTypedData(casted_from, casted_to);
|
|
return;
|
|
}
|
|
#undef COPY_TO
|
|
default:
|
|
break;
|
|
}
|
|
|
|
const Object& obj = Types::HandlifyObject(from);
|
|
FATAL1("Unexpected object: %s\n", obj.ToCString());
|
|
}
|
|
|
|
#if defined(DART_PRECOMPILED_RUNTIME)
|
|
void CopyUserdefinedInstanceAOT(typename Types::Object from,
|
|
typename Types::Object to,
|
|
UnboxedFieldBitmap bitmap) {
|
|
const intptr_t instance_size = UntagObject(from)->HeapSize();
|
|
Base::ForwardCompressedPointers(from, to, kWordSize, instance_size, bitmap);
|
|
}
|
|
#endif
|
|
|
|
void CopyUserdefinedInstance(typename Types::Object from,
|
|
typename Types::Object to) {
|
|
const intptr_t instance_size = UntagObject(from)->HeapSize();
|
|
Base::ForwardCompressedPointers(from, to, kWordSize, instance_size);
|
|
}
|
|
|
|
void CopyClosure(typename Types::Closure from, typename Types::Closure to) {
|
|
Base::StoreCompressedPointers(
|
|
from, to, OFFSET_OF(UntaggedClosure, instantiator_type_arguments_),
|
|
OFFSET_OF(UntaggedClosure, function_));
|
|
Base::ForwardCompressedPointer(from, to,
|
|
OFFSET_OF(UntaggedClosure, context_));
|
|
Base::StoreCompressedPointersNoBarrier(from, to,
|
|
OFFSET_OF(UntaggedClosure, hash_),
|
|
OFFSET_OF(UntaggedClosure, hash_));
|
|
ONLY_IN_PRECOMPILED(UntagClosure(to)->entry_point_ =
|
|
UntagClosure(from)->entry_point_);
|
|
}
|
|
|
|
void CopyArray(typename Types::Array from, typename Types::Array to) {
|
|
const intptr_t length = Smi::Value(UntagArray(from)->length());
|
|
Base::StoreCompressedArrayPointers(
|
|
length, from, to, OFFSET_OF(UntaggedArray, type_arguments_),
|
|
OFFSET_OF(UntaggedArray, type_arguments_));
|
|
Base::StoreCompressedPointersNoBarrier(from, to,
|
|
OFFSET_OF(UntaggedArray, length_),
|
|
OFFSET_OF(UntaggedArray, length_));
|
|
Base::ForwardCompressedArrayPointers(
|
|
length, from, to, Array::data_offset(),
|
|
Array::data_offset() + kCompressedWordSize * length);
|
|
}
|
|
|
|
void CopyGrowableObjectArray(typename Types::GrowableObjectArray from,
|
|
typename Types::GrowableObjectArray to) {
|
|
Base::StoreCompressedPointers(
|
|
from, to, OFFSET_OF(UntaggedGrowableObjectArray, type_arguments_),
|
|
OFFSET_OF(UntaggedGrowableObjectArray, type_arguments_));
|
|
Base::StoreCompressedPointersNoBarrier(
|
|
from, to, OFFSET_OF(UntaggedGrowableObjectArray, length_),
|
|
OFFSET_OF(UntaggedGrowableObjectArray, length_));
|
|
Base::ForwardCompressedPointer(
|
|
from, to, OFFSET_OF(UntaggedGrowableObjectArray, data_));
|
|
}
|
|
|
|
template <intptr_t one_for_set_two_for_map, typename T>
|
|
void CopyLinkedHashBase(T from,
|
|
T to,
|
|
UntaggedLinkedHashBase* from_untagged,
|
|
UntaggedLinkedHashBase* to_untagged) {
|
|
// We have to find out whether the map needs re-hashing on the receiver side
|
|
// due to keys being copied and the keys therefore possibly having different
|
|
// hash codes (e.g. due to user-defined hashCode implementation or due to
|
|
// new identity hash codes of the copied objects).
|
|
bool needs_rehashing = false;
|
|
ArrayPtr data = from_untagged->data_.Decompress(Base::heap_base_);
|
|
if (data != Array::null()) {
|
|
UntaggedArray* untagged_data = data.untag();
|
|
const intptr_t length = Smi::Value(untagged_data->length_);
|
|
auto key_value_pairs = untagged_data->data();
|
|
for (intptr_t i = 0; i < length; i += one_for_set_two_for_map) {
|
|
ObjectPtr key = key_value_pairs[i].Decompress(Base::heap_base_);
|
|
if (key->IsHeapObject()) {
|
|
if (MightNeedReHashing(key)) {
|
|
needs_rehashing = true;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
Base::StoreCompressedPointers(
|
|
from, to, OFFSET_OF(UntaggedLinkedHashBase, type_arguments_),
|
|
OFFSET_OF(UntaggedLinkedHashBase, type_arguments_));
|
|
|
|
// Compared with the snapshot-based (de)serializer we do preserve the same
|
|
// backing store (i.e. used_data/deleted_keys/data) and therefore do not
|
|
// magically shrink backing store based on usage.
|
|
//
|
|
// We do this to avoid making assumptions about the object graph and the
|
|
// linked hash map (e.g. assuming there's no other references to the data,
|
|
// assuming the linked hashmap is in a consistent state)
|
|
if (needs_rehashing) {
|
|
to_untagged->hash_mask_ = Smi::New(0);
|
|
to_untagged->index_ = TypedData::RawCast(Object::null());
|
|
Base::EnqueueObjectToRehash(to);
|
|
}
|
|
|
|
// From this point on we shouldn't use the raw pointers, since GC might
|
|
// happen when forwarding objects.
|
|
from_untagged = nullptr;
|
|
to_untagged = nullptr;
|
|
|
|
if (!needs_rehashing) {
|
|
Base::ForwardCompressedPointer(from, to,
|
|
OFFSET_OF(UntaggedLinkedHashBase, index_));
|
|
Base::StoreCompressedPointersNoBarrier(
|
|
from, to, OFFSET_OF(UntaggedLinkedHashBase, hash_mask_),
|
|
OFFSET_OF(UntaggedLinkedHashBase, hash_mask_));
|
|
}
|
|
Base::ForwardCompressedPointer(from, to,
|
|
OFFSET_OF(UntaggedLinkedHashBase, data_));
|
|
Base::StoreCompressedPointersNoBarrier(
|
|
from, to, OFFSET_OF(UntaggedLinkedHashBase, used_data_),
|
|
OFFSET_OF(UntaggedLinkedHashBase, used_data_));
|
|
Base::StoreCompressedPointersNoBarrier(
|
|
from, to, OFFSET_OF(UntaggedLinkedHashMap, deleted_keys_),
|
|
OFFSET_OF(UntaggedLinkedHashMap, deleted_keys_));
|
|
}
|
|
|
|
void CopyLinkedHashMap(typename Types::LinkedHashMap from,
|
|
typename Types::LinkedHashMap to) {
|
|
CopyLinkedHashBase<2, typename Types::LinkedHashMap>(
|
|
from, to, UntagLinkedHashMap(from), UntagLinkedHashMap(to));
|
|
}
|
|
void CopyLinkedHashSet(typename Types::LinkedHashSet from,
|
|
typename Types::LinkedHashSet to) {
|
|
CopyLinkedHashBase<1, typename Types::LinkedHashSet>(
|
|
from, to, UntagLinkedHashSet(from), UntagLinkedHashSet(to));
|
|
}
|
|
|
|
void CopyDouble(typename Types::Double from, typename Types::Double to) {
|
|
#if !defined(DART_PRECOMPILED_RUNTIME)
|
|
auto raw_from = UntagDouble(from);
|
|
auto raw_to = UntagDouble(to);
|
|
raw_to->value_ = raw_from->value_;
|
|
#else
|
|
// Will be shared and not copied.
|
|
UNREACHABLE();
|
|
#endif
|
|
}
|
|
|
|
void CopyFloat32x4(typename Types::Float32x4 from,
|
|
typename Types::Float32x4 to) {
|
|
#if !defined(DART_PRECOMPILED_RUNTIME)
|
|
auto raw_from = UntagFloat32x4(from);
|
|
auto raw_to = UntagFloat32x4(to);
|
|
raw_to->value_[0] = raw_from->value_[0];
|
|
raw_to->value_[1] = raw_from->value_[1];
|
|
raw_to->value_[2] = raw_from->value_[2];
|
|
raw_to->value_[3] = raw_from->value_[3];
|
|
#else
|
|
// Will be shared and not copied.
|
|
UNREACHABLE();
|
|
#endif
|
|
}
|
|
|
|
void CopyFloat64x2(typename Types::Float64x2 from,
|
|
typename Types::Float64x2 to) {
|
|
#if !defined(DART_PRECOMPILED_RUNTIME)
|
|
auto raw_from = UntagFloat64x2(from);
|
|
auto raw_to = UntagFloat64x2(to);
|
|
raw_to->value_[0] = raw_from->value_[0];
|
|
raw_to->value_[1] = raw_from->value_[1];
|
|
#else
|
|
// Will be shared and not copied.
|
|
UNREACHABLE();
|
|
#endif
|
|
}
|
|
|
|
void CopyTypedData(typename Types::TypedData from,
|
|
typename Types::TypedData to) {
|
|
auto raw_from = UntagTypedData(from);
|
|
auto raw_to = UntagTypedData(to);
|
|
const intptr_t cid = Types::GetTypedDataPtr(from)->GetClassId();
|
|
raw_to->length_ = raw_from->length_;
|
|
raw_to->RecomputeDataField();
|
|
const intptr_t length =
|
|
TypedData::ElementSizeInBytes(cid) * Smi::Value(raw_from->length_);
|
|
memmove(raw_to->data_, raw_from->data_, length);
|
|
}
|
|
|
|
void CopyTypedDataView(typename Types::TypedDataView from,
|
|
typename Types::TypedDataView to) {
|
|
// This will forward & initialize the typed data.
|
|
Base::ForwardCompressedPointer(
|
|
from, to, OFFSET_OF(UntaggedTypedDataView, typed_data_));
|
|
|
|
auto raw_from = UntagTypedDataView(from);
|
|
auto raw_to = UntagTypedDataView(to);
|
|
raw_to->length_ = raw_from->length_;
|
|
raw_to->offset_in_bytes_ = raw_from->offset_in_bytes_;
|
|
raw_to->data_ = nullptr;
|
|
|
|
if (raw_to->typed_data_.Decompress(Base::heap_base_) == Object::null()) {
|
|
ASSERT(Base::exception_msg_ != nullptr);
|
|
return;
|
|
}
|
|
|
|
const bool is_external =
|
|
raw_from->data_ != raw_from->DataFieldForInternalTypedData();
|
|
if (is_external) {
|
|
// The raw_to is fully initialized at this point (see handling of external
|
|
// typed data in [ForwardCompressedPointer])
|
|
raw_to->RecomputeDataField();
|
|
} else {
|
|
// The raw_to isn't initialized yet, but it's address is valid, so we can
|
|
// compute the data field it would use.
|
|
raw_to->RecomputeDataFieldForInternalTypedData();
|
|
}
|
|
const bool is_external2 =
|
|
raw_to->data_ != raw_to->DataFieldForInternalTypedData();
|
|
ASSERT(is_external == is_external2);
|
|
}
|
|
|
|
void CopyExternalTypedData(typename Types::ExternalTypedData from,
|
|
typename Types::ExternalTypedData to) {
|
|
// The external typed data is initialized on the forwarding pass (where
|
|
// normally allocation but not initialization happens), so views on it
|
|
// can be initialized immediately.
|
|
#if defined(DEBUG)
|
|
auto raw_from = UntagExternalTypedData(from);
|
|
auto raw_to = UntagExternalTypedData(to);
|
|
ASSERT(raw_to->data_ != nullptr);
|
|
ASSERT(raw_to->length_ == raw_from->length_);
|
|
#endif
|
|
}
|
|
|
|
void CopyTransferableTypedData(typename Types::TransferableTypedData from,
|
|
typename Types::TransferableTypedData to) {
|
|
// The [TransferableTypedData] is an empty object with an associated heap
|
|
// peer object.
|
|
// -> We'll validate that there's a peer and enqueue the transferable to be
|
|
// transferred if the transitive copy is successful.
|
|
auto fpeer = static_cast<TransferableTypedDataPeer*>(
|
|
Base::heap_->GetPeer(Types::GetTransferableTypedDataPtr(from)));
|
|
ASSERT(fpeer != nullptr);
|
|
if (fpeer->data() == nullptr) {
|
|
Base::exception_msg_ =
|
|
"Illegal argument in isolate message"
|
|
" : (TransferableTypedData has been transferred already)";
|
|
return;
|
|
}
|
|
Base::EnqueueTransferable(from, to);
|
|
}
|
|
|
|
#define DEFINE_UNSUPPORTED(clazz) \
|
|
void Copy##clazz(typename Types::clazz from, typename Types::clazz to) { \
|
|
FATAL("Objects of type " #clazz " should not occur in object graphs"); \
|
|
}
|
|
|
|
FOR_UNSUPPORTED_CLASSES(DEFINE_UNSUPPORTED)
|
|
|
|
#undef DEFINE_UNSUPPORTED
|
|
|
|
UntaggedObject* UntagObject(typename Types::Object obj) {
|
|
return Types::GetObjectPtr(obj).Decompress(Base::heap_base_).untag();
|
|
}
|
|
|
|
#define DO(V) \
|
|
DART_FORCE_INLINE \
|
|
Untagged##V* Untag##V(typename Types::V obj) { \
|
|
return Types::Get##V##Ptr(obj).Decompress(Base::heap_base_).untag(); \
|
|
}
|
|
CLASS_LIST_FOR_HANDLES(DO)
|
|
#undef DO
|
|
};
|
|
|
|
class FastObjectCopy : public ObjectCopy<FastObjectCopyBase> {
|
|
public:
|
|
explicit FastObjectCopy(Thread* thread) : ObjectCopy(thread) {}
|
|
~FastObjectCopy() {}
|
|
|
|
ObjectPtr TryCopyGraphFast(ObjectPtr root) {
|
|
NoSafepointScope no_safepoint_scope;
|
|
|
|
ObjectPtr root_copy = Forward(TagsFromUntaggedObject(root.untag()), root);
|
|
if (root_copy == Marker()) {
|
|
return root_copy;
|
|
}
|
|
while (fast_forward_map_.fill_cursor_ <
|
|
fast_forward_map_.raw_from_to_.length()) {
|
|
const intptr_t index = fast_forward_map_.fill_cursor_;
|
|
ObjectPtr from = fast_forward_map_.raw_from_to_[index];
|
|
ObjectPtr to = fast_forward_map_.raw_from_to_[index + 1];
|
|
FastCopyObject(from, to);
|
|
if (exception_msg_ != nullptr) {
|
|
return root_copy;
|
|
}
|
|
fast_forward_map_.fill_cursor_ += 2;
|
|
}
|
|
if (root_copy != Marker()) {
|
|
TryBuildArrayOfObjectsToRehash();
|
|
}
|
|
return root_copy;
|
|
}
|
|
|
|
void TryBuildArrayOfObjectsToRehash() {
|
|
const auto& objects_to_rehash = fast_forward_map_.raw_objects_to_rehash_;
|
|
const intptr_t length = objects_to_rehash.length();
|
|
if (length == 0) return;
|
|
|
|
const intptr_t size = Array::InstanceSize(length);
|
|
const uword array_addr = new_space_->TryAllocate(thread_, size);
|
|
if (array_addr == 0) {
|
|
exception_msg_ = kFastAllocationFailed;
|
|
return;
|
|
}
|
|
|
|
const uword header_size =
|
|
UntaggedObject::SizeTag::SizeFits(size) ? size : 0;
|
|
ArrayPtr array(reinterpret_cast<UntaggedArray*>(array_addr));
|
|
SetNewSpaceTaggingWord(array, kArrayCid, header_size);
|
|
StoreCompressedPointerNoBarrier(array, OFFSET_OF(UntaggedArray, length_),
|
|
Smi::New(length));
|
|
StoreCompressedPointerNoBarrier(array,
|
|
OFFSET_OF(UntaggedArray, type_arguments_),
|
|
TypeArguments::null());
|
|
auto array_data = array.untag()->data();
|
|
for (intptr_t i = 0; i < length; ++i) {
|
|
array_data[i] = objects_to_rehash[i];
|
|
}
|
|
raw_objects_to_rehash_ = array;
|
|
}
|
|
|
|
private:
|
|
friend class ObjectGraphCopier;
|
|
|
|
void FastCopyObject(ObjectPtr from, ObjectPtr to) {
|
|
const uword tags = TagsFromUntaggedObject(from.untag());
|
|
const intptr_t cid = UntaggedObject::ClassIdTag::decode(tags);
|
|
const intptr_t size = UntaggedObject::SizeTag::decode(tags);
|
|
|
|
// Ensure the last word is GC-safe (our heap objects are 2-word aligned, the
|
|
// object header stores the size in multiples of kObjectAlignment, the GC
|
|
// uses the information from the header and therefore might visit one slot
|
|
// more than the actual size of the instance).
|
|
*reinterpret_cast<ObjectPtr*>(UntaggedObject::ToAddr(to) +
|
|
from.untag()->HeapSize() - kWordSize) = 0;
|
|
SetNewSpaceTaggingWord(to, cid, size);
|
|
|
|
// Fall back to virtual variant for predefined classes
|
|
if (cid < kNumPredefinedCids && cid != kInstanceCid) {
|
|
CopyPredefinedInstance(from, to, cid);
|
|
return;
|
|
}
|
|
#if defined(DART_PRECOMPILED_RUNTIME)
|
|
const auto bitmap =
|
|
class_table_->shared_class_table()->GetUnboxedFieldsMapAt(cid);
|
|
CopyUserdefinedInstanceAOT(Instance::RawCast(from), Instance::RawCast(to),
|
|
bitmap);
|
|
#else
|
|
CopyUserdefinedInstance(Instance::RawCast(from), Instance::RawCast(to));
|
|
#endif
|
|
}
|
|
|
|
ArrayPtr raw_objects_to_rehash_ = Array::null();
|
|
};
|
|
|
|
class SlowObjectCopy : public ObjectCopy<SlowObjectCopyBase> {
|
|
public:
|
|
explicit SlowObjectCopy(Thread* thread)
|
|
: ObjectCopy(thread), objects_to_rehash_(Array::Handle(thread->zone())) {}
|
|
~SlowObjectCopy() {}
|
|
|
|
ObjectPtr ContinueCopyGraphSlow(const Object& root,
|
|
const Object& fast_root_copy) {
|
|
auto& root_copy = Object::Handle(Z, fast_root_copy.ptr());
|
|
if (root_copy.ptr() == Marker()) {
|
|
root_copy = Forward(TagsFromUntaggedObject(root.ptr().untag()), root);
|
|
}
|
|
|
|
Object& from = Object::Handle(Z);
|
|
Object& to = Object::Handle(Z);
|
|
while (slow_forward_map_.fill_cursor_ <
|
|
slow_forward_map_.from_to_.length()) {
|
|
const intptr_t index = slow_forward_map_.fill_cursor_;
|
|
from = slow_forward_map_.from_to_[index]->ptr();
|
|
to = slow_forward_map_.from_to_[index + 1]->ptr();
|
|
CopyObject(from, to);
|
|
slow_forward_map_.fill_cursor_ += 2;
|
|
if (exception_msg_ != nullptr) {
|
|
return Marker();
|
|
}
|
|
}
|
|
BuildArrayOfObjectsToRehash();
|
|
return root_copy.ptr();
|
|
}
|
|
|
|
void BuildArrayOfObjectsToRehash() {
|
|
const auto& objects_to_rehash = slow_forward_map_.objects_to_rehash_;
|
|
const intptr_t length = objects_to_rehash.length();
|
|
if (length == 0) return;
|
|
|
|
objects_to_rehash_ = Array::New(length);
|
|
for (intptr_t i = 0; i < length; ++i) {
|
|
objects_to_rehash_.SetAt(i, *objects_to_rehash[i]);
|
|
}
|
|
}
|
|
|
|
private:
|
|
friend class ObjectGraphCopier;
|
|
|
|
void CopyObject(const Object& from, const Object& to) {
|
|
const auto cid = from.GetClassId();
|
|
|
|
// Fall back to virtual variant for predefined classes
|
|
if (cid < kNumPredefinedCids && cid != kInstanceCid) {
|
|
CopyPredefinedInstance(from, to, cid);
|
|
return;
|
|
}
|
|
#if defined(DART_PRECOMPILED_RUNTIME)
|
|
const auto bitmap =
|
|
class_table_->shared_class_table()->GetUnboxedFieldsMapAt(cid);
|
|
CopyUserdefinedInstanceAOT(from, to, bitmap);
|
|
#else
|
|
CopyUserdefinedInstance(from, to);
|
|
#endif
|
|
}
|
|
|
|
Array& objects_to_rehash_;
|
|
};
|
|
|
|
class ObjectGraphCopier {
|
|
public:
|
|
explicit ObjectGraphCopier(Thread* thread)
|
|
: thread_(thread),
|
|
zone_(thread->zone()),
|
|
fast_object_copy_(thread_),
|
|
slow_object_copy_(thread_) {
|
|
thread_->isolate()->set_forward_table_new(new WeakTable());
|
|
thread_->isolate()->set_forward_table_old(new WeakTable());
|
|
}
|
|
~ObjectGraphCopier() {
|
|
thread_->isolate()->set_forward_table_new(nullptr);
|
|
thread_->isolate()->set_forward_table_old(nullptr);
|
|
}
|
|
|
|
// Result will be [<msg>, <objects-in-msg-to-rehash>]
|
|
ObjectPtr CopyObjectGraph(const Object& root) {
|
|
const char* exception_msg = nullptr;
|
|
auto& result = Object::Handle(zone_);
|
|
|
|
{
|
|
LongJumpScope jump; // e.g. for OOMs.
|
|
if (setjmp(*jump.Set()) == 0) {
|
|
result = CopyObjectGraphInternal(root, &exception_msg);
|
|
// Any allocated external typed data must have finalizers attached so
|
|
// memory will get free()ed.
|
|
slow_object_copy_.slow_forward_map_.FinalizeExternalTypedData();
|
|
} else {
|
|
// Any allocated external typed data must have finalizers attached so
|
|
// memory will get free()ed.
|
|
slow_object_copy_.slow_forward_map_.FinalizeExternalTypedData();
|
|
|
|
// The copy failed due to non-application error (e.g. OOM error),
|
|
// propagate this error.
|
|
result = thread_->StealStickyError();
|
|
RELEASE_ASSERT(result.IsError());
|
|
}
|
|
}
|
|
|
|
if (result.IsError()) {
|
|
Exceptions::PropagateError(Error::Cast(result));
|
|
UNREACHABLE();
|
|
}
|
|
if (result.ptr() == Marker()) {
|
|
ASSERT(exception_msg != nullptr);
|
|
ThrowException(exception_msg);
|
|
UNREACHABLE();
|
|
}
|
|
|
|
// The copy was successful, then detach transferable data from the sender
|
|
// and attach to the copied graph.
|
|
slow_object_copy_.slow_forward_map_.FinalizeTransferables();
|
|
return result.ptr();
|
|
}
|
|
|
|
private:
|
|
ObjectPtr CopyObjectGraphInternal(const Object& root,
|
|
const char** exception_msg) {
|
|
const auto& result_array = Array::Handle(zone_, Array::New(2));
|
|
if (!root.ptr()->IsHeapObject()) {
|
|
result_array.SetAt(0, root);
|
|
return result_array.ptr();
|
|
}
|
|
const uword tags = TagsFromUntaggedObject(root.ptr().untag());
|
|
if (CanShareObject(tags)) {
|
|
result_array.SetAt(0, root);
|
|
return result_array.ptr();
|
|
}
|
|
if (!fast_object_copy_.CanCopyObject(tags, root.ptr())) {
|
|
ASSERT(fast_object_copy_.exception_msg_ != nullptr);
|
|
*exception_msg = fast_object_copy_.exception_msg_;
|
|
return Marker();
|
|
}
|
|
|
|
// We try a fast new-space only copy first that will not use any barriers.
|
|
auto& result = Object::Handle(Z, Marker());
|
|
|
|
// All allocated but non-initialized heap objects have to be made GC-visible
|
|
// at this point.
|
|
if (FLAG_enable_fast_object_copy) {
|
|
{
|
|
NoSafepointScope no_safepoint_scope;
|
|
|
|
result = fast_object_copy_.TryCopyGraphFast(root.ptr());
|
|
if (result.ptr() != Marker()) {
|
|
if (fast_object_copy_.exception_msg_ == nullptr) {
|
|
result_array.SetAt(0, result);
|
|
fast_object_copy_.tmp_ = fast_object_copy_.raw_objects_to_rehash_;
|
|
result_array.SetAt(1, fast_object_copy_.tmp_);
|
|
HandlifyExternalTypedData();
|
|
HandlifyTransferables();
|
|
return result_array.ptr();
|
|
}
|
|
|
|
// There are left-over uninitialized objects we'll have to make GC
|
|
// visible.
|
|
SwitchToSlowFowardingList();
|
|
}
|
|
}
|
|
|
|
if (FLAG_gc_on_foc_slow_path) {
|
|
// We use kLowMemory to force the GC to compact, which is more likely to
|
|
// discover untracked pointers (and other issues, like incorrect class
|
|
// table).
|
|
thread_->heap()->CollectAllGarbage(Heap::kLowMemory);
|
|
}
|
|
|
|
// Fast copy failed due to
|
|
// - either failure to allocate into new space
|
|
// - or failure to copy object which we cannot copy
|
|
ASSERT(fast_object_copy_.exception_msg_ != nullptr);
|
|
if (fast_object_copy_.exception_msg_ != kFastAllocationFailed) {
|
|
*exception_msg = fast_object_copy_.exception_msg_;
|
|
return Marker();
|
|
}
|
|
ASSERT(fast_object_copy_.exception_msg_ == kFastAllocationFailed);
|
|
}
|
|
|
|
// Use the slow copy approach.
|
|
result = slow_object_copy_.ContinueCopyGraphSlow(root, result);
|
|
ASSERT((result.ptr() == Marker()) ==
|
|
(slow_object_copy_.exception_msg_ != nullptr));
|
|
if (result.ptr() == Marker()) {
|
|
*exception_msg = slow_object_copy_.exception_msg_;
|
|
return Marker();
|
|
}
|
|
|
|
result_array.SetAt(0, result);
|
|
result_array.SetAt(1, slow_object_copy_.objects_to_rehash_);
|
|
return result_array.ptr();
|
|
}
|
|
|
|
void SwitchToSlowFowardingList() {
|
|
auto& fast_forward_map = fast_object_copy_.fast_forward_map_;
|
|
auto& slow_forward_map = slow_object_copy_.slow_forward_map_;
|
|
|
|
MakeUninitializedNewSpaceObjectsGCSafe();
|
|
HandlifyTransferables();
|
|
HandlifyExternalTypedData();
|
|
HandlifyObjectsToReHash();
|
|
HandlifyFromToObjects();
|
|
slow_forward_map.fill_cursor_ = fast_forward_map.fill_cursor_;
|
|
}
|
|
|
|
void MakeUninitializedNewSpaceObjectsGCSafe() {
|
|
auto& fast_forward_map = fast_object_copy_.fast_forward_map_;
|
|
const auto length = fast_forward_map.raw_from_to_.length();
|
|
const auto cursor = fast_forward_map.fill_cursor_;
|
|
for (intptr_t i = cursor; i < length; i += 2) {
|
|
auto from = fast_forward_map.raw_from_to_[i];
|
|
auto to = fast_forward_map.raw_from_to_[i + 1];
|
|
const uword tags = TagsFromUntaggedObject(from.untag());
|
|
const intptr_t cid = UntaggedObject::ClassIdTag::decode(tags);
|
|
const intptr_t size = UntaggedObject::SizeTag::decode(tags);
|
|
// External typed data is already initialized.
|
|
if (!IsExternalTypedDataClassId(cid) && !IsTypedDataViewClassId(cid)) {
|
|
memset(to.untag(), 0x0, from.untag()->HeapSize());
|
|
SetNewSpaceTaggingWord(to, cid, size);
|
|
UpdateLengthField(cid, from, to);
|
|
}
|
|
}
|
|
}
|
|
void HandlifyTransferables() {
|
|
auto& raw_transferables =
|
|
fast_object_copy_.fast_forward_map_.raw_transferables_from_to_;
|
|
const auto length = raw_transferables.length();
|
|
if (length > 0) {
|
|
auto& transferables =
|
|
slow_object_copy_.slow_forward_map_.transferables_from_to_;
|
|
transferables.Resize(length);
|
|
for (intptr_t i = 0; i < length; i++) {
|
|
transferables[i] =
|
|
&TransferableTypedData::Handle(Z, raw_transferables[i]);
|
|
}
|
|
raw_transferables.Clear();
|
|
}
|
|
}
|
|
void HandlifyExternalTypedData() {
|
|
auto& raw_external_typed_data =
|
|
fast_object_copy_.fast_forward_map_.raw_external_typed_data_to_;
|
|
const auto length = raw_external_typed_data.length();
|
|
if (length > 0) {
|
|
auto& external_typed_data =
|
|
slow_object_copy_.slow_forward_map_.external_typed_data_;
|
|
external_typed_data.Resize(length);
|
|
for (intptr_t i = 0; i < length; i++) {
|
|
external_typed_data[i] =
|
|
&ExternalTypedData::Handle(Z, raw_external_typed_data[i]);
|
|
}
|
|
raw_external_typed_data.Clear();
|
|
}
|
|
}
|
|
void HandlifyObjectsToReHash() {
|
|
auto& fast_forward_map = fast_object_copy_.fast_forward_map_;
|
|
auto& slow_forward_map = slow_object_copy_.slow_forward_map_;
|
|
const auto length = fast_forward_map.raw_transferables_from_to_.length();
|
|
if (length > 0) {
|
|
slow_forward_map.objects_to_rehash_.Resize(length);
|
|
for (intptr_t i = 0; i < length; i++) {
|
|
slow_forward_map.objects_to_rehash_[i] =
|
|
&Object::Handle(Z, fast_forward_map.raw_objects_to_rehash_[i]);
|
|
}
|
|
fast_forward_map.raw_objects_to_rehash_.Clear();
|
|
}
|
|
}
|
|
void HandlifyFromToObjects() {
|
|
auto& fast_forward_map = fast_object_copy_.fast_forward_map_;
|
|
auto& slow_forward_map = slow_object_copy_.slow_forward_map_;
|
|
|
|
const intptr_t cursor = fast_forward_map.fill_cursor_;
|
|
const intptr_t length = fast_forward_map.raw_from_to_.length();
|
|
|
|
slow_forward_map.from_to_.Resize(length);
|
|
for (intptr_t i = 2; i < length; i += 2) {
|
|
slow_forward_map.from_to_[i] =
|
|
i < cursor ? nullptr
|
|
: &Object::Handle(Z, fast_forward_map.raw_from_to_[i]);
|
|
slow_forward_map.from_to_[i + 1] =
|
|
&Object::Handle(Z, fast_forward_map.raw_from_to_[i + 1]);
|
|
}
|
|
fast_forward_map.raw_from_to_.Clear();
|
|
}
|
|
void ThrowException(const char* exception_msg) {
|
|
const auto& msg_obj = String::Handle(Z, String::New(exception_msg));
|
|
const auto& args = Array::Handle(Z, Array::New(1));
|
|
args.SetAt(0, msg_obj);
|
|
Exceptions::ThrowByType(Exceptions::kArgument, args);
|
|
UNREACHABLE();
|
|
}
|
|
|
|
Thread* thread_;
|
|
Zone* zone_;
|
|
FastObjectCopy fast_object_copy_;
|
|
SlowObjectCopy slow_object_copy_;
|
|
};
|
|
|
|
ObjectPtr CopyMutableObjectGraph(const Object& object) {
|
|
auto thread = Thread::Current();
|
|
ObjectGraphCopier copier(thread);
|
|
return copier.CopyObjectGraph(object);
|
|
}
|
|
|
|
} // namespace dart
|