mirror of
https://github.com/dart-lang/sdk
synced 2024-10-14 23:28:09 +00:00
0713bce1fe
This was sometimes responsible for the largest GC pause in IsolateSendExitLatency, with a scavenge waiting for a concurrent marker to finish a whole array and pause. TEST=ci Change-Id: Ib2d8417ffe6cee3e8436a8c382d2db660e1eef41 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/371180 Commit-Queue: Ryan Macnak <rmacnak@google.com> Reviewed-by: Alexander Aprelev <aam@google.com>
2738 lines
104 KiB
C++
2738 lines
104 KiB
C++
// Copyright (c) 2021, the Dart project authors. Please see the AUTHORS file
|
|
// for details. All rights reserved. Use of this source code is governed by a
|
|
// BSD-style license that can be found in the LICENSE file.
|
|
|
|
#include "vm/object_graph_copy.h"
|
|
|
|
#include <memory>
|
|
|
|
#include "vm/dart_api_state.h"
|
|
#include "vm/flags.h"
|
|
#include "vm/heap/weak_table.h"
|
|
#include "vm/longjump.h"
|
|
#include "vm/object.h"
|
|
#include "vm/object_store.h"
|
|
#include "vm/snapshot.h"
|
|
#include "vm/symbols.h"
|
|
#include "vm/timeline.h"
|
|
|
|
#define Z zone_
|
|
|
|
// The list here contains two kinds of classes of objects
|
|
// * objects that will be shared and we will therefore never need to copy
|
|
// * objects that user object graphs should never reference
|
|
#define FOR_UNSUPPORTED_CLASSES(V) \
|
|
V(AbstractType) \
|
|
V(ApiError) \
|
|
V(Bool) \
|
|
V(CallSiteData) \
|
|
V(Capability) \
|
|
V(Class) \
|
|
V(ClosureData) \
|
|
V(Code) \
|
|
V(CodeSourceMap) \
|
|
V(CompressedStackMaps) \
|
|
V(ContextScope) \
|
|
V(DynamicLibrary) \
|
|
V(Error) \
|
|
V(ExceptionHandlers) \
|
|
V(FfiTrampolineData) \
|
|
V(Field) \
|
|
V(Finalizer) \
|
|
V(FinalizerBase) \
|
|
V(FinalizerEntry) \
|
|
V(NativeFinalizer) \
|
|
V(Function) \
|
|
V(FunctionType) \
|
|
V(FutureOr) \
|
|
V(ICData) \
|
|
V(Instance) \
|
|
V(Instructions) \
|
|
V(InstructionsSection) \
|
|
V(InstructionsTable) \
|
|
V(Int32x4) \
|
|
V(Integer) \
|
|
V(KernelProgramInfo) \
|
|
V(LanguageError) \
|
|
V(Library) \
|
|
V(LibraryPrefix) \
|
|
V(LoadingUnit) \
|
|
V(LocalVarDescriptors) \
|
|
V(MegamorphicCache) \
|
|
V(Mint) \
|
|
V(MirrorReference) \
|
|
V(MonomorphicSmiableCall) \
|
|
V(Namespace) \
|
|
V(Number) \
|
|
V(ObjectPool) \
|
|
V(PatchClass) \
|
|
V(PcDescriptors) \
|
|
V(Pointer) \
|
|
V(ReceivePort) \
|
|
V(RecordType) \
|
|
V(RegExp) \
|
|
V(Script) \
|
|
V(Sentinel) \
|
|
V(SendPort) \
|
|
V(SingleTargetCache) \
|
|
V(Smi) \
|
|
V(StackTrace) \
|
|
V(SubtypeTestCache) \
|
|
V(SuspendState) \
|
|
V(Type) \
|
|
V(TypeArguments) \
|
|
V(TypeParameter) \
|
|
V(TypeParameters) \
|
|
V(TypedDataBase) \
|
|
V(UnhandledException) \
|
|
V(UnlinkedCall) \
|
|
V(UnwindError) \
|
|
V(UserTag) \
|
|
V(WeakArray) \
|
|
V(WeakSerializationReference)
|
|
|
|
namespace dart {
|
|
|
|
DEFINE_FLAG(bool,
|
|
enable_fast_object_copy,
|
|
true,
|
|
"Enable fast path for fast object copy.");
|
|
DEFINE_FLAG(bool,
|
|
gc_on_foc_slow_path,
|
|
false,
|
|
"Cause a GC when falling off the fast path for fast object copy.");
|
|
|
|
const char* kFastAllocationFailed = "fast allocation failed";
|
|
|
|
struct PtrTypes {
|
|
using Object = ObjectPtr;
|
|
static const dart::UntaggedObject* UntagObject(Object arg) {
|
|
return arg.untag();
|
|
}
|
|
static const dart::ObjectPtr GetObjectPtr(Object arg) { return arg; }
|
|
static const dart::Object& HandlifyObject(ObjectPtr arg) {
|
|
return dart::Object::Handle(arg);
|
|
}
|
|
|
|
#define DO(V) \
|
|
using V = V##Ptr; \
|
|
static Untagged##V* Untag##V(V##Ptr arg) { return arg.untag(); } \
|
|
static V##Ptr Get##V##Ptr(V##Ptr arg) { return arg; } \
|
|
static V##Ptr Cast##V(ObjectPtr arg) { return dart::V::RawCast(arg); }
|
|
CLASS_LIST_FOR_HANDLES(DO)
|
|
#undef DO
|
|
};
|
|
|
|
struct HandleTypes {
|
|
using Object = const dart::Object&;
|
|
static const dart::UntaggedObject* UntagObject(Object arg) {
|
|
return arg.ptr().untag();
|
|
}
|
|
static dart::ObjectPtr GetObjectPtr(Object arg) { return arg.ptr(); }
|
|
static Object HandlifyObject(Object arg) { return arg; }
|
|
|
|
#define DO(V) \
|
|
using V = const dart::V&; \
|
|
static Untagged##V* Untag##V(V arg) { return arg.ptr().untag(); } \
|
|
static V##Ptr Get##V##Ptr(V arg) { return arg.ptr(); } \
|
|
static V Cast##V(const dart::Object& arg) { return dart::V::Cast(arg); }
|
|
CLASS_LIST_FOR_HANDLES(DO)
|
|
#undef DO
|
|
};
|
|
|
|
DART_FORCE_INLINE
|
|
static ObjectPtr Marker() {
|
|
return Object::unknown_constant().ptr();
|
|
}
|
|
|
|
DART_FORCE_INLINE
|
|
static bool CanShareObject(ObjectPtr obj, uword tags) {
|
|
if ((tags & UntaggedObject::CanonicalBit::mask_in_place()) != 0) {
|
|
return true;
|
|
}
|
|
const auto cid = UntaggedObject::ClassIdTag::decode(tags);
|
|
if ((tags & UntaggedObject::ImmutableBit::mask_in_place()) != 0) {
|
|
if (IsUnmodifiableTypedDataViewClassId(cid)) {
|
|
// Unmodifiable typed data views may have mutable backing stores.
|
|
return TypedDataView::RawCast(obj)
|
|
->untag()
|
|
->typed_data()
|
|
->untag()
|
|
->IsImmutable();
|
|
}
|
|
|
|
// All other objects that have immutability bit set are deeply immutable.
|
|
return true;
|
|
}
|
|
|
|
// TODO(https://dartbug.com/55136): Mark Closures as shallowly imutable.
|
|
// And move this into the if above.
|
|
if (cid == kClosureCid) {
|
|
// We can share a closure iff it doesn't close over any state.
|
|
return Closure::RawCast(obj)->untag()->context() == Object::null();
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
bool CanShareObjectAcrossIsolates(ObjectPtr obj) {
|
|
if (!obj->IsHeapObject()) return true;
|
|
const uword tags = TagsFromUntaggedObject(obj.untag());
|
|
return CanShareObject(obj, tags);
|
|
}
|
|
|
|
// Whether executing `get:hashCode` (possibly in a different isolate) on an
|
|
// object with the given [tags] might return a different answer than the source
|
|
// object (if copying is needed) or on the same object (if the object is
|
|
// shared).
|
|
DART_FORCE_INLINE
|
|
static bool MightNeedReHashing(ObjectPtr object) {
|
|
const uword tags = TagsFromUntaggedObject(object.untag());
|
|
const auto cid = UntaggedObject::ClassIdTag::decode(tags);
|
|
// These use structural hash codes and will therefore always result in the
|
|
// same hash codes.
|
|
if (cid == kOneByteStringCid) return false;
|
|
if (cid == kTwoByteStringCid) return false;
|
|
if (cid == kMintCid) return false;
|
|
if (cid == kDoubleCid) return false;
|
|
if (cid == kBoolCid) return false;
|
|
if (cid == kSendPortCid) return false;
|
|
if (cid == kCapabilityCid) return false;
|
|
if (cid == kNullCid) return false;
|
|
|
|
// These are shared and use identity hash codes. If they are used as a key in
|
|
// a map or a value in a set, they will already have the identity hash code
|
|
// set.
|
|
if (cid == kRegExpCid) return false;
|
|
if (cid == kInt32x4Cid) return false;
|
|
|
|
// If the [tags] indicates this is a canonical object we'll share it instead
|
|
// of copying it. That would suggest we don't have to re-hash maps/sets
|
|
// containing this object on the receiver side.
|
|
//
|
|
// Though the object can be a constant of a user-defined class with a
|
|
// custom hash code that is misbehaving (e.g one that depends on global field
|
|
// state, ...). To be on the safe side we'll force re-hashing if such objects
|
|
// are encountered in maps/sets.
|
|
//
|
|
// => We might want to consider changing the implementation to avoid rehashing
|
|
// in such cases in the future and disambiguate the documentation.
|
|
return true;
|
|
}
|
|
|
|
DART_FORCE_INLINE
|
|
uword TagsFromUntaggedObject(UntaggedObject* obj) {
|
|
return obj->tags_;
|
|
}
|
|
|
|
DART_FORCE_INLINE
|
|
void SetNewSpaceTaggingWord(ObjectPtr to, classid_t cid, uint32_t size) {
|
|
uword tags = 0;
|
|
|
|
tags = UntaggedObject::SizeTag::update(size, tags);
|
|
tags = UntaggedObject::ClassIdTag::update(cid, tags);
|
|
tags = UntaggedObject::AlwaysSetBit::update(true, tags);
|
|
tags = UntaggedObject::NotMarkedBit::update(true, tags);
|
|
tags = UntaggedObject::OldAndNotRememberedBit::update(false, tags);
|
|
tags = UntaggedObject::CanonicalBit::update(false, tags);
|
|
tags = UntaggedObject::NewOrEvacuationCandidateBit::update(true, tags);
|
|
tags = UntaggedObject::ImmutableBit::update(
|
|
IsUnmodifiableTypedDataViewClassId(cid), tags);
|
|
#if defined(HASH_IN_OBJECT_HEADER)
|
|
tags = UntaggedObject::HashTag::update(0, tags);
|
|
#endif
|
|
to.untag()->tags_ = tags;
|
|
}
|
|
|
|
DART_FORCE_INLINE
|
|
ObjectPtr AllocateObject(intptr_t cid,
|
|
intptr_t size,
|
|
intptr_t allocated_bytes) {
|
|
#if defined(DART_COMPRESSED_POINTERS)
|
|
const bool compressed = true;
|
|
#else
|
|
const bool compressed = false;
|
|
#endif
|
|
const intptr_t kLargeMessageThreshold = 16 * MB;
|
|
const Heap::Space space =
|
|
allocated_bytes > kLargeMessageThreshold ? Heap::kOld : Heap::kNew;
|
|
// Mimic the old initialization behavior of Object::InitializeObject where
|
|
// the contents are initialized to Object::null(), except for TypedDataBase
|
|
// subclasses which are initialized to 0, as the contents of the original
|
|
// are translated and copied over prior to returning the object graph root.
|
|
if (IsTypedDataBaseClassId(cid)) {
|
|
return Object::Allocate(cid, size, space, compressed,
|
|
Object::from_offset<TypedDataBase>(),
|
|
Object::to_offset<TypedDataBase>());
|
|
|
|
} else {
|
|
// Remember that ptr_field_end_offset is the offset to the last Ptr
|
|
// field, not the offset just past it.
|
|
const uword ptr_field_end_offset =
|
|
size - (compressed ? kCompressedWordSize : kWordSize);
|
|
return Object::Allocate(cid, size, space, compressed,
|
|
Object::from_offset<Object>(),
|
|
ptr_field_end_offset);
|
|
}
|
|
}
|
|
|
|
DART_FORCE_INLINE
|
|
void UpdateLengthField(intptr_t cid, ObjectPtr from, ObjectPtr to) {
|
|
// We share these objects - never copy them.
|
|
ASSERT(!IsStringClassId(cid));
|
|
|
|
// We update any in-heap variable sized object with the length to keep the
|
|
// length and the size in the object header in-sync for the GC.
|
|
if (cid == kArrayCid || cid == kImmutableArrayCid) {
|
|
static_cast<UntaggedArray*>(to.untag())->length_ =
|
|
static_cast<UntaggedArray*>(from.untag())->length_;
|
|
} else if (cid == kContextCid) {
|
|
static_cast<UntaggedContext*>(to.untag())->num_variables_ =
|
|
static_cast<UntaggedContext*>(from.untag())->num_variables_;
|
|
} else if (IsTypedDataClassId(cid)) {
|
|
static_cast<UntaggedTypedDataBase*>(to.untag())->length_ =
|
|
static_cast<UntaggedTypedDataBase*>(from.untag())->length_;
|
|
} else if (cid == kRecordCid) {
|
|
static_cast<UntaggedRecord*>(to.untag())->shape_ =
|
|
static_cast<UntaggedRecord*>(from.untag())->shape_;
|
|
}
|
|
}
|
|
|
|
void InitializeExternalTypedData(intptr_t cid,
|
|
ExternalTypedDataPtr from,
|
|
ExternalTypedDataPtr to) {
|
|
auto raw_from = from.untag();
|
|
auto raw_to = to.untag();
|
|
const intptr_t length =
|
|
TypedData::ElementSizeInBytes(cid) * Smi::Value(raw_from->length_);
|
|
|
|
auto buffer = static_cast<uint8_t*>(malloc(length));
|
|
memmove(buffer, raw_from->data_, length);
|
|
raw_to->length_ = raw_from->length_;
|
|
raw_to->data_ = buffer;
|
|
}
|
|
|
|
template <typename T>
|
|
void CopyTypedDataBaseWithSafepointChecks(Thread* thread,
|
|
const T& from,
|
|
const T& to,
|
|
intptr_t length) {
|
|
constexpr intptr_t kChunkSize = 100 * 1024;
|
|
|
|
const intptr_t chunks = length / kChunkSize;
|
|
const intptr_t remainder = length % kChunkSize;
|
|
|
|
// Notice we re-load the data pointer, since T may be TypedData in which case
|
|
// the interior pointer may change after checking into safepoints.
|
|
for (intptr_t i = 0; i < chunks; ++i) {
|
|
memmove(to.ptr().untag()->data_ + i * kChunkSize,
|
|
from.ptr().untag()->data_ + i * kChunkSize, kChunkSize);
|
|
|
|
thread->CheckForSafepoint();
|
|
}
|
|
if (remainder > 0) {
|
|
memmove(to.ptr().untag()->data_ + chunks * kChunkSize,
|
|
from.ptr().untag()->data_ + chunks * kChunkSize, remainder);
|
|
}
|
|
}
|
|
|
|
void InitializeExternalTypedDataWithSafepointChecks(
|
|
Thread* thread,
|
|
intptr_t cid,
|
|
const ExternalTypedData& from,
|
|
const ExternalTypedData& to) {
|
|
const intptr_t length_in_elements = from.Length();
|
|
const intptr_t length_in_bytes =
|
|
TypedData::ElementSizeInBytes(cid) * length_in_elements;
|
|
|
|
uint8_t* to_data = static_cast<uint8_t*>(malloc(length_in_bytes));
|
|
to.ptr().untag()->data_ = to_data;
|
|
to.ptr().untag()->length_ = Smi::New(length_in_elements);
|
|
|
|
CopyTypedDataBaseWithSafepointChecks(thread, from, to, length_in_bytes);
|
|
}
|
|
|
|
void InitializeTypedDataView(TypedDataViewPtr obj) {
|
|
obj.untag()->typed_data_ = TypedDataBase::null();
|
|
obj.untag()->offset_in_bytes_ = Smi::New(0);
|
|
obj.untag()->length_ = Smi::New(0);
|
|
}
|
|
|
|
void FreeExternalTypedData(void* isolate_callback_data, void* buffer) {
|
|
free(buffer);
|
|
}
|
|
|
|
void FreeTransferablePeer(void* isolate_callback_data, void* peer) {
|
|
delete static_cast<TransferableTypedDataPeer*>(peer);
|
|
}
|
|
|
|
class SlowFromTo {
|
|
public:
|
|
explicit SlowFromTo(const GrowableObjectArray& storage) : storage_(storage) {}
|
|
|
|
ObjectPtr At(intptr_t index) { return storage_.At(index); }
|
|
void Add(const Object& key, const Object& value) {
|
|
storage_.Add(key);
|
|
storage_.Add(value);
|
|
}
|
|
intptr_t Length() { return storage_.Length(); }
|
|
|
|
private:
|
|
const GrowableObjectArray& storage_;
|
|
};
|
|
|
|
class FastFromTo {
|
|
public:
|
|
explicit FastFromTo(GrowableArray<ObjectPtr>& storage) : storage_(storage) {}
|
|
|
|
ObjectPtr At(intptr_t index) { return storage_.At(index); }
|
|
void Add(ObjectPtr key, ObjectPtr value) {
|
|
intptr_t i = storage_.length();
|
|
storage_.Resize(i + 2);
|
|
storage_[i + 0] = key;
|
|
storage_[i + 1] = value;
|
|
}
|
|
intptr_t Length() { return storage_.length(); }
|
|
|
|
private:
|
|
GrowableArray<ObjectPtr>& storage_;
|
|
};
|
|
|
|
static ObjectPtr Ptr(ObjectPtr obj) {
|
|
return obj;
|
|
}
|
|
static ObjectPtr Ptr(const Object& obj) {
|
|
return obj.ptr();
|
|
}
|
|
|
|
#if defined(HASH_IN_OBJECT_HEADER)
|
|
class IdentityMap {
|
|
public:
|
|
explicit IdentityMap(Thread* thread) : thread_(thread) {
|
|
hash_table_used_ = 0;
|
|
hash_table_capacity_ = 32;
|
|
hash_table_ = reinterpret_cast<uint32_t*>(
|
|
malloc(hash_table_capacity_ * sizeof(uint32_t)));
|
|
memset(hash_table_, 0, hash_table_capacity_ * sizeof(uint32_t));
|
|
}
|
|
~IdentityMap() { free(hash_table_); }
|
|
|
|
template <typename S, typename T>
|
|
DART_FORCE_INLINE ObjectPtr ForwardedObject(const S& object, T from_to) {
|
|
intptr_t mask = hash_table_capacity_ - 1;
|
|
intptr_t probe = GetHeaderHash(Ptr(object)) & mask;
|
|
for (;;) {
|
|
intptr_t index = hash_table_[probe];
|
|
if (index == 0) {
|
|
return Marker();
|
|
}
|
|
if (from_to.At(index) == Ptr(object)) {
|
|
return from_to.At(index + 1);
|
|
}
|
|
probe = (probe + 1) & mask;
|
|
}
|
|
}
|
|
|
|
template <typename S, typename T>
|
|
DART_FORCE_INLINE void Insert(const S& from,
|
|
const S& to,
|
|
T from_to,
|
|
bool check_for_safepoint) {
|
|
ASSERT(ForwardedObject(from, from_to) == Marker());
|
|
const auto id = from_to.Length();
|
|
from_to.Add(from, to); // Must occur before rehashing.
|
|
intptr_t mask = hash_table_capacity_ - 1;
|
|
intptr_t probe = GetHeaderHash(Ptr(from)) & mask;
|
|
for (;;) {
|
|
intptr_t index = hash_table_[probe];
|
|
if (index == 0) {
|
|
hash_table_[probe] = id;
|
|
break;
|
|
}
|
|
probe = (probe + 1) & mask;
|
|
}
|
|
hash_table_used_++;
|
|
if (hash_table_used_ * 2 > hash_table_capacity_) {
|
|
Rehash(hash_table_capacity_ * 2, from_to, check_for_safepoint);
|
|
}
|
|
}
|
|
|
|
private:
|
|
DART_FORCE_INLINE
|
|
uint32_t GetHeaderHash(ObjectPtr object) {
|
|
uint32_t hash = Object::GetCachedHash(object);
|
|
if (hash == 0) {
|
|
switch (object->GetClassId()) {
|
|
case kMintCid:
|
|
hash = Mint::Value(static_cast<MintPtr>(object));
|
|
// Don't write back: doesn't agree with dart:core's identityHash.
|
|
break;
|
|
case kDoubleCid:
|
|
hash =
|
|
bit_cast<uint64_t>(Double::Value(static_cast<DoublePtr>(object)));
|
|
// Don't write back: doesn't agree with dart:core's identityHash.
|
|
break;
|
|
case kOneByteStringCid:
|
|
case kTwoByteStringCid:
|
|
hash = String::Hash(static_cast<StringPtr>(object));
|
|
hash = Object::SetCachedHashIfNotSet(object, hash);
|
|
break;
|
|
default:
|
|
do {
|
|
hash = thread_->random()->NextUInt32();
|
|
} while (hash == 0 || !Smi::IsValid(hash));
|
|
hash = Object::SetCachedHashIfNotSet(object, hash);
|
|
break;
|
|
}
|
|
}
|
|
return hash;
|
|
}
|
|
|
|
template <typename T>
|
|
void Rehash(intptr_t new_capacity, T from_to, bool check_for_safepoint) {
|
|
hash_table_capacity_ = new_capacity;
|
|
hash_table_used_ = 0;
|
|
free(hash_table_);
|
|
hash_table_ = reinterpret_cast<uint32_t*>(
|
|
malloc(hash_table_capacity_ * sizeof(uint32_t)));
|
|
for (intptr_t i = 0; i < hash_table_capacity_; i++) {
|
|
hash_table_[i] = 0;
|
|
if (check_for_safepoint && (((i + 1) % kSlotsPerInterruptCheck) == 0)) {
|
|
thread_->CheckForSafepoint();
|
|
}
|
|
}
|
|
for (intptr_t id = 2; id < from_to.Length(); id += 2) {
|
|
ObjectPtr obj = from_to.At(id);
|
|
intptr_t mask = hash_table_capacity_ - 1;
|
|
intptr_t probe = GetHeaderHash(obj) & mask;
|
|
for (;;) {
|
|
if (hash_table_[probe] == 0) {
|
|
hash_table_[probe] = id;
|
|
hash_table_used_++;
|
|
break;
|
|
}
|
|
probe = (probe + 1) & mask;
|
|
}
|
|
if (check_for_safepoint && (((id + 2) % kSlotsPerInterruptCheck) == 0)) {
|
|
thread_->CheckForSafepoint();
|
|
}
|
|
}
|
|
}
|
|
|
|
Thread* thread_;
|
|
uint32_t* hash_table_;
|
|
uint32_t hash_table_capacity_;
|
|
uint32_t hash_table_used_;
|
|
};
|
|
#else // defined(HASH_IN_OBJECT_HEADER)
|
|
class IdentityMap {
|
|
public:
|
|
explicit IdentityMap(Thread* thread) : isolate_(thread->isolate()) {
|
|
isolate_->set_forward_table_new(new WeakTable());
|
|
isolate_->set_forward_table_old(new WeakTable());
|
|
}
|
|
~IdentityMap() {
|
|
isolate_->set_forward_table_new(nullptr);
|
|
isolate_->set_forward_table_old(nullptr);
|
|
}
|
|
|
|
template <typename S, typename T>
|
|
DART_FORCE_INLINE ObjectPtr ForwardedObject(const S& object, T from_to) {
|
|
const intptr_t id = GetObjectId(Ptr(object));
|
|
if (id == 0) return Marker();
|
|
return from_to.At(id + 1);
|
|
}
|
|
|
|
template <typename S, typename T>
|
|
DART_FORCE_INLINE void Insert(const S& from,
|
|
const S& to,
|
|
T from_to,
|
|
bool check_for_safepoint) {
|
|
ASSERT(ForwardedObject(from, from_to) == Marker());
|
|
const auto id = from_to.Length();
|
|
// May take >100ms and cannot yield to safepoints.
|
|
SetObjectId(Ptr(from), id);
|
|
from_to.Add(from, to);
|
|
}
|
|
|
|
private:
|
|
DART_FORCE_INLINE
|
|
intptr_t GetObjectId(ObjectPtr object) {
|
|
if (object->IsNewObject()) {
|
|
return isolate_->forward_table_new()->GetValueExclusive(object);
|
|
} else {
|
|
return isolate_->forward_table_old()->GetValueExclusive(object);
|
|
}
|
|
}
|
|
|
|
DART_FORCE_INLINE
|
|
void SetObjectId(ObjectPtr object, intptr_t id) {
|
|
if (object->IsNewObject()) {
|
|
isolate_->forward_table_new()->SetValueExclusive(object, id);
|
|
} else {
|
|
isolate_->forward_table_old()->SetValueExclusive(object, id);
|
|
}
|
|
}
|
|
|
|
Isolate* isolate_;
|
|
};
|
|
#endif // defined(HASH_IN_OBJECT_HEADER)
|
|
|
|
class ForwardMapBase {
|
|
public:
|
|
explicit ForwardMapBase(Thread* thread)
|
|
: thread_(thread), zone_(thread->zone()) {}
|
|
|
|
protected:
|
|
friend class ObjectGraphCopier;
|
|
|
|
void FinalizeTransferable(const TransferableTypedData& from,
|
|
const TransferableTypedData& to) {
|
|
// Get the old peer.
|
|
auto fpeer = static_cast<TransferableTypedDataPeer*>(
|
|
thread_->heap()->GetPeer(from.ptr()));
|
|
ASSERT(fpeer != nullptr && fpeer->data() != nullptr);
|
|
const intptr_t length = fpeer->length();
|
|
|
|
// Allocate new peer object with (data, length).
|
|
auto tpeer = new TransferableTypedDataPeer(fpeer->data(), length);
|
|
thread_->heap()->SetPeer(to.ptr(), tpeer);
|
|
|
|
// Move the handle itself to the new object.
|
|
fpeer->handle()->EnsureFreedExternal(thread_->isolate_group());
|
|
FinalizablePersistentHandle* finalizable_ref =
|
|
FinalizablePersistentHandle::New(thread_->isolate_group(), to, tpeer,
|
|
FreeTransferablePeer, length,
|
|
/*auto_delete=*/true);
|
|
ASSERT(finalizable_ref != nullptr);
|
|
tpeer->set_handle(finalizable_ref);
|
|
fpeer->ClearData();
|
|
}
|
|
|
|
void FinalizeExternalTypedData(const ExternalTypedData& to) {
|
|
to.AddFinalizer(to.DataAddr(0), &FreeExternalTypedData, to.LengthInBytes());
|
|
}
|
|
|
|
Thread* thread_;
|
|
Zone* zone_;
|
|
|
|
private:
|
|
DISALLOW_COPY_AND_ASSIGN(ForwardMapBase);
|
|
};
|
|
|
|
class FastForwardMap : public ForwardMapBase {
|
|
public:
|
|
explicit FastForwardMap(Thread* thread, IdentityMap* map)
|
|
: ForwardMapBase(thread),
|
|
map_(map),
|
|
raw_from_to_(thread->zone(), 20),
|
|
raw_transferables_from_to_(thread->zone(), 0),
|
|
raw_objects_to_rehash_(thread->zone(), 0),
|
|
raw_expandos_to_rehash_(thread->zone(), 0) {
|
|
raw_from_to_.Resize(2);
|
|
raw_from_to_[0] = Object::null();
|
|
raw_from_to_[1] = Object::null();
|
|
fill_cursor_ = 2;
|
|
}
|
|
|
|
ObjectPtr ForwardedObject(ObjectPtr object) {
|
|
return map_->ForwardedObject(object, FastFromTo(raw_from_to_));
|
|
}
|
|
|
|
void Insert(ObjectPtr from, ObjectPtr to, intptr_t size) {
|
|
map_->Insert(from, to, FastFromTo(raw_from_to_),
|
|
/*check_for_safepoint*/ false);
|
|
allocated_bytes += size;
|
|
}
|
|
|
|
void AddTransferable(TransferableTypedDataPtr from,
|
|
TransferableTypedDataPtr to) {
|
|
raw_transferables_from_to_.Add(from);
|
|
raw_transferables_from_to_.Add(to);
|
|
}
|
|
void AddWeakProperty(WeakPropertyPtr from) { raw_weak_properties_.Add(from); }
|
|
void AddWeakReference(WeakReferencePtr from) {
|
|
raw_weak_references_.Add(from);
|
|
}
|
|
void AddExternalTypedData(ExternalTypedDataPtr to) {
|
|
raw_external_typed_data_to_.Add(to);
|
|
}
|
|
|
|
void AddObjectToRehash(ObjectPtr to) { raw_objects_to_rehash_.Add(to); }
|
|
void AddExpandoToRehash(ObjectPtr to) { raw_expandos_to_rehash_.Add(to); }
|
|
|
|
private:
|
|
friend class FastObjectCopy;
|
|
friend class ObjectGraphCopier;
|
|
|
|
IdentityMap* map_;
|
|
GrowableArray<ObjectPtr> raw_from_to_;
|
|
GrowableArray<TransferableTypedDataPtr> raw_transferables_from_to_;
|
|
GrowableArray<ExternalTypedDataPtr> raw_external_typed_data_to_;
|
|
GrowableArray<ObjectPtr> raw_objects_to_rehash_;
|
|
GrowableArray<ObjectPtr> raw_expandos_to_rehash_;
|
|
GrowableArray<WeakPropertyPtr> raw_weak_properties_;
|
|
GrowableArray<WeakReferencePtr> raw_weak_references_;
|
|
intptr_t fill_cursor_ = 0;
|
|
intptr_t allocated_bytes = 0;
|
|
|
|
DISALLOW_COPY_AND_ASSIGN(FastForwardMap);
|
|
};
|
|
|
|
class SlowForwardMap : public ForwardMapBase {
|
|
public:
|
|
explicit SlowForwardMap(Thread* thread, IdentityMap* map)
|
|
: ForwardMapBase(thread),
|
|
map_(map),
|
|
from_to_transition_(thread->zone(), 2),
|
|
from_to_(GrowableObjectArray::Handle(thread->zone(),
|
|
GrowableObjectArray::New(2))),
|
|
transferables_from_to_(thread->zone(), 0) {
|
|
from_to_transition_.Resize(2);
|
|
from_to_transition_[0] = &PassiveObject::Handle();
|
|
from_to_transition_[1] = &PassiveObject::Handle();
|
|
from_to_.Add(Object::null_object());
|
|
from_to_.Add(Object::null_object());
|
|
fill_cursor_ = 2;
|
|
}
|
|
|
|
ObjectPtr ForwardedObject(ObjectPtr object) {
|
|
return map_->ForwardedObject(object, SlowFromTo(from_to_));
|
|
}
|
|
void Insert(const Object& from, const Object& to, intptr_t size) {
|
|
map_->Insert(from, to, SlowFromTo(from_to_),
|
|
/* check_for_safepoint */ true);
|
|
allocated_bytes += size;
|
|
}
|
|
|
|
void AddTransferable(const TransferableTypedData& from,
|
|
const TransferableTypedData& to) {
|
|
transferables_from_to_.Add(&TransferableTypedData::Handle(from.ptr()));
|
|
transferables_from_to_.Add(&TransferableTypedData::Handle(to.ptr()));
|
|
}
|
|
void AddWeakProperty(const WeakProperty& from) {
|
|
weak_properties_.Add(&WeakProperty::Handle(from.ptr()));
|
|
}
|
|
void AddWeakReference(const WeakReference& from) {
|
|
weak_references_.Add(&WeakReference::Handle(from.ptr()));
|
|
}
|
|
const ExternalTypedData& AddExternalTypedData(ExternalTypedDataPtr to) {
|
|
auto to_handle = &ExternalTypedData::Handle(to);
|
|
external_typed_data_.Add(to_handle);
|
|
return *to_handle;
|
|
}
|
|
void AddObjectToRehash(const Object& to) {
|
|
objects_to_rehash_.Add(&Object::Handle(to.ptr()));
|
|
}
|
|
void AddExpandoToRehash(const Object& to) {
|
|
expandos_to_rehash_.Add(&Object::Handle(to.ptr()));
|
|
}
|
|
|
|
void FinalizeTransferables() {
|
|
for (intptr_t i = 0; i < transferables_from_to_.length(); i += 2) {
|
|
auto from = transferables_from_to_[i];
|
|
auto to = transferables_from_to_[i + 1];
|
|
FinalizeTransferable(*from, *to);
|
|
}
|
|
}
|
|
|
|
void FinalizeExternalTypedData() {
|
|
for (intptr_t i = 0; i < external_typed_data_.length(); i++) {
|
|
auto to = external_typed_data_[i];
|
|
ForwardMapBase::FinalizeExternalTypedData(*to);
|
|
}
|
|
}
|
|
|
|
private:
|
|
friend class SlowObjectCopy;
|
|
friend class SlowObjectCopyBase;
|
|
friend class ObjectGraphCopier;
|
|
|
|
IdentityMap* map_;
|
|
GrowableArray<const PassiveObject*> from_to_transition_;
|
|
GrowableObjectArray& from_to_;
|
|
GrowableArray<const TransferableTypedData*> transferables_from_to_;
|
|
GrowableArray<const ExternalTypedData*> external_typed_data_;
|
|
GrowableArray<const Object*> objects_to_rehash_;
|
|
GrowableArray<const Object*> expandos_to_rehash_;
|
|
GrowableArray<const WeakProperty*> weak_properties_;
|
|
GrowableArray<const WeakReference*> weak_references_;
|
|
intptr_t fill_cursor_ = 0;
|
|
intptr_t allocated_bytes = 0;
|
|
|
|
DISALLOW_COPY_AND_ASSIGN(SlowForwardMap);
|
|
};
|
|
|
|
class ObjectCopyBase {
|
|
public:
|
|
explicit ObjectCopyBase(Thread* thread)
|
|
: thread_(thread),
|
|
heap_base_(thread->heap_base()),
|
|
zone_(thread->zone()),
|
|
heap_(thread->isolate_group()->heap()),
|
|
class_table_(thread->isolate_group()->class_table()),
|
|
new_space_(heap_->new_space()),
|
|
tmp_(Object::Handle(thread->zone())),
|
|
to_(Object::Handle(thread->zone())),
|
|
expando_cid_(Class::GetClassId(
|
|
thread->isolate_group()->object_store()->expando_class())),
|
|
exception_unexpected_object_(Object::Handle(thread->zone())) {}
|
|
~ObjectCopyBase() {}
|
|
|
|
protected:
|
|
static ObjectPtr LoadPointer(ObjectPtr src, intptr_t offset) {
|
|
return src.untag()->LoadPointer(reinterpret_cast<ObjectPtr*>(
|
|
reinterpret_cast<uint8_t*>(src.untag()) + offset));
|
|
}
|
|
static CompressedObjectPtr LoadCompressedPointer(ObjectPtr src,
|
|
intptr_t offset) {
|
|
return src.untag()->LoadPointer(reinterpret_cast<CompressedObjectPtr*>(
|
|
reinterpret_cast<uint8_t*>(src.untag()) + offset));
|
|
}
|
|
static compressed_uword LoadCompressedNonPointerWord(ObjectPtr src,
|
|
intptr_t offset) {
|
|
return *reinterpret_cast<compressed_uword*>(
|
|
reinterpret_cast<uint8_t*>(src.untag()) + offset);
|
|
}
|
|
static void StorePointerBarrier(ObjectPtr obj,
|
|
intptr_t offset,
|
|
ObjectPtr value) {
|
|
obj.untag()->StorePointer(
|
|
reinterpret_cast<ObjectPtr*>(reinterpret_cast<uint8_t*>(obj.untag()) +
|
|
offset),
|
|
value);
|
|
}
|
|
static void StoreCompressedPointerBarrier(ObjectPtr obj,
|
|
intptr_t offset,
|
|
ObjectPtr value) {
|
|
obj.untag()->StoreCompressedPointer(
|
|
reinterpret_cast<CompressedObjectPtr*>(
|
|
reinterpret_cast<uint8_t*>(obj.untag()) + offset),
|
|
value);
|
|
}
|
|
void StoreCompressedLargeArrayPointerBarrier(ObjectPtr obj,
|
|
intptr_t offset,
|
|
ObjectPtr value) {
|
|
obj.untag()->StoreCompressedArrayPointer(
|
|
reinterpret_cast<CompressedObjectPtr*>(
|
|
reinterpret_cast<uint8_t*>(obj.untag()) + offset),
|
|
value, thread_);
|
|
}
|
|
static void StorePointerNoBarrier(ObjectPtr obj,
|
|
intptr_t offset,
|
|
ObjectPtr value) {
|
|
*reinterpret_cast<ObjectPtr*>(reinterpret_cast<uint8_t*>(obj.untag()) +
|
|
offset) = value;
|
|
}
|
|
template <typename T = ObjectPtr>
|
|
static void StoreCompressedPointerNoBarrier(ObjectPtr obj,
|
|
intptr_t offset,
|
|
T value) {
|
|
*reinterpret_cast<CompressedObjectPtr*>(
|
|
reinterpret_cast<uint8_t*>(obj.untag()) + offset) = value;
|
|
}
|
|
static void StoreCompressedNonPointerWord(ObjectPtr obj,
|
|
intptr_t offset,
|
|
compressed_uword value) {
|
|
*reinterpret_cast<compressed_uword*>(
|
|
reinterpret_cast<uint8_t*>(obj.untag()) + offset) = value;
|
|
}
|
|
|
|
DART_FORCE_INLINE
|
|
bool CanCopyObject(uword tags, ObjectPtr object) {
|
|
const auto cid = UntaggedObject::ClassIdTag::decode(tags);
|
|
if (Class::IsIsolateUnsendable(class_table_->At(cid))) {
|
|
exception_msg_ = OS::SCreate(
|
|
zone_,
|
|
"Illegal argument in isolate message: object is unsendable - %s ("
|
|
"see restrictions listed at `SendPort.send()` documentation "
|
|
"for more information)",
|
|
Class::Handle(class_table_->At(cid)).ToCString());
|
|
exception_unexpected_object_ = object;
|
|
return false;
|
|
}
|
|
if (cid > kNumPredefinedCids) {
|
|
return true;
|
|
}
|
|
#define HANDLE_ILLEGAL_CASE(Type) \
|
|
case k##Type##Cid: { \
|
|
exception_msg_ = \
|
|
"Illegal argument in isolate message: " \
|
|
"(object is a " #Type ")"; \
|
|
exception_unexpected_object_ = object; \
|
|
return false; \
|
|
}
|
|
|
|
switch (cid) {
|
|
// From "dart:ffi" we handle only Pointer/DynamicLibrary specially, since
|
|
// those are the only non-abstract classes (so we avoid checking more cids
|
|
// here that cannot happen in reality)
|
|
HANDLE_ILLEGAL_CASE(DynamicLibrary)
|
|
HANDLE_ILLEGAL_CASE(Finalizer)
|
|
HANDLE_ILLEGAL_CASE(NativeFinalizer)
|
|
HANDLE_ILLEGAL_CASE(MirrorReference)
|
|
HANDLE_ILLEGAL_CASE(Pointer)
|
|
HANDLE_ILLEGAL_CASE(ReceivePort)
|
|
HANDLE_ILLEGAL_CASE(SuspendState)
|
|
HANDLE_ILLEGAL_CASE(UserTag)
|
|
default:
|
|
return true;
|
|
}
|
|
}
|
|
|
|
Thread* thread_;
|
|
uword heap_base_;
|
|
Zone* zone_;
|
|
Heap* heap_;
|
|
ClassTable* class_table_;
|
|
Scavenger* new_space_;
|
|
Object& tmp_;
|
|
Object& to_;
|
|
intptr_t expando_cid_;
|
|
|
|
const char* exception_msg_ = nullptr;
|
|
Object& exception_unexpected_object_;
|
|
};
|
|
|
|
class RetainingPath {
|
|
class Visitor : public ObjectPointerVisitor {
|
|
public:
|
|
Visitor(IsolateGroup* isolate_group,
|
|
RetainingPath* retaining_path,
|
|
MallocGrowableArray<ObjectPtr>* const working_list,
|
|
TraversalRules traversal_rules)
|
|
: ObjectPointerVisitor(isolate_group),
|
|
retaining_path_(retaining_path),
|
|
working_list_(working_list),
|
|
traversal_rules_(traversal_rules) {}
|
|
|
|
void VisitObject(ObjectPtr obj) {
|
|
if (!obj->IsHeapObject()) {
|
|
return;
|
|
}
|
|
// Skip canonical objects when rules are for messages internal to
|
|
// an isolate group. Otherwise, need to inspect canonical objects
|
|
// as well.
|
|
if (traversal_rules_ == TraversalRules::kInternalToIsolateGroup &&
|
|
obj->untag()->IsCanonical()) {
|
|
return;
|
|
}
|
|
if (retaining_path_->WasVisited(obj)) {
|
|
return;
|
|
}
|
|
retaining_path_->MarkVisited(obj);
|
|
working_list_->Add(obj);
|
|
}
|
|
|
|
void VisitPointers(ObjectPtr* from, ObjectPtr* to) override {
|
|
for (ObjectPtr* ptr = from; ptr <= to; ptr++) {
|
|
VisitObject(*ptr);
|
|
}
|
|
}
|
|
|
|
#if defined(DART_COMPRESSED_POINTERS)
|
|
void VisitCompressedPointers(uword heap_base,
|
|
CompressedObjectPtr* from,
|
|
CompressedObjectPtr* to) override {
|
|
for (CompressedObjectPtr* ptr = from; ptr <= to; ptr++) {
|
|
VisitObject(ptr->Decompress(heap_base));
|
|
}
|
|
}
|
|
#endif
|
|
|
|
RetainingPath* retaining_path_;
|
|
MallocGrowableArray<ObjectPtr>* const working_list_;
|
|
TraversalRules traversal_rules_;
|
|
};
|
|
|
|
public:
|
|
RetainingPath(Zone* zone,
|
|
Isolate* isolate,
|
|
const Object& from,
|
|
const Object& to,
|
|
TraversalRules traversal_rules)
|
|
: zone_(zone),
|
|
isolate_(isolate),
|
|
from_(from),
|
|
to_(to),
|
|
traversal_rules_(traversal_rules) {
|
|
isolate_->set_forward_table_new(new WeakTable());
|
|
isolate_->set_forward_table_old(new WeakTable());
|
|
}
|
|
|
|
~RetainingPath() {
|
|
isolate_->set_forward_table_new(nullptr);
|
|
isolate_->set_forward_table_old(nullptr);
|
|
}
|
|
|
|
bool WasVisited(ObjectPtr object) {
|
|
if (object->IsNewObject()) {
|
|
return isolate_->forward_table_new()->GetValueExclusive(object) != 0;
|
|
} else {
|
|
return isolate_->forward_table_old()->GetValueExclusive(object) != 0;
|
|
}
|
|
}
|
|
|
|
void MarkVisited(ObjectPtr object) {
|
|
if (object->IsNewObject()) {
|
|
isolate_->forward_table_new()->SetValueExclusive(object, 1);
|
|
} else {
|
|
isolate_->forward_table_old()->SetValueExclusive(object, 1);
|
|
}
|
|
}
|
|
|
|
const char* FindPath() {
|
|
MallocGrowableArray<ObjectPtr>* const working_list =
|
|
isolate_->pointers_to_verify_at_exit();
|
|
ASSERT(working_list->length() == 0);
|
|
|
|
Visitor visitor(isolate_->group(), this, working_list, traversal_rules_);
|
|
|
|
MarkVisited(from_.ptr());
|
|
working_list->Add(from_.ptr());
|
|
|
|
Thread* thread = Thread::Current();
|
|
ClassTable* class_table = isolate_->group()->class_table();
|
|
Closure& closure = Closure::Handle(zone_);
|
|
Array& array = Array::Handle(zone_);
|
|
Class& klass = Class::Handle(zone_);
|
|
|
|
while (!working_list->is_empty()) {
|
|
thread->CheckForSafepoint();
|
|
|
|
// Keep node in the list, separated by null value so that
|
|
// if we are to add children, children can find it in case
|
|
// they are on retaining path.
|
|
ObjectPtr raw = working_list->Last();
|
|
if (raw == Object::null()) {
|
|
// If all children of a node were processed, then skip the separator,
|
|
working_list->RemoveLast();
|
|
// then skip the parent since it has already been processed too.
|
|
working_list->RemoveLast();
|
|
continue;
|
|
}
|
|
|
|
if (raw == to_.ptr()) {
|
|
return CollectPath(working_list);
|
|
}
|
|
|
|
// Separator null object indicates children goes next in the working_list
|
|
working_list->Add(Object::null());
|
|
int length = working_list->length();
|
|
|
|
do { // This loop is here so that we can skip children processing
|
|
const intptr_t cid = raw->GetClassId();
|
|
|
|
if (traversal_rules_ == TraversalRules::kInternalToIsolateGroup) {
|
|
if (CanShareObjectAcrossIsolates(raw)) {
|
|
break;
|
|
}
|
|
if (cid == kClosureCid) {
|
|
closure ^= raw;
|
|
// Only context has to be checked.
|
|
working_list->Add(closure.RawContext());
|
|
break;
|
|
}
|
|
// These we are not expected to drill into as they can't be on
|
|
// retaining path, they are illegal to send.
|
|
klass = class_table->At(cid);
|
|
if (klass.is_isolate_unsendable()) {
|
|
break;
|
|
}
|
|
} else {
|
|
ASSERT(traversal_rules_ ==
|
|
TraversalRules::kExternalBetweenIsolateGroups);
|
|
// Skip classes that are illegal to send across isolate groups.
|
|
// (keep the list in sync with message_snapshot.cc)
|
|
bool skip = false;
|
|
switch (cid) {
|
|
case kClosureCid:
|
|
case kFinalizerCid:
|
|
case kFinalizerEntryCid:
|
|
case kFunctionTypeCid:
|
|
case kMirrorReferenceCid:
|
|
case kNativeFinalizerCid:
|
|
case kReceivePortCid:
|
|
case kRecordCid:
|
|
case kRecordTypeCid:
|
|
case kRegExpCid:
|
|
case kStackTraceCid:
|
|
case kSuspendStateCid:
|
|
case kUserTagCid:
|
|
case kWeakPropertyCid:
|
|
case kWeakReferenceCid:
|
|
case kWeakArrayCid:
|
|
case kDynamicLibraryCid:
|
|
case kPointerCid:
|
|
case kInstanceCid:
|
|
skip = true;
|
|
break;
|
|
default:
|
|
if (cid >= kNumPredefinedCids) {
|
|
skip = true;
|
|
}
|
|
}
|
|
if (skip) {
|
|
break;
|
|
}
|
|
}
|
|
if (cid == kArrayCid) {
|
|
array ^= Array::RawCast(raw);
|
|
visitor.VisitObject(array.GetTypeArguments());
|
|
const intptr_t batch_size = (2 << 14) - 1;
|
|
for (intptr_t i = 0; i < array.Length(); ++i) {
|
|
ObjectPtr ptr = array.At(i);
|
|
visitor.VisitObject(ptr);
|
|
if ((i & batch_size) == batch_size) {
|
|
thread->CheckForSafepoint();
|
|
}
|
|
}
|
|
break;
|
|
} else {
|
|
raw->untag()->VisitPointers(&visitor);
|
|
}
|
|
} while (false);
|
|
|
|
// If no children were added, remove null separator and the node.
|
|
// If children were added, the node will be removed once last child
|
|
// is processed, only separator null remains.
|
|
if (working_list->length() == length) {
|
|
RELEASE_ASSERT(working_list->RemoveLast() == Object::null());
|
|
RELEASE_ASSERT(working_list->RemoveLast() == raw);
|
|
}
|
|
}
|
|
// `to` was not found in the graph rooted in `from`, empty retaining path
|
|
return "";
|
|
}
|
|
|
|
private:
|
|
Zone* zone_;
|
|
Isolate* isolate_;
|
|
const Object& from_;
|
|
const Object& to_;
|
|
TraversalRules traversal_rules_;
|
|
|
|
class FindObjectVisitor : public ObjectPointerVisitor {
|
|
public:
|
|
FindObjectVisitor(IsolateGroup* isolate_group, ObjectPtr target)
|
|
: ObjectPointerVisitor(isolate_group), target_(target), index_(0) {}
|
|
|
|
void VisitPointers(ObjectPtr* from, ObjectPtr* to) override {
|
|
for (ObjectPtr* ptr = from; ptr <= to; ptr++, index_++) {
|
|
if (*ptr == target_) {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
#if defined(DART_COMPRESSED_POINTERS)
|
|
void VisitCompressedPointers(uword heap_base,
|
|
CompressedObjectPtr* from,
|
|
CompressedObjectPtr* to) override {
|
|
for (CompressedObjectPtr* ptr = from; ptr <= to; ptr++, index_++) {
|
|
if (ptr->Decompress(heap_base) == target_) {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
#endif
|
|
|
|
intptr_t index() { return index_; }
|
|
|
|
private:
|
|
ObjectPtr target_;
|
|
intptr_t index_;
|
|
};
|
|
|
|
const char* CollectPath(MallocGrowableArray<ObjectPtr>* const working_list) {
|
|
Object& previous_object = Object::Handle(zone_);
|
|
Object& object = Object::Handle(zone_);
|
|
Field& field = Field::Handle(zone_);
|
|
Class& klass = Class::Handle(zone_);
|
|
Library& library = Library::Handle(zone_);
|
|
String& library_url = String::Handle(zone_);
|
|
Context& context = Context::Handle(zone_);
|
|
Closure& closure = Closure::Handle(zone_);
|
|
Function& function = Function::Handle(zone_);
|
|
#if !defined(DART_PRECOMPILED_RUNTIME)
|
|
Code& code = Code::Handle(zone_);
|
|
LocalVarDescriptors& var_descriptors = LocalVarDescriptors::Handle(zone_);
|
|
String& name = String::Handle(zone_);
|
|
#endif
|
|
|
|
const char* saved_context_location = nullptr;
|
|
intptr_t saved_context_object_index = -1;
|
|
intptr_t saved_context_depth = 0;
|
|
const char* retaining_path = "";
|
|
|
|
ObjectPtr raw = to_.ptr();
|
|
do {
|
|
previous_object = raw;
|
|
// Skip all remaining children until null-separator, so we get the parent
|
|
do {
|
|
raw = working_list->RemoveLast();
|
|
} while (raw != Object::null() && raw != from_.ptr());
|
|
if (raw == Object::null()) {
|
|
raw = working_list->RemoveLast();
|
|
object = raw;
|
|
klass = object.clazz();
|
|
|
|
const char* location = object.ToCString();
|
|
|
|
if (object.IsContext()) {
|
|
context ^= raw;
|
|
if (saved_context_object_index == -1) {
|
|
// If this is the first context, remember index of the
|
|
// [previous_object] in the Context.
|
|
// We will need it later if get to see the Closure next.
|
|
saved_context_depth = 0;
|
|
for (intptr_t i = 0; i < context.num_variables(); i++) {
|
|
if (context.At(i) == previous_object.ptr()) {
|
|
saved_context_object_index = i;
|
|
break;
|
|
}
|
|
}
|
|
} else {
|
|
// Keep track of context depths in case of nested contexts;
|
|
saved_context_depth++;
|
|
}
|
|
} else {
|
|
if (object.IsInstance()) {
|
|
if (object.IsClosure()) {
|
|
closure ^= raw;
|
|
function ^= closure.function();
|
|
// Use function's class when looking for a library information.
|
|
klass ^= function.Owner();
|
|
#if defined(DART_PRECOMPILED_RUNTIME)
|
|
// Use function's name instead of closure's.
|
|
location = function.QualifiedUserVisibleNameCString();
|
|
#else // defined(DART_PRECOMPILED_RUNTIME) \
|
|
// Attempt to convert "instance <- Context+ <- Closure" into \
|
|
// "instance <- local var name in Closure".
|
|
if (!function.ForceOptimize()) {
|
|
function.EnsureHasCompiledUnoptimizedCode();
|
|
}
|
|
code ^= function.unoptimized_code();
|
|
ASSERT(!code.IsNull());
|
|
var_descriptors ^= code.GetLocalVarDescriptors();
|
|
for (intptr_t i = 0; i < var_descriptors.Length(); i++) {
|
|
UntaggedLocalVarDescriptors::VarInfo info;
|
|
var_descriptors.GetInfo(i, &info);
|
|
if (info.scope_id == -saved_context_depth &&
|
|
info.kind() ==
|
|
UntaggedLocalVarDescriptors::VarInfoKind::kContextVar &&
|
|
info.index() == saved_context_object_index) {
|
|
name ^= var_descriptors.GetName(i);
|
|
location =
|
|
OS::SCreate(zone_, "field %s in %s", name.ToCString(),
|
|
function.QualifiedUserVisibleNameCString());
|
|
// Won't need saved context location after all.
|
|
saved_context_location = nullptr;
|
|
break;
|
|
}
|
|
}
|
|
#endif // defined(DART_PRECOMPILED_RUNTIME)
|
|
} else {
|
|
// Attempt to find field name for the field that holds the
|
|
// [previous_object] instance.
|
|
FindObjectVisitor visitor(isolate_->group(),
|
|
previous_object.ptr());
|
|
raw->untag()->VisitPointers(&visitor);
|
|
field ^= klass.FieldFromIndex(visitor.index());
|
|
if (!field.IsNull()) {
|
|
location =
|
|
OS::SCreate(zone_, "%s in %s",
|
|
field.UserVisibleNameCString(), location);
|
|
}
|
|
}
|
|
}
|
|
// Saved context object index stays up for only one cycle - just to
|
|
// accommodate short chains Closure -> Context -> instance.
|
|
saved_context_object_index = -1;
|
|
saved_context_depth = -1;
|
|
}
|
|
// Add library url to the location if library is available.
|
|
library = klass.library();
|
|
if (!library.IsNull()) {
|
|
library_url = library.url();
|
|
location = OS::SCreate(zone_, "%s (from %s)", location,
|
|
library_url.ToCString());
|
|
}
|
|
|
|
if (object.IsContext()) {
|
|
// Save context string placeholder in case we don't find closure next
|
|
if (saved_context_location == nullptr) {
|
|
saved_context_location = location;
|
|
} else {
|
|
// Append saved contexts
|
|
saved_context_location = OS::SCreate(
|
|
zone_, "%s <- %s\n", saved_context_location, location);
|
|
}
|
|
} else {
|
|
if (saved_context_location != nullptr) {
|
|
// Could not use saved context, insert it into retaining path now.
|
|
retaining_path = OS::SCreate(zone_, "%s <- %s", retaining_path,
|
|
saved_context_location);
|
|
saved_context_location = nullptr;
|
|
}
|
|
retaining_path =
|
|
OS::SCreate(zone_, "%s <- %s\n", retaining_path, location);
|
|
}
|
|
}
|
|
} while (raw != from_.ptr());
|
|
ASSERT(working_list->is_empty());
|
|
return retaining_path;
|
|
}
|
|
};
|
|
|
|
const char* FindRetainingPath(Zone* zone_,
|
|
Isolate* isolate,
|
|
const Object& from,
|
|
const Object& to,
|
|
TraversalRules traversal_rules) {
|
|
RetainingPath rr(zone_, isolate, from, to, traversal_rules);
|
|
return rr.FindPath();
|
|
}
|
|
|
|
class FastObjectCopyBase : public ObjectCopyBase {
|
|
public:
|
|
using Types = PtrTypes;
|
|
|
|
FastObjectCopyBase(Thread* thread, IdentityMap* map)
|
|
: ObjectCopyBase(thread), fast_forward_map_(thread, map) {}
|
|
|
|
protected:
|
|
DART_FORCE_INLINE
|
|
void ForwardCompressedPointers(ObjectPtr src,
|
|
ObjectPtr dst,
|
|
intptr_t offset,
|
|
intptr_t end_offset) {
|
|
for (; offset < end_offset; offset += kCompressedWordSize) {
|
|
ForwardCompressedPointer(src, dst, offset);
|
|
}
|
|
}
|
|
|
|
DART_FORCE_INLINE
|
|
void ForwardCompressedPointers(ObjectPtr src,
|
|
ObjectPtr dst,
|
|
intptr_t offset,
|
|
intptr_t end_offset,
|
|
UnboxedFieldBitmap bitmap) {
|
|
if (bitmap.IsEmpty()) {
|
|
ForwardCompressedPointers(src, dst, offset, end_offset);
|
|
return;
|
|
}
|
|
intptr_t bit = offset >> kCompressedWordSizeLog2;
|
|
for (; offset < end_offset; offset += kCompressedWordSize) {
|
|
if (bitmap.Get(bit++)) {
|
|
StoreCompressedNonPointerWord(
|
|
dst, offset, LoadCompressedNonPointerWord(src, offset));
|
|
} else {
|
|
ForwardCompressedPointer(src, dst, offset);
|
|
}
|
|
}
|
|
}
|
|
|
|
void ForwardCompressedArrayPointers(intptr_t array_length,
|
|
ObjectPtr src,
|
|
ObjectPtr dst,
|
|
intptr_t offset,
|
|
intptr_t end_offset) {
|
|
for (; offset < end_offset; offset += kCompressedWordSize) {
|
|
ForwardCompressedPointer(src, dst, offset);
|
|
}
|
|
}
|
|
|
|
void ForwardCompressedContextPointers(intptr_t context_length,
|
|
ObjectPtr src,
|
|
ObjectPtr dst,
|
|
intptr_t offset,
|
|
intptr_t end_offset) {
|
|
for (; offset < end_offset; offset += kCompressedWordSize) {
|
|
ForwardCompressedPointer(src, dst, offset);
|
|
}
|
|
}
|
|
|
|
DART_FORCE_INLINE
|
|
void ForwardCompressedPointer(ObjectPtr src, ObjectPtr dst, intptr_t offset) {
|
|
auto value = LoadCompressedPointer(src, offset);
|
|
if (!value.IsHeapObject()) {
|
|
StoreCompressedPointerNoBarrier(dst, offset, value);
|
|
return;
|
|
}
|
|
auto value_decompressed = value.Decompress(heap_base_);
|
|
const uword tags = TagsFromUntaggedObject(value_decompressed.untag());
|
|
if (CanShareObject(value_decompressed, tags)) {
|
|
StoreCompressedPointerNoBarrier(dst, offset, value);
|
|
return;
|
|
}
|
|
|
|
ObjectPtr existing_to =
|
|
fast_forward_map_.ForwardedObject(value_decompressed);
|
|
if (existing_to != Marker()) {
|
|
StoreCompressedPointerNoBarrier(dst, offset, existing_to);
|
|
return;
|
|
}
|
|
|
|
if (UNLIKELY(!CanCopyObject(tags, value_decompressed))) {
|
|
ASSERT(exception_msg_ != nullptr);
|
|
StoreCompressedPointerNoBarrier(dst, offset, Object::null());
|
|
return;
|
|
}
|
|
|
|
auto to = Forward(tags, value_decompressed);
|
|
StoreCompressedPointerNoBarrier(dst, offset, to);
|
|
}
|
|
|
|
ObjectPtr Forward(uword tags, ObjectPtr from) {
|
|
const intptr_t header_size = UntaggedObject::SizeTag::decode(tags);
|
|
const auto cid = UntaggedObject::ClassIdTag::decode(tags);
|
|
const uword size =
|
|
header_size != 0 ? header_size : from.untag()->HeapSize();
|
|
if (IsAllocatableInNewSpace(size)) {
|
|
const uword alloc = new_space_->TryAllocateNoSafepoint(thread_, size);
|
|
if (alloc != 0) {
|
|
ObjectPtr to(reinterpret_cast<UntaggedObject*>(alloc));
|
|
fast_forward_map_.Insert(from, to, size);
|
|
|
|
if (IsExternalTypedDataClassId(cid)) {
|
|
SetNewSpaceTaggingWord(to, cid, header_size);
|
|
InitializeExternalTypedData(cid, ExternalTypedData::RawCast(from),
|
|
ExternalTypedData::RawCast(to));
|
|
fast_forward_map_.AddExternalTypedData(
|
|
ExternalTypedData::RawCast(to));
|
|
} else if (IsTypedDataViewClassId(cid) ||
|
|
IsUnmodifiableTypedDataViewClassId(cid)) {
|
|
// We set the views backing store to `null` to satisfy an assertion in
|
|
// GCCompactor::VisitTypedDataViewPointers().
|
|
SetNewSpaceTaggingWord(to, cid, header_size);
|
|
InitializeTypedDataView(TypedDataView::RawCast(to));
|
|
}
|
|
return to;
|
|
}
|
|
}
|
|
exception_msg_ = kFastAllocationFailed;
|
|
return Marker();
|
|
}
|
|
|
|
void EnqueueTransferable(TransferableTypedDataPtr from,
|
|
TransferableTypedDataPtr to) {
|
|
fast_forward_map_.AddTransferable(from, to);
|
|
}
|
|
void EnqueueWeakProperty(WeakPropertyPtr from) {
|
|
fast_forward_map_.AddWeakProperty(from);
|
|
}
|
|
void EnqueueWeakReference(WeakReferencePtr from) {
|
|
fast_forward_map_.AddWeakReference(from);
|
|
}
|
|
void EnqueueObjectToRehash(ObjectPtr to) {
|
|
fast_forward_map_.AddObjectToRehash(to);
|
|
}
|
|
void EnqueueExpandoToRehash(ObjectPtr to) {
|
|
fast_forward_map_.AddExpandoToRehash(to);
|
|
}
|
|
|
|
static void StoreCompressedArrayPointers(intptr_t array_length,
|
|
ObjectPtr src,
|
|
ObjectPtr dst,
|
|
intptr_t offset,
|
|
intptr_t end_offset) {
|
|
StoreCompressedPointers(src, dst, offset, end_offset);
|
|
}
|
|
static void StoreCompressedPointers(ObjectPtr src,
|
|
ObjectPtr dst,
|
|
intptr_t offset,
|
|
intptr_t end_offset) {
|
|
StoreCompressedPointersNoBarrier(src, dst, offset, end_offset);
|
|
}
|
|
static void StoreCompressedPointersNoBarrier(ObjectPtr src,
|
|
ObjectPtr dst,
|
|
intptr_t offset,
|
|
intptr_t end_offset) {
|
|
for (; offset <= end_offset; offset += kCompressedWordSize) {
|
|
StoreCompressedPointerNoBarrier(dst, offset,
|
|
LoadCompressedPointer(src, offset));
|
|
}
|
|
}
|
|
|
|
protected:
|
|
friend class ObjectGraphCopier;
|
|
|
|
FastForwardMap fast_forward_map_;
|
|
};
|
|
|
|
class SlowObjectCopyBase : public ObjectCopyBase {
|
|
public:
|
|
using Types = HandleTypes;
|
|
|
|
explicit SlowObjectCopyBase(Thread* thread, IdentityMap* map)
|
|
: ObjectCopyBase(thread), slow_forward_map_(thread, map) {}
|
|
|
|
protected:
|
|
DART_FORCE_INLINE
|
|
void ForwardCompressedPointers(const Object& src,
|
|
const Object& dst,
|
|
intptr_t offset,
|
|
intptr_t end_offset) {
|
|
for (; offset < end_offset; offset += kCompressedWordSize) {
|
|
ForwardCompressedPointer(src, dst, offset);
|
|
}
|
|
}
|
|
|
|
DART_FORCE_INLINE
|
|
void ForwardCompressedPointers(const Object& src,
|
|
const Object& dst,
|
|
intptr_t offset,
|
|
intptr_t end_offset,
|
|
UnboxedFieldBitmap bitmap) {
|
|
intptr_t bit = offset >> kCompressedWordSizeLog2;
|
|
for (; offset < end_offset; offset += kCompressedWordSize) {
|
|
if (bitmap.Get(bit++)) {
|
|
StoreCompressedNonPointerWord(
|
|
dst.ptr(), offset, LoadCompressedNonPointerWord(src.ptr(), offset));
|
|
} else {
|
|
ForwardCompressedPointer(src, dst, offset);
|
|
}
|
|
}
|
|
}
|
|
|
|
void ForwardCompressedArrayPointers(intptr_t array_length,
|
|
const Object& src,
|
|
const Object& dst,
|
|
intptr_t offset,
|
|
intptr_t end_offset) {
|
|
if (Array::UseCardMarkingForAllocation(array_length)) {
|
|
for (; offset < end_offset; offset += kCompressedWordSize) {
|
|
ForwardCompressedLargeArrayPointer(src, dst, offset);
|
|
thread_->CheckForSafepoint();
|
|
}
|
|
} else {
|
|
for (; offset < end_offset; offset += kCompressedWordSize) {
|
|
ForwardCompressedPointer(src, dst, offset);
|
|
}
|
|
}
|
|
}
|
|
|
|
void ForwardCompressedContextPointers(intptr_t context_length,
|
|
const Object& src,
|
|
const Object& dst,
|
|
intptr_t offset,
|
|
intptr_t end_offset) {
|
|
for (; offset < end_offset; offset += kCompressedWordSize) {
|
|
ForwardCompressedPointer(src, dst, offset);
|
|
}
|
|
}
|
|
|
|
DART_FORCE_INLINE
|
|
void ForwardCompressedLargeArrayPointer(const Object& src,
|
|
const Object& dst,
|
|
intptr_t offset) {
|
|
auto value = LoadCompressedPointer(src.ptr(), offset);
|
|
if (!value.IsHeapObject()) {
|
|
StoreCompressedPointerNoBarrier(dst.ptr(), offset, value);
|
|
return;
|
|
}
|
|
|
|
auto value_decompressed = value.Decompress(heap_base_);
|
|
const uword tags = TagsFromUntaggedObject(value_decompressed.untag());
|
|
if (CanShareObject(value_decompressed, tags)) {
|
|
StoreCompressedLargeArrayPointerBarrier(dst.ptr(), offset,
|
|
value_decompressed);
|
|
return;
|
|
}
|
|
|
|
ObjectPtr existing_to =
|
|
slow_forward_map_.ForwardedObject(value_decompressed);
|
|
if (existing_to != Marker()) {
|
|
StoreCompressedLargeArrayPointerBarrier(dst.ptr(), offset, existing_to);
|
|
return;
|
|
}
|
|
|
|
if (UNLIKELY(!CanCopyObject(tags, value_decompressed))) {
|
|
ASSERT(exception_msg_ != nullptr);
|
|
StoreCompressedLargeArrayPointerBarrier(dst.ptr(), offset,
|
|
Object::null());
|
|
return;
|
|
}
|
|
|
|
tmp_ = value_decompressed;
|
|
tmp_ = Forward(tags, tmp_); // Only this can cause allocation.
|
|
StoreCompressedLargeArrayPointerBarrier(dst.ptr(), offset, tmp_.ptr());
|
|
}
|
|
DART_FORCE_INLINE
|
|
void ForwardCompressedPointer(const Object& src,
|
|
const Object& dst,
|
|
intptr_t offset) {
|
|
auto value = LoadCompressedPointer(src.ptr(), offset);
|
|
if (!value.IsHeapObject()) {
|
|
StoreCompressedPointerNoBarrier(dst.ptr(), offset, value);
|
|
return;
|
|
}
|
|
auto value_decompressed = value.Decompress(heap_base_);
|
|
const uword tags = TagsFromUntaggedObject(value_decompressed.untag());
|
|
if (CanShareObject(value_decompressed, tags)) {
|
|
StoreCompressedPointerBarrier(dst.ptr(), offset, value_decompressed);
|
|
return;
|
|
}
|
|
|
|
ObjectPtr existing_to =
|
|
slow_forward_map_.ForwardedObject(value_decompressed);
|
|
if (existing_to != Marker()) {
|
|
StoreCompressedPointerBarrier(dst.ptr(), offset, existing_to);
|
|
return;
|
|
}
|
|
|
|
if (UNLIKELY(!CanCopyObject(tags, value_decompressed))) {
|
|
ASSERT(exception_msg_ != nullptr);
|
|
StoreCompressedPointerNoBarrier(dst.ptr(), offset, Object::null());
|
|
return;
|
|
}
|
|
|
|
tmp_ = value_decompressed;
|
|
tmp_ = Forward(tags, tmp_); // Only this can cause allocation.
|
|
StoreCompressedPointerBarrier(dst.ptr(), offset, tmp_.ptr());
|
|
}
|
|
|
|
ObjectPtr Forward(uword tags, const Object& from) {
|
|
const intptr_t cid = UntaggedObject::ClassIdTag::decode(tags);
|
|
intptr_t size = UntaggedObject::SizeTag::decode(tags);
|
|
if (size == 0) {
|
|
size = from.ptr().untag()->HeapSize();
|
|
}
|
|
to_ = AllocateObject(cid, size, slow_forward_map_.allocated_bytes);
|
|
UpdateLengthField(cid, from.ptr(), to_.ptr());
|
|
slow_forward_map_.Insert(from, to_, size);
|
|
ObjectPtr to = to_.ptr();
|
|
if ((cid == kArrayCid || cid == kImmutableArrayCid) &&
|
|
!IsAllocatableInNewSpace(size)) {
|
|
to.untag()->SetCardRememberedBitUnsynchronized();
|
|
}
|
|
if (IsExternalTypedDataClassId(cid)) {
|
|
const auto& external_to = slow_forward_map_.AddExternalTypedData(
|
|
ExternalTypedData::RawCast(to));
|
|
InitializeExternalTypedDataWithSafepointChecks(
|
|
thread_, cid, ExternalTypedData::Cast(from), external_to);
|
|
return external_to.ptr();
|
|
} else if (IsTypedDataViewClassId(cid) ||
|
|
IsUnmodifiableTypedDataViewClassId(cid)) {
|
|
// We set the views backing store to `null` to satisfy an assertion in
|
|
// GCCompactor::VisitTypedDataViewPointers().
|
|
InitializeTypedDataView(TypedDataView::RawCast(to));
|
|
}
|
|
return to;
|
|
}
|
|
void EnqueueTransferable(const TransferableTypedData& from,
|
|
const TransferableTypedData& to) {
|
|
slow_forward_map_.AddTransferable(from, to);
|
|
}
|
|
void EnqueueWeakProperty(const WeakProperty& from) {
|
|
slow_forward_map_.AddWeakProperty(from);
|
|
}
|
|
void EnqueueWeakReference(const WeakReference& from) {
|
|
slow_forward_map_.AddWeakReference(from);
|
|
}
|
|
void EnqueueObjectToRehash(const Object& to) {
|
|
slow_forward_map_.AddObjectToRehash(to);
|
|
}
|
|
void EnqueueExpandoToRehash(const Object& to) {
|
|
slow_forward_map_.AddExpandoToRehash(to);
|
|
}
|
|
|
|
void StoreCompressedArrayPointers(intptr_t array_length,
|
|
const Object& src,
|
|
const Object& dst,
|
|
intptr_t offset,
|
|
intptr_t end_offset) {
|
|
auto src_ptr = src.ptr();
|
|
auto dst_ptr = dst.ptr();
|
|
if (Array::UseCardMarkingForAllocation(array_length)) {
|
|
for (; offset <= end_offset; offset += kCompressedWordSize) {
|
|
StoreCompressedLargeArrayPointerBarrier(
|
|
dst_ptr, offset,
|
|
LoadCompressedPointer(src_ptr, offset).Decompress(heap_base_));
|
|
}
|
|
} else {
|
|
for (; offset <= end_offset; offset += kCompressedWordSize) {
|
|
StoreCompressedPointerBarrier(
|
|
dst_ptr, offset,
|
|
LoadCompressedPointer(src_ptr, offset).Decompress(heap_base_));
|
|
}
|
|
}
|
|
}
|
|
void StoreCompressedPointers(const Object& src,
|
|
const Object& dst,
|
|
intptr_t offset,
|
|
intptr_t end_offset) {
|
|
auto src_ptr = src.ptr();
|
|
auto dst_ptr = dst.ptr();
|
|
for (; offset <= end_offset; offset += kCompressedWordSize) {
|
|
StoreCompressedPointerBarrier(
|
|
dst_ptr, offset,
|
|
LoadCompressedPointer(src_ptr, offset).Decompress(heap_base_));
|
|
}
|
|
}
|
|
static void StoreCompressedPointersNoBarrier(const Object& src,
|
|
const Object& dst,
|
|
intptr_t offset,
|
|
intptr_t end_offset) {
|
|
auto src_ptr = src.ptr();
|
|
auto dst_ptr = dst.ptr();
|
|
for (; offset <= end_offset; offset += kCompressedWordSize) {
|
|
StoreCompressedPointerNoBarrier(dst_ptr, offset,
|
|
LoadCompressedPointer(src_ptr, offset));
|
|
}
|
|
}
|
|
|
|
protected:
|
|
friend class ObjectGraphCopier;
|
|
|
|
SlowForwardMap slow_forward_map_;
|
|
};
|
|
|
|
template <typename Base>
|
|
class ObjectCopy : public Base {
|
|
public:
|
|
using Types = typename Base::Types;
|
|
|
|
ObjectCopy(Thread* thread, IdentityMap* map) : Base(thread, map) {}
|
|
|
|
void CopyPredefinedInstance(typename Types::Object from,
|
|
typename Types::Object to,
|
|
intptr_t cid) {
|
|
if (IsImplicitFieldClassId(cid)) {
|
|
CopyUserdefinedInstanceWithoutUnboxedFields(from, to);
|
|
return;
|
|
}
|
|
switch (cid) {
|
|
#define COPY_TO(clazz) \
|
|
case clazz::kClassId: { \
|
|
typename Types::clazz casted_from = Types::Cast##clazz(from); \
|
|
typename Types::clazz casted_to = Types::Cast##clazz(to); \
|
|
Copy##clazz(casted_from, casted_to); \
|
|
return; \
|
|
}
|
|
|
|
CLASS_LIST_NO_OBJECT_NOR_STRING_NOR_ARRAY_NOR_MAP(COPY_TO)
|
|
COPY_TO(Array)
|
|
COPY_TO(GrowableObjectArray)
|
|
COPY_TO(Map)
|
|
COPY_TO(Set)
|
|
#undef COPY_TO
|
|
|
|
case ImmutableArray::kClassId: {
|
|
typename Types::Array casted_from = Types::CastArray(from);
|
|
typename Types::Array casted_to = Types::CastArray(to);
|
|
CopyArray(casted_from, casted_to);
|
|
return;
|
|
}
|
|
|
|
#define COPY_TO(clazz) case kTypedData##clazz##Cid:
|
|
|
|
CLASS_LIST_TYPED_DATA(COPY_TO) {
|
|
typename Types::TypedData casted_from = Types::CastTypedData(from);
|
|
typename Types::TypedData casted_to = Types::CastTypedData(to);
|
|
CopyTypedData(casted_from, casted_to);
|
|
return;
|
|
}
|
|
#undef COPY_TO
|
|
|
|
case kByteDataViewCid:
|
|
case kUnmodifiableByteDataViewCid:
|
|
#define COPY_TO(clazz) \
|
|
case kTypedData##clazz##ViewCid: \
|
|
case kUnmodifiableTypedData##clazz##ViewCid:
|
|
CLASS_LIST_TYPED_DATA(COPY_TO) {
|
|
typename Types::TypedDataView casted_from =
|
|
Types::CastTypedDataView(from);
|
|
typename Types::TypedDataView casted_to =
|
|
Types::CastTypedDataView(to);
|
|
CopyTypedDataView(casted_from, casted_to);
|
|
return;
|
|
}
|
|
#undef COPY_TO
|
|
|
|
#define COPY_TO(clazz) case kExternalTypedData##clazz##Cid:
|
|
|
|
CLASS_LIST_TYPED_DATA(COPY_TO) {
|
|
typename Types::ExternalTypedData casted_from =
|
|
Types::CastExternalTypedData(from);
|
|
typename Types::ExternalTypedData casted_to =
|
|
Types::CastExternalTypedData(to);
|
|
CopyExternalTypedData(casted_from, casted_to);
|
|
return;
|
|
}
|
|
#undef COPY_TO
|
|
default:
|
|
break;
|
|
}
|
|
|
|
const Object& obj = Types::HandlifyObject(from);
|
|
FATAL("Unexpected object: %s\n", obj.ToCString());
|
|
}
|
|
|
|
void CopyUserdefinedInstance(typename Types::Object from,
|
|
typename Types::Object to,
|
|
UnboxedFieldBitmap bitmap) {
|
|
const intptr_t instance_size = UntagObject(from)->HeapSize();
|
|
Base::ForwardCompressedPointers(from, to, kWordSize, instance_size, bitmap);
|
|
}
|
|
|
|
void CopyUserdefinedInstanceWithoutUnboxedFields(typename Types::Object from,
|
|
typename Types::Object to) {
|
|
const intptr_t instance_size = UntagObject(from)->HeapSize();
|
|
Base::ForwardCompressedPointers(from, to, kWordSize, instance_size);
|
|
}
|
|
void CopyClosure(typename Types::Closure from, typename Types::Closure to) {
|
|
Base::StoreCompressedPointers(
|
|
from, to, OFFSET_OF(UntaggedClosure, instantiator_type_arguments_),
|
|
OFFSET_OF(UntaggedClosure, function_));
|
|
Base::ForwardCompressedPointer(from, to,
|
|
OFFSET_OF(UntaggedClosure, context_));
|
|
Base::StoreCompressedPointersNoBarrier(from, to,
|
|
OFFSET_OF(UntaggedClosure, hash_),
|
|
OFFSET_OF(UntaggedClosure, hash_));
|
|
ONLY_IN_PRECOMPILED(UntagClosure(to)->entry_point_ =
|
|
UntagClosure(from)->entry_point_);
|
|
}
|
|
|
|
void CopyContext(typename Types::Context from, typename Types::Context to) {
|
|
const intptr_t length = Context::NumVariables(Types::GetContextPtr(from));
|
|
|
|
UntagContext(to)->num_variables_ = UntagContext(from)->num_variables_;
|
|
|
|
Base::ForwardCompressedPointer(from, to,
|
|
OFFSET_OF(UntaggedContext, parent_));
|
|
Base::ForwardCompressedContextPointers(
|
|
length, from, to, Context::variable_offset(0),
|
|
Context::variable_offset(0) + Context::kBytesPerElement * length);
|
|
}
|
|
|
|
void CopyArray(typename Types::Array from, typename Types::Array to) {
|
|
const intptr_t length = Smi::Value(UntagArray(from)->length());
|
|
Base::StoreCompressedArrayPointers(
|
|
length, from, to, OFFSET_OF(UntaggedArray, type_arguments_),
|
|
OFFSET_OF(UntaggedArray, type_arguments_));
|
|
Base::StoreCompressedPointersNoBarrier(from, to,
|
|
OFFSET_OF(UntaggedArray, length_),
|
|
OFFSET_OF(UntaggedArray, length_));
|
|
Base::ForwardCompressedArrayPointers(
|
|
length, from, to, Array::data_offset(),
|
|
Array::data_offset() + kCompressedWordSize * length);
|
|
}
|
|
|
|
void CopyGrowableObjectArray(typename Types::GrowableObjectArray from,
|
|
typename Types::GrowableObjectArray to) {
|
|
Base::StoreCompressedPointers(
|
|
from, to, OFFSET_OF(UntaggedGrowableObjectArray, type_arguments_),
|
|
OFFSET_OF(UntaggedGrowableObjectArray, type_arguments_));
|
|
Base::StoreCompressedPointersNoBarrier(
|
|
from, to, OFFSET_OF(UntaggedGrowableObjectArray, length_),
|
|
OFFSET_OF(UntaggedGrowableObjectArray, length_));
|
|
Base::ForwardCompressedPointer(
|
|
from, to, OFFSET_OF(UntaggedGrowableObjectArray, data_));
|
|
}
|
|
|
|
void CopyRecord(typename Types::Record from, typename Types::Record to) {
|
|
const intptr_t num_fields = Record::NumFields(Types::GetRecordPtr(from));
|
|
Base::StoreCompressedPointersNoBarrier(from, to,
|
|
OFFSET_OF(UntaggedRecord, shape_),
|
|
OFFSET_OF(UntaggedRecord, shape_));
|
|
Base::ForwardCompressedPointers(
|
|
from, to, Record::field_offset(0),
|
|
Record::field_offset(0) + Record::kBytesPerElement * num_fields);
|
|
}
|
|
|
|
template <intptr_t one_for_set_two_for_map, typename T>
|
|
void CopyLinkedHashBase(T from,
|
|
T to,
|
|
UntaggedLinkedHashBase* from_untagged,
|
|
UntaggedLinkedHashBase* to_untagged) {
|
|
// We have to find out whether the map needs re-hashing on the receiver side
|
|
// due to keys being copied and the keys therefore possibly having different
|
|
// hash codes (e.g. due to user-defined hashCode implementation or due to
|
|
// new identity hash codes of the copied objects).
|
|
bool needs_rehashing = false;
|
|
ArrayPtr data = from_untagged->data_.Decompress(Base::heap_base_);
|
|
if (data != Array::null()) {
|
|
UntaggedArray* untagged_data = data.untag();
|
|
const intptr_t length = Smi::Value(untagged_data->length_);
|
|
auto key_value_pairs = untagged_data->data();
|
|
for (intptr_t i = 0; i < length; i += one_for_set_two_for_map) {
|
|
ObjectPtr key = key_value_pairs[i].Decompress(Base::heap_base_);
|
|
const bool is_deleted_entry = key == data;
|
|
if (key->IsHeapObject()) {
|
|
if (!is_deleted_entry && MightNeedReHashing(key)) {
|
|
needs_rehashing = true;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
Base::StoreCompressedPointers(
|
|
from, to, OFFSET_OF(UntaggedLinkedHashBase, type_arguments_),
|
|
OFFSET_OF(UntaggedLinkedHashBase, type_arguments_));
|
|
|
|
// Compared with the snapshot-based (de)serializer we do preserve the same
|
|
// backing store (i.e. used_data/deleted_keys/data) and therefore do not
|
|
// magically shrink backing store based on usage.
|
|
//
|
|
// We do this to avoid making assumptions about the object graph and the
|
|
// linked hash map (e.g. assuming there's no other references to the data,
|
|
// assuming the linked hashmap is in a consistent state)
|
|
if (needs_rehashing) {
|
|
to_untagged->hash_mask_ = Smi::New(0);
|
|
to_untagged->index_ = TypedData::RawCast(Object::null());
|
|
to_untagged->deleted_keys_ = Smi::New(0);
|
|
}
|
|
|
|
// From this point on we shouldn't use the raw pointers, since GC might
|
|
// happen when forwarding objects.
|
|
from_untagged = nullptr;
|
|
to_untagged = nullptr;
|
|
|
|
if (!needs_rehashing) {
|
|
Base::ForwardCompressedPointer(from, to,
|
|
OFFSET_OF(UntaggedLinkedHashBase, index_));
|
|
Base::StoreCompressedPointersNoBarrier(
|
|
from, to, OFFSET_OF(UntaggedLinkedHashBase, hash_mask_),
|
|
OFFSET_OF(UntaggedLinkedHashBase, hash_mask_));
|
|
Base::StoreCompressedPointersNoBarrier(
|
|
from, to, OFFSET_OF(UntaggedMap, deleted_keys_),
|
|
OFFSET_OF(UntaggedMap, deleted_keys_));
|
|
}
|
|
Base::ForwardCompressedPointer(from, to,
|
|
OFFSET_OF(UntaggedLinkedHashBase, data_));
|
|
Base::StoreCompressedPointersNoBarrier(
|
|
from, to, OFFSET_OF(UntaggedLinkedHashBase, used_data_),
|
|
OFFSET_OF(UntaggedLinkedHashBase, used_data_));
|
|
|
|
if (Base::exception_msg_ == nullptr && needs_rehashing) {
|
|
Base::EnqueueObjectToRehash(to);
|
|
}
|
|
}
|
|
|
|
void CopyMap(typename Types::Map from, typename Types::Map to) {
|
|
CopyLinkedHashBase<2, typename Types::Map>(from, to, UntagMap(from),
|
|
UntagMap(to));
|
|
}
|
|
|
|
void CopySet(typename Types::Set from, typename Types::Set to) {
|
|
CopyLinkedHashBase<1, typename Types::Set>(from, to, UntagSet(from),
|
|
UntagSet(to));
|
|
}
|
|
|
|
void CopyDouble(typename Types::Double from, typename Types::Double to) {
|
|
#if !defined(DART_PRECOMPILED_RUNTIME)
|
|
auto raw_from = UntagDouble(from);
|
|
auto raw_to = UntagDouble(to);
|
|
raw_to->value_ = raw_from->value_;
|
|
#else
|
|
// Will be shared and not copied.
|
|
UNREACHABLE();
|
|
#endif
|
|
}
|
|
|
|
void CopyFloat32x4(typename Types::Float32x4 from,
|
|
typename Types::Float32x4 to) {
|
|
#if !defined(DART_PRECOMPILED_RUNTIME)
|
|
auto raw_from = UntagFloat32x4(from);
|
|
auto raw_to = UntagFloat32x4(to);
|
|
raw_to->value_[0] = raw_from->value_[0];
|
|
raw_to->value_[1] = raw_from->value_[1];
|
|
raw_to->value_[2] = raw_from->value_[2];
|
|
raw_to->value_[3] = raw_from->value_[3];
|
|
#else
|
|
// Will be shared and not copied.
|
|
UNREACHABLE();
|
|
#endif
|
|
}
|
|
|
|
void CopyFloat64x2(typename Types::Float64x2 from,
|
|
typename Types::Float64x2 to) {
|
|
#if !defined(DART_PRECOMPILED_RUNTIME)
|
|
auto raw_from = UntagFloat64x2(from);
|
|
auto raw_to = UntagFloat64x2(to);
|
|
raw_to->value_[0] = raw_from->value_[0];
|
|
raw_to->value_[1] = raw_from->value_[1];
|
|
#else
|
|
// Will be shared and not copied.
|
|
UNREACHABLE();
|
|
#endif
|
|
}
|
|
|
|
void CopyTypedData(TypedDataPtr from, TypedDataPtr to) {
|
|
auto raw_from = from.untag();
|
|
auto raw_to = to.untag();
|
|
const intptr_t cid = Types::GetTypedDataPtr(from)->GetClassId();
|
|
raw_to->length_ = raw_from->length_;
|
|
raw_to->RecomputeDataField();
|
|
const intptr_t length =
|
|
TypedData::ElementSizeInBytes(cid) * Smi::Value(raw_from->length_);
|
|
memmove(raw_to->data_, raw_from->data_, length);
|
|
}
|
|
|
|
void CopyTypedData(const TypedData& from, const TypedData& to) {
|
|
auto raw_from = from.ptr().untag();
|
|
auto raw_to = to.ptr().untag();
|
|
const intptr_t cid = Types::GetTypedDataPtr(from)->GetClassId();
|
|
ASSERT(raw_to->length_ == raw_from->length_);
|
|
raw_to->RecomputeDataField();
|
|
const intptr_t length =
|
|
TypedData::ElementSizeInBytes(cid) * Smi::Value(raw_from->length_);
|
|
CopyTypedDataBaseWithSafepointChecks(Base::thread_, from, to, length);
|
|
}
|
|
|
|
void CopyTypedDataView(typename Types::TypedDataView from,
|
|
typename Types::TypedDataView to) {
|
|
// This will forward & initialize the typed data.
|
|
Base::ForwardCompressedPointer(
|
|
from, to, OFFSET_OF(UntaggedTypedDataView, typed_data_));
|
|
|
|
auto raw_from = UntagTypedDataView(from);
|
|
auto raw_to = UntagTypedDataView(to);
|
|
raw_to->length_ = raw_from->length_;
|
|
raw_to->offset_in_bytes_ = raw_from->offset_in_bytes_;
|
|
raw_to->data_ = nullptr;
|
|
|
|
auto forwarded_backing_store =
|
|
raw_to->typed_data_.Decompress(Base::heap_base_);
|
|
if (forwarded_backing_store == Marker() ||
|
|
forwarded_backing_store == Object::null()) {
|
|
// Ensure the backing store is never "sentinel" - the scavenger doesn't
|
|
// like it.
|
|
Base::StoreCompressedPointerNoBarrier(
|
|
Types::GetTypedDataViewPtr(to),
|
|
OFFSET_OF(UntaggedTypedDataView, typed_data_), Object::null());
|
|
raw_to->length_ = Smi::New(0);
|
|
raw_to->offset_in_bytes_ = Smi::New(0);
|
|
ASSERT(Base::exception_msg_ != nullptr);
|
|
return;
|
|
}
|
|
|
|
const bool is_external =
|
|
raw_from->data_ != raw_from->DataFieldForInternalTypedData();
|
|
if (is_external) {
|
|
// The raw_to is fully initialized at this point (see handling of external
|
|
// typed data in [ForwardCompressedPointer])
|
|
raw_to->RecomputeDataField();
|
|
} else {
|
|
// The raw_to isn't initialized yet, but it's address is valid, so we can
|
|
// compute the data field it would use.
|
|
raw_to->RecomputeDataFieldForInternalTypedData();
|
|
}
|
|
const bool is_external2 =
|
|
raw_to->data_ != raw_to->DataFieldForInternalTypedData();
|
|
ASSERT(is_external == is_external2);
|
|
}
|
|
|
|
void CopyExternalTypedData(typename Types::ExternalTypedData from,
|
|
typename Types::ExternalTypedData to) {
|
|
// The external typed data is initialized on the forwarding pass (where
|
|
// normally allocation but not initialization happens), so views on it
|
|
// can be initialized immediately.
|
|
#if defined(DEBUG)
|
|
auto raw_from = UntagExternalTypedData(from);
|
|
auto raw_to = UntagExternalTypedData(to);
|
|
ASSERT(raw_to->data_ != nullptr);
|
|
ASSERT(raw_to->length_ == raw_from->length_);
|
|
#endif
|
|
}
|
|
|
|
void CopyTransferableTypedData(typename Types::TransferableTypedData from,
|
|
typename Types::TransferableTypedData to) {
|
|
// The [TransferableTypedData] is an empty object with an associated heap
|
|
// peer object.
|
|
// -> We'll validate that there's a peer and enqueue the transferable to be
|
|
// transferred if the transitive copy is successful.
|
|
auto fpeer = static_cast<TransferableTypedDataPeer*>(
|
|
Base::heap_->GetPeer(Types::GetTransferableTypedDataPtr(from)));
|
|
ASSERT(fpeer != nullptr);
|
|
if (fpeer->data() == nullptr) {
|
|
Base::exception_msg_ =
|
|
"Illegal argument in isolate message"
|
|
" : (TransferableTypedData has been transferred already)";
|
|
Base::exception_unexpected_object_ =
|
|
Types::GetTransferableTypedDataPtr(from);
|
|
return;
|
|
}
|
|
Base::EnqueueTransferable(from, to);
|
|
}
|
|
|
|
void CopyWeakProperty(typename Types::WeakProperty from,
|
|
typename Types::WeakProperty to) {
|
|
// We store `null`s as keys/values and let the main algorithm know that
|
|
// we should check reachability of the key again after the fixpoint (if it
|
|
// became reachable, forward the key/value).
|
|
Base::StoreCompressedPointerNoBarrier(Types::GetWeakPropertyPtr(to),
|
|
OFFSET_OF(UntaggedWeakProperty, key_),
|
|
Object::null());
|
|
Base::StoreCompressedPointerNoBarrier(
|
|
Types::GetWeakPropertyPtr(to), OFFSET_OF(UntaggedWeakProperty, value_),
|
|
Object::null());
|
|
// To satisfy some ASSERT()s in GC we'll use Object:null() explicitly here.
|
|
Base::StoreCompressedPointerNoBarrier(
|
|
Types::GetWeakPropertyPtr(to),
|
|
OFFSET_OF(UntaggedWeakProperty, next_seen_by_gc_), Object::null());
|
|
Base::EnqueueWeakProperty(from);
|
|
}
|
|
|
|
void CopyWeakReference(typename Types::WeakReference from,
|
|
typename Types::WeakReference to) {
|
|
// We store `null` as target and let the main algorithm know that
|
|
// we should check reachability of the target again after the fixpoint (if
|
|
// it became reachable, forward the target).
|
|
Base::StoreCompressedPointerNoBarrier(
|
|
Types::GetWeakReferencePtr(to),
|
|
OFFSET_OF(UntaggedWeakReference, target_), Object::null());
|
|
// Type argument should always be copied.
|
|
Base::ForwardCompressedPointer(
|
|
from, to, OFFSET_OF(UntaggedWeakReference, type_arguments_));
|
|
// To satisfy some ASSERT()s in GC we'll use Object:null() explicitly here.
|
|
Base::StoreCompressedPointerNoBarrier(
|
|
Types::GetWeakReferencePtr(to),
|
|
OFFSET_OF(UntaggedWeakReference, next_seen_by_gc_), Object::null());
|
|
Base::EnqueueWeakReference(from);
|
|
}
|
|
|
|
// clang-format off
|
|
#define DEFINE_UNSUPPORTED(clazz) \
|
|
void Copy##clazz(typename Types::clazz from, typename Types::clazz to) { \
|
|
FATAL("Objects of type " #clazz " should not occur in object graphs"); \
|
|
}
|
|
|
|
FOR_UNSUPPORTED_CLASSES(DEFINE_UNSUPPORTED)
|
|
|
|
#undef DEFINE_UNSUPPORTED
|
|
// clang-format on
|
|
|
|
UntaggedObject* UntagObject(typename Types::Object obj) {
|
|
return Types::GetObjectPtr(obj).Decompress(Base::heap_base_).untag();
|
|
}
|
|
|
|
#define DO(V) \
|
|
DART_FORCE_INLINE \
|
|
Untagged##V* Untag##V(typename Types::V obj) { \
|
|
return Types::Get##V##Ptr(obj).Decompress(Base::heap_base_).untag(); \
|
|
}
|
|
CLASS_LIST_FOR_HANDLES(DO)
|
|
#undef DO
|
|
};
|
|
|
|
class FastObjectCopy : public ObjectCopy<FastObjectCopyBase> {
|
|
public:
|
|
FastObjectCopy(Thread* thread, IdentityMap* map) : ObjectCopy(thread, map) {}
|
|
~FastObjectCopy() {}
|
|
|
|
ObjectPtr TryCopyGraphFast(ObjectPtr root) {
|
|
NoSafepointScope no_safepoint_scope;
|
|
|
|
ObjectPtr root_copy = Forward(TagsFromUntaggedObject(root.untag()), root);
|
|
if (root_copy == Marker()) {
|
|
return root_copy;
|
|
}
|
|
auto& from_weak_property = WeakProperty::Handle(zone_);
|
|
auto& to_weak_property = WeakProperty::Handle(zone_);
|
|
auto& weak_property_key = Object::Handle(zone_);
|
|
while (true) {
|
|
if (fast_forward_map_.fill_cursor_ ==
|
|
fast_forward_map_.raw_from_to_.length()) {
|
|
break;
|
|
}
|
|
|
|
// Run fixpoint to copy all objects.
|
|
while (fast_forward_map_.fill_cursor_ <
|
|
fast_forward_map_.raw_from_to_.length()) {
|
|
const intptr_t index = fast_forward_map_.fill_cursor_;
|
|
ObjectPtr from = fast_forward_map_.raw_from_to_[index];
|
|
ObjectPtr to = fast_forward_map_.raw_from_to_[index + 1];
|
|
FastCopyObject(from, to);
|
|
if (exception_msg_ != nullptr) {
|
|
return root_copy;
|
|
}
|
|
fast_forward_map_.fill_cursor_ += 2;
|
|
|
|
// To maintain responsiveness we regularly check whether safepoints are
|
|
// requested - if so, we bail to slow path which will then checkin.
|
|
if (thread_->IsSafepointRequested()) {
|
|
exception_msg_ = kFastAllocationFailed;
|
|
return root_copy;
|
|
}
|
|
}
|
|
|
|
// Possibly forward values of [WeakProperty]s if keys became reachable.
|
|
intptr_t i = 0;
|
|
auto& weak_properties = fast_forward_map_.raw_weak_properties_;
|
|
while (i < weak_properties.length()) {
|
|
from_weak_property = weak_properties[i];
|
|
weak_property_key =
|
|
fast_forward_map_.ForwardedObject(from_weak_property.key());
|
|
if (weak_property_key.ptr() != Marker()) {
|
|
to_weak_property ^=
|
|
fast_forward_map_.ForwardedObject(from_weak_property.ptr());
|
|
|
|
// The key became reachable so we'll change the forwarded
|
|
// [WeakProperty]'s key to the new key (it is `null` at this point).
|
|
to_weak_property.set_key(weak_property_key);
|
|
|
|
// Since the key has become strongly reachable in the copied graph,
|
|
// we'll also need to forward the value.
|
|
ForwardCompressedPointer(from_weak_property.ptr(),
|
|
to_weak_property.ptr(),
|
|
OFFSET_OF(UntaggedWeakProperty, value_));
|
|
|
|
// We don't need to process this [WeakProperty] again.
|
|
const intptr_t last = weak_properties.length() - 1;
|
|
if (i < last) {
|
|
weak_properties[i] = weak_properties[last];
|
|
weak_properties.SetLength(last);
|
|
continue;
|
|
}
|
|
}
|
|
i++;
|
|
}
|
|
}
|
|
// After the fix point with [WeakProperty]s do [WeakReference]s.
|
|
auto& from_weak_reference = WeakReference::Handle(zone_);
|
|
auto& to_weak_reference = WeakReference::Handle(zone_);
|
|
auto& weak_reference_target = Object::Handle(zone_);
|
|
auto& weak_references = fast_forward_map_.raw_weak_references_;
|
|
for (intptr_t i = 0; i < weak_references.length(); i++) {
|
|
from_weak_reference = weak_references[i];
|
|
weak_reference_target =
|
|
fast_forward_map_.ForwardedObject(from_weak_reference.target());
|
|
if (weak_reference_target.ptr() != Marker()) {
|
|
to_weak_reference ^=
|
|
fast_forward_map_.ForwardedObject(from_weak_reference.ptr());
|
|
|
|
// The target became reachable so we'll change the forwarded
|
|
// [WeakReference]'s target to the new target (it is `null` at this
|
|
// point).
|
|
to_weak_reference.set_target(weak_reference_target);
|
|
}
|
|
}
|
|
if (root_copy != Marker()) {
|
|
ObjectPtr array;
|
|
array = TryBuildArrayOfObjectsToRehash(
|
|
fast_forward_map_.raw_objects_to_rehash_);
|
|
if (array == Marker()) return root_copy;
|
|
raw_objects_to_rehash_ = Array::RawCast(array);
|
|
|
|
array = TryBuildArrayOfObjectsToRehash(
|
|
fast_forward_map_.raw_expandos_to_rehash_);
|
|
if (array == Marker()) return root_copy;
|
|
raw_expandos_to_rehash_ = Array::RawCast(array);
|
|
}
|
|
return root_copy;
|
|
}
|
|
|
|
ObjectPtr TryBuildArrayOfObjectsToRehash(
|
|
const GrowableArray<ObjectPtr>& objects_to_rehash) {
|
|
const intptr_t length = objects_to_rehash.length();
|
|
if (length == 0) return Object::null();
|
|
|
|
const intptr_t size = Array::InstanceSize(length);
|
|
const uword array_addr = new_space_->TryAllocateNoSafepoint(thread_, size);
|
|
if (array_addr == 0) {
|
|
exception_msg_ = kFastAllocationFailed;
|
|
return Marker();
|
|
}
|
|
|
|
const uword header_size =
|
|
UntaggedObject::SizeTag::SizeFits(size) ? size : 0;
|
|
ArrayPtr array(reinterpret_cast<UntaggedArray*>(array_addr));
|
|
SetNewSpaceTaggingWord(array, kArrayCid, header_size);
|
|
StoreCompressedPointerNoBarrier(array, OFFSET_OF(UntaggedArray, length_),
|
|
Smi::New(length));
|
|
StoreCompressedPointerNoBarrier(array,
|
|
OFFSET_OF(UntaggedArray, type_arguments_),
|
|
TypeArguments::null());
|
|
auto array_data = array.untag()->data();
|
|
for (intptr_t i = 0; i < length; ++i) {
|
|
array_data[i] = objects_to_rehash[i];
|
|
}
|
|
return array;
|
|
}
|
|
|
|
private:
|
|
friend class ObjectGraphCopier;
|
|
|
|
void FastCopyObject(ObjectPtr from, ObjectPtr to) {
|
|
const uword tags = TagsFromUntaggedObject(from.untag());
|
|
const intptr_t cid = UntaggedObject::ClassIdTag::decode(tags);
|
|
const intptr_t size = UntaggedObject::SizeTag::decode(tags);
|
|
|
|
// Ensure the last word is GC-safe (our heap objects are 2-word aligned, the
|
|
// object header stores the size in multiples of kObjectAlignment, the GC
|
|
// uses the information from the header and therefore might visit one slot
|
|
// more than the actual size of the instance).
|
|
*reinterpret_cast<ObjectPtr*>(UntaggedObject::ToAddr(to) +
|
|
from.untag()->HeapSize() - kWordSize) =
|
|
nullptr;
|
|
SetNewSpaceTaggingWord(to, cid, size);
|
|
|
|
// Fall back to virtual variant for predefined classes
|
|
if (cid < kNumPredefinedCids && cid != kInstanceCid) {
|
|
CopyPredefinedInstance(from, to, cid);
|
|
return;
|
|
}
|
|
const auto bitmap = class_table_->GetUnboxedFieldsMapAt(cid);
|
|
CopyUserdefinedInstance(Instance::RawCast(from), Instance::RawCast(to),
|
|
bitmap);
|
|
if (cid == expando_cid_) {
|
|
EnqueueExpandoToRehash(to);
|
|
}
|
|
}
|
|
|
|
ArrayPtr raw_objects_to_rehash_ = Array::null();
|
|
ArrayPtr raw_expandos_to_rehash_ = Array::null();
|
|
};
|
|
|
|
class SlowObjectCopy : public ObjectCopy<SlowObjectCopyBase> {
|
|
public:
|
|
SlowObjectCopy(Thread* thread, IdentityMap* map)
|
|
: ObjectCopy(thread, map),
|
|
objects_to_rehash_(Array::Handle(thread->zone())),
|
|
expandos_to_rehash_(Array::Handle(thread->zone())) {}
|
|
~SlowObjectCopy() {}
|
|
|
|
ObjectPtr ContinueCopyGraphSlow(const Object& root,
|
|
const Object& fast_root_copy) {
|
|
auto& root_copy = Object::Handle(Z, fast_root_copy.ptr());
|
|
if (root_copy.ptr() == Marker()) {
|
|
root_copy = Forward(TagsFromUntaggedObject(root.ptr().untag()), root);
|
|
}
|
|
|
|
WeakProperty& weak_property = WeakProperty::Handle(Z);
|
|
Object& from = Object::Handle(Z);
|
|
Object& to = Object::Handle(Z);
|
|
while (true) {
|
|
if (slow_forward_map_.fill_cursor_ ==
|
|
slow_forward_map_.from_to_.Length()) {
|
|
break;
|
|
}
|
|
|
|
// Run fixpoint to copy all objects.
|
|
while (slow_forward_map_.fill_cursor_ <
|
|
slow_forward_map_.from_to_.Length()) {
|
|
const intptr_t index = slow_forward_map_.fill_cursor_;
|
|
from = slow_forward_map_.from_to_.At(index);
|
|
to = slow_forward_map_.from_to_.At(index + 1);
|
|
CopyObject(from, to);
|
|
slow_forward_map_.fill_cursor_ += 2;
|
|
if (exception_msg_ != nullptr) {
|
|
return Marker();
|
|
}
|
|
// To maintain responsiveness we regularly check whether safepoints are
|
|
// requested.
|
|
thread_->CheckForSafepoint();
|
|
}
|
|
|
|
// Possibly forward values of [WeakProperty]s if keys became reachable.
|
|
intptr_t i = 0;
|
|
auto& weak_properties = slow_forward_map_.weak_properties_;
|
|
while (i < weak_properties.length()) {
|
|
const auto& from_weak_property = *weak_properties[i];
|
|
to = slow_forward_map_.ForwardedObject(from_weak_property.key());
|
|
if (to.ptr() != Marker()) {
|
|
weak_property ^=
|
|
slow_forward_map_.ForwardedObject(from_weak_property.ptr());
|
|
|
|
// The key became reachable so we'll change the forwarded
|
|
// [WeakProperty]'s key to the new key (it is `null` at this point).
|
|
weak_property.set_key(to);
|
|
|
|
// Since the key has become strongly reachable in the copied graph,
|
|
// we'll also need to forward the value.
|
|
ForwardCompressedPointer(from_weak_property, weak_property,
|
|
OFFSET_OF(UntaggedWeakProperty, value_));
|
|
|
|
// We don't need to process this [WeakProperty] again.
|
|
const intptr_t last = weak_properties.length() - 1;
|
|
if (i < last) {
|
|
weak_properties[i] = weak_properties[last];
|
|
weak_properties.SetLength(last);
|
|
continue;
|
|
}
|
|
}
|
|
i++;
|
|
}
|
|
}
|
|
|
|
// After the fix point with [WeakProperty]s do [WeakReference]s.
|
|
WeakReference& weak_reference = WeakReference::Handle(Z);
|
|
auto& weak_references = slow_forward_map_.weak_references_;
|
|
for (intptr_t i = 0; i < weak_references.length(); i++) {
|
|
const auto& from_weak_reference = *weak_references[i];
|
|
to = slow_forward_map_.ForwardedObject(from_weak_reference.target());
|
|
if (to.ptr() != Marker()) {
|
|
weak_reference ^=
|
|
slow_forward_map_.ForwardedObject(from_weak_reference.ptr());
|
|
|
|
// The target became reachable so we'll change the forwarded
|
|
// [WeakReference]'s target to the new target (it is `null` at this
|
|
// point).
|
|
weak_reference.set_target(to);
|
|
}
|
|
}
|
|
|
|
objects_to_rehash_ =
|
|
BuildArrayOfObjectsToRehash(slow_forward_map_.objects_to_rehash_);
|
|
expandos_to_rehash_ =
|
|
BuildArrayOfObjectsToRehash(slow_forward_map_.expandos_to_rehash_);
|
|
return root_copy.ptr();
|
|
}
|
|
|
|
ArrayPtr BuildArrayOfObjectsToRehash(
|
|
const GrowableArray<const Object*>& objects_to_rehash) {
|
|
const intptr_t length = objects_to_rehash.length();
|
|
if (length == 0) return Array::null();
|
|
|
|
const auto& array = Array::Handle(zone_, Array::New(length));
|
|
for (intptr_t i = 0; i < length; ++i) {
|
|
array.SetAt(i, *objects_to_rehash[i]);
|
|
}
|
|
return array.ptr();
|
|
}
|
|
|
|
private:
|
|
friend class ObjectGraphCopier;
|
|
|
|
void CopyObject(const Object& from, const Object& to) {
|
|
const auto cid = from.GetClassId();
|
|
|
|
// Fall back to virtual variant for predefined classes
|
|
if (cid < kNumPredefinedCids && cid != kInstanceCid) {
|
|
CopyPredefinedInstance(from, to, cid);
|
|
return;
|
|
}
|
|
const auto bitmap = class_table_->GetUnboxedFieldsMapAt(cid);
|
|
CopyUserdefinedInstance(from, to, bitmap);
|
|
if (cid == expando_cid_) {
|
|
EnqueueExpandoToRehash(to);
|
|
}
|
|
}
|
|
|
|
Array& objects_to_rehash_;
|
|
Array& expandos_to_rehash_;
|
|
};
|
|
|
|
class ObjectGraphCopier : public StackResource {
|
|
public:
|
|
explicit ObjectGraphCopier(Thread* thread)
|
|
: StackResource(thread),
|
|
thread_(thread),
|
|
zone_(thread->zone()),
|
|
map_(thread),
|
|
fast_object_copy_(thread_, &map_),
|
|
slow_object_copy_(thread_, &map_) {}
|
|
|
|
// Result will be
|
|
// [
|
|
// <message>,
|
|
// <collection-lib-objects-to-rehash>,
|
|
// <core-lib-objects-to-rehash>,
|
|
// ]
|
|
ObjectPtr CopyObjectGraph(const Object& root) {
|
|
const char* volatile exception_msg = nullptr;
|
|
auto& result = Object::Handle(zone_);
|
|
|
|
{
|
|
LongJumpScope jump; // e.g. for OOMs.
|
|
if (setjmp(*jump.Set()) == 0) {
|
|
result = CopyObjectGraphInternal(root, &exception_msg);
|
|
// Any allocated external typed data must have finalizers attached so
|
|
// memory will get free()ed.
|
|
slow_object_copy_.slow_forward_map_.FinalizeExternalTypedData();
|
|
} else {
|
|
// Any allocated external typed data must have finalizers attached so
|
|
// memory will get free()ed.
|
|
slow_object_copy_.slow_forward_map_.FinalizeExternalTypedData();
|
|
|
|
// The copy failed due to non-application error (e.g. OOM error),
|
|
// propagate this error.
|
|
result = thread_->StealStickyError();
|
|
RELEASE_ASSERT(result.IsError());
|
|
}
|
|
}
|
|
|
|
if (result.IsError()) {
|
|
Exceptions::PropagateError(Error::Cast(result));
|
|
UNREACHABLE();
|
|
}
|
|
ASSERT(result.IsArray());
|
|
auto& result_array = Array::Cast(result);
|
|
if (result_array.At(0) == Marker()) {
|
|
ASSERT(exception_msg != nullptr);
|
|
auto& unexpected_object_ = Object::Handle(zone_, result_array.At(1));
|
|
if (!unexpected_object_.IsNull()) {
|
|
exception_msg =
|
|
OS::SCreate(zone_, "%s\n%s", exception_msg,
|
|
FindRetainingPath(
|
|
zone_, thread_->isolate(), root, unexpected_object_,
|
|
TraversalRules::kInternalToIsolateGroup));
|
|
}
|
|
ThrowException(exception_msg);
|
|
UNREACHABLE();
|
|
}
|
|
|
|
// The copy was successful, then detach transferable data from the sender
|
|
// and attach to the copied graph.
|
|
slow_object_copy_.slow_forward_map_.FinalizeTransferables();
|
|
return result.ptr();
|
|
}
|
|
|
|
intptr_t allocated_bytes() { return allocated_bytes_; }
|
|
|
|
intptr_t copied_objects() { return copied_objects_; }
|
|
|
|
private:
|
|
ObjectPtr CopyObjectGraphInternal(const Object& root,
|
|
const char* volatile* exception_msg) {
|
|
const auto& result_array = Array::Handle(zone_, Array::New(3));
|
|
if (!root.ptr()->IsHeapObject()) {
|
|
result_array.SetAt(0, root);
|
|
return result_array.ptr();
|
|
}
|
|
const uword tags = TagsFromUntaggedObject(root.ptr().untag());
|
|
if (CanShareObject(root.ptr(), tags)) {
|
|
result_array.SetAt(0, root);
|
|
return result_array.ptr();
|
|
}
|
|
if (!fast_object_copy_.CanCopyObject(tags, root.ptr())) {
|
|
ASSERT(fast_object_copy_.exception_msg_ != nullptr);
|
|
*exception_msg = fast_object_copy_.exception_msg_;
|
|
result_array.SetAt(0, Object::Handle(zone_, Marker()));
|
|
result_array.SetAt(1, fast_object_copy_.exception_unexpected_object_);
|
|
return result_array.ptr();
|
|
}
|
|
|
|
// We try a fast new-space only copy first that will not use any barriers.
|
|
auto& result = Object::Handle(Z, Marker());
|
|
|
|
// All allocated but non-initialized heap objects have to be made GC-visible
|
|
// at this point.
|
|
if (FLAG_enable_fast_object_copy) {
|
|
{
|
|
NoSafepointScope no_safepoint_scope;
|
|
|
|
result = fast_object_copy_.TryCopyGraphFast(root.ptr());
|
|
if (result.ptr() != Marker()) {
|
|
if (fast_object_copy_.exception_msg_ == nullptr) {
|
|
result_array.SetAt(0, result);
|
|
fast_object_copy_.tmp_ = fast_object_copy_.raw_objects_to_rehash_;
|
|
result_array.SetAt(1, fast_object_copy_.tmp_);
|
|
fast_object_copy_.tmp_ = fast_object_copy_.raw_expandos_to_rehash_;
|
|
result_array.SetAt(2, fast_object_copy_.tmp_);
|
|
HandlifyExternalTypedData();
|
|
HandlifyTransferables();
|
|
allocated_bytes_ =
|
|
fast_object_copy_.fast_forward_map_.allocated_bytes;
|
|
copied_objects_ =
|
|
fast_object_copy_.fast_forward_map_.fill_cursor_ / 2 -
|
|
/*null_entry=*/1;
|
|
return result_array.ptr();
|
|
}
|
|
|
|
// There are left-over uninitialized objects we'll have to make GC
|
|
// visible.
|
|
SwitchToSlowForwardingList();
|
|
}
|
|
}
|
|
|
|
if (FLAG_gc_on_foc_slow_path) {
|
|
// We force the GC to compact, which is more likely to discover
|
|
// untracked pointers (and other issues, like incorrect class table).
|
|
thread_->heap()->CollectAllGarbage(GCReason::kDebugging,
|
|
/*compact=*/true);
|
|
}
|
|
|
|
ObjectifyFromToObjects();
|
|
|
|
// Fast copy failed due to
|
|
// - either failure to allocate into new space
|
|
// - or failure to copy object which we cannot copy
|
|
ASSERT(fast_object_copy_.exception_msg_ != nullptr);
|
|
if (fast_object_copy_.exception_msg_ != kFastAllocationFailed) {
|
|
*exception_msg = fast_object_copy_.exception_msg_;
|
|
result_array.SetAt(0, Object::Handle(zone_, Marker()));
|
|
result_array.SetAt(1, fast_object_copy_.exception_unexpected_object_);
|
|
return result_array.ptr();
|
|
}
|
|
ASSERT(fast_object_copy_.exception_msg_ == kFastAllocationFailed);
|
|
}
|
|
|
|
// Use the slow copy approach.
|
|
result = slow_object_copy_.ContinueCopyGraphSlow(root, result);
|
|
ASSERT((result.ptr() == Marker()) ==
|
|
(slow_object_copy_.exception_msg_ != nullptr));
|
|
if (result.ptr() == Marker()) {
|
|
*exception_msg = slow_object_copy_.exception_msg_;
|
|
result_array.SetAt(0, Object::Handle(zone_, Marker()));
|
|
result_array.SetAt(1, slow_object_copy_.exception_unexpected_object_);
|
|
return result_array.ptr();
|
|
}
|
|
|
|
result_array.SetAt(0, result);
|
|
result_array.SetAt(1, slow_object_copy_.objects_to_rehash_);
|
|
result_array.SetAt(2, slow_object_copy_.expandos_to_rehash_);
|
|
allocated_bytes_ = slow_object_copy_.slow_forward_map_.allocated_bytes;
|
|
copied_objects_ =
|
|
slow_object_copy_.slow_forward_map_.fill_cursor_ / 2 - /*null_entry=*/1;
|
|
return result_array.ptr();
|
|
}
|
|
|
|
void SwitchToSlowForwardingList() {
|
|
auto& fast_forward_map = fast_object_copy_.fast_forward_map_;
|
|
auto& slow_forward_map = slow_object_copy_.slow_forward_map_;
|
|
|
|
MakeUninitializedNewSpaceObjectsGCSafe();
|
|
HandlifyTransferables();
|
|
HandlifyWeakProperties();
|
|
HandlifyWeakReferences();
|
|
HandlifyExternalTypedData();
|
|
HandlifyObjectsToReHash();
|
|
HandlifyExpandosToReHash();
|
|
HandlifyFromToObjects();
|
|
slow_forward_map.fill_cursor_ = fast_forward_map.fill_cursor_;
|
|
slow_forward_map.allocated_bytes = fast_forward_map.allocated_bytes;
|
|
}
|
|
|
|
void MakeUninitializedNewSpaceObjectsGCSafe() {
|
|
auto& fast_forward_map = fast_object_copy_.fast_forward_map_;
|
|
const auto length = fast_forward_map.raw_from_to_.length();
|
|
const auto cursor = fast_forward_map.fill_cursor_;
|
|
for (intptr_t i = cursor; i < length; i += 2) {
|
|
auto from = fast_forward_map.raw_from_to_[i];
|
|
auto to = fast_forward_map.raw_from_to_[i + 1];
|
|
const uword tags = TagsFromUntaggedObject(from.untag());
|
|
const intptr_t cid = UntaggedObject::ClassIdTag::decode(tags);
|
|
// External typed data is already initialized.
|
|
if (!IsExternalTypedDataClassId(cid) && !IsTypedDataViewClassId(cid) &&
|
|
!IsUnmodifiableTypedDataViewClassId(cid)) {
|
|
#if defined(DART_COMPRESSED_POINTERS)
|
|
const bool compressed = true;
|
|
#else
|
|
const bool compressed = false;
|
|
#endif
|
|
// Mimic the old initialization behavior of Object::InitializeObject
|
|
// where the contents are initialized to Object::null(), except for
|
|
// TypedDataBase subclasses which are initialized to 0, as the contents
|
|
// of the original are translated and copied over prior to returning
|
|
// the object graph root.
|
|
if (IsTypedDataBaseClassId(cid)) {
|
|
Object::InitializeObject(reinterpret_cast<uword>(to.untag()), cid,
|
|
from.untag()->HeapSize(), compressed,
|
|
Object::from_offset<TypedDataBase>(),
|
|
Object::to_offset<TypedDataBase>());
|
|
|
|
} else {
|
|
// Remember that ptr_field_end_offset is the offset to the last Ptr
|
|
// field, not the offset just past it.
|
|
const uword ptr_field_end_offset =
|
|
from.untag()->HeapSize() -
|
|
(compressed ? kCompressedWordSize : kWordSize);
|
|
Object::InitializeObject(reinterpret_cast<uword>(to.untag()), cid,
|
|
from.untag()->HeapSize(), compressed,
|
|
Object::from_offset<Object>(),
|
|
ptr_field_end_offset);
|
|
}
|
|
UpdateLengthField(cid, from, to);
|
|
}
|
|
}
|
|
}
|
|
void HandlifyTransferables() {
|
|
Handlify(&fast_object_copy_.fast_forward_map_.raw_transferables_from_to_,
|
|
&slow_object_copy_.slow_forward_map_.transferables_from_to_);
|
|
}
|
|
void HandlifyWeakProperties() {
|
|
Handlify(&fast_object_copy_.fast_forward_map_.raw_weak_properties_,
|
|
&slow_object_copy_.slow_forward_map_.weak_properties_);
|
|
}
|
|
void HandlifyWeakReferences() {
|
|
Handlify(&fast_object_copy_.fast_forward_map_.raw_weak_references_,
|
|
&slow_object_copy_.slow_forward_map_.weak_references_);
|
|
}
|
|
void HandlifyExternalTypedData() {
|
|
Handlify(&fast_object_copy_.fast_forward_map_.raw_external_typed_data_to_,
|
|
&slow_object_copy_.slow_forward_map_.external_typed_data_);
|
|
}
|
|
void HandlifyObjectsToReHash() {
|
|
Handlify(&fast_object_copy_.fast_forward_map_.raw_objects_to_rehash_,
|
|
&slow_object_copy_.slow_forward_map_.objects_to_rehash_);
|
|
}
|
|
void HandlifyExpandosToReHash() {
|
|
Handlify(&fast_object_copy_.fast_forward_map_.raw_expandos_to_rehash_,
|
|
&slow_object_copy_.slow_forward_map_.expandos_to_rehash_);
|
|
}
|
|
template <typename PtrType, typename HandleType>
|
|
void Handlify(GrowableArray<PtrType>* from,
|
|
GrowableArray<const HandleType*>* to) {
|
|
const auto length = from->length();
|
|
if (length > 0) {
|
|
to->Resize(length);
|
|
for (intptr_t i = 0; i < length; i++) {
|
|
(*to)[i] = &HandleType::Handle(Z, (*from)[i]);
|
|
}
|
|
from->Clear();
|
|
}
|
|
}
|
|
void HandlifyFromToObjects() {
|
|
auto& fast_forward_map = fast_object_copy_.fast_forward_map_;
|
|
auto& slow_forward_map = slow_object_copy_.slow_forward_map_;
|
|
const intptr_t length = fast_forward_map.raw_from_to_.length();
|
|
slow_forward_map.from_to_transition_.Resize(length);
|
|
for (intptr_t i = 0; i < length; i++) {
|
|
slow_forward_map.from_to_transition_[i] =
|
|
&PassiveObject::Handle(Z, fast_forward_map.raw_from_to_[i]);
|
|
}
|
|
ASSERT(slow_forward_map.from_to_transition_.length() == length);
|
|
fast_forward_map.raw_from_to_.Clear();
|
|
}
|
|
void ObjectifyFromToObjects() {
|
|
auto& from_to_transition =
|
|
slow_object_copy_.slow_forward_map_.from_to_transition_;
|
|
auto& from_to = slow_object_copy_.slow_forward_map_.from_to_;
|
|
intptr_t length = from_to_transition.length();
|
|
from_to = GrowableObjectArray::New(length, Heap::kOld);
|
|
for (intptr_t i = 0; i < length; i++) {
|
|
from_to.Add(*from_to_transition[i]);
|
|
}
|
|
ASSERT(from_to.Length() == length);
|
|
from_to_transition.Clear();
|
|
}
|
|
|
|
void ThrowException(const char* exception_msg) {
|
|
const auto& msg_obj = String::Handle(Z, String::New(exception_msg));
|
|
const auto& args = Array::Handle(Z, Array::New(1));
|
|
args.SetAt(0, msg_obj);
|
|
Exceptions::ThrowByType(Exceptions::kArgument, args);
|
|
UNREACHABLE();
|
|
}
|
|
|
|
Thread* thread_;
|
|
Zone* zone_;
|
|
IdentityMap map_;
|
|
FastObjectCopy fast_object_copy_;
|
|
SlowObjectCopy slow_object_copy_;
|
|
intptr_t copied_objects_ = 0;
|
|
intptr_t allocated_bytes_ = 0;
|
|
};
|
|
|
|
ObjectPtr CopyMutableObjectGraph(const Object& object) {
|
|
auto thread = Thread::Current();
|
|
TIMELINE_DURATION(thread, Isolate, "CopyMutableObjectGraph");
|
|
ObjectGraphCopier copier(thread);
|
|
ObjectPtr result = copier.CopyObjectGraph(object);
|
|
#if defined(SUPPORT_TIMELINE)
|
|
if (tbes.enabled()) {
|
|
tbes.SetNumArguments(2);
|
|
tbes.FormatArgument(0, "CopiedObjects", "%" Pd, copier.copied_objects());
|
|
tbes.FormatArgument(1, "AllocatedBytes", "%" Pd, copier.allocated_bytes());
|
|
}
|
|
#endif
|
|
return result;
|
|
}
|
|
|
|
} // namespace dart
|