[kernel] Change constant references; remove need for map

Previously constants were referenced via there relative binary offset,
i.e. a constant reference saying 42 meant at byte offset 42 relative to
the start of the constant table in the binary. This was done to be able
to (on the VM side) read the needed constants lazily. It meant, though,
that constants had to be stored in a map, mapping from the byte position
to the constant.

This change adds a level of indirection when needing the lazy reading,
but lets the constant references reference the constant number instead
so that a constant reference saying 42 means constant number 42,
i.e. constants can be stored in a list instead of in a map.
This is done on the dart side, but the VM still stores it in a map.

The level of indirection is a tabel next to the constant table where
each entry has constant size (4 bytes) from which one can read the
relative byte offset into the constant table from the constant number,
thus still being able to read needed constants lazily.

This CL also cleans up a leftover where for instance double constants
had their textual representation saved as a string in the string indexer
(and thus the output dill) even though they were never referenced.

File size changes:
* Platform: increses 7,816 bytes.
* Compile of dart2js (including platform): decreases 71,424 bytes.

Speed changes:
* Adding `UserTag`s to the code and looking at observatories cpu profile
  on a `pkg/kernel/test/binary_bench.dart --golem AstFromBinaryLazy`
  run of a compile of dart2js reading the constant table has gone
  from ~10% to ~5%.
* Doing statistics on
  `pkg/kernel/test/binary_bench.dart --raw AstFromBinaryLazy` run of a
  compile of dart2js says -6.28169% +/- 4.97269%.
* Golem runs has nothing above the noise level.
  It does say "AstFromBinaryLazy (Intel Xeon) -4.006% (0.4 noise)" and
  "AstFromBinaryEagerP50 (Intel Core i5) -8.929% (0.6 noise)" though.

TEST=All tests running the VM tests this.

Change-Id: I07ead457527a4477de803ee55ba742f5557413d3
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/196925
Commit-Queue: Jens Johansen <jensj@google.com>
Reviewed-by: Martin Kustermann <kustermann@google.com>
Reviewed-by: Johnni Winther <johnniwinther@google.com>
This commit is contained in:
Jens Johansen 2021-05-04 11:08:31 +00:00 committed by commit-bot@chromium.org
parent 4839c1e689
commit da4343348e
15 changed files with 216 additions and 133 deletions

View file

@ -29,6 +29,7 @@ class BinaryMdDillReader {
Map _dillStringsPointer;
int verboseLevel = 0;
bool _ranSetup = false;
List<String> readingStack = [];
BinaryMdDillReader(this._binaryMdContent, this._dillContent);
@ -101,11 +102,13 @@ class BinaryMdDillReader {
int numLibs;
int binaryOffsetForSourceTable;
int binaryOffsetForConstantTable;
int binaryOffsetForConstantTableIndex;
int binaryOffsetForCanonicalNames;
int binaryOffsetForMetadataPayloads;
int binaryOffsetForMetadataMappings;
int binaryOffsetForStringTable;
int binaryOffsetForConstantTable;
int binaryOffsetForStartOfComponentIndex;
int mainMethodReference;
/// Read the dill file data, parsing it into a Map.
@ -119,11 +122,15 @@ class BinaryMdDillReader {
// Skip to the start of the index.
_binaryOffset = _dillContent.length -
((numLibs + 1) + 10 /* number of fixed fields */) * 4;
((numLibs + 1) + 12 /* number of fixed fields */) * 4;
// Read index.
binaryOffsetForSourceTable = _peekUint32();
_binaryOffset += 4;
binaryOffsetForConstantTable = _peekUint32();
_binaryOffset += 4;
binaryOffsetForConstantTableIndex = _peekUint32();
_binaryOffset += 4;
binaryOffsetForCanonicalNames = _peekUint32();
_binaryOffset += 4;
binaryOffsetForMetadataPayloads = _peekUint32();
@ -132,7 +139,7 @@ class BinaryMdDillReader {
_binaryOffset += 4;
binaryOffsetForStringTable = _peekUint32();
_binaryOffset += 4;
binaryOffsetForConstantTable = _peekUint32();
binaryOffsetForStartOfComponentIndex = _peekUint32();
_binaryOffset += 4;
mainMethodReference = _peekUint32();
_binaryOffset += 4;
@ -141,13 +148,13 @@ class BinaryMdDillReader {
_binaryOffset = binaryOffsetForStringTable;
var saved = _readingInstructions["ComponentFile"];
_readingInstructions["ComponentFile"] = ["StringTable strings;"];
_readBinary("ComponentFile");
_readBinary("ComponentFile", "");
_readingInstructions["ComponentFile"] = saved;
_binaryOffset = 0;
_depth = 0;
// Hack end.
Map componentFile = _readBinary("ComponentFile");
Map componentFile = _readBinary("ComponentFile", "");
if (_binaryOffset != _dillContent.length) {
throw "Didn't read the entire binary: "
"Only read $_binaryOffset of ${_dillContent.length} bytes. "
@ -384,7 +391,7 @@ class BinaryMdDillReader {
/// Actually read the binary dill file. Read type [what] at the current
/// binary position as specified by field [_binaryOffset].
dynamic _readBinary(String what) {
dynamic _readBinary(String what, String callerInstruction) {
++_depth;
what = _remapWhat(what);
@ -417,6 +424,9 @@ class BinaryMdDillReader {
throw "Didn't find instructions for '$what'";
}
readingStack.add(orgWhat);
readingStack.add(callerInstruction);
Map<String, dynamic> vars = {};
if (verboseLevel > 1) {
print("".padLeft(_depth * 2) + " -> $what ($orgWhat @ $orgPosition)");
@ -516,25 +526,36 @@ class BinaryMdDillReader {
count == "libraryCount + 1") {
intCount = numLibs + 1;
}
if (intCount < 0 &&
type == "UInt32" &&
_depth == 2 &&
count == "length" &&
readingStack.last == "RList<UInt32> constantsMapping;") {
int prevBinaryOffset = _binaryOffset;
// Hack: Use the ComponentIndex to go to the end and read the length.
_binaryOffset = binaryOffsetForCanonicalNames - 4;
intCount = _peekUint32();
_binaryOffset = prevBinaryOffset;
}
if (intCount >= 0) {
readNothingIsOk = intCount == 0;
List<dynamic> value = new List.filled(intCount, null);
for (int i = 0; i < intCount; ++i) {
int oldOffset2 = _binaryOffset;
value[i] = _readBinary(type);
value[i] = _readBinary(type, instruction);
if (_binaryOffset <= oldOffset2) {
throw "Didn't read anything for $type @ $_binaryOffset";
}
}
vars[name] = value;
} else {
throw "Array of unknown size ($count)";
throw "Array of unknown size ($instruction, $type, $count)";
}
} else {
// Not an array, read the single field recursively.
type = _lookupGenericType(typeNames, type, types);
dynamic value = _readBinary(type);
dynamic value = _readBinary(type, instruction);
vars[name] = value;
_checkTag(instruction, value);
}
@ -551,6 +572,8 @@ class BinaryMdDillReader {
}
--_depth;
readingStack.removeLast();
readingStack.removeLast();
return vars;
}

View file

@ -940,6 +940,7 @@ rarely
ratio
rb
rc
rd
re
reach
reachabilities

View file

@ -87,7 +87,7 @@ type StringReference {
}
type ConstantReference {
UInt offset; // Byte offset into the Component's constants.
UInt index; // Index into [constantsMapping] and [constants].
}
type SourceInfo {
@ -147,16 +147,17 @@ type CanonicalName {
type ComponentFile {
UInt32 magic = 0x90ABCDEF;
UInt32 formatVersion = 63;
UInt32 formatVersion = 64;
Byte[10] shortSdkHash;
List<String> problemsAsJson; // Described in problems.md.
Library[] libraries;
UriSource sourceMap;
List<Constant> constants;
RList<UInt32> constantsMapping; // Byte offset into the Component's constants.
List<CanonicalName> canonicalNames;
MetadataPayload[] metadataPayloads;
RList<MetadataMapping> metadataMappings;
StringTable strings;
List<Constant> constants;
ComponentIndex componentIndex;
}
@ -179,11 +180,13 @@ type MetadataMapping {
type ComponentIndex {
Byte[] 8bitAlignment; // 0-bytes to make the entire component (!) 8-byte aligned.
UInt32 binaryOffsetForSourceTable;
UInt32 binaryOffsetForConstantTable;
UInt32 binaryOffsetForConstantTableIndex;
UInt32 binaryOffsetForCanonicalNames;
UInt32 binaryOffsetForMetadataPayloads;
UInt32 binaryOffsetForMetadataMappings;
UInt32 binaryOffsetForStringTable;
UInt32 binaryOffsetForConstantTable;
UInt32 binaryOffsetForStartOfComponentIndex;
UInt32 mainMethodReference; // This is a ProcedureReference with a fixed-size integer.
UInt32 compilationMode; // enum NonNullableByDefaultCompiledMode { Disabled = 0, Weak = 1, Strong = 2, Agnostic = 3 } with a fixed-size integer.
UInt32[libraryCount + 1] libraryOffsets;

View file

@ -61,7 +61,7 @@ class CompilationModeError {
}
class _ComponentIndex {
static const int numberOfFixedFields = 10;
static const int numberOfFixedFields = 12;
final int binaryOffsetForSourceTable;
final int binaryOffsetForCanonicalNames;
@ -69,6 +69,8 @@ class _ComponentIndex {
final int binaryOffsetForMetadataMappings;
final int binaryOffsetForStringTable;
final int binaryOffsetForConstantTable;
final int binaryOffsetForConstantTableIndex;
final int binaryOffsetForStartOfComponentIndex;
final int mainMethodReference;
final NonNullableByDefaultCompiledMode compiledMode;
final List<int> libraryOffsets;
@ -82,6 +84,8 @@ class _ComponentIndex {
required this.binaryOffsetForMetadataMappings,
required this.binaryOffsetForStringTable,
required this.binaryOffsetForConstantTable,
required this.binaryOffsetForConstantTableIndex,
required this.binaryOffsetForStartOfComponentIndex,
required this.mainMethodReference,
required this.compiledMode,
required this.libraryOffsets,
@ -122,7 +126,7 @@ class BinaryBuilder {
int _byteOffset = 0;
List<String> _stringTable = const [];
List<Uri> _sourceUriTable = const [];
Map<int, Constant> _constantTable = <int, Constant>{};
List<Constant> _constantTable = const <Constant>[];
late List<CanonicalName> _linkTable;
int _transformerFlags = 0;
Library? _currentLibrary;
@ -329,9 +333,12 @@ class BinaryBuilder {
void readConstantTable() {
final int length = readUInt30();
final int startOffset = byteOffset;
// Because of "back-references" (e.g. the 10th constant referencing the 3rd
// constant) we can't use List.generate.
_constantTable =
new List<Constant>.filled(length, dummyConstant, growable: false);
for (int i = 0; i < length; i++) {
_constantTable[byteOffset - startOffset] = readConstantTableEntry();
_constantTable[i] = readConstantTableEntry();
}
}
@ -457,10 +464,11 @@ class BinaryBuilder {
}
Constant readConstantReference() {
final int offset = readUInt30();
Constant? constant = _constantTable[offset];
assert(constant != null, "No constant found at offset $offset.");
return constant!;
final int index = readUInt30();
Constant constant = _constantTable[index];
assert(!identical(constant, dummyConstant),
"No constant found at index $index.");
return constant;
}
List<Constant> _readConstantReferenceList() {
@ -702,11 +710,15 @@ class BinaryBuilder {
// Now read the component index.
int binaryOffsetForSourceTable = _componentStartOffset + readUint32();
int binaryOffsetForConstantTable = _componentStartOffset + readUint32();
int binaryOffsetForConstantTableIndex =
_componentStartOffset + readUint32();
int binaryOffsetForCanonicalNames = _componentStartOffset + readUint32();
int binaryOffsetForMetadataPayloads = _componentStartOffset + readUint32();
int binaryOffsetForMetadataMappings = _componentStartOffset + readUint32();
int binaryOffsetForStringTable = _componentStartOffset + readUint32();
int binaryOffsetForConstantTable = _componentStartOffset + readUint32();
int binaryOffsetForStartOfComponentIndex =
_componentStartOffset + readUint32();
int mainMethodReference = readUint32();
NonNullableByDefaultCompiledMode compiledMode =
NonNullableByDefaultCompiledMode.values[readUint32()];
@ -726,6 +738,9 @@ class BinaryBuilder {
binaryOffsetForMetadataMappings: binaryOffsetForMetadataMappings,
binaryOffsetForStringTable: binaryOffsetForStringTable,
binaryOffsetForConstantTable: binaryOffsetForConstantTable,
binaryOffsetForConstantTableIndex: binaryOffsetForConstantTableIndex,
binaryOffsetForStartOfComponentIndex:
binaryOffsetForStartOfComponentIndex,
mainMethodReference: mainMethodReference,
compiledMode: compiledMode);
}
@ -804,6 +819,7 @@ class BinaryBuilder {
_byteOffset = index.binaryOffsetForConstantTable;
readConstantTable();
// We don't need the constant table index on the dart side.
int numberOfLibraries = index.libraryCount;

View file

@ -23,7 +23,7 @@ class BinaryPrinter implements Visitor<void>, BinarySink {
SwitchCaseIndexer? _switchCaseIndexer;
TypeParameterIndexer _typeParameterIndexer = new TypeParameterIndexer();
final StringIndexer stringIndexer;
late ConstantIndexer _constantIndexer;
final ConstantIndexer _constantIndexer;
final UriIndexer _sourceUriIndexer = new UriIndexer();
bool _currentlyInNonimplementation = false;
final List<bool> _sourcesFromRealImplementation = <bool>[];
@ -36,8 +36,6 @@ class BinaryPrinter implements Visitor<void>, BinarySink {
final BufferedSink _mainSink;
final BufferedSink _metadataSink;
final BytesSink _constantsBytesSink;
late BufferedSink _constantsSink;
late BufferedSink _sink;
final bool includeSources;
final bool includeOffsets;
@ -51,9 +49,11 @@ class BinaryPrinter implements Visitor<void>, BinarySink {
int _binaryOffsetForMetadataPayloads = -1;
int _binaryOffsetForMetadataMappings = -1;
int _binaryOffsetForStringTable = -1;
int _binaryOffsetForConstantTableIndex = -1;
int _binaryOffsetForConstantTable = -1;
late List<CanonicalName> _canonicalNameList;
bool _canonicalNameListDone = false;
Set<CanonicalName> _knownCanonicalNameNonRootTops = new Set<CanonicalName>();
Library? _currentLibrary;
@ -69,10 +69,8 @@ class BinaryPrinter implements Visitor<void>, BinarySink {
this.includeOffsets = true})
: _mainSink = new BufferedSink(sink),
_metadataSink = new BufferedSink(new BytesSink()),
_constantsBytesSink = new BytesSink(),
stringIndexer = stringIndexer ?? new StringIndexer() {
_constantsSink = new BufferedSink(_constantsBytesSink);
_constantIndexer = new ConstantIndexer(this.stringIndexer, this);
stringIndexer = stringIndexer ?? new StringIndexer(),
_constantIndexer = new ConstantIndexer() {
_sink = _mainSink;
}
@ -181,21 +179,34 @@ class BinaryPrinter implements Visitor<void>, BinarySink {
writeUInt30(_constantIndexer.put(constant));
}
void writeConstantTable(ConstantIndexer indexer) {
void writeConstantTable() {
_binaryOffsetForConstantTable = getBufferOffset();
writeUInt30(indexer.entries.length);
assert(identical(_sink, _mainSink));
_constantsSink.flushAndDestroy();
writeBytes(_constantsBytesSink.builder.takeBytes());
writeUInt30(_constantIndexer.entries.length);
assert(_constantIndexer.entries.length == _constantIndexer.offsets.length);
for (int i = 0; i < _constantIndexer.entries.length; i++) {
final Constant entry = _constantIndexer.entries[i];
_constantIndexer.offsets[i] =
getBufferOffset() - _binaryOffsetForConstantTable;
writeConstantTableEntry(entry);
}
}
int writeConstantTableEntry(Constant constant) {
void writeConstantTableIndex() {
_binaryOffsetForConstantTableIndex = getBufferOffset();
assert(identical(_sink, _mainSink));
assert(_constantIndexer.entries.length == _constantIndexer.offsets.length);
for (int i = 0; i < _constantIndexer.offsets.length; i++) {
final int relativeOffset = _constantIndexer.offsets[i];
assert(relativeOffset >= 0);
writeUInt32(relativeOffset);
}
writeUInt32(_constantIndexer.entries.length);
}
void writeConstantTableEntry(Constant constant) {
TypeParameterIndexer oldTypeParameterIndexer = _typeParameterIndexer;
_typeParameterIndexer = new TypeParameterIndexer();
BufferedSink oldSink = _sink;
_sink = _constantsSink;
int initialOffset = _sink.offset;
if (constant is NullConstant) {
writeByte(ConstantTag.NullConstant);
} else if (constant is BoolConstant) {
@ -264,9 +275,7 @@ class BinaryPrinter implements Visitor<void>, BinarySink {
} else {
throw new ArgumentError('Unsupported constant $constant');
}
_sink = oldSink;
_typeParameterIndexer = oldTypeParameterIndexer;
return _constantsSink.offset - initialOffset;
}
void writeDartType(DartType type) {
@ -491,6 +500,7 @@ class BinaryPrinter implements Visitor<void>, BinarySink {
void writeLinkTable(Component component) {
_binaryOffsetForLinkTable = getBufferOffset();
writeList(_canonicalNameList, writeCanonicalNameEntry);
_canonicalNameListDone = true;
}
void indexLinkTable(Component component) {
@ -506,6 +516,7 @@ class BinaryPrinter implements Visitor<void>, BinarySink {
void _indexLinkTableInternal(CanonicalName node) {
node.index = _canonicalNameList.length;
assert(!_canonicalNameListDone);
_canonicalNameList.add(node);
Iterable<CanonicalName>? children = node.childrenOrNull;
if (children != null) {
@ -557,10 +568,14 @@ class BinaryPrinter implements Visitor<void>, BinarySink {
}
writeLibraries(component);
writeUriToSource(component.uriToSource);
// Writing constants can add both strings and canonical names.
writeConstantTable();
writeConstantTableIndex();
// Writing canonical names can add strings.
writeLinkTable(component);
// Writing metadata sections can add strings.
_writeMetadataSection(component);
writeStringTable(stringIndexer);
writeConstantTable(_constantIndexer);
List<Library> libraries = component.libraries;
if (libraryFilter != null) {
List<Library> librariesNew = <Library>[];
@ -727,9 +742,10 @@ class BinaryPrinter implements Visitor<void>, BinarySink {
const int kernelFileAlignment = 8;
// Keep this in sync with number of writeUInt32 below.
int numComponentIndexEntries = 8 + libraryOffsets.length + 3;
int numComponentIndexEntries = 10 + libraryOffsets.length + 3;
int componentIndexOffset = getBufferOffset();
int unalignedSize = getBufferOffset() + numComponentIndexEntries * 4;
int unalignedSize = componentIndexOffset + numComponentIndexEntries * 4;
int padding =
((unalignedSize + kernelFileAlignment - 1) & -kernelFileAlignment) -
unalignedSize;
@ -740,6 +756,10 @@ class BinaryPrinter implements Visitor<void>, BinarySink {
// Fixed-size ints at the end used as an index.
assert(_binaryOffsetForSourceTable >= 0);
writeUInt32(_binaryOffsetForSourceTable);
assert(_binaryOffsetForConstantTable >= 0);
writeUInt32(_binaryOffsetForConstantTable);
assert(_binaryOffsetForConstantTableIndex >= 0);
writeUInt32(_binaryOffsetForConstantTableIndex);
assert(_binaryOffsetForLinkTable >= 0);
writeUInt32(_binaryOffsetForLinkTable);
assert(_binaryOffsetForMetadataPayloads >= 0);
@ -748,8 +768,8 @@ class BinaryPrinter implements Visitor<void>, BinarySink {
writeUInt32(_binaryOffsetForMetadataMappings);
assert(_binaryOffsetForStringTable >= 0);
writeUInt32(_binaryOffsetForStringTable);
assert(_binaryOffsetForConstantTable >= 0);
writeUInt32(_binaryOffsetForConstantTable);
assert(componentIndexOffset >= 0);
writeUInt32(componentIndexOffset);
Procedure? mainMethod = component.mainMethod;
if (mainMethod == null) {
@ -911,6 +931,7 @@ class BinaryPrinter implements Visitor<void>, BinarySink {
}
checkCanonicalName(node.parent!);
node.index = _canonicalNameList.length;
assert(!_canonicalNameListDone);
_canonicalNameList.add(node);
}
@ -2772,48 +2793,28 @@ class SwitchCaseIndexer {
}
class ConstantIndexer extends RecursiveResultVisitor {
final StringIndexer stringIndexer;
final List<Constant> entries = <Constant>[];
final Map<Constant, int> offsets = <Constant, int>{};
int nextOffset = 0;
final BinaryPrinter _printer;
ConstantIndexer(this.stringIndexer, this._printer);
final List<int> offsets = <int>[];
final Map<Constant, int> index = <Constant, int>{};
int put(Constant constant) {
final int? oldOffset = offsets[constant];
if (oldOffset != null) return oldOffset;
final int? oldIndex = index[constant];
if (oldIndex != null) return oldIndex;
// Traverse DAG in post-order to ensure children have their offsets assigned
// before the parent.
constant.visitChildren(this);
if (constant is StringConstant) {
stringIndexer.put(constant.value);
} else if (constant is SymbolConstant) {
stringIndexer.put(constant.name);
} else if (constant is DoubleConstant) {
stringIndexer.put('${constant.value}');
} else if (constant is IntConstant) {
final int value = constant.value;
if ((value.abs() >> 30) != 0) {
stringIndexer.put('$value');
}
}
final int newOffset = nextOffset;
final int newIndex = entries.length;
entries.add(constant);
nextOffset += _printer.writeConstantTableEntry(constant);
return offsets[constant] = newOffset;
offsets.add(-1); // placeholder.
assert(entries.length == offsets.length);
return index[constant] = newIndex;
}
defaultConstantReference(Constant node) {
put(node);
}
int? operator [](Constant node) => offsets[node];
}
class TypeParameterIndexer {

View file

@ -174,7 +174,7 @@ class Tag {
/// Internal version of kernel binary format.
/// Bump it when making incompatible changes in kernel binaries.
/// Keep in sync with runtime/vm/kernel_binary.h, pkg/kernel/binary.md.
static const int BinaryFormatVersion = 63;
static const int BinaryFormatVersion = 64;
}
abstract class ConstantTag {

View file

@ -10,6 +10,10 @@ namespace kernel {
#define Z (zone_)
#define H (translation_helper_)
// Note: If changing how the constants are saved in the binary (and thus how
// they are read here) be aware that there's also some reading going on in
// KernelLoader::ReadVMAnnotations which then also has to be updated!
ConstantReader::ConstantReader(KernelReaderHelper* helper,
ActiveClass* active_class)
: helper_(helper),
@ -70,7 +74,7 @@ ObjectPtr ConstantReader::ReadAnnotations() {
return H.Canonicalize(metadata_values);
}
InstancePtr ConstantReader::ReadConstant(intptr_t constant_offset) {
InstancePtr ConstantReader::ReadConstant(intptr_t constant_index) {
ASSERT(!H.constants().IsNull());
ASSERT(!H.constants_table().IsNull()); // raw bytes
@ -84,30 +88,30 @@ InstancePtr ConstantReader::ReadConstant(intptr_t constant_offset) {
SafepointMutexLocker ml(
H.thread()->isolate_group()->kernel_constants_mutex());
KernelConstantsMap constant_map(H.info().constants());
result_ ^= constant_map.GetOrNull(constant_offset);
result_ ^= constant_map.GetOrNull(constant_index);
ASSERT(constant_map.Release().ptr() == H.info().constants());
}
// On miss, evaluate, and insert value.
if (result_.IsNull()) {
LeaveCompilerScope cs(H.thread());
result_ = ReadConstantInternal(constant_offset);
result_ = ReadConstantInternal(constant_index);
SafepointMutexLocker ml(
H.thread()->isolate_group()->kernel_constants_mutex());
KernelConstantsMap constant_map(H.info().constants());
auto insert = constant_map.InsertNewOrGetValue(constant_offset, result_);
auto insert = constant_map.InsertNewOrGetValue(constant_index, result_);
ASSERT(insert == result_.ptr());
H.info().set_constants(constant_map.Release()); // update!
}
return result_.ptr();
}
bool ConstantReader::IsInstanceConstant(intptr_t constant_offset,
bool ConstantReader::IsInstanceConstant(intptr_t constant_index,
const Class& clazz) {
// Get reader directly into raw bytes of constant table.
// Get reader directly into raw bytes of constant table/constant mapping.
KernelReaderHelper reader(Z, &H, script_, H.constants_table(), 0);
reader.ReadUInt(); // skip variable-sized int for adjusted constant offset
reader.SetOffset(reader.ReaderOffset() + constant_offset);
NavigateToIndex(&reader, constant_index);
// Peek for an instance of the given clazz.
if (reader.ReadByte() == kInstanceConstant) {
const NameIndex index = reader.ReadCanonicalNameReference();
@ -116,12 +120,29 @@ bool ConstantReader::IsInstanceConstant(intptr_t constant_offset,
return false;
}
InstancePtr ConstantReader::ReadConstantInternal(intptr_t constant_offset) {
// Get reader directly into raw bytes of constant table.
intptr_t ConstantReader::NavigateToIndex(KernelReaderHelper* reader,
intptr_t constant_index) {
// Get reader directly into raw bytes of constant table/constant mapping.
// Get the length of the constants (at the end of the mapping).
reader->SetOffset(reader->ReaderSize() - 4);
const intptr_t num_constants = reader->ReadUInt32();
// Get the binary offset of the constant at the wanted index.
reader->SetOffset(reader->ReaderSize() - 4 - (num_constants * 4) +
(constant_index * 4));
const intptr_t constant_offset = reader->ReadUInt32();
reader->SetOffset(constant_offset);
return constant_offset;
}
InstancePtr ConstantReader::ReadConstantInternal(intptr_t constant_index) {
// Get reader directly into raw bytes of constant table/constant mapping.
bool null_safety = H.thread()->isolate_group()->null_safety();
KernelReaderHelper reader(Z, &H, script_, H.constants_table(), 0);
reader.ReadUInt(); // skip variable-sized int for adjusted constant offset
reader.SetOffset(reader.ReaderOffset() + constant_offset);
const intptr_t constant_offset = NavigateToIndex(&reader, constant_index);
// No function types returned as part of any types built should reference
// free parent type args, ensured by clearing the enclosing function type.
ActiveEnclosingFunctionScope scope(active_class_, nullptr);
@ -221,9 +242,9 @@ InstancePtr ConstantReader::ReadConstantInternal(intptr_t constant_offset) {
for (intptr_t j = 0; j < length; ++j) {
// Recurse into lazily evaluating all "sub" constants
// needed to evaluate the current constant.
const intptr_t entry_offset = reader.ReadUInt();
ASSERT(entry_offset < constant_offset); // DAG!
constant = ReadConstant(entry_offset);
const intptr_t entry_index = reader.ReadUInt();
ASSERT(entry_index < constant_offset); // DAG!
constant = ReadConstant(entry_index);
array.SetAt(j, constant);
}
instance = array.ptr();
@ -273,9 +294,9 @@ InstancePtr ConstantReader::ReadConstantInternal(intptr_t constant_offset) {
reader.ReadCanonicalNameReference());
// Recurse into lazily evaluating all "sub" constants
// needed to evaluate the current constant.
const intptr_t entry_offset = reader.ReadUInt();
ASSERT(entry_offset < constant_offset); // DAG!
constant = ReadConstant(entry_offset);
const intptr_t entry_index = reader.ReadUInt();
ASSERT(entry_index < constant_offset); // DAG!
constant = ReadConstant(entry_index);
instance.SetField(field, constant);
}
break;
@ -283,9 +304,9 @@ InstancePtr ConstantReader::ReadConstantInternal(intptr_t constant_offset) {
case kPartialInstantiationConstant: {
// Recurse into lazily evaluating the "sub" constant
// needed to evaluate the current constant.
const intptr_t entry_offset = reader.ReadUInt();
ASSERT(entry_offset < constant_offset); // DAG!
const auto& constant = Instance::Handle(Z, ReadConstant(entry_offset));
const intptr_t entry_index = reader.ReadUInt();
ASSERT(entry_index < constant_offset); // DAG!
const auto& constant = Instance::Handle(Z, ReadConstant(entry_index));
ASSERT(!constant.IsNull());
// Build type from the raw bytes (needs temporary translator).

View file

@ -27,16 +27,17 @@ class ConstantReader {
InstancePtr ReadConstantExpression();
ObjectPtr ReadAnnotations();
// Peeks to see if constant at the given offset will evaluate to
// Peeks to see if constant at the given index will evaluate to
// instance of the given clazz.
bool IsInstanceConstant(intptr_t constant_offset, const Class& clazz);
bool IsInstanceConstant(intptr_t constant_index, const Class& clazz);
// Reads a constant at the given offset (possibly by recursing
// Reads a constant at the given index (possibly by recursing
// into sub-constants).
InstancePtr ReadConstant(intptr_t constant_offset);
InstancePtr ReadConstant(intptr_t constant_index);
private:
InstancePtr ReadConstantInternal(intptr_t constant_offset);
InstancePtr ReadConstantInternal(intptr_t constant_index);
intptr_t NavigateToIndex(KernelReaderHelper* reader, intptr_t constant_index);
KernelReaderHelper* helper_;
Zone* zone_;

View file

@ -3760,9 +3760,9 @@ Fragment StreamingFlowGraphBuilder::BuildConstantExpression(
SkipDartType();
}
if (position != nullptr) *position = p;
const intptr_t constant_offset = ReadUInt();
const intptr_t constant_index = ReadUInt();
Fragment result = Constant(
Object::ZoneHandle(Z, constant_reader_.ReadConstant(constant_offset)));
Object::ZoneHandle(Z, constant_reader_.ReadConstant(constant_index)));
return result;
}

View file

@ -1717,10 +1717,10 @@ InferredTypeMetadata InferredTypeMetadataHelper::GetInferredType(
const Object* constant_value = &Object::null_object();
if ((flags & InferredTypeMetadata::kFlagConstant) != 0) {
const intptr_t constant_offset = helper_->ReadUInt();
const intptr_t constant_index = helper_->ReadUInt();
if (read_constant) {
constant_value = &Object::ZoneHandle(
H.zone(), constant_reader_->ReadConstant(constant_offset));
H.zone(), constant_reader_->ReadConstant(constant_index));
}
}
@ -1958,6 +1958,10 @@ intptr_t KernelReaderHelper::ReaderOffset() const {
return reader_.offset();
}
intptr_t KernelReaderHelper::ReaderSize() const {
return reader_.size();
}
void KernelReaderHelper::SetOffset(intptr_t offset) {
reader_.set_offset(offset);
}
@ -2713,17 +2717,13 @@ TokenPosition KernelReaderHelper::ReadPosition() {
return position;
}
intptr_t KernelReaderHelper::SourceTableFieldCountFromFirstLibraryOffset() {
return SourceTableFieldCountFromFirstLibraryOffset41Plus;
}
intptr_t KernelReaderHelper::SourceTableSize() {
AlternativeReadingScope alt(&reader_);
intptr_t library_count = reader_.ReadFromIndexNoReset(
reader_.size(), LibraryCountFieldCountFromEnd, 1, 0);
const intptr_t count_from_first_library_offset =
SourceTableFieldCountFromFirstLibraryOffset();
SourceTableFieldCountFromFirstLibraryOffset;
intptr_t source_table_offset = reader_.ReadFromIndexNoReset(
reader_.size(),
@ -2740,7 +2740,7 @@ intptr_t KernelReaderHelper::GetOffsetForSourceInfo(intptr_t index) {
reader_.size(), LibraryCountFieldCountFromEnd, 1, 0);
const intptr_t count_from_first_library_offset =
SourceTableFieldCountFromFirstLibraryOffset();
SourceTableFieldCountFromFirstLibraryOffset;
intptr_t source_table_offset = reader_.ReadFromIndexNoReset(
reader_.size(),

View file

@ -1204,6 +1204,7 @@ class KernelReaderHelper {
}
intptr_t ReaderOffset() const;
intptr_t ReaderSize() const;
void SkipBytes(intptr_t skip);
bool ReadBool();
uint8_t ReadByte();
@ -1252,7 +1253,6 @@ class KernelReaderHelper {
Nullability ReadNullability();
Variance ReadVariance();
intptr_t SourceTableFieldCountFromFirstLibraryOffset();
intptr_t SourceTableSize();
intptr_t GetOffsetForSourceInfo(intptr_t index);
String& SourceTableUriFor(intptr_t index);

View file

@ -99,6 +99,7 @@ class Program {
return metadata_mappings_offset_;
}
intptr_t constant_table_offset() { return constant_table_offset_; }
intptr_t component_index_offset() { return component_index_offset_; }
intptr_t library_count() { return library_count_; }
NNBDCompiledMode compilation_mode() const { return compilation_mode_; }
@ -122,6 +123,9 @@ class Program {
// The offset from the start of the binary to the start of the constant table.
intptr_t constant_table_offset_;
// The offset from the start of the binary to the start of the ending-index.
intptr_t component_index_offset_;
// The offset from the start of the binary to the canonical name table.
intptr_t name_table_offset_;

View file

@ -171,17 +171,21 @@ std::unique_ptr<Program> Program::ReadFrom(Reader* reader, const char** error) {
program->library_count_ = reader->ReadFromIndexNoReset(
reader->size_, LibraryCountFieldCountFromEnd, 1, 0);
intptr_t count_from_first_library_offset =
SourceTableFieldCountFromFirstLibraryOffset41Plus;
SourceTableFieldCountFromFirstLibraryOffset;
program->source_table_offset_ = reader->ReadFromIndexNoReset(
reader->size_,
LibraryCountFieldCountFromEnd + 1 + program->library_count_ + 1 +
count_from_first_library_offset,
1, 0);
program->constant_table_offset_ = reader->ReadUInt32();
reader->ReadUInt32(); // offset for constant table index.
program->name_table_offset_ = reader->ReadUInt32();
program->metadata_payloads_offset_ = reader->ReadUInt32();
program->metadata_mappings_offset_ = reader->ReadUInt32();
program->string_table_offset_ = reader->ReadUInt32();
program->constant_table_offset_ = reader->ReadUInt32();
// The below includes any 8-bit alignment; denotes the end of the previous
// block.
program->component_index_offset_ = reader->ReadUInt32();
program->main_method_reference_ = NameIndex(reader->ReadUInt32() - 1);
NNBDCompiledMode compilation_mode =

View file

@ -20,8 +20,8 @@ namespace kernel {
static const uint32_t kMagicProgramFile = 0x90ABCDEFu;
// Both version numbers are inclusive.
static const uint32_t kMinSupportedKernelFormatVersion = 63;
static const uint32_t kMaxSupportedKernelFormatVersion = 63;
static const uint32_t kMinSupportedKernelFormatVersion = 64;
static const uint32_t kMaxSupportedKernelFormatVersion = 64;
// Keep in sync with package:kernel/lib/binary/tag.dart
#define KERNEL_TAG_LIST(V) \
@ -217,8 +217,7 @@ enum class NamedTypeFlags : uint8_t {
static const int SpecializedIntLiteralBias = 3;
static const int LibraryCountFieldCountFromEnd = 1;
static const int KernelFormatVersionOffset = 4;
static const int SourceTableFieldCountFromFirstLibraryOffsetPre41 = 6;
static const int SourceTableFieldCountFromFirstLibraryOffset41Plus = 7;
static const int SourceTableFieldCountFromFirstLibraryOffset = 9;
static const int HeaderSize = 8; // 'magic', 'formatVersion'.

View file

@ -399,11 +399,11 @@ void KernelLoader::InitializeFields(UriToSourceTable* uri_to_source_table) {
Z,
reader.ExternalDataFromTo(reader.offset(), reader.offset() + end_offset));
// Create a view of the constants table. The trailing ComponentIndex is
// negligible in size.
// Create a view of the constants table (first part)
// and the constant table index (second part).
const ExternalTypedData& constants_table = ExternalTypedData::Handle(
Z, reader.ExternalDataFromTo(program_->constant_table_offset(),
program_->kernel_data_size()));
program_->name_table_offset()));
// Copy the canonical names into the VM's heap. Encode them as unsigned, so
// the parent indexes are adjusted when extracted.
@ -550,10 +550,10 @@ void KernelLoader::AnnotateNativeProcedures() {
// We have a candidate. Let's look if it's an instance of the
// ExternalName class.
const intptr_t constant_table_offset = helper_.ReadUInt();
if (constant_reader.IsInstanceConstant(constant_table_offset,
const intptr_t constant_table_index = helper_.ReadUInt();
if (constant_reader.IsInstanceConstant(constant_table_index,
external_name_class_)) {
constant = constant_reader.ReadConstant(constant_table_offset);
constant = constant_reader.ReadConstant(constant_table_index);
ASSERT(constant.clazz() == external_name_class_.ptr());
// We found the annotation, let's flag the function as native and
// set the native name!
@ -667,10 +667,10 @@ void KernelLoader::LoadNativeExtensionLibraries() {
// We have a candidate. Let's look if it's an instance of the
// ExternalName class.
const intptr_t constant_table_offset = helper_.ReadUInt();
if (constant_reader.IsInstanceConstant(constant_table_offset,
const intptr_t constant_table_index = helper_.ReadUInt();
if (constant_reader.IsInstanceConstant(constant_table_index,
external_name_class_)) {
constant = constant_reader.ReadConstant(constant_table_offset);
constant = constant_reader.ReadConstant(constant_table_index);
ASSERT(constant.clazz() == external_name_class_.ptr());
uri_path ^= constant.GetField(external_name_field_);
}
@ -1899,7 +1899,7 @@ void KernelLoader::ReadVMAnnotations(const Library& library,
helper_.ReadByte(); // Skip the tag.
helper_.ReadPosition(); // Skip fileOffset.
helper_.SkipDartType(); // Skip type.
const intptr_t offset_in_constant_table = helper_.ReadUInt();
const intptr_t index_in_constant_table = helper_.ReadUInt();
AlternativeReadingScopeWithNewData scope(
&helper_.reader_,
@ -1909,8 +1909,18 @@ void KernelLoader::ReadVMAnnotations(const Library& library,
// Seek into the position within the constant table where we can inspect
// this constant's Kernel representation.
helper_.ReadUInt(); // skip constant table size
helper_.SkipBytes(offset_in_constant_table);
// Get the length of the constants (at the end of the mapping).
helper_.SetOffset(helper_.ReaderSize() - 4);
const intptr_t num_constants = helper_.ReadUInt32();
// Get the binary offset of the constant at the wanted index.
helper_.SetOffset(helper_.ReaderSize() - 4 - (num_constants * 4) +
(index_in_constant_table * 4));
const intptr_t constant_offset = helper_.ReadUInt32();
helper_.SetOffset(constant_offset);
uint8_t tag = helper_.ReadTag();
if (tag == kInstanceConstant) {
*has_pragma_annotation =
@ -1937,15 +1947,15 @@ void KernelLoader::ReadVMAnnotations(const Library& library,
helper_.ReadPosition(); // Skip fileOffset.
helper_.SkipDartType(); // Skip type.
}
const intptr_t constant_table_offset = helper_.ReadUInt();
const intptr_t constant_table_index = helper_.ReadUInt();
// We have a candidate. Let's look if it's an instance of the
// ExternalName or Pragma class.
if (constant_reader.IsInstanceConstant(constant_table_offset,
if (constant_reader.IsInstanceConstant(constant_table_index,
external_name_class_)) {
constant = constant_reader.ReadConstant(constant_table_offset);
constant = constant_reader.ReadConstant(constant_table_index);
ASSERT(constant.clazz() == external_name_class_.ptr());
*native_name ^= constant.GetField(external_name_field_);
} else if (constant_reader.IsInstanceConstant(constant_table_offset,
} else if (constant_reader.IsInstanceConstant(constant_table_index,
pragma_class_)) {
*has_pragma_annotation = true;
}