[cfe] Fix more cases of kernel AST serializer missing file offset.

Dart2JS uses the file offset on these nodes to create JS source maps. We want to maintain these file offsets across the serialization layer when we compile from dill files rather than directly from sources.

This last CL resolves all diffs observed in test files in pkg/compiler/test/inference/data.

TEST=Updating tests for all of these in fasta offsets tests.

Change-Id: If84c7542b61c8d8b894b202c68841ccfe91dafa5
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/328080
Reviewed-by: Johnni Winther <johnniwinther@google.com>
Commit-Queue: Nate Biggs <natebiggs@google.com>
Reviewed-by: Alexander Markov <alexmarkov@google.com>
This commit is contained in:
Nate Biggs 2023-09-27 22:12:58 +00:00 committed by Commit Queue
parent 0742604758
commit 175c949398
11 changed files with 132 additions and 27 deletions

View file

@ -147,7 +147,7 @@ type CanonicalName {
type ComponentFile {
UInt32 magic = 0x90ABCDEF;
UInt32 formatVersion = 110;
UInt32 formatVersion = 111;
Byte[10] shortSdkHash;
List<String> problemsAsJson; // Described in problems.md.
Library[] libraries;
@ -1000,6 +1000,7 @@ type AsExpression extends Expression {
type StringLiteral extends Expression {
Byte tag = 39;
FileOffset fileOffset;
StringReference value;
}
@ -1008,42 +1009,51 @@ type IntegerLiteral extends Expression {}
type SpecializedIntLiteral extends IntegerLiteral {
Byte tag = 240 + N; // Where 0 <= N < 8.
// Integer literal with value (N - 3), that is, an integer in range -3..4.
FileOffset fileOffset;
}
type PositiveIntLiteral extends IntegerLiteral {
Byte tag = 55;
FileOffset fileOffset;
UInt value;
}
type NegativeIntLiteral extends IntegerLiteral {
Byte tag = 56;
FileOffset fileOffset;
UInt absoluteValue;
}
type BigIntLiteral extends IntegerLiteral {
Byte tag = 57;
FileOffset fileOffset;
StringReference valueString;
}
type DoubleLiteral extends Expression {
Byte tag = 40;
FileOffset fileOffset;
Double value;
}
type TrueLiteral extends Expression {
Byte tag = 41;
FileOffset fileOffset;
}
type FalseLiteral extends Expression {
Byte tag = 42;
FileOffset fileOffset;
}
type NullLiteral extends Expression {
Byte tag = 43;
FileOffset fileOffset;
}
type SymbolLiteral extends Expression {
Byte tag = 44;
FileOffset fileOffset;
StringReference value; // Everything strictly after the '#'.
}
@ -1055,6 +1065,7 @@ type TypeLiteral extends Expression {
type ThisExpression extends Expression {
Byte tag = 46;
FileOffset fileOffset;
}
type Rethrow extends Expression {
@ -1438,6 +1449,7 @@ type ReturnStatement extends Statement {
type TryCatch extends Statement {
Byte tag = 75;
FileOffset fileOffset;
Statement body;
// "any catch needs a stacktrace" means it has a stacktrace variable.
Byte flags (anyCatchNeedsStackTrace, isSynthesized);
@ -1454,6 +1466,7 @@ type Catch {
type TryFinally extends Statement {
Byte tag = 76;
FileOffset fileOffset;
Statement body;
Statement finalizer;
}

View file

@ -2736,44 +2736,56 @@ class BinaryBuilder {
}
Expression _readStringLiteral() {
return new StringLiteral(readStringReference());
int offset = readOffset();
return new StringLiteral(readStringReference())..fileOffset = offset;
}
Expression _readSpecializedIntLiteral(int tagByte) {
int biasedValue = tagByte & Tag.SpecializedPayloadMask;
return new IntLiteral(biasedValue - Tag.SpecializedIntLiteralBias);
return new IntLiteral(biasedValue - Tag.SpecializedIntLiteralBias)
..fileOffset = readOffset();
}
Expression _readPositiveIntLiteral() {
return new IntLiteral(readUInt30());
int offset = readOffset();
int value = readUInt30();
return new IntLiteral(value)..fileOffset = offset;
}
Expression _readNegativeIntLiteral() {
return new IntLiteral(-readUInt30());
int offset = readOffset();
int value = -readUInt30();
return new IntLiteral(value)..fileOffset = offset;
}
Expression _readBigIntLiteral() {
return new IntLiteral(int.parse(readStringReference()));
int offset = readOffset();
int value = int.parse(readStringReference());
return new IntLiteral(value)..fileOffset = offset;
}
Expression _readDoubleLiteral() {
return new DoubleLiteral(readDouble());
int offset = readOffset();
double value = readDouble();
return new DoubleLiteral(value)..fileOffset = offset;
}
Expression _readTrueLiteral() {
return new BoolLiteral(true);
return new BoolLiteral(true)..fileOffset = readOffset();
}
Expression _readFalseLiteral() {
return new BoolLiteral(false);
return new BoolLiteral(false)..fileOffset = readOffset();
}
Expression _readNullLiteral() {
return new NullLiteral();
return new NullLiteral()..fileOffset = readOffset();
}
Expression _readSymbolLiteral() {
return new SymbolLiteral(readStringReference());
int offset = readOffset();
String value = readStringReference();
return new SymbolLiteral(value)..fileOffset = offset;
}
Expression _readTypeLiteral() {
@ -2782,7 +2794,7 @@ class BinaryBuilder {
}
Expression _readThisLiteral() {
return new ThisExpression();
return new ThisExpression()..fileOffset = readOffset();
}
Expression _readRethrow() {
@ -3615,13 +3627,17 @@ class BinaryBuilder {
}
Statement _readTryCatch() {
int offset = readOffset();
Statement body = readStatement();
int flags = readByte();
return new TryCatch(body, readCatchList(), isSynthetic: flags & 2 == 2);
return new TryCatch(body, readCatchList(), isSynthetic: flags & 2 == 2)
..fileOffset = offset;
}
Statement _readTryFinally() {
return new TryFinally(readStatement(), readStatement());
int offset = readOffset();
return new TryFinally(readStatement(), readStatement())
..fileOffset = offset;
}
Statement _readYieldStatement() {

View file

@ -223,7 +223,7 @@ class BinaryPrinter implements Visitor<void>, BinarySink {
writeByte(constant.value ? 1 : 0);
} else if (constant is IntConstant) {
writeByte(ConstantTag.IntConstant);
writeInteger(constant.value);
writeInteger(constant.value, TreeNode.noOffset);
} else if (constant is DoubleConstant) {
writeByte(ConstantTag.DoubleConstant);
writeDouble(constant.value);
@ -1998,30 +1998,35 @@ class BinaryPrinter implements Visitor<void>, BinarySink {
@override
void visitStringLiteral(StringLiteral node) {
writeByte(Tag.StringLiteral);
writeOffset(node.fileOffset);
writeStringReference(node.value);
}
@override
void visitIntLiteral(IntLiteral node) {
writeInteger(node.value);
writeInteger(node.value, node.fileOffset);
}
void writeInteger(int value) {
void writeInteger(int value, int fileOffset) {
int biasedValue = value + Tag.SpecializedIntLiteralBias;
if (biasedValue >= 0 &&
biasedValue & Tag.SpecializedPayloadMask == biasedValue) {
writeByte(Tag.SpecializedIntLiteral + biasedValue);
writeOffset(fileOffset);
} else if (value.abs() >> 30 == 0) {
if (value < 0) {
writeByte(Tag.NegativeIntLiteral);
writeOffset(fileOffset);
writeUInt30(-value);
} else {
writeByte(Tag.PositiveIntLiteral);
writeOffset(fileOffset);
writeUInt30(value);
}
} else {
// TODO: Pick a better format for big int literals.
writeByte(Tag.BigIntLiteral);
writeOffset(fileOffset);
writeStringReference('$value');
}
}
@ -2029,6 +2034,7 @@ class BinaryPrinter implements Visitor<void>, BinarySink {
@override
void visitDoubleLiteral(DoubleLiteral node) {
writeByte(Tag.DoubleLiteral);
writeOffset(node.fileOffset);
writeDouble(node.value);
}
@ -2039,16 +2045,19 @@ class BinaryPrinter implements Visitor<void>, BinarySink {
@override
void visitBoolLiteral(BoolLiteral node) {
writeByte(node.value ? Tag.TrueLiteral : Tag.FalseLiteral);
writeOffset(node.fileOffset);
}
@override
void visitNullLiteral(NullLiteral node) {
writeByte(Tag.NullLiteral);
writeOffset(node.fileOffset);
}
@override
void visitSymbolLiteral(SymbolLiteral node) {
writeByte(Tag.SymbolLiteral);
writeOffset(node.fileOffset);
writeStringReference(node.value);
}
@ -2062,6 +2071,7 @@ class BinaryPrinter implements Visitor<void>, BinarySink {
@override
void visitThisExpression(ThisExpression node) {
writeByte(Tag.ThisExpression);
writeOffset(node.fileOffset);
}
@override
@ -2362,6 +2372,7 @@ class BinaryPrinter implements Visitor<void>, BinarySink {
@override
void visitTryCatch(TryCatch node) {
writeByte(Tag.TryCatch);
writeOffset(node.fileOffset);
writeNode(node.body);
bool needsStackTrace = node.catches.any((Catch c) => c.stackTrace != null);
writeByte(_encodeTryCatchFlags(needsStackTrace, node.isSynthetic));
@ -2385,6 +2396,7 @@ class BinaryPrinter implements Visitor<void>, BinarySink {
@override
void visitTryFinally(TryFinally node) {
writeByte(Tag.TryFinally);
writeOffset(node.fileOffset);
writeNode(node.body);
writeNode(node.finalizer);
}

View file

@ -226,7 +226,7 @@ class Tag {
/// Internal version of kernel binary format.
/// Bump it when making incompatible changes in kernel binaries.
/// Keep in sync with runtime/vm/kernel_binary.h, pkg/kernel/binary.md.
static const int BinaryFormatVersion = 110;
static const int BinaryFormatVersion = 111;
}
abstract class ConstantTag {

View file

@ -256,22 +256,26 @@ InstancePtr ConstantReader::ReadConstantInternal(intptr_t constant_index) {
Tag integer_tag = reader.ReadTag(&payload); // read tag.
switch (integer_tag) {
case kBigIntLiteral: {
reader.ReadPosition();
const String& value = H.DartString(reader.ReadStringReference());
instance = Integer::New(value, Heap::kOld);
break;
}
case kSpecializedIntLiteral: {
reader.ReadPosition();
const int64_t value =
static_cast<int32_t>(payload) - SpecializedIntLiteralBias;
instance = Integer::New(value, Heap::kOld);
break;
}
case kNegativeIntLiteral: {
reader.ReadPosition();
const int64_t value = -static_cast<int64_t>(reader.ReadUInt());
instance = Integer::New(value, Heap::kOld);
break;
}
case kPositiveIntLiteral: {
reader.ReadPosition();
const int64_t value = reader.ReadUInt();
instance = Integer::New(value, Heap::kOld);
break;

View file

@ -3696,6 +3696,7 @@ void StreamingFlowGraphBuilder::FlattenStringConcatenation(
switch (PeekTag()) {
case kStringLiteral: {
ReadTag();
ReadPosition();
const String& s = H.DartSymbolPlain(ReadStringReference());
// Skip empty strings.
if (!s.Equals("")) {
@ -3976,6 +3977,7 @@ Fragment StreamingFlowGraphBuilder::BuildTypeLiteral(TokenPosition* position) {
Fragment StreamingFlowGraphBuilder::BuildThisExpression(
TokenPosition* position) {
ReadPosition(); // ignore file offset.
if (position != nullptr) *position = TokenPosition::kNoSource;
return LoadLocal(parsed_function()->receiver_var());
@ -4254,6 +4256,7 @@ Fragment StreamingFlowGraphBuilder::BuildBlockExpression() {
Fragment StreamingFlowGraphBuilder::BuildBigIntLiteral(
TokenPosition* position) {
ReadPosition(); // ignore file offset.
if (position != nullptr) *position = TokenPosition::kNoSource;
const String& value =
@ -4270,6 +4273,7 @@ Fragment StreamingFlowGraphBuilder::BuildBigIntLiteral(
Fragment StreamingFlowGraphBuilder::BuildStringLiteral(
TokenPosition* position) {
ReadPosition(); // ignore file offset.
if (position != nullptr) *position = TokenPosition::kNoSource;
return Constant(H.DartSymbolPlain(
@ -4278,6 +4282,7 @@ Fragment StreamingFlowGraphBuilder::BuildStringLiteral(
Fragment StreamingFlowGraphBuilder::BuildIntLiteral(uint8_t payload,
TokenPosition* position) {
ReadPosition(); // ignore file offset.
if (position != nullptr) *position = TokenPosition::kNoSource;
int64_t value = static_cast<int32_t>(payload) - SpecializedIntLiteralBias;
@ -4286,6 +4291,7 @@ Fragment StreamingFlowGraphBuilder::BuildIntLiteral(uint8_t payload,
Fragment StreamingFlowGraphBuilder::BuildIntLiteral(bool is_negative,
TokenPosition* position) {
ReadPosition(); // ignore file offset.
if (position != nullptr) *position = TokenPosition::kNoSource;
int64_t value = is_negative ? -static_cast<int64_t>(ReadUInt())
@ -4295,6 +4301,7 @@ Fragment StreamingFlowGraphBuilder::BuildIntLiteral(bool is_negative,
Fragment StreamingFlowGraphBuilder::BuildDoubleLiteral(
TokenPosition* position) {
ReadPosition(); // ignore file offset.
if (position != nullptr) *position = TokenPosition::kNoSource;
Double& constant = Double::ZoneHandle(
@ -4304,12 +4311,14 @@ Fragment StreamingFlowGraphBuilder::BuildDoubleLiteral(
Fragment StreamingFlowGraphBuilder::BuildBoolLiteral(bool value,
TokenPosition* position) {
ReadPosition(); // ignore file offset.
if (position != nullptr) *position = TokenPosition::kNoSource;
return Constant(Bool::Get(value));
}
Fragment StreamingFlowGraphBuilder::BuildNullLiteral(TokenPosition* position) {
ReadPosition(); // ignore file offset.
if (position != nullptr) *position = TokenPosition::kNoSource;
return Constant(Instance::ZoneHandle(Z, Instance::null()));
@ -5466,6 +5475,9 @@ Fragment StreamingFlowGraphBuilder::BuildTryCatch(TokenPosition* position) {
ASSERT(block_expression_depth() == 0); // no try-catch in block-expr
InlineBailout("kernel::FlowgraphBuilder::VisitTryCatch");
const TokenPosition pos = ReadPosition(); // read position.
if (position != nullptr) *position = pos;
intptr_t try_handler_index = AllocateTryIndex();
Fragment try_body = TryCatch(try_handler_index);
JoinEntryInstr* after_try = BuildJoinEntry();
@ -5591,6 +5603,9 @@ Fragment StreamingFlowGraphBuilder::BuildTryFinally(TokenPosition* position) {
InlineBailout("kernel::FlowgraphBuilder::VisitTryFinally");
const TokenPosition pos = ReadPosition(); // read position.
if (position != nullptr) *position = pos;
// There are 5 different cases where we need to execute the finally block:
//
// a) 1/2/3th case: Special control flow going out of `node->body()`:

View file

@ -571,6 +571,7 @@ void KernelFingerprintHelper::CalculateExpressionFingerprint() {
CalculateDartTypeFingerprint(); // read type.
return;
case kThisExpression:
ReadPosition(); // read position.
return;
case kRethrow:
ReadPosition(); // read position.
@ -636,20 +637,26 @@ void KernelFingerprintHelper::CalculateExpressionFingerprint() {
CalculateListOfDartTypesFingerprint(); // read type arguments.
return;
case kBigIntLiteral:
ReadPosition(); // read position.
CalculateStringReferenceFingerprint(); // read string reference.
return;
case kStringLiteral:
ReadPosition(); // read position.
CalculateStringReferenceFingerprint(); // read string reference.
return;
case kSpecializedIntLiteral:
ReadPosition(); // read position.
return;
case kNegativeIntLiteral:
ReadPosition(); // read position.
BuildHash(ReadUInt()); // read value.
return;
case kPositiveIntLiteral:
ReadPosition(); // read position.
BuildHash(ReadUInt()); // read value.
return;
case kDoubleLiteral: {
ReadPosition(); // read position.
double value = ReadDouble(); // read value.
uint64_t data = bit_cast<uint64_t>(value);
BuildHash(static_cast<uint32_t>(data >> 32));
@ -657,10 +664,13 @@ void KernelFingerprintHelper::CalculateExpressionFingerprint() {
return;
}
case kTrueLiteral:
ReadPosition(); // read position.
return;
case kFalseLiteral:
ReadPosition(); // read position.
return;
case kNullLiteral:
ReadPosition(); // read position.
return;
case kConstantExpression:
ReadPosition();
@ -803,6 +813,7 @@ void KernelFingerprintHelper::CalculateStatementFingerprint() {
return;
}
case kTryCatch: {
ReadPosition(); // read position.
CalculateStatementFingerprint(); // read body.
BuildHash(ReadByte()); // read flags
intptr_t catch_count = ReadListLength(); // read number of catches.
@ -824,6 +835,7 @@ void KernelFingerprintHelper::CalculateStatementFingerprint() {
return;
}
case kTryFinally:
ReadPosition(); // read position.
CalculateStatementFingerprint(); // read body.
CalculateStatementFingerprint(); // read finalizer.
return;

View file

@ -2667,6 +2667,7 @@ void KernelReaderHelper::SkipExpression() {
SkipDartType(); // read type.
return;
case kThisExpression:
ReadPosition(); // read position.
return;
case kRethrow:
ReadPosition(); // read position.
@ -2732,27 +2733,36 @@ void KernelReaderHelper::SkipExpression() {
SkipListOfDartTypes(); // read type arguments.
return;
case kBigIntLiteral:
ReadPosition(); // read position.
SkipStringReference(); // read string reference.
return;
case kStringLiteral:
ReadPosition(); // read position.
SkipStringReference(); // read string reference.
return;
case kSpecializedIntLiteral:
ReadPosition(); // read position.
return;
case kNegativeIntLiteral:
ReadUInt(); // read value.
ReadPosition(); // read position.
ReadUInt(); // read value.
return;
case kPositiveIntLiteral:
ReadUInt(); // read value.
ReadPosition(); // read position.
ReadUInt(); // read value.
return;
case kDoubleLiteral:
ReadDouble(); // read value.
ReadPosition(); // read position.
ReadDouble(); // read value.
return;
case kTrueLiteral:
ReadPosition(); // read position.
return;
case kFalseLiteral:
ReadPosition(); // read position.
return;
case kNullLiteral:
ReadPosition(); // read position.
return;
case kConstantExpression:
ReadPosition(); // read position.
@ -2893,6 +2903,7 @@ void KernelReaderHelper::SkipStatement() {
return;
}
case kTryCatch: {
ReadPosition(); // read position
SkipStatement(); // read body.
ReadByte(); // read flags
intptr_t catch_count = ReadListLength(); // read number of catches.
@ -2912,6 +2923,7 @@ void KernelReaderHelper::SkipStatement() {
return;
}
case kTryFinally:
ReadPosition(); // read position
SkipStatement(); // read body.
SkipStatement(); // read finalizer.
return;

View file

@ -848,6 +848,7 @@ void ScopeBuilder::VisitExpression() {
return;
case kThisExpression:
HandleLoadReceiver();
helper_.ReadPosition(); // read file offset.
return;
case kRethrow:
helper_.ReadPosition(); // read position.
@ -933,27 +934,36 @@ void ScopeBuilder::VisitExpression() {
return;
}
case kBigIntLiteral:
helper_.ReadPosition(); // read position.
helper_.SkipStringReference(); // read string reference.
return;
case kStringLiteral:
helper_.ReadPosition(); // read position.
helper_.SkipStringReference(); // read string reference.
return;
case kSpecializedIntLiteral:
helper_.ReadPosition(); // read position.
return;
case kNegativeIntLiteral:
helper_.ReadUInt(); // read value.
helper_.ReadPosition(); // read position.
helper_.ReadUInt(); // read value.
return;
case kPositiveIntLiteral:
helper_.ReadUInt(); // read value.
helper_.ReadPosition(); // read position.
helper_.ReadUInt(); // read value.
return;
case kDoubleLiteral:
helper_.ReadDouble(); // read value.
helper_.ReadPosition(); // read position.
helper_.ReadDouble(); // read value.
return;
case kTrueLiteral:
helper_.ReadPosition(); // read position.
return;
case kFalseLiteral:
helper_.ReadPosition(); // read position.
return;
case kNullLiteral:
helper_.ReadPosition(); // read position.
return;
case kConstantExpression:
helper_.ReadPosition();
@ -1175,7 +1185,8 @@ void ScopeBuilder::VisitStatement() {
case kTryCatch: {
++depth_.try_;
AddTryVariables();
VisitStatement(); // read body.
helper_.ReadPosition(); // read position.
VisitStatement(); // read body.
--depth_.try_;
++depth_.catch_;
@ -1216,7 +1227,8 @@ void ScopeBuilder::VisitStatement() {
++depth_.finally_;
AddTryVariables();
VisitStatement(); // read body.
helper_.ReadPosition(); // read position.
VisitStatement(); // read body.
--depth_.finally_;
--depth_.try_;

View file

@ -18,7 +18,7 @@ namespace kernel {
// package:kernel/binary.md.
static const uint32_t kMagicProgramFile = 0x90ABCDEFu;
static const uint32_t kSupportedKernelFormatVersion = 110;
static const uint32_t kSupportedKernelFormatVersion = 111;
// Keep in sync with package:kernel/lib/binary/tag.dart
#define KERNEL_TAG_LIST(V) \

View file

@ -50,6 +50,7 @@ class SimpleExpressionConverter {
Tag tag = helper_->ReadTag(&payload); // read tag.
switch (tag) {
case kBigIntLiteral: {
helper_->ReadPosition();
const String& literal_str =
H.DartString(helper_->ReadStringReference(),
Heap::kOld); // read index into string table.
@ -63,10 +64,12 @@ class SimpleExpressionConverter {
return true;
}
case kStringLiteral:
helper_->ReadPosition();
simple_value_ = &H.DartSymbolPlain(
helper_->ReadStringReference()); // read index into string table.
return true;
case kSpecializedIntLiteral:
helper_->ReadPosition();
simple_value_ =
&Integer::ZoneHandle(Z, Integer::New(static_cast<int32_t>(payload) -
SpecializedIntLiteralBias,
@ -74,29 +77,35 @@ class SimpleExpressionConverter {
*simple_value_ = H.Canonicalize(*simple_value_);
return true;
case kNegativeIntLiteral:
helper_->ReadPosition();
simple_value_ = &Integer::ZoneHandle(
Z, Integer::New(-static_cast<int64_t>(helper_->ReadUInt()),
Heap::kOld)); // read value.
*simple_value_ = H.Canonicalize(*simple_value_);
return true;
case kPositiveIntLiteral:
helper_->ReadPosition();
simple_value_ = &Integer::ZoneHandle(
Z, Integer::New(static_cast<int64_t>(helper_->ReadUInt()),
Heap::kOld)); // read value.
*simple_value_ = H.Canonicalize(*simple_value_);
return true;
case kDoubleLiteral:
helper_->ReadPosition();
simple_value_ = &Double::ZoneHandle(
Z, Double::New(helper_->ReadDouble(), Heap::kOld)); // read value.
*simple_value_ = H.Canonicalize(*simple_value_);
return true;
case kTrueLiteral:
helper_->ReadPosition();
simple_value_ = &Bool::Handle(Z, Bool::Get(true).ptr());
return true;
case kFalseLiteral:
helper_->ReadPosition();
simple_value_ = &Bool::Handle(Z, Bool::Get(false).ptr());
return true;
case kNullLiteral:
helper_->ReadPosition();
simple_value_ = &Instance::ZoneHandle(Z, Instance::null());
return true;
default: