languages/dart-sdk
languages/dart-sdk
1
0
Fork 0
mirror of https://github.com/dart-lang/sdk synced 2024-09-15 22:41:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

[kernel] Calculate scopes for expression evaluation differently

Browse source 
This CL introduces an alternative way to calculate scopes for
expression evaluations. Currently the previous method is kept as well.

What this does is that it:

* Finds all nodes that match the offset and uri. (Ideally there's only
  one, but that's not always the case --- it's the case for less than
  60% of cases in the test added actually).
* Calculate the scope along the way.
* Return the scopes of all the found nodes that offset.

A test that asks for ~all file offsets in the dill files in the sdk and
compares the result with the scope the binary serialization computes is
added.
A single point can't always be found (for all dills in the sdk
only ~58.93% returns a single result), but for each query the wanted
scope is among the results returned.
For the non-outline dills in the sdk, between ~87.93% and ~94.52% of
the results are either a single result or multiple results with the
same actual scope in all of them.

Note that the test asks for ~all offsets, not just "debuggable" or
"stopable" offsets (which will likely vary depending on the vm/web
backend etc) --- likely the percentage will be higher for those points
though.

When this returns multiple results one can likely be picked by the
client based on information it has about names of variables in scope
etc.

Note that the test is rather slow, on my machine with the platforms
available there it makes 3,908,181 queries in ~3.5 minutes.
(Which corresponds to roughly 18,610 per second or under 0.06 ms per
query on average though, so it doesn't seem like a concert for actually
using it.)

Change-Id: I40b5360fcd935c70629543737e89a787de36ca16
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/332204
Reviewed-by: Johnni Winther <johnniwinther@google.com>
Commit-Queue: Jens Johansen <jensj@google.com>
This commit is contained in:
Jens Johansen 2023-11-06 09:38:40 +00:00 committed by Commit Queue
parent dc68ce5c68
commit ed64348f68
7 changed files with 938 additions and 63 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

37
pkg/kernel/lib/ast.dart
View file

@ -153,6 +153,9 @@ abstract class TreeNode extends Node {
/// not available (this is the default if none is specifically set).
int fileOffset = noOffset;
/// Returns List<int> if this node has more offsets than [fileOffset].
List<int>? get fileOffsetsIfMultiple => null;
@override
R accept<R>(TreeVisitor<R> v);
@override
@ -1021,6 +1024,10 @@ class Class extends NamedNode implements TypeDeclaration {
/// (this is the default if none is specifically set).
int fileEndOffset = TreeNode.noOffset;
@override
List<int>? get fileOffsetsIfMultiple =>
[fileOffset, startFileOffset, fileEndOffset];
/// List of metadata annotations on the class.
///
/// This defaults to an immutable empty list. Use [addAnnotation] to add
@ -2024,6 +2031,9 @@ sealed class Member extends NamedNode implements Annotatable, FileUriNode {
/// set).
int fileEndOffset = TreeNode.noOffset;
@override
List<int>? get fileOffsetsIfMultiple => [fileOffset, fileEndOffset];
/// List of metadata annotations on the member.
///
/// This defaults to an immutable empty list. Use [addAnnotation] to add
@ -2573,6 +2583,10 @@ class Constructor extends Member {
/// set).
int startFileOffset = TreeNode.noOffset;
@override
List<int>? get fileOffsetsIfMultiple =>
[fileOffset, startFileOffset, fileEndOffset];
int flags = 0;
@override
@ -2923,6 +2937,10 @@ class Procedure extends Member implements GenericFunction {
/// set).
int fileStartOffset = TreeNode.noOffset;
@override
List<int>? get fileOffsetsIfMultiple =>
[fileOffset, fileStartOffset, fileEndOffset];
final ProcedureKind kind;
int flags = 0;
@ -3686,6 +3704,9 @@ class FunctionNode extends TreeNode {
/// (this is the default if none is specifically set).
int fileEndOffset = TreeNode.noOffset;
@override
List<int>? get fileOffsetsIfMultiple => [fileOffset, fileEndOffset];
/// Kernel async marker for the function.
///
/// See also [dartAsyncMarker].
@ -9163,6 +9184,9 @@ class Block extends Statement {
/// (this is the default if none is specifically set).
int fileEndOffset = TreeNode.noOffset;
@override
List<int>? get fileOffsetsIfMultiple => [fileOffset, fileEndOffset];
Block(this.statements) {
// Ensure statements is mutable.
assert(checkListIsMutable(statements, dummyStatement));
@ -9301,6 +9325,10 @@ class AssertStatement extends Statement {
/// Note: This is not the offset into the UTF8 encoded `List<int>` source.
int conditionEndOffset;
@override
List<int>? get fileOffsetsIfMultiple =>
[fileOffset, conditionStartOffset, conditionEndOffset];
AssertStatement(this.condition,
{this.message,
required this.conditionStartOffset,
@ -9673,6 +9701,9 @@ class ForInStatement extends Statement {
/// offset is not available (this is the default if none is specifically set).
int bodyOffset = TreeNode.noOffset;
@override
List<int>? get fileOffsetsIfMultiple => [fileOffset, bodyOffset];
VariableDeclaration variable; // Has no initializer.
Expression iterable;
Statement body;
@ -9945,6 +9976,9 @@ class SwitchCase extends TreeNode {
}
}
@override
List<int>? get fileOffsetsIfMultiple => [fileOffset, ...expressionOffsets];
@override
R accept<R>(TreeVisitor<R> v) => v.visitSwitchCase(this);
@ -10460,6 +10494,9 @@ class VariableDeclaration extends Statement implements Annotatable {
/// (this is the default if none is specifically set).
int fileEqualsOffset = TreeNode.noOffset;
@override
List<int>? get fileOffsetsIfMultiple => [fileOffset, fileEqualsOffset];
/// List of metadata annotations on the variable declaration.
///
/// This defaults to an immutable empty list. Use [addAnnotation] to add

40
pkg/kernel/lib/binary/ast_to_binary.dart
View file

@ -17,6 +17,7 @@ import 'tag.dart';
/// A [BinaryPrinter] can be used to write one file and must then be
/// discarded.
class BinaryPrinter implements Visitor<void>, BinarySink {
final VariableIndexer Function() _newVariableIndexer;
VariableIndexer? _variableIndexer;
LabelIndexer? _labelIndexer;
SwitchCaseIndexer? _switchCaseIndexer;
@ -67,11 +68,14 @@ class BinaryPrinter implements Visitor<void>, BinarySink {
StringIndexer? stringIndexer,
this.includeSources = true,
this.includeSourceBytes = true,
this.includeOffsets = true})
this.includeOffsets = true,
VariableIndexer Function()? newVariableIndexerForTesting})
: _mainSink = new BufferedSink(sink),
_metadataSink = new BufferedSink(new BytesSink()),
stringIndexer = stringIndexer ?? new StringIndexer(),
_constantIndexer = new ConstantIndexer() {
_constantIndexer = new ConstantIndexer(),
_newVariableIndexer =
newVariableIndexerForTesting ?? VariableIndexer.new {
_sink = _mainSink;
}
@ -80,7 +84,7 @@ class BinaryPrinter implements Visitor<void>, BinarySink {
}
int _getVariableIndex(VariableDeclaration variable) {
int? index = (_variableIndexer ??= new VariableIndexer())[variable];
int? index = (_variableIndexer ??= _newVariableIndexer())[variable];
assert(index != null, "No index found for ${variable}");
return index!;
}
@ -696,7 +700,7 @@ class BinaryPrinter implements Visitor<void>, BinarySink {
_variableIndexer = null;
}
if (variableScope) {
_variableIndexer ??= new VariableIndexer();
_variableIndexer ??= _newVariableIndexer();
_variableIndexer!.pushScope();
}
}
@ -729,7 +733,7 @@ class BinaryPrinter implements Visitor<void>, BinarySink {
_variableIndexer = null;
}
if (variableScope) {
_variableIndexer ??= new VariableIndexer();
_variableIndexer ??= _newVariableIndexer();
_variableIndexer!.pushScope();
}
}
@ -1326,7 +1330,7 @@ class BinaryPrinter implements Visitor<void>, BinarySink {
assert(node.function.typeParameters.isEmpty);
writeFunctionNode(node.function);
// Parameters are in scope in the initializers.
_variableIndexer ??= new VariableIndexer();
_variableIndexer ??= _newVariableIndexer();
_variableIndexer!.restoreScope(node.function.positionalParameters.length +
node.function.namedParameters.length);
writeNodeList(node.initializers);
@ -2152,7 +2156,7 @@ class BinaryPrinter implements Visitor<void>, BinarySink {
writeByte(Tag.Let);
writeOffset(node.fileOffset);
VariableIndexer variableIndexer =
_variableIndexer ??= new VariableIndexer();
_variableIndexer ??= _newVariableIndexer();
variableIndexer.pushScope();
writeVariableDeclaration(node.variable);
writeNode(node.body);
@ -2164,7 +2168,7 @@ class BinaryPrinter implements Visitor<void>, BinarySink {
writeByte(Tag.BlockExpression);
writeOffset(node.fileOffset);
VariableIndexer variableIndexer =
_variableIndexer ??= new VariableIndexer();
_variableIndexer ??= _newVariableIndexer();
variableIndexer.pushScope();
writeNodeList(node.body.statements);
writeNode(node.value);
@ -2210,7 +2214,7 @@ class BinaryPrinter implements Visitor<void>, BinarySink {
@override
void visitBlock(Block node) {
VariableIndexer variableIndexer =
_variableIndexer ??= new VariableIndexer();
_variableIndexer ??= _newVariableIndexer();
variableIndexer.pushScope();
writeByte(Tag.Block);
writeOffset(node.fileOffset);
@ -2222,7 +2226,7 @@ class BinaryPrinter implements Visitor<void>, BinarySink {
@override
void visitAssertBlock(AssertBlock node) {
VariableIndexer variableIndexer =
_variableIndexer ??= new VariableIndexer();
_variableIndexer ??= _newVariableIndexer();
variableIndexer.pushScope();
writeByte(Tag.AssertBlock);
writeNodeList(node.statements);
@ -2295,7 +2299,7 @@ class BinaryPrinter implements Visitor<void>, BinarySink {
@override
void visitForStatement(ForStatement node) {
VariableIndexer variableIndexer =
_variableIndexer ??= new VariableIndexer();
_variableIndexer ??= _newVariableIndexer();
variableIndexer.pushScope();
writeByte(Tag.ForStatement);
writeOffset(node.fileOffset);
@ -2309,7 +2313,7 @@ class BinaryPrinter implements Visitor<void>, BinarySink {
@override
void visitForInStatement(ForInStatement node) {
VariableIndexer variableIndexer =
_variableIndexer ??= new VariableIndexer();
_variableIndexer ??= _newVariableIndexer();
variableIndexer.pushScope();
writeByte(node.isAsync ? Tag.AsyncForInStatement : Tag.ForInStatement);
writeOffset(node.fileOffset);
@ -2389,7 +2393,7 @@ class BinaryPrinter implements Visitor<void>, BinarySink {
void visitCatch(Catch node) {
// Note: there is no tag on Catch.
VariableIndexer variableIndexer =
_variableIndexer ??= new VariableIndexer();
_variableIndexer ??= _newVariableIndexer();
variableIndexer.pushScope();
writeOffset(node.fileOffset);
writeNode(node.guard);
@ -2435,7 +2439,7 @@ class BinaryPrinter implements Visitor<void>, BinarySink {
writeOptionalNode(node.initializer);
// Declare the variable after its initializer. It is not in scope in its
// own initializer.
(_variableIndexer ??= new VariableIndexer()).declare(node);
(_variableIndexer ??= _newVariableIndexer()).declare(node);
}
void writeVariableDeclarationList(List<VariableDeclaration> nodes) {
@ -3294,6 +3298,14 @@ class BinaryPrinter implements Visitor<void>, BinarySink {
throw new UnsupportedError("serialization of auxiliary constant reference "
"${node} (${node.runtimeType}).");
}
VariableIndexer? getVariableIndexerForTesting() {
return _variableIndexer;
}
TypeParameterIndexer getTypeParameterIndexerForTesting() {
return _typeParameterIndexer;
}
}
typedef bool LibraryFilter(Library _);

309
pkg/kernel/lib/dart_scope_calculator.dart
View file

@ -4,19 +4,31 @@
import 'package:kernel/ast.dart'
show
AssertBlock,
Block,
BlockExpression,
Catch,
Class,
Component,
Constructor,
DartType,
Extension,
ExtensionTypeDeclaration,
Field,
FileUriNode,
ForInStatement,
ForStatement,
FunctionNode,
Initializer,
Let,
Library,
Member,
Node,
Procedure,
TreeNode,
TypeParameter,
Typedef,
TypedefTearOff,
VariableDeclaration,
VisitorDefault,
VisitorNullMixin,
@ -26,6 +38,7 @@ import 'package:kernel/ast.dart'
///
/// Provides information about symbols available inside a dart scope.
class DartScope {
final TreeNode? node;
final Library library;
final Class? cls;
final Member? member;
@ -33,7 +46,7 @@ class DartScope {
final Map<String, DartType> definitions;
final List<TypeParameter> typeParameters;
DartScope(this.library, this.cls, this.member, this.definitions,
DartScope(this.node, this.library, this.cls, this.member, this.definitions,
this.typeParameters)
: isStatic = member is Procedure ? member.isStatic : false;
@ -86,7 +99,8 @@ class DartScopeBuilder extends VisitorDefault<void> with VisitorVoidMixin {
DartScope? build() {
if (_offset < 0 || _library == null) return null;
return DartScope(_library!, _cls, _member, _definitions, _typeParameters);
return DartScope(
null, _library!, _cls, _member, _definitions, _typeParameters);
}
@override
@ -244,3 +258,294 @@ class FileEndOffsetCalculator extends VisitorDefault<int?>
return noOffset;
}
}
class DartScopeBuilder2 extends VisitorDefault<void> with VisitorVoidMixin {
final Library _library;
final Uri _scriptUri;
final int _offset;
final List<DartScope> findScopes = [];
final List<List<VariableDeclaration>> scopes = [];
final List<List<TypeParameter>> typeParameterScopes = [];
Class? _currentCls = null;
Member? _currentMember = null;
bool checkClasses = true;
Uri _currentUri;
DartScopeBuilder2._(this._library, this._scriptUri, this._offset)
: _currentUri = _library.fileUri;
void addFound(TreeNode node) {
Map<String, DartType> definitions = {};
for (List<VariableDeclaration> scope in scopes) {
for (VariableDeclaration decl in scope) {
String? name = decl.name;
if (name != null && name != "") {
definitions[name] = decl.type;
}
}
}
List<TypeParameter> typeParameters = [];
for (List<TypeParameter> typeParameterScope in typeParameterScopes) {
typeParameters.addAll(typeParameterScope);
}
DartScope findScope = new DartScope(node, _library, _currentCls,
_currentMember, definitions, typeParameters);
findScopes.add(findScope);
}
@override
void defaultDartType(DartType node) {
return;
}
@override
void defaultTreeNode(TreeNode node) {
Uri prevUri = _currentUri;
if (node is FileUriNode) {
_currentUri = node.fileUri;
}
_checkOffset(node);
node.visitChildren(this);
_currentUri = prevUri;
}
@override
void visitAssertBlock(AssertBlock node) {
scopes.add([]);
super.visitAssertBlock(node);
scopes.removeLast();
}
@override
void visitBlock(Block node) {
scopes.add([]);
_checkOffset(node);
bool shouldSkip;
// See also the test [DillBlockChecker] which checks that we can do this
// pruning!
if (_currentUri != _scriptUri) {
shouldSkip = true;
} else if (node.parent is ForInStatement) {
// E.g. dart2js implicit cast in for-in loop
shouldSkip = false;
} else if (node.parent?.parent is ForStatement) {
// A vm transformation turns
// `for (var foo in bar) {}`
// into
// `for(;iterator.moveNext; ) { var foo = iterator.current; {} }`
// where the block directly containing `foo` has the original blocks
// offset, i.e. after the variable declaration, but it still contain
// it. So we pretend it has no offsets.
shouldSkip = false;
} else if (node.fileOffset >= 0 &&
node.fileEndOffset >= 0 &&
node.fileOffset != node.fileEndOffset) {
if (_offset < node.fileOffset || _offset > node.fileEndOffset) {
// Not contained in the block.
shouldSkip = true;
} else {
// Contained in the block.
shouldSkip = false;
}
} else {
// The block doesn't have valid offsets.
shouldSkip = false;
}
if (!shouldSkip) {
node.visitChildren(this);
}
scopes.removeLast();
}
@override
void visitBlockExpression(BlockExpression node) {
scopes.add([]);
super.visitBlockExpression(node);
scopes.removeLast();
}
@override
void visitCatch(Catch node) {
scopes.add([]);
super.visitCatch(node);
scopes.removeLast();
}
@override
void visitClass(Class node) {
if (!checkClasses) {
return;
}
_currentCls = node;
typeParameterScopes.add([...node.typeParameters]);
scopes.add([]);
super.visitClass(node);
scopes.clear();
typeParameterScopes.removeLast();
assert(typeParameterScopes.isEmpty);
_currentCls = null;
}
@override
void visitConstructor(Constructor node) {
Uri prevUri = _currentUri;
_currentUri = node.fileUri;
_currentMember = node;
scopes.clear();
scopes.add([]);
_checkOffset(node);
// The constructor is special in that the parameters from the contained
// function node is in scope in the initializers.
node.function.accept(this);
for (VariableDeclaration param in node.function.positionalParameters) {
scopes.last.add(param);
}
for (VariableDeclaration param in node.function.namedParameters) {
scopes.last.add(param);
}
for (Initializer initializer in node.initializers) {
initializer.accept(this);
}
scopes.clear();
_currentMember = null;
_currentUri = prevUri;
}
@override
void visitExtension(Extension node) {
typeParameterScopes.add([...node.typeParameters]);
super.visitExtension(node);
typeParameterScopes.removeLast();
}
@override
void visitExtensionTypeDeclaration(ExtensionTypeDeclaration node) {
typeParameterScopes.add([...node.typeParameters]);
super.visitExtensionTypeDeclaration(node);
typeParameterScopes.removeLast();
}
@override
void visitField(Field node) {
_currentMember = node;
scopes.clear();
scopes.add([]);
super.visitField(node);
scopes.clear();
_currentMember = null;
}
@override
void visitForInStatement(ForInStatement node) {
scopes.add([]);
super.visitForInStatement(node);
scopes.removeLast();
}
@override
void visitForStatement(ForStatement node) {
scopes.add([]);
super.visitForStatement(node);
scopes.removeLast();
}
@override
void visitFunctionNode(FunctionNode node) {
typeParameterScopes.add([...node.typeParameters]);
scopes.add([]);
super.visitFunctionNode(node);
scopes.removeLast();
typeParameterScopes.removeLast();
}
@override
void visitLet(Let node) {
scopes.add([]);
super.visitLet(node);
scopes.removeLast();
}
@override
void visitLibrary(Library node) {
scopes.add([]);
super.visitLibrary(node);
scopes.clear();
}
@override
void visitProcedure(Procedure node) {
_currentMember = node;
scopes.clear();
scopes.add([]);
super.visitProcedure(node);
scopes.clear();
_currentMember = null;
}
@override
void visitTypedef(Typedef node) {
scopes.clear();
scopes.add([]);
typeParameterScopes.add([...node.typeParameters]);
super.visitTypedef(node);
typeParameterScopes.removeLast();
scopes.clear();
}
@override
void visitTypedefTearOff(TypedefTearOff node) {
typeParameterScopes.add([...node.typeParameters]);
super.visitTypedefTearOff(node);
typeParameterScopes.removeLast();
}
@override
void visitVariableDeclaration(VariableDeclaration node) {
super.visitVariableDeclaration(node);
// Declare it after.
scopes.last.add(node);
}
void _checkOffset(TreeNode node) {
if (_currentUri == _scriptUri) {
if (node.fileOffset == _offset) {
addFound(node);
} else {
List<int>? allOffsets = node.fileOffsetsIfMultiple;
if (allOffsets != null) {
for (final int offset in allOffsets) {
if (offset == _offset) {
addFound(node);
break;
}
}
}
}
}
}
static List<DartScope> findScopeFromOffsetAndClass(
Library library, Uri scriptUri, Class? cls, int offset) {
DartScopeBuilder2 builder = DartScopeBuilder2._(library, scriptUri, offset);
if (cls != null) {
builder.visitClass(cls);
} else {
builder.checkClasses = false;
builder.visitLibrary(library);
}
return builder.findScopes;
}
}

50
pkg/kernel/test/binary/can_read_platform_test.dart
View file

@ -7,54 +7,10 @@ import 'dart:io';
import 'package:kernel/binary/ast_from_binary.dart';
import 'package:kernel/kernel.dart';
import 'find_sdk_dills.dart';
void main() {
File exe = new File(Platform.resolvedExecutable).absolute;
int steps = 0;
Directory parent = exe.parent.parent;
while (true) {
Set<String> foundDirs = {};
for (FileSystemEntity entry in parent.listSync(recursive: false)) {
if (entry is Directory) {
List<String> pathSegments = entry.uri.pathSegments;
String name = pathSegments[pathSegments.length - 2];
foundDirs.add(name);
}
}
if (foundDirs.contains("pkg") &&
foundDirs.contains("tools") &&
foundDirs.contains("tests")) {
break;
}
steps++;
if (parent.uri == parent.parent.uri) {
throw "Reached end without finding the root.";
}
parent = parent.parent;
}
// We had to go $steps steps to reach the "root" --- now we should go 2 steps
// shorter to be in the "compiled dir".
parent = exe.parent;
for (int i = steps - 2; i >= 0; i--) {
parent = parent.parent;
}
List<File> dills = [];
for (FileSystemEntity entry in parent.listSync(recursive: false)) {
if (entry is File) {
if (entry.path.toLowerCase().endsWith(".dill")) {
dills.add(entry);
}
}
}
Directory sdk = new Directory.fromUri(parent.uri.resolve("dart-sdk/"));
for (FileSystemEntity entry in sdk.listSync(recursive: true)) {
if (entry is File) {
if (entry.path.toLowerCase().endsWith(".dill")) {
dills.add(entry);
}
}
}
List<File> dills = findSdkDills();
print("Found ${dills.length} dills!");
List<File> errors = [];

99
pkg/kernel/test/binary/find_sdk_dills.dart Normal file
View file

@ -0,0 +1,99 @@
// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
import 'dart:io';
/// Finds unique dills in the sdk.
///
/// Note that it reads all dills found to create and compare a checksum to
/// remove duplicates (and re-read on checksum matches).
List<File> findSdkDills() {
File exe = new File(Platform.resolvedExecutable).absolute;
int steps = 0;
Directory parent = exe.parent.parent;
while (true) {
Set<String> foundDirs = {};
for (FileSystemEntity entry in parent.listSync(recursive: false)) {
if (entry is Directory) {
List<String> pathSegments = entry.uri.pathSegments;
String name = pathSegments[pathSegments.length - 2];
foundDirs.add(name);
}
}
if (foundDirs.contains("pkg") &&
foundDirs.contains("tools") &&
foundDirs.contains("tests")) {
break;
}
steps++;
if (parent.uri == parent.parent.uri) {
throw "Reached end without finding the root.";
}
parent = parent.parent;
}
// We had to go $steps steps to reach the "root" --- now we should go 2 steps
// shorter to be in the "compiled dir".
parent = exe.parent;
for (int i = steps - 2; i >= 0; i--) {
parent = parent.parent;
}
List<File> dills = [];
List<int> checksums = [];
for (FileSystemEntity entry in parent.listSync(recursive: false)) {
if (entry is File) {
if (entry.path.toLowerCase().endsWith(".dill")) {
_addIfNotDuplicate(entry, dills, checksums);
}
}
}
Directory sdk = new Directory.fromUri(parent.uri.resolve("dart-sdk/"));
for (FileSystemEntity entry in sdk.listSync(recursive: true)) {
if (entry is File) {
if (entry.path.toLowerCase().endsWith(".dill")) {
_addIfNotDuplicate(entry, dills, checksums);
}
}
}
return dills;
}
void _addIfNotDuplicate(File f, List<File> existingFiles, List<int> checksums) {
List<int> content = f.readAsBytesSync();
int adler = adler32(content);
for (int i = 0; i < checksums.length; i++) {
if (checksums[i] == adler) {
List<int> existingContent = existingFiles[i].readAsBytesSync();
bool duplicate = existingContent.length == content.length;
if (duplicate) {
for (int j = 0; j < existingContent.length; j++) {
if (existingContent[j] != content[j]) {
duplicate = false;
break;
}
}
}
if (duplicate) return;
}
}
checksums.add(adler);
existingFiles.add(f);
}
int adler32(List<int> data) {
int a = 1;
int b = 0;
for (int i = 0; i < data.length; i++) {
a += data[i];
b += a;
if (i & 255 == 255) {
a %= 65521;
b %= 65521;
}
}
a %= 65521;
b %= 65521;
return (b << 16) | a;
}

465
pkg/kernel/test/dart_scope_calculator_test.dart Normal file
View file

@ -0,0 +1,465 @@
// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
import 'dart:io';
import 'package:kernel/ast.dart';
import 'package:kernel/binary/ast_from_binary.dart';
import 'package:kernel/binary/ast_to_binary.dart';
import 'package:kernel/dart_scope_calculator.dart';
import 'binary/find_sdk_dills.dart';
void main() {
List<File> dills = findSdkDills();
print("Found ${dills.length} dill files.");
for (int i = 0; i < dills.length; i++) {
print("");
testDill(dills[i]);
print("Finished dill ${i + 1} out of ${dills.length}");
}
List<MapEntry<String, int>> fromWhereList = fromWhereMap.entries.toList();
fromWhereList.sort((a, b) => b.value - a.value);
print("");
print("More-than-ones came from here:");
for (MapEntry<String, int> entry in fromWhereList.take(5)) {
print(" => ${entry.value}: ${entry.key}");
}
print("");
print("Valid block offsets: ${DillBlockChecker.validBlockOffset}");
print("Empty block offsets: ${DillBlockChecker.emptyBlockOffset}");
print("No block offsets: ${DillBlockChecker.noBlockOffset}");
print("");
if (errors.isNotEmpty) {
print("");
print("GOT ERRORS!");
for (String e in errors) {
print(" - $e");
}
throw "Errors detected.";
} else {
print("No direct errors found.");
}
}
List<String> errors = [];
void testDill(File dill) {
print("Looking at $dill");
Component component = new Component();
new BinaryBuilder(dill.readAsBytesSync()).readComponent(component);
DillBlockChecker dillBlockChecker = new DillBlockChecker();
component.accept(dillBlockChecker);
ScopeTestingBinaryPrinter binaryPrinter = new ScopeTestingBinaryPrinter();
binaryPrinter.writeComponentFile(component);
print("${binaryPrinter.exact} out of ${binaryPrinter.total} "
"(${binaryPrinter.moreButAgree}/${binaryPrinter.total - binaryPrinter.exact})");
int totalAgree = binaryPrinter.exact + binaryPrinter.moreButAgree;
print(" => ${totalAgree * 100 / binaryPrinter.total}%");
}
class DevNullSink<T> implements Sink<T> {
const DevNullSink();
@override
void add(T data) {}
@override
void close() {}
}
/// Checks that each block (except for a few known bad cases) contains all
/// offsets inside it, thus verifying that we can use the blocks offsets to
/// skip/prune parts of the tree while searching for node(s) with a specific
/// offset.
class DillBlockChecker extends VisitorDefault<void> with VisitorVoidMixin {
Uri _currentUri = Uri.parse("dummy:uri");
int start = -1;
int end = -1;
bool insideType = false;
static int validBlockOffset = 0;
static int emptyBlockOffset = 0;
static int noBlockOffset = 0;
@override
void defaultDartType(DartType node) {
bool oldInsideType = insideType;
insideType = true;
super.defaultDartType(node);
insideType = oldInsideType;
}
@override
void defaultTreeNode(TreeNode node) {
if (insideType) {
throw "Got to a treenode from inside a type.";
}
Uri prevUri = _currentUri;
if (node is FileUriNode) {
_currentUri = node.fileUri;
}
if (prevUri == _currentUri && start >= 0 && end >= 0) {
// This node should be contained.
for (int offset in [node.fileOffset, ...?node.fileOffsetsIfMultiple]) {
if (offset >= 0) {
if (offset < start || offset > end) {
// Not contained.
throw "Error on $node; $offset not in [$start, $end] "
"(${node.parent.runtimeType} "
"${node.parent?.parent.runtimeType}) "
"${node.parent?.parent?.parent.runtimeType})";
}
}
}
}
bool hasVisited = false;
if (node is Block) {
if (node.fileOffset == node.fileEndOffset) {
emptyBlockOffset++;
} else if (node.fileOffset < 0 || node.fileEndOffset < 0) {
noBlockOffset++;
} else {
validBlockOffset++;
}
if (node.parent is ForInStatement) {
// E.g. dart2js implicit cast in for-in loop
} else if (node.parent?.parent is ForStatement) {
// A vm transformation turns
// `for (var foo in bar) {}`
// into
// `for(;iterator.moveNext; ) { var foo = iterator.current; {} }`
// where the block directly containing `foo` has the original blocks
// offset, i.e. after the variable declaration, but it still contain
// it. So we pretend it has no offsets.
} else if (node.fileOffset >= 0 &&
node.fileEndOffset >= 0 &&
node.fileOffset != node.fileEndOffset) {
int prevStart = start;
int prevEnd = end;
start = node.fileOffset;
end = node.fileEndOffset;
node.visitChildren(this);
hasVisited = true;
end = prevEnd;
start = prevStart;
}
}
if (!hasVisited) {
node.visitChildren(this);
}
_currentUri = prevUri;
}
}
final Map<String, int> fromWhereMap = {};
class ScopeTestingBinaryPrinter extends BinaryPrinter {
Library? currentLibrary;
Class? currentClass;
Member? currentMember;
Uri? currentUri;
bool checkOffset = false;
Set<Member> skipMembers = {};
int exact = 0;
int total = 0;
int moreButAgree = 0;
ScopeTestingBinaryPrinter()
: super(const DevNullSink(),
newVariableIndexerForTesting: VariableIndexer2.new);
@override
void visitClass(Class node) {
currentClass = node;
Uri? prevUri = currentUri;
currentUri = node.fileUri;
super.visitClass(node);
currentUri = prevUri;
currentClass = null;
}
@override
void visitConstructor(Constructor node) {
currentMember = node;
Uri? prevUri = currentUri;
currentUri = node.fileUri;
super.visitConstructor(node);
currentUri = prevUri;
currentMember = null;
}
@override
void visitExtension(Extension node) {
for (ExtensionMemberDescriptor memberDescriptor in node.memberDescriptors) {
// The tear off procedures have two enclosing function nodes with the same
// offsets, but with (possibly) different type parameters. Skip them.
Member? skip = memberDescriptor.tearOffReference?.asMember;
if (skip != null) skipMembers.add(skip);
}
Uri? prevUri = currentUri;
currentUri = node.fileUri;
super.visitExtension(node);
currentUri = prevUri;
}
@override
void visitExtensionTypeDeclaration(ExtensionTypeDeclaration node) {
Uri? prevUri = currentUri;
currentUri = node.fileUri;
super.visitExtensionTypeDeclaration(node);
currentUri = prevUri;
}
@override
void visitField(Field node) {
currentMember = node;
Uri? prevUri = currentUri;
currentUri = node.fileUri;
super.visitField(node);
currentUri = prevUri;
currentMember = null;
}
@override
void visitFileUriExpression(FileUriExpression node) {
Uri? prevUri = currentUri;
currentUri = node.fileUri;
super.visitFileUriExpression(node);
currentUri = prevUri;
}
@override
void visitFunctionNode(FunctionNode node) {
bool oldCheckOffset = checkOffset;
checkOffset = true;
super.visitFunctionNode(node);
checkOffset = oldCheckOffset;
}
@override
void visitLibrary(Library node) {
currentLibrary = node;
Uri? prevUri = currentUri;
currentUri = node.fileUri;
super.visitLibrary(node);
currentUri = prevUri;
currentLibrary = null;
}
@override
void visitProcedure(Procedure node) {
currentMember = node;
Uri? prevUri = currentUri;
currentUri = node.fileUri;
super.visitProcedure(node);
currentUri = prevUri;
currentMember = null;
}
@override
void visitTypedef(Typedef node) {
Uri? prevUri = currentUri;
currentUri = node.fileUri;
super.visitTypedef(node);
currentUri = prevUri;
}
@override
void writeOffset(int offset) {
// TODO(jensj): Currently, e.g. for function node, we write the end offset
// before the data --- and the actual code finding it sees it before too,
// but really, if we've asked for the scope at the end we've seen everything
// inside --- although if that's theoretically in scope or not is probably
// up for debate.
if (checkOffset && offset >= 0 && !skipMembers.contains(currentMember)) {
List<DartScope> nodesAtPoint =
DartScopeBuilder2.findScopeFromOffsetAndClass(
currentLibrary!, currentUri!, currentClass, offset);
List<Object> expectedTypeParameters =
getTypeParameterIndexerForTesting().index.keys.toList();
VariableIndexer2? varIndexer =
getVariableIndexerForTesting() as VariableIndexer2?;
Map<String, DartType> expectedVariablesMap = {};
for (VariableDeclaration variable in varIndexer?.declsOrder ?? const []) {
String? name = variable.name;
if (name != null && name != "") {
expectedVariablesMap[name] = variable.type;
}
}
total++;
if (nodesAtPoint.length == 0) {
String msg = "Didn't find any scope for "
"${currentLibrary!.fileUri} $currentUri and $offset";
errors.add(msg);
print(msg);
} else if (nodesAtPoint.length == 1) {
exact++;
if (!compareScopes(expectedTypeParameters, expectedVariablesMap,
nodesAtPoint.single, offset,
doPrint: true)) {
errors.add("Found 1 scope, but it didn't match for "
"${currentLibrary!.fileUri} $currentUri and $offset");
}
} else {
// Does one that agree exist?
bool foundMatch = false;
bool allMatch = true;
for (DartScope compareMe in nodesAtPoint) {
if (compareScopes(
expectedTypeParameters, expectedVariablesMap, compareMe, offset,
doPrint: false)) {
foundMatch = true;
} else {
allMatch = false;
}
}
if (!foundMatch) {
String msg =
"Found ${nodesAtPoint.length} scopes, but didn't one matching "
"${currentLibrary!.fileUri} $currentUri and $offset";
print(msg);
errors.add(msg);
}
if (allMatch) {
moreButAgree++;
} else {
String fromWhere =
StackTrace.current.toString().split("\n").skip(1).first;
fromWhereMap[fromWhere] = (fromWhereMap[fromWhere] ?? 0) + 1;
}
}
}
super.writeOffset(offset);
}
bool compareScopes(
List<Object> expectedTypeParameters,
Map<String, DartType> expectedVariablesMap,
DartScope compareWith,
int offsetForErrorMessage,
{required bool doPrint}) {
bool compareOk = true;
if (expectedTypeParameters.length != compareWith.typeParameters.length) {
compareOk = false;
if (doPrint) {
print("Failure on type parameters for "
"${currentLibrary!.fileUri} $currentUri and "
"$offsetForErrorMessage -- "
"${compareWith.typeParameters} vs $expectedTypeParameters");
}
} else {
// Do they agree?
for (int i = 0; i < expectedTypeParameters.length; i++) {
TypeParameter a = expectedTypeParameters[i] as TypeParameter;
TypeParameter b = compareWith.typeParameters[i];
if (!identical(a, b)) {
compareOk = false;
if (doPrint) {
print("$a != $b");
}
}
}
}
if (compareWith.cls != currentClass) {
compareOk = false;
if (doPrint) {
print("Failure on class for "
"${currentLibrary!.fileUri} $currentUri and "
"$offsetForErrorMessage -- "
"${compareWith.cls} vs $currentClass");
}
}
if (expectedVariablesMap.length != compareWith.definitions.length) {
compareOk = false;
if (doPrint) {
print("Failure on definitions for "
"${currentLibrary!.fileUri} $currentUri and "
"$offsetForErrorMessage -- "
"${compareWith.definitions} vs $expectedVariablesMap");
}
} else {
// Do they agree?
for (String variableName in expectedVariablesMap.keys) {
DartType? a = expectedVariablesMap[variableName];
DartType? b = compareWith.definitions[variableName];
if (a != b) {
compareOk = false;
if (doPrint) {
print("$a != $b");
}
}
}
}
return compareOk;
}
@override
void writeVariableDeclaration(VariableDeclaration node) {
bool oldCheckOffset = checkOffset;
checkOffset = true;
super.writeVariableDeclaration(node);
checkOffset = oldCheckOffset;
}
}
class VariableIndexer2 implements VariableIndexer {
List<VariableDeclaration> declsOrder = [];
List<VariableDeclaration> declsOrderPopped = [];
@override
Map<VariableDeclaration, int>? index;
@override
List<int>? scopes;
@override
int stackHeight = 0;
@override
int? operator [](VariableDeclaration node) {
return index == null ? null : index![node];
}
@override
void declare(VariableDeclaration node) {
(index ??= <VariableDeclaration, int>{})[node] = stackHeight++;
declsOrder.add(node);
}
@override
void popScope() {
stackHeight = scopes!.removeLast();
while (declsOrder.length > stackHeight) {
declsOrderPopped.add(declsOrder.removeLast());
}
}
@override
void pushScope() {
(scopes ??= <int>[]).add(stackHeight);
}
@override
void restoreScope(int numberOfVariables) {
stackHeight += numberOfVariables;
while (declsOrder.length < stackHeight) {
declsOrder.add(declsOrderPopped.removeLast());
}
}
}

1
pkg/pkg.status
View file

@ -63,6 +63,7 @@ front_end/test/incremental_compiler_leak_test: Pass, ExtraSlow
front_end/test/incremental_dart2js_test: Pass, Slow
front_end/testcases/*: Skip # These are not tests but input for tests.
front_end/tool/incremental_perf_test: Slow, Pass
kernel/test/dart_scope_calculator_test: Slow, Pass
kernel/testcases/*: Skip # These are not tests but input for tests.
vm/test/kernel_front_end_test: Slow, Pass
vm/test/transformations/type_flow/transformer_test: Slow, Pass