From c768f80a96774ed465b606921ab21f5b7ec200de Mon Sep 17 00:00:00 2001 From: Stephen Adams Date: Tue, 28 Feb 2023 03:00:18 +0000 Subject: [PATCH] [dart2js] Implement `@pragma('dart2js:resource-identifier')` Calls to methods annotated with `@pragma('dart2js:resource-identifier')` are tracked, with their primitive constant arguments, through to the `.js` file which contains the call. - JavaScript annotations are attached to the JavaScript AST node for the call. - At the time of `.js` file printing, the JavaScript annotations are collected and attributed to the file. This allows the construction of a map from `.js` files to the 'resource identifiers' contained in the file. - Alongside the `main.js` file the resource identifiers are emitted in a file called `main.js.resources.json`. This is controlled by the `--write-resources` command line option. - Serialization of JavaScript ASTs now serializes the attached JavaScript annotations. - The internal method used to implement deferred library loading is annotated, to allow analysis of which deferred library parts load other libraries. pkg/js_ast was tweaked to make the pkg/js_ast was tweaked to make the - pkg/js_ast was tweaked to make propagating the JavaScript annotations through the async transforms easier. TODO: - Annotate const constructors - Add golden-style tests Change-Id: Iea77550e22ee98f81dca61dfd713c09f734583d2 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/284492 Reviewed-by: Nate Biggs Commit-Queue: Stephen Adams --- pkg/compiler/doc/pragmas.md | 1 + pkg/compiler/doc/resource_identifiers.md | 151 +++++++++++++++ pkg/compiler/lib/compiler_api.dart | 3 + pkg/compiler/lib/src/commandline_options.dart | 2 + pkg/compiler/lib/src/common/codegen.dart | 67 ++++++- pkg/compiler/lib/src/dart2js.dart | 1 + pkg/compiler/lib/src/js/js.dart | 23 ++- pkg/compiler/lib/src/js/rewrite_async.dart | 3 +- .../lib/src/js_backend/annotations.dart | 21 ++- .../src/js_emitter/resource_info_emitter.dart | 176 ++++++++++++++++++ .../startup_emitter/model_emitter.dart | 39 +++- pkg/compiler/lib/src/options.dart | 5 + .../lib/src/source_file_provider.dart | 1 + pkg/compiler/lib/src/ssa/builder.dart | 7 +- pkg/compiler/lib/src/ssa/codegen.dart | 50 +++++ .../lib/src/universe/resource_identifier.dart | 162 ++++++++++++++++ .../test/end_to_end/output_type_test.dart | 13 ++ pkg/js_ast/lib/src/nodes.dart | 34 ++++ .../_internal/js_runtime/lib/js_helper.dart | 1 + 19 files changed, 735 insertions(+), 25 deletions(-) create mode 100644 pkg/compiler/doc/resource_identifiers.md create mode 100644 pkg/compiler/lib/src/js_emitter/resource_info_emitter.dart create mode 100644 pkg/compiler/lib/src/universe/resource_identifier.dart diff --git a/pkg/compiler/doc/pragmas.md b/pkg/compiler/doc/pragmas.md index bf4059612bd..569951678b0 100644 --- a/pkg/compiler/doc/pragmas.md +++ b/pkg/compiler/doc/pragmas.md @@ -12,6 +12,7 @@ | `dart2js:noElision` | Disables an optimization whereby unused fields or unused parameters are removed | | `dart2js:load-priority:normal` | [Affects deferred library loading](#load-priority) | | `dart2js:load-priority:high` | [Affects deferred library loading](#load-priority) | +| `dart2js:resource-identifer` | [Collects data references to resources](resource-identifers.md) | ## Unsafe pragmas for general use diff --git a/pkg/compiler/doc/resource_identifiers.md b/pkg/compiler/doc/resource_identifiers.md new file mode 100644 index 00000000000..e3e3dbc5905 --- /dev/null +++ b/pkg/compiler/doc/resource_identifiers.md @@ -0,0 +1,151 @@ +# Resource Identifiers + +Content TBD. Work in progress and details in flux. + +### Example output + +TODO: Reference goldens in tests rather than keep the example below. + +The call to `loadDeferredLibrary` in the Dart js_runtime is annotated with +`@pragma('dart2js:resource-identifier')`. This means that an app that uses +deferred loaded libraries will generate a section in the `.resources.json`. + + +In the Dart sdk directory, compile: + +```sh +dart compile js --write-resources --out=somedir/o.js \ + benchmarks/OmnibusDeferred/dart/OmnibusDeferred.dart +``` + +`somedir/o.js.resource_identifiers.json`: + +```json +{ + "environment": { + "dart.web.assertions_enabled": "false" + }, + "identifiers": [ + { + "name": "loadDeferredLibrary", + "uri": "org-dartlang-sdk:///lib/_internal/js_runtime/lib/js_helper.dart", + "nonconstant": false, + "files": [ + { + "filename": "o.js", + "references": [ + { + "@": { + "uri": "benchmarks/OmnibusDeferred/dart/OmnibusDeferred.dart", + "line": 15, + "column": 17 + }, + "1": "lib_BigIntParsePrint", + "2": 0 + }, + { + "@": { + "uri": "benchmarks/OmnibusDeferred/dart/OmnibusDeferred.dart", + "line": 16, + "column": 56 + }, + "1": "lib_ListCopy", + "2": 0 + }, + { + "@": { + "uri": "benchmarks/OmnibusDeferred/dart/OmnibusDeferred.dart", + "line": 17, + "column": 54 + }, + "1": "lib_MapCopy", + "2": 0 + }, + { + "@": { + "uri": "benchmarks/OmnibusDeferred/dart/OmnibusDeferred.dart", + "line": 18, + "column": 46 + }, + "1": "lib_MD5", + "2": 0 + }, + { + "@": { + "uri": "benchmarks/OmnibusDeferred/dart/OmnibusDeferred.dart", + "line": 19, + "column": 62 + }, + "1": "lib_RuntimeType", + "2": 0 + }, + { + "@": { + "uri": "benchmarks/OmnibusDeferred/dart/OmnibusDeferred.dart", + "line": 20, + "column": 48 + }, + "1": "lib_SHA1", + "2": 0 + }, + { + "@": { + "uri": "benchmarks/OmnibusDeferred/dart/OmnibusDeferred.dart", + "line": 21, + "column": 52 + }, + "1": "lib_SHA256", + "2": 0 + }, + { + "@": { + "uri": "benchmarks/OmnibusDeferred/dart/OmnibusDeferred.dart", + "line": 23, + "column": 17 + }, + "1": "lib_SkeletalAnimation", + "2": 0 + }, + { + "@": { + "uri": "benchmarks/OmnibusDeferred/dart/OmnibusDeferred.dart", + "line": 25, + "column": 17 + }, + "1": "lib_SkeletalAnimationSIMD", + "2": 0 + }, + { + "@": { + "uri": "benchmarks/OmnibusDeferred/dart/OmnibusDeferred.dart", + "line": 27, + "column": 17 + }, + "1": "lib_TypedDataDuplicate", + "2": 0 + }, + { + "@": { + "uri": "benchmarks/OmnibusDeferred/dart/OmnibusDeferred.dart", + "line": 28, + "column": 60 + }, + "1": "lib_Utf8Decode", + "2": 0 + }, + { + "@": { + "uri": "benchmarks/OmnibusDeferred/dart/OmnibusDeferred.dart", + "line": 29, + "column": 60 + }, + "1": "lib_Utf8Encode", + "2": 0 + } + ] + } + ] + } + ] +} +``` diff --git a/pkg/compiler/lib/compiler_api.dart b/pkg/compiler/lib/compiler_api.dart index 61f5be79c0e..de802922f89 100644 --- a/pkg/compiler/lib/compiler_api.dart +++ b/pkg/compiler/lib/compiler_api.dart @@ -142,6 +142,9 @@ enum OutputType { /// Unused libraries output. dumpUnusedLibraries, + /// Resource identifiers output. + resourceIdentifiers, + /// Implementation specific output used for debugging the compiler. debug, } diff --git a/pkg/compiler/lib/src/commandline_options.dart b/pkg/compiler/lib/src/commandline_options.dart index c4a438e8080..93c48bca1d2 100644 --- a/pkg/compiler/lib/src/commandline_options.dart +++ b/pkg/compiler/lib/src/commandline_options.dart @@ -133,6 +133,8 @@ class Flags { static const String noSoundNullSafety = '--no-sound-null-safety'; static const String mergeFragmentsThreshold = '--merge-fragments-threshold'; + static const String writeResources = '--write-resources'; + /// Flag for a combination of flags for 'production' mode. static const String benchmarkingProduction = '--benchmarking-production'; diff --git a/pkg/compiler/lib/src/common/codegen.dart b/pkg/compiler/lib/src/common/codegen.dart index efc3bacc58b..c6c43926d02 100644 --- a/pkg/compiler/lib/src/common/codegen.dart +++ b/pkg/compiler/lib/src/common/codegen.dart @@ -30,6 +30,7 @@ import '../js_model/type_recipe.dart' show TypeRecipe; import '../native/behavior.dart'; import '../serialization/serialization.dart'; import '../universe/feature.dart'; +import '../universe/resource_identifier.dart' show ResourceIdentifier; import '../universe/selector.dart'; import '../universe/use.dart' show ConstantUse, DynamicUse, StaticUse, TypeUse; import '../universe/world_impact.dart' show WorldImpact, WorldImpactBuilderImpl; @@ -789,6 +790,11 @@ class JsNodeTags { static const String deferredHolderExpression = 'js-deferredHolderExpression'; } +enum JsAnnotationKind { + string, + resourceIdentifier, +} + /// Visitor that serializes a [js.Node] into a [DataSinkWriter]. class JsNodeSerializer implements js.NodeVisitor { final DataSinkWriter sink; @@ -819,11 +825,37 @@ class JsNodeSerializer implements js.NodeVisitor { } void _writeInfo(js.Node node) { - sink.writeCached( - node.sourceInformation as SourceInformation?, - (SourceInformation sourceInformation) { - SourceInformation.writeToDataSink(sink, sourceInformation); - }); + final sourceInformation = node.sourceInformation as SourceInformation?; + final annotations = node.annotations; + // Low bit encodes presence of `sourceInformation`, higher bits the number + // of annotations. + final infoCode = + (sourceInformation == null ? 0 : 1) + 2 * annotations.length; + sink.writeInt(infoCode); + final hasSourceInformation = infoCode.isOdd; + final annotationCount = infoCode ~/ 2; + if (hasSourceInformation) { + sink.writeCached(sourceInformation, + (SourceInformation sourceInformation) { + SourceInformation.writeToDataSink(sink, sourceInformation); + }); + } + for (int i = 0; i < annotationCount; i++) { + _writeAnnotation(annotations[i]); + } + } + + void _writeAnnotation(Object annotation) { + if (annotation is String) { + sink.writeEnum(JsAnnotationKind.string); + sink.writeString(annotation); + } else if (annotation is ResourceIdentifier) { + sink.writeEnum(JsAnnotationKind.resourceIdentifier); + annotation.writeToDataSink(sink); + } else { + throw UnsupportedError( + 'JsNodeAnnotation ${annotation.runtimeType}: $annotation'); + } } @override @@ -1891,18 +1923,35 @@ class JsNodeDeserializer { source.end(JsNodeTags.deferredHolderExpression); break; } - final sourceInformation = source.readCachedOrNull(() { - return SourceInformation.readFromDataSource(source); - }); - if (sourceInformation != null) { + + final infoCode = source.readInt(); + final hasSourceInformation = infoCode.isOdd; + final annotationCount = infoCode ~/ 2; + if (hasSourceInformation) { + final sourceInformation = source.readCachedOrNull(() { + return SourceInformation.readFromDataSource(source); + }); node = node.withSourceInformation(sourceInformation); } + for (int i = 0; i < annotationCount; i++) { + node = node.withAnnotation(_readAnnotation()); + } return node as T; } List readList() { return source.readList(read); } + + Object _readAnnotation() { + final kind = source.readEnum(JsAnnotationKind.values); + switch (kind) { + case JsAnnotationKind.string: + return source.readString(); + case JsAnnotationKind.resourceIdentifier: + return ResourceIdentifier.readFromDataSource(source); + } + } } class CodegenReaderImpl implements CodegenReader { diff --git a/pkg/compiler/lib/src/dart2js.dart b/pkg/compiler/lib/src/dart2js.dart index 8ee2cb267ae..06e2252391d 100644 --- a/pkg/compiler/lib/src/dart2js.dart +++ b/pkg/compiler/lib/src/dart2js.dart @@ -664,6 +664,7 @@ Future compile(List argv, _OneOption(Flags.soundNullSafety, setNullSafetyMode), _OneOption(Flags.noSoundNullSafety, setNullSafetyMode), _OneOption(Flags.dumpUnusedLibraries, passThrough), + _OneOption(Flags.writeResources, passThrough), // TODO(floitsch): remove conditional directives flag. // We don't provide the info-message yet, since we haven't publicly diff --git a/pkg/compiler/lib/src/js/js.dart b/pkg/compiler/lib/src/js/js.dart index 2e5c6bdfa14..7d915e2355c 100644 --- a/pkg/compiler/lib/src/js/js.dart +++ b/pkg/compiler/lib/src/js/js.dart @@ -34,6 +34,8 @@ String prettyPrint(Node node, CodeBuffer createCodeBuffer(Node node, CompilerOptions compilerOptions, JavaScriptSourceInformationStrategy sourceInformationStrategy, {DumpInfoTask? monitor, + JavaScriptAnnotationMonitor annotationMonitor = + const JavaScriptAnnotationMonitor(), bool allowVariableMinification = true, List listeners = const []}) { JavaScriptPrintingOptions options = JavaScriptPrintingOptions( @@ -44,21 +46,32 @@ CodeBuffer createCodeBuffer(Node node, CompilerOptions compilerOptions, SourceInformationProcessor sourceInformationProcessor = sourceInformationStrategy.createProcessor( SourceMapperProviderImpl(outBuffer), const SourceInformationReader()); + Dart2JSJavaScriptPrintingContext context = Dart2JSJavaScriptPrintingContext( - monitor, outBuffer, sourceInformationProcessor); + monitor, outBuffer, sourceInformationProcessor, annotationMonitor); Printer printer = Printer(options, context); printer.visit(node); sourceInformationProcessor.process(node, outBuffer); return outBuffer; } +class JavaScriptAnnotationMonitor { + const JavaScriptAnnotationMonitor(); + + /// Called for each non-empty list of annotations in the JavaScript tree. + void onAnnotations(List annotations) { + // Should the position of the annotated node be recorded? + } +} + class Dart2JSJavaScriptPrintingContext implements JavaScriptPrintingContext { final DumpInfoTask? monitor; final CodeBuffer outBuffer; final CodePositionListener codePositionListener; + final JavaScriptAnnotationMonitor annotationMonitor; - Dart2JSJavaScriptPrintingContext( - this.monitor, this.outBuffer, this.codePositionListener); + Dart2JSJavaScriptPrintingContext(this.monitor, this.outBuffer, + this.codePositionListener, this.annotationMonitor); @override void error(String message) { @@ -83,6 +96,10 @@ class Dart2JSJavaScriptPrintingContext implements JavaScriptPrintingContext { monitor?.exitNode(node, startPosition, endPosition, closingPosition); codePositionListener.onPositions( node, startPosition, endPosition, closingPosition); + final annotations = node.annotations; + if (annotations.isNotEmpty) { + annotationMonitor.onAnnotations(annotations); + } } @override diff --git a/pkg/compiler/lib/src/js/rewrite_async.dart b/pkg/compiler/lib/src/js/rewrite_async.dart index 0ebc1a1e12f..5085aede389 100644 --- a/pkg/compiler/lib/src/js/rewrite_async.dart +++ b/pkg/compiler/lib/src/js/rewrite_async.dart @@ -931,8 +931,7 @@ abstract class AsyncRewriterBase extends js.NodeVisitor { bool storeTarget = node.arguments.any(shouldTransform); return withCallTargetExpression(node.target, (target) { return withExpressions(node.arguments, (List arguments) { - return js.Call(target, arguments) - .withSourceInformation(node.sourceInformation); + return js.Call(target, arguments).withInformationFrom(node); }); }, store: storeTarget); } diff --git a/pkg/compiler/lib/src/js_backend/annotations.dart b/pkg/compiler/lib/src/js_backend/annotations.dart index 9742c09e18c..035bc995298 100644 --- a/pkg/compiler/lib/src/js_backend/annotations.dart +++ b/pkg/compiler/lib/src/js_backend/annotations.dart @@ -137,6 +137,9 @@ class PragmaAnnotation { static const PragmaAnnotation loadLibraryPriorityHigh = PragmaAnnotation(21, 'load-priority:high'); + static const PragmaAnnotation resourceIdentifier = + PragmaAnnotation(22, 'resource-identifier'); + static const List values = [ noInline, tryInline, @@ -160,6 +163,7 @@ class PragmaAnnotation { lateCheck, loadLibraryPriorityNormal, loadLibraryPriorityHigh, + resourceIdentifier, ]; static const Map> implies = { @@ -168,7 +172,7 @@ class PragmaAnnotation { }; static const Map> excludes = { noInline: {tryInline}, - tryInline: {noInline}, + tryInline: {noInline, resourceIdentifier}, typesTrust: {typesCheck, parameterCheck, downcastCheck}, typesCheck: {typesTrust, parameterTrust, downcastTrust}, parameterTrust: {parameterCheck}, @@ -181,6 +185,7 @@ class PragmaAnnotation { lateCheck: {lateTrust}, loadLibraryPriorityNormal: {loadLibraryPriorityHigh}, loadLibraryPriorityHigh: {loadLibraryPriorityNormal}, + resourceIdentifier: {tryInline}, }; static const Map> requires = { noThrows: {noInline}, @@ -391,6 +396,9 @@ abstract class AnnotationsData { /// Indicates that the `fetchpriority` attribute should be set to the /// specified value on the injected script tag used to load the library. LoadLibraryPriority getLoadLibraryPriorityAt(ir.LoadLibrary node); + + /// Determines whether [member] is annotated as a resource identifier. + bool methodIsResourceIdentifier(FunctionEntity member); } class AnnotationsDataImpl implements AnnotationsData { @@ -635,6 +643,17 @@ class AnnotationsDataImpl implements AnnotationsData { } return null; } + + @override + bool methodIsResourceIdentifier(MemberEntity member) { + EnumSet? annotations = pragmaAnnotations[member]; + if (annotations != null) { + if (annotations.contains(PragmaAnnotation.resourceIdentifier)) { + return true; + } + } + return false; + } } class AnnotationsDataBuilder { diff --git a/pkg/compiler/lib/src/js_emitter/resource_info_emitter.dart b/pkg/compiler/lib/src/js_emitter/resource_info_emitter.dart new file mode 100644 index 00000000000..01e714450d2 --- /dev/null +++ b/pkg/compiler/lib/src/js_emitter/resource_info_emitter.dart @@ -0,0 +1,176 @@ +// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +/// Emitter for resource identifiers embedded in the program. +/// +/// See [documentation](../../../doc/resource_identifiers.md) for examples. +/// +/// .../foo_resource.dart: +/// +/// @pragma('dart2js:resource-identifer') +/// Resource getResource(String group, int index) { ... } +/// +/// .../my_resources.dart: +/// +/// ... +/// getResource('group1', 1); +/// ... +/// getResource('group2', 2); +/// ... +/// getResource('group1', 10); // optimized away. +/// ... +/// getResource('group1', 100); +/// ... +/// getResource('group1', 1000); // optimized away. +/// ... +/// getResource('group1', 1001); +/// +/// +/// Some of the calls above are tree-shaken. Some are placed in one 'part' file +/// and the others in a different 'part' file. The generated resources file +/// contains the constant arguments to the calls, arranged by resource identifer +/// and 'part' file. +/// +/// `main.js.resources.json`: +/// ```json +/// {... +/// "environment": { // Command-line environment +/// "foo": "bar", // -Dfoo=bar +/// } +/// "identifiers": [ +/// {"name": "getResource", +/// "uri": ".../foo_resource.dart", +/// "nonconstant": false, // No calls without a constant. +/// "files": [ +/// {"filename": "main.js_13.part.js", +/// "references": [ +/// {"1": "group1", "2": 1}, +/// {"1": "group1", "2": 1001}, +/// {"1": "group2", "2": 2} +/// ]} +/// {"filename": "main.js_282.part.js", +/// "references": [ +/// {"1": "group1", "2": 100}, +/// ]} +/// ]}, +/// -- next identifer +/// ] +/// } +/// ``` +/// +/// To appear in the output, arguments must be primitive constants i.e. int, +/// double, String, bool, null. Other constants (e.g. enums, const objects) will +/// simply be missing as though they were not constants. + +library js_emitter.resource_info_emitter; + +import 'dart:convert' show jsonDecode; +import 'dart:io' show Platform; + +import 'package:front_end/src/api_unstable/dart2js.dart' as fe; + +import '../js/js.dart' as js; +import '../universe/resource_identifier.dart' + show ResourceIdentifier, ResourceIdentifierLocation; + +class _AnnotationMonitor implements js.JavaScriptAnnotationMonitor { + final ResourceInfoCollector _collector; + final String _filename; + _AnnotationMonitor(this._collector, this._filename); + + @override + void onAnnotations(List annotations) { + for (Object annotation in annotations) { + if (annotation is ResourceIdentifier) { + _collector._register(_filename, annotation); + } + } + } +} + +class ResourceInfoCollector { + final Map<_ResourceIdentifierKey, _ResourceIdentifierInfo> _identifierMap = + {}; + + js.JavaScriptAnnotationMonitor monitorFor(String fileName) { + return _AnnotationMonitor(this, fileName); + } + + void _register(String filename, ResourceIdentifier identifier) { + final key = _ResourceIdentifierKey(identifier.name, identifier.uri); + final info = _identifierMap[key] ??= _ResourceIdentifierInfo(key); + if (identifier.nonconstant) info.nonconstant = true; + (info._files[filename] ??= []).add(identifier); + } + + Object finish(Map environment) { + Map json = { + '_comment': r'Resources referenced by annotated resource identifers', + 'AppTag': 'TBD', + 'environment': environment, + 'identifiers': _identifierMap.values.toList() + ..sort(_ResourceIdentifierInfo.compare) + }; + return json; + } +} + +class _ResourceIdentifierKey { + final String name; + final Uri uri; + + _ResourceIdentifierKey(this.name, this.uri); + + @override + bool operator ==(Object other) => + other is _ResourceIdentifierKey && name == other.name && uri == other.uri; + + @override + late final int hashCode = Object.hash(name, uri); +} + +class _ResourceIdentifierInfo { + final _ResourceIdentifierKey _key; + bool nonconstant = false; + final Map> _files = {}; + _ResourceIdentifierInfo(this._key); + + static int compare(_ResourceIdentifierInfo a, _ResourceIdentifierInfo b) { + int r = a._key.name.compareTo(b._key.name); + if (r != 0) return r; + return a._key.uri.toString().compareTo(b._key.uri.toString()); + } + + Map toJson() { + final files = _files.entries.toList() + ..sort((a, b) => a.key.compareTo(b.key)); + return { + "name": _key.name, + "uri": _key.uri.toString(), + "nonconstant": nonconstant, + "files": [ + for (final entry in files) + { + "filename": entry.key, + "references": [ + for (final resourceIdentifier in entry.value) + { + if (resourceIdentifier.location != null) + '@': _locationToJson(resourceIdentifier.location!), + ...jsonDecode(resourceIdentifier.arguments) + } + ] + } + ] + }; + } + + Map _locationToJson(ResourceIdentifierLocation location) { + return { + 'uri': fe.relativizeUri(Uri.base, location.uri, Platform.isWindows), + if (location.line != null) 'line': location.line, + if (location.column != null) 'column': location.column, + }; + } +} diff --git a/pkg/compiler/lib/src/js_emitter/startup_emitter/model_emitter.dart b/pkg/compiler/lib/src/js_emitter/startup_emitter/model_emitter.dart index 5a19539bd09..f84f7b083ad 100644 --- a/pkg/compiler/lib/src/js_emitter/startup_emitter/model_emitter.dart +++ b/pkg/compiler/lib/src/js_emitter/startup_emitter/model_emitter.dart @@ -89,6 +89,7 @@ import '../js_emitter.dart'; import '../constant_ordering.dart' show ConstantOrdering; import '../headers.dart'; import '../model.dart'; +import '../resource_info_emitter.dart' show ResourceInfoCollector; import 'fragment_merger.dart'; part 'fragment_emitter.dart'; @@ -105,6 +106,7 @@ class ModelEmitter { final DiagnosticReporter _reporter; final api.CompilerOutput _outputProvider; final DumpInfoTask _dumpInfoTask; + final ResourceInfoCollector _resourceInfoCollector = ResourceInfoCollector(); final Namer _namer; final CompilerTask _task; final Emitter _emitter; @@ -353,6 +355,10 @@ class ModelEmitter { writeDeferredMap(); } + if (_options.writeResources) { + writeResourceIdentifiers(); + } + // Return the total program size. return emittedOutputBuffers.values.fold(0, (a, b) => a + b.length); } @@ -434,7 +440,9 @@ var ${startupMetricsGlobal} = CodeBuffer buffer = js.createCodeBuffer(program, _options, _sourceInformationStrategy as JavaScriptSourceInformationStrategy, - monitor: _dumpInfoTask); + monitor: _dumpInfoTask, + annotationMonitor: _resourceInfoCollector + .monitorFor(_options.outputUri?.pathSegments.last ?? 'out')); _task.measureSubtask('emit buffers', () { mainOutput.addBuffer(buffer); }); @@ -495,7 +503,7 @@ var ${startupMetricsGlobal} = outputFileName, deferredExtension, api.OutputType.jsPart), outputListeners); - writeCodeFragments(fragmentCode, fragmentHashes, output); + writeCodeFragments(fragmentCode, fragmentHashes, output, outputFileName); if (_shouldGenerateSourceMap) { _task.measureSubtask('source-maps', () { @@ -529,8 +537,11 @@ var ${startupMetricsGlobal} = } /// Writes a list of [CodeFragments] to [CodeOutput]. - void writeCodeFragments(List fragmentCode, - Map fragmentHashes, CodeOutput output) { + void writeCodeFragments( + List fragmentCode, + Map fragmentHashes, + CodeOutput output, + String outputFileName) { bool isFirst = true; for (var emittedCodeFragment in fragmentCode) { var codeFragment = emittedCodeFragment.codeFragment; @@ -538,7 +549,8 @@ var ${startupMetricsGlobal} = for (var outputUnit in codeFragment.outputUnits) { emittedOutputBuffers[outputUnit] = output; } - fragmentHashes[codeFragment] = writeCodeFragment(output, code, isFirst); + fragmentHashes[codeFragment] = + writeCodeFragment(output, code, isFirst, outputFileName); isFirst = false; } } @@ -548,8 +560,8 @@ var ${startupMetricsGlobal} = // Returns the deferred fragment's hash. // // Updates the shared [outputBuffers] field with the output. - String writeCodeFragment( - CodeOutput output, js.Expression code, bool isFirst) { + String writeCodeFragment(CodeOutput output, js.Expression code, bool isFirst, + String outputFileName) { // The [code] contains the function that must be invoked when the deferred // hunk is loaded. // That function must be in a map from its hashcode to the function. Since @@ -568,7 +580,9 @@ var ${startupMetricsGlobal} = Hasher hasher = Hasher(); CodeBuffer buffer = js.createCodeBuffer(program, _options, _sourceInformationStrategy as JavaScriptSourceInformationStrategy, - monitor: _dumpInfoTask, listeners: [hasher]); + monitor: _dumpInfoTask, + listeners: [hasher], + annotationMonitor: _resourceInfoCollector.monitorFor(outputFileName)); _task.measureSubtask('emit buffers', () { output.addBuffer(buffer); }); @@ -600,4 +614,13 @@ var ${startupMetricsGlobal} = ..add(const JsonEncoder.withIndent(" ").convert(mapping)) ..close(); } + + /// Writes out all the referenced resource identifiers as a JSON file. + void writeResourceIdentifiers() { + _outputProvider.createOutputSink( + '', 'resources.json', api.OutputType.resourceIdentifiers) + ..add(JsonEncoder.withIndent(' ') + .convert(_resourceInfoCollector.finish(_options.environment))) + ..close(); + } } diff --git a/pkg/compiler/lib/src/options.dart b/pkg/compiler/lib/src/options.dart index ff50a754d00..ed00750d6c7 100644 --- a/pkg/compiler/lib/src/options.dart +++ b/pkg/compiler/lib/src/options.dart @@ -391,6 +391,10 @@ class CompilerOptions implements DiagnosticOptions { /// this RegExp pattern. String? dumpSsaPattern = null; + /// Whether to generate a `.resources.json` file detailing the use of resource + /// identifiers. + bool writeResources = false; + /// Whether we allow passing an extra argument to `assert`, containing a /// reason for why an assertion fails. (experimental) /// @@ -654,6 +658,7 @@ class CompilerOptions implements DiagnosticOptions { _hasOption(options, "${Flags.dumpInfo}=binary") ..dumpSsaPattern = _extractStringOption(options, '${Flags.dumpSsa}=', null) + ..writeResources = _hasOption(options, Flags.writeResources) ..enableMinification = _hasOption(options, Flags.minify) .._disableMinification = _hasOption(options, Flags.noMinify) ..omitLateNames = _hasOption(options, Flags.omitLateNames) diff --git a/pkg/compiler/lib/src/source_file_provider.dart b/pkg/compiler/lib/src/source_file_provider.dart index cf86d48510d..e016948f0b6 100644 --- a/pkg/compiler/lib/src/source_file_provider.dart +++ b/pkg/compiler/lib/src/source_file_provider.dart @@ -342,6 +342,7 @@ class RandomAccessFileOutputProvider implements api.CompilerOutput { case api.OutputType.dumpInfo: case api.OutputType.dumpUnusedLibraries: case api.OutputType.deferredMap: + case api.OutputType.resourceIdentifiers: if (name == '') { name = out.pathSegments.last; } diff --git a/pkg/compiler/lib/src/ssa/builder.dart b/pkg/compiler/lib/src/ssa/builder.dart index 133a7ecde82..1aea1632a45 100644 --- a/pkg/compiler/lib/src/ssa/builder.dart +++ b/pkg/compiler/lib/src/ssa/builder.dart @@ -1846,12 +1846,14 @@ class KernelSsaGraphBuilder extends ir.Visitor with ir.VisitorVoidMixin { String loadId = closedWorld.outputUnitData.getImportDeferName( _elementMap.getSpannable(targetElement, loadLibrary), _elementMap.getImport(loadLibrary.import)); - // TODO(efortuna): Source information! final priority = closedWorld.annotationsData.getLoadLibraryPriorityAt(loadLibrary); final flag = priority.index; + final sourceInformation = + _sourceInformationBuilder.buildCall(loadLibrary, loadLibrary); + push(HInvokeStatic( _commonElements.loadDeferredLibrary, [ @@ -1860,7 +1862,8 @@ class KernelSsaGraphBuilder extends ir.Visitor with ir.VisitorVoidMixin { ], _abstractValueDomain.nonNullType, const [], - targetCanThrow: false)); + targetCanThrow: false) + ..sourceInformation = sourceInformation); } @override diff --git a/pkg/compiler/lib/src/ssa/codegen.dart b/pkg/compiler/lib/src/ssa/codegen.dart index 9a089cf0ad8..74dbb76c992 100644 --- a/pkg/compiler/lib/src/ssa/codegen.dart +++ b/pkg/compiler/lib/src/ssa/codegen.dart @@ -39,6 +39,7 @@ import '../native/behavior.dart'; import '../options.dart'; import '../tracer.dart' show Tracer; import '../universe/call_structure.dart' show CallStructure; +import '../universe/resource_identifier.dart'; import '../universe/selector.dart' show Selector; import '../universe/use.dart' show ConstantUse, DynamicUse, StaticUse, TypeUse; import 'codegen_helpers.dart'; @@ -2101,6 +2102,7 @@ class SsaCodeGenerator implements HVisitor, HBlockInformationVisitor { node.sourceInformation)); } else { StaticUse staticUse; + Object? resourceIdentifierAnnotation; if (element is ConstructorEntity) { CallStructure callStructure = CallStructure.unnamed(arguments.length, node.typeArguments.length); @@ -2115,11 +2117,59 @@ class SsaCodeGenerator implements HVisitor, HBlockInformationVisitor { CallStructure.unnamed(arguments.length, node.typeArguments.length); staticUse = StaticUse.staticInvoke(element, callStructure, node.typeArguments); + if (_closedWorld.annotationsData.methodIsResourceIdentifier(element)) { + resourceIdentifierAnnotation = _methodResourceIdentifier( + element, callStructure, node.inputs, node.sourceInformation); + } } _registry.registerStaticUse(staticUse); push(_emitter.staticFunctionAccess(element)); push( js.Call(pop(), arguments, sourceInformation: node.sourceInformation)); + if (resourceIdentifierAnnotation != null) { + push(pop().withAnnotation(resourceIdentifierAnnotation)); + } + } + } + + ResourceIdentifier _methodResourceIdentifier( + FunctionEntity element, + CallStructure callStructure, + List arguments, + SourceInformation? sourceInformation) { + ConstantValue? findConstant(HInstruction node) { + while (node is HLateValue) node = node.target; + return node is HConstant ? node.constant : null; + } + + final definition = _closedWorld.elementMap.getMemberDefinition(element); + final uri = definition.location.uri; + + final builder = ResourceIdentifierBuilder(element.name!, uri); + + if (sourceInformation != null) { + _addSourceInformationToResourceIdentiferBuilder( + builder, sourceInformation); + } + for (int i = 0; i < arguments.length; i++) { + builder.add('${i + 1}', findConstant(arguments[i])); + } + + return builder.finish(); + } + + void _addSourceInformationToResourceIdentiferBuilder( + ResourceIdentifierBuilder builder, SourceInformation sourceInformation) { + SourceLocation? location = sourceInformation.startPosition ?? + sourceInformation.innerPosition ?? + sourceInformation.endPosition; + if (location != null) { + final sourceUri = location.sourceUri; + if (sourceUri != null) { + // Is [sourceUri] normalized in some way or does that need to be done + // here? + builder.addLocation(sourceUri, location.line, location.column); + } } } diff --git a/pkg/compiler/lib/src/universe/resource_identifier.dart b/pkg/compiler/lib/src/universe/resource_identifier.dart new file mode 100644 index 00000000000..e13057be0ed --- /dev/null +++ b/pkg/compiler/lib/src/universe/resource_identifier.dart @@ -0,0 +1,162 @@ +// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:convert' show jsonEncode; + +import '../constants/values.dart'; +import '../serialization/serialization.dart'; + +class ResourceIdentifier { + static const String tag = 'resource-identifier'; + + /// Name of the class or method that is a resource identifier. + final String name; + + /// When the class or method is defined. + final Uri uri; + + /// Location of the resource identifer instance. This is `null` for constant + /// resource identifiers. For other resource identifer instances this is the + /// call site to the constructor or method. + final ResourceIdentifierLocation? location; + + /// True if some argument is missing from [arguments] because it is not a + /// constant. + final bool nonconstant; + + /// JSON encoded map from class field names or function parameter positions to + /// primitive values for arguments that are constant. + // TODO(sra): Consider holding as a map with ConstantValue values. + final String arguments; + + ResourceIdentifier( + this.name, this.uri, this.location, this.nonconstant, this.arguments); + + factory ResourceIdentifier.readFromDataSource(DataSourceReader source) { + source.begin(tag); + String name = source.readString(); + Uri uri = source.readUri(); + + bool hasLocation = source.readBool(); + ResourceIdentifierLocation? location = hasLocation + ? ResourceIdentifierLocation.readFromDataSource(source) + : null; + + bool nonconstant = source.readBool(); + String arguments = source.readString(); + source.end(tag); + return ResourceIdentifier(name, uri, location, nonconstant, arguments); + } + + void writeToDataSink(DataSinkWriter sink) { + sink.begin(tag); + sink.writeString(name); + sink.writeUri(uri); + + if (location == null) { + sink.writeBool(false); + } else { + sink.writeBool(true); + location!.writeToDataSink(sink); + } + + sink.writeBool(nonconstant); + sink.writeString(arguments); + sink.end(tag); + } + + @override + bool operator ==(Object other) => + other is ResourceIdentifier && + name == other.name && + uri == other.uri && + location == other.location && + arguments == other.arguments; + + @override + int get hashCode => Object.hash(name, uri, location, arguments); + + @override + String toString() { + return 'ResourceIdentifier($name @ $uri, $location, $arguments)'; + } +} + +class ResourceIdentifierLocation { + final Uri uri; + final int? line; + final int? column; + ResourceIdentifierLocation._(this.uri, this.line, this.column); + + factory ResourceIdentifierLocation.readFromDataSource( + DataSourceReader source) { + final uri = source.readUri(); + final line = source.readIntOrNull(); + final column = source.readIntOrNull(); + return ResourceIdentifierLocation._(uri, line, column); + } + + void writeToDataSink(DataSinkWriter sink) { + sink.writeUri(uri); + sink.writeIntOrNull(line); + sink.writeIntOrNull(column); + } + + @override + bool operator ==(Object other) => + other is ResourceIdentifierLocation && + uri == other.uri && + line == other.line && + column == other.column; + + @override + late int hashCode = Object.hash(uri, line, column); + + @override + String toString() => 'ResourceIdentifierLocation($uri:$line:$column)'; +} + +class ResourceIdentifierBuilder { + final String name; + final Uri uri; + bool _nonconstant = false; + ResourceIdentifierLocation? _location; + final Map _arguments = {}; + + ResourceIdentifierBuilder(this.name, this.uri); + + ResourceIdentifier finish() { + return ResourceIdentifier( + name, uri, _location, _nonconstant, jsonEncode(_arguments)); + } + + void add(String argumentName, ConstantValue? constant) { + if (constant != null) { + final value = _findValue(constant); + if (!identical(value, _unknown)) { + _arguments[argumentName] = value; + return; + } + } + _nonconstant = true; + } + + void addLocation(Uri uri, int? line, int? column) { + _location = ResourceIdentifierLocation._(uri, line, column); + } + + Object? _findValue(ConstantValue constant) { + if (constant is IntConstantValue) { + final value = constant.intValue; + return value.isValidInt ? value.toInt() : _unknown; + } + if (constant is StringConstantValue) return constant.stringValue; + if (constant is BoolConstantValue) return constant.boolValue; + if (constant is DoubleConstantValue) return constant.doubleValue; + if (constant is NullConstantValue) return null; + return _unknown; + } + + static final Object _unknown = Object(); +} diff --git a/pkg/compiler/test/end_to_end/output_type_test.dart b/pkg/compiler/test/end_to_end/output_type_test.dart index 1b80a7ba75b..a9859f18a16 100644 --- a/pkg/compiler/test/end_to_end/output_type_test.dart +++ b/pkg/compiler/test/end_to_end/output_type_test.dart @@ -115,6 +115,19 @@ main() { '--csp', ...additionOptionals, ], expectedOutput); + + // If we add the '--write-resources' flag, we get another file + // `out.js.resources.json'. + await test([ + 'pkg/compiler/test/deferred/data/deferred_helper.dart', + '--no-sound-null-safety', + '--csp', + Flags.writeResources, + ...additionOptionals, + ], [ + ...expectedOutput, + 'out.js.resources.json', + ]); } asyncTest(() async { diff --git a/pkg/js_ast/lib/src/nodes.dart b/pkg/js_ast/lib/src/nodes.dart index b55eb71ecd5..d2db44f7222 100644 --- a/pkg/js_ast/lib/src/nodes.dart +++ b/pkg/js_ast/lib/src/nodes.dart @@ -641,6 +641,18 @@ abstract class Node { sourceInformation, List.unmodifiable([...annotations, newAnnotation])); } + /// Returns a node equivalent to [this] but with the same source information + /// and annotations as [node]. + Node withInformationFrom(Node node) { + return _hasSameInformationAs(node) + ? this + : (_clone().._sourceInformation = node._sourceInformation); + } + + bool _hasSameInformationAs(Node node) { + return node._sourceInformation == _sourceInformation; + } + bool get isCommaOperator => false; Statement toStatement() { @@ -714,6 +726,14 @@ abstract class Statement extends Node { _replacementSourceInformation(newSourceInformation); } + // Override for refined return type. + @override + Statement withInformationFrom(Node node) { + return _hasSameInformationAs(node) + ? this + : (_clone().._sourceInformation = node._sourceInformation); + } + @override Statement toStatement() => this; } @@ -1379,6 +1399,20 @@ abstract class Expression extends Node { _replacementSourceInformation(newSourceInformation); } + // Override for refined return type. + @override + Expression withAnnotation(Object newAnnotation) { + return _clone().._sourceInformation = _appendedAnnotation(newAnnotation); + } + + // Override for refined return type. + @override + Expression withInformationFrom(Node node) { + return _hasSameInformationAs(node) + ? this + : (_clone().._sourceInformation = node._sourceInformation); + } + @override Statement toStatement() => ExpressionStatement(this); } diff --git a/sdk/lib/_internal/js_runtime/lib/js_helper.dart b/sdk/lib/_internal/js_runtime/lib/js_helper.dart index 75d98b01d1c..35cbf04fcae 100644 --- a/sdk/lib/_internal/js_runtime/lib/js_helper.dart +++ b/sdk/lib/_internal/js_runtime/lib/js_helper.dart @@ -2706,6 +2706,7 @@ DeferredLoadCallback? deferredLoadHook; /// /// - `0` for `LoadLibraryPriority.normal` /// - `1` for `LoadLibraryPriority.high` +@pragma('dart2js:resource-identifier') Future loadDeferredLibrary(String loadId, int priority) { // Validate the priority using the index to allow the actual enum to get // tree-shaken.