From b48584d3d0eb480943d72958104baf8e0e366074 Mon Sep 17 00:00:00 2001 From: Sigmund Cherem Date: Mon, 7 Aug 2017 08:41:25 -0700 Subject: [PATCH] Switch FE to use the libraries.json format. This CL: * introduces the Dart API to operate over libraries specifications and describes the format we intend to use (see libraries_spec.dart) * implements serialization/deserialization for this API * switches over the front_end to use these APIs * public options accept a URI to the JSON file and no longer accept a `dartLibraries` map * internal code uses the LibrariesSpecification API * switches other dependencies on these APIs (resynthesizer_test and patch_sdk.dart) This is the first step in migrating over to use the libraries.json format and eventually remove the patched_sdk step. In particular, some of the next steps include: * add a build step to generate .json files from .yaml files * add a libraries.yaml file for the sdk * split the patched_sdk step in two: * patching files * generating .dill files * add any missing support for patch-files in fasta * finally remove the patching files step, and only have a build step for generating .dill files BUG= R=ahe@google.com, paulberry@google.com, scheglov@google.com Committed: https://github.com/dart-lang/sdk/commit/abf2d23af2315fae6dc688741147ef677ec34835 Review-Url: https://codereview.chromium.org/2986303003 . --- .../src/summary/resynthesize_kernel_test.dart | 12 +- pkg/compiler/lib/src/library_loader.dart | 1 - .../compiler_with_invalidation.dart | 14 - pkg/front_end/lib/compiler_options.dart | 19 +- .../lib/incremental_kernel_generator.dart | 11 +- .../lib/src/base/libraries_specification.dart | 221 +++++++++++++ .../lib/src/base/processed_options.dart | 115 +++++-- .../lib/src/fasta/fasta_codes_generated.dart | 51 +++ .../lib/src/fasta/uri_translator_impl.dart | 80 +---- .../lib/src/testing/compiler_common.dart | 26 +- pkg/front_end/messages.yaml | 7 + pkg/front_end/test/fasta/testing/suite.dart | 12 +- .../test/fasta/uri_translator_test.dart | 20 +- .../incremental_kernel_generator_test.dart | 6 +- pkg/front_end/test/kernel_generator_test.dart | 9 +- .../base/libraries_specification_test.dart | 309 ++++++++++++++++++ .../test/src/base/processed_options_test.dart | 59 +++- .../test/src/incremental/file_state_test.dart | 6 +- .../src/incremental/hot_reload_e2e_test.dart | 12 +- .../src/incremental/kernel_driver_test.dart | 5 +- .../test/src/incremental/mock_sdk.dart | 22 +- .../test/subpackage_relationships_test.dart | 1 + pkg/front_end/tool/fasta_perf.dart | 10 +- tools/patch_sdk.dart | 54 ++- 24 files changed, 871 insertions(+), 211 deletions(-) create mode 100644 pkg/front_end/lib/src/base/libraries_specification.dart create mode 100644 pkg/front_end/test/src/base/libraries_specification_test.dart diff --git a/pkg/analyzer/test/src/summary/resynthesize_kernel_test.dart b/pkg/analyzer/test/src/summary/resynthesize_kernel_test.dart index cb43e6d3e37..e7ec0cebf27 100644 --- a/pkg/analyzer/test/src/summary/resynthesize_kernel_test.dart +++ b/pkg/analyzer/test/src/summary/resynthesize_kernel_test.dart @@ -15,6 +15,7 @@ import 'package:analyzer/src/summary/resynthesize.dart'; import 'package:front_end/compiler_options.dart'; import 'package:front_end/file_system.dart'; import 'package:front_end/src/base/performace_logger.dart'; +import 'package:front_end/src/base/libraries_specification.dart'; import 'package:front_end/src/base/processed_options.dart'; import 'package:front_end/src/fasta/uri_translator_impl.dart'; import 'package:front_end/src/incremental/byte_store.dart'; @@ -74,13 +75,16 @@ class ResynthesizeKernelStrongTest extends ResynthesizeTest { Uri testUri = testFile.toUri(); String testUriStr = testUri.toString(); - Map dartLibraries = {}; + Map dartLibraries = {}; MockSdk.FULL_URI_MAP.forEach((dartUri, path) { - dartLibraries[Uri.parse(dartUri).path] = Uri.parse('file://$path'); + var name = Uri.parse(dartUri).path; + dartLibraries[name] = + new LibraryInfo(name, Uri.parse('file://$path'), const []); }); - var uriTranslator = - new UriTranslatorImpl(dartLibraries, {}, Packages.noPackages); + var uriTranslator = new UriTranslatorImpl( + new TargetLibrariesSpecification('none', dartLibraries), + Packages.noPackages); var options = new ProcessedOptions(new CompilerOptions() ..target = new NoneTarget(new TargetFlags(strongMode: isStrongMode)) ..reportMessages = false diff --git a/pkg/compiler/lib/src/library_loader.dart b/pkg/compiler/lib/src/library_loader.dart index 6bea483442e..a76dcc07100 100644 --- a/pkg/compiler/lib/src/library_loader.dart +++ b/pkg/compiler/lib/src/library_loader.dart @@ -858,7 +858,6 @@ class KernelLibraryLoaderTask extends CompilerTask var options = new fe.CompilerOptions() ..fileSystem = new CompilerFileSystem(compilerInput) ..target = new Dart2jsTarget(new TargetFlags()) - ..compileSdk = true ..linkedDependencies = [ sdkRoot.resolve('_internal/dart2js_platform.dill') ] diff --git a/pkg/front_end/example/incremental_reload/compiler_with_invalidation.dart b/pkg/front_end/example/incremental_reload/compiler_with_invalidation.dart index c5d97602703..061771c9e54 100644 --- a/pkg/front_end/example/incremental_reload/compiler_with_invalidation.dart +++ b/pkg/front_end/example/incremental_reload/compiler_with_invalidation.dart @@ -9,7 +9,6 @@ library front_end.example.incremental_reload.compiler_with_invalidation; import 'dart:io'; import 'dart:async'; -import 'dart:convert' show JSON; import 'package:front_end/compiler_options.dart'; import 'package:front_end/incremental_kernel_generator.dart'; @@ -33,7 +32,6 @@ Future createIncrementalCompiler(String entry, ..sdkRoot = sdkRoot ..packagesFileUri = Uri.base.resolve('.packages') ..strongMode = false - ..dartLibraries = loadDartLibraries(sdkRoot) // Note: we do not report error on the console because the incremental // compiler is an ongoing background service that shouldn't polute stdout. // TODO(sigmund): do something with the errors. @@ -43,18 +41,6 @@ Future createIncrementalCompiler(String entry, return IncrementalCompiler.create(options, entryUri); } -/// Reads the `libraries.json` file for an SDK to provide the location of the -/// SDK files. -// TODO(sigmund): this should be handled by package:front_end internally. -Map loadDartLibraries(Uri sdkRoot) { - var libraries = sdkRoot.resolve('lib/libraries.json'); - var map = - JSON.decode(new File.fromUri(libraries).readAsStringSync())['libraries']; - var dartLibraries = {}; - map.forEach((k, v) => dartLibraries[k] = libraries.resolve(v)); - return dartLibraries; -} - /// An incremental compiler that monitors file modifications on disk and /// invalidates only files that have been modified since the previous time the /// compiler was invoked. diff --git a/pkg/front_end/lib/compiler_options.dart b/pkg/front_end/lib/compiler_options.dart index d20aa834fb8..28993863c5f 100644 --- a/pkg/front_end/lib/compiler_options.dart +++ b/pkg/front_end/lib/compiler_options.dart @@ -25,18 +25,17 @@ class CompilerOptions { /// [Platform.resolvedExecutable] as a starting point. Uri sdkRoot; - /// Map of `dart:*` libraries to URIs in the [fileSystem]. + /// Uri to a platform libraries specification file. /// - /// Keys in the map are the name of the library with no `dart:` prefix, for - /// example: + /// A libraries specification file is a JSON file that describes how to map + /// `dart:*` libraries to URIs in the underlying [fileSystem]. See + /// `package:front_end/src/base/libraries_specification.dart` for details on + /// the format. /// - /// {'core': 'file:///sdk/lib/core/core.dart'} - /// - /// If `null`, the default set of libraries will be loaded from - /// `sdkRoot/lib/libraries.json`. - // TODO(sigmund): also provide an option to specify the .json file, then - // consider dropping this option. - Map dartLibraries; + /// If a value is not specified and `compileSdk = true`, the compiler will + /// infer at a default location under [sdkRoot], typically under + /// `lib/libraries.json`. + Uri librariesSpecificationUri; /// Callback to which compilation errors should be delivered. /// diff --git a/pkg/front_end/lib/incremental_kernel_generator.dart b/pkg/front_end/lib/incremental_kernel_generator.dart index 9d52c218845..93e88ee60ed 100644 --- a/pkg/front_end/lib/incremental_kernel_generator.dart +++ b/pkg/front_end/lib/incremental_kernel_generator.dart @@ -6,6 +6,7 @@ import 'dart:async'; import 'package:front_end/src/base/processed_options.dart'; import 'package:front_end/src/incremental_kernel_generator_impl.dart'; +import 'package:front_end/src/fasta/compiler_context.dart'; import 'package:kernel/kernel.dart'; import 'compiler_options.dart'; @@ -100,9 +101,11 @@ abstract class IncrementalKernelGenerator { CompilerOptions options, Uri entryPoint, {WatchUsedFilesFn watch}) async { var processedOptions = new ProcessedOptions(options, false, [entryPoint]); - var uriTranslator = await processedOptions.getUriTranslator(); - return new IncrementalKernelGeneratorImpl( - processedOptions, uriTranslator, entryPoint, - watch: watch); + return await CompilerContext.runWithOptions(processedOptions, (_) async { + var uriTranslator = await processedOptions.getUriTranslator(); + return new IncrementalKernelGeneratorImpl( + processedOptions, uriTranslator, entryPoint, + watch: watch); + }); } } diff --git a/pkg/front_end/lib/src/base/libraries_specification.dart b/pkg/front_end/lib/src/base/libraries_specification.dart new file mode 100644 index 00000000000..f21c8a2579a --- /dev/null +++ b/pkg/front_end/lib/src/base/libraries_specification.dart @@ -0,0 +1,221 @@ +// Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +/// Library specification in-memory representation. +/// +/// Many dart tools are configurable to support different target platforms. For +/// a given target, they need to know what libraries are available and where are +/// the sources and target-specific patches. +/// +/// Here we define APIs to represent this specification and implement +/// serialization to (and deserialization from) a JSON file. +/// +/// Here is an example specification JSON file: +/// +/// { +/// "vm": { +/// "libraries": { +/// "core": { +/// "uri": "async/core.dart", +/// "patches": [ +/// "path/to/core_patch.dart", +/// "path/to/list_patch.dart" +/// ] +/// } +/// "async": { +/// "uri": "async/async.dart", +/// "patches": "path/to/async_patch.dart" +/// } +/// "convert": { +/// "uri": "convert/convert.dart", +/// } +/// } +/// } +/// } +/// +/// The format contains: +/// - a top level entry for each target. Keys are target names (e.g. "vm" +/// above), and values contain the entire specification of a target. +/// +/// - each target specification is a map. Today only one key ("libraries") is +/// supported, but this may be extended in the future to add more +/// information on each target. +/// +/// - The "libraries" entry contains details for how each platform library is +/// implemented. The entry is a map, where keys are the name of the platform +/// library and values contain details for where to find the implementation +/// fo that library. +/// +/// - The name of the library is a single token (e.g. "core") that matches the +/// Uri path used after `dart:` (e.g. "dart:core"). +/// +/// - The "uri" entry on the library information is mandatory. The value is a +/// string URI reference. The "patches" entry is optional and may have as a +/// value a string URI reference or a list of URI references. +/// +/// All URI references can either be a file URI or a relative URI path, +/// which will be resolved relative to the location of the library +/// specification file. +/// +/// +/// Note: we currently have several different files that need to be updated +/// when changing libraries, sources, and patch files: +/// * .platform files (for dart2js) +/// * .gypi files (for vm) +/// * sdk_library_metadata/lib/libraries.dart (for analyzer, ddc) +/// +/// we are in the process of unifying them all under this format (see +/// https://github.com/dart-lang/sdk/issues/28836), but for now we need to pay +/// close attention to change them consistently. + +// TODO(sigmund): move this file to a shared package. +import 'dart:convert' show JSON; + +import '../fasta/util/relativize.dart'; + +/// Contents from a single library specification file. +/// +/// Contains information about all libraries on all target platforms defined in +/// that file. +class LibrariesSpecification { + final Map _targets; + + const LibrariesSpecification( + [this._targets = const {}]); + + /// The library specification for a given [target], or null if none is + /// available. + TargetLibrariesSpecification specificationFor(String target) => + _targets[target]; + + /// Parse the given [json] as a library specification, resolving any relative + /// paths from [baseUri]. + /// + /// May throw an exception if [json] is not properly formatted or contains + /// invalid values. + static LibrariesSpecification parse(Uri baseUri, String json) { + if (json == null) return const LibrariesSpecification(); + var jsonData; + try { + var data = JSON.decode(json); + if (data is! Map) { + return _reportError('top-level specification is not a map'); + } + jsonData = data as Map; + } on FormatException catch (e) { + throw new LibrariesSpecificationException(e); + } + var targets = {}; + jsonData.forEach((String targetName, targetData) { + Map libraries = {}; + if (targetData is! Map) { + return _reportError( + "target specification for '$targetName' is not a map"); + } + if (!targetData.containsKey("libraries")) { + return _reportError("target specification " + "for '$targetName' doesn't have a libraries entry"); + } + var librariesData = targetData["libraries"]; + if (librariesData is! Map) { + return _reportError("libraries entry for '$targetName' is not a map"); + } + librariesData.forEach((String name, data) { + if (data is! Map) { + return _reportError( + "library data for '$name' in target '$targetName' is not a map"); + } + Uri checkAndResolve(uriString) { + if (uriString is! String) { + return _reportError("uri value '$uriString' is not a string" + "(from library '$name' in target '$targetName')"); + } + var uri = Uri.parse(uriString); + if (uri.scheme != '' && uri.scheme != 'file') { + return _reportError("uri scheme in '$uriString' is not supported."); + } + return baseUri.resolveUri(uri); + } + + var uri = checkAndResolve(data['uri']); + var patches; + if (data['patches'] is List) { + patches = data['patches'].map(baseUri.resolve).toList(); + } else if (data['patches'] is String) { + patches = [checkAndResolve(data['patches'])]; + } else if (data['patches'] == null) { + patches = const []; + } else { + return _reportError( + "patches entry for '$name' is not a list or a string"); + } + libraries[name] = new LibraryInfo(name, uri, patches); + }); + targets[targetName] = + new TargetLibrariesSpecification(targetName, libraries); + }); + return new LibrariesSpecification(targets); + } + + static _reportError(String error) => + throw new LibrariesSpecificationException(error); + + /// Serialize this specification to json. + /// + /// If possible serializes paths relative to [outputUri]. + String toJsonString(Uri outputUri) => JSON.encode(toJsonMap(outputUri)); + + Map toJsonMap(Uri outputUri) { + var result = {}; + var dir = outputUri.resolve('.'); + String pathFor(Uri uri) => relativizeUri(uri, base: dir); + _targets.forEach((targetName, target) { + var libraries = {}; + target._libraries.forEach((name, lib) { + libraries[name] = { + 'uri': pathFor(lib.uri), + 'patches': lib.patches.map(pathFor).toList(), + }; + }); + result[targetName] = {'libraries': libraries}; + }); + return result; + } +} + +/// Specifies information about all libraries supported by a given target. +class TargetLibrariesSpecification { + /// Name of the target platform. + final String targetName; + + final Map _libraries; + + const TargetLibrariesSpecification(this.targetName, + [this._libraries = const {}]); + + /// Details about a library whose import is `dart:$name`. + LibraryInfo libraryInfoFor(String name) => _libraries[name]; +} + +/// Information about a `dart:` library in a specific target platform. +class LibraryInfo { + /// The name of the library, which is the path developers use to import this + /// library (as `dart:$name`). + final String name; + + /// The file defining the main implementation of the library. + final Uri uri; + + /// Patch files used for this library in the target platform, if any. + final List patches; + + const LibraryInfo(this.name, this.uri, this.patches); +} + +class LibrariesSpecificationException { + Object error; + LibrariesSpecificationException(this.error); + + String toString() => '$error'; +} diff --git a/pkg/front_end/lib/src/base/processed_options.dart b/pkg/front_end/lib/src/base/processed_options.dart index ea1d6ac526b..5ba11fafdd2 100644 --- a/pkg/front_end/lib/src/base/processed_options.dart +++ b/pkg/front_end/lib/src/base/processed_options.dart @@ -28,6 +28,8 @@ import 'package:source_span/source_span.dart' show SourceSpan, SourceLocation; import 'package:front_end/src/fasta/command_line_reporting.dart' as command_line_reporting; +import 'libraries_specification.dart'; + /// All options needed for the front end implementation. /// /// This includes: all of [CompilerOptions] in a form useful to the @@ -76,10 +78,22 @@ class ProcessedOptions { /// The location of the SDK, or `null` if the location hasn't been determined /// yet. Uri _sdkRoot; - Uri get sdkRoot => _sdkRoot ??= _normalizeSdkRoot(); + Uri get sdkRoot { + _ensureSdkDefaults(); + return _sdkRoot; + } Uri _sdkSummary; - Uri get sdkSummary => _sdkSummary ??= _computeSdkSummaryUri(); + Uri get sdkSummary { + _ensureSdkDefaults(); + return _sdkSummary; + } + + Uri _librariesSpecificationUri; + Uri get librariesSpecificationUri { + _ensureSdkDefaults(); + return _librariesSpecificationUri; + } Ticker ticker; @@ -274,20 +288,45 @@ class ProcessedOptions { /// required to locate/read the packages file as well as SDK metadata. Future getUriTranslator() async { if (_uriTranslator == null) { - await _getPackages(); - // TODO(scheglov) Load SDK libraries from whatever format we decide. - // TODO(scheglov) Remove the field "_raw.dartLibraries". - var libraries = _raw.dartLibraries ?? await _parseDartLibraries(); - _uriTranslator = new UriTranslatorImpl( - libraries, const >{}, _packages); + ticker.logMs("Started building UriTranslator"); + var libraries = await _computeLibrarySpecification(); + ticker.logMs("Read libraries file"); + var packages = await _getPackages(); ticker.logMs("Read packages file"); + _uriTranslator = new UriTranslatorImpl(libraries, packages); } return _uriTranslator; } - Future> _parseDartLibraries() async { - Uri librariesJson = _raw.sdkRoot?.resolve("lib/libraries.json"); - return await computeDartLibraries(fileSystem, librariesJson); + Future _computeLibrarySpecification() async { + var name = target.name; + // TODO(sigmund): Eek! We should get to the point where there is no + // fasta-specific targets and the target names are meaningful. + if (name.endsWith('_fasta')) name = name.substring(0, name.length - 6); + + if (librariesSpecificationUri == null || + !await fileSystem.entityForUri(librariesSpecificationUri).exists()) { + if (compileSdk) { + reportWithoutLocation( + templateSdkSpecificationNotFound + .withArguments(librariesSpecificationUri), + Severity.error); + } + return new TargetLibrariesSpecification(name); + } + + var json = + await fileSystem.entityForUri(librariesSpecificationUri).readAsString(); + try { + var spec = + await LibrariesSpecification.parse(librariesSpecificationUri, json); + return spec.specificationFor(name); + } on LibrariesSpecificationException catch (e) { + reportWithoutLocation( + templateCannotReadSdkSpecification.withArguments('${e.error}'), + Severity.error); + return new TargetLibrariesSpecification(name); + } } /// Get the package map which maps package names to URIs. @@ -398,32 +437,42 @@ class ProcessedOptions { return Packages.noPackages; } - /// Get the location of the SDK. - Uri _normalizeSdkRoot() { - // If an SDK summary location was provided, the SDK itself should not be - // needed. - assert(_raw.sdkSummary == null); - if (_raw.sdkRoot == null) { + bool _computedSdkDefaults = false; + + /// Ensure [_sdkRoot], [_sdkSummary] and [_librarySpecUri] are initialized. + /// + /// If they are not set explicitly, they are infered based on the default + /// behavior described in [CompilerOptions]. + void _ensureSdkDefaults() { + if (_computedSdkDefaults) return; + _computedSdkDefaults = true; + var root = _raw.sdkRoot; + if (root != null) { + // Normalize to always end in '/' + if (!root.path.endsWith('/')) { + root = root.replace(path: root.path + '/'); + } + _sdkRoot = root; + } else if (compileSdk) { // TODO(paulberry): implement the algorithm for finding the SDK // automagically. - return unimplemented('infer the default sdk location', -1, null); + unimplemented('infer the default sdk location', -1, null); } - var root = _raw.sdkRoot; - if (!root.path.endsWith('/')) { - root = root.replace(path: root.path + '/'); + + if (_raw.sdkSummary != null) { + _sdkSummary = _raw.sdkSummary; + } else if (!compileSdk) { + // Infer based on the sdkRoot, but only when `compileSdk` is false, + // otherwise the default intent was to compile the sdk from sources and + // not to load an sdk summary file. + _sdkSummary = root?.resolve('outline.dill'); } - return root; - } - /// Get or infer the location of the SDK summary. - Uri _computeSdkSummaryUri() { - if (_raw.sdkSummary != null) return _raw.sdkSummary; - - // Infer based on the sdkRoot, but only when `compileSdk` is false, - // otherwise the default intent was to compile the sdk from sources and not - // to load an sdk summary file. - if (_raw.compileSdk) return null; - return sdkRoot.resolve('outline.dill'); + if (_raw.librariesSpecificationUri != null) { + _librariesSpecificationUri = _raw.librariesSpecificationUri; + } else if (compileSdk) { + _librariesSpecificationUri = sdkRoot.resolve('lib/libraries.json'); + } } /// Create a [FileSystem] specific to the current options. @@ -494,6 +543,8 @@ class ProcessedOptions { sb.writeln('Compile SDK: ${compileSdk}'); sb.writeln('SDK root: ${_sdkRoot} (provided: ${_raw.sdkRoot})'); + sb.writeln('SDK specification: ${_librariesSpecificationUri} ' + '(provided: ${_raw.librariesSpecificationUri})'); sb.writeln('SDK summary: ${_sdkSummary} (provided: ${_raw.sdkSummary})'); sb.writeln('Strong: ${strongMode}'); diff --git a/pkg/front_end/lib/src/fasta/fasta_codes_generated.dart b/pkg/front_end/lib/src/fasta/fasta_codes_generated.dart index 505f593abaa..cb7c60b9ff1 100644 --- a/pkg/front_end/lib/src/fasta/fasta_codes_generated.dart +++ b/pkg/front_end/lib/src/fasta/fasta_codes_generated.dart @@ -201,6 +201,29 @@ Message _withArgumentsCannotReadPackagesFile(String string) { $string.""", arguments: {'string': string}); } +// DO NOT EDIT. THIS FILE IS GENERATED. SEE TOP OF FILE. +const Template + templateCannotReadSdkSpecification = + const Template( + messageTemplate: + r"""Unable to read the 'libraries.json' specification file: + #string.""", + withArguments: _withArgumentsCannotReadSdkSpecification); + +// DO NOT EDIT. THIS FILE IS GENERATED. SEE TOP OF FILE. +const Code codeCannotReadSdkSpecification = + const Code( + "CannotReadSdkSpecification", + templateCannotReadSdkSpecification, +); + +// DO NOT EDIT. THIS FILE IS GENERATED. SEE TOP OF FILE. +Message _withArgumentsCannotReadSdkSpecification(String string) { + return new Message(codeCannotReadSdkSpecification, + message: """Unable to read the 'libraries.json' specification file: + $string.""", arguments: {'string': string}); +} + // DO NOT EDIT. THIS FILE IS GENERATED. SEE TOP OF FILE. const Code codeCantInferPackagesFromManyInputs = messageCantInferPackagesFromManyInputs; @@ -2489,6 +2512,34 @@ Message _withArgumentsSdkRootNotFound(Uri uri_) { arguments: {'uri': uri_}); } +// DO NOT EDIT. THIS FILE IS GENERATED. SEE TOP OF FILE. +const Template< + Message Function( + Uri + uri_)> templateSdkSpecificationNotFound = const Template< + Message Function(Uri uri_)>( + messageTemplate: r"""SDK libraries specification not found: #uri.""", + tipTemplate: + r"""Normally, the specification is a file named 'libraries.json' in the Dart SDK install location.""", + withArguments: _withArgumentsSdkSpecificationNotFound); + +// DO NOT EDIT. THIS FILE IS GENERATED. SEE TOP OF FILE. +const Code codeSdkSpecificationNotFound = + const Code( + "SdkSpecificationNotFound", + templateSdkSpecificationNotFound, +); + +// DO NOT EDIT. THIS FILE IS GENERATED. SEE TOP OF FILE. +Message _withArgumentsSdkSpecificationNotFound(Uri uri_) { + String uri = relativizeUri(uri_); + return new Message(codeSdkSpecificationNotFound, + message: """SDK libraries specification not found: $uri.""", + tip: + """Normally, the specification is a file named 'libraries.json' in the Dart SDK install location.""", + arguments: {'uri': uri_}); +} + // DO NOT EDIT. THIS FILE IS GENERATED. SEE TOP OF FILE. const Template templateSdkSummaryNotFound = const Template( diff --git a/pkg/front_end/lib/src/fasta/uri_translator_impl.dart b/pkg/front_end/lib/src/fasta/uri_translator_impl.dart index b34fe69d0dc..78d9e264a92 100644 --- a/pkg/front_end/lib/src/fasta/uri_translator_impl.dart +++ b/pkg/front_end/lib/src/fasta/uri_translator_impl.dart @@ -4,65 +4,33 @@ library fasta.uri_translator_impl; -import 'dart:async' show Future; -import 'dart:convert' show JSON; - -import 'package:front_end/file_system.dart' - show FileSystem, FileSystemException; +import 'package:front_end/src/base/libraries_specification.dart' + show TargetLibrariesSpecification; import 'package:front_end/src/fasta/compiler_context.dart' show CompilerContext; import 'package:front_end/src/fasta/fasta_codes.dart'; import 'package:front_end/src/fasta/severity.dart' show Severity; import 'package:front_end/src/fasta/uri_translator.dart'; -import 'package:package_config/packages_file.dart' as packages_file show parse; import 'package:package_config/packages.dart' show Packages; -import 'package:package_config/src/packages_impl.dart' show MapPackages; - -import 'deprecated_problems.dart' show deprecated_inputError; - -/// Read the JSON file with defined SDK libraries from the given [uri] in the -/// [fileSystem] and return the mapping from parsed Dart library names (e.g. -/// `math`) to file URIs. -Future> computeDartLibraries( - FileSystem fileSystem, Uri uri) async { - if (uri == null) return const {}; - Map libraries = JSON - .decode(await fileSystem.entityForUri(uri).readAsString())["libraries"]; - Map result = {}; - libraries.forEach((String name, String path) { - result[name] = uri.resolveUri(new Uri.file(path)); - }); - return result; -} - -Future>> computeDartPatches( - FileSystem fileSystem, Uri uri) async { - // TODO(ahe): Read patch information. - return const >{}; -} /// Implementation of [UriTranslator] for absolute `dart` and `package` URIs. class UriTranslatorImpl implements UriTranslator { - /// Mapping from Dart library names (e.g. `math`) to file URIs. - final Map dartLibraries; - - // TODO(ahe): We probably want this to be `Map`, that is, just - // one patch library (with parts). - /// Mapping from Dart library names to the file URIs of patches to apply. - final Map> dartPatches; + /// Library information for platform libraries. + final TargetLibrariesSpecification dartLibraries; /// Mapping from package names (e.g. `angular`) to the file URIs. final Packages packages; - UriTranslatorImpl(this.dartLibraries, this.dartPatches, this.packages); + UriTranslatorImpl(this.dartLibraries, this.packages); @override - List getDartPatches(String libraryName) => dartPatches[libraryName]; + List getDartPatches(String libraryName) => + dartLibraries.libraryInfoFor(libraryName)?.patches; @override bool isPlatformImplementation(Uri uri) { if (uri.scheme != "dart") return false; String path = uri.path; - return dartLibraries[path] == null || path.startsWith("_"); + return dartLibraries.libraryInfoFor(path) == null || path.startsWith("_"); } @override @@ -82,11 +50,11 @@ class UriTranslatorImpl implements UriTranslator { String path = uri.path; int index = path.indexOf('/'); - if (index == -1) return dartLibraries[path]; + if (index == -1) return dartLibraries.libraryInfoFor(path)?.uri; String libraryName = path.substring(0, index); String relativePath = path.substring(index + 1); - Uri libraryFileUri = dartLibraries[libraryName]; + Uri libraryFileUri = dartLibraries.libraryInfoFor(libraryName).uri; return libraryFileUri?.resolve(relativePath); } @@ -116,32 +84,4 @@ class UriTranslatorImpl implements UriTranslator { // compiler. return null; } - - static Future parse(FileSystem fileSystem, Uri sdk, - {Uri packages}) async { - Uri librariesJson = sdk?.resolve("lib/libraries.json"); - - // TODO(ahe): Provide a value for this file. - Uri patches = null; - - packages ??= Uri.base.resolve(".packages"); - - List bytes; - try { - bytes = await fileSystem.entityForUri(packages).readAsBytes(); - } on FileSystemException catch (e) { - deprecated_inputError(packages, -1, e.message); - } - - Packages parsedPackages; - try { - parsedPackages = new MapPackages(packages_file.parse(bytes, packages)); - } on FormatException catch (e) { - return deprecated_inputError(packages, e.offset, e.message); - } - - var dartLibraries = await computeDartLibraries(fileSystem, librariesJson); - return new UriTranslatorImpl(dartLibraries, - await computeDartPatches(fileSystem, patches), parsedPackages); - } } diff --git a/pkg/front_end/lib/src/testing/compiler_common.dart b/pkg/front_end/lib/src/testing/compiler_common.dart index 02890026496..675bbbf8ecd 100644 --- a/pkg/front_end/lib/src/testing/compiler_common.dart +++ b/pkg/front_end/lib/src/testing/compiler_common.dart @@ -85,6 +85,9 @@ Future setup(CompilerOptions options, Map sources, } }); fs.entityForUri(toTestUri('.packages')).writeAsStringSync(''); + fs + .entityForUri(invalidCoreLibsSpecUri) + .writeAsStringSync(_invalidLibrariesSpec); options ..verify = true ..fileSystem = new HybridFileSystem(fs) @@ -105,13 +108,22 @@ Uri _defaultDir = Uri.parse('file:///a/b/c/'); /// helpers above. Uri toTestUri(String relativePath) => _defaultDir.resolve(relativePath); -/// A map defining the location of core libraries that purposely provides -/// invalid Uris. Used by tests that want to ensure that the sdk libraries are -/// not loaded from sources, but read from a .dill file. -Map invalidCoreLibs = { - 'core': Uri.parse('file:///non_existing_file/core.dart'), - 'async': Uri.parse('file:///non_existing_file/async.dart'), -}; +/// Uri to a libraries specification file that purposely provides +/// invalid Uris to dart:core and dart:async. Used by tests that want to ensure +/// that the sdk libraries are not loaded from sources, but read from a .dill +/// file. +Uri invalidCoreLibsSpecUri = toTestUri('invalid_sdk_libraries.json'); + +String _invalidLibrariesSpec = ''' +{ + "vm": { + "libraries": { + "core": {"uri": "/non_existing_file/core.dart"}, + "async": {"uri": "/non_existing_file/async.dart"} + } + } +} +'''; bool isDartCoreLibrary(Library lib) => isDartCore(lib.importUri); bool isDartCore(Uri uri) => uri.scheme == 'dart' && uri.path == 'core'; diff --git a/pkg/front_end/messages.yaml b/pkg/front_end/messages.yaml index b0a3d1c618d..ca0c8d66efe 100644 --- a/pkg/front_end/messages.yaml +++ b/pkg/front_end/messages.yaml @@ -823,6 +823,10 @@ SdkRootNotFound: SdkSummaryNotFound: template: "SDK summary not found: #uri." +SdkSpecificationNotFound: + template: "SDK libraries specification not found: #uri." + tip: "Normally, the specification is a file named 'libraries.json' in the Dart SDK install location." + ThisAccessInFieldInitializer: template: "Can't access 'this' in a field initializer to read '#name'." @@ -883,6 +887,9 @@ NotAnLvalue: CannotReadPackagesFile: template: "Unable to read '.packages' file:\n #string." +CannotReadSdkSpecification: + template: "Unable to read the 'libraries.json' specification file:\n #string." + CantInferPackagesFromManyInputs: template: "Can't infer a .packages file when compiling multiple inputs." tip: "Try specifying the file explicitly with the --packages option." diff --git a/pkg/front_end/test/fasta/testing/suite.dart b/pkg/front_end/test/fasta/testing/suite.dart index bde552d310d..4cdc6756c73 100644 --- a/pkg/front_end/test/fasta/testing/suite.dart +++ b/pkg/front_end/test/fasta/testing/suite.dart @@ -12,6 +12,9 @@ import 'dart:convert' show JSON; import 'package:front_end/physical_file_system.dart' show PhysicalFileSystem; +import 'package:front_end/src/base/libraries_specification.dart' + show TargetLibrariesSpecification; + import 'package:front_end/src/fasta/testing/validating_instrumentation.dart' show ValidatingInstrumentation; @@ -155,8 +158,10 @@ class FastaContext extends ChainContext { Uri sdk = await computePatchedSdk(); Uri vm = computeDartVm(sdk); Uri packages = Uri.base.resolve(".packages"); - UriTranslator uriTranslator = await UriTranslatorImpl - .parse(PhysicalFileSystem.instance, sdk, packages: packages); + var options = new ProcessedOptions(new CompilerOptions() + ..sdkRoot = sdk + ..packagesFileUri = packages); + UriTranslator uriTranslator = await options.getUriTranslator(); bool strongMode = environment.containsKey(STRONG_MODE); bool updateExpectations = environment["updateExpectations"] == "true"; bool updateComments = environment["updateComments"] == "true"; @@ -242,8 +247,7 @@ class Outline extends Step { // We create a new URI translator to avoid reading platform libraries from // file system. UriTranslatorImpl uriTranslator = new UriTranslatorImpl( - const {}, - const >{}, + const TargetLibrariesSpecification('vm'), context.uriTranslator.packages); KernelTarget sourceTarget = astKind == AstKind.Analyzer ? new AnalyzerTarget(dillTarget, uriTranslator, strongMode) diff --git a/pkg/front_end/test/fasta/uri_translator_test.dart b/pkg/front_end/test/fasta/uri_translator_test.dart index 27ca3c868e1..e15cde8cd2a 100644 --- a/pkg/front_end/test/fasta/uri_translator_test.dart +++ b/pkg/front_end/test/fasta/uri_translator_test.dart @@ -2,6 +2,7 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +import 'package:front_end/src/base/libraries_specification.dart'; import 'package:front_end/src/fasta/uri_translator_impl.dart'; import 'package:package_config/packages.dart'; import 'package:test/test.dart'; @@ -15,12 +16,16 @@ main() { @reflectiveTest class UriTranslatorImplTest { - void test_isPlatformImplementation() { - var translator = new UriTranslatorImpl({ - 'core': Uri.parse('file:///sdk/core/core.dart'), - 'math': Uri.parse('file:///sdk/math/math.dart') - }, {}, Packages.noPackages); + UriTranslatorImpl translator = new UriTranslatorImpl( + new TargetLibrariesSpecification('vm', { + 'core': new LibraryInfo( + 'core', Uri.parse('file:///sdk/core/core.dart'), const []), + 'math': new LibraryInfo( + 'core', Uri.parse('file:///sdk/math/math.dart'), const []), + }), + Packages.noPackages); + void test_isPlatformImplementation() { bool isPlatform(String uriStr) { var uri = Uri.parse(uriStr); return translator.isPlatformImplementation(uri); @@ -33,11 +38,6 @@ class UriTranslatorImplTest { } void test_translate_dart() { - var translator = new UriTranslatorImpl({ - 'core': Uri.parse('file:///sdk/core/core.dart'), - 'math': Uri.parse('file:///sdk/math/math.dart') - }, {}, Packages.noPackages); - expect(translator.translate(Uri.parse('dart:core')), Uri.parse('file:///sdk/core/core.dart')); expect(translator.translate(Uri.parse('dart:core/string.dart')), diff --git a/pkg/front_end/test/incremental_kernel_generator_test.dart b/pkg/front_end/test/incremental_kernel_generator_test.dart index 9195343d6e3..c3bbeb91f4b 100644 --- a/pkg/front_end/test/incremental_kernel_generator_test.dart +++ b/pkg/front_end/test/incremental_kernel_generator_test.dart @@ -36,18 +36,16 @@ class IncrementalKernelGeneratorTest { /// Compute the initial [Program] for the given [entryPoint]. Future getInitialState(Uri entryPoint, {bool setPackages: true}) async { - Map dartLibraries = createSdkFiles(fileSystem); + createSdkFiles(fileSystem); // TODO(scheglov) Builder the SDK kernel and set it into the options. - // TODO(scheglov) Make `.packages` file optional. - var compilerOptions = new CompilerOptions() ..fileSystem = fileSystem ..byteStore = new MemoryByteStore() // ..logger = new PerformanceLog(stdout) ..strongMode = true ..chaseDependencies = true - ..dartLibraries = dartLibraries; + ..librariesSpecificationUri = Uri.parse('file:///sdk/lib/libraries.json'); if (setPackages) { compilerOptions.packagesFileUri = Uri.parse('file:///test/.packages'); diff --git a/pkg/front_end/test/kernel_generator_test.dart b/pkg/front_end/test/kernel_generator_test.dart index 6620a094bf3..cbed95ef8e4 100644 --- a/pkg/front_end/test/kernel_generator_test.dart +++ b/pkg/front_end/test/kernel_generator_test.dart @@ -17,7 +17,7 @@ main() { test('compiler fails if it cannot find sdk sources', () async { var errors = []; var options = new CompilerOptions() - ..dartLibraries = invalidCoreLibs + ..librariesSpecificationUri = invalidCoreLibsSpecUri ..sdkSummary = null ..compileSdk = true // To prevent FE from loading an sdk-summary. ..onError = (e) => errors.add(e); @@ -42,9 +42,10 @@ main() { test('by default program is compiled using summaries', () async { var options = new CompilerOptions() - // Note: we define [dartLibraries] with broken URIs to ensure we do not - // attempt to lookup for sources of the sdk directly. - ..dartLibraries = invalidCoreLibs; + // Note: we define [librariesSpecificationUri] with a specification that + // contains broken URIs to ensure we do not attempt to lookup for + // sources of the sdk directly. + ..librariesSpecificationUri = invalidCoreLibsSpecUri; var program = await compileScript('main() => print("hi");', options: options); var core = program.libraries.firstWhere(isDartCoreLibrary); diff --git a/pkg/front_end/test/src/base/libraries_specification_test.dart b/pkg/front_end/test/src/base/libraries_specification_test.dart new file mode 100644 index 00000000000..25c3b4ac2cf --- /dev/null +++ b/pkg/front_end/test/src/base/libraries_specification_test.dart @@ -0,0 +1,309 @@ +// Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'package:front_end/src/base/libraries_specification.dart'; +import 'package:test/test.dart'; + +main() { + group('parse', () { + test('top-level must be a map', () async { + var jsonString = '[]'; + expect( + () => LibrariesSpecification.parse( + Uri.parse('org-dartlang-custom:///f.json'), jsonString), + throwsA((e) => e is LibrariesSpecificationException)); + jsonString = '""'; + expect( + () => LibrariesSpecification.parse( + Uri.parse('org-dartlang-custom:///f.json'), jsonString), + throwsA((e) => e is LibrariesSpecificationException)); + }); + + test('target entry must be a map', () async { + var jsonString = '{"vm" : []}'; + expect( + () => LibrariesSpecification.parse( + Uri.parse('org-dartlang-custom:///f.json'), jsonString), + throwsA((e) => e is LibrariesSpecificationException)); + jsonString = '{"vm" : ""}'; + expect( + () => LibrariesSpecification.parse( + Uri.parse('org-dartlang-custom:///f.json'), jsonString), + throwsA((e) => e is LibrariesSpecificationException)); + }); + + test('library entry must exist', () async { + var jsonString = '{"vm" : {}}'; + expect( + () => LibrariesSpecification.parse( + Uri.parse('org-dartlang-custom:///f.json'), jsonString), + throwsA((e) => e is LibrariesSpecificationException)); + }); + + test('library entry must be a map', () async { + var jsonString = '{"vm" : {"libraries": []}}'; + expect( + () => LibrariesSpecification.parse( + Uri.parse('org-dartlang-custom:///f.json'), jsonString), + throwsA((e) => e is LibrariesSpecificationException)); + }); + + test('uri must be a string', () async { + var jsonString = '{"vm" : {"libraries": {"core": {"uri": 3}}}'; + expect( + () => LibrariesSpecification.parse( + Uri.parse('org-dartlang-custom:///f.json'), jsonString), + throwsA((e) => e is LibrariesSpecificationException)); + }); + + test('patches must be a string or list of string', () async { + var jsonString = ''' + { + "none": { + "libraries": { + "c" : { + "uri": "c/main.dart", + "patches": 3 + } + } + } + } + '''; + expect( + () => LibrariesSpecification.parse( + Uri.parse('org-dartlang-custom:///f.json'), jsonString), + throwsA((e) => e is LibrariesSpecificationException)); + + jsonString = ''' + { + "none": { + "libraries": { + "c" : { + "uri": "c/main.dart", + "patches": "a.dart" + } + } + } + } + '''; + var spec = LibrariesSpecification.parse( + Uri.parse('org-dartlang-custom:///f.json'), jsonString); + expect(spec.specificationFor("none").libraryInfoFor("c").patches.first, + Uri.parse('org-dartlang-custom:///a.dart')); + + jsonString = ''' + { + "none": { + "libraries": { + "c" : { + "uri": "c/main.dart", + "patches": ["a.dart"] + } + } + } + } + '''; + spec = LibrariesSpecification.parse( + Uri.parse('org-dartlang-custom:///f.json'), jsonString); + expect(spec.specificationFor("none").libraryInfoFor("c").patches.first, + Uri.parse('org-dartlang-custom:///a.dart')); + }); + + test('patches are optional in the format', () async { + var jsonString = ''' + { "none": { "libraries": {"c" : { "uri": "c/main.dart" }}}} + '''; + var spec = LibrariesSpecification.parse( + Uri.parse('org-dartlang-custom:///one/two/f.json'), jsonString); + expect(spec, isNotNull); + expect( + spec.specificationFor('none').libraryInfoFor('c').patches, isEmpty); + }); + + test('library paths are resolved from spec uri', () async { + var jsonString = ''' + { "none": { "libraries": {"c" : { "uri": "c/main.dart" }}}} + '''; + + var spec = LibrariesSpecification.parse( + Uri.parse('org-dartlang-custom:///one/two/f.json'), jsonString); + expect(spec.specificationFor('none').libraryInfoFor('c').uri, + Uri.parse('org-dartlang-custom:///one/two/c/main.dart')); + }); + + test('patches paths are resolved from spec uri', () async { + var jsonString = ''' + { + "none": { + "libraries": { + "c" : { + "uri": "c/main.dart", + "patches": [ + "../a/p1.dart", + "../a/p2.dart" + ] + } + } + } + } + '''; + + var spec = LibrariesSpecification.parse( + Uri.parse('org-dartlang-custom:///one/two/f.json'), jsonString); + expect(spec.specificationFor('none').libraryInfoFor('c').patches[0], + Uri.parse('org-dartlang-custom:///one/a/p1.dart')); + expect(spec.specificationFor('none').libraryInfoFor('c').patches[1], + Uri.parse('org-dartlang-custom:///one/a/p2.dart')); + }); + + test('multiple targets are supported', () async { + var jsonString = ''' + { + "vm": { + "libraries": { + "foo" : { + "uri": "a/main.dart", + "patches": [ + "a/p1.dart", + "a/p2.dart" + ] + }, + "bar" : { + "uri": "b/main.dart", + "patches": [ + "b/p3.dart" + ] + } + } + }, + "none": { + "libraries": { + "c" : { + "uri": "c/main.dart" + } + } + } + } + '''; + + var spec = LibrariesSpecification.parse( + Uri.parse('org-dartlang-custom:///one/two/f.json'), jsonString); + + expect(spec.specificationFor('vm').libraryInfoFor('foo').uri, + Uri.parse('org-dartlang-custom:///one/two/a/main.dart')); + expect(spec.specificationFor('vm').libraryInfoFor('bar').uri, + Uri.parse('org-dartlang-custom:///one/two/b/main.dart')); + expect(spec.specificationFor('none').libraryInfoFor('c').uri, + Uri.parse('org-dartlang-custom:///one/two/c/main.dart')); + }); + }); + + group('toJson', () { + test('serialization produces same data that was parsed', () async { + var jsonString = ''' + { + "vm": { + "libraries": { + "foo" : { + "uri": "a/main.dart", + "patches": [ + "a/p1.dart", + "a/p2.dart" + ] + }, + "bar" : { + "uri": "b/main.dart", + "patches": [ + "b/p3.dart" + ] + } + } + }, + "none": { + "libraries": { + "c" : { + "uri": "c/main.dart", + "patches": [] + } + } + } + } + '''; + + var spec = LibrariesSpecification.parse( + Uri.parse('org-dartlang-custom:///one/two/f.json'), jsonString); + var newJson = + spec.toJsonString(Uri.parse('org-dartlang-custom:///one/two/g.json')); + expect(jsonString.replaceAll(new RegExp('\\s'), ''), newJson); + }); + + test('serialization can adapt to new file location', () async { + var jsonString = ''' + { + "vm": { + "libraries": { + "foo" : { + "uri": "a/main.dart", + "patches": [ + "a/p1.dart", + "a/p2.dart" + ] + }, + "bar" : { + "uri": "b/main.dart", + "patches": [ + "b/p3.dart" + ] + } + } + }, + "none": { + "libraries": { + "c" : { + "uri": "c/main.dart" + } + } + } + } + '''; + + var spec = LibrariesSpecification.parse( + Uri.parse('org-dartlang-custom:///one/two/f.json'), jsonString); + var newJson = + spec.toJsonString(Uri.parse('org-dartlang-custom:///one/g.json')); + + var expected = ''' + { + "vm": { + "libraries": { + "foo" : { + "uri": "two/a/main.dart", + "patches": [ + "two/a/p1.dart", + "two/a/p2.dart" + ] + }, + "bar" : { + "uri": "two/b/main.dart", + "patches": [ + "two/b/p3.dart" + ] + } + } + }, + "none": { + "libraries": { + "c" : { + "uri": "two/c/main.dart", + "patches": [] + } + } + } + } + '''; + + expect(expected.replaceAll(new RegExp('\\s'), ''), newJson); + }); + }); +} diff --git a/pkg/front_end/test/src/base/processed_options_test.dart b/pkg/front_end/test/src/base/processed_options_test.dart index 883dbc99558..917576f515d 100644 --- a/pkg/front_end/test/src/base/processed_options_test.dart +++ b/pkg/front_end/test/src/base/processed_options_test.dart @@ -8,6 +8,7 @@ import 'package:front_end/compiler_options.dart'; import 'package:front_end/memory_file_system.dart'; import 'package:front_end/src/base/processed_options.dart'; import 'package:front_end/src/fasta/fasta.dart' show ByteSink; +import 'package:front_end/src/fasta/compiler_context.dart'; import 'package:front_end/src/fasta/fasta_codes.dart'; import 'package:kernel/binary/ast_to_binary.dart' show BinaryPrinter; import 'package:kernel/kernel.dart' show Program, Library, CanonicalName; @@ -17,8 +18,10 @@ import 'package:test/test.dart'; import 'package:test_reflective_loader/test_reflective_loader.dart'; main() { - defineReflectiveSuite(() { - defineReflectiveTests(ProcessedOptionsTest); + CompilerContext.runWithDefaultOptions((_) { + defineReflectiveSuite(() { + defineReflectiveTests(ProcessedOptionsTest); + }); }); } @@ -84,6 +87,58 @@ class ProcessedOptionsTest { mockSummary.libraries.single.importUri); } + test_getUriTranslator_explicitLibrariesSpec() async { + fileSystem + .entityForUri(Uri.parse('file:///.packages')) + .writeAsStringSync(''); + fileSystem + .entityForUri(Uri.parse('file:///libraries.json')) + .writeAsStringSync('{"vm":{"libraries":{"foo":{"uri":"bar.dart"}}}}'); + var raw = new CompilerOptions() + ..packagesFileUri = Uri.parse('file:///.packages') + ..fileSystem = fileSystem + ..librariesSpecificationUri = Uri.parse('file:///libraries.json'); + var processed = new ProcessedOptions(raw); + var uriTranslator = await processed.getUriTranslator(); + expect(uriTranslator.dartLibraries.libraryInfoFor('foo').uri.path, + '/bar.dart'); + } + + test_getUriTranslator_inferredLibrariesSpec() async { + fileSystem + .entityForUri(Uri.parse('file:///.packages')) + .writeAsStringSync(''); + fileSystem + .entityForUri(Uri.parse('file:///mysdk/lib/libraries.json')) + .writeAsStringSync('{"vm":{"libraries":{"foo":{"uri":"bar.dart"}}}}'); + var raw = new CompilerOptions() + ..fileSystem = fileSystem + ..packagesFileUri = Uri.parse('file:///.packages') + ..compileSdk = true + ..sdkRoot = Uri.parse('file:///mysdk/'); + var processed = new ProcessedOptions(raw); + var uriTranslator = await processed.getUriTranslator(); + expect(uriTranslator.dartLibraries.libraryInfoFor('foo').uri.path, + '/mysdk/lib/bar.dart'); + } + + test_getUriTranslator_notInferredLibrariesSpec() async { + fileSystem + .entityForUri(Uri.parse('file:///.packages')) + .writeAsStringSync(''); + fileSystem + .entityForUri(Uri.parse('file:///mysdk/lib/libraries.json')) + .writeAsStringSync('{"vm":{"libraries":{"foo":{"uri":"bar.dart"}}}}'); + var raw = new CompilerOptions() + ..fileSystem = fileSystem + ..packagesFileUri = Uri.parse('file:///.packages') + ..compileSdk = false // libraries.json is only inferred if true + ..sdkRoot = Uri.parse('file:///mysdk/'); + var processed = new ProcessedOptions(raw); + var uriTranslator = await processed.getUriTranslator(); + expect(uriTranslator.dartLibraries.libraryInfoFor('foo'), isNull); + } + checkPackageExpansion( String packageName, String packageDir, Packages packages) { var input = Uri.parse('package:$packageName/a.dart'); diff --git a/pkg/front_end/test/src/incremental/file_state_test.dart b/pkg/front_end/test/src/incremental/file_state_test.dart index 2fd3a26c7be..590b2d845ec 100644 --- a/pkg/front_end/test/src/incremental/file_state_test.dart +++ b/pkg/front_end/test/src/incremental/file_state_test.dart @@ -24,16 +24,14 @@ main() { class FileSystemStateTest { final byteStore = new MemoryByteStore(); final fileSystem = new MemoryFileSystem(Uri.parse('file:///')); - final UriTranslatorImpl uriTranslator = - new UriTranslatorImpl({}, {}, Packages.noPackages); FileSystemState fsState; Uri _coreUri; List _newFileUris = []; void setUp() { - Map dartLibraries = createSdkFiles(fileSystem); - uriTranslator.dartLibraries.addAll(dartLibraries); + var uriTranslator = + new UriTranslatorImpl(createSdkFiles(fileSystem), Packages.noPackages); _coreUri = Uri.parse('dart:core'); expect(_coreUri, isNotNull); fsState = new FileSystemState(byteStore, fileSystem, uriTranslator, [], diff --git a/pkg/front_end/test/src/incremental/hot_reload_e2e_test.dart b/pkg/front_end/test/src/incremental/hot_reload_e2e_test.dart index 0740dfc5215..0571ed8efcc 100644 --- a/pkg/front_end/test/src/incremental/hot_reload_e2e_test.dart +++ b/pkg/front_end/test/src/incremental/hot_reload_e2e_test.dart @@ -175,24 +175,14 @@ Future createIncrementalCompiler( var entryUri = Uri.base.resolve(entry); var options = new CompilerOptions() ..sdkRoot = sdkRoot - ..sdkSummary = sdkRoot.resolve('outline.dill') ..packagesFileUri = Uri.parse('file:///.packages') ..strongMode = false - ..dartLibraries = loadDartLibraries() + ..compileSdk = true // the incremental generator requires the sdk sources ..fileSystem = fs ..byteStore = new MemoryByteStore(); return IncrementalKernelGenerator.newInstance(options, entryUri); } -Map loadDartLibraries() { - var libraries = sdkRoot.resolve('lib/libraries.json'); - var map = - JSON.decode(new File.fromUri(libraries).readAsStringSync())['libraries']; - var dartLibraries = {}; - map.forEach((k, v) => dartLibraries[k] = libraries.resolve(v)); - return dartLibraries; -} - Future rebuild(IncrementalKernelGenerator compiler, Uri outputUri) async { compiler.invalidate(Uri.parse("file:///a.dart")); compiler.invalidate(Uri.parse("file:///b.dart")); diff --git a/pkg/front_end/test/src/incremental/kernel_driver_test.dart b/pkg/front_end/test/src/incremental/kernel_driver_test.dart index 193a84d95bc..67f3e088c99 100644 --- a/pkg/front_end/test/src/incremental/kernel_driver_test.dart +++ b/pkg/front_end/test/src/incremental/kernel_driver_test.dart @@ -650,9 +650,8 @@ import 'b.dart'; /// Create new [KernelDriver] instance and put it into the [driver] field. void _createDriver( {Map packages, KernelDriverFileAddedFn fileAddedFn}) { - Map dartLibraries = createSdkFiles(fileSystem); - var uriTranslator = - new UriTranslatorImpl(dartLibraries, {}, new MapPackages(packages)); + var uriTranslator = new UriTranslatorImpl( + createSdkFiles(fileSystem), new MapPackages(packages)); driver = new KernelDriver( new ProcessedOptions(new CompilerOptions() ..logger = new PerformanceLog(null) diff --git a/pkg/front_end/test/src/incremental/mock_sdk.dart b/pkg/front_end/test/src/incremental/mock_sdk.dart index b005e2606c3..b4b8d81c295 100644 --- a/pkg/front_end/test/src/incremental/mock_sdk.dart +++ b/pkg/front_end/test/src/incremental/mock_sdk.dart @@ -3,18 +3,21 @@ // BSD-style license that can be found in the LICENSE file. import 'package:front_end/memory_file_system.dart'; +import 'package:front_end/src/base/libraries_specification.dart'; /// Create SDK libraries which are used by Fasta to perform kernel generation. -/// Return the mapping from the simple names of these library to the URIs -/// in the given [fileSystem]. The root of the SDK is `file:///sdk`. -Map createSdkFiles(MemoryFileSystem fileSystem) { - Map dartLibraries = {}; - +/// The root of the SDK is `file:///sdk`, it will contain a libraries +/// specification file at `lib/libraries.json`. +/// +/// Returns the [TargetLibrariesSpecification] whose contents are in +/// libraries.json. +TargetLibrariesSpecification createSdkFiles(MemoryFileSystem fileSystem) { + Map dartLibraries = {}; void addSdkLibrary(String name, String contents) { String path = '$name/$name.dart'; Uri uri = Uri.parse('file:///sdk/lib/$path'); fileSystem.entityForUri(uri).writeAsStringSync(contents); - dartLibraries[name] = uri; + dartLibraries[name] = new LibraryInfo(name, uri, const []); } addSdkLibrary('core', r''' @@ -265,5 +268,10 @@ class ExternalName { '''); addSdkLibrary('_vmservice', 'library dart._vmservice;'); - return dartLibraries; + var targetSpec = new TargetLibrariesSpecification('vm', dartLibraries); + var spec = new LibrariesSpecification({'vm': targetSpec}); + + Uri uri = Uri.parse('file:///sdk/lib/libraries.json'); + fileSystem.entityForUri(uri).writeAsStringSync(spec.toJsonString(uri)); + return targetSpec; } diff --git a/pkg/front_end/test/subpackage_relationships_test.dart b/pkg/front_end/test/subpackage_relationships_test.dart index f7fd69ae7d0..abd8e2aa62e 100644 --- a/pkg/front_end/test/subpackage_relationships_test.dart +++ b/pkg/front_end/test/subpackage_relationships_test.dart @@ -57,6 +57,7 @@ final subpackageRules = { 'lib', 'lib/src', 'lib/src/fasta', + 'lib/src/fasta/util', 'lib/src/incremental' ]), 'lib/src/codegen': new SubpackageRules(), diff --git a/pkg/front_end/tool/fasta_perf.dart b/pkg/front_end/tool/fasta_perf.dart index d8d4147c081..c083aaf3820 100644 --- a/pkg/front_end/tool/fasta_perf.dart +++ b/pkg/front_end/tool/fasta_perf.dart @@ -10,7 +10,7 @@ import 'dart:io'; import 'package:analyzer/src/fasta/ast_builder.dart'; import 'package:front_end/front_end.dart'; -import 'package:front_end/physical_file_system.dart'; +import 'package:front_end/src/base/processed_options.dart'; import 'package:front_end/src/fasta/parser.dart'; import 'package:front_end/src/fasta/scanner.dart'; import 'package:front_end/src/fasta/scanner/io.dart' show readBytesFromFileSync; @@ -18,7 +18,6 @@ import 'package:front_end/src/fasta/source/directive_listener.dart'; import 'package:front_end/src/fasta/uri_translator.dart' show UriTranslator; import 'package:front_end/src/fasta/parser/native_support.dart' show skipNativeClause; -import 'package:front_end/src/fasta/uri_translator_impl.dart'; /// Cumulative total number of chars scanned. int inputSize = 0; @@ -80,8 +79,11 @@ UriTranslator uriResolver; /// Preliminary set up to be able to correctly resolve URIs on the given /// program. Future setup(Uri entryUri) async { - uriResolver = - await UriTranslatorImpl.parse(PhysicalFileSystem.instance, sdkRoot); + var options = new CompilerOptions() + ..sdkRoot = sdkRoot + ..compileSdk = true + ..packagesFileUri = Uri.base.resolve('.packages'); + uriResolver = await new ProcessedOptions(options).getUriTranslator(); } /// Scan [contents] and return the first token produced by the scanner. diff --git a/tools/patch_sdk.dart b/tools/patch_sdk.dart index 8e86120e9c5..97e2e12c8c9 100644 --- a/tools/patch_sdk.dart +++ b/tools/patch_sdk.dart @@ -108,7 +108,7 @@ Future _main(List argv) async { libContents = _updateLibraryMetadata(sdkOut, libContents); var sdkLibraries = _getSdkLibraries(libContents); - Map locations = {}; + var locations = >{}; // Enumerate core libraries and apply patches for (SdkLibrary library in sdkLibraries) { @@ -125,7 +125,10 @@ Future _main(List argv) async { Uri packages = Uri.base.resolveUri(new Uri.file(packagesFile)); await _writeSync( - librariesJson.toFilePath(), JSON.encode({"libraries": locations})); + librariesJson.toFilePath(), + JSON.encode({ + mode: {"libraries": locations} + })); var flags = new TargetFlags(); var target = forVm @@ -141,6 +144,22 @@ Future _main(List argv) async { var base = path.fromUri(Platform.script); Uri dartDir = new Uri.directory(path.dirname(path.dirname(path.absolute(base)))); + + String vmserviceJson = JSON.encode({ + 'vm': { + "libraries": { + '_vmservice': { + 'uri': '${dartDir.resolve('sdk/lib/vmservice/vmservice.dart')}' + }, + 'vmservice_io': { + 'uri': + '${dartDir.resolve('runtime/bin/vmservice/vmservice_io.dart')}' + }, + } + } + }); + Uri vmserviceJsonUri = outDirUri.resolve("lib/vmservice_libraries.json"); + await _writeSync(vmserviceJsonUri.toFilePath(), vmserviceJson); var program = await kernelForProgram( Uri.parse('dart:$vmserviceName'), new CompilerOptions() @@ -148,11 +167,7 @@ Future _main(List argv) async { // TODO(sigmund): investigate. This should be outline, but it breaks // vm-debug tests. Issue #30111 ..sdkSummary = platform - ..dartLibraries = { - '_vmservice': dartDir.resolve('sdk/lib/vmservice/vmservice.dart'), - 'vmservice_io': - dartDir.resolve('runtime/bin/vmservice/vmservice_io.dart'), - } + ..librariesSpecificationUri = vmserviceJsonUri ..packagesFileUri = packages); Uri vmserviceUri = outDirUri.resolve('$vmserviceName.dill'); // TODO(sigmund): remove. This is a workaround because in the VM @@ -333,7 +348,7 @@ String _updateLibraryMetadata(String sdkOut, String libContents) { /// Copy internal libraries that are developed outside the sdk folder into the /// patched_sdk folder. For the VM< this includes files under 'runtime/bin/', /// for flutter, this is includes also the ui library. -_copyExtraLibraries(String sdkOut, Map locations) { +_copyExtraLibraries(String sdkOut, Map> locations) { if (forDart2js) return; var base = path.fromUri(Platform.script); var dartDir = path.dirname(path.dirname(path.absolute(base))); @@ -341,7 +356,7 @@ _copyExtraLibraries(String sdkOut, Map locations) { var builtinLibraryIn = path.join(dartDir, 'runtime', 'bin', 'builtin.dart'); var builtinLibraryOut = path.join(sdkOut, '_builtin', '_builtin.dart'); _writeSync(builtinLibraryOut, readInputFile(builtinLibraryIn)); - locations['_builtin'] = path.join('_builtin', '_builtin.dart'); + addLocation(locations, '_builtin', path.join('_builtin', '_builtin.dart')); if (forFlutter) { // Flutter repo has this layout: @@ -356,7 +371,7 @@ _copyExtraLibraries(String sdkOut, Map locations) { var uiLibraryOut = path.join(sdkOut, 'ui', name); _writeSync(uiLibraryOut, readInputFile(file.path)); } - locations['ui'] = 'ui/ui.dart'; + addLocation(locations, 'ui', path.join('ui', 'ui.dart')); if (!forFlutterRelease) { // vmservice should be present unless we build release flavor of Flutter. @@ -368,22 +383,23 @@ _copyExtraLibraries(String sdkOut, Map locations) { var libraryOut = path.join(sdkOut, 'vmservice_io', file); _writeSync(libraryOut, readInputFile(libraryIn)); } - locations['vmservice_sky'] = - path.join('vmservice_io', 'vmservice_io.dart'); - locations['_vmservice'] = path.join('vmservice', 'vmservice.dart'); + addLocation(locations, 'vmservice_sky', + path.join('vmservice_io', 'vmservice_io.dart')); + addLocation( + locations, '_vmservice', path.join('vmservice', 'vmservice.dart')); } } } _applyPatch(SdkLibrary library, String sdkLibIn, String patchIn, String sdkOut, - Map locations) { + Map> locations) { var libraryOut = path.join(sdkLibIn, library.path); var libraryIn = libraryOut; var libraryFile = getInputFile(libraryIn, canBeMissing: true); if (libraryFile != null) { - locations[Uri.parse(library.shortName).path] = - path.relative(libraryOut, from: sdkLibIn); + addLocation(locations, Uri.parse(library.shortName).path, + path.relative(libraryOut, from: sdkLibIn)); var outPaths = [libraryOut]; var libraryContents = libraryFile.readAsStringSync(); @@ -806,3 +822,9 @@ List _getSdkLibraries(String contents) { parseCompilationUnit(contents).accept(libraryBuilder); return libraryBuilder.librariesMap.sdkLibraries; } + +void addLocation(Map> locations, String libraryName, + String libraryPath) { + assert(locations[libraryName] == null); + locations[libraryName] = {'uri': '${path.toUri(libraryPath)}'}; +}