Updates frontend_server javascript bundler to be based on import uri instead of file uri.

Converts `package:` import uris into `/packages/` modules.

Also renames the output modules to append `.lib.js` instead of just `.js`. This allows us to distinguish between modules and applications based on extension.

Updates DDC source map code to be able to convert absolute file uris in sources so that they are relative to the source map.

Bug: https://github.com/dart-lang/webdev/issues/865
Change-Id: I55d70aa3761f10cc8bd7e92f5b567478040660de
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/132300
Reviewed-by: Nicholas Shahan <nshahan@google.com>
Reviewed-by: Sigmund Cherem <sigmund@google.com>
Reviewed-by: Jonah Williams <jonahwilliams@google.com>
Commit-Queue: Jake Macdonald <jakemac@google.com>
This commit is contained in:
Jacob MacDonald 2020-01-21 17:41:02 +00:00 committed by commit-bot@chromium.org
parent da1b47cace
commit 4460f95782
8 changed files with 203 additions and 111 deletions

View file

@ -305,18 +305,32 @@ Uri sourcePathToRelativeUri(String source, {bool windows}) {
return uri;
}
/// Adjusts the source paths in [sourceMap] to be relative to [sourceMapPath],
/// and returns the new map. Relative paths are in terms of URIs ('/'), not
/// local OS paths (e.g., windows '\'). Sources with a multi-root scheme
/// matching [multiRootScheme] are adjusted to be relative to
/// [multiRootOutputPath].
/// Adjusts the source paths in [sourceMap] to be relative paths,and returns
/// the new map.
///
/// Relative paths are in terms of URIs ('/'), not local OS paths (e.g.,
/// windows '\').
///
/// Source uris show up in two forms, absolute `file:` uris and custom
/// [multiRootScheme] uris (also "absolute" uris, but always relative to some
/// multi-root).
///
/// - `file:` uris are converted to be relative to [sourceMapBase], which
/// defaults to the dirname of [sourceMapPath] if not provided.
///
/// - [multiRootScheme] uris are prefixed by [multiRootOutputPath]. If the
/// path starts with `/lib`, then we strip that before making it relative
/// to the [multiRootOutputPath], and assert that [multiRootOutputPath]
/// starts with `/packages` (more explanation inline).
///
// TODO(jmesserly): find a new home for this.
Map placeSourceMap(Map sourceMap, String sourceMapPath, String multiRootScheme,
{String multiRootOutputPath}) {
{String multiRootOutputPath, String sourceMapBase}) {
var map = Map.from(sourceMap);
// Convert to a local file path if it's not.
sourceMapPath = sourcePathToUri(p.absolute(p.fromUri(sourceMapPath))).path;
var sourceMapDir = p.url.dirname(sourceMapPath);
sourceMapBase ??= sourceMapDir;
var list = (map['sources'] as List).toList();
String makeRelative(String sourcePath) {
@ -330,8 +344,17 @@ Map placeSourceMap(Map sourceMap, String sourceMapPath, String multiRootScheme,
var shortPath = uri.path
.replaceAll('/sdk/', '/dart-sdk/')
.replaceAll('/sdk_nnbd/', '/dart-sdk/');
// A multi-root uri starting with a path under `/lib` indicates that
// the multi-root is at the root of a package (typically, the
// application root package). These should be converted into a
// `/packages` path, we do that by stripping the `/lib` prefix and
// relying on the `multiRootOutputPath` to be set to the proper
// packages dir (so /packages/<package>).
if (shortPath.startsWith('/lib')) {
assert(multiRootOutputPath.startsWith('/packages'));
shortPath = shortPath.substring(4);
}
var multiRootPath = "${multiRootOutputPath ?? ''}$shortPath";
multiRootPath = multiRootPath;
multiRootPath = p.url.relative(multiRootPath, from: sourceMapDir);
return multiRootPath;
}
@ -342,7 +365,7 @@ Map placeSourceMap(Map sourceMap, String sourceMapPath, String multiRootScheme,
sourcePath = sourcePathToUri(p.absolute(p.fromUri(uri))).path;
// Fall back to a relative path against the source map itself.
sourcePath = p.url.relative(sourcePath, from: sourceMapDir);
sourcePath = p.url.relative(sourcePath, from: sourceMapBase);
// Convert from relative local path to relative URI.
return p.toUri(sourcePath).path;

View file

@ -536,11 +536,16 @@ class JSCode {
JSCode(this.code, this.sourceMap);
}
/// Converts [moduleTree] to [JSCode], using [format].
///
/// See [placeSourceMap] for a description of [sourceMapBase], [customScheme],
/// and [multiRootOutputPath] arguments.
JSCode jsProgramToCode(js_ast.Program moduleTree, ModuleFormat format,
{bool buildSourceMap = false,
bool inlineSourceMap = false,
String jsUrl,
String mapUrl,
String sourceMapBase,
String customScheme,
String multiRootOutputPath}) {
var opts = js_ast.JavaScriptPrintingOptions(
@ -562,7 +567,7 @@ JSCode jsProgramToCode(js_ast.Program moduleTree, ModuleFormat format,
Map builtMap;
if (buildSourceMap && sourceMap != null) {
builtMap = placeSourceMap(sourceMap.build(jsUrl), mapUrl, customScheme,
multiRootOutputPath: multiRootOutputPath);
multiRootOutputPath: multiRootOutputPath, sourceMapBase: sourceMapBase);
var jsDir = p.dirname(p.fromUri(jsUrl));
var relative = p.relative(p.fromUri(mapUrl), from: jsDir);
var relativeMapUrl = p.toUri(relative).toString();

View file

@ -25,6 +25,7 @@ import 'package:kernel/binary/limited_ast_to_binary.dart';
import 'package:kernel/kernel.dart'
show Component, loadComponentSourceFromBytes;
import 'package:kernel/target/targets.dart' show targets, TargetFlags;
import 'package:package_resolver/package_resolver.dart';
import 'package:path/path.dart' as path;
import 'package:usage/uuid/uuid.dart';
@ -456,7 +457,8 @@ class FrontendCompiler implements CompilerInterface {
transformer?.transform(results.component);
if (_compilerOptions.target.name == 'dartdevc') {
await writeJavascriptBundle(results, _kernelBinaryFilename);
await writeJavascriptBundle(
results, _kernelBinaryFilename, options['filesystem-scheme']);
} else {
await writeDillFile(results, _kernelBinaryFilename,
filterExternal: importDill != null,
@ -526,16 +528,14 @@ class FrontendCompiler implements CompilerInterface {
}
/// Write a JavaScript bundle containg the provided component.
Future<void> writeJavascriptBundle(
KernelCompilationResults results, String filename) async {
Future<void> writeJavascriptBundle(KernelCompilationResults results,
String filename, String fileSystemScheme) async {
var packageResolver = await PackageResolver.loadConfig(
_compilerOptions.packagesFileUri ?? '.packages');
final Component component = results.component;
// Compute strongly connected components.
final strongComponents = StrongComponents(
component,
results.loadedLibraries,
_mainSource,
_compilerOptions.packagesFileUri,
_compilerOptions.fileSystem);
final strongComponents = StrongComponents(component,
results.loadedLibraries, _mainSource, _compilerOptions.fileSystem);
await strongComponents.computeModules();
// Create JavaScript bundler.
@ -545,11 +545,12 @@ class FrontendCompiler implements CompilerInterface {
if (!sourceFile.parent.existsSync()) {
sourceFile.parent.createSync(recursive: true);
}
final bundler = JavaScriptBundler(component, strongComponents);
final bundler = JavaScriptBundler(
component, strongComponents, fileSystemScheme, packageResolver);
final sourceFileSink = sourceFile.openWrite();
final manifestFileSink = manifestFile.openWrite();
final sourceMapsFileSink = sourceMapsFile.openWrite();
bundler.compile(
await bundler.compile(
results.classHierarchy,
results.coreTypes,
results.loadedLibraries,
@ -790,7 +791,8 @@ class FrontendCompiler implements CompilerInterface {
deltaProgram.uriToSource.keys);
if (_compilerOptions.target.name == 'dartdevc') {
await writeJavascriptBundle(results, _kernelBinaryFilename);
await writeJavascriptBundle(
results, _kernelBinaryFilename, _options['filesystem-scheme']);
} else {
await writeDillFile(results, _kernelBinaryFilename,
incrementalSerializer: _generator.incrementalSerializer);

View file

@ -9,6 +9,8 @@ import 'package:dev_compiler/dev_compiler.dart';
import 'package:kernel/ast.dart';
import 'package:kernel/class_hierarchy.dart';
import 'package:kernel/core_types.dart';
import 'package:path/path.dart' as p;
import 'package:package_resolver/package_resolver.dart';
import 'strong_components.dart';
@ -22,7 +24,8 @@ import 'strong_components.dart';
/// an incremental build, a different file is written for each which contains
/// only the updated libraries.
class JavaScriptBundler {
JavaScriptBundler(this._originalComponent, this._strongComponents) {
JavaScriptBundler(this._originalComponent, this._strongComponents,
this._fileSystemScheme, this._packageResolver) {
_summaries = <Component>[];
_summaryUris = <Uri>[];
_moduleImportForSummary = <Uri, String>{};
@ -36,13 +39,15 @@ class JavaScriptBundler {
);
_summaries.add(summaryComponent);
_summaryUris.add(uri);
_moduleImportForSummary[uri] = '${uri.path}.js';
_moduleImportForSummary[uri] = '${urlForComponentUri(uri)}.lib.js';
_uriToComponent[uri] = summaryComponent;
}
}
final StrongComponents _strongComponents;
final Component _originalComponent;
final String _fileSystemScheme;
final PackageResolver _packageResolver;
List<Component> _summaries;
List<Uri> _summaryUris;
@ -50,13 +55,13 @@ class JavaScriptBundler {
Map<Uri, Component> _uriToComponent;
/// Compile each component into a single JavaScript module.
void compile(
Future<void> compile(
ClassHierarchy classHierarchy,
CoreTypes coreTypes,
Set<Library> loadedLibraries,
IOSink codeSink,
IOSink manifestSink,
IOSink sourceMapsSink) {
IOSink sourceMapsSink) async {
var codeOffset = 0;
var sourceMapOffset = 0;
final manifest = <String, Map<String, List<int>>>{};
@ -66,7 +71,8 @@ class JavaScriptBundler {
library.importUri.scheme == 'dart') {
continue;
}
final Uri moduleUri = _strongComponents.moduleAssignment[library.fileUri];
final Uri moduleUri =
_strongComponents.moduleAssignment[library.importUri];
if (visited.contains(moduleUri)) {
continue;
}
@ -81,12 +87,29 @@ class JavaScriptBundler {
);
final jsModule = compiler.emitModule(
summaryComponent, _summaries, _summaryUris, _moduleImportForSummary);
final moduleUrl = moduleUri.toString();
final code = jsProgramToCode(jsModule, ModuleFormat.amd,
inlineSourceMap: true,
buildSourceMap: true,
jsUrl: '$moduleUrl',
mapUrl: '$moduleUrl.js.map');
final moduleUrl = urlForComponentUri(moduleUri);
String sourceMapBase;
if (moduleUri.scheme == 'package') {
// Source locations come through as absolute file uris. In order to
// make relative paths in the source map we get the absolute uri for
// the module and make them relative to that.
sourceMapBase =
p.dirname((await _packageResolver.resolveUri(moduleUri)).path);
}
final code = jsProgramToCode(
jsModule,
ModuleFormat.amd,
inlineSourceMap: true,
buildSourceMap: true,
jsUrl: '$moduleUrl.lib.js',
mapUrl: '$moduleUrl.lib.js.map',
sourceMapBase: sourceMapBase,
customScheme: _fileSystemScheme,
multiRootOutputPath: moduleUri.scheme == 'package'
? '/packages/${moduleUri.pathSegments.first}'
: null,
);
final codeBytes = utf8.encode(code.code);
final sourceMapBytes = utf8.encode(json.encode(code.sourceMap));
codeSink.add(codeBytes);
@ -104,3 +127,7 @@ class JavaScriptBundler {
manifestSink.add(utf8.encode(json.encode(manifest)));
}
}
String urlForComponentUri(Uri componentUri) => componentUri.scheme == 'package'
? '/packages/${componentUri.path}'
: componentUri.path;

View file

@ -2,12 +2,9 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:io';
import 'package:kernel/ast.dart';
import 'package:kernel/util/graph.dart';
import 'package:vm/kernel_front_end.dart';
import 'package:front_end/src/api_unstable/vm.dart' show FileSystem;
/// Compute the strongly connected components for JavaScript compilation.
@ -31,7 +28,6 @@ class StrongComponents {
this.component,
this.loadedLibraries,
this.mainUri, [
this.packagesUri,
this.fileSystem,
]);
@ -47,9 +43,6 @@ class StrongComponents {
/// The main URI for thiis application.
final Uri mainUri;
/// The URI of the .packages file.
final Uri packagesUri;
/// The filesystem instance for resolving files.
final FileSystem fileSystem;
@ -72,84 +65,28 @@ class StrongComponents {
if (component.libraries.isEmpty) {
return;
}
Uri entrypointFileUri = mainUri;
if (!entrypointFileUri.isScheme('file')) {
entrypointFileUri = await asFileUri(fileSystem,
await _convertToFileUri(fileSystem, entrypointFileUri, packagesUri));
}
if (entrypointFileUri == null || !entrypointFileUri.isScheme('file')) {
throw Exception(
'Unable to map ${entrypointFileUri} back to file scheme.');
}
// If we don't have a file uri, just use the first library in the
// component.
Library entrypoint = component.libraries.firstWhere(
(Library library) => library.fileUri == entrypointFileUri,
(Library library) =>
library.fileUri == mainUri || library.importUri == mainUri,
orElse: () => null);
if (entrypoint == null) {
throw Exception(
'Could not find entrypoint ${entrypointFileUri} in Component.');
throw Exception('Could not find entrypoint ${mainUri} in Component.');
}
final List<List<Library>> results =
computeStrongComponents(_LibraryGraph(entrypoint, loadedLibraries));
for (List<Library> component in results) {
assert(component.length > 0);
final Uri moduleUri = component.first.fileUri;
final Uri moduleUri = component.first.importUri;
modules[moduleUri] = component;
for (Library componentLibrary in component) {
moduleAssignment[componentLibrary.fileUri] = moduleUri;
moduleAssignment[componentLibrary.importUri] = moduleUri;
}
}
}
// Convert package URI to file URI if it is inside one of the packages.
Future<Uri> _convertToFileUri(
FileSystem fileSystem, Uri uri, Uri packagesUri) async {
if (uri == null || uri.scheme != 'package') {
return uri;
}
// Convert virtual URI to a real file URI.
// String uriString = (await asFileUri(fileSystem, uri)).toString();
List<String> packages;
try {
packages =
await File((await asFileUri(fileSystem, packagesUri)).toFilePath())
.readAsLines();
} on IOException {
// Can't read packages file - silently give up.
return uri;
}
// package:x.y/main.dart -> file:///a/b/x/y/main.dart
for (var line in packages) {
if (line.isEmpty || line.startsWith("#")) {
continue;
}
final colon = line.indexOf(':');
if (colon == -1) {
continue;
}
final packageName = line.substring(0, colon);
if (!uri.path.startsWith('$packageName/')) {
continue;
}
String packagePath;
try {
packagePath = (await asFileUri(
fileSystem, packagesUri.resolve(line.substring(colon + 1))))
.toString();
} on FileSystemException {
// Can't resolve package path.
continue;
}
return Uri.parse(
'$packagePath${uri.path.substring(packageName.length + 1)}');
}
return uri;
}
}
class _LibraryGraph implements Graph<Library> {

View file

@ -13,6 +13,7 @@ dependencies:
front_end: ^0.1.6
kernel: ^0.3.6
args: ^1.4.4
package_resolver: ^1.0.0
dev_dependencies:
test: any
test: any

View file

@ -10,6 +10,7 @@ import 'package:frontend_server/src/strong_components.dart';
import 'package:kernel/ast.dart';
import 'package:kernel/class_hierarchy.dart';
import 'package:kernel/core_types.dart';
import 'package:package_resolver/package_resolver.dart';
import 'package:test/test.dart';
/// Additional indexed types required by the dev_compiler's NativeTypeSet.
@ -72,6 +73,9 @@ void main() {
]),
];
final packageResolver = PackageResolver.config({'a': Uri.file('/pkg/a')});
final multiRootScheme = 'org-dartlang-app';
test('compiles JavaScript code', () async {
final library = Library(
Uri.file('/c.dart'),
@ -85,21 +89,21 @@ void main() {
final strongComponents =
StrongComponents(testComponent, {}, Uri.file('/c.dart'));
strongComponents.computeModules();
final javaScriptBundler =
JavaScriptBundler(testComponent, strongComponents);
final javaScriptBundler = JavaScriptBundler(
testComponent, strongComponents, multiRootScheme, packageResolver);
final manifestSink = _MemorySink();
final codeSink = _MemorySink();
final sourcemapSink = _MemorySink();
final coreTypes = CoreTypes(testComponent);
javaScriptBundler.compile(ClassHierarchy(testComponent, coreTypes),
await javaScriptBundler.compile(ClassHierarchy(testComponent, coreTypes),
coreTypes, {}, codeSink, manifestSink, sourcemapSink);
final Map manifest = json.decode(utf8.decode(manifestSink.buffer));
final String code = utf8.decode(codeSink.buffer);
expect(manifest, {
'/c.dart.js': {
'/c.dart.lib.js': {
'code': [0, codeSink.buffer.length],
'sourcemap': [0, sourcemapSink.buffer.length],
},
@ -107,7 +111,87 @@ void main() {
expect(code, contains('ArbitrarilyChosen'));
// verify source map url is correct.
expect(code, contains('sourceMappingURL=c.dart.js.map'));
expect(code, contains('sourceMappingURL=c.dart.lib.js.map'));
});
test('converts package: uris into /packages/ uris', () async {
var importUri = Uri.parse('package:a/a.dart');
var fileUri = await packageResolver.resolveUri(importUri);
final library = Library(
importUri,
fileUri: fileUri,
procedures: [
Procedure(Name('ArbitrarilyChosen'), ProcedureKind.Method,
FunctionNode(Block([])))
],
);
final testComponent = Component(libraries: [library, ...testCoreLibraries]);
final strongComponents = StrongComponents(testComponent, {}, fileUri);
strongComponents.computeModules();
final javaScriptBundler = JavaScriptBundler(
testComponent, strongComponents, multiRootScheme, packageResolver);
final manifestSink = _MemorySink();
final codeSink = _MemorySink();
final sourcemapSink = _MemorySink();
final coreTypes = CoreTypes(testComponent);
await javaScriptBundler.compile(ClassHierarchy(testComponent, coreTypes),
coreTypes, {}, codeSink, manifestSink, sourcemapSink);
final Map manifest = json.decode(utf8.decode(manifestSink.buffer));
final String code = utf8.decode(codeSink.buffer);
expect(manifest, {
'/packages/a/a.dart.lib.js': {
'code': [0, codeSink.buffer.length],
'sourcemap': [0, sourcemapSink.buffer.length],
},
});
expect(code, contains('ArbitrarilyChosen'));
// verify source map url is correct.
expect(code, contains('sourceMappingURL=a.dart.lib.js.map'));
});
test('multi-root uris create modules relative to the root', () async {
var importUri = Uri.parse('$multiRootScheme:/web/main.dart');
var fileUri = importUri;
final library = Library(
importUri,
fileUri: fileUri,
procedures: [
Procedure(Name('ArbitrarilyChosen'), ProcedureKind.Method,
FunctionNode(Block([])))
],
);
final testComponent = Component(libraries: [library, ...testCoreLibraries]);
final strongComponents = StrongComponents(testComponent, {}, fileUri);
strongComponents.computeModules();
final javaScriptBundler = JavaScriptBundler(
testComponent, strongComponents, multiRootScheme, packageResolver);
final manifestSink = _MemorySink();
final codeSink = _MemorySink();
final sourcemapSink = _MemorySink();
final coreTypes = CoreTypes(testComponent);
await javaScriptBundler.compile(ClassHierarchy(testComponent, coreTypes),
coreTypes, {}, codeSink, manifestSink, sourcemapSink);
final Map manifest = json.decode(utf8.decode(manifestSink.buffer));
final String code = utf8.decode(codeSink.buffer);
expect(manifest, {
'${importUri.path}.lib.js': {
'code': [0, codeSink.buffer.length],
'sourcemap': [0, sourcemapSink.buffer.length],
},
});
expect(code, contains('ArbitrarilyChosen'));
// verify source map url is correct.
expect(code, contains('sourceMappingURL=main.dart.lib.js.map'));
});
test('can combine strongly connected components', () {
@ -134,8 +218,8 @@ void main() {
final strongComponents =
StrongComponents(testComponent, {}, Uri.file('/a.dart'));
strongComponents.computeModules();
final javaScriptBundler =
JavaScriptBundler(testComponent, strongComponents);
final javaScriptBundler = JavaScriptBundler(
testComponent, strongComponents, multiRootScheme, packageResolver);
final manifestSink = _MemorySink();
final codeSink = _MemorySink();
final sourcemapSink = _MemorySink();
@ -153,14 +237,14 @@ void main() {
expect(moduleHeader.allMatches(code), hasLength(2));
// verify source map url is correct.
expect(code, contains('sourceMappingURL=a.dart.js.map'));
expect(code, contains('sourceMappingURL=a.dart.lib.js.map'));
final offsets = manifest['/a.dart.js']['sourcemap'];
final offsets = manifest['/a.dart.lib.js']['sourcemap'];
final sourcemapModuleA = json.decode(
utf8.decode(sourcemapSink.buffer.sublist(offsets.first, offsets.last)));
// verify source maps are pointing at correct source files.
expect(sourcemapModuleA['file'], 'a.dart');
expect(sourcemapModuleA['file'], 'a.dart.lib.js');
});
}

View file

@ -6,6 +6,7 @@ import("../../build/dart/dart_action.gni")
import("../../runtime/runtime_args.gni")
import("../../sdk_args.gni")
import("../application_snapshot.gni")
import("../create_timestamp.gni")
group("kernel-service") {
if (create_kernel_service_snapshot) {
@ -47,9 +48,21 @@ copy("copy_kernel-service_snapshot") {
]
}
create_timestamp_file("frontend_server_files_stamp") {
path = rebase_path("../../pkg/frontend_server/lib")
output = "$target_gen_dir/frontend_server_files.stamp"
}
create_timestamp_file("ddc_files_stamp") {
path = rebase_path("../../pkg/dev_compiler/lib")
output = "$target_gen_dir/ddc_files.stamp"
}
application_snapshot("frontend_server") {
main_dart = "../../pkg/frontend_server/bin/frontend_server_starter.dart"
deps = [
":ddc_files_stamp",
":frontend_server_files_stamp",
"../../runtime/vm:kernel_platform_files($host_toolchain)",
]
sdk_root = rebase_path("$root_out_dir")