[dartdevc] build the ddk sdk with dartdevc

This:
(1) Uses dartdevc and kernel_worker directly to build ddk artifacts.

(2) Generates an outline file instead of a full dill file for ddc_sdk.dill.

(3) Fixes source maps in the shipped sdk so that urls from dart_sdk.js.map
have correct relative paths to dart files in the sdk.  This won't work with
webdev/build as that copies and serves dart_sdk.js, but it will now be
able to build the sdk directly.

Change-Id: I7b9470fe18cac9b4343c7c520fe6ffd7bd9246b4
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/104842
Reviewed-by: Jake Macdonald <jakemac@google.com>
Reviewed-by: Vyacheslav Egorov <vegorov@google.com>
Commit-Queue: Vijay Menon <vsm@google.com>

pkg/dev_compiler/lib/src/compiler/shared_command.dart

@@ -371,7 +371,7 @@ Map placeSourceMap(Map sourceMap, String sourceMapPath,
     sourcePath = sourcePathToUri(p.absolute(p.fromUri(uri))).path;
 
     // Allow bazel mappings to override.
-    var match = bazelMappings[sourcePath];
+    var match = bazelMappings != null ? bazelMappings[sourcePath] : null;
     if (match != null) return match;
 
     // Fall back to a relative path against the source map itself.
@@ -385,7 +385,8 @@ Map placeSourceMap(Map sourceMap, String sourceMapPath,
     list[i] = makeRelative(list[i] as String);
   }
   map['sources'] = list;
-  map['file'] = makeRelative(map['file'] as String);
+  map['file'] =
+      map['file'] != null ? makeRelative(map['file'] as String) : null;
   return map;
 }
 

pkg/dev_compiler/lib/src/kernel/command.dart

@@ -76,7 +76,7 @@ Future<CompilerResult> _compile(List<String> args,
   var argParser = ArgParser(allowTrailingOptions: true)
     ..addFlag('help',
         abbr: 'h', help: 'Display this message.', negatable: false)
-    ..addOption('out', abbr: 'o', help: 'Output file (required).')
+    ..addMultiOption('out', abbr: 'o', help: 'Output file (required).')
     ..addOption('packages', help: 'The package spec file to use.')
     // TODO(jmesserly): is this still useful for us, or can we remove it now?
     ..addFlag('summarize-text',
@@ -95,6 +95,8 @@ Future<CompilerResult> _compile(List<String> args,
         help: 'The directories to search when encountering uris with the '
             'specified multi-root scheme.',
         defaultsTo: [Uri.base.path])
+    ..addOption('multi-root-output-path',
+        help: 'Path to set multi-root files relative to.', hide: true)
     ..addOption('dart-sdk',
         help: '(unsupported with --kernel) path to the Dart SDK.', hide: true)
     ..addFlag('compile-sdk',
@@ -113,12 +115,15 @@ Future<CompilerResult> _compile(List<String> args,
     return CompilerResult(64);
   }
 
-  var output = argResults['out'] as String;
-  if (output == null) {
+  var outPaths = argResults['out'] as List<String>;
+  var moduleFormats = parseModuleFormatOption(argResults);
+  if (outPaths.isEmpty) {
     print('Please specify the output file location. For example:\n'
-        '    -o PATH/TO/OUTPUT_FILE.js'
-        '');
-    print(_usageMessage(argParser));
+        '    -o PATH/TO/OUTPUT_FILE.js');
+    return CompilerResult(64);
+  } else if (outPaths.length != moduleFormats.length) {
+    print('Number of output files (${outPaths.length}) must match '
+        'number of module formats (${moduleFormats.length}).');
     return CompilerResult(64);
   }
 
@@ -136,8 +141,17 @@ Future<CompilerResult> _compile(List<String> args,
   var multiRootPaths = (argResults['multi-root'] as Iterable<String>)
       .map(Uri.base.resolve)
       .toList();
-  var multiRootOutputPath = _longestPrefixingPath(
-      sourcePathToUri(p.absolute(output)), multiRootPaths);
+  var multiRootOutputPath = argResults['multi-root-output-path'] as String;
+  if (multiRootOutputPath == null) {
+    if (outPaths.length > 1) {
+      print(
+          'If multiple output files (found ${outPaths.length}) are specified, '
+          'then --multi-root-output-path must be explicitly provided.');
+      return CompilerResult(64);
+    }
+    multiRootOutputPath = _longestPrefixingPath(
+        sourcePathToUri(p.absolute(outPaths.first)), multiRootPaths);
+  }
 
   var fileSystem = MultiRootFileSystem(
       multiRootScheme, multiRootPaths, fe.StandardFileSystem.instance);
@@ -314,12 +328,15 @@ Future<CompilerResult> _compile(List<String> args,
     return CompilerResult(1, kernelState: compilerState);
   }
 
-  var file = File(output);
-  await file.parent.create(recursive: true);
-
   // Output files can be written in parallel, so collect the futures.
   var outFiles = <Future>[];
   if (argResults['summarize'] as bool) {
+    if (outPaths.length > 1) {
+      print(
+          'If multiple output files (found ${outPaths.length}) are specified, '
+          'the --summarize option is not supported.');
+      return CompilerResult(64);
+    }
     // TODO(jmesserly): CFE mutates the Kernel tree, so we can't save the dill
     // file if we successfully reused a cached library. If compiler state is
     // unchanged, it means we used the cache.
@@ -329,16 +346,22 @@ Future<CompilerResult> _compile(List<String> args,
     if (identical(compilerState, oldCompilerState)) {
       component.unbindCanonicalNames();
     }
-    var sink = File(p.withoutExtension(output) + '.dill').openWrite();
+    var sink = File(p.withoutExtension(outPaths.first) + '.dill').openWrite();
     // TODO(jmesserly): this appears to save external libraries.
     // Do we need to run them through an outlining step so they can be saved?
     kernel.BinaryPrinter(sink).writeComponentFile(component);
     outFiles.add(sink.flush().then((_) => sink.close()));
   }
   if (argResults['summarize-text'] as bool) {
+    if (outPaths.length > 1) {
+      print(
+          'If multiple output files (found ${outPaths.length}) are specified, '
+          'the --summarize-text option is not supported.');
+      return CompilerResult(64);
+    }
     StringBuffer sb = StringBuffer();
     kernel.Printer(sb, showExternal: false).writeComponentFile(component);
-    outFiles.add(File(output + '.txt').writeAsString(sb.toString()));
+    outFiles.add(File(outPaths.first + '.txt').writeAsString(sb.toString()));
   }
   if (hierarchy == null) {
     var target = compilerState.options.target as DevCompilerTarget;
@@ -351,24 +374,28 @@ Future<CompilerResult> _compile(List<String> args,
   var jsModule = compiler.emitModule(
       component, result.inputSummaries, inputSummaries, summaryModules);
 
-  // TODO(jmesserly): support for multiple output formats?
-  //
   // Also the old Analyzer backend had some code to make debugging better when
   // --single-out-file is used, but that option does not appear to be used by
   // any of our build systems.
-  var jsCode = jsProgramToCode(jsModule, options.moduleFormats.first,
-      buildSourceMap: options.sourceMap,
-      inlineSourceMap: options.inlineSourceMap,
-      jsUrl: p.toUri(output).toString(),
-      mapUrl: p.toUri(output + '.map').toString(),
-      bazelMapping: options.bazelMapping,
-      customScheme: multiRootScheme,
-      multiRootOutputPath: multiRootOutputPath);
+  for (var i = 0; i < outPaths.length; ++i) {
+    var output = outPaths[i];
+    var moduleFormat = moduleFormats[i];
+    var file = File(output);
+    await file.parent.create(recursive: true);
+    var jsCode = jsProgramToCode(jsModule, moduleFormat,
+        buildSourceMap: options.sourceMap,
+        inlineSourceMap: options.inlineSourceMap,
+        jsUrl: p.toUri(output).toString(),
+        mapUrl: p.toUri(output + '.map').toString(),
+        bazelMapping: options.bazelMapping,
+        customScheme: multiRootScheme,
+        multiRootOutputPath: multiRootOutputPath);
 
-  outFiles.add(file.writeAsString(jsCode.code));
-  if (jsCode.sourceMap != null) {
-    outFiles.add(
-        File(output + '.map').writeAsString(json.encode(jsCode.sourceMap)));
+    outFiles.add(file.writeAsString(jsCode.code));
+    if (jsCode.sourceMap != null) {
+      outFiles.add(
+          File(output + '.map').writeAsString(json.encode(jsCode.sourceMap)));
+    }
   }
 
   await Future.wait(outFiles);

pkg/dev_compiler/tool/ddb

@@ -92,8 +92,8 @@ void main(List<String> args) async {
   ProcessResult runDdc(String command, List<String> args) {
     if (debug) {
       // Use unbuilt script.  This only works from a source checkout.
-      args.insertAll(0,
-          ['--enable-asserts', p.join(ddcPath, 'bin', '${command}.dart')]);
+      args.insertAll(
+          0, ['--enable-asserts', p.join(ddcPath, 'bin', '${command}.dart')]);
       command = dartBinary;
     } else {
       // Use built snapshot.
@@ -138,18 +138,14 @@ void main(List<String> args) async {
   if (debug) {
     var sdkRoot = p.dirname(p.dirname(ddcPath));
     var buildDir = p.join(sdkRoot, Platform.isMacOS ? 'xcodebuild' : 'out');
-    sdkJsPath = p.join(buildDir, 'ReleaseX64', 'gen', 'utils', 'dartdevc',
-        kernel ? 'kernel' : 'js', mod);
-    requirePath = p.join(sdkRoot, 'third_party', 'requirejs');
-    ddcSdk = p.join(buildDir, 'ReleaseX64', 'gen', 'utils', 'dartdevc',
-        kernel ? p.join('kernel', 'ddc_sdk.dill') : 'ddc_sdk.sum');
-  } else {
-    var suffix = kernel ? p.join('kernel', mod) : mod;
-    sdkJsPath = p.join(dartSdk, 'lib', 'dev_compiler', suffix);
-    requirePath = sdkJsPath;
-    ddcSdk = p.join(
-        dartSdk, 'lib', '_internal', kernel ? 'ddc_sdk.dill' : 'ddc_sdk.sum');
+    dartSdk = p.join(buildDir, 'ReleaseX64', 'dart-sdk');
   }
+  var suffix = kernel ? p.join('kernel', mod) : mod;
+  sdkJsPath = p.join(dartSdk, 'lib', 'dev_compiler', suffix);
+  requirePath = sdkJsPath;
+  ddcSdk = p.join(
+      dartSdk, 'lib', '_internal', kernel ? 'ddc_sdk.dill' : 'ddc_sdk.sum');
+
   ProcessResult result;
   var ddcArgs = [
     if (kernel) '--kernel',

pkg/dev_compiler/tool/kernel_sdk.dart

@@ -89,9 +89,11 @@ Future main(List<String> args) async {
     var format = moduleFormats[name];
     var jsDir = p.join(outputDir, name);
     var jsPath = p.join(jsDir, 'dart_sdk.js');
+    var mapPath = '$jsPath.map';
     await Directory(jsDir).create();
-    var jsCode = jsProgramToCode(jsModule, format);
+    var jsCode = jsProgramToCode(jsModule, format,
+        jsUrl: jsPath, mapUrl: mapPath, buildSourceMap: true);
     await File(jsPath).writeAsString(jsCode.code);
-    await File('$jsPath.map').writeAsString(json.encode(jsCode.sourceMap));
+    await File(mapPath).writeAsString(json.encode(jsCode.sourceMap));
   }
 }

pkg/front_end/lib/src/fasta/kernel/body_builder.dart

@@ -1497,7 +1497,7 @@ abstract class BodyBuilder extends ScopeListener<JumpTarget>
     }
     Name kernelName = new Name(name, library.library);
     List<LocatedMessage> context;
-    if (candidate != null) {
+    if (candidate != null && candidate.location != null) {
       Uri uri = candidate.location.file;
       int offset = candidate.fileOffset;
       Message contextMessage;

pkg/front_end/lib/src/fasta/source/outline_builder.dart

@@ -645,6 +645,9 @@ class OutlineBuilder extends StackListener {
     if (isAbstract) {
       modifiers |= abstractMask;
     }
+    if (nativeMethodName != null) {
+      modifiers |= externalMask;
+    }
     List<MetadataBuilder> metadata = pop();
     checkEmpty(beginToken.charOffset);
     library
@@ -826,6 +829,9 @@ class OutlineBuilder extends StackListener {
       }
     }
     int modifiers = Modifier.validate(pop(), isAbstract: isAbstract);
+    if (nativeMethodName != null) {
+      modifiers |= externalMask;
+    }
     if ((modifiers & externalMask) != 0) {
       modifiers &= ~abstractMask;
     }
@@ -1507,6 +1513,9 @@ class OutlineBuilder extends StackListener {
     int charOffset = pop();
     Object name = pop();
     int modifiers = pop();
+    if (nativeMethodName != null) {
+      modifiers |= externalMask;
+    }
     List<MetadataBuilder> metadata = pop();
     if (name is ParserRecovery) {
       library.endNestedDeclaration("<syntax-error>");

pkg/front_end/testcases/rasta/native_is_illegal.dart.outline.expect

@@ -3,14 +3,9 @@ import self as self;
 import "dart:core" as core;
 
 class Bar extends core::Object {
-  get x() → self::Bar
-    ;
-  set x(self::Bar value) → void
-    ;
-  method f() → dynamic
-    ;
-  static factory •() → self::Bar
-    ;
+  external get x() → self::Bar;
+  external set x(self::Bar value) → void;
+  external method f() → dynamic;
+  external static factory •() → self::Bar;
 }
-static method foo() → dynamic
-  ;
+external static method foo() → dynamic;

sdk/BUILD.gn

@@ -602,7 +602,7 @@ copy("copy_dev_compiler_summary") {
   deps = [
     ":copy_libraries",
     "../utils/dartdevc:dartdevc_sdk",
-    "../utils/dartdevc:dartdevc_kernel_sdk",
+    "../utils/dartdevc:dartdevc_kernel_sdk_outline",
   ]
   gen_dir = get_label_info("../utils/dartdevc:dartdevc_sdk", "target_gen_dir")
   sources = [

utils/dartdevc/BUILD.gn

@@ -26,10 +26,11 @@ application_snapshot("dartdevc") {
   ]
 
   deps = [
-    ":dartdevc_sdk",
+    ":dartdevc_kernel_sdk",
+    ":dartdevc_kernel_sdk_outline",
   ]
 
-  inputs = [ sdk_summary ]
+  inputs = [ sdk_dill ]
 }
 
 sdk_lib_files = exec_script("../../tools/list_dart_files.py",
@@ -245,6 +246,8 @@ prebuilt_dart_action("dartdevc_test_pkg") {
     ":dartdevc_files_stamp",
     ":dartdevc_sdk",
     ":dartdevc_kernel_sdk",
+    ":dartdevc_kernel_sdk_outline",
+    ":dartdevc_kernel_sdk_libraries_json",
     "../../pkg:pkg_files_stamp",
   ]
 
@@ -314,6 +317,51 @@ prebuilt_dart_action("dartdevc_test_pkg") {
   ]
 }
 
+prebuilt_dart_action("dartdevc_kernel_sdk_outline") {
+  deps = [
+    "../../pkg:pkg_files_stamp",
+    ":dartdevc_files_stamp",
+    ":dartdevc_sdk_patch_stamp",
+  ]
+
+  inputs = [
+    "../../utils/bazel/kernel_worker.dart",
+    "$target_gen_dir/dartdevc_files.stamp",
+    "$root_gen_dir/pkg_files.stamp",
+  ]
+
+  outputs = [
+    sdk_dill,
+  ]
+
+  script = "../../utils/bazel/kernel_worker.dart"
+
+  args = [
+    "--summary-only",
+    "--target",
+    "ddc",
+    "--multi-root-scheme",
+    "org-dartlang-sdk",
+    "--multi-root",
+    rebase_path("../../sdk"),
+    "--libraries-file",
+    "file:///" + rebase_path("../../sdk/lib/libraries.json"),
+    "--output",
+    rebase_path(sdk_dill),
+    "--source",
+    "dart:core"
+  ]
+}
+
+copy("dartdevc_kernel_sdk_libraries_json") {
+  sources = [
+    rebase_path("../../sdk/lib/libraries.json"),
+  ]
+  outputs = [
+    sdk_libraries_json,
+  ]
+}
+
 # Compiles the DDC SDK's kernel summary and JS code.
 prebuilt_dart_action("dartdevc_kernel_sdk") {
   deps = [
@@ -323,14 +371,11 @@ prebuilt_dart_action("dartdevc_kernel_sdk") {
   ]
 
   inputs = [
-    "../../pkg/dev_compiler/tool/kernel_sdk.dart",
     "$target_gen_dir/dartdevc_files.stamp",
     "$root_gen_dir/pkg_files.stamp",
   ]
 
   outputs = [
-    sdk_dill,
-    sdk_libraries_json,
     "$target_gen_dir/kernel/amd/dart_sdk.js",
     "$target_gen_dir/kernel/amd/dart_sdk.js.map",
     "$target_gen_dir/kernel/common/dart_sdk.js",
@@ -341,12 +386,36 @@ prebuilt_dart_action("dartdevc_kernel_sdk") {
     "$target_gen_dir/kernel/legacy/dart_sdk.js.map",
   ]
 
-  script = "../../pkg/dev_compiler/tool/kernel_sdk.dart"
+  script = "../../pkg/dev_compiler/bin/dartdevc.dart"
 
   args = [
-    "--output",
-    rebase_path(sdk_dill),
-    "--libraries",
-    rebase_path("../../sdk/lib/libraries.json"),
+    "-k",
+    "--compile-sdk",
+    "dart:core",
+    "--no-summarize",
+    "--multi-root-scheme",
+    "org-dartlang-sdk",
+    "--multi-root",
+    rebase_path("../../sdk"),
+    "--multi-root-output-path",
+    rebase_path("$target_gen_dir/../../"),
+    "--libraries-file",
+    "file:///" + rebase_path("../../sdk/lib/libraries.json"),
+    "--modules",
+    "amd",
+    "-o",
+    rebase_path("$target_gen_dir/kernel/amd/dart_sdk.js"),
+    "--modules",
+    "common",
+    "-o",
+    rebase_path("$target_gen_dir/kernel/common/dart_sdk.js"),
+    "--modules",
+    "es6",
+    "-o",
+    rebase_path("$target_gen_dir/kernel/es6/dart_sdk.js"),
+    "--modules",
+    "legacy",
+    "-o",
+    rebase_path("$target_gen_dir/kernel/legacy/dart_sdk.js"),
   ]
 }