Support test packages in dartdevk in test.dart.

This allows dartdevk to compile tests that import packages like expect.
There are a few pieces to this:

- Add support to build_pkgs.dart to build the kernel summaries for each
  test package (in addition to the analyzer summaries it already
  builds).

- Plumb that through the dartdevc_test target in the GN build as well.

- While we're at it, use GN to build the ddc_sdk.dill file and have
  test.dart load that one instead of the manually built one from calling
  ./tool/kernel_sdk.dart.

- Add command-line arguments to dartdevk for passing in the path to the
  SDK summary and the other summaries to compile against.

- Fix a little typo in processed_options.dart that was preventing it
  from resolving "package:" URIs.

- In test.dart, when compiling a test, link in the summaries for all of
  the test packages.

At runtime, it still uses the JS for those packages generated from the
old analyzer-based front end since the kernel-based compiler isn't
complete enough to compile any of those packages yet.

With all of this, if I change a test to:

  import "package:expect/expect.dart";

  main() {
    Expect.equals("a", "b");
  }

Then it compiles but fails at runtime. The compiler is completing, but
the generated code has some bugs. I don't know enough to fix them
myself, but here's what I've found out:

- In _libraryToModule(), the Library we get from kernel has a null
  fileUri, so this returns an empty string. That in turn means the
  generated JS tries to use "$" as the module name.

  Using this works around it temporarily:

      if (moduleName.isEmpty) moduleName = library.name;

- In _emitTopLevelNameNoInterop(), it doesn't handle the case where the
  NamedNode is a static method on a class. It just generates the library
  and method name, skipping the class, so "Expect.equals(1, 2)" gets
  compiled to "expect.equals(1, 2)" instead of
  "expect.Expect.equals(1, 2)".

Change-Id: I6bd9d98bc9706965160d8fb7cf70b20eeebab3a8
Reviewed-on: https://dart-review.googlesource.com/16687
Commit-Queue: Bob Nystrom <rnystrom@google.com>
Reviewed-by: Vijay Menon <vsm@google.com>

pkg/dev_compiler/lib/src/kernel/command.dart

@@ -24,7 +24,11 @@ import 'native_types.dart';
 /// Returns `true` if the program compiled without any fatal errors.
 Future<bool> compile(List<String> args) async {
   var argParser = new ArgParser(allowTrailingOptions: true)
-    ..addOption('out', abbr: 'o', help: 'Output file (required).');
+    ..addOption('out', abbr: 'o', help: 'Output file (required).')
+    ..addOption('dart-sdk-summary',
+        help: 'The path to the Dart SDK summary file.', hide: true)
+    ..addOption('summary',
+        abbr: 's', help: 'summaries to link to', allowMultiple: true);
 
   addModuleFormatOptions(argParser, singleOutFile: false);
 
@@ -32,17 +36,23 @@ Future<bool> compile(List<String> args) async {
 
   var moduleFormat = parseModuleFormatOption(argResults).first;
   var ddcPath = path.dirname(path.dirname(path.fromUri(Platform.script)));
-  var succeeded = true;
 
+  var summaries =
+      (argResults['summary'] as List<String>).map(Uri.parse).toList();
+
+  var sdkSummaryPath = argResults['dart-sdk-summary'] ??
+      path.absolute(ddcPath, 'lib', 'sdk', 'ddc_sdk.dill');
+
+  var succeeded = true;
   void errorHandler(CompilationMessage error) {
     if (error.severity == Severity.error) succeeded = false;
   }
 
   var options = new CompilerOptions()
-    ..sdkSummary =
-        path.toUri(path.absolute(ddcPath, 'lib', 'sdk', 'ddc_sdk.dill'))
+    ..sdkSummary = path.toUri(sdkSummaryPath)
     ..packagesFileUri =
         path.toUri(path.absolute(ddcPath, '..', '..', '.packages'))
+    ..inputSummaries = summaries
     ..target = new DevCompilerTarget()
     ..onError = errorHandler
     ..reportMessages = true;

pkg/dev_compiler/tool/build_pkgs.dart

@@ -1,13 +1,25 @@
 #!/usr/bin/env dart
+import 'dart:async';
 import 'dart:io';
 
+import 'package:front_end/compilation_message.dart';
+import 'package:front_end/compiler_options.dart';
+import 'package:front_end/summary_generator.dart';
 import 'package:path/path.dart' as p;
 
-import 'package:dev_compiler/src/analyzer/command.dart';
+import 'package:dev_compiler/src/analyzer/command.dart' as analyzer;
+import 'package:dev_compiler/src/kernel/target.dart';
 
 final String scriptDirectory = p.dirname(p.fromUri(Platform.script));
+
+/// The directory that output is written to.
+///
+/// The DDC kernel SDK should be directly in this directory. The resulting
+/// packages will be placed in a "pkg" subdirectory of this.
 String outputDirectory;
 
+String get pkgDirectory => p.join(outputDirectory, "pkg");
+
 /// Compiles the packages that the DDC tests use to JS into the given output
 /// directory. Usage:
 ///
@@ -15,7 +27,7 @@ String outputDirectory;
 ///
 /// If "travis" is passed, builds the all of the modules tested on Travis.
 /// Otherwise, only builds the modules needed by the tests.
-void main(List<String> arguments) {
+Future main(List<String> arguments) async {
   var isTravis = arguments.isNotEmpty && arguments.last == "travis";
   if (isTravis) {
     arguments = arguments.sublist(0, arguments.length - 1);
@@ -27,45 +39,45 @@ void main(List<String> arguments) {
   }
 
   outputDirectory = arguments[0];
-  new Directory(outputDirectory).createSync(recursive: true);
+  new Directory(pkgDirectory).createSync(recursive: true);
 
   // Build leaf packages. These have no other package dependencies.
 
   // Under pkg.
-  compileModule('async_helper');
-  compileModule('expect', libs: ['minitest']);
-  compileModule('js', libs: ['js_util']);
-  compileModule('meta');
+  await compileModule('async_helper');
+  await compileModule('expect', libs: ['minitest']);
+  await compileModule('js', libs: ['js_util']);
+  await compileModule('meta');
   if (isTravis) {
-    compileModule('microlytics', libs: ['html_channels']);
-    compileModule('typed_mock');
+    await compileModule('microlytics', libs: ['html_channels']);
+    await compileModule('typed_mock');
   }
 
   // Under third_party/pkg.
-  compileModule('collection');
-  compileModule('matcher');
-  compileModule('path');
+  await compileModule('collection');
+  await compileModule('path');
   if (isTravis) {
-    compileModule('args', libs: ['command_runner']);
-    compileModule('charcode');
-    compileModule('fixnum');
-    compileModule('logging');
-    compileModule('markdown');
-    compileModule('mime');
-    compileModule('plugin', libs: ['manager']);
-    compileModule('typed_data');
-    compileModule('usage');
-    compileModule('utf');
+    await compileModule('args', libs: ['command_runner']);
+    await compileModule('charcode');
+    await compileModule('fixnum');
+    await compileModule('logging');
+    await compileModule('markdown');
+    await compileModule('mime');
+    await compileModule('plugin', libs: ['manager']);
+    await compileModule('typed_data');
+    await compileModule('usage');
+    await compileModule('utf');
   }
 
   // Composite packages with dependencies.
-  compileModule('stack_trace', deps: ['path']);
+  await compileModule('stack_trace', deps: ['path']);
+  await compileModule('matcher', deps: ['stack_trace']);
   if (isTravis) {
-    compileModule('async', deps: ['collection']);
+    await compileModule('async', deps: ['collection']);
   }
 
   if (!isTravis) {
-    compileModule('unittest', deps: [
+    await compileModule('unittest', deps: [
       'matcher',
       'path',
       'stack_trace'
@@ -79,27 +91,31 @@ void main(List<String> arguments) {
 
 /// Compiles a [module] with a single matching ".dart" library and additional
 /// [libs] and [deps] on other modules.
-void compileModule(String module, {List<String> libs, List<String> deps}) {
+Future compileModule(String module,
+    {List<String> libs, List<String> deps}) async {
+  compileModuleUsingAnalyzer(module, libs, deps);
+  await compileKernelSummary(module, libs, deps);
+}
+
+void compileModuleUsingAnalyzer(
+    String module, List<String> libraries, List<String> dependencies) {
   var sdkSummary = p.join(scriptDirectory, "../lib/sdk/ddc_sdk.sum");
-  var args = [
-    '--dart-sdk-summary=$sdkSummary',
-    '-o${outputDirectory}/$module.js'
-  ];
+  var args = ['--dart-sdk-summary=$sdkSummary', '-o${pkgDirectory}/$module.js'];
 
   // There is always a library that matches the module.
   args.add('package:$module/$module.dart');
 
   // Add any additional libraries.
-  if (libs != null) {
-    for (var lib in libs) {
+  if (libraries != null) {
+    for (var lib in libraries) {
       args.add('package:$module/$lib.dart');
     }
   }
 
   // Add summaries for any modules this depends on.
-  if (deps != null) {
-    for (var dep in deps) {
-      args.add('-s${outputDirectory}/$dep.sum');
+  if (dependencies != null) {
+    for (var dep in dependencies) {
+      args.add('-s${pkgDirectory}/$dep.sum');
     }
   }
 
@@ -112,6 +128,78 @@ void compileModule(String module, {List<String> libs, List<String> deps}) {
         p.join(scriptDirectory, "../test/codegen/async_helper.dart"));
   }
 
-  var exitCode = compile(args);
+  var exitCode = analyzer.compile(args);
   if (exitCode != 0) exit(exitCode);
 }
+
+Future compileKernelSummary(
+    String module, List<String> libraries, List<String> dependencies) async {
+  var succeeded = true;
+
+  void errorHandler(CompilationMessage error) {
+    if (error.severity == Severity.error) succeeded = false;
+  }
+
+  var sdk = p.toUri(p.join(outputDirectory, "ddc_sdk.dill"));
+  print(sdk);
+  var options = new CompilerOptions()
+    ..sdkSummary = sdk
+    ..packagesFileUri = _uriInRepo(".packages")
+    ..strongMode = true
+    ..debugDump = true
+    ..chaseDependencies = true
+    ..onError = errorHandler
+    ..reportMessages = true
+    ..target = new DevCompilerTarget();
+
+  var inputs = <Uri>[];
+
+  if (module == "async_helper") {
+    // TODO(rnystrom): Hack. DDC has its own forked copy of async_helper that
+    // has a couple of differences from pkg/async_helper. We should unfork them,
+    // but I'm not sure how they'll affect the other non-DDC tests. For now,
+    // just use ours.
+    inputs.add(_uriInRepo("pkg/dev_compiler/test/codegen/async_helper.dart"));
+  } else {
+    // There is always a library that matches the module.
+    inputs.add(Uri.parse("package:$module/$module.dart"));
+
+    // Add any other libraries too.
+    if (libraries != null) {
+      for (var lib in libraries) {
+        inputs.add(Uri.parse("package:$module/$lib.dart"));
+      }
+    }
+  }
+
+  // Add summaries for any modules this depends on.
+  if (dependencies != null) {
+    var uris = <Uri>[];
+
+    for (var dep in dependencies) {
+      uris.add(p.toUri(p.absolute(p.join(pkgDirectory, "$dep.dill"))));
+    }
+
+    options.inputSummaries = uris;
+  }
+
+  // Compile the summary.
+  var bytes = await summaryFor(inputs, options);
+  var dillFile = new File(p.join(pkgDirectory, "$module.dill"));
+  if (succeeded) {
+    dillFile.writeAsBytesSync(bytes);
+  } else {
+    // Don't leave the previous version of the file on failure.
+    if (dillFile.existsSync()) dillFile.deleteSync();
+
+    stderr.writeln("Could not generate kernel summary for $module.");
+    exit(1);
+  }
+}
+
+Uri _uriInRepo(String pathInRepo) {
+  // Walk up to repo root.
+  var result = p.join(scriptDirectory, "../../../");
+  result = p.join(result, pathInRepo);
+  return p.toUri(p.absolute(p.normalize(result)));
+}

pkg/dev_compiler/tool/build_pkgs.sh

@@ -2,4 +2,4 @@
 
 # TODO: This script is deprecated in favor of the Dart version. For now, forward
 # to it so existing scripts don't break. Eventually, delete this one.
-./tool/build_pkgs.dart gen/codegen_output/pkg travis
+./tool/build_pkgs.dart gen/codegen_output travis

pkg/dev_compiler/tool/kernel_sdk.dart

@@ -13,8 +13,13 @@ import 'package:path/path.dart' as path;
 
 Future main(List<String> args) async {
   Directory.current = path.dirname(path.dirname(path.fromUri(Platform.script)));
-  var target = new DevCompilerTarget();
 
+  var outputPath = path.absolute('lib/sdk/ddc_sdk.dill');
+  if (args.isNotEmpty) {
+    outputPath = args[0];
+  }
+
+  var target = new DevCompilerTarget();
   var options = new CompilerOptions()
     ..compileSdk = true
     ..chaseDependencies = true
@@ -27,6 +32,5 @@ Future main(List<String> args) async {
 
   // Useful for debugging:
   // writeProgramToText(program);
-  var output = path.absolute('lib/sdk/ddc_sdk.dill');
-  await writeProgramToBinary(program, output);
+  await writeProgramToBinary(program, outputPath);
 }

pkg/front_end/lib/src/base/processed_options.dart

@@ -197,7 +197,7 @@ class ProcessedOptions {
       // TODO(sigmund): consider validating dart/packages uri right after we
       // build the uri translator.
       if (source.scheme != 'dart' &&
-          source.scheme != 'packages' &&
+          source.scheme != 'package' &&
           !await fileSystem.entityForUri(source).exists()) {
         reportWithoutLocation(
             templateInputFileNotFound.withArguments(source), Severity.error);

tools/testing/dart/compiler_configuration.dart

@@ -477,13 +477,30 @@ class DevKernelCompilerConfiguration extends CompilerConfiguration {
       String inputFile, String outputFile, List<String> sharedOptions,
       [Map<String, String> environment = const {}]) {
     var args = sharedOptions.toList();
+
+    var sdkSummary = new Path(_configuration.buildDirectory)
+        .append("/gen/utils/dartdevc/ddc_sdk.dill")
+        .absolute
+        .toNativePath();
+
     args.addAll([
+      "--dart-sdk-summary",
+      sdkSummary,
       "-o",
       outputFile,
       inputFile,
     ]);
 
-    // TODO(rnystrom): Link to dill files for the packages used by tests.
+    // Link to the summaries for the available packages, so that they don't
+    // get recompiled into the test's own module.
+    for (var package in testPackages) {
+      var summary = new Path(_configuration.buildDirectory)
+          .append("/gen/utils/dartdevc/pkg/$package.dill")
+          .absolute
+          .toNativePath();
+      args.add("-s");
+      args.add(summary);
+    }
 
     // Use the directory containing the test as the working directory. This
     // ensures dartdevk creates a short module named based on the test name

utils/dartdevc/BUILD.gn

@@ -87,10 +87,11 @@ dart2js_compile("stack_trace_mapper") {
   out = "$root_out_dir/dev_compiler/build/web/dart_stack_trace_mapper.js"
 }
 
-# Builds everything needed to run dartdevc tests using test.dart.
+# Builds everything needed to run dartdevc and dartdevk tests using test.dart.
 group("dartdevc_test") {
   deps = [
     ":dartdevc",
+    ":dartdevc_sdk_kernel_summary",
     ":dartdevc_test_pkg",
     "../../sdk:create_sdk",
   ]
@@ -107,41 +108,74 @@ compiled_action("dartdevc_test_pkg") {
   tool = "../../runtime/bin:dart"
 
   deps = [
+    ":dartdevc_sdk_kernel_summary",
     ":dartdevc_files_stamp",
-    "../../pkg:pkg_files_stamp",
+    "../../pkg:pkg_files_stamp"
   ]
 
   inputs = [
+    "$target_gen_dir/ddc_sdk.dill",
     "$target_gen_dir/dartdevc_files.stamp",
     "$root_gen_dir/pkg_files.stamp",
   ]
 
   outputs = [
+    "$target_gen_dir/pkg/async_helper.dill",
     "$target_gen_dir/pkg/async_helper.js",
     "$target_gen_dir/pkg/async_helper.sum",
+    "$target_gen_dir/pkg/collection.dill",
     "$target_gen_dir/pkg/collection.js",
     "$target_gen_dir/pkg/collection.sum",
+    "$target_gen_dir/pkg/expect.dill",
     "$target_gen_dir/pkg/expect.js",
     "$target_gen_dir/pkg/expect.sum",
+    "$target_gen_dir/pkg/js.dill",
     "$target_gen_dir/pkg/js.js",
     "$target_gen_dir/pkg/js.sum",
+    "$target_gen_dir/pkg/matcher.dill",
     "$target_gen_dir/pkg/matcher.js",
     "$target_gen_dir/pkg/matcher.sum",
+    "$target_gen_dir/pkg/meta.dill",
     "$target_gen_dir/pkg/meta.js",
     "$target_gen_dir/pkg/meta.sum",
+    "$target_gen_dir/pkg/path.dill",
     "$target_gen_dir/pkg/path.js",
     "$target_gen_dir/pkg/path.sum",
+    "$target_gen_dir/pkg/stack_trace.dill",
     "$target_gen_dir/pkg/stack_trace.js",
     "$target_gen_dir/pkg/stack_trace.sum",
 
     # TODO(rnystrom): Remove this when unittest is no longer used. Also remove
     # any of the above packages that are only here because unittest uses them.
+    "$target_gen_dir/pkg/unittest.dill",
     "$target_gen_dir/pkg/unittest.js",
     "$target_gen_dir/pkg/unittest.sum",
   ]
 
   args = [
     rebase_path("../../pkg/dev_compiler/tool/build_pkgs.dart"),
-    rebase_path("$target_gen_dir/pkg"),
+    rebase_path("$target_gen_dir"),
+  ]
+}
+
+# Compiles the DDC SDK's kernel summary.
+compiled_action("dartdevc_sdk_kernel_summary") {
+  tool = "../../runtime/bin:dart"
+
+  deps = [
+    ":dartdevc_files_stamp",
+  ]
+
+  inputs = [
+    "$target_gen_dir/dartdevc_files.stamp",
+  ]
+
+  outputs = [
+    "$target_gen_dir/ddc_sdk.dill",
+  ]
+
+  args = [
+    rebase_path("../../pkg/dev_compiler/tool/kernel_sdk.dart"),
+    rebase_path("$target_gen_dir/ddc_sdk.dill"),
   ]
 }