[ddc] Fix running d8 with test.py

- Add build targets to compile .dill outlines for common test 
  dependencies using kernel_worker.dart.
- Build ddc modules for common test dependencies and move amd versions
  to a separate subdirectory.

Change-Id: I126b5821f6aea00b753d83560c76f00c6c5edc61
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/350081
Reviewed-by: Sigmund Cherem <sigmund@google.com>
Commit-Queue: Nicholas Shahan <nshahan@google.com>

pkg/test_runner/lib/src/browser.dart

@@ -159,7 +159,7 @@ String ddcHtml(
   var soundNullSafety = mode == NnbdMode.strong;
   var ddcGenDir = '/root_build/$genDir';
   var packagePaths =
-      testPackages.map((p) => '    "$p": "$ddcGenDir/pkg/$p",').join("\n");
+      testPackages.map((p) => '    "$p": "$ddcGenDir/pkg/amd/$p",').join("\n");
   // The native JavaScript Object prototype is sealed before loading the Dart
   // SDK module to guard against prototype pollution.
   return """

pkg/test_runner/lib/src/compiler_configuration.dart

@@ -626,12 +626,19 @@ class DevCompilerConfiguration extends CompilerConfiguration {
 
   /// The output directory under `_configuration.buildDirectory` where DDC build
   /// targets are output.
-  final String genDir;
+  static final String ddcGenDir = 'gen/utils/ddc/';
+
+  /// The subdirectory under [ddcGenDir] where build specific targets are
+  /// output.
+  ///
+  /// To keep outputs separate from each other this directory changes based on
+  /// the canary and null safety modes.
+  final String buildOptionsDir;
 
   DevCompilerConfiguration(super.configuration)
       : _soundNullSafety = configuration.nnbdMode == NnbdMode.strong,
-        genDir = [
-          'gen/utils/ddc/',
+        buildOptionsDir = [
+          ddcGenDir,
           if (configuration.ddcOptions.contains('--canary'))
             'canary'
           else
@@ -704,22 +711,24 @@ class DevCompilerConfiguration extends CompilerConfiguration {
       inputFile,
     ]);
 
-    if (!d8Runtime) {
-      // TODO(sigmund): allow caching of shared packages for DDC modules too.
-      // Link to the summaries for the available packages, so that they don't
-      // get recompiled into the test's own module.
-      for (var package in testPackages) {
-        args.add("-s");
+    for (var package in testPackages) {
+      // The common packages imported from test files are precompiled to
+      // summary .dill and JavaScript files during the SDK build.
+      // The summaries are provided here during the compilation of the test and
+      // the JavaScript will be loaded separately when the test is run.
+      args.add("-s");
+      var outlineFilename = [
+        package,
+        '_outline',
+        if (!_soundNullSafety) '_unsound',
+        '.dill'
+      ].join();
 
-        // Since the summaries for the packages are not near the tests, we give
-        // dartdevc explicit module paths for each one. When the test is run, we
-        // will tell require.js where to find each package's compiled JS.
-        var summary = Path(_configuration.buildDirectory)
-            .append('$genDir/pkg/$package.dill')
-            .absolute
-            .toNativePath();
-        args.add("$summary=$package");
-      }
+      var summary = Path(_configuration.buildDirectory)
+          .append('$ddcGenDir/$outlineFilename')
+          .absolute
+          .toNativePath();
+      args.add("$summary=$package");
     }
 
     var compilerPath = _useSdk
@@ -765,15 +774,23 @@ class DevCompilerConfiguration extends CompilerConfiguration {
       var dartLibraryPath = repositoryUri
           .resolve('pkg/dev_compiler/lib/js/ddc/ddc_module_loader.js')
           .path;
-      var sdkJsDir =
-          Uri.directory(_configuration.buildDirectory).resolve('$genDir/sdk');
+      var sdkJsDir = Uri.directory(_configuration.buildDirectory)
+          .resolve('$buildOptionsDir/sdk/ddc');
+      var pkgJsDir = Uri.directory(_configuration.buildDirectory)
+          .resolve('$buildOptionsDir/pkg/ddc');
       var sdkJsPath = 'dart_sdk.js';
       var libraryName = inputUri.path
           .substring(repositoryUri.path.length)
           .replaceAll("/", "__")
           .replaceAll("-", "_")
           .replaceAll(".dart", "");
-
+      var testPackageLoadStatements = [
+        for (var package in testPackages) 'load("$pkgJsDir/$package.js");'
+      ].join('\n');
+      // Used in the DDC module system for multi-app workflows, and are simply
+      // placeholder values here.
+      var uuid = "00000000-0000-0000-0000-000000000000";
+      var appName = 'test_app';
       // Note: this assumes that d8 is invoked with the dart2js d8.js preamble.
       // TODO(sigmund): to support other runtimes like js-shell, we may want to
       // remove the `load` statements here and instead provide those files
@@ -781,6 +798,7 @@ class DevCompilerConfiguration extends CompilerConfiguration {
       File(runFile).writeAsStringSync('''
         load("$dartLibraryPath");
         load("$sdkJsDir/$sdkJsPath");
+        $testPackageLoadStatements
         load("$outputFile");
 
         let sdk = dart_library.import("dart_sdk");
@@ -792,7 +810,7 @@ class DevCompilerConfiguration extends CompilerConfiguration {
         // Invoke main through the d8 preamble to ensure the code is running
         // within the fake event loop.
         self.dartMainRunner(function () {
-          dart_library.start("$moduleName", "$libraryName");
+          dart_library.start("$appName", "$uuid", "$moduleName", "$libraryName", false);
         });
       '''
           .replaceAll("\n        ", "\n"));

pkg/test_runner/lib/src/test_suite.dart

@@ -977,7 +977,7 @@ class StandardTestSuite extends TestSuite {
             jsDir,
             configuration.compiler,
             configuration.nnbdMode,
-            ddcConfig.genDir,
+            ddcConfig.buildOptionsDir,
             nullAssertions,
             weakNullSafetyErrors);
       } else {

utils/bazel/BUILD.gn

@@ -3,8 +3,14 @@
 # BSD-style license that can be found in the LICENSE file.
 
 import("../application_snapshot.gni")
+import("../create_timestamp.gni")
 
 application_snapshot("kernel_worker") {
   main_dart = "kernel_worker.dart"
   training_args = [ "--help" ]
 }
+
+create_timestamp_file("kernel_worker_files_stamp") {
+  path = rebase_path("kernel_worker.dart")
+  output = "$target_gen_dir/kernel_worker.stamp"
+}

utils/ddc/BUILD.gn

@@ -13,6 +13,7 @@ sdk_outline_unsound_dill = "$root_out_dir/ddc_outline_unsound.dill"
 sdk_full_unsound_dill = "$root_out_dir/ddc_platform_unsound.dill"
 
 sdk_root = "../../sdk"
+pkg_root = "../../pkg"
 
 application_snapshot("dartdevc") {
   main_dart = "../../pkg/dev_compiler/bin/dartdevc.dart"
@@ -135,6 +136,7 @@ group("ddc_stable_test_pkg") {
   deps = [
     ":async_helper_stable_js",
     ":async_helper_stable_unsound_js",
+    ":ddc_test_pkg_outline",
     ":expect_stable_js",
     ":expect_stable_unsound_js",
     ":js_stable_js",
@@ -149,6 +151,7 @@ group("ddc_canary_test_pkg") {
   deps = [
     ":async_helper_canary_js",
     ":async_helper_canary_unsound_js",
+    ":ddc_test_pkg_outline",
     ":expect_canary_js",
     ":expect_canary_unsound_js",
     ":js_canary_js",
@@ -158,6 +161,19 @@ group("ddc_canary_test_pkg") {
   ]
 }
 
+group("ddc_test_pkg_outline") {
+  deps = [
+    ":async_helper_outline",
+    ":async_helper_outline_unsound",
+    ":expect_outline",
+    ":expect_outline_unsound",
+    ":js_outline",
+    ":js_outline_unsound",
+    ":meta_outline",
+    ":meta_outline_unsound",
+  ]
+}
+
 template("ddc_compile") {
   assert(defined(invoker.package),
          "Need 'package' in $target_name (the name of the package)")
@@ -201,6 +217,7 @@ template("ddc_compile") {
     module = invoker.package
     out_dir = rebase_path("$js_gen_dir")
     sdk_path = rebase_path(sdk_outline)
+    pkg_root = rebase_path("$root_build_dir/../../pkg")
 
     deps = [
       ":ddc_files_stamp",
@@ -213,15 +230,22 @@ template("ddc_compile") {
     ]
 
     outputs = [
-      "$js_gen_dir/$module.dill",
-      "$js_gen_dir/$module.js",
+      "$js_gen_dir/amd/$module.js",
+      "$js_gen_dir/amd/$module.js.map",
+      "$js_gen_dir/ddc/$module.js",
+      "$js_gen_dir/ddc/$module.js.map",
     ]
 
     vm_args = [ "-Dsdk_hash=$sdk_hash" ]
 
     args = [
+      "--no-summarize",
+      "--multi-root-output-path=$pkg_root",
       "--dart-sdk-summary=$sdk_path",
-      "-o$out_dir/$module.js",
+      "--modules=amd",
+      "-o$out_dir/amd/$module.js",
+      "--modules=ddc",
+      "-o$out_dir/ddc/$module.js",
       "package:$module/$module.dart",
     ]
 
@@ -254,6 +278,71 @@ template("ddc_compile") {
   }
 }
 
+# Compiles a package to outline .dill file with the "dartdevc" target option.
+template("package_kernel_outline") {
+  assert(defined(invoker.package),
+         "Need 'package' in $target_name (the name of the package)")
+  assert(defined(invoker.sound_null_safety),
+         "Need 'sound_null_safety' in $target_name.")
+
+  module = invoker.package
+  output = "$target_gen_dir/${module}_outline"
+
+  if (invoker.sound_null_safety) {
+    platform_dep = ":ddc_platform"
+    sdk_outline = sdk_outline_dill
+    output = "$output.dill"
+  } else {
+    platform_dep = ":ddc_platform_unsound"
+    sdk_outline = sdk_outline_unsound_dill
+    output = "${output}_unsound.dill"
+  }
+
+  prebuilt_dart_action(target_name) {
+    deps = [
+      "../bazel:kernel_worker_files_stamp",
+      platform_dep,
+    ]
+    script = "../bazel/kernel_worker.dart"
+    outputs = [ output ]
+    args = [
+      "--packages-file",
+      rebase_path("$root_out_dir/../../.dart_tool/package_config.json"),
+      "--summary-only",
+      "--target",
+      "ddc",
+      "--dart-sdk-summary",
+      rebase_path(sdk_outline),
+      "--source",
+      rebase_path("$pkg_root/$module/lib/$module.dart"),
+      "--output",
+      rebase_path(output),
+    ]
+    if (invoker.sound_null_safety) {
+      args += [ "--sound-null-safety" ]
+    } else {
+      args += [ "--no-sound-null-safety" ]
+    }
+    if (defined(invoker.extra_libraries)) {
+      foreach(lib, invoker.extra_libraries) {
+        args += [
+          "--source",
+          rebase_path("$pkg_root/$module/lib/$lib.dart"),
+        ]
+      }
+    }
+  }
+}
+
+package_kernel_outline("async_helper_outline") {
+  package = "async_helper"
+  sound_null_safety = true
+}
+package_kernel_outline("async_helper_outline_unsound") {
+  package = "async_helper"
+  sound_null_safety = false
+}
+
 ddc_compile("async_helper_stable_js") {
   package = "async_helper"
   canary = false
@@ -278,6 +367,18 @@ ddc_compile("async_helper_canary_unsound_js") {
   sound_null_safety = false
 }
 
+package_kernel_outline("expect_outline") {
+  package = "expect"
+  extra_libraries = [ "minitest" ]
+  sound_null_safety = true
+}
+
+package_kernel_outline("expect_outline_unsound") {
+  package = "expect"
+  extra_libraries = [ "minitest" ]
+  sound_null_safety = false
+}
+
 ddc_compile("expect_stable_js") {
   package = "expect"
   extra_libraries = [ "minitest" ]
@@ -306,6 +407,18 @@ ddc_compile("expect_canary_unsound_js") {
   sound_null_safety = false
 }
 
+package_kernel_outline("js_outline") {
+  package = "js"
+  extra_libraries = [ "js_util" ]
+  sound_null_safety = true
+}
+
+package_kernel_outline("js_outline_unsound") {
+  package = "js"
+  extra_libraries = [ "js_util" ]
+  sound_null_safety = false
+}
+
 ddc_compile("js_stable_js") {
   package = "js"
   extra_libraries = [ "js_util" ]
@@ -333,6 +446,16 @@ ddc_compile("js_canary_unsound_js") {
   sound_null_safety = false
 }
 
+package_kernel_outline("meta_outline") {
+  package = "meta"
+  sound_null_safety = true
+}
+
+package_kernel_outline("meta_outline_unsound") {
+  package = "meta"
+  sound_null_safety = false
+}
+
 ddc_compile("meta_stable_js") {
   package = "meta"
   canary = false