[dart2wasm] Build platform dill and compile dart2wasm to AOT snapshots

This adds a --platform= option to dart2wasm to read the SDK libraries
from that dill file instead of compiling them from source every time.
If the option is not given, the SDK libraries are compiled, like before.

Also adds a "dart2wasm" build target, which will build the dart2wasm
platform dill and compile dart2wasm to two AOT snapshots (with and
without asserts). The dart2wasm scripts in sdk/bin are updated to run
via these snapshots and use this platform dill.

This speeds up test runs for the dart2wasm-hostasserts-linux-x64-d8
configuration by approximately 45x.

Change-Id: If2c7750a6eb39725310745f887792784d0dfc583
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/243624
Commit-Queue: Aske Simon Christensen <askesc@google.com>
Reviewed-by: William Hesse <whesse@google.com>
Reviewed-by: Joshua Litt <joshualitt@google.com>

BUILD.gn

@@ -103,6 +103,15 @@ group("dart2js") {
   deps = [ "utils/compiler:dart2js" ]
 }
 
+group("dart2wasm") {
+  deps = [
+    ":runtime_precompiled",
+    "utils/dart2wasm:compile_dart2wasm_platform",
+    "utils/dart2wasm:dart2wasm_asserts_snapshot",
+    "utils/dart2wasm:dart2wasm_snapshot",
+  ]
+}
+
 group("dartanalyzer") {
   deps = [ "utils/dartanalyzer" ]
 }

pkg/_fe_analyzer_shared/lib/src/messages/codes_generated.dart

@@ -4124,7 +4124,7 @@ const MessageCode messageFastaUsageLong =
     Read the SDK platform from <file>, which should be in Dill/Kernel IR format
     and contain the Dart SDK.
 
-  --target=dart2js|dart2js_server|dart_runner|dartdevc|flutter|flutter_runner|none|vm
+  --target=dart2js|dart2js_server|dart2wasm|dart_runner|dartdevc|flutter|flutter_runner|none|vm
     Specify the target configuration.
 
   --enable-asserts

pkg/dart2wasm/bin/dart2wasm.dart

@@ -40,6 +40,7 @@ Never usage(String message) {
   print("");
   print("Options:");
   print("  --dart-sdk=<path>");
+  print("  --platform=<path>");
   print("");
   for (String option in boolOptionMap.keys) {
     print("  --[no-]$option");
@@ -55,6 +56,7 @@ Never usage(String message) {
 
 Future<int> main(List<String> args) async {
   Uri sdkPath = Platform.script.resolve("../../../sdk");
+  Uri? platformPath = null;
   TranslatorOptions options = TranslatorOptions();
   List<String> nonOptions = [];
   void Function(TranslatorOptions, int)? intOptionFun = null;
@@ -65,6 +67,9 @@ Future<int> main(List<String> args) async {
     } else if (arg.startsWith("--dart-sdk=")) {
       String path = arg.substring("--dart-sdk=".length);
       sdkPath = Uri.file(Directory(path).absolute.path);
+    } else if (arg.startsWith("--platform=")) {
+      String path = arg.substring("--platform=".length);
+      platformPath = Uri.file(Directory(path).absolute.path);
     } else if (arg.startsWith("--no-")) {
       var optionFun = boolOptionMap[arg.substring(5)];
       if (optionFun == null) usage("Unknown option $arg");
@@ -95,8 +100,8 @@ Future<int> main(List<String> args) async {
   String output = nonOptions[1];
   Uri mainUri = resolveInputUri(input);
 
-  Uint8List? module = await compileToModule(mainUri, sdkPath, options,
+  Uint8List? module = await compileToModule(mainUri, sdkPath, platformPath,
-      (message) => printDiagnosticMessage(message, print));
+      options, (message) => printDiagnosticMessage(message, print));
 
   if (module == null) {
     exitCode = 1;

pkg/dart2wasm/dart2wasm.md

@@ -11,6 +11,7 @@ where *options* include:
 | Option                                  | Default | Description |
 | --------------------------------------- | ------- | ----------- |
 | `--dart-sdk=`*path*                     | relative to script | The location of the `sdk` directory inside the Dart SDK, containing the core library sources.
+| `--platform=`*path*                     | none    | The location of the platform `dill` file containing the compiled core libraries.
 | `--`[`no-`]`export-all`                 | no      | Export all functions; otherwise, just export `main`.
 | `--`[`no-`]`import-shared-memory`       | no      | Import a shared memory buffer. If this is on, `--shared-memory-max-pages` must also be specified.
 | `--`[`no-`]`inlining`                   | no      | Inline small functions.

pkg/dart2wasm/lib/compile.dart

@@ -33,6 +33,7 @@ import 'package:dart2wasm/translator.dart';
 Future<Uint8List?> compileToModule(
     Uri mainUri,
     Uri sdkRoot,
+    Uri? platformDill,
     TranslatorOptions options,
     void Function(DiagnosticMessage) handleDiagnosticMessage) async {
   var succeeded = true;
@@ -46,13 +47,18 @@ Future<Uint8List?> compileToModule(
   Target target = WasmTarget();
   CompilerOptions compilerOptions = CompilerOptions()
     ..target = target
-    ..compileSdk = true
     ..sdkRoot = sdkRoot
     ..environmentDefines = {}
     ..verbose = false
     ..onDiagnostic = diagnosticMessageHandler
     ..nnbdMode = NnbdMode.Strong;
 
+  if (platformDill != null) {
+    compilerOptions.sdkSummary = platformDill;
+  } else {
+    compilerOptions.compileSdk = true;
+  }
+
   CompilerResult? compilerResult =
       await kernelForProgram(mainUri, compilerOptions);
   if (compilerResult == null || !succeeded) {

pkg/dart2wasm/lib/target.dart

@@ -51,6 +51,8 @@ class WasmTarget extends Target {
         'dart:nativewrappers',
         'dart:js_util_wasm',
         'dart:js_wasm',
+        'dart:wasm',
+        'dart:developer',
       ];
 
   @override

pkg/front_end/messages.yaml

@@ -1964,7 +1964,7 @@ FastaUsageLong:
         Read the SDK platform from <file>, which should be in Dill/Kernel IR format
         and contain the Dart SDK.
 
-      --target=dart2js|dart2js_server|dart_runner|dartdevc|flutter|flutter_runner|none|vm
+      --target=dart2js|dart2js_server|dart2wasm|dart_runner|dartdevc|flutter|flutter_runner|none|vm
         Specify the target configuration.
 
       --enable-asserts

pkg/front_end/pubspec.yaml

@@ -23,6 +23,8 @@ dev_dependencies:
     path: ../build_integration
   compiler:
     path: ../compiler
+  dart2wasm:
+    path: ../dart2wasm
   dart_style: ^2.0.0
   dev_compiler:
     path: ../dev_compiler

pkg/front_end/test/spell_checking_list_common.txt

@@ -723,6 +723,7 @@ cycles
 cyclic
 dart
 dart2js
+dart2wasm
 dartdevc
 data
 date
@@ -3339,6 +3340,7 @@ warn
 warning
 warnings
 was
+wasm
 wasn't
 way
 ways

pkg/front_end/tool/_fasta/additional_targets.dart

@@ -10,6 +10,8 @@ import 'package:compiler/src/kernel/dart2js_target.dart' show Dart2jsTarget;
 
 import 'package:dev_compiler/src/kernel/target.dart' show DevCompilerTarget;
 
+import 'package:dart2wasm/target.dart' show WasmTarget;
+
 import 'package:vm/target/install.dart' as vm_target_install
     show installAdditionalTargets;
 
@@ -21,5 +23,6 @@ void installAdditionalTargets() {
   targets["dart2js_server"] =
       (TargetFlags flags) => new Dart2jsTarget("dart2js_server", flags);
   targets["dartdevc"] = (TargetFlags flags) => new DevCompilerTarget(flags);
+  targets["dart2wasm"] = (TargetFlags flags) => new WasmTarget();
   vm_target_install.installAdditionalTargets();
 }

sdk/bin/dart2wasm

@@ -19,21 +19,39 @@ function follow_links() {
 PROG_NAME="$(follow_links "$BASH_SOURCE")"
 
 # Handle the case where dart-sdk/bin has been symlinked to.
-BIN_DIR="$(cd "${PROG_NAME%/*}" ; pwd -P)"
+PROG_DIR="$(cd "${PROG_NAME%/*}" ; pwd -P)"
-SDK_DIR="$(cd "${BIN_DIR}/.." ; pwd -P)"
+SDK_DIR="$(cd "${PROG_DIR}/../.." ; pwd -P)"
 
+# Locate build directory, containing executables, snapshots and platform dill.
+if [[ `uname` == 'Darwin' ]]; then
+  OUT_DIR="$SDK_DIR/xcodebuild"
+else
+  OUT_DIR="$SDK_DIR/out"
+fi
+DART_CONFIGURATION=${DART_CONFIGURATION:-ReleaseX64}
+BIN_DIR="$OUT_DIR/$DART_CONFIGURATION"
+
+DART_PRECOMPILED_RUNTIME="$BIN_DIR/dart_precompiled_runtime"
+
+# Point to SDK directory.
 SDK_ARG="--dart-sdk=$SDK_DIR"
 
-DART="$BIN_DIR/dart"
+# Point to built platform dill.
+PLATFORM="$BIN_DIR/dart2wasm_platform.dill"
+PLATFORM_ARG="--platform=$PLATFORM"
 
 unset EXTRA_VM_OPTIONS
 declare -a EXTRA_VM_OPTIONS
 
+# Choose snapshot with or without asserts enabled.
+SNAPSHOT_NAME="dart2wasm"
 case $0 in
   *_developer)
     EXTRA_VM_OPTIONS+=('--enable_asserts')
+    SNAPSHOT_NAME="dart2wasm_asserts"
     ;;
 esac
+SNAPSHOT="$BIN_DIR/$SNAPSHOT_NAME.snapshot"
 
 # We allow extra vm options to be passed in through an environment variable.
 if [[ $DART_VM_OPTIONS ]]; then
@@ -43,6 +61,4 @@ fi
 
 DART_ROOT="$(cd "${SDK_DIR}/.." ; pwd -P)"
 
-DART2WASM_COMPILER="$DART_ROOT/pkg/dart2wasm/bin/dart2wasm.dart"
+exec "$DART_PRECOMPILED_RUNTIME" "--packages=$DART_ROOT/.packages" "${EXTRA_VM_OPTIONS[@]}" "$SNAPSHOT" "$SDK_ARG" "$PLATFORM_ARG" "$@"
-
-exec "$DART" "--packages=$DART_ROOT/.packages" "${EXTRA_VM_OPTIONS[@]}" "$DART2WASM_COMPILER" "$SDK_ARG" "$@"

tools/bots/test_matrix.json

@@ -3026,7 +3026,8 @@
           "name": "build dart",
           "script": "tools/build.py",
           "arguments": [
-            "runtime"
+            "runtime",
+            "dart2wasm"
           ]
         },
         {

utils/aot_snapshot.gni

@@ -0,0 +1,118 @@
+# Copyright (c) 2022, the Dart project authors.  Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+import("../build/dart/dart_action.gni")
+import("../sdk_args.gni")
+
+_dart_root = get_path_info("..", "abspath")
+
+template("aot_snapshot") {
+  assert(defined(invoker.main_dart), "Must specify 'main_dart'")
+  gen_kernel_args = []
+  if (defined(invoker.gen_kernel_args)) {
+    gen_kernel_args = invoker.gen_kernel_args
+  }
+  gen_snapshot_args = []
+  if (defined(invoker.gen_snapshot_args)) {
+    gen_snapshot_args = invoker.gen_snapshot_args
+  }
+  main_dart = invoker.main_dart
+  name = target_name
+  if (defined(invoker.name)) {
+    name = invoker.name
+  }
+  extra_deps = []
+  if (defined(invoker.deps)) {
+    extra_deps += invoker.deps
+  }
+  extra_inputs = [ main_dart ]
+  if (defined(invoker.inputs)) {
+    extra_inputs += invoker.inputs
+  }
+  if (defined(invoker.dot_packages)) {
+    dot_packages = invoker.dot_packages
+  } else {
+    dot_packages = rebase_path("$_dart_root/.dart_tool/package_config.json")
+  }
+  output = "$root_out_dir/$name.snapshot"
+  if (defined(invoker.output)) {
+    output = invoker.output
+  }
+
+  dill = "$target_gen_dir/$name.dart.dill"
+
+  # Build the kernel file using the prebuilt VM to speed up the debug and
+  # simulator builds.
+  prebuilt_dart_action(target_name + "_dill") {
+    if (defined(invoker.pool)) {
+      pool = invoker.pool
+    }
+    deps = extra_deps + [
+             "$_dart_root/runtime/vm:kernel_platform_files($host_toolchain)",
+             "$_dart_root/runtime/vm:vm_platform",
+             "$_dart_root/utils/gen_kernel:bootstrap_gen_kernel",
+           ]
+    gen_kernel_kernel =
+        get_label_info("$_dart_root/utils/gen_kernel:bootstrap_gen_kernel",
+                       "target_gen_dir") + "/bootstrap_gen_kernel.dill"
+    platform_dill = "$root_out_dir/vm_platform_strong.dill"
+
+    inputs = extra_inputs + [
+               gen_kernel_kernel,
+               platform_dill,
+               main_dart,
+               dot_packages,
+             ]
+    output = dill
+    outputs = [ output ]
+
+    depfile = "$output.d"
+
+    vm_args = [
+      # Ensure gen_kernel.dart will use this SDK hash when consuming/producing kernel.
+      "-Dsdk_hash=$sdk_hash",
+    ]
+
+    script = gen_kernel_kernel
+
+    args = [
+      "--packages=" + rebase_path(dot_packages),
+      "--platform=" + rebase_path(platform_dill),
+      "--aot",
+      "--output=" + rebase_path(output, root_build_dir),
+      "--depfile=" + rebase_path(depfile),
+
+      # Ensure the compiled appliation (e.g. kernel-service, frontend-server,
+      # ...) will use this SDK hash when consuming/producing kernel.
+      #
+      # (Instead of ensuring every user of the "application_snapshot" /
+      # "kernel_snapshot" passes this if needed, we always pass it)
+      "-Dsdk_hash=$sdk_hash",
+    ]
+    args += gen_kernel_args
+    args += [ rebase_path(main_dart) ]
+  }
+
+  # Create a snapshot from kernel built above.
+  gen_snapshot_action(target_name) {
+    if (defined(invoker.pool)) {
+      pool = invoker.pool
+    }
+    deps = extra_deps + [ ":${target_name}_dill" ]
+
+    inputs = extra_inputs
+
+    outputs = [ output ]
+
+    abs_output = rebase_path(output)
+
+    vm_args = [
+                "--deterministic",
+                "--snapshot-kind=app-aot-elf",
+                "--elf=$abs_output",
+              ] + gen_snapshot_args
+
+    args = [ rebase_path(dill) ]
+  }
+}

utils/dart2wasm/BUILD.gn

@@ -0,0 +1,38 @@
+# Copyright (c) 2022, the Dart project authors.  Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+import("../aot_snapshot.gni")
+import("../compile_platform.gni")
+
+sdk_root = "../../sdk"
+
+aot_snapshot("dart2wasm_snapshot") {
+  main_dart = "../../pkg/dart2wasm/bin/dart2wasm.dart"
+  name = "dart2wasm"
+}
+
+aot_snapshot("dart2wasm_asserts_snapshot") {
+  main_dart = "../../pkg/dart2wasm/bin/dart2wasm.dart"
+  name = "dart2wasm_asserts"
+  gen_kernel_args = [ "--enable-asserts" ]
+  gen_snapshot_args = [ "--enable-asserts" ]
+}
+
+compile_platform("compile_dart2wasm_platform") {
+  single_root_scheme = "org-dartlang-sdk"
+  single_root_base = rebase_path("$sdk_root/")
+  libraries_specification_uri = "org-dartlang-sdk:///lib/libraries.json"
+
+  outputs = [
+    "$root_out_dir/dart2wasm_platform.dill",
+    "$root_out_dir/dart2wasm_outline.dill",
+  ]
+
+  args = [
+    "--target=dart2wasm",
+    "--no-defines",
+    "dart:core",
+    "--nnbd-strong",
+  ]
+}