[tools] Allow precompiling gen_kernel and compile_platform

When iterating on core library changes or changes in the AOT compiler
many seconds are wasted waiting on gen_kernel/compile_platform to
parse Dart code. This happens because we are running these tools
from sources on prebuilt Dart SDK.

This CL allows SDK developer to opt-in into AOT compiling these
tools by adding `precompile_tools=true` to their DART_GN_ARGS.

AOT compilation is performed using prebuilt SDK - so these
executables do not need to be recompiled if core libraries or
VM changes reducing iteration cycles.

pkg/vm/tool/precompiler2 is tweaked to detect when DART_GN_ARGS
contains `precompile_tools=true` and use precompiled
gen_kernel.exe instead of running it from source.

Using precompiled compile_platform takes vm_platform_strong.dill
build from 20 seconds to 3 seconds.

Using precompiled gen_kernel takes small benchmark build from
~10 seconds to 2 seconds.

This relands 5cda2a871c with fixes
for Flutter build.

TEST=manually tested

Change-Id: I552861c80c152890655e41baaf6ea3fb3b03a57e
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/367961
Reviewed-by: Martin Kustermann <kustermann@google.com>
Commit-Queue: Slava Egorov <vegorov@google.com>

BUILD.gn

@@ -150,6 +150,13 @@ group("analysis_server") {
   deps = [ "utils/analysis_server" ]
 }
 
+group("tools") {
+  deps = [
+    "utils:compile_platform.exe",
+    "utils:gen_kernel.exe",
+  ]
+}
+
 # This is the target that is built on the dart2js build bots.
 # It must depend on anything that is required by the dart2js
 # test suites.

build/gn_dart_compile_exe.py

@@ -0,0 +1,151 @@
+#!/usr/bin/env python3
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""Helper script for GN to run `dart compile exe` and produce a depfile.
+
+Run with:
+  python3 gn_dart_compile_exe.py             \
+    --dart-binary <path to dart binary>      \
+    --entry-point <path to dart entry point>       \
+    --output <path to resulting executable>  \
+    --sdk-hash <SDK hash>                    \
+    --packages <path to package config file> \
+    --depfile <path to depfile to write>
+
+This is workaround for `dart compile exe` not supporting --depfile option
+in the current version of prebuilt SDK. Once we roll in a new version
+of checked in SDK we can remove this helper.
+"""
+
+import argparse
+import os
+import sys
+import subprocess
+from tempfile import TemporaryDirectory
+
+
+def parse_args(argv):
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--dart-sdk",
+                        required=True,
+                        help="Path to the prebuilt Dart SDK")
+    parser.add_argument("--sdk-hash", required=True, help="SDK hash")
+    parser.add_argument("--entry-point",
+                        required=True,
+                        help="Dart entry point to precompile")
+    parser.add_argument("--output",
+                        required=True,
+                        help="Path to resulting executable   ")
+    parser.add_argument("--packages",
+                        required=True,
+                        help="Path to package config file")
+    parser.add_argument("--depfile",
+                        required=True,
+                        help="Path to depfile to write")
+    return parser.parse_args(argv)
+
+
+# Run a command, swallowing the output unless there is an error.
+def run_command(command):
+    try:
+        subprocess.check_output(command, stderr=subprocess.STDOUT)
+        return True
+    except subprocess.CalledProcessError as e:
+        print("Command failed: " + " ".join(command) + "\n" + "output: " +
+              _decode(e.output))
+        return False
+    except OSError as e:
+        print("Command failed: " + " ".join(command) + "\n" + "output: " +
+              _decode(e.strerror))
+        return False
+
+
+def _decode(bytes):
+    return bytes.decode("utf-8")
+
+
+def main(argv):
+    args = parse_args(argv[1:])
+
+    # Unless the path is absolute, this script is designed to run binaries
+    # produced by the current build, which is the current working directory when
+    # this script is run.
+    prebuilt_sdk = os.path.abspath(args.dart_sdk)
+
+    dart_binary = os.path.join(prebuilt_sdk, "bin", "dart")
+    if not os.path.isfile(dart_binary):
+        print("Binary not found: " + dart_binary)
+        return 1
+
+    dartaotruntime_binary = os.path.join(prebuilt_sdk, "bin", "dartaotruntime")
+    if not os.path.isfile(dartaotruntime_binary):
+        print("Binary not found: " + dartaotruntime_binary)
+        return 1
+
+    gen_kernel_snapshot = os.path.join(prebuilt_sdk, "bin", "snapshots",
+                                       "gen_kernel_aot.dart.snapshot")
+    if not os.path.isfile(gen_kernel_snapshot):
+        print("Binary not found: " + gen_kernel_snapshot)
+        return 1
+
+    platform_dill = os.path.join(prebuilt_sdk, "lib", "_internal",
+                                 "vm_platform_strong.dill")
+    if not os.path.isfile(platform_dill):
+        print("Binary not found: " + platform_dill)
+        return 1
+
+    # Compile the executable.
+    ok = run_command([
+        dart_binary,
+        "compile",
+        "exe",
+        "--packages",
+        args.packages,
+        f"-Dsdk_hash={args.sdk_hash}",
+        "-o",
+        args.output,
+        args.entry_point,
+    ])
+    if not ok:
+        return 1
+
+    # Collect dependencies by using gen_kernel.
+    with TemporaryDirectory() as tmpdir:
+        output_dill = os.path.join(tmpdir, "output.dill")
+        ok = run_command([
+            dartaotruntime_binary,
+            gen_kernel_snapshot,
+            "--platform",
+            platform_dill,
+            "--packages",
+            args.packages,
+            "--depfile",
+            args.depfile,
+            "-o",
+            output_dill,
+            args.entry_point,
+        ])
+        if not ok:
+            return 1
+
+        # Fix generated depfile to refer to the output file name instead
+        # of referring to the temporary dill file we have generated.
+        with open(args.depfile, "r") as f:
+            content = f.read()
+        (target_name, deps) = content.split(": ", 1)
+        if target_name != output_dill:
+            print(
+                "ERROR: Something is wrong with generated depfile: expected {output_dill} as target, but got {target_name}"
+            )
+            return 1
+        with open(args.depfile, "w") as f:
+            f.write(args.output)
+            f.write(": ")
+            f.write(deps)
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))

pkg/front_end/tool/_fasta/entry_points.dart

@@ -476,6 +476,11 @@ Future<void> compilePlatformInternal(CompilerContext c, Uri fullOutput,
 }
 
 Future<List<Uri>> computeHostDependencies(Uri hostPlatform) {
+  // Do not try to parse compile_platform if it was precompiled into a binary.
+  if (!Platform.script.toFilePath().endsWith('.dart')) {
+    return Future.value([]);
+  }
+
   // Returns a list of source files that make up the Fasta compiler (the files
   // the Dart VM reads to run Fasta). Until Fasta is self-hosting (in strong
   // mode), this is only an approximation, albeit accurate.  Once Fasta is

pkg/vm/tool/precompiler2

@@ -14,6 +14,23 @@
 
 set -e
 
+function follow_links() {
+  file="$1"
+  while [ -h "$file" ]; do
+    # On Mac OS, readlink -f doesn't work.
+    file="$(readlink "$file")"
+  done
+  echo "$file"
+}
+
+# Unlike $0, $BASH_SOURCE points to the absolute path of this file.
+PROG_NAME="$(follow_links "$BASH_SOURCE")"
+
+# Handle the case where dart-sdk/bin has been symlinked to.
+CUR_DIR="$(cd "${PROG_NAME%/*}" ; pwd -P)"
+
+SDK_DIR="$CUR_DIR/../../.."
+
 OPTIONS=()
 GEN_KERNEL_OPTIONS=()
 PACKAGES=
@@ -22,6 +39,9 @@ BUILD_ASM=0
 ARGV=()
 for arg in "$@"; do
   case $arg in
+    --packages=sdk)
+    PACKAGES="$SDK_DIR/.dart_tool/package_config.json"
+    ;;
     --packages=*)
     PACKAGES="$arg"
     ;;
@@ -71,50 +91,47 @@ else
   GEN_SNAPSHOT_FILENAME="--elf=${SNAPSHOT_FILE}"
 fi
 
-function follow_links() {
-  file="$1"
-  while [ -h "$file" ]; do
-    # On Mac OS, readlink -f doesn't work.
-    file="$(readlink "$file")"
-  done
-  echo "$file"
-}
-
-# Unlike $0, $BASH_SOURCE points to the absolute path of this file.
-PROG_NAME="$(follow_links "$BASH_SOURCE")"
-
-# Handle the case where dart-sdk/bin has been symlinked to.
-CUR_DIR="$(cd "${PROG_NAME%/*}" ; pwd -P)"
-
-SDK_DIR="$CUR_DIR/../../.."
-
 if [[ `uname` == 'Darwin' ]]; then
   OUT_DIR="$SDK_DIR/xcodebuild"
 else
   OUT_DIR="$SDK_DIR/out"
 fi
 
-export DART_CONFIGURATION=${DART_CONFIGURATION:-ReleaseX64}
-BIN_DIR="$OUT_DIR/$DART_CONFIGURATION"
+HOST_ARCH="X64"
+if [[ `uname -m` == 'arm64' ]]; then
+  HOST_ARCH="ARM64"
+fi
+
+export DART_CONFIGURATION=${DART_CONFIGURATION:-Release$HOST_ARCH}
+BUILD_DIR="$OUT_DIR/$DART_CONFIGURATION"
 
 DART="${SDK_DIR}/tools/sdks/dart-sdk/bin/dart"
 if [ ! -f "$DART" ]; then
-  DART="$BIN_DIR/dart"
+  DART="$BUILD_DIR/dart"
 fi
 
+function gen_kernel() {
+  if [[ "$DART_GN_ARGS" == *"precompile_tools=true"* ]]; then
+    # Precompile gen_kernel to an AOT app.
+    ninja -C "$BUILD_DIR" gen_kernel.exe
+    "$BUILD_DIR/gen_kernel.exe" $@
+  else
+    $DART ${DART_VM_FLAGS} "${SDK_DIR}/pkg/vm/bin/gen_kernel.dart" $@
+  fi
+}
+
 # Step 1: Generate Kernel binary from the input Dart source.
-$DART                                                                          \
-     ${DART_VM_FLAGS}                                                          \
-     "${SDK_DIR}/pkg/vm/bin/gen_kernel.dart"                                   \
-     --platform "${BIN_DIR}/vm_platform_strong.dill"                           \
+gen_kernel --platform "${BUILD_DIR}/vm_platform_strong.dill"                   \
      --aot                                                                     \
      "${GEN_KERNEL_OPTIONS[@]}"                                                \
      $PACKAGES                                                                 \
      -o "$SNAPSHOT_FILE.dill"                                                  \
      "$SOURCE_FILE"
 
+
+
 # Step 2: Generate snapshot from the Kernel binary.
-"$BIN_DIR"/gen_snapshot                                                        \
+"$BUILD_DIR"/gen_snapshot                                                      \
      ${GEN_SNAPSHOT_FLAGS}                                                     \
      "$GEN_SNAPSHOT_OPTION"                                                    \
      "$GEN_SNAPSHOT_FILENAME"                                                  \

sdk_args.gni

@@ -32,6 +32,12 @@ declare_args() {
 
   # The location in the build output directory of the built Dart SDK.
   dart_sdk_output = "dart-sdk"
+
+  # When set to `true` will cause compile_platform action to use a precompiled
+  # compile_platform.dart script instead of running it from source. This
+  # can significantly improve iteration time when iteration on changes in
+  # core libraries.
+  precompile_tools = false
 }
 
 if (default_git_folder == "") {

utils/BUILD.gn

@@ -0,0 +1,59 @@
+# Copyright (c) 2024, the Dart project authors.  Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+import("../build/dart/dart_action.gni")
+import("../sdk_args.gni")
+
+_dart_root = get_path_info("..", "abspath")
+
+template("aot_compile_using_prebuilt_sdk") {
+  action(target_name) {
+    forward_variables_from(invoker,
+                           [
+                             "deps",
+                             "pool",
+                             "testonly",
+                             "visibility",
+                           ])
+
+    script = "$_dart_root/build/gn_dart_compile_exe.py"
+
+    inputs = [
+      invoker.entry_point,
+      invoker.package_config,
+    ]
+
+    outputs = [ invoker.output ]
+
+    depfile = invoker.output + ".d"
+
+    # TODO(vegorov): support RBE by using rewrapper script.
+    args = [
+      "--dart-sdk",
+      rebase_path("$_dart_root/tools/sdks/dart-sdk", root_build_dir),
+      "--sdk-hash",
+      "$sdk_hash",
+      "--entry-point",
+      rebase_path(invoker.entry_point, root_build_dir),
+      "--output",
+      rebase_path(invoker.output, root_build_dir),
+      "--packages",
+      rebase_path(invoker.package_config, root_build_dir),
+      "--depfile",
+      rebase_path(depfile, root_build_dir),
+    ]
+  }
+}
+
+aot_compile_using_prebuilt_sdk("compile_platform.exe") {
+  entry_point = "$_dart_root/pkg/front_end/tool/_fasta/compile_platform.dart"
+  output = "$root_out_dir/compile_platform.exe"
+  package_config = "$_dart_root/.dart_tool/package_config.json"
+}
+
+aot_compile_using_prebuilt_sdk("gen_kernel.exe") {
+  entry_point = "$_dart_root/pkg/vm/bin/gen_kernel.dart"
+  output = "$root_out_dir/gen_kernel.exe"
+  package_config = "$_dart_root/.dart_tool/package_config.json"
+}

utils/compile_platform.gni

@@ -40,49 +40,101 @@ template("compile_platform") {
     outline = invoker.outline
   }
 
-  prebuilt_dart_action(target_name) {
-    if (defined(invoker.pool)) {
-      pool = invoker.pool
-    }
-    script = "$_dart_root/pkg/front_end/tool/_fasta/compile_platform.dart"
+  if (precompile_tools) {
+    action(target_name) {
+      if (defined(invoker.pool)) {
+        pool = invoker.pool
+      }
 
-    packages = "$_dart_root/.dart_tool/package_config.json"
+      outputs = invoker.outputs
 
-    outputs = invoker.outputs
+      compile_platform_tool = "//utils:compile_platform.exe($host_toolchain)"
 
-    vm_args = [ "-Dsdk_hash=$sdk_hash" ]
+      deps = [ compile_platform_tool ]
+      if (defined(invoker.deps)) {
+        deps += invoker.deps
+      }
 
-    inputs = []
-    deps = []
-    args = []
-    if (defined(invoker.deps)) {
-      deps += invoker.deps
-    }
+      inputs = []
+      if (defined(invoker.inputs)) {
+        inputs += invoker.inputs
+      }
 
-    if (defined(invoker.inputs)) {
-      inputs += invoker.inputs
-    }
+      if (add_implicit_vm_platform_dependency) {
+        inputs += [ "$root_out_dir/$outline" ]
+        deps += [ "$_dart_root/runtime/vm:vm_platform" ]
+      }
 
-    if (add_implicit_vm_platform_dependency) {
-      inputs += [ "$root_out_dir/$outline" ]
-      deps += [ "$_dart_root/runtime/vm:vm_platform" ]
-    }
-    depfile = outputs[0] + ".d"
+      depfile = outputs[0] + ".d"
 
-    args += invoker.args
-    if (defined(invoker.single_root_scheme)) {
-      args += [ "--single-root-scheme=" + invoker.single_root_scheme ]
+      script = "$_dart_root/build/gn_run_binary.py"
+      args = [
+        "compiled_action",
+        rebase_path(get_label_info(compile_platform_tool, "root_out_dir") +
+                        "/compile_platform.exe",
+                    root_build_dir),
+      ]
+      args += invoker.args
+      if (defined(invoker.single_root_scheme)) {
+        args += [ "--single-root-scheme=" + invoker.single_root_scheme ]
+      }
+      if (defined(invoker.single_root_base)) {
+        args += [ "--single-root-base=" + invoker.single_root_base ]
+      }
+      if (defined(invoker.single_root_scheme)) {
+        args += [ invoker.libraries_specification_uri ]
+      } else {
+        args +=
+            [ rebase_path(invoker.libraries_specification_uri, root_build_dir) ]
+      }
+      args += [ rebase_path("$root_out_dir/$outline", root_build_dir) ]
+      args += rebase_path(outputs, root_build_dir)
     }
-    if (defined(invoker.single_root_base)) {
-      args += [ "--single-root-base=" + invoker.single_root_base ]
+  } else {
+    prebuilt_dart_action(target_name) {
+      if (defined(invoker.pool)) {
+        pool = invoker.pool
+      }
+      script = "$_dart_root/pkg/front_end/tool/_fasta/compile_platform.dart"
+
+      packages = "$_dart_root/.dart_tool/package_config.json"
+
+      outputs = invoker.outputs
+
+      vm_args = [ "-Dsdk_hash=$sdk_hash" ]
+
+      inputs = []
+      deps = []
+      args = []
+      if (defined(invoker.deps)) {
+        deps += invoker.deps
+      }
+
+      if (defined(invoker.inputs)) {
+        inputs += invoker.inputs
+      }
+
+      if (add_implicit_vm_platform_dependency) {
+        inputs += [ "$root_out_dir/$outline" ]
+        deps += [ "$_dart_root/runtime/vm:vm_platform" ]
+      }
+      depfile = outputs[0] + ".d"
+
+      args += invoker.args
+      if (defined(invoker.single_root_scheme)) {
+        args += [ "--single-root-scheme=" + invoker.single_root_scheme ]
+      }
+      if (defined(invoker.single_root_base)) {
+        args += [ "--single-root-base=" + invoker.single_root_base ]
+      }
+      if (defined(invoker.single_root_scheme)) {
+        args += [ invoker.libraries_specification_uri ]
+      } else {
+        args +=
+            [ rebase_path(invoker.libraries_specification_uri, root_build_dir) ]
+      }
+      args += [ rebase_path("$root_out_dir/$outline", root_build_dir) ]
+      args += rebase_path(outputs, root_build_dir)
     }
-    if (defined(invoker.single_root_scheme)) {
-      args += [ invoker.libraries_specification_uri ]
-    } else {
-      args +=
-          [ rebase_path(invoker.libraries_specification_uri, root_build_dir) ]
-    }
-    args += [ rebase_path("$root_out_dir/$outline", root_build_dir) ]
-    args += rebase_path(outputs, root_build_dir)
   }
 }