diff --git a/BUILD.gn b/BUILD.gn index 4f11d9b2c70..3dd681d9d41 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -150,6 +150,13 @@ group("analysis_server") { deps = [ "utils/analysis_server" ] } +group("tools") { + deps = [ + "utils:compile_platform.exe", + "utils:gen_kernel.exe", + ] +} + # This is the target that is built on the dart2js build bots. # It must depend on anything that is required by the dart2js # test suites. diff --git a/build/gn_dart_compile_exe.py b/build/gn_dart_compile_exe.py new file mode 100755 index 00000000000..656bd4c7e95 --- /dev/null +++ b/build/gn_dart_compile_exe.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""Helper script for GN to run `dart compile exe` and produce a depfile. + +Run with: + python3 gn_dart_compile_exe.py \ + --dart-binary \ + --entry-point \ + --output \ + --sdk-hash \ + --packages \ + --depfile + +This is workaround for `dart compile exe` not supporting --depfile option +in the current version of prebuilt SDK. Once we roll in a new version +of checked in SDK we can remove this helper. +""" + +import argparse +import os +import sys +import subprocess +from tempfile import TemporaryDirectory + + +def parse_args(argv): + parser = argparse.ArgumentParser() + parser.add_argument("--dart-sdk", + required=True, + help="Path to the prebuilt Dart SDK") + parser.add_argument("--sdk-hash", required=True, help="SDK hash") + parser.add_argument("--entry-point", + required=True, + help="Dart entry point to precompile") + parser.add_argument("--output", + required=True, + help="Path to resulting executable ") + parser.add_argument("--packages", + required=True, + help="Path to package config file") + parser.add_argument("--depfile", + required=True, + help="Path to depfile to write") + return parser.parse_args(argv) + + +# Run a command, swallowing the output unless there is an error. +def run_command(command): + try: + subprocess.check_output(command, stderr=subprocess.STDOUT) + return True + except subprocess.CalledProcessError as e: + print("Command failed: " + " ".join(command) + "\n" + "output: " + + _decode(e.output)) + return False + except OSError as e: + print("Command failed: " + " ".join(command) + "\n" + "output: " + + _decode(e.strerror)) + return False + + +def _decode(bytes): + return bytes.decode("utf-8") + + +def main(argv): + args = parse_args(argv[1:]) + + # Unless the path is absolute, this script is designed to run binaries + # produced by the current build, which is the current working directory when + # this script is run. + prebuilt_sdk = os.path.abspath(args.dart_sdk) + + dart_binary = os.path.join(prebuilt_sdk, "bin", "dart") + if not os.path.isfile(dart_binary): + print("Binary not found: " + dart_binary) + return 1 + + dartaotruntime_binary = os.path.join(prebuilt_sdk, "bin", "dartaotruntime") + if not os.path.isfile(dartaotruntime_binary): + print("Binary not found: " + dartaotruntime_binary) + return 1 + + gen_kernel_snapshot = os.path.join(prebuilt_sdk, "bin", "snapshots", + "gen_kernel_aot.dart.snapshot") + if not os.path.isfile(gen_kernel_snapshot): + print("Binary not found: " + gen_kernel_snapshot) + return 1 + + platform_dill = os.path.join(prebuilt_sdk, "lib", "_internal", + "vm_platform_strong.dill") + if not os.path.isfile(platform_dill): + print("Binary not found: " + platform_dill) + return 1 + + # Compile the executable. + ok = run_command([ + dart_binary, + "compile", + "exe", + "--packages", + args.packages, + f"-Dsdk_hash={args.sdk_hash}", + "-o", + args.output, + args.entry_point, + ]) + if not ok: + return 1 + + # Collect dependencies by using gen_kernel. + with TemporaryDirectory() as tmpdir: + output_dill = os.path.join(tmpdir, "output.dill") + ok = run_command([ + dartaotruntime_binary, + gen_kernel_snapshot, + "--platform", + platform_dill, + "--packages", + args.packages, + "--depfile", + args.depfile, + "-o", + output_dill, + args.entry_point, + ]) + if not ok: + return 1 + + # Fix generated depfile to refer to the output file name instead + # of referring to the temporary dill file we have generated. + with open(args.depfile, "r") as f: + content = f.read() + (target_name, deps) = content.split(": ", 1) + if target_name != output_dill: + print( + "ERROR: Something is wrong with generated depfile: expected {output_dill} as target, but got {target_name}" + ) + return 1 + with open(args.depfile, "w") as f: + f.write(args.output) + f.write(": ") + f.write(deps) + + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/pkg/front_end/tool/_fasta/entry_points.dart b/pkg/front_end/tool/_fasta/entry_points.dart index 7df8904477d..de197e78a79 100644 --- a/pkg/front_end/tool/_fasta/entry_points.dart +++ b/pkg/front_end/tool/_fasta/entry_points.dart @@ -476,6 +476,11 @@ Future compilePlatformInternal(CompilerContext c, Uri fullOutput, } Future> computeHostDependencies(Uri hostPlatform) { + // Do not try to parse compile_platform if it was precompiled into a binary. + if (!Platform.script.toFilePath().endsWith('.dart')) { + return Future.value([]); + } + // Returns a list of source files that make up the Fasta compiler (the files // the Dart VM reads to run Fasta). Until Fasta is self-hosting (in strong // mode), this is only an approximation, albeit accurate. Once Fasta is diff --git a/pkg/vm/tool/precompiler2 b/pkg/vm/tool/precompiler2 index d92585d7be4..9fb1d0c3447 100755 --- a/pkg/vm/tool/precompiler2 +++ b/pkg/vm/tool/precompiler2 @@ -14,6 +14,23 @@ set -e +function follow_links() { + file="$1" + while [ -h "$file" ]; do + # On Mac OS, readlink -f doesn't work. + file="$(readlink "$file")" + done + echo "$file" +} + +# Unlike $0, $BASH_SOURCE points to the absolute path of this file. +PROG_NAME="$(follow_links "$BASH_SOURCE")" + +# Handle the case where dart-sdk/bin has been symlinked to. +CUR_DIR="$(cd "${PROG_NAME%/*}" ; pwd -P)" + +SDK_DIR="$CUR_DIR/../../.." + OPTIONS=() GEN_KERNEL_OPTIONS=() PACKAGES= @@ -22,6 +39,9 @@ BUILD_ASM=0 ARGV=() for arg in "$@"; do case $arg in + --packages=sdk) + PACKAGES="$SDK_DIR/.dart_tool/package_config.json" + ;; --packages=*) PACKAGES="$arg" ;; @@ -71,50 +91,47 @@ else GEN_SNAPSHOT_FILENAME="--elf=${SNAPSHOT_FILE}" fi -function follow_links() { - file="$1" - while [ -h "$file" ]; do - # On Mac OS, readlink -f doesn't work. - file="$(readlink "$file")" - done - echo "$file" -} - -# Unlike $0, $BASH_SOURCE points to the absolute path of this file. -PROG_NAME="$(follow_links "$BASH_SOURCE")" - -# Handle the case where dart-sdk/bin has been symlinked to. -CUR_DIR="$(cd "${PROG_NAME%/*}" ; pwd -P)" - -SDK_DIR="$CUR_DIR/../../.." - if [[ `uname` == 'Darwin' ]]; then OUT_DIR="$SDK_DIR/xcodebuild" else OUT_DIR="$SDK_DIR/out" fi -export DART_CONFIGURATION=${DART_CONFIGURATION:-ReleaseX64} -BIN_DIR="$OUT_DIR/$DART_CONFIGURATION" +HOST_ARCH="X64" +if [[ `uname -m` == 'arm64' ]]; then + HOST_ARCH="ARM64" +fi + +export DART_CONFIGURATION=${DART_CONFIGURATION:-Release$HOST_ARCH} +BUILD_DIR="$OUT_DIR/$DART_CONFIGURATION" DART="${SDK_DIR}/tools/sdks/dart-sdk/bin/dart" if [ ! -f "$DART" ]; then - DART="$BIN_DIR/dart" + DART="$BUILD_DIR/dart" fi +function gen_kernel() { + if [[ "$DART_GN_ARGS" == *"precompile_tools=true"* ]]; then + # Precompile gen_kernel to an AOT app. + ninja -C "$BUILD_DIR" gen_kernel.exe + "$BUILD_DIR/gen_kernel.exe" $@ + else + $DART ${DART_VM_FLAGS} "${SDK_DIR}/pkg/vm/bin/gen_kernel.dart" $@ + fi +} + # Step 1: Generate Kernel binary from the input Dart source. -$DART \ - ${DART_VM_FLAGS} \ - "${SDK_DIR}/pkg/vm/bin/gen_kernel.dart" \ - --platform "${BIN_DIR}/vm_platform_strong.dill" \ +gen_kernel --platform "${BUILD_DIR}/vm_platform_strong.dill" \ --aot \ "${GEN_KERNEL_OPTIONS[@]}" \ $PACKAGES \ -o "$SNAPSHOT_FILE.dill" \ "$SOURCE_FILE" + + # Step 2: Generate snapshot from the Kernel binary. -"$BIN_DIR"/gen_snapshot \ +"$BUILD_DIR"/gen_snapshot \ ${GEN_SNAPSHOT_FLAGS} \ "$GEN_SNAPSHOT_OPTION" \ "$GEN_SNAPSHOT_FILENAME" \ diff --git a/sdk_args.gni b/sdk_args.gni index b511f76d3e9..542d7b88521 100644 --- a/sdk_args.gni +++ b/sdk_args.gni @@ -32,6 +32,12 @@ declare_args() { # The location in the build output directory of the built Dart SDK. dart_sdk_output = "dart-sdk" + + # When set to `true` will cause compile_platform action to use a precompiled + # compile_platform.dart script instead of running it from source. This + # can significantly improve iteration time when iteration on changes in + # core libraries. + precompile_tools = false } if (default_git_folder == "") { diff --git a/utils/BUILD.gn b/utils/BUILD.gn new file mode 100644 index 00000000000..8376e332db5 --- /dev/null +++ b/utils/BUILD.gn @@ -0,0 +1,59 @@ +# Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file +# for details. All rights reserved. Use of this source code is governed by a +# BSD-style license that can be found in the LICENSE file. + +import("../build/dart/dart_action.gni") +import("../sdk_args.gni") + +_dart_root = get_path_info("..", "abspath") + +template("aot_compile_using_prebuilt_sdk") { + action(target_name) { + forward_variables_from(invoker, + [ + "deps", + "pool", + "testonly", + "visibility", + ]) + + script = "$_dart_root/build/gn_dart_compile_exe.py" + + inputs = [ + invoker.entry_point, + invoker.package_config, + ] + + outputs = [ invoker.output ] + + depfile = invoker.output + ".d" + + # TODO(vegorov): support RBE by using rewrapper script. + args = [ + "--dart-sdk", + rebase_path("$_dart_root/tools/sdks/dart-sdk", root_build_dir), + "--sdk-hash", + "$sdk_hash", + "--entry-point", + rebase_path(invoker.entry_point, root_build_dir), + "--output", + rebase_path(invoker.output, root_build_dir), + "--packages", + rebase_path(invoker.package_config, root_build_dir), + "--depfile", + rebase_path(depfile, root_build_dir), + ] + } +} + +aot_compile_using_prebuilt_sdk("compile_platform.exe") { + entry_point = "$_dart_root/pkg/front_end/tool/_fasta/compile_platform.dart" + output = "$root_out_dir/compile_platform.exe" + package_config = "$_dart_root/.dart_tool/package_config.json" +} + +aot_compile_using_prebuilt_sdk("gen_kernel.exe") { + entry_point = "$_dart_root/pkg/vm/bin/gen_kernel.dart" + output = "$root_out_dir/gen_kernel.exe" + package_config = "$_dart_root/.dart_tool/package_config.json" +} diff --git a/utils/compile_platform.gni b/utils/compile_platform.gni index 0523ac32039..4bb7f8f636c 100644 --- a/utils/compile_platform.gni +++ b/utils/compile_platform.gni @@ -40,49 +40,101 @@ template("compile_platform") { outline = invoker.outline } - prebuilt_dart_action(target_name) { - if (defined(invoker.pool)) { - pool = invoker.pool - } - script = "$_dart_root/pkg/front_end/tool/_fasta/compile_platform.dart" + if (precompile_tools) { + action(target_name) { + if (defined(invoker.pool)) { + pool = invoker.pool + } - packages = "$_dart_root/.dart_tool/package_config.json" + outputs = invoker.outputs - outputs = invoker.outputs + compile_platform_tool = "//utils:compile_platform.exe($host_toolchain)" - vm_args = [ "-Dsdk_hash=$sdk_hash" ] + deps = [ compile_platform_tool ] + if (defined(invoker.deps)) { + deps += invoker.deps + } - inputs = [] - deps = [] - args = [] - if (defined(invoker.deps)) { - deps += invoker.deps - } + inputs = [] + if (defined(invoker.inputs)) { + inputs += invoker.inputs + } - if (defined(invoker.inputs)) { - inputs += invoker.inputs - } + if (add_implicit_vm_platform_dependency) { + inputs += [ "$root_out_dir/$outline" ] + deps += [ "$_dart_root/runtime/vm:vm_platform" ] + } - if (add_implicit_vm_platform_dependency) { - inputs += [ "$root_out_dir/$outline" ] - deps += [ "$_dart_root/runtime/vm:vm_platform" ] - } - depfile = outputs[0] + ".d" + depfile = outputs[0] + ".d" - args += invoker.args - if (defined(invoker.single_root_scheme)) { - args += [ "--single-root-scheme=" + invoker.single_root_scheme ] + script = "$_dart_root/build/gn_run_binary.py" + args = [ + "compiled_action", + rebase_path(get_label_info(compile_platform_tool, "root_out_dir") + + "/compile_platform.exe", + root_build_dir), + ] + args += invoker.args + if (defined(invoker.single_root_scheme)) { + args += [ "--single-root-scheme=" + invoker.single_root_scheme ] + } + if (defined(invoker.single_root_base)) { + args += [ "--single-root-base=" + invoker.single_root_base ] + } + if (defined(invoker.single_root_scheme)) { + args += [ invoker.libraries_specification_uri ] + } else { + args += + [ rebase_path(invoker.libraries_specification_uri, root_build_dir) ] + } + args += [ rebase_path("$root_out_dir/$outline", root_build_dir) ] + args += rebase_path(outputs, root_build_dir) } - if (defined(invoker.single_root_base)) { - args += [ "--single-root-base=" + invoker.single_root_base ] + } else { + prebuilt_dart_action(target_name) { + if (defined(invoker.pool)) { + pool = invoker.pool + } + script = "$_dart_root/pkg/front_end/tool/_fasta/compile_platform.dart" + + packages = "$_dart_root/.dart_tool/package_config.json" + + outputs = invoker.outputs + + vm_args = [ "-Dsdk_hash=$sdk_hash" ] + + inputs = [] + deps = [] + args = [] + if (defined(invoker.deps)) { + deps += invoker.deps + } + + if (defined(invoker.inputs)) { + inputs += invoker.inputs + } + + if (add_implicit_vm_platform_dependency) { + inputs += [ "$root_out_dir/$outline" ] + deps += [ "$_dart_root/runtime/vm:vm_platform" ] + } + depfile = outputs[0] + ".d" + + args += invoker.args + if (defined(invoker.single_root_scheme)) { + args += [ "--single-root-scheme=" + invoker.single_root_scheme ] + } + if (defined(invoker.single_root_base)) { + args += [ "--single-root-base=" + invoker.single_root_base ] + } + if (defined(invoker.single_root_scheme)) { + args += [ invoker.libraries_specification_uri ] + } else { + args += + [ rebase_path(invoker.libraries_specification_uri, root_build_dir) ] + } + args += [ rebase_path("$root_out_dir/$outline", root_build_dir) ] + args += rebase_path(outputs, root_build_dir) } - if (defined(invoker.single_root_scheme)) { - args += [ invoker.libraries_specification_uri ] - } else { - args += - [ rebase_path(invoker.libraries_specification_uri, root_build_dir) ] - } - args += [ rebase_path("$root_out_dir/$outline", root_build_dir) ] - args += rebase_path(outputs, root_build_dir) } }