mirror of
https://github.com/dart-lang/sdk
synced 2024-09-30 03:48:30 +00:00
[tools] Allow precompiling gen_kernel and compile_platform
When iterating on core library changes or changes in the AOT compiler many seconds are wasted waiting on gen_kernel/compile_platform to parse Dart code. This happens because we are running these tools from sources on prebuilt Dart SDK. This CL allows SDK developer to opt-in into AOT compiling these tools by adding `precompile_tools=true` to their DART_GN_ARGS. AOT compilation is performed using prebuilt SDK - so these executables do not need to be recompiled if core libraries or VM changes reducing iteration cycles. pkg/vm/tool/precompiler2 is tweaked to detect when DART_GN_ARGS contains `precompile_tools=true` and use precompiled gen_kernel.exe instead of running it from source. Using precompiled compile_platform takes vm_platform_strong.dill build from 20 seconds to 3 seconds. Using precompiled gen_kernel takes small benchmark build from ~10 seconds to 2 seconds. TEST=manually tested Change-Id: Ieec6ad4e1081023d140eb744f0a3cd0c754414ca Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/367940 Commit-Queue: Slava Egorov <vegorov@google.com> Reviewed-by: Martin Kustermann <kustermann@google.com>
This commit is contained in:
parent
6f10e620fa
commit
5cda2a871c
7
BUILD.gn
7
BUILD.gn
|
@ -150,6 +150,13 @@ group("analysis_server") {
|
|||
deps = [ "utils/analysis_server" ]
|
||||
}
|
||||
|
||||
group("tools") {
|
||||
deps = [
|
||||
"utils:compile_platform.exe",
|
||||
"utils:gen_kernel.exe",
|
||||
]
|
||||
}
|
||||
|
||||
# This is the target that is built on the dart2js build bots.
|
||||
# It must depend on anything that is required by the dart2js
|
||||
# test suites.
|
||||
|
|
|
@ -35,7 +35,7 @@ if (_is_dart) {
|
|||
import("../toolchain/rbe.gni")
|
||||
}
|
||||
|
||||
template("_compiled_action") {
|
||||
template("compiled_action") {
|
||||
assert(defined(invoker.tool), "tool must be defined for $target_name")
|
||||
assert(defined(invoker.outputs), "outputs must be defined for $target_name")
|
||||
assert(defined(invoker.args), "args must be defined for $target_name")
|
||||
|
@ -244,7 +244,7 @@ template("_built_tool_action") {
|
|||
vm_args += invoker.vm_args
|
||||
}
|
||||
|
||||
_compiled_action(target_name) {
|
||||
compiled_action(target_name) {
|
||||
forward_variables_from(invoker,
|
||||
[
|
||||
"depfile",
|
||||
|
|
151
build/gn_dart_compile_exe.py
Executable file
151
build/gn_dart_compile_exe.py
Executable file
|
@ -0,0 +1,151 @@
|
|||
#!/usr/bin/env python3
|
||||
# Copyright 2014 The Chromium Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
"""Helper script for GN to run `dart compile exe` and produce a depfile.
|
||||
|
||||
Run with:
|
||||
python3 gn_dart_compile_exe.py \
|
||||
--dart-binary <path to dart binary> \
|
||||
--entry-point <path to dart entry point> \
|
||||
--output <path to resulting executable> \
|
||||
--sdk-hash <SDK hash> \
|
||||
--packages <path to package config file> \
|
||||
--depfile <path to depfile to write>
|
||||
|
||||
This is workaround for `dart compile exe` not supporting --depfile option
|
||||
in the current version of prebuilt SDK. Once we roll in a new version
|
||||
of checked in SDK we can remove this helper.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
import subprocess
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
|
||||
def parse_args(argv):
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--dart-sdk",
|
||||
required=True,
|
||||
help="Path to the prebuilt Dart SDK")
|
||||
parser.add_argument("--sdk-hash", required=True, help="SDK hash")
|
||||
parser.add_argument("--entry-point",
|
||||
required=True,
|
||||
help="Dart entry point to precompile")
|
||||
parser.add_argument("--output",
|
||||
required=True,
|
||||
help="Path to resulting executable ")
|
||||
parser.add_argument("--packages",
|
||||
required=True,
|
||||
help="Path to package config file")
|
||||
parser.add_argument("--depfile",
|
||||
required=True,
|
||||
help="Path to depfile to write")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
# Run a command, swallowing the output unless there is an error.
|
||||
def run_command(command):
|
||||
try:
|
||||
subprocess.check_output(command, stderr=subprocess.STDOUT)
|
||||
return True
|
||||
except subprocess.CalledProcessError as e:
|
||||
print("Command failed: " + " ".join(command) + "\n" + "output: " +
|
||||
_decode(e.output))
|
||||
return False
|
||||
except OSError as e:
|
||||
print("Command failed: " + " ".join(command) + "\n" + "output: " +
|
||||
_decode(e.strerror))
|
||||
return False
|
||||
|
||||
|
||||
def _decode(bytes):
|
||||
return bytes.decode("utf-8")
|
||||
|
||||
|
||||
def main(argv):
|
||||
args = parse_args(argv[1:])
|
||||
|
||||
# Unless the path is absolute, this script is designed to run binaries
|
||||
# produced by the current build, which is the current working directory when
|
||||
# this script is run.
|
||||
prebuilt_sdk = os.path.abspath(args.dart_sdk)
|
||||
|
||||
dart_binary = os.path.join(prebuilt_sdk, "bin", "dart")
|
||||
if not os.path.isfile(dart_binary):
|
||||
print("Binary not found: " + dart_binary)
|
||||
return 1
|
||||
|
||||
dartaotruntime_binary = os.path.join(prebuilt_sdk, "bin", "dartaotruntime")
|
||||
if not os.path.isfile(dartaotruntime_binary):
|
||||
print("Binary not found: " + dartaotruntime_binary)
|
||||
return 1
|
||||
|
||||
gen_kernel_snapshot = os.path.join(prebuilt_sdk, "bin", "snapshots",
|
||||
"gen_kernel_aot.dart.snapshot")
|
||||
if not os.path.isfile(gen_kernel_snapshot):
|
||||
print("Binary not found: " + gen_kernel_snapshot)
|
||||
return 1
|
||||
|
||||
platform_dill = os.path.join(prebuilt_sdk, "lib", "_internal",
|
||||
"vm_platform_strong.dill")
|
||||
if not os.path.isfile(platform_dill):
|
||||
print("Binary not found: " + platform_dill)
|
||||
return 1
|
||||
|
||||
# Compile the executable.
|
||||
ok = run_command([
|
||||
dart_binary,
|
||||
"compile",
|
||||
"exe",
|
||||
"--packages",
|
||||
args.packages,
|
||||
f"-Dsdk_hash={args.sdk_hash}",
|
||||
"-o",
|
||||
args.output,
|
||||
args.entry_point,
|
||||
])
|
||||
if not ok:
|
||||
return 1
|
||||
|
||||
# Collect dependencies by using gen_kernel.
|
||||
with TemporaryDirectory() as tmpdir:
|
||||
output_dill = os.path.join(tmpdir, "output.dill")
|
||||
ok = run_command([
|
||||
dartaotruntime_binary,
|
||||
gen_kernel_snapshot,
|
||||
"--platform",
|
||||
platform_dill,
|
||||
"--packages",
|
||||
args.packages,
|
||||
"--depfile",
|
||||
args.depfile,
|
||||
"-o",
|
||||
output_dill,
|
||||
args.entry_point,
|
||||
])
|
||||
if not ok:
|
||||
return 1
|
||||
|
||||
# Fix generated depfile to refer to the output file name instead
|
||||
# of referring to the temporary dill file we have generated.
|
||||
with open(args.depfile, "r") as f:
|
||||
content = f.read()
|
||||
(target_name, deps) = content.split(": ", 1)
|
||||
if target_name != output_dill:
|
||||
print(
|
||||
"ERROR: Something is wrong with generated depfile: expected {output_dill} as target, but got {target_name}"
|
||||
)
|
||||
return 1
|
||||
with open(args.depfile, "w") as f:
|
||||
f.write(args.output)
|
||||
f.write(": ")
|
||||
f.write(deps)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv))
|
|
@ -476,6 +476,11 @@ Future<void> compilePlatformInternal(CompilerContext c, Uri fullOutput,
|
|||
}
|
||||
|
||||
Future<List<Uri>> computeHostDependencies(Uri hostPlatform) {
|
||||
// Do not try to parse compile_platform if it was precompiled into a binary.
|
||||
if (!Platform.script.toFilePath().endsWith('.dart')) {
|
||||
return Future.value([]);
|
||||
}
|
||||
|
||||
// Returns a list of source files that make up the Fasta compiler (the files
|
||||
// the Dart VM reads to run Fasta). Until Fasta is self-hosting (in strong
|
||||
// mode), this is only an approximation, albeit accurate. Once Fasta is
|
||||
|
|
|
@ -14,6 +14,23 @@
|
|||
|
||||
set -e
|
||||
|
||||
function follow_links() {
|
||||
file="$1"
|
||||
while [ -h "$file" ]; do
|
||||
# On Mac OS, readlink -f doesn't work.
|
||||
file="$(readlink "$file")"
|
||||
done
|
||||
echo "$file"
|
||||
}
|
||||
|
||||
# Unlike $0, $BASH_SOURCE points to the absolute path of this file.
|
||||
PROG_NAME="$(follow_links "$BASH_SOURCE")"
|
||||
|
||||
# Handle the case where dart-sdk/bin has been symlinked to.
|
||||
CUR_DIR="$(cd "${PROG_NAME%/*}" ; pwd -P)"
|
||||
|
||||
SDK_DIR="$CUR_DIR/../../.."
|
||||
|
||||
OPTIONS=()
|
||||
GEN_KERNEL_OPTIONS=()
|
||||
PACKAGES=
|
||||
|
@ -22,6 +39,9 @@ BUILD_ASM=0
|
|||
ARGV=()
|
||||
for arg in "$@"; do
|
||||
case $arg in
|
||||
--packages=sdk)
|
||||
PACKAGES="$SDK_DIR/.dart_tool/package_config.json"
|
||||
;;
|
||||
--packages=*)
|
||||
PACKAGES="$arg"
|
||||
;;
|
||||
|
@ -71,50 +91,47 @@ else
|
|||
GEN_SNAPSHOT_FILENAME="--elf=${SNAPSHOT_FILE}"
|
||||
fi
|
||||
|
||||
function follow_links() {
|
||||
file="$1"
|
||||
while [ -h "$file" ]; do
|
||||
# On Mac OS, readlink -f doesn't work.
|
||||
file="$(readlink "$file")"
|
||||
done
|
||||
echo "$file"
|
||||
}
|
||||
|
||||
# Unlike $0, $BASH_SOURCE points to the absolute path of this file.
|
||||
PROG_NAME="$(follow_links "$BASH_SOURCE")"
|
||||
|
||||
# Handle the case where dart-sdk/bin has been symlinked to.
|
||||
CUR_DIR="$(cd "${PROG_NAME%/*}" ; pwd -P)"
|
||||
|
||||
SDK_DIR="$CUR_DIR/../../.."
|
||||
|
||||
if [[ `uname` == 'Darwin' ]]; then
|
||||
OUT_DIR="$SDK_DIR/xcodebuild"
|
||||
else
|
||||
OUT_DIR="$SDK_DIR/out"
|
||||
fi
|
||||
|
||||
export DART_CONFIGURATION=${DART_CONFIGURATION:-ReleaseX64}
|
||||
BIN_DIR="$OUT_DIR/$DART_CONFIGURATION"
|
||||
HOST_ARCH="X64"
|
||||
if [[ `uname -m` == 'arm64' ]]; then
|
||||
HOST_ARCH="ARM64"
|
||||
fi
|
||||
|
||||
export DART_CONFIGURATION=${DART_CONFIGURATION:-Release$HOST_ARCH}
|
||||
BUILD_DIR="$OUT_DIR/$DART_CONFIGURATION"
|
||||
|
||||
DART="${SDK_DIR}/tools/sdks/dart-sdk/bin/dart"
|
||||
if [ ! -f "$DART" ]; then
|
||||
DART="$BIN_DIR/dart"
|
||||
DART="$BUILD_DIR/dart"
|
||||
fi
|
||||
|
||||
function gen_kernel() {
|
||||
if [[ "$DART_GN_ARGS" == *"precompile_tools=true"* ]]; then
|
||||
# Precompile gen_kernel to an AOT app.
|
||||
ninja -C "$BUILD_DIR" gen_kernel.exe
|
||||
"$BUILD_DIR/gen_kernel.exe" $@
|
||||
else
|
||||
$DART ${DART_VM_FLAGS} "${SDK_DIR}/pkg/vm/bin/gen_kernel.dart" $@
|
||||
fi
|
||||
}
|
||||
|
||||
# Step 1: Generate Kernel binary from the input Dart source.
|
||||
$DART \
|
||||
${DART_VM_FLAGS} \
|
||||
"${SDK_DIR}/pkg/vm/bin/gen_kernel.dart" \
|
||||
--platform "${BIN_DIR}/vm_platform_strong.dill" \
|
||||
gen_kernel --platform "${BUILD_DIR}/vm_platform_strong.dill" \
|
||||
--aot \
|
||||
"${GEN_KERNEL_OPTIONS[@]}" \
|
||||
$PACKAGES \
|
||||
-o "$SNAPSHOT_FILE.dill" \
|
||||
"$SOURCE_FILE"
|
||||
|
||||
|
||||
|
||||
# Step 2: Generate snapshot from the Kernel binary.
|
||||
"$BIN_DIR"/gen_snapshot \
|
||||
"$BUILD_DIR"/gen_snapshot \
|
||||
${GEN_SNAPSHOT_FLAGS} \
|
||||
"$GEN_SNAPSHOT_OPTION" \
|
||||
"$GEN_SNAPSHOT_FILENAME" \
|
||||
|
|
|
@ -32,6 +32,12 @@ declare_args() {
|
|||
|
||||
# The location in the build output directory of the built Dart SDK.
|
||||
dart_sdk_output = "dart-sdk"
|
||||
|
||||
# When set to `true` will cause compile_platform action to use a precompiled
|
||||
# compile_platform.dart script instead of running it from source. This
|
||||
# can significantly improve iteration time when iteration on changes in
|
||||
# core libraries.
|
||||
precompile_tools = false
|
||||
}
|
||||
|
||||
if (default_git_folder == "") {
|
||||
|
|
59
utils/BUILD.gn
Normal file
59
utils/BUILD.gn
Normal file
|
@ -0,0 +1,59 @@
|
|||
# Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file
|
||||
# for details. All rights reserved. Use of this source code is governed by a
|
||||
# BSD-style license that can be found in the LICENSE file.
|
||||
|
||||
import("../build/dart/dart_action.gni")
|
||||
import("../sdk_args.gni")
|
||||
|
||||
_dart_root = get_path_info("..", "abspath")
|
||||
|
||||
template("aot_compile_using_prebuilt_sdk") {
|
||||
action(target_name) {
|
||||
forward_variables_from(invoker,
|
||||
[
|
||||
"deps",
|
||||
"pool",
|
||||
"testonly",
|
||||
"visibility",
|
||||
])
|
||||
|
||||
script = "$_dart_root/build/gn_dart_compile_exe.py"
|
||||
|
||||
inputs = [
|
||||
invoker.entry_point,
|
||||
invoker.package_config,
|
||||
]
|
||||
|
||||
outputs = [ invoker.output ]
|
||||
|
||||
depfile = invoker.output + ".d"
|
||||
|
||||
# TODO(vegorov): support RBE by using rewrapper script.
|
||||
args = [
|
||||
"--dart-sdk",
|
||||
rebase_path("$_dart_root/tools/sdks/dart-sdk", root_build_dir),
|
||||
"--sdk-hash",
|
||||
"$sdk_hash",
|
||||
"--entry-point",
|
||||
rebase_path(invoker.entry_point, root_build_dir),
|
||||
"--output",
|
||||
rebase_path(invoker.output, root_build_dir),
|
||||
"--packages",
|
||||
rebase_path(invoker.package_config, root_build_dir),
|
||||
"--depfile",
|
||||
rebase_path(depfile, root_build_dir),
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
aot_compile_using_prebuilt_sdk("compile_platform.exe") {
|
||||
entry_point = "$_dart_root/pkg/front_end/tool/_fasta/compile_platform.dart"
|
||||
output = "$root_out_dir/compile_platform.exe"
|
||||
package_config = "$_dart_root/.dart_tool/package_config.json"
|
||||
}
|
||||
|
||||
aot_compile_using_prebuilt_sdk("gen_kernel.exe") {
|
||||
entry_point = "$_dart_root/pkg/vm/bin/gen_kernel.dart"
|
||||
output = "$root_out_dir/gen_kernel.exe"
|
||||
package_config = "$_dart_root/.dart_tool/package_config.json"
|
||||
}
|
|
@ -40,49 +40,93 @@ template("compile_platform") {
|
|||
outline = invoker.outline
|
||||
}
|
||||
|
||||
prebuilt_dart_action(target_name) {
|
||||
if (defined(invoker.pool)) {
|
||||
pool = invoker.pool
|
||||
}
|
||||
script = "$_dart_root/pkg/front_end/tool/_fasta/compile_platform.dart"
|
||||
if (precompile_tools) {
|
||||
compiled_action(target_name) {
|
||||
if (defined(invoker.pool)) {
|
||||
pool = invoker.pool
|
||||
}
|
||||
tool = "//utils:compile_platform.exe"
|
||||
|
||||
packages = "$_dart_root/.dart_tool/package_config.json"
|
||||
outputs = invoker.outputs
|
||||
|
||||
outputs = invoker.outputs
|
||||
inputs = []
|
||||
deps = []
|
||||
args = []
|
||||
if (defined(invoker.deps)) {
|
||||
deps += invoker.deps
|
||||
}
|
||||
|
||||
vm_args = [ "-Dsdk_hash=$sdk_hash" ]
|
||||
if (defined(invoker.inputs)) {
|
||||
inputs += invoker.inputs
|
||||
}
|
||||
|
||||
inputs = []
|
||||
deps = []
|
||||
args = []
|
||||
if (defined(invoker.deps)) {
|
||||
deps += invoker.deps
|
||||
}
|
||||
if (add_implicit_vm_platform_dependency) {
|
||||
inputs += [ "$root_out_dir/$outline" ]
|
||||
deps += [ "$_dart_root/runtime/vm:vm_platform" ]
|
||||
}
|
||||
depfile = outputs[0] + ".d"
|
||||
|
||||
if (defined(invoker.inputs)) {
|
||||
inputs += invoker.inputs
|
||||
args += invoker.args
|
||||
if (defined(invoker.single_root_scheme)) {
|
||||
args += [ "--single-root-scheme=" + invoker.single_root_scheme ]
|
||||
}
|
||||
if (defined(invoker.single_root_base)) {
|
||||
args += [ "--single-root-base=" + invoker.single_root_base ]
|
||||
}
|
||||
if (defined(invoker.single_root_scheme)) {
|
||||
args += [ invoker.libraries_specification_uri ]
|
||||
} else {
|
||||
args +=
|
||||
[ rebase_path(invoker.libraries_specification_uri, root_build_dir) ]
|
||||
}
|
||||
args += [ rebase_path("$root_out_dir/$outline", root_build_dir) ]
|
||||
args += rebase_path(outputs, root_build_dir)
|
||||
}
|
||||
} else {
|
||||
prebuilt_dart_action(target_name) {
|
||||
if (defined(invoker.pool)) {
|
||||
pool = invoker.pool
|
||||
}
|
||||
script = "$_dart_root/pkg/front_end/tool/_fasta/compile_platform.dart"
|
||||
|
||||
if (add_implicit_vm_platform_dependency) {
|
||||
inputs += [ "$root_out_dir/$outline" ]
|
||||
deps += [ "$_dart_root/runtime/vm:vm_platform" ]
|
||||
}
|
||||
depfile = outputs[0] + ".d"
|
||||
packages = "$_dart_root/.dart_tool/package_config.json"
|
||||
|
||||
args += invoker.args
|
||||
if (defined(invoker.single_root_scheme)) {
|
||||
args += [ "--single-root-scheme=" + invoker.single_root_scheme ]
|
||||
outputs = invoker.outputs
|
||||
|
||||
vm_args = [ "-Dsdk_hash=$sdk_hash" ]
|
||||
|
||||
inputs = []
|
||||
deps = []
|
||||
args = []
|
||||
if (defined(invoker.deps)) {
|
||||
deps += invoker.deps
|
||||
}
|
||||
|
||||
if (defined(invoker.inputs)) {
|
||||
inputs += invoker.inputs
|
||||
}
|
||||
|
||||
if (add_implicit_vm_platform_dependency) {
|
||||
inputs += [ "$root_out_dir/$outline" ]
|
||||
deps += [ "$_dart_root/runtime/vm:vm_platform" ]
|
||||
}
|
||||
depfile = outputs[0] + ".d"
|
||||
|
||||
args += invoker.args
|
||||
if (defined(invoker.single_root_scheme)) {
|
||||
args += [ "--single-root-scheme=" + invoker.single_root_scheme ]
|
||||
}
|
||||
if (defined(invoker.single_root_base)) {
|
||||
args += [ "--single-root-base=" + invoker.single_root_base ]
|
||||
}
|
||||
if (defined(invoker.single_root_scheme)) {
|
||||
args += [ invoker.libraries_specification_uri ]
|
||||
} else {
|
||||
args +=
|
||||
[ rebase_path(invoker.libraries_specification_uri, root_build_dir) ]
|
||||
}
|
||||
args += [ rebase_path("$root_out_dir/$outline", root_build_dir) ]
|
||||
args += rebase_path(outputs, root_build_dir)
|
||||
}
|
||||
if (defined(invoker.single_root_base)) {
|
||||
args += [ "--single-root-base=" + invoker.single_root_base ]
|
||||
}
|
||||
if (defined(invoker.single_root_scheme)) {
|
||||
args += [ invoker.libraries_specification_uri ]
|
||||
} else {
|
||||
args +=
|
||||
[ rebase_path(invoker.libraries_specification_uri, root_build_dir) ]
|
||||
}
|
||||
args += [ rebase_path("$root_out_dir/$outline", root_build_dir) ]
|
||||
args += rebase_path(outputs, root_build_dir)
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue