mirror of
https://github.com/dart-lang/sdk
synced 2024-07-05 17:30:16 +00:00
Revert "[infra] RBE all the things."
This reverts commit b3253d9c8b
.
Reason for revert: Flutter build is broken, it also has a GN variable rbi_dir, and dart_action.gni now imports two conflicting definitions.
Original change's description:
> [infra] RBE all the things.
>
> This change offloads all Dart actions during the build to RBE using
> the new rewrapper_dart.py script that understands all the different
> command line invocations of Dart programs and translates it into the
> appropriate rewrapper invocation, with a full list of input and output
> files and no absolute paths.
>
> The dart actions are all considered to be expensive RBE compilation
> steps that may be much slower on RBE than locally due to the low
> parallelism of the build during these stages and the comperatively
> slower bots in the RBE pools. The build uses the racing strategy by
> default for these compilations, such that cache hits are still used
> if available, and otherwise a local build is transparently used if it
> happens to be faster. The bots will use the remote strategy by default
> unlike developers, such that the rewrapper_dart.py script does not fail.
>
> rewrapper_dart.py contains a tiny dart import proprocessor written
> in python and a big argument parser that understands every command
> invoked during the build and the semantic meaning of every option.
>
> The absolute paths used during the Dart SDK build is not solved in
> this changelist, which just works around them initially, but these
> will be fixed in follow up changes with the appropriate teams now
> that this change proves they are not needed.
>
> Bug: b/333595242
> Change-Id: I36603ec1bf16f4ac87d56635cc1c98e8686a4028
> Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/335827
> Reviewed-by: William Hesse <whesse@google.com>
> Commit-Queue: Jonas Termansen <sortie@google.com>
Bug: b/333595242
Change-Id: I56c9df11e672e7bd096f2a6e7d0243dccd0396c9
No-Presubmit: true
No-Tree-Checks: true
No-Try: true
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/362385
Bot-Commit: Rubber Stamper <rubber-stamper@appspot.gserviceaccount.com>
Auto-Submit: William Hesse <whesse@google.com>
Commit-Queue: Alexander Thomas <athom@google.com>
Reviewed-by: Alexander Thomas <athom@google.com>
This commit is contained in:
parent
5d3dd79d95
commit
1b249f5496
|
@ -3,7 +3,6 @@
|
||||||
# BSD-style license that can be found in the LICENSE file.
|
# BSD-style license that can be found in the LICENSE file.
|
||||||
|
|
||||||
import("../executable_suffix.gni")
|
import("../executable_suffix.gni")
|
||||||
import("../toolchain/rbe.gni")
|
|
||||||
|
|
||||||
# This file defines templates for running and compiling Dart code during
|
# This file defines templates for running and compiling Dart code during
|
||||||
# Dart's build.
|
# Dart's build.
|
||||||
|
@ -83,22 +82,10 @@ template("_compiled_action") {
|
||||||
|
|
||||||
# The "compiled_action" argument to gn_run_binary.py indicates that
|
# The "compiled_action" argument to gn_run_binary.py indicates that
|
||||||
# it will exit with a non-zero status when the target program does.
|
# it will exit with a non-zero status when the target program does.
|
||||||
args = [ "compiled_action" ]
|
args = [
|
||||||
|
"compiled_action",
|
||||||
if (use_rbe && (target_os == rbe_os ||
|
rebase_path(host_executable, root_build_dir),
|
||||||
(target_os == "android" && rbe_os == "linux")) &&
|
] + invoker.args
|
||||||
target_cpu == rbe_cpu) {
|
|
||||||
args += [
|
|
||||||
"/usr/bin/python3",
|
|
||||||
rebase_path("//build/rbe/rewrapper_dart.py", root_build_dir),
|
|
||||||
] + rewrapper_args +
|
|
||||||
[
|
|
||||||
"--exec_strategy=" + rbe_expensive_exec_strategy,
|
|
||||||
"--",
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
||||||
args += [ rebase_path(host_executable, root_build_dir) ] + invoker.args
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -135,22 +122,10 @@ template("_prebuilt_tool_action") {
|
||||||
inputs += [ invoker.packages ]
|
inputs += [ invoker.packages ]
|
||||||
}
|
}
|
||||||
|
|
||||||
args = [ "compiled_action" ]
|
args = [
|
||||||
|
"compiled_action",
|
||||||
if (use_rbe && (target_os == rbe_os ||
|
rebase_path(invoker.binary, root_build_dir),
|
||||||
(target_os == "android" && rbe_os == "linux")) &&
|
] + vm_args
|
||||||
target_cpu == rbe_cpu) {
|
|
||||||
args += [
|
|
||||||
"/usr/bin/python3",
|
|
||||||
rebase_path("//build/rbe/rewrapper_dart.py", root_build_dir),
|
|
||||||
] + rewrapper_args +
|
|
||||||
[
|
|
||||||
"--exec_strategy=" + rbe_expensive_exec_strategy,
|
|
||||||
"--",
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
||||||
args += [ rebase_path(invoker.binary, root_build_dir) ] + vm_args
|
|
||||||
if (defined(invoker.packages)) {
|
if (defined(invoker.packages)) {
|
||||||
args += [ "--packages=" + rebase_path(invoker.packages, root_build_dir) ]
|
args += [ "--packages=" + rebase_path(invoker.packages, root_build_dir) ]
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,761 +0,0 @@
|
||||||
#!/usr/bin/env python3
|
|
||||||
# Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file
|
|
||||||
# for details. All rights reserved. Use of this source code is governed by a
|
|
||||||
# BSD-style license that can be found in the LICENSE file.
|
|
||||||
|
|
||||||
# This program executes Dart programs on RBE using rewrapper by parsing the
|
|
||||||
# source code to locate the input files, and recognizes the command line options
|
|
||||||
# of well known programs to determine the output files.
|
|
||||||
|
|
||||||
# New executions during the Dart SDK build needs to be supported here in order
|
|
||||||
# to speed up the build with RBE. See the argument parser below.
|
|
||||||
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import re
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
|
|
||||||
|
|
||||||
# Run a command, swallowing the output unless there is an error.
|
|
||||||
def run_command(command, strategy):
|
|
||||||
try:
|
|
||||||
subprocess.check_output(command, stderr=subprocess.STDOUT)
|
|
||||||
return 0
|
|
||||||
except subprocess.CalledProcessError as e:
|
|
||||||
print(e.output.decode("utf-8"))
|
|
||||||
if strategy == 'remote':
|
|
||||||
joined = ' '.join(command)
|
|
||||||
print(f'''Failed to run command remotely: {joined}
|
|
||||||
|
|
||||||
If you're seeing this error on a bot and it doesn't happen locally, then you may
|
|
||||||
need to teach the script build/rbe/rewrapper_dart.py what the appropriate input
|
|
||||||
and outputs files of the command are. You can see the list of used inputs and
|
|
||||||
outputs in the rewrapper invocation above. To reproduce this error locally, try
|
|
||||||
forcing a remote build by setting the RBE_exec_strategy=remote environment
|
|
||||||
variable.
|
|
||||||
''')
|
|
||||||
sys.exit(1)
|
|
||||||
except OSError as e:
|
|
||||||
print(e.strerror)
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
|
|
||||||
# Loads the package config file.
|
|
||||||
def load_package_config(exec_root):
|
|
||||||
path = os.path.join(exec_root, '.dart_tool', 'package_config.json')
|
|
||||||
with open(path, 'r') as file:
|
|
||||||
return json.load(file)
|
|
||||||
|
|
||||||
|
|
||||||
# Resolves a Dart import URI using the package config.
|
|
||||||
def resolve_uri(uri, exec_root, package_config, whole_dir=False):
|
|
||||||
if uri.startswith('package:'):
|
|
||||||
match = re.search(r'package:([^/]*)/(.*)', uri)
|
|
||||||
package_name = match.groups()[0]
|
|
||||||
relative = match.groups()[1]
|
|
||||||
package_data = next(pkg for pkg in package_config['packages']
|
|
||||||
if pkg['name'] == package_name)
|
|
||||||
package_root = package_data['rootUri']
|
|
||||||
package_root = package_root[3:] # Remove leading ../
|
|
||||||
package_uri = package_data['packageUri']
|
|
||||||
if whole_dir:
|
|
||||||
uri = package_root + '/' + package_uri
|
|
||||||
else:
|
|
||||||
uri = package_root + '/' + package_uri + relative
|
|
||||||
return uri
|
|
||||||
|
|
||||||
|
|
||||||
# Lists the imports of a Dart file uri using the package config and a rough
|
|
||||||
# parser that recognizes fairly traditional imports. This is designed to be much
|
|
||||||
# faster than actually invoking the front end.
|
|
||||||
def list_imports(uri, exec_root, package_config):
|
|
||||||
if uri.startswith('dart:'):
|
|
||||||
return set()
|
|
||||||
path = os.path.join(exec_root, resolve_uri(uri, exec_root, package_config))
|
|
||||||
file = open(path, 'r')
|
|
||||||
imports = set()
|
|
||||||
for line in file.readlines():
|
|
||||||
tokens = [token for token in re.split(r'\s+', line) if token != '']
|
|
||||||
if not tokens or tokens[0] in [
|
|
||||||
'//', '///', '/*', '*/', '#!', 'library', 'show'
|
|
||||||
]:
|
|
||||||
continue
|
|
||||||
# Imports must happen before definitions.
|
|
||||||
if tokens[0] in ['const', 'class', 'enum']:
|
|
||||||
break
|
|
||||||
if 2 <= len(tokens
|
|
||||||
) and tokens[0] == 'if' and tokens[1] == '(dart.library.io)':
|
|
||||||
tokens = ['import'] + tokens[2:]
|
|
||||||
if tokens[0] not in ['import', 'export', 'part']:
|
|
||||||
continue
|
|
||||||
if len(tokens) < 2:
|
|
||||||
raise Exception(f'Bad import statement: {path}: {line}')
|
|
||||||
if tokens[0] == 'part' and tokens[1] == 'of':
|
|
||||||
continue
|
|
||||||
token = tokens[1].replace('"', '').replace("'", '').replace(';', '')
|
|
||||||
if token.startswith('dart:'):
|
|
||||||
continue
|
|
||||||
if not ':' in token:
|
|
||||||
dirname = os.path.dirname(uri)
|
|
||||||
while token.startswith('..'):
|
|
||||||
token = token[3:]
|
|
||||||
dirname = os.path.dirname(dirname)
|
|
||||||
token = dirname + '/' + token
|
|
||||||
imports.add(token)
|
|
||||||
file.close()
|
|
||||||
return imports
|
|
||||||
|
|
||||||
|
|
||||||
# Transitively compute the set of dart files needed to execute the specified
|
|
||||||
# entry point using the package config.
|
|
||||||
def find_inputs(uris, exec_root, package_config):
|
|
||||||
inputs = set(uris)
|
|
||||||
unexplored = set(uris)
|
|
||||||
while unexplored:
|
|
||||||
uri = unexplored.pop()
|
|
||||||
imports = list_imports(uri, exec_root, package_config)
|
|
||||||
for uri in imports:
|
|
||||||
if not uri in inputs:
|
|
||||||
inputs.add(uri)
|
|
||||||
unexplored.add(uri)
|
|
||||||
return inputs
|
|
||||||
|
|
||||||
|
|
||||||
# Rewrite absolute paths in an argument to be relative.
|
|
||||||
def rewrite_absolute(arg, exec_root, working_directory):
|
|
||||||
# The file:// schema does not work with relative paths as they are parsed as
|
|
||||||
# the authority by the dart Uri class.
|
|
||||||
arg = arg.replace('file:///' + exec_root, '../../')
|
|
||||||
arg = arg.replace('file://' + exec_root, '../../')
|
|
||||||
# Replace the absolute exec root by a relative path to the exec root.
|
|
||||||
arg = arg.replace(exec_root, '../../')
|
|
||||||
# Simplify paths going to the exec root and back into the out directory.
|
|
||||||
# Carefully ensure the whole path isn't optimized away.
|
|
||||||
if arg.endswith(f'../../{working_directory}/'):
|
|
||||||
arg = arg.replace(f'../../{working_directory}/', '.')
|
|
||||||
else:
|
|
||||||
arg = arg.replace(f'../../{working_directory}/', '')
|
|
||||||
return arg
|
|
||||||
|
|
||||||
|
|
||||||
# Parse the command line execution to recognize well known programs during the
|
|
||||||
# Dart SDK build, so the inputs and output files can be determined, and the
|
|
||||||
# command can be offloaded to RBE.
|
|
||||||
#
|
|
||||||
# RBE needs a command to run, a list of input files, and a list of output files,
|
|
||||||
# and it then executes the command remotely and caches the result. Absolute
|
|
||||||
# paths must not occur in the command as the remote execution will happen in
|
|
||||||
# another directory. However, since we currently rely on absolute paths, we work
|
|
||||||
# around the issue and rewrite the absolute paths accordingly until the problem
|
|
||||||
# is fixed on our end.
|
|
||||||
#
|
|
||||||
# This is a parser that handles nested commands executing each other, taking
|
|
||||||
# care to know whose options it is currently parsing, and extracting the
|
|
||||||
# appropriate information from each argument. Every invoked program and option
|
|
||||||
# during the build needs to be supported here, otherwise the remote command may
|
|
||||||
# be inaccurate and not have right inputs and outputs. Although maintaining this
|
|
||||||
# parser takes some effort, it is being paid back in the builders being sped up
|
|
||||||
# massively on cache hits, as well as speeding up any local developers that
|
|
||||||
# build code already built by the bots.
|
|
||||||
#
|
|
||||||
# To add a new program, recognize the entry point and define its parser method.
|
|
||||||
# To add a new option, parse the option in the appropriate method and either
|
|
||||||
# ignore it or recognize any input and output files. All invoked options needs
|
|
||||||
# be allowlisted here know we didn't accidentally misunderstand the invoked
|
|
||||||
# command when running it remotely.
|
|
||||||
class Rewrapper:
|
|
||||||
|
|
||||||
def __init__(self, argv):
|
|
||||||
self.dart_subdir = None
|
|
||||||
self.depfiles = None
|
|
||||||
self.entry_points = set()
|
|
||||||
self.exec_root = None
|
|
||||||
self.exec_strategy = 'remote'
|
|
||||||
self.exec_strategy_explicit = False
|
|
||||||
self.extra_paths = set()
|
|
||||||
self.outputs = []
|
|
||||||
self.no_remote = None
|
|
||||||
self.argv = argv
|
|
||||||
self.optarg = None
|
|
||||||
self.optind = 0
|
|
||||||
self.parse()
|
|
||||||
|
|
||||||
@property
|
|
||||||
def has_next_arg(self):
|
|
||||||
return self.optind + 1 < len(self.argv)
|
|
||||||
|
|
||||||
def next_arg(self):
|
|
||||||
self.optind += 1
|
|
||||||
return self.argv[self.optind]
|
|
||||||
|
|
||||||
def get_option(self, options):
|
|
||||||
arg = self.argv[self.optind]
|
|
||||||
for option in options:
|
|
||||||
if arg == option:
|
|
||||||
self.optind += 1
|
|
||||||
self.optarg = self.argv[self.optind]
|
|
||||||
return True
|
|
||||||
elif option.startswith('--') and arg.startswith(f'{option}='):
|
|
||||||
self.optarg = arg[len(f'{option}='):]
|
|
||||||
return True
|
|
||||||
elif option[0] == '-' and option[1] != '-' and arg.startswith(
|
|
||||||
option):
|
|
||||||
self.optarg = arg[len(option):]
|
|
||||||
return True
|
|
||||||
return False
|
|
||||||
|
|
||||||
def unsupported(self, state, arg):
|
|
||||||
raise Exception(f'''Unsupported operand in state {state}: {arg}
|
|
||||||
|
|
||||||
You need to recognize the argument/option in the build/rbe/rewrapper_dart.py
|
|
||||||
script in order to execute this command remotely on RBE. Read the big comments
|
|
||||||
in the file explaining what this script is and how it works. Follow this stack
|
|
||||||
trace to find the place to insert the appropriate support.
|
|
||||||
''')
|
|
||||||
|
|
||||||
def rebase(self, path):
|
|
||||||
if path.startswith('package:'):
|
|
||||||
return path
|
|
||||||
# Handle the use of paths starting with an extra slash.
|
|
||||||
if path.startswith('org-dartlang-kernel-service:///'):
|
|
||||||
path = os.path.join(self.exec_root,
|
|
||||||
path[len('org-dartlang-kernel-service:///'):])
|
|
||||||
if path.startswith('org-dartlang-kernel-service://'):
|
|
||||||
path = os.path.join(self.exec_root,
|
|
||||||
path[len('org-dartlang-kernel-service://'):])
|
|
||||||
# Handle the use of paths starting with an extra slash.
|
|
||||||
if path.startswith('file:////'):
|
|
||||||
path = path[len('file:///'):]
|
|
||||||
elif path.startswith('file://'):
|
|
||||||
path = path[len('file://'):]
|
|
||||||
path = os.path.abspath(path)
|
|
||||||
if not path.startswith(self.exec_root):
|
|
||||||
raise Exception(f"Path isn't inside exec_root: {path}")
|
|
||||||
return path[len(self.exec_root):]
|
|
||||||
|
|
||||||
def parse(self):
|
|
||||||
while self.has_next_arg:
|
|
||||||
arg = self.next_arg()
|
|
||||||
if arg == 'rewrapper' or arg.endswith('/rewrapper'):
|
|
||||||
return self.parse_rewrapper()
|
|
||||||
else:
|
|
||||||
self.unsupported('rewrapper_dart', arg)
|
|
||||||
|
|
||||||
def parse_rewrapper(self):
|
|
||||||
while self.has_next_arg:
|
|
||||||
arg = self.next_arg()
|
|
||||||
if self.get_option(['--cfg']):
|
|
||||||
with open(self.optarg, 'r') as fp:
|
|
||||||
for line in fp.readlines():
|
|
||||||
key, value = fp.split('=')
|
|
||||||
if key == 'exec_root':
|
|
||||||
self.exec_root = value
|
|
||||||
elif key == 'exec_strategy':
|
|
||||||
self.exec_strategy = value
|
|
||||||
elif self.get_option(['--exec_root']):
|
|
||||||
self.exec_root = os.path.abspath(self.optarg)
|
|
||||||
if not self.exec_root.endswith('/'):
|
|
||||||
self.exec_root += '/'
|
|
||||||
elif self.get_option(['--exec_strategy']):
|
|
||||||
self.exec_strategy = self.optarg
|
|
||||||
self.exec_strategy_explicit = True
|
|
||||||
elif arg == '--':
|
|
||||||
env_exec_strategy = os.environ.get('RBE_exec_strategy')
|
|
||||||
if env_exec_strategy and not self.exec_strategy_explicit:
|
|
||||||
self.exec_strategy = env_exec_strategy
|
|
||||||
elif arg.startswith('-'):
|
|
||||||
pass # Ignore unknown rewrapper options.
|
|
||||||
elif arg.endswith('/dart'):
|
|
||||||
self.dart_subdir = os.path.dirname(arg)
|
|
||||||
return self.parse_dart()
|
|
||||||
elif arg.endswith('/gen_snapshot') or arg.endswith(
|
|
||||||
'/gen_snapshot_product'):
|
|
||||||
return self.parse_gen_snapshot()
|
|
||||||
else:
|
|
||||||
self.unsupported('rewrapper', arg)
|
|
||||||
|
|
||||||
def parse_dart(self):
|
|
||||||
while self.has_next_arg:
|
|
||||||
arg = self.next_arg()
|
|
||||||
if self.get_option(['--dfe']):
|
|
||||||
self.extra_paths.add(self.rebase(self.optarg))
|
|
||||||
elif self.get_option(['--snapshot']):
|
|
||||||
self.outputs.append(self.rebase(self.optarg))
|
|
||||||
elif self.get_option(['--depfile']):
|
|
||||||
self.depfiles = [self.rebase(self.optarg)]
|
|
||||||
elif self.get_option(['--snapshot-depfile']):
|
|
||||||
self.depfiles = [self.rebase(self.optarg)]
|
|
||||||
elif self.get_option([
|
|
||||||
'--packages', '-D', '--snapshot-kind',
|
|
||||||
'--depfile_output_filename'
|
|
||||||
]):
|
|
||||||
pass
|
|
||||||
elif arg in ['--deterministic', '--sound-null-safety']:
|
|
||||||
pass
|
|
||||||
elif arg == 'compile':
|
|
||||||
self.extra_paths.add(
|
|
||||||
self.rebase(
|
|
||||||
os.path.join(self.dart_subdir,
|
|
||||||
'snapshots/dartdev.dart.snapshot')))
|
|
||||||
self.extra_paths.add(
|
|
||||||
self.rebase(os.path.join(self.dart_subdir, '../lib')))
|
|
||||||
return self.parse_compile()
|
|
||||||
elif arg == '../../pkg/compiler/lib/src/dart2js.dart':
|
|
||||||
self.entry_points.add(self.rebase(arg))
|
|
||||||
return self.parse_dart2js()
|
|
||||||
elif arg == 'gen/utils/compiler/dart2js.dart.dill':
|
|
||||||
self.extra_paths.add(self.rebase(arg))
|
|
||||||
return self.parse_dart2js()
|
|
||||||
elif arg == '../../pkg/dev_compiler/bin/dartdevc.dart':
|
|
||||||
self.entry_points.add(self.rebase(arg))
|
|
||||||
return self.parse_dartdevc()
|
|
||||||
elif arg == 'gen/utils/ddc/dartdevc.dart.dill':
|
|
||||||
self.extra_paths.add(self.rebase(arg))
|
|
||||||
return self.parse_dartdevc()
|
|
||||||
elif arg == 'gen/utils/dartanalyzer/dartanalyzer.dart.dill':
|
|
||||||
self.extra_paths.add(self.rebase(arg))
|
|
||||||
return self.parse_dartanalyzer()
|
|
||||||
elif arg == 'gen/utils/analysis_server/analysis_server.dart.dill':
|
|
||||||
self.extra_paths.add(self.rebase(arg))
|
|
||||||
return self.parse_analysis_server()
|
|
||||||
elif arg == '../../pkg/front_end/tool/_fasta/compile_platform.dart':
|
|
||||||
self.entry_points.add(self.rebase(arg))
|
|
||||||
return self.parse_compile_platform()
|
|
||||||
elif arg == '../../utils/compiler/create_snapshot_entry.dart':
|
|
||||||
self.entry_points.add(self.rebase(arg))
|
|
||||||
self.extra_paths.add('tools/make_version.py')
|
|
||||||
# This step is very cheap and python3 isn't in the docker image.
|
|
||||||
self.no_remote = True
|
|
||||||
return self.parse_create_snapshot_entry()
|
|
||||||
elif arg == '../../utils/bazel/kernel_worker.dart':
|
|
||||||
self.entry_points.add(self.rebase(arg))
|
|
||||||
return self.parse_kernel_worker()
|
|
||||||
elif arg == '../../pkg/vm/bin/gen_kernel.dart':
|
|
||||||
self.entry_points.add(self.rebase(arg))
|
|
||||||
return self.parse_gen_kernel()
|
|
||||||
elif arg == 'gen/utils/kernel-service/frontend_server.dart.dill':
|
|
||||||
self.extra_paths.add(self.rebase(arg))
|
|
||||||
return self.parse_frontend_server()
|
|
||||||
elif arg == 'gen/utils/dtd/generate_dtd_snapshot.dart.dill':
|
|
||||||
self.extra_paths.add(self.rebase(arg))
|
|
||||||
return self.parse_generate_dtd_snapshot()
|
|
||||||
elif arg == 'gen/utils/dds/generate_dds_snapshot.dart.dill':
|
|
||||||
self.extra_paths.add(self.rebase(arg))
|
|
||||||
return self.parse_generate_dds_snapshot()
|
|
||||||
elif arg == 'gen/utils/bazel/kernel_worker.dart.dill':
|
|
||||||
self.extra_paths.add(self.rebase(arg))
|
|
||||||
return self.parse_kernel_worker()
|
|
||||||
elif arg == 'gen/utils/dartdev/generate_dartdev_snapshot.dart.dill':
|
|
||||||
self.extra_paths.add(self.rebase(arg))
|
|
||||||
return self.parse_generate_dartdev_snapshot()
|
|
||||||
elif arg == 'gen/utils/gen_kernel/bootstrap_gen_kernel.dill':
|
|
||||||
self.extra_paths.add(self.rebase(arg))
|
|
||||||
return self.parse_bootstrap_gen_kernel()
|
|
||||||
elif arg == 'gen/utils/kernel-service/kernel-service_snapshot.dart.dill':
|
|
||||||
self.extra_paths.add(self.rebase(arg))
|
|
||||||
self.extra_paths.add(
|
|
||||||
self.rebase(
|
|
||||||
os.path.join(self.dart_subdir,
|
|
||||||
'vm_platform_strong.dill')))
|
|
||||||
return self.parse_kernel_service_snapshot()
|
|
||||||
else:
|
|
||||||
self.unsupported('dart', arg)
|
|
||||||
|
|
||||||
def parse_compile(self):
|
|
||||||
while self.has_next_arg:
|
|
||||||
arg = self.next_arg()
|
|
||||||
if arg == 'js':
|
|
||||||
self.extra_paths.add(
|
|
||||||
self.rebase(
|
|
||||||
os.path.join(self.dart_subdir,
|
|
||||||
'snapshots/dart2js.dart.snapshot')))
|
|
||||||
return self.parse_dart2js()
|
|
||||||
else:
|
|
||||||
self.unsupported('compile', arg)
|
|
||||||
|
|
||||||
def parse_dart2js(self):
|
|
||||||
while self.has_next_arg:
|
|
||||||
arg = self.next_arg()
|
|
||||||
if self.get_option(['-o', '--output']):
|
|
||||||
self.outputs.append(self.rebase(self.optarg))
|
|
||||||
self.outputs.append(
|
|
||||||
self.rebase(self.optarg.replace('.js', '.js.map')))
|
|
||||||
elif self.get_option(['--platform-binaries']):
|
|
||||||
self.extra_paths.add(
|
|
||||||
self.rebase(
|
|
||||||
os.path.join(self.optarg, 'dart2js_platform.dill')))
|
|
||||||
elif self.get_option([
|
|
||||||
'--invoker', '--packages', '--libraries-spec',
|
|
||||||
'--snapshot-kind', '--depfile_output_filename'
|
|
||||||
]):
|
|
||||||
pass
|
|
||||||
elif arg in [
|
|
||||||
'--canary',
|
|
||||||
'--enable-asserts',
|
|
||||||
'-m',
|
|
||||||
'--minify',
|
|
||||||
'--no-source-maps',
|
|
||||||
]:
|
|
||||||
pass
|
|
||||||
elif not arg.startswith('-'):
|
|
||||||
self.entry_points.add(self.rebase(arg))
|
|
||||||
else:
|
|
||||||
self.unsupported('dart2js', arg)
|
|
||||||
|
|
||||||
def parse_dartdevc(self):
|
|
||||||
while self.has_next_arg:
|
|
||||||
arg = self.next_arg()
|
|
||||||
if self.get_option(['-o', '--output']):
|
|
||||||
self.outputs.append(self.rebase(self.optarg))
|
|
||||||
self.outputs.append(
|
|
||||||
self.rebase(self.optarg.replace('.js', '.js.map')))
|
|
||||||
self.outputs.append(
|
|
||||||
self.rebase(self.optarg.replace('.js', '.dill')))
|
|
||||||
elif self.get_option(['--dart-sdk-summary']):
|
|
||||||
self.extra_paths.add(self.rebase(self.optarg))
|
|
||||||
elif self.get_option([
|
|
||||||
'--multi-root-scheme', '--multi-root-output-path',
|
|
||||||
'--modules'
|
|
||||||
]):
|
|
||||||
pass
|
|
||||||
elif arg in [
|
|
||||||
'--canary', '--no-summarize', '--sound-null-safety',
|
|
||||||
'--no-sound-null-safety'
|
|
||||||
]:
|
|
||||||
pass
|
|
||||||
elif not arg.startswith('-'):
|
|
||||||
if arg.endswith('.dart'):
|
|
||||||
self.entry_points.add(self.rebase(arg))
|
|
||||||
else:
|
|
||||||
self.extra_paths.add(self.rebase(arg))
|
|
||||||
else:
|
|
||||||
self.unsupported('dartdevc', arg)
|
|
||||||
|
|
||||||
def parse_dartanalyzer(self):
|
|
||||||
while self.has_next_arg:
|
|
||||||
arg = self.next_arg()
|
|
||||||
if arg in ['--help']:
|
|
||||||
pass
|
|
||||||
else:
|
|
||||||
self.unsupported('dartanalyzer', arg)
|
|
||||||
|
|
||||||
def parse_analysis_server(self):
|
|
||||||
while self.has_next_arg:
|
|
||||||
arg = self.next_arg()
|
|
||||||
if self.get_option(['--sdk']):
|
|
||||||
self.extra_paths.add(self.rebase(self.optarg))
|
|
||||||
elif self.get_option(['--train-using']):
|
|
||||||
self.extra_paths.add(self.rebase(self.optarg))
|
|
||||||
self.entry_points.add(
|
|
||||||
self.rebase(os.path.join(self.optarg, 'compiler_api.dart')))
|
|
||||||
# This file isn't referenced from compiler_api.dart.
|
|
||||||
self.entry_points.add(
|
|
||||||
self.rebase(
|
|
||||||
os.path.join(self.optarg, 'src/io/mapped_file.dart')))
|
|
||||||
else:
|
|
||||||
self.unsupported('analysis_server', arg)
|
|
||||||
|
|
||||||
def parse_compile_platform(self):
|
|
||||||
compile_platform_args = []
|
|
||||||
single_root_scheme = None
|
|
||||||
single_root_base = None
|
|
||||||
while self.has_next_arg:
|
|
||||||
arg = self.next_arg()
|
|
||||||
if self.get_option(['--single-root-scheme']):
|
|
||||||
single_root_scheme = self.optarg
|
|
||||||
elif self.get_option(['--single-root-base']):
|
|
||||||
single_root_base = self.optarg
|
|
||||||
# Remove trailing slash to avoid duplicate slashes later.
|
|
||||||
if 1 < len(single_root_base) and single_root_base[-1] == '/':
|
|
||||||
single_root_base = single_root_base[:-1]
|
|
||||||
elif self.get_option(['-D', '--target']):
|
|
||||||
pass
|
|
||||||
elif arg in [
|
|
||||||
'--no-defines',
|
|
||||||
'--nnbd-strong',
|
|
||||||
'--nnbd-weak',
|
|
||||||
'--nnbd-agnostic',
|
|
||||||
'--exclude-source',
|
|
||||||
]:
|
|
||||||
pass
|
|
||||||
elif not arg.startswith('-'):
|
|
||||||
if len(compile_platform_args) == 0:
|
|
||||||
pass # e.g. dart:core
|
|
||||||
elif len(compile_platform_args) == 1:
|
|
||||||
sdk = arg # sdk via libraries.json
|
|
||||||
if sdk.startswith(f'{single_root_scheme}:///'):
|
|
||||||
sdk = sdk[len(f'{single_root_scheme}:///'):]
|
|
||||||
sdk = os.path.join(single_root_base, sdk)
|
|
||||||
if sdk.endswith('libraries.json'):
|
|
||||||
sdk = os.path.dirname(sdk)
|
|
||||||
self.extra_paths.add(self.rebase(sdk))
|
|
||||||
elif len(compile_platform_args) == 2: # vm_outline_strong dill
|
|
||||||
arg = self.rebase(arg)
|
|
||||||
elif len(compile_platform_args) == 3: # platform dill
|
|
||||||
arg = self.rebase(arg)
|
|
||||||
self.outputs.append(arg)
|
|
||||||
elif len(compile_platform_args) == 4: # outline dill
|
|
||||||
arg = self.rebase(arg)
|
|
||||||
self.outputs.append(arg)
|
|
||||||
if arg != compile_platform_args[2]:
|
|
||||||
self.extra_paths.add(compile_platform_args[2])
|
|
||||||
else:
|
|
||||||
self.unsupported('compile_platform', arg)
|
|
||||||
compile_platform_args.append(arg)
|
|
||||||
else:
|
|
||||||
self.unsupported('compile_platform', arg)
|
|
||||||
|
|
||||||
def parse_create_snapshot_entry(self):
|
|
||||||
while self.has_next_arg:
|
|
||||||
arg = self.next_arg()
|
|
||||||
if self.get_option(['--output_dir']):
|
|
||||||
self.outputs.append(self.rebase(self.optarg))
|
|
||||||
elif arg in ['--no-git-hash']:
|
|
||||||
pass
|
|
||||||
else:
|
|
||||||
self.unsupported('create_snapshot_entry', arg)
|
|
||||||
|
|
||||||
def parse_kernel_worker(self):
|
|
||||||
while self.has_next_arg:
|
|
||||||
arg = self.next_arg()
|
|
||||||
if self.get_option(['-o', '--output']):
|
|
||||||
self.outputs.append(self.rebase(self.optarg))
|
|
||||||
elif self.get_option(['--dart-sdk-summary']):
|
|
||||||
self.extra_paths.add(self.rebase(self.optarg))
|
|
||||||
elif self.get_option(['--source']):
|
|
||||||
self.entry_points.add(self.rebase(self.optarg))
|
|
||||||
elif self.get_option(
|
|
||||||
['--packages-file', '--target', '--dart-sdk-summary']):
|
|
||||||
pass
|
|
||||||
elif arg in [
|
|
||||||
'--summary-only',
|
|
||||||
'--sound-null-safety',
|
|
||||||
'--no-sound-null-safety',
|
|
||||||
]:
|
|
||||||
pass
|
|
||||||
else:
|
|
||||||
self.unsupported('kernel_worker', arg)
|
|
||||||
|
|
||||||
def parse_gen_kernel(self):
|
|
||||||
while self.has_next_arg:
|
|
||||||
arg = self.next_arg()
|
|
||||||
if self.get_option(['-o', '--output']):
|
|
||||||
self.outputs.append(self.rebase(self.optarg))
|
|
||||||
elif self.get_option(['--platform']):
|
|
||||||
self.extra_paths.add(self.rebase(self.optarg))
|
|
||||||
elif self.get_option([
|
|
||||||
'--packages', '-D', '--filesystem-root',
|
|
||||||
'--filesystem-scheme'
|
|
||||||
]):
|
|
||||||
pass
|
|
||||||
elif arg in ['--no-aot', '--no-embed-sources']:
|
|
||||||
pass
|
|
||||||
elif not arg.startswith('-'):
|
|
||||||
self.entry_points.add(self.rebase(arg))
|
|
||||||
else:
|
|
||||||
self.unsupported('gen_kernel', arg)
|
|
||||||
|
|
||||||
def parse_bootstrap_gen_kernel(self):
|
|
||||||
while self.has_next_arg:
|
|
||||||
arg = self.next_arg()
|
|
||||||
if self.get_option(['-o', '--output']):
|
|
||||||
self.outputs.append(self.rebase(self.optarg))
|
|
||||||
elif self.get_option(['--platform']):
|
|
||||||
self.extra_paths.add(self.rebase(self.optarg))
|
|
||||||
elif self.get_option(['--packages', '-D']):
|
|
||||||
pass
|
|
||||||
elif arg in [
|
|
||||||
'--aot',
|
|
||||||
'--no-aot',
|
|
||||||
'--no-embed-sources',
|
|
||||||
'--no-link-platform',
|
|
||||||
'--enable-asserts',
|
|
||||||
]:
|
|
||||||
pass
|
|
||||||
elif self.get_option(['--depfile']):
|
|
||||||
self.depfiles = [self.rebase(self.optarg)]
|
|
||||||
elif not arg.startswith('-'):
|
|
||||||
self.entry_points.add(self.rebase(arg))
|
|
||||||
else:
|
|
||||||
self.unsupported('bootstrap_gen_kernel', arg)
|
|
||||||
|
|
||||||
def parse_kernel_service_snapshot(self):
|
|
||||||
while self.has_next_arg:
|
|
||||||
arg = self.next_arg()
|
|
||||||
if self.get_option(['--train']):
|
|
||||||
self.entry_points.add(self.rebase(self.optarg))
|
|
||||||
else:
|
|
||||||
self.unsupported('kernel_service_snapshot', arg)
|
|
||||||
|
|
||||||
def parse_frontend_server(self):
|
|
||||||
while self.has_next_arg:
|
|
||||||
arg = self.next_arg()
|
|
||||||
if self.get_option(['--platform']):
|
|
||||||
self.extra_paths.add(self.rebase(self.optarg))
|
|
||||||
elif self.get_option(['--sdk-root']):
|
|
||||||
pass
|
|
||||||
elif arg in ['--train']:
|
|
||||||
pass
|
|
||||||
elif not arg.startswith('-'):
|
|
||||||
self.entry_points.add(self.rebase(arg))
|
|
||||||
else:
|
|
||||||
self.unsupported('frontend_server', arg)
|
|
||||||
|
|
||||||
def parse_generate_dtd_snapshot(self):
|
|
||||||
while self.has_next_arg:
|
|
||||||
arg = self.next_arg()
|
|
||||||
if arg in ['--train']:
|
|
||||||
pass
|
|
||||||
else:
|
|
||||||
self.unsupported('generate_dtd_snapshot', arg)
|
|
||||||
|
|
||||||
def parse_generate_dds_snapshot(self):
|
|
||||||
while self.has_next_arg:
|
|
||||||
arg = self.next_arg()
|
|
||||||
if arg in ['--help']:
|
|
||||||
pass
|
|
||||||
else:
|
|
||||||
self.unsupported('generate_dds_snapshot', arg)
|
|
||||||
|
|
||||||
def parse_kernel_worker(self):
|
|
||||||
while self.has_next_arg:
|
|
||||||
arg = self.next_arg()
|
|
||||||
if arg in ['--help']:
|
|
||||||
pass
|
|
||||||
elif self.get_option(['-o', '--output']):
|
|
||||||
self.outputs.append(self.rebase(self.optarg))
|
|
||||||
elif self.get_option(['--packages-file']):
|
|
||||||
self.extra_paths.add(self.rebase(self.optarg))
|
|
||||||
elif self.get_option(['--dart-sdk-summary']):
|
|
||||||
self.extra_paths.add(self.rebase(self.optarg))
|
|
||||||
elif self.get_option(['--source']):
|
|
||||||
self.entry_points.add(self.rebase(self.optarg))
|
|
||||||
elif self.get_option(['--target']):
|
|
||||||
pass
|
|
||||||
elif arg in [
|
|
||||||
'--sound-null-safety', '--no-sound-null-safety',
|
|
||||||
'--summary-only'
|
|
||||||
]:
|
|
||||||
pass
|
|
||||||
else:
|
|
||||||
self.unsupported('kernel_worker', arg)
|
|
||||||
|
|
||||||
def parse_generate_dartdev_snapshot(self):
|
|
||||||
while self.has_next_arg:
|
|
||||||
arg = self.next_arg()
|
|
||||||
if arg in ['--help']:
|
|
||||||
pass
|
|
||||||
else:
|
|
||||||
self.unsupported('generate_dartdev_snapshot', arg)
|
|
||||||
|
|
||||||
def parse_gen_snapshot(self):
|
|
||||||
while self.has_next_arg:
|
|
||||||
arg = self.next_arg()
|
|
||||||
if self.get_option(['-o', '--output']):
|
|
||||||
self.outputs.append(self.rebase(self.optarg))
|
|
||||||
elif self.get_option([
|
|
||||||
'--vm_snapshot_data',
|
|
||||||
'--vm_snapshot_instructions',
|
|
||||||
'--isolate_snapshot_data',
|
|
||||||
'--isolate_snapshot_instructions',
|
|
||||||
'--elf',
|
|
||||||
]):
|
|
||||||
self.outputs.append(self.rebase(self.optarg))
|
|
||||||
elif self.get_option(['--snapshot_kind', '--snapshot-kind']):
|
|
||||||
pass
|
|
||||||
elif arg in [
|
|
||||||
'--sound-null-safety',
|
|
||||||
'--deterministic',
|
|
||||||
'--enable-asserts',
|
|
||||||
]:
|
|
||||||
pass
|
|
||||||
elif not arg.startswith('-'):
|
|
||||||
self.extra_paths.add(self.rebase(arg))
|
|
||||||
else:
|
|
||||||
self.unsupported('gen_snapshot', arg)
|
|
||||||
|
|
||||||
|
|
||||||
def main(argv):
|
|
||||||
# Like gn_run_binary, run programs relative to the build directory. The
|
|
||||||
# command is assumed to invoke rewrapper and end its rewrapper arguments
|
|
||||||
# with an -- argument.
|
|
||||||
rewrapper_end = 0
|
|
||||||
for i in range(len(argv)):
|
|
||||||
if argv[i] == '--' and rewrapper_end == 0:
|
|
||||||
rewrapper_end = i + 1
|
|
||||||
if not '/' in argv[i + 1]:
|
|
||||||
argv[i + 1] = './' + argv[i + 1]
|
|
||||||
break
|
|
||||||
|
|
||||||
rewrapper = Rewrapper(argv)
|
|
||||||
|
|
||||||
if rewrapper.exec_root == None:
|
|
||||||
raise Exception('No rewrapper --exec_root was specified')
|
|
||||||
|
|
||||||
if not rewrapper.outputs:
|
|
||||||
raise Exception('No output files were recognized')
|
|
||||||
|
|
||||||
# Run the command directly if it's not supported for remote builds.
|
|
||||||
if rewrapper.no_remote:
|
|
||||||
run_command(argv[rewrapper_end:], 'local')
|
|
||||||
return 0
|
|
||||||
|
|
||||||
# Determine the set of input and output files.
|
|
||||||
package_config = load_package_config(rewrapper.exec_root)
|
|
||||||
if not rewrapper.depfiles:
|
|
||||||
rewrapper.depfiles = [output + '.d' for output in rewrapper.outputs]
|
|
||||||
output_files = rewrapper.outputs + rewrapper.depfiles
|
|
||||||
inputs = find_inputs(rewrapper.entry_points, rewrapper.exec_root,
|
|
||||||
package_config)
|
|
||||||
paths = set(
|
|
||||||
resolve_uri(uri, rewrapper.exec_root, package_config, whole_dir=True)
|
|
||||||
for uri in inputs)
|
|
||||||
paths.add(os.path.join('.dart_tool', 'package_config.json'))
|
|
||||||
for path in rewrapper.extra_paths:
|
|
||||||
paths.add(path)
|
|
||||||
# Ensure the working directory is included if no inputs are inside it.
|
|
||||||
working_directory = rewrapper.rebase('.')
|
|
||||||
if not any([path.startswith(working_directory) for path in paths]):
|
|
||||||
paths.add(rewrapper.rebase('build.ninja.stamp'))
|
|
||||||
paths = list(paths)
|
|
||||||
paths.sort()
|
|
||||||
|
|
||||||
# Construct the final rewrapped command line.
|
|
||||||
command = [argv[1]]
|
|
||||||
command.append('--labels=type=tool')
|
|
||||||
command.append('--inputs=' + ','.join(paths))
|
|
||||||
command.append('--output_files=' + ','.join(output_files))
|
|
||||||
# Absolute paths must not be used with RBE, but since the build currently
|
|
||||||
# heavily relies on them, work around this issue by rewriting the command
|
|
||||||
# to instead use relative paths. The Dart SDK build rules needs to be fixed
|
|
||||||
# rather than doing this, but this is an initial step towards that goal
|
|
||||||
# which will land in subsequent follow up changes.
|
|
||||||
command += argv[2:rewrapper_end] + [
|
|
||||||
rewrite_absolute(arg, rewrapper.exec_root, working_directory)
|
|
||||||
for arg in argv[rewrapper_end:]
|
|
||||||
]
|
|
||||||
|
|
||||||
# Finally execute the command remotely.
|
|
||||||
run_command(command, rewrapper.exec_strategy)
|
|
||||||
|
|
||||||
# Until the depfiles are fixed so they don't contain absoiute paths, we need
|
|
||||||
# to rewrite the absoute paths appropriately.
|
|
||||||
for depfile in rewrapper.depfiles:
|
|
||||||
lines = []
|
|
||||||
try:
|
|
||||||
with open(os.path.join(rewrapper.exec_root, depfile), 'r') as file:
|
|
||||||
lines = file.readlines()
|
|
||||||
lines = [
|
|
||||||
line.replace('/b/f/w', rewrapper.exec_root) for line in lines
|
|
||||||
]
|
|
||||||
with open(os.path.join(rewrapper.exec_root, depfile), 'w') as file:
|
|
||||||
file.writelines(lines)
|
|
||||||
except FileNotFoundError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
sys.exit(main(sys.argv))
|
|
|
@ -17,12 +17,9 @@ declare_args() {
|
||||||
# Set to the docker image to use in the RBE.
|
# Set to the docker image to use in the RBE.
|
||||||
rbe_image = "docker://gcr.io/cloud-marketplace/google/debian11@sha256:69e2789c9f3d28c6a0f13b25062c240ee7772be1f5e6d41bb4680b63eae6b304"
|
rbe_image = "docker://gcr.io/cloud-marketplace/google/debian11@sha256:69e2789c9f3d28c6a0f13b25062c240ee7772be1f5e6d41bb4680b63eae6b304"
|
||||||
|
|
||||||
# Do expensive RBE compilations both locally and remotely, whatever is faster.
|
rbe_exec_root = rebase_path("//")
|
||||||
rbe_expensive_exec_strategy = "racing"
|
|
||||||
|
|
||||||
rbe_exec_root = rebase_path("//", root_build_dir)
|
rbe_dir = rebase_path("//buildtools/reclient")
|
||||||
|
|
||||||
rbe_dir = rebase_path("//buildtools/reclient", root_build_dir)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
declare_args() {
|
declare_args() {
|
||||||
|
|
|
@ -312,11 +312,10 @@ def Main():
|
||||||
|
|
||||||
exit_code = Build(configs, env, options)
|
exit_code = Build(configs, env, options)
|
||||||
|
|
||||||
endtime = time.time()
|
|
||||||
|
|
||||||
StopRBE(env)
|
StopRBE(env)
|
||||||
|
|
||||||
if exit_code == 0:
|
if exit_code == 0:
|
||||||
|
endtime = time.time()
|
||||||
print("The build took %.3f seconds" % (endtime - starttime))
|
print("The build took %.3f seconds" % (endtime - starttime))
|
||||||
return exit_code
|
return exit_code
|
||||||
|
|
||||||
|
|
|
@ -294,10 +294,6 @@ def ToGnArgs(args, mode, arch, target_os, sanitizer, verify_sdk_hash,
|
||||||
|
|
||||||
gn_args['use_rbe'] = args.rbe
|
gn_args['use_rbe'] = args.rbe
|
||||||
|
|
||||||
if args.rbe_expensive_exec_strategy:
|
|
||||||
gn_args[
|
|
||||||
'rbe_expensive_exec_strategy'] = args.rbe_expensive_exec_strategy
|
|
||||||
|
|
||||||
# Code coverage requires -O0 to be set.
|
# Code coverage requires -O0 to be set.
|
||||||
if enable_code_coverage:
|
if enable_code_coverage:
|
||||||
gn_args['dart_debug_optimization_level'] = 0
|
gn_args['dart_debug_optimization_level'] = 0
|
||||||
|
@ -431,10 +427,6 @@ def AddCommonGnOptionArgs(parser):
|
||||||
parser.set_defaults(rbe=os.environ.get('RBE') == '1' or \
|
parser.set_defaults(rbe=os.environ.get('RBE') == '1' or \
|
||||||
os.environ.get('DART_RBE') == '1' or \
|
os.environ.get('DART_RBE') == '1' or \
|
||||||
os.environ.get('RBE_cfg') != None)
|
os.environ.get('RBE_cfg') != None)
|
||||||
parser.add_argument('--rbe-expensive-exec-strategy',
|
|
||||||
default=os.environ.get('RBE_exec_strategy'),
|
|
||||||
help='Strategy for expensive RBE compilations',
|
|
||||||
type=str)
|
|
||||||
|
|
||||||
# Disable git hashes when remote compiling to ensure cache hits of the final
|
# Disable git hashes when remote compiling to ensure cache hits of the final
|
||||||
# output artifacts when nothing has changed.
|
# output artifacts when nothing has changed.
|
||||||
|
|
Loading…
Reference in New Issue
Block a user