From 1b249f5496d7fd4923b966577c45cd30060350e0 Mon Sep 17 00:00:00 2001 From: William Hesse Date: Fri, 12 Apr 2024 10:08:28 +0000 Subject: [PATCH] Revert "[infra] RBE all the things." This reverts commit b3253d9c8b2436060096234b69860caac67efbac. Reason for revert: Flutter build is broken, it also has a GN variable rbi_dir, and dart_action.gni now imports two conflicting definitions. Original change's description: > [infra] RBE all the things. > > This change offloads all Dart actions during the build to RBE using > the new rewrapper_dart.py script that understands all the different > command line invocations of Dart programs and translates it into the > appropriate rewrapper invocation, with a full list of input and output > files and no absolute paths. > > The dart actions are all considered to be expensive RBE compilation > steps that may be much slower on RBE than locally due to the low > parallelism of the build during these stages and the comperatively > slower bots in the RBE pools. The build uses the racing strategy by > default for these compilations, such that cache hits are still used > if available, and otherwise a local build is transparently used if it > happens to be faster. The bots will use the remote strategy by default > unlike developers, such that the rewrapper_dart.py script does not fail. > > rewrapper_dart.py contains a tiny dart import proprocessor written > in python and a big argument parser that understands every command > invoked during the build and the semantic meaning of every option. > > The absolute paths used during the Dart SDK build is not solved in > this changelist, which just works around them initially, but these > will be fixed in follow up changes with the appropriate teams now > that this change proves they are not needed. > > Bug: b/333595242 > Change-Id: I36603ec1bf16f4ac87d56635cc1c98e8686a4028 > Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/335827 > Reviewed-by: William Hesse > Commit-Queue: Jonas Termansen Bug: b/333595242 Change-Id: I56c9df11e672e7bd096f2a6e7d0243dccd0396c9 No-Presubmit: true No-Tree-Checks: true No-Try: true Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/362385 Bot-Commit: Rubber Stamper Auto-Submit: William Hesse Commit-Queue: Alexander Thomas Reviewed-by: Alexander Thomas --- build/dart/dart_action.gni | 41 +- build/rbe/rewrapper_dart.py | 761 ------------------------------------ build/toolchain/rbe.gni | 7 +- tools/build.py | 3 +- tools/gn.py | 8 - 5 files changed, 11 insertions(+), 809 deletions(-) delete mode 100644 build/rbe/rewrapper_dart.py diff --git a/build/dart/dart_action.gni b/build/dart/dart_action.gni index f25d9458afc..33002af1ad6 100644 --- a/build/dart/dart_action.gni +++ b/build/dart/dart_action.gni @@ -3,7 +3,6 @@ # BSD-style license that can be found in the LICENSE file. import("../executable_suffix.gni") -import("../toolchain/rbe.gni") # This file defines templates for running and compiling Dart code during # Dart's build. @@ -83,22 +82,10 @@ template("_compiled_action") { # The "compiled_action" argument to gn_run_binary.py indicates that # it will exit with a non-zero status when the target program does. - args = [ "compiled_action" ] - - if (use_rbe && (target_os == rbe_os || - (target_os == "android" && rbe_os == "linux")) && - target_cpu == rbe_cpu) { - args += [ - "/usr/bin/python3", - rebase_path("//build/rbe/rewrapper_dart.py", root_build_dir), - ] + rewrapper_args + - [ - "--exec_strategy=" + rbe_expensive_exec_strategy, - "--", - ] - } - - args += [ rebase_path(host_executable, root_build_dir) ] + invoker.args + args = [ + "compiled_action", + rebase_path(host_executable, root_build_dir), + ] + invoker.args } } @@ -135,22 +122,10 @@ template("_prebuilt_tool_action") { inputs += [ invoker.packages ] } - args = [ "compiled_action" ] - - if (use_rbe && (target_os == rbe_os || - (target_os == "android" && rbe_os == "linux")) && - target_cpu == rbe_cpu) { - args += [ - "/usr/bin/python3", - rebase_path("//build/rbe/rewrapper_dart.py", root_build_dir), - ] + rewrapper_args + - [ - "--exec_strategy=" + rbe_expensive_exec_strategy, - "--", - ] - } - - args += [ rebase_path(invoker.binary, root_build_dir) ] + vm_args + args = [ + "compiled_action", + rebase_path(invoker.binary, root_build_dir), + ] + vm_args if (defined(invoker.packages)) { args += [ "--packages=" + rebase_path(invoker.packages, root_build_dir) ] } diff --git a/build/rbe/rewrapper_dart.py b/build/rbe/rewrapper_dart.py deleted file mode 100644 index fec2d03505c..00000000000 --- a/build/rbe/rewrapper_dart.py +++ /dev/null @@ -1,761 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file -# for details. All rights reserved. Use of this source code is governed by a -# BSD-style license that can be found in the LICENSE file. - -# This program executes Dart programs on RBE using rewrapper by parsing the -# source code to locate the input files, and recognizes the command line options -# of well known programs to determine the output files. - -# New executions during the Dart SDK build needs to be supported here in order -# to speed up the build with RBE. See the argument parser below. - -import json -import os -import re -import subprocess -import sys - - -# Run a command, swallowing the output unless there is an error. -def run_command(command, strategy): - try: - subprocess.check_output(command, stderr=subprocess.STDOUT) - return 0 - except subprocess.CalledProcessError as e: - print(e.output.decode("utf-8")) - if strategy == 'remote': - joined = ' '.join(command) - print(f'''Failed to run command remotely: {joined} - -If you're seeing this error on a bot and it doesn't happen locally, then you may -need to teach the script build/rbe/rewrapper_dart.py what the appropriate input -and outputs files of the command are. You can see the list of used inputs and -outputs in the rewrapper invocation above. To reproduce this error locally, try -forcing a remote build by setting the RBE_exec_strategy=remote environment -variable. -''') - sys.exit(1) - except OSError as e: - print(e.strerror) - sys.exit(1) - - -# Loads the package config file. -def load_package_config(exec_root): - path = os.path.join(exec_root, '.dart_tool', 'package_config.json') - with open(path, 'r') as file: - return json.load(file) - - -# Resolves a Dart import URI using the package config. -def resolve_uri(uri, exec_root, package_config, whole_dir=False): - if uri.startswith('package:'): - match = re.search(r'package:([^/]*)/(.*)', uri) - package_name = match.groups()[0] - relative = match.groups()[1] - package_data = next(pkg for pkg in package_config['packages'] - if pkg['name'] == package_name) - package_root = package_data['rootUri'] - package_root = package_root[3:] # Remove leading ../ - package_uri = package_data['packageUri'] - if whole_dir: - uri = package_root + '/' + package_uri - else: - uri = package_root + '/' + package_uri + relative - return uri - - -# Lists the imports of a Dart file uri using the package config and a rough -# parser that recognizes fairly traditional imports. This is designed to be much -# faster than actually invoking the front end. -def list_imports(uri, exec_root, package_config): - if uri.startswith('dart:'): - return set() - path = os.path.join(exec_root, resolve_uri(uri, exec_root, package_config)) - file = open(path, 'r') - imports = set() - for line in file.readlines(): - tokens = [token for token in re.split(r'\s+', line) if token != ''] - if not tokens or tokens[0] in [ - '//', '///', '/*', '*/', '#!', 'library', 'show' - ]: - continue - # Imports must happen before definitions. - if tokens[0] in ['const', 'class', 'enum']: - break - if 2 <= len(tokens - ) and tokens[0] == 'if' and tokens[1] == '(dart.library.io)': - tokens = ['import'] + tokens[2:] - if tokens[0] not in ['import', 'export', 'part']: - continue - if len(tokens) < 2: - raise Exception(f'Bad import statement: {path}: {line}') - if tokens[0] == 'part' and tokens[1] == 'of': - continue - token = tokens[1].replace('"', '').replace("'", '').replace(';', '') - if token.startswith('dart:'): - continue - if not ':' in token: - dirname = os.path.dirname(uri) - while token.startswith('..'): - token = token[3:] - dirname = os.path.dirname(dirname) - token = dirname + '/' + token - imports.add(token) - file.close() - return imports - - -# Transitively compute the set of dart files needed to execute the specified -# entry point using the package config. -def find_inputs(uris, exec_root, package_config): - inputs = set(uris) - unexplored = set(uris) - while unexplored: - uri = unexplored.pop() - imports = list_imports(uri, exec_root, package_config) - for uri in imports: - if not uri in inputs: - inputs.add(uri) - unexplored.add(uri) - return inputs - - -# Rewrite absolute paths in an argument to be relative. -def rewrite_absolute(arg, exec_root, working_directory): - # The file:// schema does not work with relative paths as they are parsed as - # the authority by the dart Uri class. - arg = arg.replace('file:///' + exec_root, '../../') - arg = arg.replace('file://' + exec_root, '../../') - # Replace the absolute exec root by a relative path to the exec root. - arg = arg.replace(exec_root, '../../') - # Simplify paths going to the exec root and back into the out directory. - # Carefully ensure the whole path isn't optimized away. - if arg.endswith(f'../../{working_directory}/'): - arg = arg.replace(f'../../{working_directory}/', '.') - else: - arg = arg.replace(f'../../{working_directory}/', '') - return arg - - -# Parse the command line execution to recognize well known programs during the -# Dart SDK build, so the inputs and output files can be determined, and the -# command can be offloaded to RBE. -# -# RBE needs a command to run, a list of input files, and a list of output files, -# and it then executes the command remotely and caches the result. Absolute -# paths must not occur in the command as the remote execution will happen in -# another directory. However, since we currently rely on absolute paths, we work -# around the issue and rewrite the absolute paths accordingly until the problem -# is fixed on our end. -# -# This is a parser that handles nested commands executing each other, taking -# care to know whose options it is currently parsing, and extracting the -# appropriate information from each argument. Every invoked program and option -# during the build needs to be supported here, otherwise the remote command may -# be inaccurate and not have right inputs and outputs. Although maintaining this -# parser takes some effort, it is being paid back in the builders being sped up -# massively on cache hits, as well as speeding up any local developers that -# build code already built by the bots. -# -# To add a new program, recognize the entry point and define its parser method. -# To add a new option, parse the option in the appropriate method and either -# ignore it or recognize any input and output files. All invoked options needs -# be allowlisted here know we didn't accidentally misunderstand the invoked -# command when running it remotely. -class Rewrapper: - - def __init__(self, argv): - self.dart_subdir = None - self.depfiles = None - self.entry_points = set() - self.exec_root = None - self.exec_strategy = 'remote' - self.exec_strategy_explicit = False - self.extra_paths = set() - self.outputs = [] - self.no_remote = None - self.argv = argv - self.optarg = None - self.optind = 0 - self.parse() - - @property - def has_next_arg(self): - return self.optind + 1 < len(self.argv) - - def next_arg(self): - self.optind += 1 - return self.argv[self.optind] - - def get_option(self, options): - arg = self.argv[self.optind] - for option in options: - if arg == option: - self.optind += 1 - self.optarg = self.argv[self.optind] - return True - elif option.startswith('--') and arg.startswith(f'{option}='): - self.optarg = arg[len(f'{option}='):] - return True - elif option[0] == '-' and option[1] != '-' and arg.startswith( - option): - self.optarg = arg[len(option):] - return True - return False - - def unsupported(self, state, arg): - raise Exception(f'''Unsupported operand in state {state}: {arg} - -You need to recognize the argument/option in the build/rbe/rewrapper_dart.py -script in order to execute this command remotely on RBE. Read the big comments -in the file explaining what this script is and how it works. Follow this stack -trace to find the place to insert the appropriate support. -''') - - def rebase(self, path): - if path.startswith('package:'): - return path - # Handle the use of paths starting with an extra slash. - if path.startswith('org-dartlang-kernel-service:///'): - path = os.path.join(self.exec_root, - path[len('org-dartlang-kernel-service:///'):]) - if path.startswith('org-dartlang-kernel-service://'): - path = os.path.join(self.exec_root, - path[len('org-dartlang-kernel-service://'):]) - # Handle the use of paths starting with an extra slash. - if path.startswith('file:////'): - path = path[len('file:///'):] - elif path.startswith('file://'): - path = path[len('file://'):] - path = os.path.abspath(path) - if not path.startswith(self.exec_root): - raise Exception(f"Path isn't inside exec_root: {path}") - return path[len(self.exec_root):] - - def parse(self): - while self.has_next_arg: - arg = self.next_arg() - if arg == 'rewrapper' or arg.endswith('/rewrapper'): - return self.parse_rewrapper() - else: - self.unsupported('rewrapper_dart', arg) - - def parse_rewrapper(self): - while self.has_next_arg: - arg = self.next_arg() - if self.get_option(['--cfg']): - with open(self.optarg, 'r') as fp: - for line in fp.readlines(): - key, value = fp.split('=') - if key == 'exec_root': - self.exec_root = value - elif key == 'exec_strategy': - self.exec_strategy = value - elif self.get_option(['--exec_root']): - self.exec_root = os.path.abspath(self.optarg) - if not self.exec_root.endswith('/'): - self.exec_root += '/' - elif self.get_option(['--exec_strategy']): - self.exec_strategy = self.optarg - self.exec_strategy_explicit = True - elif arg == '--': - env_exec_strategy = os.environ.get('RBE_exec_strategy') - if env_exec_strategy and not self.exec_strategy_explicit: - self.exec_strategy = env_exec_strategy - elif arg.startswith('-'): - pass # Ignore unknown rewrapper options. - elif arg.endswith('/dart'): - self.dart_subdir = os.path.dirname(arg) - return self.parse_dart() - elif arg.endswith('/gen_snapshot') or arg.endswith( - '/gen_snapshot_product'): - return self.parse_gen_snapshot() - else: - self.unsupported('rewrapper', arg) - - def parse_dart(self): - while self.has_next_arg: - arg = self.next_arg() - if self.get_option(['--dfe']): - self.extra_paths.add(self.rebase(self.optarg)) - elif self.get_option(['--snapshot']): - self.outputs.append(self.rebase(self.optarg)) - elif self.get_option(['--depfile']): - self.depfiles = [self.rebase(self.optarg)] - elif self.get_option(['--snapshot-depfile']): - self.depfiles = [self.rebase(self.optarg)] - elif self.get_option([ - '--packages', '-D', '--snapshot-kind', - '--depfile_output_filename' - ]): - pass - elif arg in ['--deterministic', '--sound-null-safety']: - pass - elif arg == 'compile': - self.extra_paths.add( - self.rebase( - os.path.join(self.dart_subdir, - 'snapshots/dartdev.dart.snapshot'))) - self.extra_paths.add( - self.rebase(os.path.join(self.dart_subdir, '../lib'))) - return self.parse_compile() - elif arg == '../../pkg/compiler/lib/src/dart2js.dart': - self.entry_points.add(self.rebase(arg)) - return self.parse_dart2js() - elif arg == 'gen/utils/compiler/dart2js.dart.dill': - self.extra_paths.add(self.rebase(arg)) - return self.parse_dart2js() - elif arg == '../../pkg/dev_compiler/bin/dartdevc.dart': - self.entry_points.add(self.rebase(arg)) - return self.parse_dartdevc() - elif arg == 'gen/utils/ddc/dartdevc.dart.dill': - self.extra_paths.add(self.rebase(arg)) - return self.parse_dartdevc() - elif arg == 'gen/utils/dartanalyzer/dartanalyzer.dart.dill': - self.extra_paths.add(self.rebase(arg)) - return self.parse_dartanalyzer() - elif arg == 'gen/utils/analysis_server/analysis_server.dart.dill': - self.extra_paths.add(self.rebase(arg)) - return self.parse_analysis_server() - elif arg == '../../pkg/front_end/tool/_fasta/compile_platform.dart': - self.entry_points.add(self.rebase(arg)) - return self.parse_compile_platform() - elif arg == '../../utils/compiler/create_snapshot_entry.dart': - self.entry_points.add(self.rebase(arg)) - self.extra_paths.add('tools/make_version.py') - # This step is very cheap and python3 isn't in the docker image. - self.no_remote = True - return self.parse_create_snapshot_entry() - elif arg == '../../utils/bazel/kernel_worker.dart': - self.entry_points.add(self.rebase(arg)) - return self.parse_kernel_worker() - elif arg == '../../pkg/vm/bin/gen_kernel.dart': - self.entry_points.add(self.rebase(arg)) - return self.parse_gen_kernel() - elif arg == 'gen/utils/kernel-service/frontend_server.dart.dill': - self.extra_paths.add(self.rebase(arg)) - return self.parse_frontend_server() - elif arg == 'gen/utils/dtd/generate_dtd_snapshot.dart.dill': - self.extra_paths.add(self.rebase(arg)) - return self.parse_generate_dtd_snapshot() - elif arg == 'gen/utils/dds/generate_dds_snapshot.dart.dill': - self.extra_paths.add(self.rebase(arg)) - return self.parse_generate_dds_snapshot() - elif arg == 'gen/utils/bazel/kernel_worker.dart.dill': - self.extra_paths.add(self.rebase(arg)) - return self.parse_kernel_worker() - elif arg == 'gen/utils/dartdev/generate_dartdev_snapshot.dart.dill': - self.extra_paths.add(self.rebase(arg)) - return self.parse_generate_dartdev_snapshot() - elif arg == 'gen/utils/gen_kernel/bootstrap_gen_kernel.dill': - self.extra_paths.add(self.rebase(arg)) - return self.parse_bootstrap_gen_kernel() - elif arg == 'gen/utils/kernel-service/kernel-service_snapshot.dart.dill': - self.extra_paths.add(self.rebase(arg)) - self.extra_paths.add( - self.rebase( - os.path.join(self.dart_subdir, - 'vm_platform_strong.dill'))) - return self.parse_kernel_service_snapshot() - else: - self.unsupported('dart', arg) - - def parse_compile(self): - while self.has_next_arg: - arg = self.next_arg() - if arg == 'js': - self.extra_paths.add( - self.rebase( - os.path.join(self.dart_subdir, - 'snapshots/dart2js.dart.snapshot'))) - return self.parse_dart2js() - else: - self.unsupported('compile', arg) - - def parse_dart2js(self): - while self.has_next_arg: - arg = self.next_arg() - if self.get_option(['-o', '--output']): - self.outputs.append(self.rebase(self.optarg)) - self.outputs.append( - self.rebase(self.optarg.replace('.js', '.js.map'))) - elif self.get_option(['--platform-binaries']): - self.extra_paths.add( - self.rebase( - os.path.join(self.optarg, 'dart2js_platform.dill'))) - elif self.get_option([ - '--invoker', '--packages', '--libraries-spec', - '--snapshot-kind', '--depfile_output_filename' - ]): - pass - elif arg in [ - '--canary', - '--enable-asserts', - '-m', - '--minify', - '--no-source-maps', - ]: - pass - elif not arg.startswith('-'): - self.entry_points.add(self.rebase(arg)) - else: - self.unsupported('dart2js', arg) - - def parse_dartdevc(self): - while self.has_next_arg: - arg = self.next_arg() - if self.get_option(['-o', '--output']): - self.outputs.append(self.rebase(self.optarg)) - self.outputs.append( - self.rebase(self.optarg.replace('.js', '.js.map'))) - self.outputs.append( - self.rebase(self.optarg.replace('.js', '.dill'))) - elif self.get_option(['--dart-sdk-summary']): - self.extra_paths.add(self.rebase(self.optarg)) - elif self.get_option([ - '--multi-root-scheme', '--multi-root-output-path', - '--modules' - ]): - pass - elif arg in [ - '--canary', '--no-summarize', '--sound-null-safety', - '--no-sound-null-safety' - ]: - pass - elif not arg.startswith('-'): - if arg.endswith('.dart'): - self.entry_points.add(self.rebase(arg)) - else: - self.extra_paths.add(self.rebase(arg)) - else: - self.unsupported('dartdevc', arg) - - def parse_dartanalyzer(self): - while self.has_next_arg: - arg = self.next_arg() - if arg in ['--help']: - pass - else: - self.unsupported('dartanalyzer', arg) - - def parse_analysis_server(self): - while self.has_next_arg: - arg = self.next_arg() - if self.get_option(['--sdk']): - self.extra_paths.add(self.rebase(self.optarg)) - elif self.get_option(['--train-using']): - self.extra_paths.add(self.rebase(self.optarg)) - self.entry_points.add( - self.rebase(os.path.join(self.optarg, 'compiler_api.dart'))) - # This file isn't referenced from compiler_api.dart. - self.entry_points.add( - self.rebase( - os.path.join(self.optarg, 'src/io/mapped_file.dart'))) - else: - self.unsupported('analysis_server', arg) - - def parse_compile_platform(self): - compile_platform_args = [] - single_root_scheme = None - single_root_base = None - while self.has_next_arg: - arg = self.next_arg() - if self.get_option(['--single-root-scheme']): - single_root_scheme = self.optarg - elif self.get_option(['--single-root-base']): - single_root_base = self.optarg - # Remove trailing slash to avoid duplicate slashes later. - if 1 < len(single_root_base) and single_root_base[-1] == '/': - single_root_base = single_root_base[:-1] - elif self.get_option(['-D', '--target']): - pass - elif arg in [ - '--no-defines', - '--nnbd-strong', - '--nnbd-weak', - '--nnbd-agnostic', - '--exclude-source', - ]: - pass - elif not arg.startswith('-'): - if len(compile_platform_args) == 0: - pass # e.g. dart:core - elif len(compile_platform_args) == 1: - sdk = arg # sdk via libraries.json - if sdk.startswith(f'{single_root_scheme}:///'): - sdk = sdk[len(f'{single_root_scheme}:///'):] - sdk = os.path.join(single_root_base, sdk) - if sdk.endswith('libraries.json'): - sdk = os.path.dirname(sdk) - self.extra_paths.add(self.rebase(sdk)) - elif len(compile_platform_args) == 2: # vm_outline_strong dill - arg = self.rebase(arg) - elif len(compile_platform_args) == 3: # platform dill - arg = self.rebase(arg) - self.outputs.append(arg) - elif len(compile_platform_args) == 4: # outline dill - arg = self.rebase(arg) - self.outputs.append(arg) - if arg != compile_platform_args[2]: - self.extra_paths.add(compile_platform_args[2]) - else: - self.unsupported('compile_platform', arg) - compile_platform_args.append(arg) - else: - self.unsupported('compile_platform', arg) - - def parse_create_snapshot_entry(self): - while self.has_next_arg: - arg = self.next_arg() - if self.get_option(['--output_dir']): - self.outputs.append(self.rebase(self.optarg)) - elif arg in ['--no-git-hash']: - pass - else: - self.unsupported('create_snapshot_entry', arg) - - def parse_kernel_worker(self): - while self.has_next_arg: - arg = self.next_arg() - if self.get_option(['-o', '--output']): - self.outputs.append(self.rebase(self.optarg)) - elif self.get_option(['--dart-sdk-summary']): - self.extra_paths.add(self.rebase(self.optarg)) - elif self.get_option(['--source']): - self.entry_points.add(self.rebase(self.optarg)) - elif self.get_option( - ['--packages-file', '--target', '--dart-sdk-summary']): - pass - elif arg in [ - '--summary-only', - '--sound-null-safety', - '--no-sound-null-safety', - ]: - pass - else: - self.unsupported('kernel_worker', arg) - - def parse_gen_kernel(self): - while self.has_next_arg: - arg = self.next_arg() - if self.get_option(['-o', '--output']): - self.outputs.append(self.rebase(self.optarg)) - elif self.get_option(['--platform']): - self.extra_paths.add(self.rebase(self.optarg)) - elif self.get_option([ - '--packages', '-D', '--filesystem-root', - '--filesystem-scheme' - ]): - pass - elif arg in ['--no-aot', '--no-embed-sources']: - pass - elif not arg.startswith('-'): - self.entry_points.add(self.rebase(arg)) - else: - self.unsupported('gen_kernel', arg) - - def parse_bootstrap_gen_kernel(self): - while self.has_next_arg: - arg = self.next_arg() - if self.get_option(['-o', '--output']): - self.outputs.append(self.rebase(self.optarg)) - elif self.get_option(['--platform']): - self.extra_paths.add(self.rebase(self.optarg)) - elif self.get_option(['--packages', '-D']): - pass - elif arg in [ - '--aot', - '--no-aot', - '--no-embed-sources', - '--no-link-platform', - '--enable-asserts', - ]: - pass - elif self.get_option(['--depfile']): - self.depfiles = [self.rebase(self.optarg)] - elif not arg.startswith('-'): - self.entry_points.add(self.rebase(arg)) - else: - self.unsupported('bootstrap_gen_kernel', arg) - - def parse_kernel_service_snapshot(self): - while self.has_next_arg: - arg = self.next_arg() - if self.get_option(['--train']): - self.entry_points.add(self.rebase(self.optarg)) - else: - self.unsupported('kernel_service_snapshot', arg) - - def parse_frontend_server(self): - while self.has_next_arg: - arg = self.next_arg() - if self.get_option(['--platform']): - self.extra_paths.add(self.rebase(self.optarg)) - elif self.get_option(['--sdk-root']): - pass - elif arg in ['--train']: - pass - elif not arg.startswith('-'): - self.entry_points.add(self.rebase(arg)) - else: - self.unsupported('frontend_server', arg) - - def parse_generate_dtd_snapshot(self): - while self.has_next_arg: - arg = self.next_arg() - if arg in ['--train']: - pass - else: - self.unsupported('generate_dtd_snapshot', arg) - - def parse_generate_dds_snapshot(self): - while self.has_next_arg: - arg = self.next_arg() - if arg in ['--help']: - pass - else: - self.unsupported('generate_dds_snapshot', arg) - - def parse_kernel_worker(self): - while self.has_next_arg: - arg = self.next_arg() - if arg in ['--help']: - pass - elif self.get_option(['-o', '--output']): - self.outputs.append(self.rebase(self.optarg)) - elif self.get_option(['--packages-file']): - self.extra_paths.add(self.rebase(self.optarg)) - elif self.get_option(['--dart-sdk-summary']): - self.extra_paths.add(self.rebase(self.optarg)) - elif self.get_option(['--source']): - self.entry_points.add(self.rebase(self.optarg)) - elif self.get_option(['--target']): - pass - elif arg in [ - '--sound-null-safety', '--no-sound-null-safety', - '--summary-only' - ]: - pass - else: - self.unsupported('kernel_worker', arg) - - def parse_generate_dartdev_snapshot(self): - while self.has_next_arg: - arg = self.next_arg() - if arg in ['--help']: - pass - else: - self.unsupported('generate_dartdev_snapshot', arg) - - def parse_gen_snapshot(self): - while self.has_next_arg: - arg = self.next_arg() - if self.get_option(['-o', '--output']): - self.outputs.append(self.rebase(self.optarg)) - elif self.get_option([ - '--vm_snapshot_data', - '--vm_snapshot_instructions', - '--isolate_snapshot_data', - '--isolate_snapshot_instructions', - '--elf', - ]): - self.outputs.append(self.rebase(self.optarg)) - elif self.get_option(['--snapshot_kind', '--snapshot-kind']): - pass - elif arg in [ - '--sound-null-safety', - '--deterministic', - '--enable-asserts', - ]: - pass - elif not arg.startswith('-'): - self.extra_paths.add(self.rebase(arg)) - else: - self.unsupported('gen_snapshot', arg) - - -def main(argv): - # Like gn_run_binary, run programs relative to the build directory. The - # command is assumed to invoke rewrapper and end its rewrapper arguments - # with an -- argument. - rewrapper_end = 0 - for i in range(len(argv)): - if argv[i] == '--' and rewrapper_end == 0: - rewrapper_end = i + 1 - if not '/' in argv[i + 1]: - argv[i + 1] = './' + argv[i + 1] - break - - rewrapper = Rewrapper(argv) - - if rewrapper.exec_root == None: - raise Exception('No rewrapper --exec_root was specified') - - if not rewrapper.outputs: - raise Exception('No output files were recognized') - - # Run the command directly if it's not supported for remote builds. - if rewrapper.no_remote: - run_command(argv[rewrapper_end:], 'local') - return 0 - - # Determine the set of input and output files. - package_config = load_package_config(rewrapper.exec_root) - if not rewrapper.depfiles: - rewrapper.depfiles = [output + '.d' for output in rewrapper.outputs] - output_files = rewrapper.outputs + rewrapper.depfiles - inputs = find_inputs(rewrapper.entry_points, rewrapper.exec_root, - package_config) - paths = set( - resolve_uri(uri, rewrapper.exec_root, package_config, whole_dir=True) - for uri in inputs) - paths.add(os.path.join('.dart_tool', 'package_config.json')) - for path in rewrapper.extra_paths: - paths.add(path) - # Ensure the working directory is included if no inputs are inside it. - working_directory = rewrapper.rebase('.') - if not any([path.startswith(working_directory) for path in paths]): - paths.add(rewrapper.rebase('build.ninja.stamp')) - paths = list(paths) - paths.sort() - - # Construct the final rewrapped command line. - command = [argv[1]] - command.append('--labels=type=tool') - command.append('--inputs=' + ','.join(paths)) - command.append('--output_files=' + ','.join(output_files)) - # Absolute paths must not be used with RBE, but since the build currently - # heavily relies on them, work around this issue by rewriting the command - # to instead use relative paths. The Dart SDK build rules needs to be fixed - # rather than doing this, but this is an initial step towards that goal - # which will land in subsequent follow up changes. - command += argv[2:rewrapper_end] + [ - rewrite_absolute(arg, rewrapper.exec_root, working_directory) - for arg in argv[rewrapper_end:] - ] - - # Finally execute the command remotely. - run_command(command, rewrapper.exec_strategy) - - # Until the depfiles are fixed so they don't contain absoiute paths, we need - # to rewrite the absoute paths appropriately. - for depfile in rewrapper.depfiles: - lines = [] - try: - with open(os.path.join(rewrapper.exec_root, depfile), 'r') as file: - lines = file.readlines() - lines = [ - line.replace('/b/f/w', rewrapper.exec_root) for line in lines - ] - with open(os.path.join(rewrapper.exec_root, depfile), 'w') as file: - file.writelines(lines) - except FileNotFoundError: - pass - - return 0 - - -if __name__ == '__main__': - sys.exit(main(sys.argv)) diff --git a/build/toolchain/rbe.gni b/build/toolchain/rbe.gni index dda0ca5f8c9..961e33e9eb5 100644 --- a/build/toolchain/rbe.gni +++ b/build/toolchain/rbe.gni @@ -17,12 +17,9 @@ declare_args() { # Set to the docker image to use in the RBE. rbe_image = "docker://gcr.io/cloud-marketplace/google/debian11@sha256:69e2789c9f3d28c6a0f13b25062c240ee7772be1f5e6d41bb4680b63eae6b304" - # Do expensive RBE compilations both locally and remotely, whatever is faster. - rbe_expensive_exec_strategy = "racing" + rbe_exec_root = rebase_path("//") - rbe_exec_root = rebase_path("//", root_build_dir) - - rbe_dir = rebase_path("//buildtools/reclient", root_build_dir) + rbe_dir = rebase_path("//buildtools/reclient") } declare_args() { diff --git a/tools/build.py b/tools/build.py index c6bab348bd6..62a57022c44 100755 --- a/tools/build.py +++ b/tools/build.py @@ -312,11 +312,10 @@ def Main(): exit_code = Build(configs, env, options) - endtime = time.time() - StopRBE(env) if exit_code == 0: + endtime = time.time() print("The build took %.3f seconds" % (endtime - starttime)) return exit_code diff --git a/tools/gn.py b/tools/gn.py index 11e88fd9437..3984e591715 100755 --- a/tools/gn.py +++ b/tools/gn.py @@ -294,10 +294,6 @@ def ToGnArgs(args, mode, arch, target_os, sanitizer, verify_sdk_hash, gn_args['use_rbe'] = args.rbe - if args.rbe_expensive_exec_strategy: - gn_args[ - 'rbe_expensive_exec_strategy'] = args.rbe_expensive_exec_strategy - # Code coverage requires -O0 to be set. if enable_code_coverage: gn_args['dart_debug_optimization_level'] = 0 @@ -431,10 +427,6 @@ def AddCommonGnOptionArgs(parser): parser.set_defaults(rbe=os.environ.get('RBE') == '1' or \ os.environ.get('DART_RBE') == '1' or \ os.environ.get('RBE_cfg') != None) - parser.add_argument('--rbe-expensive-exec-strategy', - default=os.environ.get('RBE_exec_strategy'), - help='Strategy for expensive RBE compilations', - type=str) # Disable git hashes when remote compiling to ensure cache hits of the final # output artifacts when nothing has changed.