dart-sdk/tools/gn.py

785 lines
29 KiB
Python
Raw Normal View History

#!/usr/bin/env python3
# Copyright 2016 The Dart project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import argparse
import os
import platform
import subprocess
import socket
import sys
import time
import utils
HOST_OS = utils.GuessOS()
HOST_ARCH = utils.GuessArchitecture()
SCRIPT_DIR = os.path.dirname(sys.argv[0])
DART_ROOT = os.path.realpath(os.path.join(SCRIPT_DIR, '..'))
AVAILABLE_ARCHS = utils.ARCH_FAMILY.keys()
# Environment variables for default settings.
DART_USE_TOOLCHAIN = "DART_USE_TOOLCHAIN" # Use instead of --toolchain-prefix
DART_USE_SYSROOT = "DART_USE_SYSROOT" # Use instead of --target-sysroot
DART_USE_CRASHPAD = "DART_USE_CRASHPAD" # Use instead of --use-crashpad
# use instead of --platform-sdk
DART_MAKE_PLATFORM_SDK = "DART_MAKE_PLATFORM_SDK"
DART_GN_ARGS = "DART_GN_ARGS"
def ToolchainPrefix(args):
if args.toolchain_prefix:
return args.toolchain_prefix
return os.environ.get(DART_USE_TOOLCHAIN)
def TargetSysroot(args):
if args.target_sysroot:
return args.target_sysroot
return os.environ.get(DART_USE_SYSROOT)
def MakePlatformSDK():
return DART_MAKE_PLATFORM_SDK in os.environ
def GetGNArgs(args):
if args.gn_args != None:
return args.gn_args
args = os.environ.get(DART_GN_ARGS) or ""
return args.split()
def GetOutDir(mode, arch, target_os, sanitizer):
return utils.GetBuildRoot(HOST_OS, mode, arch, target_os, sanitizer)
def ToCommandLine(gn_args):
def merge(key, value):
if type(value) is bool:
return '%s=%s' % (key, 'true' if value else 'false')
elif type(value) is int:
return '%s=%d' % (key, value)
return '%s="%s"' % (key, value)
return [merge(x, y) for x, y in gn_args.items()]
# The C compiler's target under the host toolchain (DART_HOST_ARCH_***).
def HostCpuForArch(arch):
arch = arch.split("_")[-1]
# For each target architecture, we prefer in descending order
# - using the same architecture for the host (supports all architectures)
# - using a host architecture with the same word size (supports arm and riscv, which have simulators)
# - using a host architecture with a different word size (supports only AOT and only 32-bit target on 64-bit host)
if arch in ['ia32']:
candidates = ['x86']
elif arch in ['x64', 'x64c', 'simx64', 'simx64c']:
candidates = ['x64', 'arm64']
elif arch in ['arm', 'simarm']:
candidates = ['arm', 'x86', 'riscv32', 'arm64', 'x64', 'riscv64']
elif arch in ['arm64', 'arm64c', 'simarm64', 'simarm64c']:
candidates = ['arm64', 'x64', 'riscv64']
elif arch in ['riscv32', 'simriscv32']:
candidates = ['riscv32', 'arm', 'x86', 'riscv64', 'arm64', 'x64']
elif arch in ['riscv64', 'simriscv64']:
candidates = ['riscv64', 'arm64', 'x64']
else:
raise Exception("Unknown Dart architecture: %s" % arch)
available = utils.HostArchitectures()
for candidate in candidates:
if candidate in available:
return candidate
raise Exception(
"Failed to find a C host architecture for %s. Need one of %s but only %s are available."
% (arch, candidates, available))
# The C compiler's target under the target toolchain (DART_HOST_ARCH_***).
def TargetCpuForArch(arch):
# Real target architectures
if arch.startswith('ia32'):
return 'x86'
elif arch.startswith('x64'):
return 'x64'
elif arch.startswith('arm64'):
return 'arm64'
elif arch.startswith('arm'):
return 'arm'
elif arch.startswith('riscv32'):
return 'riscv32'
elif arch.startswith('riscv64'):
return 'riscv64'
# Simulators
if arch.endswith('_x64'):
return 'x64'
elif arch.endswith('_arm64'):
return 'arm64'
elif arch.endswith('_riscv64'):
return 'riscv64'
elif arch in ['simarm', 'simriscv32']:
candidates = ['arm', 'riscv32', 'x86']
elif arch in ['simx64', 'simx64c', 'simarm64', 'simarm64c', 'simriscv64']:
candidates = ['arm64', 'riscv64', 'x64']
else:
raise Exception("Unknown Dart architecture: %s" % arch)
available = utils.HostArchitectures()
for candidate in candidates:
if candidate in available:
return candidate
raise Exception(
"Failed to find a C target architecture for %s. Need one of %s but only %s are available."
% (arch, candidates, available))
# The Dart compiler's target (DART_TARGET_ARCH_***)
def DartTargetCpuForArch(arch):
arch = arch.split("_")[0]
if arch.startswith("sim"):
arch = arch[3:]
if arch.endswith("c"):
arch = arch[:-1]
return arch
def IsCompressedPointerArch(arch):
return "64c" in arch
def HostOsForGn(host_os):
if host_os.startswith('macos'):
return 'mac'
if host_os.startswith('win'):
return 'win'
return host_os
# Where string_map is formatted as X1=Y1,X2=Y2 etc.
# If key is X1, returns Y1.
def ParseStringMap(key, string_map):
for m in string_map.split(','):
l = m.split('=')
if l[0] == key:
return l[1]
return None
def UseSysroot(args, gn_args):
# Don't try to use a Linux sysroot if we aren't on Linux.
if gn_args['target_os'] != 'linux' and HOST_OS != 'linux':
return False
# Don't use the sysroot if we're given another sysroot.
if TargetSysroot(args):
return False
# Don't use the sysroot if we're given another toolchain.
if not gn_args['is_clang']:
return False
# Fuchsia's Linux sysroot doesn't support RISCV32
if gn_args['target_cpu'] in ['riscv32']:
[vm] Support RISC-V. Implements a backend targeting RV32GC and RV64GC, based on Linux standardizing around GC. The assembler is written to make it easy to disable usage of C, but because the sizes of some instruction sequences are compile-time constants, an additional build configuration would need to be defined to make use of it. The assembler and disassembler cover every RV32/64GC instruction. The simulator covers all instructions except accessing CSRs and the floating point state accessible through such, include accrued exceptions and dynamic rounding mode. Quirks: - RISC-V is a compare-and-branch architecture, but some existing "architecture-independent" parts of the Dart compiler assume a condition code architecture. To avoid rewriting these parts, we use a peephole in the assembler to map to compare-and-branch. See Assembler::BranchIf. Luckily nothing depended on taking multiple branches on the same condition code set. - There are no hardware overflow checks, so we must use Hacker's Delight style software checks. Often these are very cheap: if the sign of one operand is known, a single branch is needed. - The ranges of RISC-V branches and jumps are such that we use 3 levels of generation for forward branches, instead of the 2 levels of near and far branches used on ARM[64]. Nearly all code is handled by the first two levels with 20-bits of range, with enormous regex matchers triggering the third level that uses aupic+jalr to get 32-bits of range. - For PC-relative calls in AOT, we always generate auipc+jalr pairs with 32-bits of range, so we never generate trampolines. - Only a subset of registers are available in some compressed instructions, so we assign the most popular uses to these registers. In particular, THR, TMP[2], CODE and PP. This has the effect of assigning CODE and PP to volatile registers in the C calling convention, whereas they are assigned preserved registers on the other architectures. As on ARM64, PP is untagged; this is so short indices can be accessed with a compressed instruction. - There are no push or pop instructions, so combining pushes and pops is preferred so we can update SP once. - The C calling convention has a strongly aligned stack, but unlike on ARM64 we don't need to use an alternate stack pointer. The author ensured language was added to the RISC-V psABI making the OS responsible for realigning the stack pointer for signal handlers, allowing Dart to leave the stack pointer misaligned from the C calling convention's point of view until a foreign call. - We don't bother with the link register tracking done on ARM[64]. Instead we make use of an alternate link register to avoid inline spilling in the write barrier. Unimplemented: - non-trivial FFI cases - Compressed pointers - No intention to implement. - Unboxed SIMD - We might make use of the V extension registers when the V extension is ratified. - BigInt intrinsics TEST=existing tests for IL level, new tests for assembler/disassembler/simulator Bug: https://github.com/dart-lang/sdk/issues/38587 Bug: https://github.com/dart-lang/sdk/issues/48164 Change-Id: I991d1df4be5bf55efec5371b767b332d37dfa3e0 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/217289 Reviewed-by: Alexander Markov <alexmarkov@google.com> Reviewed-by: Daco Harkes <dacoharkes@google.com> Reviewed-by: Slava Egorov <vegorov@google.com> Commit-Queue: Ryan Macnak <rmacnak@google.com>
2022-01-20 00:57:57 +00:00
return False
# Otherwise use the sysroot.
return True
def ToGnArgs(args, mode, arch, target_os, sanitizer, verify_sdk_hash,
git_version):
gn_args = {}
host_os = HostOsForGn(HOST_OS)
if target_os == 'host':
gn_args['target_os'] = host_os
else:
gn_args['target_os'] = target_os
gn_args['host_cpu'] = HostCpuForArch(arch)
gn_args['target_cpu'] = TargetCpuForArch(arch)
gn_args['dart_target_arch'] = DartTargetCpuForArch(arch)
gn_args['dart_use_compressed_pointers'] = IsCompressedPointerArch(arch)
# Configure Crashpad library if it is used.
gn_args['dart_use_crashpad'] = ((args.use_crashpad or
DART_USE_CRASHPAD in os.environ) and
gn_args['target_cpu'] in ['x86', 'x64'])
if gn_args['dart_use_crashpad']:
# Tell Crashpad's BUILD files which checkout layout to use.
gn_args['crashpad_dependencies'] = 'dart'
if DartTargetCpuForArch(arch) != HostCpuForArch(arch):
# Training an app-jit snapshot under a simulator is slow. Use script
# snapshots instead.
gn_args['dart_snapshot_kind'] = 'kernel'
else:
gn_args['dart_snapshot_kind'] = 'app-jit'
# We only want the fallback root certs in the standalone VM on
# Linux and Windows.
if gn_args['target_os'] in ['linux', 'win']:
gn_args['dart_use_fallback_root_certificates'] = True
gn_args['bssl_use_clang_integrated_as'] = True
if gn_args['target_os'] == 'linux':
if gn_args['target_cpu'] == 'arm':
# Default to -mfloat-abi=hard and -mfpu=neon for arm on Linux as we're
# specifying a gnueabihf compiler in //build/toolchain/linux/BUILD.gn.
floatabi = 'hard' if args.arm_float_abi == '' else args.arm_float_abi
gn_args['arm_version'] = 7
gn_args['arm_float_abi'] = floatabi
gn_args['arm_use_neon'] = True
if gn_args['target_os'] == 'fuchsia':
gn_args['fuchsia_target_api_level'] = 19
gn_args['is_debug'] = mode == 'debug'
gn_args['is_release'] = mode == 'release'
gn_args['is_product'] = mode == 'product'
gn_args['dart_debug'] = mode == 'debug'
# This setting is only meaningful for Flutter. Standalone builds of the VM
# should leave this set to 'develop', which causes the build to defer to
# 'is_debug', 'is_release' and 'is_product'.
if mode == 'product':
gn_args['dart_runtime_mode'] = 'release'
else:
gn_args['dart_runtime_mode'] = 'develop'
gn_args['exclude_kernel_service'] = args.exclude_kernel_service
gn_args['is_clang'] = args.clang
enable_code_coverage = args.code_coverage and gn_args['is_clang']
gn_args['dart_vm_code_coverage'] = enable_code_coverage
gn_args['is_asan'] = sanitizer == 'asan'
gn_args['is_lsan'] = sanitizer == 'lsan'
gn_args['is_msan'] = sanitizer == 'msan'
gn_args['is_tsan'] = sanitizer == 'tsan'
gn_args['is_ubsan'] = sanitizer == 'ubsan'
gn_args['is_qemu'] = args.use_qemu
if not args.platform_sdk:
gn_args['dart_platform_sdk'] = args.platform_sdk
Reland "[vm] Build dart2native dependencies with the normal "create_sdk" target" The flutter engine DEPS got new icu/boringssl so this CL can safely land and roll into engine. A small change is made in Patchset 1..3. This removes special logic for creating the `dart-sdk` we distribute which used to build release and product mode and copied some binaries from the latter into the former, before the SDK was actuallly ready to test and distribute. This changes the GN build rules to build the necessary gen_snapshot/dart_precompiled_runtime product binaries during the normal release build. Normally during --mode=product builds the global build config in //build/config/BUILDCONFIG.gn will set `-fvisibility=false`. => Doing so results in much smaller binaries - because only explicitly exported symbols are visible, the rest can be tree shaken by the linker. Since we are building --mode=release, the `-fvisibility=false` will not be set. In order to set the flag for the 2 special product-mode binaries we need to add -fvisibility=hidden manually, in: * dart_product_config: Which is used for compiling VM sources. * 3rd party double-conversion library * 3rd party boringssl library * 3rd party icu library The upstream CLs are: * BoringSSL: https://dart-review.googlesource.com/c/boringssl_gen/+/150482 * ICU: https://chromium-review.googlesource.com/c/chromium/deps/icu/+/2236407 Issue https://github.com/dart-lang/sdk/issues/42230 Change-Id: I8d9b37acbb5eb94e44d97652f838948a946fd372 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/151505 Commit-Queue: Martin Kustermann <kustermann@google.com> Reviewed-by: Alexander Thomas <athom@google.com>
2020-06-17 16:12:26 +00:00
# We don't support stripping on Windows
if host_os != 'win':
gn_args['dart_stripped_binary'] = 'exe.stripped/dart'
gn_args['dart_precompiled_runtime_stripped_binary'] = (
'exe.stripped/dart_precompiled_runtime_product')
gn_args['gen_snapshot_stripped_binary'] = (
'exe.stripped/gen_snapshot_product')
gn_args['analyze_snapshot_binary'] = ('exe.stripped/analyze_snapshot')
gn_args['wasm_opt_stripped_binary'] = 'exe.stripped/wasm-opt'
# Setup the user-defined sysroot.
if UseSysroot(args, gn_args):
gn_args['dart_sysroot'] = 'debian'
else:
sysroot = TargetSysroot(args)
if sysroot:
gn_args['target_sysroot'] = ParseStringMap(arch, sysroot)
toolchain = ToolchainPrefix(args)
if toolchain:
for arch in ['ia32', 'x64', 'arm', 'arm64', 'riscv32', 'riscv64']:
prefix = ParseStringMap(arch, toolchain)
if prefix != None:
gn_args[arch + '_toolchain_prefix'] = prefix
gn_args['use_rbe'] = args.rbe
Reland "[infra] RBE all the things." This is a reland of commit b3253d9c8b2436060096234b69860caac67efbac Fix the gn rules so the RBE offloading only happens in the Dart build and the conflicting RBE configuration is not included otherwise. This ensures this change can roll into Flutter while we design a better solution to compose the projects' RBE usages. Original change's description: > [infra] RBE all the things. > > This change offloads all Dart actions during the build to RBE using > the new rewrapper_dart.py script that understands all the different > command line invocations of Dart programs and translates it into the > appropriate rewrapper invocation, with a full list of input and output > files and no absolute paths. > > The dart actions are all considered to be expensive RBE compilation > steps that may be much slower on RBE than locally due to the low > parallelism of the build during these stages and the comperatively > slower bots in the RBE pools. The build uses the racing strategy by > default for these compilations, such that cache hits are still used > if available, and otherwise a local build is transparently used if it > happens to be faster. The bots will use the remote strategy by default > unlike developers, such that the rewrapper_dart.py script does not fail. > > rewrapper_dart.py contains a tiny dart import proprocessor written > in python and a big argument parser that understands every command > invoked during the build and the semantic meaning of every option. > > The absolute paths used during the Dart SDK build is not solved in > this changelist, which just works around them initially, but these > will be fixed in follow up changes with the appropriate teams now > that this change proves they are not needed. > > Bug: b/333595242 > Change-Id: I36603ec1bf16f4ac87d56635cc1c98e8686a4028 > Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/335827 > Reviewed-by: William Hesse <whesse@google.com> > Commit-Queue: Jonas Termansen <sortie@google.com> Bug: b/333595242 Change-Id: I43bec2e7fcad375e972ca447aa13ae65b0ce09ae Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/362661 Reviewed-by: William Hesse <whesse@google.com> Commit-Queue: Jonas Termansen <sortie@google.com>
2024-04-12 22:06:32 +00:00
if args.rbe_expensive_exec_strategy:
gn_args[
'rbe_expensive_exec_strategy'] = args.rbe_expensive_exec_strategy
# Code coverage requires -O0 to be set.
if enable_code_coverage:
gn_args['dart_debug_optimization_level'] = 0
gn_args['debug_optimization_level'] = 0
elif args.debug_opt_level:
gn_args['dart_debug_optimization_level'] = args.debug_opt_level
gn_args['debug_optimization_level'] = args.debug_opt_level
gn_args['verify_sdk_hash'] = verify_sdk_hash
gn_args['dart_version_git_info'] = git_version
if args.codesigning_identity != '':
gn_args['codesigning_identity'] = args.codesigning_identity
return gn_args
def ProcessOsOption(os_name):
if os_name == 'host':
return HOST_OS
return os_name
def ProcessOptions(args):
if args.verify_sdk_hash == None:
args.verify_sdk_hash = not args.rbe
if args.git_version == None:
args.git_version = not args.rbe
if args.arch == 'all':
if platform.system() == 'Darwin':
# Targeting 32 bits not supported on MacOS.
# See HostArchitectures in utils.py.
args.arch = 'x64,simarm64,x64c,simarm64c,simriscv64'
else:
args.arch = 'ia32,x64,simarm,simarm64,x64c,simarm64c,simriscv32,simriscv64'
if args.mode == 'all':
args.mode = 'debug,release,product'
if args.os == 'all':
args.os = 'host,android,fuchsia'
if args.sanitizer == 'all':
args.sanitizer = 'none,asan,lsan,msan,tsan,ubsan'
args.mode = args.mode.split(',')
args.arch = args.arch.split(',')
args.os = args.os.split(',')
args.sanitizer = args.sanitizer.split(',')
for mode in args.mode:
if not mode in ['debug', 'release', 'product']:
print("Unknown mode %s" % mode)
return False
for i, arch in enumerate(args.arch):
args.arch[i] = arch.lower()
oses = [ProcessOsOption(os_name) for os_name in args.os]
for os_name in oses:
if not os_name in [
'android', 'freebsd', 'linux', 'macos', 'win32', 'fuchsia'
]:
print("Unknown os %s" % os_name)
return False
if os_name == 'android':
if not HOST_OS in ['linux', 'macos']:
print(
"Cross-compilation to %s is not supported on host os %s." %
(os_name, HOST_OS))
return False
if not arch in [
[vm] Support RISC-V. Implements a backend targeting RV32GC and RV64GC, based on Linux standardizing around GC. The assembler is written to make it easy to disable usage of C, but because the sizes of some instruction sequences are compile-time constants, an additional build configuration would need to be defined to make use of it. The assembler and disassembler cover every RV32/64GC instruction. The simulator covers all instructions except accessing CSRs and the floating point state accessible through such, include accrued exceptions and dynamic rounding mode. Quirks: - RISC-V is a compare-and-branch architecture, but some existing "architecture-independent" parts of the Dart compiler assume a condition code architecture. To avoid rewriting these parts, we use a peephole in the assembler to map to compare-and-branch. See Assembler::BranchIf. Luckily nothing depended on taking multiple branches on the same condition code set. - There are no hardware overflow checks, so we must use Hacker's Delight style software checks. Often these are very cheap: if the sign of one operand is known, a single branch is needed. - The ranges of RISC-V branches and jumps are such that we use 3 levels of generation for forward branches, instead of the 2 levels of near and far branches used on ARM[64]. Nearly all code is handled by the first two levels with 20-bits of range, with enormous regex matchers triggering the third level that uses aupic+jalr to get 32-bits of range. - For PC-relative calls in AOT, we always generate auipc+jalr pairs with 32-bits of range, so we never generate trampolines. - Only a subset of registers are available in some compressed instructions, so we assign the most popular uses to these registers. In particular, THR, TMP[2], CODE and PP. This has the effect of assigning CODE and PP to volatile registers in the C calling convention, whereas they are assigned preserved registers on the other architectures. As on ARM64, PP is untagged; this is so short indices can be accessed with a compressed instruction. - There are no push or pop instructions, so combining pushes and pops is preferred so we can update SP once. - The C calling convention has a strongly aligned stack, but unlike on ARM64 we don't need to use an alternate stack pointer. The author ensured language was added to the RISC-V psABI making the OS responsible for realigning the stack pointer for signal handlers, allowing Dart to leave the stack pointer misaligned from the C calling convention's point of view until a foreign call. - We don't bother with the link register tracking done on ARM[64]. Instead we make use of an alternate link register to avoid inline spilling in the write barrier. Unimplemented: - non-trivial FFI cases - Compressed pointers - No intention to implement. - Unboxed SIMD - We might make use of the V extension registers when the V extension is ratified. - BigInt intrinsics TEST=existing tests for IL level, new tests for assembler/disassembler/simulator Bug: https://github.com/dart-lang/sdk/issues/38587 Bug: https://github.com/dart-lang/sdk/issues/48164 Change-Id: I991d1df4be5bf55efec5371b767b332d37dfa3e0 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/217289 Reviewed-by: Alexander Markov <alexmarkov@google.com> Reviewed-by: Daco Harkes <dacoharkes@google.com> Reviewed-by: Slava Egorov <vegorov@google.com> Commit-Queue: Ryan Macnak <rmacnak@google.com>
2022-01-20 00:57:57 +00:00
'ia32',
'x64',
'arm',
'arm_x64',
'arm64',
'x64c',
'arm64c',
'riscv64',
]:
print(
"Cross-compilation to %s is not supported for architecture %s."
% (os_name, arch))
return False
elif os_name == 'fuchsia':
if not HOST_OS in ['linux', 'macos']:
print(
"Cross-compilation to %s is not supported on host os %s." %
(os_name, HOST_OS))
return False
if not arch in ['x64', 'arm64', 'x64c', 'arm64c', 'riscv64']:
print(
"Cross-compilation to %s is not supported for architecture %s."
% (os_name, arch))
return False
elif os_name != HOST_OS:
print("Unsupported target os %s" % os_name)
return False
if HOST_OS != 'win' and args.use_crashpad:
print("Crashpad is only supported on Windows")
return False
if not args.rbe and \
(socket.getfqdn().endswith('.corp.google.com') or
socket.getfqdn().endswith('.c.googlers.com')):
print('You can speed up your build by following: go/dart-rbe')
old_rbe_cfg = 'win-intel.cfg' if HOST_OS == 'win32' else 'linux-intel.cfg'
new_rbe_cfg = 'windows.cfg' if HOST_OS == 'win32' else 'unix.cfg'
if os.environ.get('RBE_cfg') == os.path.join(os.getcwd(), 'build', 'rbe',
old_rbe_cfg):
print(f'warning: {old_rbe_cfg} is deprecated, please update your '
f'RBE_cfg variable to {new_rbe_cfg} use RBE=1 instead per '
'go/dart-rbe')
return True
def os_has_ide(host_os):
return host_os.startswith('win') or host_os.startswith('mac')
def ide_switch(host_os):
if host_os.startswith('win'):
return '--ide=vs'
elif host_os.startswith('mac'):
return '--ide=xcode'
else:
return '--ide=json'
def AddCommonGnOptionArgs(parser):
"""Adds arguments that will change the default GN arguments."""
parser.add_argument('--rbe', help='Use rbe', action='store_true')
parser.add_argument('--no-rbe',
help='Disable rbe',
dest='rbe',
action='store_false')
parser.set_defaults(rbe=os.environ.get('RBE') == '1' or \
os.environ.get('DART_RBE') == '1' or \
os.environ.get('RBE_cfg') != None)
Reland "[infra] RBE all the things." This is a reland of commit b3253d9c8b2436060096234b69860caac67efbac Fix the gn rules so the RBE offloading only happens in the Dart build and the conflicting RBE configuration is not included otherwise. This ensures this change can roll into Flutter while we design a better solution to compose the projects' RBE usages. Original change's description: > [infra] RBE all the things. > > This change offloads all Dart actions during the build to RBE using > the new rewrapper_dart.py script that understands all the different > command line invocations of Dart programs and translates it into the > appropriate rewrapper invocation, with a full list of input and output > files and no absolute paths. > > The dart actions are all considered to be expensive RBE compilation > steps that may be much slower on RBE than locally due to the low > parallelism of the build during these stages and the comperatively > slower bots in the RBE pools. The build uses the racing strategy by > default for these compilations, such that cache hits are still used > if available, and otherwise a local build is transparently used if it > happens to be faster. The bots will use the remote strategy by default > unlike developers, such that the rewrapper_dart.py script does not fail. > > rewrapper_dart.py contains a tiny dart import proprocessor written > in python and a big argument parser that understands every command > invoked during the build and the semantic meaning of every option. > > The absolute paths used during the Dart SDK build is not solved in > this changelist, which just works around them initially, but these > will be fixed in follow up changes with the appropriate teams now > that this change proves they are not needed. > > Bug: b/333595242 > Change-Id: I36603ec1bf16f4ac87d56635cc1c98e8686a4028 > Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/335827 > Reviewed-by: William Hesse <whesse@google.com> > Commit-Queue: Jonas Termansen <sortie@google.com> Bug: b/333595242 Change-Id: I43bec2e7fcad375e972ca447aa13ae65b0ce09ae Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/362661 Reviewed-by: William Hesse <whesse@google.com> Commit-Queue: Jonas Termansen <sortie@google.com>
2024-04-12 22:06:32 +00:00
parser.add_argument('--rbe-expensive-exec-strategy',
default=os.environ.get('RBE_exec_strategy'),
help='Strategy for expensive RBE compilations',
type=str)
# Disable git hashes when remote compiling to ensure cache hits of the final
# output artifacts when nothing has changed.
parser.add_argument('--verify-sdk-hash',
help='Enable SDK hash checks',
dest='verify_sdk_hash',
action='store_true')
parser.add_argument('-nvh',
'--no-verify-sdk-hash',
help='Disable SDK hash checks',
dest='verify_sdk_hash',
action='store_false')
parser.set_defaults(verify_sdk_hash=None)
parser.add_argument('--git-version',
help='Enable git commit in version',
dest='git_version',
action='store_true')
parser.add_argument('-ngv',
'--no-git-version',
help='Disable git commit in version',
dest='git_version',
action='store_false')
parser.set_defaults(git_version=None)
parser.add_argument('--clang', help='Use Clang', action='store_true')
parser.add_argument('--no-clang',
help='Disable Clang',
dest='clang',
action='store_false')
parser.set_defaults(clang=True)
parser.add_argument(
'--platform-sdk',
help='Directs the create_sdk target to create a smaller "Platform" SDK',
default=MakePlatformSDK(),
action='store_true')
parser.add_argument('--use-crashpad',
default=False,
dest='use_crashpad',
action='store_true')
parser.add_argument('--use-qemu',
default=False,
dest='use_qemu',
action='store_true')
parser.add_argument('--exclude-kernel-service',
help='Exclude the kernel service.',
default=False,
dest='exclude_kernel_service',
action='store_true')
parser.add_argument('--arm-float-abi',
type=str,
help='The ARM float ABI (soft, softfp, hard)',
metavar='[soft,softfp,hard]',
default='')
parser.add_argument('--code-coverage',
help='Enable code coverage for the standalone VM',
default=False,
dest="code_coverage",
action='store_true')
parser.add_argument('--debug-opt-level',
'-d',
help='The optimization level to use for debug builds',
type=str)
parser.add_argument('--gn-args',
help='Set extra GN args',
dest='gn_args',
action='append')
parser.add_argument(
'--toolchain-prefix',
'-t',
type=str,
help='Comma-separated list of arch=/path/to/toolchain-prefix mappings')
parser.add_argument('--ide',
help='Generate an IDE file.',
default=os_has_ide(HOST_OS),
action='store_true')
parser.add_argument('--export-compile-commands',
help='Export compile_commands.json database file.',
default=False,
action='store_true')
parser.add_argument(
'--target-sysroot',
'-s',
type=str,
help='Comma-separated list of arch=/path/to/sysroot mappings')
parser.add_argument('--use-mallinfo2',
help='Use mallinfo2 to collect malloc stats.',
default=False,
dest='use_mallinfo2',
action='store_true')
parser.add_argument('--codesigning-identity',
help='Sign executables using the given identity.',
default='',
type=str)
def AddCommonConfigurationArgs(parser):
"""Adds arguments that influence which configuration will be built."""
parser.add_argument("-a",
"--arch",
type=str,
help='Target architectures (comma-separated).',
metavar='[all,' + ','.join(AVAILABLE_ARCHS) + ']',
default=utils.GuessArchitecture())
parser.add_argument('--mode',
'-m',
type=str,
help='Build variants (comma-separated).',
metavar='[all,debug,release,product]',
default='debug')
parser.add_argument('--os',
type=str,
help='Target OSs (comma-separated).',
metavar='[all,host,android,fuchsia]',
default='host')
parser.add_argument('--sanitizer',
type=str,
help='Build variants (comma-separated).',
metavar='[all,none,asan,lsan,msan,tsan,ubsan]',
default='none')
def AddOtherArgs(parser):
"""Adds miscellaneous arguments to the parser."""
parser.add_argument("-v",
"--verbose",
help='Verbose output.',
default=False,
action="store_true")
parser.add_argument("--test",
help='Test this script.',
default=False,
action="store_true")
def parse_args(args):
args = args[1:]
parser = argparse.ArgumentParser(
description='A script to run `gn gen`.',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
config_group = parser.add_argument_group('Configuration Related Arguments')
AddCommonConfigurationArgs(config_group)
gn_group = parser.add_argument_group('GN Related Arguments')
AddCommonGnOptionArgs(gn_group)
other_group = parser.add_argument_group('Other Arguments')
AddOtherArgs(other_group)
options = parser.parse_args(args)
if not ProcessOptions(options):
parser.print_help()
return None
return options
def InitializeRBE(out_dir, env):
RBE_cfg = 'RBE_CFG' if HOST_OS == 'win32' else 'RBE_cfg'
RBE_server_address = ('RBE_SERVER_ADDRESS'
if HOST_OS == 'win32' else 'RBE_server_address')
# Default RBE_cfg to the appropriate configuration file.
if not RBE_cfg in env:
env[RBE_cfg] = os.path.join(
os.getcwd(), 'build', 'rbe',
'windows.cfg' if HOST_OS == 'win32' else 'unix.cfg')
# Default RBE_server_address to inside the build directory.
if not RBE_server_address in env:
with open(env[RBE_cfg], 'r') as f:
if not any([l.startswith('server_address') for l in f.readlines()]):
schema = 'pipe' if HOST_OS == 'win32' else 'unix'
socket = os.path.join(os.getcwd(), out_dir, 'reproxy.sock')
if HOST_OS == 'win32':
socket = socket.replace('\\', '_').replace(':', '_')
env[RBE_server_address] = f'{schema}://{socket}'
def ExecutableName(basename):
if utils.IsWindows():
return f'{basename}.exe'
return basename
def BuildGnCommand(args, mode, arch, target_os, sanitizer, out_dir):
if utils.IsWindows():
gn = os.path.join(DART_ROOT, 'buildtools', 'win', 'gn.exe')
else:
gn = os.path.join(DART_ROOT, 'buildtools', 'gn')
if not os.path.isfile(gn):
raise Exception("Couldn't find the gn binary at path: " + gn)
# TODO(infra): Re-enable --check. Many targets fail to use
# public_deps to re-expose header files to their dependents.
# See dartbug.com/32364
command = [gn, 'gen', out_dir]
gn_args = ToCommandLine(
ToGnArgs(args, mode, arch, target_os, sanitizer, args.verify_sdk_hash,
args.git_version))
gn_args += GetGNArgs(args)
if args.ide:
command.append(ide_switch(HOST_OS))
if args.export_compile_commands:
command.append('--export-compile-commands')
command.append('--args=%s' % ' '.join(gn_args))
return command
def RunGnOnConfiguredConfigurations(args, env={}):
initialized_rbe = False
commands = []
for target_os in args.os:
for mode in args.mode:
for arch in args.arch:
for sanitizer in args.sanitizer:
out_dir = GetOutDir(mode, arch, target_os, sanitizer)
if args.rbe and not initialized_rbe:
InitializeRBE(out_dir, env)
initialized_rbe = True
commands.append(
BuildGnCommand(args, mode, arch, target_os, sanitizer,
out_dir))
if args.verbose:
print("gn gen --check in %s" % out_dir)
active_commands = []
def cleanup(command):
print("Command failed: " + ' '.join(command))
for (_, process) in active_commands:
process.terminate()
for command in commands:
try:
process = subprocess.Popen(command, cwd=DART_ROOT, env=env)
active_commands.append([command, process])
except Exception as e:
print('Error: %s' % e)
cleanup(command)
return 1
while active_commands:
time.sleep(0.1)
for active_command in active_commands:
(command, process) = active_command
if process.poll() is not None:
active_commands.remove(active_command)
if process.returncode != 0:
cleanup(command)
return 1
return 0
def ExpectEquals(actual, expected):
if actual != expected:
raise Exception(f"Actual: {actual} Expected: {expected}")
def RunTests():
host_arch = utils.HostArchitectures()[0]
host_arch_or_x64 = host_arch
if 'x64' in utils.HostArchitectures():
# Rosetta means 'x64' may be built directly.
host_arch_or_x64 = 'x64'
ExpectEquals(HostCpuForArch("arm64"), host_arch)
ExpectEquals(HostCpuForArch("arm64c"), host_arch)
ExpectEquals(HostCpuForArch("simarm64"), host_arch)
ExpectEquals(HostCpuForArch("simarm64_x64"), host_arch_or_x64)
ExpectEquals(HostCpuForArch("simarm64_arm64"), host_arch)
ExpectEquals(HostCpuForArch("simarm64_riscv64"), host_arch)
ExpectEquals(HostCpuForArch("x64"), host_arch_or_x64)
ExpectEquals(HostCpuForArch("simx64"), host_arch_or_x64)
ExpectEquals(HostCpuForArch("simx64_x64"), host_arch_or_x64)
ExpectEquals(HostCpuForArch("simx64_arm64"), host_arch)
ExpectEquals(HostCpuForArch("simx64_riscv64"), host_arch)
ExpectEquals(TargetCpuForArch("arm64"), "arm64")
ExpectEquals(TargetCpuForArch("arm64c"), "arm64")
ExpectEquals(TargetCpuForArch("simarm64"), host_arch)
ExpectEquals(TargetCpuForArch("simarm64_x64"), "x64")
ExpectEquals(TargetCpuForArch("simarm64_arm64"), "arm64")
ExpectEquals(TargetCpuForArch("simarm64_riscv64"), "riscv64")
ExpectEquals(TargetCpuForArch("x64"), "x64")
ExpectEquals(TargetCpuForArch("simx64"), host_arch)
ExpectEquals(TargetCpuForArch("simx64_x64"), "x64")
ExpectEquals(TargetCpuForArch("simx64_arm64"), "arm64")
ExpectEquals(TargetCpuForArch("simx64_riscv64"), "riscv64")
ExpectEquals(DartTargetCpuForArch("arm64"), "arm64")
ExpectEquals(DartTargetCpuForArch("arm64c"), "arm64")
ExpectEquals(DartTargetCpuForArch("simarm64"), "arm64")
ExpectEquals(DartTargetCpuForArch("simarm64_x64"), "arm64")
ExpectEquals(DartTargetCpuForArch("simarm64_arm64"), "arm64")
ExpectEquals(DartTargetCpuForArch("simarm64_riscv64"), "arm64")
ExpectEquals(DartTargetCpuForArch("x64"), "x64")
ExpectEquals(DartTargetCpuForArch("simx64"), "x64")
ExpectEquals(DartTargetCpuForArch("simx64_x64"), "x64")
ExpectEquals(DartTargetCpuForArch("simx64_arm64"), "x64")
ExpectEquals(DartTargetCpuForArch("simx64_riscv64"), "x64")
ExpectEquals(IsCompressedPointerArch("arm64c"), True)
ExpectEquals(IsCompressedPointerArch("simarm64c"), True)
ExpectEquals(IsCompressedPointerArch("simarm64c_x64"), True)
ExpectEquals(IsCompressedPointerArch("x64c"), True)
ExpectEquals(IsCompressedPointerArch("simx64c"), True)
ExpectEquals(IsCompressedPointerArch("simx64c_x64"), True)
ExpectEquals(IsCompressedPointerArch("arm64"), False)
ExpectEquals(IsCompressedPointerArch("simarm64"), False)
ExpectEquals(IsCompressedPointerArch("simarm64_x64"), False)
ExpectEquals(IsCompressedPointerArch("x64"), False)
ExpectEquals(IsCompressedPointerArch("simx64"), False)
ExpectEquals(IsCompressedPointerArch("simx64_x64"), False)
# Our Android bots:
ExpectEquals(HostCpuForArch("arm64c"), host_arch)
ExpectEquals(TargetCpuForArch("arm64c"), 'arm64')
ExpectEquals(DartTargetCpuForArch("arm64c"), 'arm64')
ExpectEquals(HostCpuForArch("arm_x64"), host_arch_or_x64)
ExpectEquals(TargetCpuForArch("arm_x64"), 'arm')
ExpectEquals(DartTargetCpuForArch("arm_x64"), 'arm')
def Main(argv):
starttime = time.time()
args = parse_args(argv)
if args is None:
return 1
if args.test:
RunTests()
print("Tests passed.")
return 0
result = RunGnOnConfiguredConfigurations(args)
if args.verbose:
endtime = time.time()
print("GN Time: %.3f seconds" % (endtime - starttime))
return result
if __name__ == '__main__':
sys.exit(Main(sys.argv))