languages/dart-sdk
languages/dart-sdk
1
0
Fork 0
mirror of https://github.com/dart-lang/sdk synced 2024-10-01 17:12:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

[infra] Generalize RBE across operating systems.

Browse source 
Bug: b/296994239
Change-Id: Ie8630bcece5c2f62c5693afe654fb9d6367b33e4
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/345286
Commit-Queue: Jonas Termansen <sortie@google.com>
Reviewed-by: Alexander Thomas <athom@google.com>
This commit is contained in:
Jonas Termansen 2024-01-10 11:38:43 +00:00 committed by Commit Queue
parent 4ad4501a16
commit 5678fd48cd
10 changed files with 128 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
DEPS
View file

@ -79,7 +79,7 @@ vars = {
# https://chrome-infra-packages.appspot.com/p/gn/gn
"gn_version": "git_revision:e4702d7409069c4f12d45ea7b7f0890717ca3f4b",
"reclient_version": "git_revision:f75cfb7bca0c04516330f27867a855e8d1186677",
"reclient_version": "git_revision:f3883c2237b0eb9cc9524cb571b5ab8378f257e4",
# Update from https://chrome-infra-packages.appspot.com/p/fuchsia/sdk/core
"fuchsia_sdk_version": "version:16.20231105.3.1",
@ -530,7 +530,6 @@ Var("dart_root") + "/third_party/pkg/tar":
"version": Var("clang_version"),
},
],
"condition": "host_cpu == x64 and host_os == linux",
"dep_type": "cipd",
},
Var("dart_root") + "/buildtools/mac-x64/clang": {
@ -581,7 +580,8 @@ Var("dart_root") + "/third_party/pkg/tar":
'version': Var('reclient_version'),
}
],
'condition': 'host_os == "linux" and host_cpu == "x64"',
# Download reclient only on the platforms where it has packages available.
'condition': '(host_os == "linux" or host_os == "win" or host_os == "mac" ) and host_cpu == "x64" or host_os == "mac" and host_cpu == "arm64"',
'dep_type': 'cipd',
},

5
build/config/mac/mac_sdk.gni
View file

@ -3,6 +3,7 @@
# found in the LICENSE file.
import("//build/toolchain/goma.gni")
import("//build/toolchain/rbe.gni")
declare_args() {
# Minimum supported version of the Mac SDK.
@ -17,7 +18,7 @@ declare_args() {
# mac_sdk_min is used.
mac_sdk_path = ""
mac_enable_relative_sdk_path = mac_use_goma_rbe
mac_enable_relative_sdk_path = use_rbe || use_goma
}
find_sdk_args = [
@ -25,7 +26,7 @@ find_sdk_args = [
mac_sdk_min,
]
if (mac_use_goma_rbe) {
if (use_rbe || use_goma) {
find_sdk_args += [
"--create_symlink_at",

5
build/rbe/llvm.sh Executable file
View file

@ -0,0 +1,5 @@
#!/bin/sh
os=$(uname -s | tr '[A-Z]' '[a-z'])
arch=$(uname -m | tr '[A-Z]' '[a-z'] | sed -E 's/^x86_64$/x64/')
cp "../../buildtools/$os-$arch/clang/bin/llvm" "$1"
"$@"

22
build/toolchain/android/BUILD.gn
View file

@ -26,19 +26,17 @@ template("android_toolchain") {
compiler_prefix = "$goma_dir/gomacc "
link_prefix = "$goma_dir/gomacc "
} else if (use_rbe) {
assert(!use_goma)
rewrapper_args = [
"$rbe_dir/rewrapper",
"--exec_root=$rbe_exec_root",
"--platform=$rbe_platform",
]
assembler_args = []
compiler_args =
rewrapper_args + [ "--labels=type=compile,compiler=clang,lang=cpp " ]
link_args = [] # rewrapper_args + [ "--labels=type=link,tool=clang " ]
assembler_prefix = string_join(" ", assembler_args)
compiler_prefix = string_join(" ", compiler_args)
link_prefix = string_join(" ", link_args)
rewrapper_args + [ "--labels=type=compile,compiler=clang,lang=cpp" ]
if (rbe_os != host_os || rbe_cpu != host_cpu) {
compiler_args += [
"--inputs=build/rbe,buildtools/$rbe_os-$rbe_cpu/clang/bin/llvm",
"--remote_wrapper=../../build/rbe/llvm.sh",
]
}
assembler_prefix = ""
compiler_prefix = string_join(" ", compiler_args) + " "
link_prefix = ""
} else if (use_ccache) {
assembler_prefix = "ccache "
compiler_prefix = "ccache "

4
build/toolchain/goma.gni
View file

@ -19,8 +19,4 @@ declare_args() {
# Absolute directory containing the Goma source code.
goma_dir = getenv("HOME") + "/goma"
}
# Sets relative symlink path to Xcode SDK path.
# Based on http://crrev.com/c/1946174.
mac_use_goma_rbe = false
}

52
build/toolchain/linux/BUILD.gn
View file

@ -24,19 +24,17 @@ if (use_goma) {
link_prefix = "$goma_dir/gomacc "
gcc_compiler_prefix = compiler_prefix
} else if (use_rbe) {
assert(!use_goma)
rewrapper_args = [
"$rbe_dir/rewrapper",
"--exec_root=$rbe_exec_root",
"--platform=$rbe_platform",
]
assembler_args = []
compiler_args =
rewrapper_args + [ "--labels=type=compile,compiler=clang,lang=cpp " ]
link_args = [] # rewrapper_args + [ "--labels=type=link,tool=clang " ]
assembler_prefix = string_join(" ", assembler_args)
compiler_prefix = string_join(" ", compiler_args)
link_prefix = string_join(" ", link_args)
rewrapper_args + [ "--labels=type=compile,compiler=clang,lang=cpp" ]
if (rbe_os != host_os || rbe_cpu != host_cpu) {
compiler_args += [
"--inputs=build/rbe,buildtools/$rbe_os-$rbe_cpu/clang/bin/llvm",
"--remote_wrapper=../../build/rbe/llvm.sh",
]
}
assembler_prefix = ""
compiler_prefix = string_join(" ", compiler_args) + " "
link_prefix = ""
gcc_compiler_prefix = ""
} else if (use_ccache) {
assembler_prefix = "ccache "
@ -88,6 +86,11 @@ gcc_toolchain("clang_arm") {
cxx = "${compiler_prefix}${prefix}/clang++"
asm = "${assembler_prefix}${prefix}/clang"
if (use_rbe) {
cc = "${cc} --target=arm-linux-gnueabihf"
cxx = "${cxx} --target=arm-linux-gnueabihf"
}
readelf = "${prefix}/llvm-readelf"
nm = "${prefix}/llvm-nm"
ar = "${prefix}/llvm-ar"
@ -126,6 +129,11 @@ gcc_toolchain("clang_arm64") {
cxx = "${compiler_prefix}${prefix}/clang++"
asm = "${assembler_prefix}${prefix}/clang"
if (use_rbe) {
cc = "${cc} --target=aarch64-linux-gnu"
cxx = "${cxx} --target=aarch64-linux-gnu"
}
readelf = "${prefix}/llvm-readelf"
nm = "${prefix}/llvm-nm"
ar = "${prefix}/llvm-ar"
@ -143,6 +151,11 @@ gcc_toolchain("clang_x86") {
cxx = "${compiler_prefix}${prefix}/clang++"
asm = "${assembler_prefix}${prefix}/clang"
if (use_rbe) {
cc = "${cc} --target=i386-linux-gnu"
cxx = "${cxx} --target=i386-linux-gnu"
}
readelf = "${prefix}/llvm-readelf"
nm = "${prefix}/llvm-nm"
ar = "${prefix}/llvm-ar"
@ -180,6 +193,11 @@ gcc_toolchain("clang_x64") {
cxx = "${compiler_prefix}${prefix}/clang++"
asm = "${assembler_prefix}${prefix}/clang"
if (use_rbe) {
cc = "${cc} --target=x86_64-linux-gnu"
cxx = "${cxx} --target=x86_64-linux-gnu"
}
readelf = "${prefix}/llvm-readelf"
nm = "${prefix}/llvm-nm"
ar = "${prefix}/llvm-ar"
@ -238,6 +256,11 @@ gcc_toolchain("clang_riscv32") {
cxx = "${compiler_prefix}${prefix}/clang++"
asm = "${assembler_prefix}${prefix}/clang"
if (use_rbe) {
cc = "${cc} --target=x86_64-linux-gnu"
cxx = "${cxx} --target=x86_64-linux-gnu"
}
readelf = "${prefix}/llvm-readelf"
nm = "${prefix}/llvm-nm"
ar = "${prefix}/llvm-ar"
@ -276,6 +299,11 @@ gcc_toolchain("clang_riscv64") {
cxx = "${compiler_prefix}${prefix}/clang++"
asm = "${assembler_prefix}${prefix}/clang"
if (use_rbe) {
cc = "${cc} --target=riscv64-linux-gnu"
cxx = "${cxx} --target=riscv64-linux-gnu"
}
readelf = "${prefix}/llvm-readelf"
nm = "${prefix}/llvm-nm"
ar = "${prefix}/llvm-ar"

20
build/toolchain/mac/BUILD.gn
View file

@ -20,6 +20,18 @@ if (use_goma) {
assembler_prefix = "$goma_dir/gomacc "
compiler_prefix = "$goma_dir/gomacc "
link_prefix = "$goma_dir/gomacc "
} else if (use_rbe) {
compiler_args =
rewrapper_args + [ "--labels=type=compile,compiler=clang,lang=cpp" ]
if (rbe_os != host_os || rbe_cpu != host_cpu) {
compiler_args += [
"--inputs=build/rbe,buildtools/$rbe_os-$rbe_cpu/clang/bin/llvm",
"--remote_wrapper=../../build/rbe/llvm.sh",
]
}
assembler_prefix = ""
compiler_prefix = string_join(" ", compiler_args) + " "
link_prefix = ""
} else {
assembler_prefix = ""
compiler_prefix = ""
@ -270,6 +282,10 @@ mac_toolchain("clang_x64") {
prefix = rebased_clang_dir
cc = "${compiler_prefix}${prefix}/clang"
cxx = "${compiler_prefix}${prefix}/clang++"
if (use_rbe) {
cc = "${cc} --target=x86_64-apple-macos"
cxx = "${cxx} --target=x86_64-apple-macos"
}
asm = "${assembler_prefix}${prefix}/clang"
ar = "${prefix}/llvm-ar"
ld = "${link_prefix}${prefix}/clang++"
@ -288,6 +304,10 @@ mac_toolchain("clang_arm64") {
prefix = rebased_clang_dir
cc = "${compiler_prefix}${prefix}/clang"
cxx = "${compiler_prefix}${prefix}/clang++"
if (use_rbe) {
cc = "${cc} --target=arm64-apple-macos"
cxx = "${cxx} --target=arm64-apple-macos"
}
asm = "${assembler_prefix}${prefix}/clang"
ar = "${prefix}/llvm-ar"
ld = "${link_prefix}${prefix}/clang++"

32
build/toolchain/rbe.gni
View file

@ -8,9 +8,37 @@ declare_args() {
# Set to true to enable distributed compilation using RBE.
use_rbe = false
rbe_exec_root = rebase_path("//")
# Set to the operating system to use in the RBE pool
rbe_os = "linux"
rbe_platform = "container-image=docker://gcr.io/cloud-marketplace/google/debian11@sha256:69e2789c9f3d28c6a0f13b25062c240ee7772be1f5e6d41bb4680b63eae6b304,OSFamily=Linux"
# Set to the architecture used in the RBE pool.
rbe_cpu = "x64"
# Set to the docker image to use in the RBE.
rbe_image = "docker://gcr.io/cloud-marketplace/google/debian11@sha256:69e2789c9f3d28c6a0f13b25062c240ee7772be1f5e6d41bb4680b63eae6b304"
rbe_exec_root = rebase_path("//")
rbe_dir = rebase_path("//buildtools/reclient")
}
declare_args() {
if (rbe_os == "linux") {
rbe_os_family = "Linux"
} else if (rbe_os == "win") {
rbe_os_family = "Windows"
} else if (rbe_os == "mac") {
rbe_os_family = "Mac"
} else {
rbe_os_family = rbe_os
}
# Set to the desired fully qualified RBE platform.
rbe_platform = "container-image=$rbe_image,OSFamily=$rbe_os_family"
}
rewrapper_args = [
"$rbe_dir/rewrapper",
"--exec_root=$rbe_exec_root",
"--platform=$rbe_platform",
]

17
build/toolchain/win/BUILD.gn
View file

@ -27,6 +27,16 @@ ninja_path = rebase_path("//buildtools/ninja/ninja")
if (use_goma) {
compiler_prefix = "$goma_dir/gomacc.exe "
} else if (use_rbe) {
compiler_args =
rewrapper_args + [ "--labels=type=compile,compiler=clang-cl,lang=cpp" ]
if (rbe_os != host_os || rbe_cpu != host_cpu) {
compiler_args += [
"--inputs=build/rbe,buildtools/$rbe_os-$rbe_cpu/clang/bin/llvm",
"--remote_wrapper=../../build/rbe/llvm.sh",
]
}
compiler_prefix = string_join(" ", compiler_args) + " "
} else {
compiler_prefix = ""
}
@ -272,6 +282,13 @@ template("win_toolchains") {
environment = "environment." + toolchain_arch
prefix = rebase_path("$clang_base_path/bin", root_build_dir)
cl = "${compiler_prefix}$prefix/clang-cl.exe"
if (use_rbe) {
if (toolchain_arch == "x86") {
cl = "${cl} --target=i686-pc-windows-msvc"
} else if (toolchain_arch == "x64") {
cl = "${cl} --target=x86_64-pc-windows-msvc"
}
}
toolchain_args = {
if (defined(invoker.toolchain_args)) {
forward_variables_from(invoker.toolchain_args, "*")

4
tools/gn.py
View file

@ -318,12 +318,8 @@ def ToGnArgs(args, mode, arch, target_os, sanitizer, verify_sdk_hash,
gn_args['use_goma'] = False
gn_args['goma_dir'] = None
if gn_args['target_os'] == 'mac' and gn_args['use_goma']:
gn_args['mac_use_goma_rbe'] = True
gn_args['use_rbe'] = args.rbe
# Code coverage requires -O0 to be set.
if enable_code_coverage:
gn_args['dart_debug_optimization_level'] = 0