mirror of
https://github.com/rust-lang/rust
synced 2024-10-07 01:00:41 +00:00
Port PGO/LTO/BOLT optimized build pipeline to Rust
This commit is contained in:
Jakub Beránek
Jakub Beránek
parent
03247fbbe8
commit
91d2fb2e2b
2
.github/workflows/ci.yml
vendored
2
.github/workflows/ci.yml
vendored
|
|
@ -408,7 +408,7 @@ jobs:
|
|||
- name: dist-x86_64-msvc
|
||||
env:
|
||||
RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --host=x86_64-pc-windows-msvc --target=x86_64-pc-windows-msvc --enable-full-tools --enable-profiler"
|
||||
SCRIPT: PGO_HOST=x86_64-pc-windows-msvc python src/ci/stage-build.py python x.py dist bootstrap --include-default-paths
|
||||
SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist python x.py dist bootstrap --include-default-paths
|
||||
DIST_REQUIRE_ALL_TOOLS: 1
|
||||
os: windows-2019-8core-32gb
|
||||
- name: dist-i686-msvc
|
||||
|
|
|
|||
440
Cargo.lock
440
Cargo.lock
|
|
@ -172,6 +172,9 @@ name = "anyhow"
|
|||
version = "1.0.71"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8"
|
||||
dependencies = [
|
||||
"backtrace",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ar_archive_writer"
|
||||
|
|
@ -258,6 +261,12 @@ dependencies = [
|
|||
"rustc-demangle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
version = "0.21.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d"
|
||||
|
||||
[[package]]
|
||||
name = "basic-toml"
|
||||
version = "0.1.2"
|
||||
|
|
@ -328,6 +337,10 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "build_helper"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bump-stage0"
|
||||
|
|
@ -685,6 +698,16 @@ dependencies = [
|
|||
"rand_xorshift",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "core-foundation"
|
||||
version = "0.9.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146"
|
||||
dependencies = [
|
||||
"core-foundation-sys",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "core-foundation-sys"
|
||||
version = "0.8.4"
|
||||
|
|
@ -985,6 +1008,15 @@ dependencies = [
|
|||
"log",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "encoding_rs"
|
||||
version = "0.8.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "env_logger"
|
||||
version = "0.7.1"
|
||||
|
|
@ -1174,6 +1206,21 @@ version = "1.0.7"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
|
||||
|
||||
[[package]]
|
||||
name = "foreign-types"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
|
||||
dependencies = [
|
||||
"foreign-types-shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "foreign-types-shared"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
|
||||
|
||||
[[package]]
|
||||
name = "form_urlencoded"
|
||||
version = "1.2.0"
|
||||
|
|
@ -1199,6 +1246,12 @@ version = "2.9.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0845fa252299212f0389d64ba26f34fa32cfe41588355f21ed507c59a0f64541"
|
||||
|
||||
[[package]]
|
||||
name = "fs_extra"
|
||||
version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
|
||||
|
||||
[[package]]
|
||||
name = "futf"
|
||||
version = "0.1.5"
|
||||
|
|
@ -1396,6 +1449,25 @@ dependencies = [
|
|||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "h2"
|
||||
version = "0.3.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d357c7ae988e7d2182f7d7871d0b963962420b0678b0997ce7de72001aeab782"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"fnv",
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
"futures-util",
|
||||
"http",
|
||||
"indexmap 1.9.3",
|
||||
"slab",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "handlebars"
|
||||
version = "4.3.7"
|
||||
|
|
@ -1494,6 +1566,49 @@ dependencies = [
|
|||
"syn 1.0.109",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "http"
|
||||
version = "0.2.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"fnv",
|
||||
"itoa",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "http-body"
|
||||
version = "0.4.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"http",
|
||||
"pin-project-lite",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "httparse"
|
||||
version = "1.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904"
|
||||
|
||||
[[package]]
|
||||
name = "httpdate"
|
||||
version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421"
|
||||
|
||||
[[package]]
|
||||
name = "humansize"
|
||||
version = "2.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6cb51c9a029ddc91b07a787f1d86b53ccfa49b0e86688c946ebe8d3555685dd7"
|
||||
dependencies = [
|
||||
"libm 0.2.7",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "humantime"
|
||||
version = "1.3.0"
|
||||
|
|
@ -1509,6 +1624,43 @@ version = "2.1.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
|
||||
|
||||
[[package]]
|
||||
name = "hyper"
|
||||
version = "0.14.22"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "abfba89e19b959ca163c7752ba59d737c1ceea53a5d31a149c805446fc958064"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"h2",
|
||||
"http",
|
||||
"http-body",
|
||||
"httparse",
|
||||
"httpdate",
|
||||
"itoa",
|
||||
"pin-project-lite",
|
||||
"socket2",
|
||||
"tokio",
|
||||
"tower-service",
|
||||
"tracing",
|
||||
"want",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hyper-tls"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"hyper",
|
||||
"native-tls",
|
||||
"tokio",
|
||||
"tokio-native-tls",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "iana-time-zone"
|
||||
version = "0.1.57"
|
||||
|
|
@ -1717,6 +1869,12 @@ dependencies = [
|
|||
"windows-sys 0.48.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ipnet"
|
||||
version = "2.7.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "12b6ee2129af8d4fb011108c73d99a1b83a85977f23b82460c0ae2e25bb4b57f"
|
||||
|
||||
[[package]]
|
||||
name = "is-terminal"
|
||||
version = "0.4.8"
|
||||
|
|
@ -1866,6 +2024,12 @@ version = "0.1.4"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7fc7aa29613bd6a620df431842069224d8bc9011086b1db4c0e0cd47fa03ec9a"
|
||||
|
||||
[[package]]
|
||||
name = "libm"
|
||||
version = "0.2.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4"
|
||||
|
||||
[[package]]
|
||||
name = "libz-sys"
|
||||
version = "1.1.9"
|
||||
|
|
@ -2104,6 +2268,17 @@ dependencies = [
|
|||
"rustc-std-workspace-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mio"
|
||||
version = "0.8.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"wasi",
|
||||
"windows-sys 0.48.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "miow"
|
||||
version = "0.5.0"
|
||||
|
|
@ -2141,6 +2316,24 @@ dependencies = [
|
|||
"regex",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "native-tls"
|
||||
version = "0.2.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e"
|
||||
dependencies = [
|
||||
"lazy_static",
|
||||
"libc",
|
||||
"log",
|
||||
"openssl",
|
||||
"openssl-probe",
|
||||
"openssl-sys",
|
||||
"schannel",
|
||||
"security-framework",
|
||||
"security-framework-sys",
|
||||
"tempfile",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "new_debug_unreachable"
|
||||
version = "1.0.4"
|
||||
|
|
@ -2169,6 +2362,15 @@ dependencies = [
|
|||
"minimal-lexical",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ntapi"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4"
|
||||
dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nu-ansi-term"
|
||||
version = "0.46.0"
|
||||
|
|
@ -2240,6 +2442,32 @@ dependencies = [
|
|||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "openssl"
|
||||
version = "0.10.55"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "345df152bc43501c5eb9e4654ff05f794effb78d4efe3d53abc158baddc0703d"
|
||||
dependencies = [
|
||||
"bitflags 1.3.2",
|
||||
"cfg-if",
|
||||
"foreign-types",
|
||||
"libc",
|
||||
"once_cell",
|
||||
"openssl-macros",
|
||||
"openssl-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "openssl-macros"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "openssl-probe"
|
||||
version = "0.1.5"
|
||||
|
|
@ -2258,6 +2486,28 @@ dependencies = [
|
|||
"vcpkg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "opt-dist"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"build_helper",
|
||||
"camino",
|
||||
"env_logger 0.10.0",
|
||||
"fs_extra",
|
||||
"glob",
|
||||
"humansize",
|
||||
"humantime 2.1.0",
|
||||
"log",
|
||||
"reqwest",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sysinfo",
|
||||
"tar",
|
||||
"xz",
|
||||
"zip",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "overload"
|
||||
version = "0.1.1"
|
||||
|
|
@ -2277,7 +2527,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "a1914cd452d8fccd6f9db48147b29fd4ae05bea9dc5d9ad578509f72415de282"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libm",
|
||||
"libm 0.1.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -2730,6 +2980,43 @@ dependencies = [
|
|||
"walkdir",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "reqwest"
|
||||
version = "0.11.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"bytes",
|
||||
"encoding_rs",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"h2",
|
||||
"http",
|
||||
"http-body",
|
||||
"hyper",
|
||||
"hyper-tls",
|
||||
"ipnet",
|
||||
"js-sys",
|
||||
"log",
|
||||
"mime",
|
||||
"native-tls",
|
||||
"once_cell",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_urlencoded",
|
||||
"tokio",
|
||||
"tokio-native-tls",
|
||||
"tower-service",
|
||||
"url",
|
||||
"wasm-bindgen",
|
||||
"wasm-bindgen-futures",
|
||||
"web-sys",
|
||||
"winreg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rls"
|
||||
version = "2.0.0"
|
||||
|
|
@ -4296,6 +4583,29 @@ version = "1.1.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
|
||||
|
||||
[[package]]
|
||||
name = "security-framework"
|
||||
version = "2.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fc758eb7bffce5b308734e9b0c1468893cae9ff70ebf13e7090be8dcbcc83a8"
|
||||
dependencies = [
|
||||
"bitflags 1.3.2",
|
||||
"core-foundation",
|
||||
"core-foundation-sys",
|
||||
"libc",
|
||||
"security-framework-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "security-framework-sys"
|
||||
version = "2.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f51d0c0d83bec45f16480d0ce0058397a69e48fcdc52d1dc8855fb68acbd31a7"
|
||||
dependencies = [
|
||||
"core-foundation-sys",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "self_cell"
|
||||
version = "0.10.2"
|
||||
|
|
@ -4352,6 +4662,18 @@ dependencies = [
|
|||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_urlencoded"
|
||||
version = "0.7.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd"
|
||||
dependencies = [
|
||||
"form_urlencoded",
|
||||
"itoa",
|
||||
"ryu",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha1"
|
||||
version = "0.10.5"
|
||||
|
|
@ -4630,6 +4952,20 @@ dependencies = [
|
|||
"unicode-xid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sysinfo"
|
||||
version = "0.29.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9557d0845b86eea8182f7b10dff120214fb6cd9fd937b6f4917714e546a38695"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"core-foundation-sys",
|
||||
"libc",
|
||||
"ntapi",
|
||||
"once_cell",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sysroot"
|
||||
version = "0.0.0"
|
||||
|
|
@ -4849,7 +5185,36 @@ dependencies = [
|
|||
"autocfg",
|
||||
"backtrace",
|
||||
"bytes",
|
||||
"libc",
|
||||
"mio",
|
||||
"num_cpus",
|
||||
"pin-project-lite",
|
||||
"socket2",
|
||||
"windows-sys 0.48.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-native-tls"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2"
|
||||
dependencies = [
|
||||
"native-tls",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-util"
|
||||
version = "0.7.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f988a1a1adc2fb21f9c12aa96441da33a1728193ae0b95d2be22dbd17fcb4e5c"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
"pin-project-lite",
|
||||
"tokio",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -4901,6 +5266,12 @@ version = "0.2.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ea68304e134ecd095ac6c3574494fc62b909f416c4fca77e440530221e549d3d"
|
||||
|
||||
[[package]]
|
||||
name = "tower-service"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"
|
||||
|
||||
[[package]]
|
||||
name = "tracing"
|
||||
version = "0.1.37"
|
||||
|
|
@ -4986,6 +5357,12 @@ dependencies = [
|
|||
"tracing-subscriber",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "try-lock"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed"
|
||||
|
||||
[[package]]
|
||||
name = "twox-hash"
|
||||
version = "1.6.3"
|
||||
|
|
@ -5304,6 +5681,15 @@ dependencies = [
|
|||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "want"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e"
|
||||
dependencies = [
|
||||
"try-lock",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
version = "0.11.0+wasi-snapshot-preview1"
|
||||
|
|
@ -5340,6 +5726,18 @@ dependencies = [
|
|||
"wasm-bindgen-shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-futures"
|
||||
version = "0.4.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
"web-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro"
|
||||
version = "0.2.87"
|
||||
|
|
@ -5369,6 +5767,16 @@ version = "0.2.87"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1"
|
||||
|
||||
[[package]]
|
||||
name = "web-sys"
|
||||
version = "0.3.61"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97"
|
||||
dependencies = [
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi"
|
||||
version = "0.3.9"
|
||||
|
|
@ -5563,6 +5971,15 @@ dependencies = [
|
|||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winreg"
|
||||
version = "0.10.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d"
|
||||
dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "writeable"
|
||||
version = "0.5.2"
|
||||
|
|
@ -5578,6 +5995,15 @@ dependencies = [
|
|||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "xz"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3c887690ff2a2e233e8e49633461521f98ec57fbff9d59a884c9a4f04ec1da34"
|
||||
dependencies = [
|
||||
"xz2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "xz2"
|
||||
version = "0.1.7"
|
||||
|
|
@ -5683,3 +6109,15 @@ dependencies = [
|
|||
"syn 1.0.109",
|
||||
"synstructure 0.12.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zip"
|
||||
version = "0.6.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"crc32fast",
|
||||
"crossbeam-utils",
|
||||
"flate2",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -43,6 +43,7 @@ members = [
|
|||
"src/tools/suggest-tests",
|
||||
"src/tools/generate-windows-sys",
|
||||
"src/tools/rustdoc-gui-test",
|
||||
"src/tools/opt-dist",
|
||||
]
|
||||
|
||||
exclude = [
|
||||
|
|
|
|||
|
|
@ -85,6 +85,10 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "build_helper"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
|
|
|
|||
|
|
@ -665,6 +665,7 @@ macro_rules! describe {
|
|||
llvm::Lld,
|
||||
llvm::CrtBeginEnd,
|
||||
tool::RustdocGUITest,
|
||||
tool::OptimizedDist
|
||||
),
|
||||
Kind::Check | Kind::Clippy | Kind::Fix => describe!(
|
||||
check::Std,
|
||||
|
|
|
|||
|
|
@ -7,7 +7,10 @@
|
|||
use crate::builder::{Builder, Step};
|
||||
use crate::util::t;
|
||||
use crate::Build;
|
||||
use serde_derive::{Deserialize, Serialize};
|
||||
use build_helper::metrics::{
|
||||
JsonInvocation, JsonInvocationSystemStats, JsonNode, JsonRoot, JsonStepSystemStats, Test,
|
||||
TestOutcome, TestSuite, TestSuiteMetadata,
|
||||
};
|
||||
use std::cell::RefCell;
|
||||
use std::fs::File;
|
||||
use std::io::BufWriter;
|
||||
|
|
@ -241,98 +244,7 @@ struct StepMetrics {
|
|||
test_suites: Vec<TestSuite>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
struct JsonRoot {
|
||||
#[serde(default)] // For version 0 the field was not present.
|
||||
format_version: usize,
|
||||
system_stats: JsonInvocationSystemStats,
|
||||
invocations: Vec<JsonInvocation>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
struct JsonInvocation {
|
||||
// Unix timestamp in seconds
|
||||
//
|
||||
// This is necessary to easily correlate this invocation with logs or other data.
|
||||
start_time: u64,
|
||||
duration_including_children_sec: f64,
|
||||
children: Vec<JsonNode>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case")]
|
||||
enum JsonNode {
|
||||
RustbuildStep {
|
||||
#[serde(rename = "type")]
|
||||
type_: String,
|
||||
debug_repr: String,
|
||||
|
||||
duration_excluding_children_sec: f64,
|
||||
system_stats: JsonStepSystemStats,
|
||||
|
||||
children: Vec<JsonNode>,
|
||||
},
|
||||
TestSuite(TestSuite),
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
struct TestSuite {
|
||||
metadata: TestSuiteMetadata,
|
||||
tests: Vec<Test>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case")]
|
||||
pub(crate) enum TestSuiteMetadata {
|
||||
CargoPackage {
|
||||
crates: Vec<String>,
|
||||
target: String,
|
||||
host: String,
|
||||
stage: u32,
|
||||
},
|
||||
Compiletest {
|
||||
suite: String,
|
||||
mode: String,
|
||||
compare_mode: Option<String>,
|
||||
target: String,
|
||||
host: String,
|
||||
stage: u32,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
pub(crate) struct Test {
|
||||
name: String,
|
||||
#[serde(flatten)]
|
||||
outcome: TestOutcome,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
#[serde(tag = "outcome", rename_all = "snake_case")]
|
||||
pub(crate) enum TestOutcome {
|
||||
Passed,
|
||||
Failed,
|
||||
Ignored { ignore_reason: Option<String> },
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
struct JsonInvocationSystemStats {
|
||||
cpu_threads_count: usize,
|
||||
cpu_model: String,
|
||||
|
||||
memory_total_bytes: u64,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
struct JsonStepSystemStats {
|
||||
cpu_utilization_percent: f64,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[derive(serde_derive::Deserialize)]
|
||||
struct OnlyFormatVersion {
|
||||
#[serde(default)] // For version 0 the field was not present.
|
||||
format_version: usize,
|
||||
|
|
|
|||
|
|
@ -141,9 +141,9 @@ fn render_test_outcome(&mut self, outcome: Outcome<'_>, test: &TestOutcome) {
|
|||
self.builder.metrics.record_test(
|
||||
&test.name,
|
||||
match outcome {
|
||||
Outcome::Ok | Outcome::BenchOk => crate::metrics::TestOutcome::Passed,
|
||||
Outcome::Failed => crate::metrics::TestOutcome::Failed,
|
||||
Outcome::Ignored { reason } => crate::metrics::TestOutcome::Ignored {
|
||||
Outcome::Ok | Outcome::BenchOk => build_helper::metrics::TestOutcome::Passed,
|
||||
Outcome::Failed => build_helper::metrics::TestOutcome::Failed,
|
||||
Outcome::Ignored { reason } => build_helper::metrics::TestOutcome::Ignored {
|
||||
ignore_reason: reason.map(|s| s.to_string()),
|
||||
},
|
||||
},
|
||||
|
|
|
|||
|
|
@ -340,7 +340,7 @@ fn run(self, builder: &Builder<'_>) {
|
|||
|
||||
#[cfg(feature = "build-metrics")]
|
||||
builder.metrics.begin_test_suite(
|
||||
crate::metrics::TestSuiteMetadata::CargoPackage {
|
||||
build_helper::metrics::TestSuiteMetadata::CargoPackage {
|
||||
crates: vec!["cargo".into()],
|
||||
target: self.host.triple.to_string(),
|
||||
host: self.host.triple.to_string(),
|
||||
|
|
@ -1827,7 +1827,7 @@ fn run(self, builder: &Builder<'_>) {
|
|||
|
||||
#[cfg(feature = "build-metrics")]
|
||||
builder.metrics.begin_test_suite(
|
||||
crate::metrics::TestSuiteMetadata::Compiletest {
|
||||
build_helper::metrics::TestSuiteMetadata::Compiletest {
|
||||
suite: suite.into(),
|
||||
mode: mode.into(),
|
||||
compare_mode: None,
|
||||
|
|
@ -1852,7 +1852,7 @@ fn run(self, builder: &Builder<'_>) {
|
|||
|
||||
#[cfg(feature = "build-metrics")]
|
||||
builder.metrics.begin_test_suite(
|
||||
crate::metrics::TestSuiteMetadata::Compiletest {
|
||||
build_helper::metrics::TestSuiteMetadata::Compiletest {
|
||||
suite: suite.into(),
|
||||
mode: mode.into(),
|
||||
compare_mode: Some(compare_mode.into()),
|
||||
|
|
@ -2200,7 +2200,7 @@ fn run_cargo_test(
|
|||
|
||||
#[cfg(feature = "build-metrics")]
|
||||
builder.metrics.begin_test_suite(
|
||||
crate::metrics::TestSuiteMetadata::CargoPackage {
|
||||
build_helper::metrics::TestSuiteMetadata::CargoPackage {
|
||||
crates: crates.iter().map(|c| c.to_string()).collect(),
|
||||
target: target.triple.to_string(),
|
||||
host: compiler.host.triple.to_string(),
|
||||
|
|
|
|||
|
|
@ -303,6 +303,7 @@ fn run(self, builder: &Builder<'_>) -> PathBuf {
|
|||
SuggestTests, "src/tools/suggest-tests", "suggest-tests";
|
||||
GenerateWindowsSys, "src/tools/generate-windows-sys", "generate-windows-sys";
|
||||
RustdocGUITest, "src/tools/rustdoc-gui-test", "rustdoc-gui-test", is_unstable_tool = true, allow_features = "test";
|
||||
OptimizedDist, "src/tools/opt-dist", "opt-dist";
|
||||
);
|
||||
|
||||
#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, Ord, PartialOrd)]
|
||||
|
|
|
|||
|
|
@ -54,7 +54,8 @@ COPY host-x86_64/dist-x86_64-linux/build-clang.sh /tmp/
|
|||
RUN ./build-clang.sh
|
||||
ENV CC=clang CXX=clang++
|
||||
|
||||
# rustc-perf version from 2023-03-15
|
||||
# rustc-perf version from 2023-05-30
|
||||
# Should also be changed in the opt-dist tool for other environments.
|
||||
ENV PERF_COMMIT 8b2ac3042e1ff2c0074455a0a3618adef97156b1
|
||||
RUN curl -LS -o perf.zip https://github.com/rust-lang/rustc-perf/archive/$PERF_COMMIT.zip && \
|
||||
unzip perf.zip && \
|
||||
|
|
@ -81,7 +82,9 @@ ENV RUST_CONFIGURE_ARGS \
|
|||
--set rust.jemalloc \
|
||||
--set rust.use-lld=true \
|
||||
--set rust.lto=thin
|
||||
ENV SCRIPT python3 ../src/ci/stage-build.py python3 ../x.py dist \
|
||||
|
||||
ENV SCRIPT python3 ../x.py build --set rust.debug=true opt-dist && \
|
||||
./build/$HOSTS/stage0-tools-bin/opt-dist python3 ../x.py dist \
|
||||
--host $HOSTS --target $HOSTS \
|
||||
--include-default-paths \
|
||||
build-manifest bootstrap
|
||||
|
|
|
|||
|
|
@ -643,7 +643,7 @@ jobs:
|
|||
--target=x86_64-pc-windows-msvc
|
||||
--enable-full-tools
|
||||
--enable-profiler
|
||||
SCRIPT: PGO_HOST=x86_64-pc-windows-msvc python src/ci/stage-build.py python x.py dist bootstrap --include-default-paths
|
||||
SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist python x.py dist bootstrap --include-default-paths
|
||||
DIST_REQUIRE_ALL_TOOLS: 1
|
||||
<<: *job-windows-8c
|
||||
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -6,3 +6,5 @@ edition = "2021"
|
|||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
serde = "1"
|
||||
serde_derive = "1"
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
pub mod ci;
|
||||
pub mod git;
|
||||
pub mod metrics;
|
||||
pub mod util;
|
||||
|
|
|
|||
92
src/tools/build_helper/src/metrics.rs
Normal file
92
src/tools/build_helper/src/metrics.rs
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
use serde_derive::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub struct JsonRoot {
|
||||
#[serde(default)] // For version 0 the field was not present.
|
||||
pub format_version: usize,
|
||||
pub system_stats: JsonInvocationSystemStats,
|
||||
pub invocations: Vec<JsonInvocation>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub struct JsonInvocation {
|
||||
// Unix timestamp in seconds
|
||||
//
|
||||
// This is necessary to easily correlate this invocation with logs or other data.
|
||||
pub start_time: u64,
|
||||
pub duration_including_children_sec: f64,
|
||||
pub children: Vec<JsonNode>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case")]
|
||||
pub enum JsonNode {
|
||||
RustbuildStep {
|
||||
#[serde(rename = "type")]
|
||||
type_: String,
|
||||
debug_repr: String,
|
||||
|
||||
duration_excluding_children_sec: f64,
|
||||
system_stats: JsonStepSystemStats,
|
||||
|
||||
children: Vec<JsonNode>,
|
||||
},
|
||||
TestSuite(TestSuite),
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
pub struct TestSuite {
|
||||
pub metadata: TestSuiteMetadata,
|
||||
pub tests: Vec<Test>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case")]
|
||||
pub enum TestSuiteMetadata {
|
||||
CargoPackage {
|
||||
crates: Vec<String>,
|
||||
target: String,
|
||||
host: String,
|
||||
stage: u32,
|
||||
},
|
||||
Compiletest {
|
||||
suite: String,
|
||||
mode: String,
|
||||
compare_mode: Option<String>,
|
||||
target: String,
|
||||
host: String,
|
||||
stage: u32,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
pub struct Test {
|
||||
pub name: String,
|
||||
#[serde(flatten)]
|
||||
pub outcome: TestOutcome,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
#[serde(tag = "outcome", rename_all = "snake_case")]
|
||||
pub enum TestOutcome {
|
||||
Passed,
|
||||
Failed,
|
||||
Ignored { ignore_reason: Option<String> },
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub struct JsonInvocationSystemStats {
|
||||
pub cpu_threads_count: usize,
|
||||
pub cpu_model: String,
|
||||
|
||||
pub memory_total_bytes: u64,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub struct JsonStepSystemStats {
|
||||
pub cpu_utilization_percent: f64,
|
||||
}
|
||||
22
src/tools/opt-dist/Cargo.toml
Normal file
22
src/tools/opt-dist/Cargo.toml
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
[package]
|
||||
name = "opt-dist"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
[dependencies]
|
||||
build_helper = { path = "../build_helper" }
|
||||
env_logger = "0.10"
|
||||
log = "0.4"
|
||||
anyhow = { version = "1", features = ["backtrace"] }
|
||||
humantime = "2"
|
||||
humansize = "2"
|
||||
sysinfo = { version = "0.29", default-features = false }
|
||||
fs_extra = "1"
|
||||
camino = "1"
|
||||
reqwest = { version = "0.11", features = ["blocking"] }
|
||||
zip = { version = "0.6", default-features = false, features = ["deflate"] }
|
||||
tar = "0.4"
|
||||
xz = "0.1"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
glob = "0.3"
|
||||
7
src/tools/opt-dist/README.md
Normal file
7
src/tools/opt-dist/README.md
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
# Optimized build pipeline
|
||||
This binary implements a heavily optimized build pipeline for `rustc` and `LLVM` artifacts that are used for both for
|
||||
benchmarking using the perf. bot and for final distribution to users.
|
||||
|
||||
It uses LTO, PGO and BOLT to optimize the compiler and LLVM as much as possible.
|
||||
This logic is not part of bootstrap, because it needs to invoke bootstrap multiple times, force-rebuild various
|
||||
artifacts repeatedly and sometimes go around bootstrap's cache mechanism.
|
||||
54
src/tools/opt-dist/src/environment/linux.rs
Normal file
54
src/tools/opt-dist/src/environment/linux.rs
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
use crate::environment::Environment;
|
||||
use crate::exec::cmd;
|
||||
use crate::utils::io::copy_directory;
|
||||
use camino::{Utf8Path, Utf8PathBuf};
|
||||
|
||||
pub(super) struct LinuxEnvironment;
|
||||
|
||||
impl Environment for LinuxEnvironment {
|
||||
fn python_binary(&self) -> &'static str {
|
||||
"python3"
|
||||
}
|
||||
|
||||
fn checkout_path(&self) -> Utf8PathBuf {
|
||||
Utf8PathBuf::from("/checkout")
|
||||
}
|
||||
|
||||
fn downloaded_llvm_dir(&self) -> Utf8PathBuf {
|
||||
Utf8PathBuf::from("/rustroot")
|
||||
}
|
||||
|
||||
fn opt_artifacts(&self) -> Utf8PathBuf {
|
||||
Utf8PathBuf::from("/tmp/tmp-multistage/opt-artifacts")
|
||||
}
|
||||
|
||||
fn build_root(&self) -> Utf8PathBuf {
|
||||
self.checkout_path().join("obj")
|
||||
}
|
||||
|
||||
fn prepare_rustc_perf(&self) -> anyhow::Result<()> {
|
||||
// /tmp/rustc-perf comes from the x64 dist Dockerfile
|
||||
copy_directory(Utf8Path::new("/tmp/rustc-perf"), &self.rustc_perf_dir())?;
|
||||
cmd(&[self.cargo_stage_0().as_str(), "build", "-p", "collector"])
|
||||
.workdir(&self.rustc_perf_dir())
|
||||
.env("RUSTC", &self.rustc_stage_0().into_string())
|
||||
.env("RUSTC_BOOTSTRAP", "1")
|
||||
.run()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn supports_bolt(&self) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn executable_extension(&self) -> &'static str {
|
||||
""
|
||||
}
|
||||
|
||||
fn skipped_tests(&self) -> &'static [&'static str] {
|
||||
&[
|
||||
// Fails because of linker errors, as of June 2023.
|
||||
"tests/ui/process/nofile-limit.rs",
|
||||
]
|
||||
}
|
||||
}
|
||||
75
src/tools/opt-dist/src/environment/mod.rs
Normal file
75
src/tools/opt-dist/src/environment/mod.rs
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
use camino::Utf8PathBuf;
|
||||
|
||||
#[cfg(target_family = "unix")]
|
||||
mod linux;
|
||||
#[cfg(target_family = "windows")]
|
||||
mod windows;
|
||||
|
||||
pub trait Environment {
|
||||
fn host_triple(&self) -> String {
|
||||
std::env::var("PGO_HOST").expect("PGO_HOST environment variable missing")
|
||||
}
|
||||
|
||||
fn python_binary(&self) -> &'static str;
|
||||
|
||||
/// The rustc checkout, where the compiler source is located.
|
||||
fn checkout_path(&self) -> Utf8PathBuf;
|
||||
|
||||
/// Path to the downloaded host LLVM.
|
||||
fn downloaded_llvm_dir(&self) -> Utf8PathBuf;
|
||||
|
||||
/// Directory where the optimization artifacts (PGO/BOLT profiles, etc.)
|
||||
/// will be stored.
|
||||
fn opt_artifacts(&self) -> Utf8PathBuf;
|
||||
|
||||
/// The main directory where the build occurs.
|
||||
fn build_root(&self) -> Utf8PathBuf;
|
||||
|
||||
fn build_artifacts(&self) -> Utf8PathBuf {
|
||||
self.build_root().join("build").join(self.host_triple())
|
||||
}
|
||||
|
||||
fn cargo_stage_0(&self) -> Utf8PathBuf {
|
||||
self.build_artifacts()
|
||||
.join("stage0")
|
||||
.join("bin")
|
||||
.join(format!("cargo{}", self.executable_extension()))
|
||||
}
|
||||
|
||||
fn rustc_stage_0(&self) -> Utf8PathBuf {
|
||||
self.build_artifacts()
|
||||
.join("stage0")
|
||||
.join("bin")
|
||||
.join(format!("rustc{}", self.executable_extension()))
|
||||
}
|
||||
|
||||
fn rustc_stage_2(&self) -> Utf8PathBuf {
|
||||
self.build_artifacts()
|
||||
.join("stage2")
|
||||
.join("bin")
|
||||
.join(format!("rustc{}", self.executable_extension()))
|
||||
}
|
||||
|
||||
/// Path to the built rustc-perf benchmark suite.
|
||||
fn rustc_perf_dir(&self) -> Utf8PathBuf {
|
||||
self.opt_artifacts().join("rustc-perf")
|
||||
}
|
||||
|
||||
/// Download and/or compile rustc-perf.
|
||||
fn prepare_rustc_perf(&self) -> anyhow::Result<()>;
|
||||
|
||||
fn supports_bolt(&self) -> bool;
|
||||
|
||||
/// What is the extension of binary executables in this environment?
|
||||
fn executable_extension(&self) -> &'static str;
|
||||
|
||||
/// List of test paths that should be skipped when testing the optimized artifacts.
|
||||
fn skipped_tests(&self) -> &'static [&'static str];
|
||||
}
|
||||
|
||||
pub fn create_environment() -> Box<dyn Environment> {
|
||||
#[cfg(target_family = "unix")]
|
||||
return Box::new(linux::LinuxEnvironment);
|
||||
#[cfg(target_family = "windows")]
|
||||
return Box::new(windows::WindowsEnvironment::new());
|
||||
}
|
||||
78
src/tools/opt-dist/src/environment/windows.rs
Normal file
78
src/tools/opt-dist/src/environment/windows.rs
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
use crate::environment::Environment;
|
||||
use crate::exec::cmd;
|
||||
use crate::utils::io::move_directory;
|
||||
use camino::Utf8PathBuf;
|
||||
use std::io::Cursor;
|
||||
use zip::ZipArchive;
|
||||
|
||||
pub(super) struct WindowsEnvironment {
|
||||
checkout_dir: Utf8PathBuf,
|
||||
}
|
||||
|
||||
impl WindowsEnvironment {
|
||||
pub fn new() -> Self {
|
||||
Self { checkout_dir: std::env::current_dir().unwrap().try_into().unwrap() }
|
||||
}
|
||||
}
|
||||
|
||||
impl Environment for WindowsEnvironment {
|
||||
fn python_binary(&self) -> &'static str {
|
||||
"python"
|
||||
}
|
||||
|
||||
fn checkout_path(&self) -> Utf8PathBuf {
|
||||
self.checkout_dir.clone()
|
||||
}
|
||||
|
||||
fn downloaded_llvm_dir(&self) -> Utf8PathBuf {
|
||||
self.checkout_path().join("citools").join("clang-rust")
|
||||
}
|
||||
|
||||
fn opt_artifacts(&self) -> Utf8PathBuf {
|
||||
self.checkout_path().join("opt-artifacts")
|
||||
}
|
||||
|
||||
fn build_root(&self) -> Utf8PathBuf {
|
||||
self.checkout_path()
|
||||
}
|
||||
|
||||
fn prepare_rustc_perf(&self) -> anyhow::Result<()> {
|
||||
// FIXME: add some mechanism for synchronization of this commit SHA with
|
||||
// Linux (which builds rustc-perf in a Dockerfile)
|
||||
// rustc-perf version from 2023-05-30
|
||||
const PERF_COMMIT: &str = "8b2ac3042e1ff2c0074455a0a3618adef97156b1";
|
||||
|
||||
let url = format!("https://github.com/rust-lang/rustc-perf/archive/{PERF_COMMIT}.zip");
|
||||
let response = reqwest::blocking::get(url)?.error_for_status()?.bytes()?.to_vec();
|
||||
|
||||
let mut archive = ZipArchive::new(Cursor::new(response))?;
|
||||
archive.extract(self.rustc_perf_dir())?;
|
||||
move_directory(
|
||||
&self.rustc_perf_dir().join(format!("rustc-perf-{PERF_COMMIT}")),
|
||||
&self.rustc_perf_dir(),
|
||||
)?;
|
||||
|
||||
cmd(&[self.cargo_stage_0().as_str(), "build", "-p", "collector"])
|
||||
.workdir(&self.rustc_perf_dir())
|
||||
.env("RUSTC", &self.rustc_stage_0().into_string())
|
||||
.env("RUSTC_BOOTSTRAP", "1")
|
||||
.run()?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn supports_bolt(&self) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn executable_extension(&self) -> &'static str {
|
||||
".exe"
|
||||
}
|
||||
|
||||
fn skipped_tests(&self) -> &'static [&'static str] {
|
||||
&[
|
||||
// Fails as of June 2023.
|
||||
"tests\\codegen\\vec-shrink-panik.rs",
|
||||
]
|
||||
}
|
||||
}
|
||||
169
src/tools/opt-dist/src/exec.rs
Normal file
169
src/tools/opt-dist/src/exec.rs
Normal file
|
|
@ -0,0 +1,169 @@
|
|||
use crate::environment::Environment;
|
||||
use crate::metrics::{load_metrics, record_metrics};
|
||||
use crate::timer::TimerSection;
|
||||
use crate::training::{LlvmBoltProfile, LlvmPGOProfile, RustcPGOProfile};
|
||||
use camino::{Utf8Path, Utf8PathBuf};
|
||||
use std::collections::BTreeMap;
|
||||
use std::fs::File;
|
||||
use std::process::{Command, Stdio};
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct CmdBuilder {
|
||||
args: Vec<String>,
|
||||
env: BTreeMap<String, String>,
|
||||
workdir: Option<Utf8PathBuf>,
|
||||
output: Option<Utf8PathBuf>,
|
||||
}
|
||||
|
||||
impl CmdBuilder {
|
||||
pub fn arg(mut self, arg: &str) -> Self {
|
||||
self.args.push(arg.to_string());
|
||||
self
|
||||
}
|
||||
|
||||
pub fn env(mut self, name: &str, value: &str) -> Self {
|
||||
self.env.insert(name.to_string(), value.to_string());
|
||||
self
|
||||
}
|
||||
|
||||
pub fn workdir(mut self, path: &Utf8Path) -> Self {
|
||||
self.workdir = Some(path.to_path_buf());
|
||||
self
|
||||
}
|
||||
|
||||
pub fn redirect_output(mut self, path: Utf8PathBuf) -> Self {
|
||||
self.output = Some(path);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn run(self) -> anyhow::Result<()> {
|
||||
let mut cmd_str = String::new();
|
||||
cmd_str.push_str(
|
||||
&self
|
||||
.env
|
||||
.iter()
|
||||
.map(|(key, value)| format!("{key}={value}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join(" "),
|
||||
);
|
||||
if !self.env.is_empty() {
|
||||
cmd_str.push(' ');
|
||||
}
|
||||
cmd_str.push_str(&self.args.join(" "));
|
||||
if let Some(ref path) = self.output {
|
||||
cmd_str.push_str(&format!(" > {path:?}"));
|
||||
}
|
||||
cmd_str.push_str(&format!(
|
||||
" [at {}]",
|
||||
self.workdir
|
||||
.clone()
|
||||
.unwrap_or_else(|| std::env::current_dir().unwrap().try_into().unwrap())
|
||||
));
|
||||
log::info!("Executing `{cmd_str}`");
|
||||
|
||||
let mut cmd = Command::new(&self.args[0]);
|
||||
cmd.stdin(Stdio::null());
|
||||
cmd.args(self.args.iter().skip(1));
|
||||
for (key, value) in &self.env {
|
||||
cmd.env(key, value);
|
||||
}
|
||||
if let Some(ref output) = self.output {
|
||||
cmd.stdout(File::create(output.clone().into_std_path_buf())?);
|
||||
}
|
||||
if let Some(ref workdir) = self.workdir {
|
||||
cmd.current_dir(workdir.clone().into_std_path_buf());
|
||||
}
|
||||
let exit_status = cmd.spawn()?.wait()?;
|
||||
if !exit_status.success() {
|
||||
Err(anyhow::anyhow!(
|
||||
"Command {cmd_str} has failed with exit code {:?}",
|
||||
exit_status.code(),
|
||||
))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn cmd(args: &[&str]) -> CmdBuilder {
|
||||
assert!(!args.is_empty());
|
||||
CmdBuilder { args: args.iter().map(|s| s.to_string()).collect(), ..Default::default() }
|
||||
}
|
||||
|
||||
pub struct Bootstrap {
|
||||
cmd: CmdBuilder,
|
||||
metrics_path: Utf8PathBuf,
|
||||
}
|
||||
|
||||
impl Bootstrap {
|
||||
pub fn build(env: &dyn Environment) -> Self {
|
||||
let metrics_path = env.build_root().join("build").join("metrics.json");
|
||||
let cmd = cmd(&[
|
||||
env.python_binary(),
|
||||
env.checkout_path().join("x.py").as_str(),
|
||||
"build",
|
||||
"--target",
|
||||
&env.host_triple(),
|
||||
"--host",
|
||||
&env.host_triple(),
|
||||
"--stage",
|
||||
"2",
|
||||
"library/std",
|
||||
])
|
||||
.env("RUST_BACKTRACE", "full");
|
||||
Self { cmd, metrics_path }
|
||||
}
|
||||
|
||||
pub fn dist(env: &dyn Environment, dist_args: &[String]) -> Self {
|
||||
let metrics_path = env.build_root().join("build").join("metrics.json");
|
||||
let cmd = cmd(&dist_args.iter().map(|arg| arg.as_str()).collect::<Vec<_>>())
|
||||
.env("RUST_BACKTRACE", "full");
|
||||
Self { cmd, metrics_path }
|
||||
}
|
||||
|
||||
pub fn llvm_pgo_instrument(mut self, profile_dir: &Utf8Path) -> Self {
|
||||
self.cmd = self
|
||||
.cmd
|
||||
.arg("--llvm-profile-generate")
|
||||
.env("LLVM_PROFILE_DIR", profile_dir.join("prof-%p").as_str());
|
||||
self
|
||||
}
|
||||
|
||||
pub fn llvm_pgo_optimize(mut self, profile: &LlvmPGOProfile) -> Self {
|
||||
self.cmd = self.cmd.arg("--llvm-profile-use").arg(profile.0.as_str());
|
||||
self
|
||||
}
|
||||
|
||||
pub fn rustc_pgo_instrument(mut self, profile_dir: &Utf8Path) -> Self {
|
||||
self.cmd = self.cmd.arg("--rust-profile-generate").arg(profile_dir.as_str());
|
||||
self
|
||||
}
|
||||
|
||||
pub fn rustc_pgo_optimize(mut self, profile: &RustcPGOProfile) -> Self {
|
||||
self.cmd = self.cmd.arg("--rust-profile-use").arg(profile.0.as_str());
|
||||
self
|
||||
}
|
||||
|
||||
pub fn llvm_bolt_instrument(mut self) -> Self {
|
||||
self.cmd = self.cmd.arg("--llvm-bolt-profile-generate");
|
||||
self
|
||||
}
|
||||
|
||||
pub fn llvm_bolt_optimize(mut self, profile: &LlvmBoltProfile) -> Self {
|
||||
self.cmd = self.cmd.arg("--llvm-bolt-profile-use").arg(profile.0.as_str());
|
||||
self
|
||||
}
|
||||
|
||||
/// Do not rebuild rustc, and use a previously built rustc sysroot instead.
|
||||
pub fn avoid_rustc_rebuild(mut self) -> Self {
|
||||
self.cmd = self.cmd.arg("--keep-stage").arg("0").arg("--keep-stage").arg("1");
|
||||
self
|
||||
}
|
||||
|
||||
pub fn run(self, timer: &mut TimerSection) -> anyhow::Result<()> {
|
||||
self.cmd.run()?;
|
||||
let metrics = load_metrics(&self.metrics_path)?;
|
||||
record_metrics(&metrics, timer);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
175
src/tools/opt-dist/src/main.rs
Normal file
175
src/tools/opt-dist/src/main.rs
Normal file
|
|
@ -0,0 +1,175 @@
|
|||
use anyhow::Context;
|
||||
use log::LevelFilter;
|
||||
|
||||
use crate::environment::{create_environment, Environment};
|
||||
use crate::exec::Bootstrap;
|
||||
use crate::tests::run_tests;
|
||||
use crate::timer::Timer;
|
||||
use crate::training::{gather_llvm_bolt_profiles, gather_llvm_profiles, gather_rustc_profiles};
|
||||
use crate::utils::io::reset_directory;
|
||||
use crate::utils::{clear_llvm_files, format_env_variables, print_free_disk_space};
|
||||
|
||||
mod environment;
|
||||
mod exec;
|
||||
mod metrics;
|
||||
mod tests;
|
||||
mod timer;
|
||||
mod training;
|
||||
mod utils;
|
||||
|
||||
fn is_try_build() -> bool {
|
||||
std::env::var("DIST_TRY_BUILD").unwrap_or_else(|_| "0".to_string()) != "0"
|
||||
}
|
||||
|
||||
fn execute_pipeline(
|
||||
env: &dyn Environment,
|
||||
timer: &mut Timer,
|
||||
dist_args: Vec<String>,
|
||||
) -> anyhow::Result<()> {
|
||||
reset_directory(&env.opt_artifacts())?;
|
||||
env.prepare_rustc_perf()?;
|
||||
|
||||
// Stage 1: Build PGO instrumented rustc
|
||||
// We use a normal build of LLVM, because gathering PGO profiles for LLVM and `rustc` at the
|
||||
// same time can cause issues, because the host and in-tree LLVM versions can diverge.
|
||||
let rustc_pgo_profile = timer.section("Stage 1 (Rustc PGO)", |stage| {
|
||||
let rustc_profile_dir_root = env.opt_artifacts().join("rustc-pgo");
|
||||
|
||||
stage.section("Build PGO instrumented rustc and LLVM", |section| {
|
||||
Bootstrap::build(env).rustc_pgo_instrument(&rustc_profile_dir_root).run(section)
|
||||
})?;
|
||||
|
||||
let profile = stage
|
||||
.section("Gather profiles", |_| gather_rustc_profiles(env, &rustc_profile_dir_root))?;
|
||||
print_free_disk_space()?;
|
||||
|
||||
stage.section("Build PGO optimized rustc", |section| {
|
||||
Bootstrap::build(env).rustc_pgo_optimize(&profile).run(section)
|
||||
})?;
|
||||
|
||||
Ok(profile)
|
||||
})?;
|
||||
|
||||
// Stage 2: Gather LLVM PGO profiles
|
||||
// Here we build a PGO instrumented LLVM, reusing the previously PGO optimized rustc.
|
||||
// Then we use the instrumented LLVM to gather LLVM PGO profiles.
|
||||
let llvm_pgo_profile = timer.section("Stage 2 (LLVM PGO)", |stage| {
|
||||
// Remove the previous, uninstrumented build of LLVM.
|
||||
clear_llvm_files(env)?;
|
||||
|
||||
let llvm_profile_dir_root = env.opt_artifacts().join("llvm-pgo");
|
||||
|
||||
stage.section("Build PGO instrumented LLVM", |section| {
|
||||
Bootstrap::build(env)
|
||||
.llvm_pgo_instrument(&llvm_profile_dir_root)
|
||||
.avoid_rustc_rebuild()
|
||||
.run(section)
|
||||
})?;
|
||||
|
||||
let profile = stage
|
||||
.section("Gather profiles", |_| gather_llvm_profiles(env, &llvm_profile_dir_root))?;
|
||||
|
||||
print_free_disk_space()?;
|
||||
|
||||
// Proactively delete the instrumented artifacts, to avoid using them by accident in
|
||||
// follow-up stages.
|
||||
clear_llvm_files(env)?;
|
||||
|
||||
Ok(profile)
|
||||
})?;
|
||||
|
||||
let llvm_bolt_profile = if env.supports_bolt() {
|
||||
// Stage 3: Build BOLT instrumented LLVM
|
||||
// We build a PGO optimized LLVM in this step, then instrument it with BOLT and gather BOLT profiles.
|
||||
// Note that we don't remove LLVM artifacts after this step, so that they are reused in the final dist build.
|
||||
// BOLT instrumentation is performed "on-the-fly" when the LLVM library is copied to the sysroot of rustc,
|
||||
// therefore the LLVM artifacts on disk are not "tainted" with BOLT instrumentation and they can be reused.
|
||||
timer.section("Stage 3 (LLVM BOLT)", |stage| {
|
||||
stage.section("Build BOLT instrumented LLVM", |stage| {
|
||||
Bootstrap::build(env)
|
||||
.llvm_bolt_instrument()
|
||||
.llvm_pgo_optimize(&llvm_pgo_profile)
|
||||
.avoid_rustc_rebuild()
|
||||
.run(stage)
|
||||
})?;
|
||||
|
||||
let profile = stage.section("Gather profiles", |_| gather_llvm_bolt_profiles(env))?;
|
||||
print_free_disk_space()?;
|
||||
|
||||
// LLVM is not being cleared here, we want to reuse the previous PGO-optimized build
|
||||
|
||||
Ok(Some(profile))
|
||||
})?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let mut dist = Bootstrap::dist(env, &dist_args)
|
||||
.llvm_pgo_optimize(&llvm_pgo_profile)
|
||||
.rustc_pgo_optimize(&rustc_pgo_profile)
|
||||
.avoid_rustc_rebuild();
|
||||
|
||||
if let Some(llvm_bolt_profile) = llvm_bolt_profile {
|
||||
dist = dist.llvm_bolt_optimize(&llvm_bolt_profile);
|
||||
}
|
||||
|
||||
// Final stage: Assemble the dist artifacts
|
||||
// The previous PGO optimized rustc build and PGO optimized LLVM builds should be reused.
|
||||
timer.section("Stage 4 (final build)", |stage| dist.run(stage))?;
|
||||
|
||||
// After dist has finished, run a subset of the test suite on the optimized artifacts to discover
|
||||
// possible regressions.
|
||||
// The tests are not executed for try builds, which can be in various broken states, so we don't
|
||||
// want to gatekeep them with tests.
|
||||
if !is_try_build() {
|
||||
timer.section("Run tests", |_| run_tests(env))?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn main() -> anyhow::Result<()> {
|
||||
// Make sure that we get backtraces for easier debugging in CI
|
||||
std::env::set_var("RUST_BACKTRACE", "1");
|
||||
|
||||
env_logger::builder()
|
||||
.filter_level(LevelFilter::Info)
|
||||
.format_timestamp_millis()
|
||||
.parse_default_env()
|
||||
.init();
|
||||
|
||||
let mut build_args: Vec<String> = std::env::args().skip(1).collect();
|
||||
log::info!("Running optimized build pipeline with args `{}`", build_args.join(" "));
|
||||
log::info!("Environment values\n{}", format_env_variables());
|
||||
|
||||
if let Ok(config) = std::fs::read_to_string("config.toml") {
|
||||
log::info!("Contents of `config.toml`:\n{config}");
|
||||
}
|
||||
|
||||
// Skip components that are not needed for try builds to speed them up
|
||||
if is_try_build() {
|
||||
log::info!("Skipping building of unimportant components for a try build");
|
||||
for target in [
|
||||
"rust-docs",
|
||||
"rustc-docs",
|
||||
"rust-docs-json",
|
||||
"rust-analyzer",
|
||||
"rustc-src",
|
||||
"clippy",
|
||||
"miri",
|
||||
"rustfmt",
|
||||
] {
|
||||
build_args.extend(["--exclude".to_string(), target.to_string()]);
|
||||
}
|
||||
}
|
||||
|
||||
let mut timer = Timer::new();
|
||||
let env = create_environment();
|
||||
|
||||
let result = execute_pipeline(env.as_ref(), &mut timer, build_args);
|
||||
log::info!("Timer results\n{}", timer.format_stats());
|
||||
|
||||
print_free_disk_space()?;
|
||||
|
||||
result.context("Optimized build pipeline has failed")
|
||||
}
|
||||
106
src/tools/opt-dist/src/metrics.rs
Normal file
106
src/tools/opt-dist/src/metrics.rs
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
use crate::timer::TimerSection;
|
||||
use build_helper::metrics::{JsonNode, JsonRoot};
|
||||
use camino::Utf8Path;
|
||||
use std::time::Duration;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct BuildStep {
|
||||
r#type: String,
|
||||
children: Vec<BuildStep>,
|
||||
duration: Duration,
|
||||
}
|
||||
|
||||
impl BuildStep {
|
||||
pub fn find_all_by_type(&self, r#type: &str) -> Vec<&BuildStep> {
|
||||
let mut result = Vec::new();
|
||||
self.find_by_type(r#type, &mut result);
|
||||
result
|
||||
}
|
||||
fn find_by_type<'a>(&'a self, r#type: &str, result: &mut Vec<&'a BuildStep>) {
|
||||
if self.r#type == r#type {
|
||||
result.push(self);
|
||||
}
|
||||
for child in &self.children {
|
||||
child.find_by_type(r#type, result);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Loads the metrics of the most recent bootstrap execution from a metrics.json file.
|
||||
pub fn load_metrics(path: &Utf8Path) -> anyhow::Result<BuildStep> {
|
||||
let content = std::fs::read(path.as_std_path())?;
|
||||
let mut metrics = serde_json::from_slice::<JsonRoot>(&content)?;
|
||||
let invocation = metrics
|
||||
.invocations
|
||||
.pop()
|
||||
.ok_or_else(|| anyhow::anyhow!("No bootstrap invocation found in metrics file"))?;
|
||||
|
||||
fn parse(node: JsonNode) -> Option<BuildStep> {
|
||||
match node {
|
||||
JsonNode::RustbuildStep {
|
||||
type_: kind,
|
||||
children,
|
||||
duration_excluding_children_sec,
|
||||
..
|
||||
} => {
|
||||
let children: Vec<_> = children.into_iter().filter_map(parse).collect();
|
||||
let children_duration = children.iter().map(|c| c.duration).sum::<Duration>();
|
||||
Some(BuildStep {
|
||||
r#type: kind.to_string(),
|
||||
children,
|
||||
duration: children_duration
|
||||
+ Duration::from_secs_f64(duration_excluding_children_sec),
|
||||
})
|
||||
}
|
||||
JsonNode::TestSuite(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
let duration = Duration::from_secs_f64(invocation.duration_including_children_sec);
|
||||
let children: Vec<_> = invocation.children.into_iter().filter_map(parse).collect();
|
||||
Ok(BuildStep { r#type: "root".to_string(), children, duration })
|
||||
}
|
||||
|
||||
/// Logs the individual metrics in a table and add Rustc and LLVM durations to the passed
|
||||
/// timer.
|
||||
pub fn record_metrics(metrics: &BuildStep, timer: &mut TimerSection) {
|
||||
let llvm_steps = metrics.find_all_by_type("bootstrap::llvm::Llvm");
|
||||
let llvm_duration: Duration = llvm_steps.into_iter().map(|s| s.duration).sum();
|
||||
|
||||
let rustc_steps = metrics.find_all_by_type("bootstrap::compile::Rustc");
|
||||
let rustc_duration: Duration = rustc_steps.into_iter().map(|s| s.duration).sum();
|
||||
|
||||
// The LLVM step is part of the Rustc step
|
||||
let rustc_duration = rustc_duration.saturating_sub(llvm_duration);
|
||||
|
||||
if !llvm_duration.is_zero() {
|
||||
timer.add_duration("LLVM", llvm_duration);
|
||||
}
|
||||
if !rustc_duration.is_zero() {
|
||||
timer.add_duration("Rustc", rustc_duration);
|
||||
}
|
||||
|
||||
log_metrics(metrics);
|
||||
}
|
||||
|
||||
fn log_metrics(metrics: &BuildStep) {
|
||||
use std::fmt::Write;
|
||||
|
||||
let mut substeps: Vec<(u32, &BuildStep)> = Vec::new();
|
||||
|
||||
fn visit<'a>(step: &'a BuildStep, level: u32, substeps: &mut Vec<(u32, &'a BuildStep)>) {
|
||||
substeps.push((level, step));
|
||||
for child in &step.children {
|
||||
visit(child, level + 1, substeps);
|
||||
}
|
||||
}
|
||||
|
||||
visit(metrics, 0, &mut substeps);
|
||||
|
||||
let mut output = String::new();
|
||||
for (level, step) in substeps {
|
||||
let label = format!("{}{}", ".".repeat(level as usize), step.r#type);
|
||||
writeln!(output, "{label:<65}{:>8.2}s", step.duration.as_secs_f64()).unwrap();
|
||||
}
|
||||
log::info!("Build step durations\n{output}");
|
||||
}
|
||||
101
src/tools/opt-dist/src/tests.rs
Normal file
101
src/tools/opt-dist/src/tests.rs
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
use crate::environment::Environment;
|
||||
use crate::exec::cmd;
|
||||
use crate::utils::io::{copy_directory, unpack_archive};
|
||||
use anyhow::Context;
|
||||
use camino::Utf8PathBuf;
|
||||
|
||||
/// Run tests on optimized dist artifacts.
|
||||
pub fn run_tests(env: &dyn Environment) -> anyhow::Result<()> {
|
||||
// After `dist` is executed, we extract its archived components into a sysroot directory,
|
||||
// and then use that extracted rustc as a stage0 compiler.
|
||||
// Then we run a subset of tests using that compiler, to have a basic smoke test which checks
|
||||
// whether the optimization pipeline hasn't broken something.
|
||||
let build_dir = env.build_root().join("build");
|
||||
let dist_dir = build_dir.join("dist");
|
||||
let unpacked_dist_dir = build_dir.join("unpacked-dist");
|
||||
std::fs::create_dir_all(&unpacked_dist_dir)?;
|
||||
|
||||
let extract_dist_dir = |name: &str| -> anyhow::Result<Utf8PathBuf> {
|
||||
unpack_archive(&dist_dir.join(format!("{name}.tar.xz")), &unpacked_dist_dir)?;
|
||||
let extracted_path = unpacked_dist_dir.join(name);
|
||||
assert!(extracted_path.is_dir());
|
||||
Ok(extracted_path)
|
||||
};
|
||||
let host_triple = env.host_triple();
|
||||
|
||||
// Extract rustc, libstd, cargo and src archives to create the optimized sysroot
|
||||
let rustc_dir = extract_dist_dir(&format!("rustc-nightly-{host_triple}"))?.join("rustc");
|
||||
let libstd_dir = extract_dist_dir(&format!("rust-std-nightly-{host_triple}"))?
|
||||
.join(format!("rust-std-{host_triple}"));
|
||||
let cargo_dir = extract_dist_dir(&format!("cargo-nightly-{host_triple}"))?.join("cargo");
|
||||
let extracted_src_dir = extract_dist_dir("rust-src-nightly")?.join("rust-src");
|
||||
|
||||
// We need to manually copy libstd to the extracted rustc sysroot
|
||||
copy_directory(
|
||||
&libstd_dir.join("lib").join("rustlib").join(&host_triple).join("lib"),
|
||||
&rustc_dir.join("lib").join("rustlib").join(&host_triple).join("lib"),
|
||||
)?;
|
||||
|
||||
// Extract sources - they aren't in the `rustc-nightly-{host}` tarball, so we need to manually copy libstd
|
||||
// sources to the extracted sysroot. We need sources available so that `-Zsimulate-remapped-rust-src-base`
|
||||
// works correctly.
|
||||
copy_directory(
|
||||
&extracted_src_dir.join("lib").join("rustlib").join("src"),
|
||||
&rustc_dir.join("lib").join("rustlib").join("src"),
|
||||
)?;
|
||||
|
||||
let rustc_path = rustc_dir.join("bin").join(format!("rustc{}", env.executable_extension()));
|
||||
assert!(rustc_path.is_file());
|
||||
let cargo_path = cargo_dir.join("bin").join(format!("cargo{}", env.executable_extension()));
|
||||
assert!(cargo_path.is_file());
|
||||
|
||||
// Specify path to a LLVM config so that LLVM is not rebuilt.
|
||||
// It doesn't really matter which LLVM config we choose, because no sysroot will be compiled.
|
||||
let llvm_config = env
|
||||
.build_artifacts()
|
||||
.join("llvm")
|
||||
.join("bin")
|
||||
.join(format!("llvm-config{}", env.executable_extension()));
|
||||
assert!(llvm_config.is_file());
|
||||
|
||||
let config_content = format!(
|
||||
r#"profile = "user"
|
||||
changelog-seen = 2
|
||||
|
||||
[build]
|
||||
rustc = "{rustc}"
|
||||
cargo = "{cargo}"
|
||||
|
||||
[target.{host_triple}]
|
||||
llvm-config = "{llvm_config}"
|
||||
"#,
|
||||
rustc = rustc_path.to_string().replace('\\', "/"),
|
||||
cargo = cargo_path.to_string().replace('\\', "/"),
|
||||
llvm_config = llvm_config.to_string().replace('\\', "/")
|
||||
);
|
||||
log::info!("Using following `config.toml` for running tests:\n{config_content}");
|
||||
|
||||
// Simulate a stage 0 compiler with the extracted optimized dist artifacts.
|
||||
std::fs::write("config.toml", config_content)?;
|
||||
|
||||
let x_py = env.checkout_path().join("x.py");
|
||||
let mut args = vec![
|
||||
env.python_binary(),
|
||||
x_py.as_str(),
|
||||
"test",
|
||||
"--stage",
|
||||
"0",
|
||||
"tests/assembly",
|
||||
"tests/codegen",
|
||||
"tests/codegen-units",
|
||||
"tests/incremental",
|
||||
"tests/mir-opt",
|
||||
"tests/pretty",
|
||||
"tests/run-pass-valgrind",
|
||||
"tests/ui",
|
||||
];
|
||||
for test_path in env.skipped_tests() {
|
||||
args.extend(["--exclude", test_path]);
|
||||
}
|
||||
cmd(&args).env("COMPILETEST_FORCE_STAGE0", "1").run().context("Cannot execute tests")
|
||||
}
|
||||
167
src/tools/opt-dist/src/timer.rs
Normal file
167
src/tools/opt-dist/src/timer.rs
Normal file
|
|
@ -0,0 +1,167 @@
|
|||
use std::ops::{Deref, DerefMut};
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
pub struct Timer {
|
||||
root: TimerSection,
|
||||
}
|
||||
|
||||
impl Timer {
|
||||
pub fn new() -> Self {
|
||||
Timer { root: TimerSection::new(None) }
|
||||
}
|
||||
|
||||
pub fn format_stats(&self) -> String {
|
||||
use std::fmt::Write;
|
||||
|
||||
let mut items = Vec::new();
|
||||
for (name, child) in &self.root.children {
|
||||
match child {
|
||||
SectionEntry::SubSection(section) => {
|
||||
section.collect_levels(0, name, &mut items);
|
||||
}
|
||||
SectionEntry::Duration(duration) => items.push((0, name, *duration)),
|
||||
}
|
||||
}
|
||||
|
||||
let rows: Vec<(String, Duration)> = items
|
||||
.into_iter()
|
||||
.map(|(level, name, duration)| (format!("{}{name}:", " ".repeat(level)), duration))
|
||||
.collect();
|
||||
|
||||
let total_duration = self.total_duration();
|
||||
let total_duration_label = "Total duration:".to_string();
|
||||
|
||||
const SPACE_AFTER_LABEL: usize = 2;
|
||||
let max_label_length = 16.max(rows.iter().map(|(label, _)| label.len()).max().unwrap_or(0))
|
||||
+ SPACE_AFTER_LABEL;
|
||||
|
||||
let table_width = max_label_length + 23;
|
||||
let divider = "-".repeat(table_width);
|
||||
|
||||
let mut output = String::new();
|
||||
writeln!(output, "{divider}").unwrap();
|
||||
for (label, duration) in rows {
|
||||
let pct = (duration.as_millis() as f64 / total_duration.as_millis() as f64) * 100.0;
|
||||
let duration_fmt = format!("{:>12.2}s ({pct:>5.2}%)", duration.as_secs_f64());
|
||||
writeln!(output, "{label:<0$} {duration_fmt}", max_label_length).unwrap();
|
||||
}
|
||||
output.push('\n');
|
||||
|
||||
let total_duration = Duration::new(total_duration.as_secs(), 0);
|
||||
let total_duration = format!(
|
||||
"{:>1$}",
|
||||
humantime::format_duration(total_duration).to_string(),
|
||||
table_width - total_duration_label.len()
|
||||
);
|
||||
writeln!(output, "{total_duration_label}{total_duration}").unwrap();
|
||||
|
||||
writeln!(output, "{divider}").unwrap();
|
||||
output
|
||||
}
|
||||
}
|
||||
|
||||
impl Deref for Timer {
|
||||
type Target = TimerSection;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.root
|
||||
}
|
||||
}
|
||||
|
||||
impl DerefMut for Timer {
|
||||
fn deref_mut(&mut self) -> &mut Self::Target {
|
||||
&mut self.root
|
||||
}
|
||||
}
|
||||
|
||||
pub struct TimerSection {
|
||||
name: Option<String>,
|
||||
children: Vec<(String, SectionEntry)>,
|
||||
duration_excluding_children: Duration,
|
||||
}
|
||||
|
||||
impl TimerSection {
|
||||
pub fn new(name: Option<String>) -> Self {
|
||||
TimerSection {
|
||||
name,
|
||||
children: Default::default(),
|
||||
duration_excluding_children: Duration::ZERO,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn section<F: FnOnce(&mut TimerSection) -> anyhow::Result<R>, R>(
|
||||
&mut self,
|
||||
name: &str,
|
||||
func: F,
|
||||
) -> anyhow::Result<R> {
|
||||
let full_name = match &self.name {
|
||||
Some(current_name) => {
|
||||
format!("{current_name} > {name}")
|
||||
}
|
||||
None => name.to_string(),
|
||||
};
|
||||
log::info!("Section `{full_name}` starts");
|
||||
let mut child = TimerSection {
|
||||
name: Some(full_name.clone()),
|
||||
children: Default::default(),
|
||||
duration_excluding_children: Duration::ZERO,
|
||||
};
|
||||
|
||||
let start = SystemTime::now();
|
||||
let result = func(&mut child);
|
||||
let duration = start.elapsed().unwrap();
|
||||
|
||||
let msg = match result {
|
||||
Ok(_) => "OK",
|
||||
Err(_) => "FAIL",
|
||||
};
|
||||
|
||||
child.duration_excluding_children = duration.saturating_sub(child.total_duration());
|
||||
|
||||
log::info!("Section `{full_name}` ended: {msg} ({:.2}s)`", duration.as_secs_f64());
|
||||
self.children.push((name.to_string(), SectionEntry::SubSection(child)));
|
||||
result
|
||||
}
|
||||
|
||||
pub fn add_duration(&mut self, name: &str, duration: Duration) {
|
||||
self.children.push((name.to_string(), SectionEntry::Duration(duration)));
|
||||
}
|
||||
|
||||
fn total_duration(&self) -> Duration {
|
||||
self.duration_excluding_children
|
||||
+ self.children.iter().map(|(_, child)| child.total_duration()).sum::<Duration>()
|
||||
}
|
||||
|
||||
fn collect_levels<'a>(
|
||||
&'a self,
|
||||
level: usize,
|
||||
name: &'a str,
|
||||
items: &mut Vec<(usize, &'a str, Duration)>,
|
||||
) {
|
||||
items.push((level, name, self.total_duration()));
|
||||
for (name, child) in &self.children {
|
||||
match &child {
|
||||
SectionEntry::Duration(duration) => {
|
||||
items.push((level + 1, name, *duration));
|
||||
}
|
||||
SectionEntry::SubSection(section) => {
|
||||
section.collect_levels(level + 1, name, items);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
enum SectionEntry {
|
||||
Duration(Duration),
|
||||
SubSection(TimerSection),
|
||||
}
|
||||
|
||||
impl SectionEntry {
|
||||
fn total_duration(&self) -> Duration {
|
||||
match self {
|
||||
SectionEntry::Duration(duration) => *duration,
|
||||
SectionEntry::SubSection(timer) => timer.total_duration(),
|
||||
}
|
||||
}
|
||||
}
|
||||
202
src/tools/opt-dist/src/training.rs
Normal file
202
src/tools/opt-dist/src/training.rs
Normal file
|
|
@ -0,0 +1,202 @@
|
|||
use crate::environment::Environment;
|
||||
use crate::exec::{cmd, CmdBuilder};
|
||||
use crate::utils::io::{count_files, delete_directory};
|
||||
use anyhow::Context;
|
||||
use camino::{Utf8Path, Utf8PathBuf};
|
||||
use humansize::BINARY;
|
||||
|
||||
const LLVM_PGO_CRATES: &[&str] = &[
|
||||
"syn-1.0.89",
|
||||
"cargo-0.60.0",
|
||||
"serde-1.0.136",
|
||||
"ripgrep-13.0.0",
|
||||
"regex-1.5.5",
|
||||
"clap-3.1.6",
|
||||
"hyper-0.14.18",
|
||||
];
|
||||
|
||||
const RUSTC_PGO_CRATES: &[&str] = &[
|
||||
"externs",
|
||||
"ctfe-stress-5",
|
||||
"cargo-0.60.0",
|
||||
"token-stream-stress",
|
||||
"match-stress",
|
||||
"tuple-stress",
|
||||
"diesel-1.4.8",
|
||||
"bitmaps-3.1.0",
|
||||
];
|
||||
|
||||
const LLVM_BOLT_CRATES: &[&str] = LLVM_PGO_CRATES;
|
||||
|
||||
fn init_compiler_benchmarks(
|
||||
env: &dyn Environment,
|
||||
profiles: &[&str],
|
||||
scenarios: &[&str],
|
||||
crates: &[&str],
|
||||
) -> CmdBuilder {
|
||||
// Run rustc-perf benchmarks
|
||||
// Benchmark using profile_local with eprintln, which essentially just means
|
||||
// don't actually benchmark -- just make sure we run rustc a bunch of times.
|
||||
cmd(&[
|
||||
env.cargo_stage_0().as_str(),
|
||||
"run",
|
||||
"-p",
|
||||
"collector",
|
||||
"--bin",
|
||||
"collector",
|
||||
"--",
|
||||
"profile_local",
|
||||
"eprintln",
|
||||
env.rustc_stage_2().as_str(),
|
||||
"--id",
|
||||
"Test",
|
||||
"--cargo",
|
||||
env.cargo_stage_0().as_str(),
|
||||
"--profiles",
|
||||
profiles.join(",").as_str(),
|
||||
"--scenarios",
|
||||
scenarios.join(",").as_str(),
|
||||
"--include",
|
||||
crates.join(",").as_str(),
|
||||
])
|
||||
.env("RUST_LOG", "collector=debug")
|
||||
.env("RUSTC", env.rustc_stage_0().as_str())
|
||||
.env("RUSTC_BOOTSTRAP", "1")
|
||||
.workdir(&env.rustc_perf_dir())
|
||||
}
|
||||
|
||||
fn merge_llvm_profiles(
|
||||
env: &dyn Environment,
|
||||
merged_path: &Utf8Path,
|
||||
profile_dir: &Utf8Path,
|
||||
) -> anyhow::Result<()> {
|
||||
cmd(&[
|
||||
env.downloaded_llvm_dir().join("bin/llvm-profdata").as_str(),
|
||||
"merge",
|
||||
"-o",
|
||||
merged_path.as_str(),
|
||||
profile_dir.as_str(),
|
||||
])
|
||||
.run()
|
||||
.context("Cannot merge LLVM profiles")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn log_profile_stats(
|
||||
name: &str,
|
||||
merged_profile: &Utf8Path,
|
||||
profile_root: &Utf8Path,
|
||||
) -> anyhow::Result<()> {
|
||||
log::info!("{name} PGO statistics");
|
||||
log::info!(
|
||||
"{merged_profile}: {}",
|
||||
humansize::format_size(std::fs::metadata(merged_profile.as_std_path())?.len(), BINARY)
|
||||
);
|
||||
log::info!(
|
||||
"{profile_root}: {}",
|
||||
humansize::format_size(fs_extra::dir::get_size(profile_root.as_std_path())?, BINARY)
|
||||
);
|
||||
log::info!("Profile file count: {}", count_files(profile_root)?);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub struct LlvmPGOProfile(pub Utf8PathBuf);
|
||||
|
||||
pub fn gather_llvm_profiles(
|
||||
env: &dyn Environment,
|
||||
profile_root: &Utf8Path,
|
||||
) -> anyhow::Result<LlvmPGOProfile> {
|
||||
log::info!("Running benchmarks with PGO instrumented LLVM");
|
||||
|
||||
init_compiler_benchmarks(env, &["Debug", "Opt"], &["Full"], LLVM_PGO_CRATES)
|
||||
.run()
|
||||
.context("Cannot gather LLVM PGO profiles")?;
|
||||
|
||||
let merged_profile = env.opt_artifacts().join("llvm-pgo.profdata");
|
||||
log::info!("Merging LLVM PGO profiles to {merged_profile}");
|
||||
|
||||
merge_llvm_profiles(env, &merged_profile, profile_root)?;
|
||||
log_profile_stats("LLVM", &merged_profile, profile_root)?;
|
||||
|
||||
// We don't need the individual .profraw files now that they have been merged
|
||||
// into a final .profdata
|
||||
delete_directory(profile_root)?;
|
||||
|
||||
Ok(LlvmPGOProfile(merged_profile))
|
||||
}
|
||||
|
||||
pub struct RustcPGOProfile(pub Utf8PathBuf);
|
||||
|
||||
pub fn gather_rustc_profiles(
|
||||
env: &dyn Environment,
|
||||
profile_root: &Utf8Path,
|
||||
) -> anyhow::Result<RustcPGOProfile> {
|
||||
log::info!("Running benchmarks with PGO instrumented rustc");
|
||||
|
||||
// The profile data is written into a single filepath that is being repeatedly merged when each
|
||||
// rustc invocation ends. Empirically, this can result in some profiling data being lost. That's
|
||||
// why we override the profile path to include the PID. This will produce many more profiling
|
||||
// files, but the resulting profile will produce a slightly faster rustc binary.
|
||||
let profile_template = profile_root.join("default_%m_%p.profraw");
|
||||
|
||||
// Here we're profiling the `rustc` frontend, so we also include `Check`.
|
||||
// The benchmark set includes various stress tests that put the frontend under pressure.
|
||||
init_compiler_benchmarks(env, &["Check", "Debug", "Opt"], &["All"], RUSTC_PGO_CRATES)
|
||||
.env("LLVM_PROFILE_FILE", profile_template.as_str())
|
||||
.run()
|
||||
.context("Cannot gather rustc PGO profiles")?;
|
||||
|
||||
let merged_profile = env.opt_artifacts().join("rustc-pgo.profdata");
|
||||
log::info!("Merging Rustc PGO profiles to {merged_profile}");
|
||||
|
||||
merge_llvm_profiles(env, &merged_profile, profile_root)?;
|
||||
log_profile_stats("Rustc", &merged_profile, profile_root)?;
|
||||
|
||||
// We don't need the individual .profraw files now that they have been merged
|
||||
// into a final .profdata
|
||||
delete_directory(profile_root)?;
|
||||
|
||||
Ok(RustcPGOProfile(merged_profile))
|
||||
}
|
||||
|
||||
pub struct LlvmBoltProfile(pub Utf8PathBuf);
|
||||
|
||||
pub fn gather_llvm_bolt_profiles(env: &dyn Environment) -> anyhow::Result<LlvmBoltProfile> {
|
||||
log::info!("Running benchmarks with BOLT instrumented LLVM");
|
||||
|
||||
init_compiler_benchmarks(env, &["Check", "Debug", "Opt"], &["Full"], LLVM_BOLT_CRATES)
|
||||
.run()
|
||||
.context("Cannot gather LLVM BOLT profiles")?;
|
||||
|
||||
let merged_profile = env.opt_artifacts().join("bolt.profdata");
|
||||
let profile_root = Utf8PathBuf::from("/tmp/prof.fdata");
|
||||
log::info!("Merging LLVM BOLT profiles to {merged_profile}");
|
||||
|
||||
let profiles: Vec<_> =
|
||||
glob::glob(&format!("{profile_root}*"))?.into_iter().collect::<Result<Vec<_>, _>>()?;
|
||||
|
||||
let mut merge_args = vec!["merge-fdata"];
|
||||
merge_args.extend(profiles.iter().map(|p| p.to_str().unwrap()));
|
||||
|
||||
cmd(&merge_args)
|
||||
.redirect_output(merged_profile.clone())
|
||||
.run()
|
||||
.context("Cannot merge BOLT profiles")?;
|
||||
|
||||
log::info!("LLVM BOLT statistics");
|
||||
log::info!(
|
||||
"{merged_profile}: {}",
|
||||
humansize::format_size(std::fs::metadata(merged_profile.as_std_path())?.len(), BINARY)
|
||||
);
|
||||
|
||||
let size = profiles
|
||||
.iter()
|
||||
.map(|p| std::fs::metadata(p).map(|metadata| metadata.len()))
|
||||
.collect::<Result<Vec<_>, _>>()?
|
||||
.into_iter()
|
||||
.sum::<u64>();
|
||||
log::info!("{profile_root}: {}", humansize::format_size(size, BINARY));
|
||||
log::info!("Profile file count: {}", profiles.len());
|
||||
|
||||
Ok(LlvmBoltProfile(merged_profile))
|
||||
}
|
||||
48
src/tools/opt-dist/src/utils/io.rs
Normal file
48
src/tools/opt-dist/src/utils/io.rs
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
use anyhow::Context;
|
||||
use camino::Utf8Path;
|
||||
use fs_extra::dir::CopyOptions;
|
||||
use std::fs::File;
|
||||
|
||||
/// Delete and re-create the directory.
|
||||
pub fn reset_directory(path: &Utf8Path) -> anyhow::Result<()> {
|
||||
log::info!("Resetting directory {path}");
|
||||
let _ = std::fs::remove_dir(path);
|
||||
std::fs::create_dir_all(path)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn copy_directory(src: &Utf8Path, dst: &Utf8Path) -> anyhow::Result<()> {
|
||||
log::info!("Copying directory {src} to {dst}");
|
||||
fs_extra::dir::copy(src, dst, &CopyOptions::default().copy_inside(true))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
pub fn move_directory(src: &Utf8Path, dst: &Utf8Path) -> anyhow::Result<()> {
|
||||
log::info!("Moving directory {src} to {dst}");
|
||||
fs_extra::dir::move_dir(src, dst, &CopyOptions::default().content_only(true))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Counts all children of a directory (non-recursively).
|
||||
pub fn count_files(dir: &Utf8Path) -> anyhow::Result<u64> {
|
||||
Ok(std::fs::read_dir(dir)?.count() as u64)
|
||||
}
|
||||
|
||||
pub fn delete_directory(path: &Utf8Path) -> anyhow::Result<()> {
|
||||
log::info!("Deleting directory `{path}`");
|
||||
std::fs::remove_dir_all(path.as_std_path())
|
||||
.context(format!("Cannot remove directory {path}"))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn unpack_archive(path: &Utf8Path, dest_dir: &Utf8Path) -> anyhow::Result<()> {
|
||||
log::info!("Unpacking directory `{path}` into `{dest_dir}`");
|
||||
|
||||
assert!(path.as_str().ends_with(".tar.xz"));
|
||||
let file = File::open(path.as_std_path())?;
|
||||
let file = xz::read::XzDecoder::new(file);
|
||||
let mut archive = tar::Archive::new(file);
|
||||
archive.unpack(dest_dir.as_std_path())?;
|
||||
Ok(())
|
||||
}
|
||||
36
src/tools/opt-dist/src/utils/mod.rs
Normal file
36
src/tools/opt-dist/src/utils/mod.rs
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
pub mod io;
|
||||
|
||||
use crate::environment::Environment;
|
||||
use crate::utils::io::delete_directory;
|
||||
use humansize::BINARY;
|
||||
use sysinfo::{DiskExt, RefreshKind, System, SystemExt};
|
||||
|
||||
pub fn format_env_variables() -> String {
|
||||
let vars = std::env::vars().map(|(key, value)| format!("{key}={value}")).collect::<Vec<_>>();
|
||||
vars.join("\n")
|
||||
}
|
||||
|
||||
pub fn print_free_disk_space() -> anyhow::Result<()> {
|
||||
let sys = System::new_with_specifics(RefreshKind::default().with_disks_list().with_disks());
|
||||
let available_space: u64 = sys.disks().iter().map(|d| d.available_space()).sum();
|
||||
let total_space: u64 = sys.disks().iter().map(|d| d.total_space()).sum();
|
||||
let used_space = total_space - available_space;
|
||||
|
||||
log::info!(
|
||||
"Free disk space: {} out of total {} ({:.2}% used)",
|
||||
humansize::format_size(available_space, BINARY),
|
||||
humansize::format_size(total_space, BINARY),
|
||||
(used_space as f64 / total_space as f64) * 100.0
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn clear_llvm_files(env: &dyn Environment) -> anyhow::Result<()> {
|
||||
// Bootstrap currently doesn't support rebuilding LLVM when PGO options
|
||||
// change (or any other llvm-related options); so just clear out the relevant
|
||||
// directories ourselves.
|
||||
log::info!("Clearing LLVM build files");
|
||||
delete_directory(&env.build_artifacts().join("llvm"))?;
|
||||
delete_directory(&env.build_artifacts().join("lld"))?;
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -40,10 +40,12 @@
|
|||
("ar_archive_writer", "Apache-2.0 WITH LLVM-exception"), // rustc
|
||||
("colored", "MPL-2.0"), // rustfmt
|
||||
("dissimilar", "Apache-2.0"), // rustdoc, rustc_lexer (few tests) via expect-test, (dev deps)
|
||||
("encoding_rs", "(Apache-2.0 OR MIT) AND BSD-3-Clause"), // opt-dist
|
||||
("fluent-langneg", "Apache-2.0"), // rustc (fluent translations)
|
||||
("fortanix-sgx-abi", "MPL-2.0"), // libstd but only for `sgx` target. FIXME: this dependency violates the documentation comment above.
|
||||
("instant", "BSD-3-Clause"), // rustc_driver/tracing-subscriber/parking_lot
|
||||
("mdbook", "MPL-2.0"), // mdbook
|
||||
("openssl", "Apache-2.0"), // opt-dist
|
||||
("ryu", "Apache-2.0 OR BSL-1.0"), // cargo/... (because of serde)
|
||||
("self_cell", "Apache-2.0"), // rustc (fluent translations)
|
||||
("snap", "BSD-3-Clause"), // rustc
|
||||
|
|
|
|||
Loading…
Reference in a new issue