mirror of
https://github.com/rust-lang/rust
synced 2024-10-14 04:23:37 +00:00
Refactor Environment
This commit is contained in:
Jakub Beránek
parent
95500f4941
commit
f17047bc90
2
.github/workflows/ci.yml
vendored
2
.github/workflows/ci.yml
vendored
|
|
@ -393,7 +393,7 @@ jobs:
|
|||
- name: dist-x86_64-msvc
|
||||
env:
|
||||
RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --host=x86_64-pc-windows-msvc --target=x86_64-pc-windows-msvc --enable-full-tools --enable-profiler"
|
||||
SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist python x.py dist bootstrap --include-default-paths
|
||||
SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist windows-ci -- python x.py dist bootstrap --include-default-paths
|
||||
DIST_REQUIRE_ALL_TOOLS: 1
|
||||
os: windows-2019-8core-32gb
|
||||
- name: dist-i686-msvc
|
||||
|
|
|
|||
76
Cargo.lock
76
Cargo.lock
|
|
@ -858,14 +858,38 @@ dependencies = [
|
|||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "darling"
|
||||
version = "0.14.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850"
|
||||
dependencies = [
|
||||
"darling_core 0.14.4",
|
||||
"darling_macro 0.14.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "darling"
|
||||
version = "0.20.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e"
|
||||
dependencies = [
|
||||
"darling_core",
|
||||
"darling_macro",
|
||||
"darling_core 0.20.3",
|
||||
"darling_macro 0.20.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "darling_core"
|
||||
version = "0.14.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0"
|
||||
dependencies = [
|
||||
"fnv",
|
||||
"ident_case",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"strsim",
|
||||
"syn 1.0.109",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -882,13 +906,24 @@ dependencies = [
|
|||
"syn 2.0.29",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "darling_macro"
|
||||
version = "0.14.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e"
|
||||
dependencies = [
|
||||
"darling_core 0.14.4",
|
||||
"quote",
|
||||
"syn 1.0.109",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "darling_macro"
|
||||
version = "0.20.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5"
|
||||
dependencies = [
|
||||
"darling_core",
|
||||
"darling_core 0.20.3",
|
||||
"quote",
|
||||
"syn 2.0.29",
|
||||
]
|
||||
|
|
@ -919,6 +954,37 @@ dependencies = [
|
|||
"syn 1.0.109",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "derive_builder"
|
||||
version = "0.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8"
|
||||
dependencies = [
|
||||
"derive_builder_macro",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "derive_builder_core"
|
||||
version = "0.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f"
|
||||
dependencies = [
|
||||
"darling 0.14.4",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 1.0.109",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "derive_builder_macro"
|
||||
version = "0.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e"
|
||||
dependencies = [
|
||||
"derive_builder_core",
|
||||
"syn 1.0.109",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "derive_more"
|
||||
version = "0.99.17"
|
||||
|
|
@ -938,7 +1004,7 @@ version = "0.1.6"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4e8ef033054e131169b8f0f9a7af8f5533a9436fadf3c500ed547f730f07090d"
|
||||
dependencies = [
|
||||
"darling",
|
||||
"darling 0.20.3",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.29",
|
||||
|
|
@ -2584,6 +2650,8 @@ dependencies = [
|
|||
"anyhow",
|
||||
"build_helper",
|
||||
"camino",
|
||||
"clap",
|
||||
"derive_builder",
|
||||
"env_logger 0.10.0",
|
||||
"fs_extra",
|
||||
"glob",
|
||||
|
|
|
|||
|
|
@ -87,7 +87,7 @@ ENV RUST_CONFIGURE_ARGS \
|
|||
--set rust.lto=thin
|
||||
|
||||
ENV SCRIPT python3 ../x.py build --set rust.debug=true opt-dist && \
|
||||
./build/$HOSTS/stage0-tools-bin/opt-dist python3 ../x.py dist \
|
||||
./build/$HOSTS/stage0-tools-bin/opt-dist linux-ci -- python3 ../x.py dist \
|
||||
--host $HOSTS --target $HOSTS \
|
||||
--include-default-paths \
|
||||
build-manifest bootstrap
|
||||
|
|
|
|||
|
|
@ -624,7 +624,7 @@ jobs:
|
|||
--target=x86_64-pc-windows-msvc
|
||||
--enable-full-tools
|
||||
--enable-profiler
|
||||
SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist python x.py dist bootstrap --include-default-paths
|
||||
SCRIPT: python x.py build --set rust.debug=true opt-dist && PGO_HOST=x86_64-pc-windows-msvc ./build/x86_64-pc-windows-msvc/stage0-tools-bin/opt-dist windows-ci -- python x.py dist bootstrap --include-default-paths
|
||||
DIST_REQUIRE_ALL_TOOLS: 1
|
||||
<<: *job-windows-8c
|
||||
|
||||
|
|
|
|||
|
|
@ -21,3 +21,5 @@ serde = { version = "1", features = ["derive"] }
|
|||
serde_json = "1"
|
||||
glob = "0.3"
|
||||
tempfile = "3.5"
|
||||
derive_builder = "0.12"
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
|
|
|
|||
100
src/tools/opt-dist/src/environment.rs
Normal file
100
src/tools/opt-dist/src/environment.rs
Normal file
|
|
@ -0,0 +1,100 @@
|
|||
use camino::Utf8PathBuf;
|
||||
use derive_builder::Builder;
|
||||
|
||||
#[derive(Builder)]
|
||||
pub struct Environment {
|
||||
host_triple: String,
|
||||
python_binary: String,
|
||||
/// The rustc checkout, where the compiler source is located.
|
||||
checkout_dir: Utf8PathBuf,
|
||||
/// The main directory where the build occurs.
|
||||
build_dir: Utf8PathBuf,
|
||||
/// Directory where the optimization artifacts (PGO/BOLT profiles, etc.)
|
||||
/// will be stored.
|
||||
artifact_dir: Utf8PathBuf,
|
||||
/// Path to the host LLVM used to compile LLVM in `src/llvm-project`.
|
||||
host_llvm_dir: Utf8PathBuf,
|
||||
/// List of test paths that should be skipped when testing the optimized artifacts.
|
||||
skipped_tests: Vec<&'static str>,
|
||||
use_bolt: bool,
|
||||
shared_llvm: bool,
|
||||
}
|
||||
|
||||
impl Environment {
|
||||
pub fn host_triple(&self) -> &str {
|
||||
&self.host_triple
|
||||
}
|
||||
|
||||
pub fn python_binary(&self) -> &str {
|
||||
&self.python_binary
|
||||
}
|
||||
|
||||
pub fn checkout_path(&self) -> Utf8PathBuf {
|
||||
self.checkout_dir.clone()
|
||||
}
|
||||
|
||||
pub fn build_root(&self) -> Utf8PathBuf {
|
||||
self.build_dir.clone()
|
||||
}
|
||||
|
||||
pub fn build_artifacts(&self) -> Utf8PathBuf {
|
||||
self.build_root().join("build").join(&self.host_triple)
|
||||
}
|
||||
|
||||
pub fn artifact_dir(&self) -> Utf8PathBuf {
|
||||
self.artifact_dir.clone()
|
||||
}
|
||||
|
||||
pub fn cargo_stage_0(&self) -> Utf8PathBuf {
|
||||
self.build_artifacts()
|
||||
.join("stage0")
|
||||
.join("bin")
|
||||
.join(format!("cargo{}", executable_extension()))
|
||||
}
|
||||
|
||||
pub fn rustc_stage_0(&self) -> Utf8PathBuf {
|
||||
self.build_artifacts()
|
||||
.join("stage0")
|
||||
.join("bin")
|
||||
.join(format!("rustc{}", executable_extension()))
|
||||
}
|
||||
|
||||
pub fn rustc_stage_2(&self) -> Utf8PathBuf {
|
||||
self.build_artifacts()
|
||||
.join("stage2")
|
||||
.join("bin")
|
||||
.join(format!("rustc{}", executable_extension()))
|
||||
}
|
||||
|
||||
/// Path to the built rustc-perf benchmark suite.
|
||||
pub fn rustc_perf_dir(&self) -> Utf8PathBuf {
|
||||
self.artifact_dir.join("rustc-perf")
|
||||
}
|
||||
|
||||
pub fn host_llvm_dir(&self) -> Utf8PathBuf {
|
||||
self.host_llvm_dir.clone()
|
||||
}
|
||||
|
||||
pub fn use_bolt(&self) -> bool {
|
||||
self.use_bolt
|
||||
}
|
||||
|
||||
pub fn supports_shared_llvm(&self) -> bool {
|
||||
self.shared_llvm
|
||||
}
|
||||
|
||||
pub fn skipped_tests(&self) -> &[&'static str] {
|
||||
&self.skipped_tests
|
||||
}
|
||||
}
|
||||
|
||||
/// What is the extension of binary executables on this platform?
|
||||
#[cfg(target_family = "unix")]
|
||||
pub fn executable_extension() -> &'static str {
|
||||
""
|
||||
}
|
||||
|
||||
#[cfg(target_family = "windows")]
|
||||
pub fn executable_extension() -> &'static str {
|
||||
".exe"
|
||||
}
|
||||
|
|
@ -1,66 +0,0 @@
|
|||
use crate::environment::Environment;
|
||||
use crate::exec::cmd;
|
||||
use crate::utils::io::copy_directory;
|
||||
use camino::{Utf8Path, Utf8PathBuf};
|
||||
|
||||
pub(super) struct LinuxEnvironment {
|
||||
target_triple: String,
|
||||
}
|
||||
|
||||
impl LinuxEnvironment {
|
||||
pub fn new(target_triple: String) -> Self {
|
||||
Self { target_triple }
|
||||
}
|
||||
}
|
||||
|
||||
impl Environment for LinuxEnvironment {
|
||||
fn host_triple(&self) -> &str {
|
||||
&self.target_triple
|
||||
}
|
||||
|
||||
fn python_binary(&self) -> &'static str {
|
||||
"python3"
|
||||
}
|
||||
|
||||
fn checkout_path(&self) -> Utf8PathBuf {
|
||||
Utf8PathBuf::from("/checkout")
|
||||
}
|
||||
|
||||
fn host_llvm_dir(&self) -> Utf8PathBuf {
|
||||
Utf8PathBuf::from("/rustroot")
|
||||
}
|
||||
|
||||
fn opt_artifacts(&self) -> Utf8PathBuf {
|
||||
Utf8PathBuf::from("/tmp/tmp-multistage/opt-artifacts")
|
||||
}
|
||||
|
||||
fn build_root(&self) -> Utf8PathBuf {
|
||||
self.checkout_path().join("obj")
|
||||
}
|
||||
|
||||
fn prepare_rustc_perf(&self) -> anyhow::Result<()> {
|
||||
// /tmp/rustc-perf comes from the x64 dist Dockerfile
|
||||
copy_directory(Utf8Path::new("/tmp/rustc-perf"), &self.rustc_perf_dir())?;
|
||||
cmd(&[self.cargo_stage_0().as_str(), "build", "-p", "collector"])
|
||||
.workdir(&self.rustc_perf_dir())
|
||||
.env("RUSTC", &self.rustc_stage_0().into_string())
|
||||
.env("RUSTC_BOOTSTRAP", "1")
|
||||
.run()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn supports_bolt(&self) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn supports_shared_llvm(&self) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn skipped_tests(&self) -> &'static [&'static str] {
|
||||
&[
|
||||
// Fails because of linker errors, as of June 2023.
|
||||
"tests/ui/process/nofile-limit.rs",
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
@ -1,83 +0,0 @@
|
|||
use camino::Utf8PathBuf;
|
||||
|
||||
#[cfg(target_family = "unix")]
|
||||
mod linux;
|
||||
#[cfg(target_family = "windows")]
|
||||
mod windows;
|
||||
|
||||
pub trait Environment {
|
||||
fn host_triple(&self) -> &str;
|
||||
|
||||
fn python_binary(&self) -> &'static str;
|
||||
|
||||
/// The rustc checkout, where the compiler source is located.
|
||||
fn checkout_path(&self) -> Utf8PathBuf;
|
||||
|
||||
/// Path to the host LLVM used to compile LLVM in `src/llvm-project`.
|
||||
fn host_llvm_dir(&self) -> Utf8PathBuf;
|
||||
|
||||
/// Directory where the optimization artifacts (PGO/BOLT profiles, etc.)
|
||||
/// will be stored.
|
||||
fn opt_artifacts(&self) -> Utf8PathBuf;
|
||||
|
||||
/// The main directory where the build occurs.
|
||||
fn build_root(&self) -> Utf8PathBuf;
|
||||
|
||||
fn build_artifacts(&self) -> Utf8PathBuf {
|
||||
self.build_root().join("build").join(self.host_triple())
|
||||
}
|
||||
|
||||
fn cargo_stage_0(&self) -> Utf8PathBuf {
|
||||
self.build_artifacts()
|
||||
.join("stage0")
|
||||
.join("bin")
|
||||
.join(format!("cargo{}", executable_extension()))
|
||||
}
|
||||
|
||||
fn rustc_stage_0(&self) -> Utf8PathBuf {
|
||||
self.build_artifacts()
|
||||
.join("stage0")
|
||||
.join("bin")
|
||||
.join(format!("rustc{}", executable_extension()))
|
||||
}
|
||||
|
||||
fn rustc_stage_2(&self) -> Utf8PathBuf {
|
||||
self.build_artifacts()
|
||||
.join("stage2")
|
||||
.join("bin")
|
||||
.join(format!("rustc{}", executable_extension()))
|
||||
}
|
||||
|
||||
/// Path to the built rustc-perf benchmark suite.
|
||||
fn rustc_perf_dir(&self) -> Utf8PathBuf {
|
||||
self.opt_artifacts().join("rustc-perf")
|
||||
}
|
||||
|
||||
/// Download and/or compile rustc-perf.
|
||||
fn prepare_rustc_perf(&self) -> anyhow::Result<()>;
|
||||
|
||||
fn supports_bolt(&self) -> bool;
|
||||
|
||||
fn supports_shared_llvm(&self) -> bool;
|
||||
|
||||
/// List of test paths that should be skipped when testing the optimized artifacts.
|
||||
fn skipped_tests(&self) -> &'static [&'static str];
|
||||
}
|
||||
|
||||
pub fn create_environment(target_triple: String) -> Box<dyn Environment> {
|
||||
#[cfg(target_family = "unix")]
|
||||
return Box::new(linux::LinuxEnvironment::new(target_triple));
|
||||
#[cfg(target_family = "windows")]
|
||||
return Box::new(windows::WindowsEnvironment::new(target_triple));
|
||||
}
|
||||
|
||||
/// What is the extension of binary executables on this platform?
|
||||
#[cfg(target_family = "unix")]
|
||||
pub fn executable_extension() -> &'static str {
|
||||
""
|
||||
}
|
||||
|
||||
#[cfg(target_family = "windows")]
|
||||
pub fn executable_extension() -> &'static str {
|
||||
".exe"
|
||||
}
|
||||
|
|
@ -1,93 +0,0 @@
|
|||
use crate::environment::Environment;
|
||||
use crate::exec::cmd;
|
||||
use crate::utils::io::move_directory;
|
||||
use crate::utils::retry_action;
|
||||
use camino::Utf8PathBuf;
|
||||
use std::io::Cursor;
|
||||
use std::time::Duration;
|
||||
use zip::ZipArchive;
|
||||
|
||||
pub(super) struct WindowsEnvironment {
|
||||
checkout_dir: Utf8PathBuf,
|
||||
target_triple: String,
|
||||
}
|
||||
|
||||
impl WindowsEnvironment {
|
||||
pub fn new(target_triple: String) -> Self {
|
||||
Self { checkout_dir: std::env::current_dir().unwrap().try_into().unwrap(), target_triple }
|
||||
}
|
||||
}
|
||||
|
||||
impl Environment for WindowsEnvironment {
|
||||
fn host_triple(&self) -> &str {
|
||||
&self.target_triple
|
||||
}
|
||||
|
||||
fn python_binary(&self) -> &'static str {
|
||||
"python"
|
||||
}
|
||||
|
||||
fn checkout_path(&self) -> Utf8PathBuf {
|
||||
self.checkout_dir.clone()
|
||||
}
|
||||
|
||||
fn host_llvm_dir(&self) -> Utf8PathBuf {
|
||||
self.checkout_path().join("citools").join("clang-rust")
|
||||
}
|
||||
|
||||
fn opt_artifacts(&self) -> Utf8PathBuf {
|
||||
self.checkout_path().join("opt-artifacts")
|
||||
}
|
||||
|
||||
fn build_root(&self) -> Utf8PathBuf {
|
||||
self.checkout_path()
|
||||
}
|
||||
|
||||
fn prepare_rustc_perf(&self) -> anyhow::Result<()> {
|
||||
// FIXME: add some mechanism for synchronization of this commit SHA with
|
||||
// Linux (which builds rustc-perf in a Dockerfile)
|
||||
// rustc-perf version from 2023-05-30
|
||||
const PERF_COMMIT: &str = "8b2ac3042e1ff2c0074455a0a3618adef97156b1";
|
||||
|
||||
let url = format!("https://ci-mirrors.rust-lang.org/rustc/rustc-perf-{PERF_COMMIT}.zip");
|
||||
let client = reqwest::blocking::Client::builder()
|
||||
.timeout(Duration::from_secs(60 * 2))
|
||||
.connect_timeout(Duration::from_secs(60 * 2))
|
||||
.build()?;
|
||||
let response = retry_action(
|
||||
|| Ok(client.get(&url).send()?.error_for_status()?.bytes()?.to_vec()),
|
||||
"Download rustc-perf archive",
|
||||
5,
|
||||
)?;
|
||||
|
||||
let mut archive = ZipArchive::new(Cursor::new(response))?;
|
||||
archive.extract(self.rustc_perf_dir())?;
|
||||
move_directory(
|
||||
&self.rustc_perf_dir().join(format!("rustc-perf-{PERF_COMMIT}")),
|
||||
&self.rustc_perf_dir(),
|
||||
)?;
|
||||
|
||||
cmd(&[self.cargo_stage_0().as_str(), "build", "-p", "collector"])
|
||||
.workdir(&self.rustc_perf_dir())
|
||||
.env("RUSTC", &self.rustc_stage_0().into_string())
|
||||
.env("RUSTC_BOOTSTRAP", "1")
|
||||
.run()?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn supports_bolt(&self) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn supports_shared_llvm(&self) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn skipped_tests(&self) -> &'static [&'static str] {
|
||||
&[
|
||||
// Fails as of June 2023.
|
||||
"tests\\codegen\\vec-shrink-panik.rs",
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
@ -96,7 +96,7 @@ pub struct Bootstrap {
|
|||
}
|
||||
|
||||
impl Bootstrap {
|
||||
pub fn build(env: &dyn Environment) -> Self {
|
||||
pub fn build(env: &Environment) -> Self {
|
||||
let metrics_path = env.build_root().join("build").join("metrics.json");
|
||||
let cmd = cmd(&[
|
||||
env.python_binary(),
|
||||
|
|
@ -114,7 +114,7 @@ pub fn build(env: &dyn Environment) -> Self {
|
|||
Self { cmd, metrics_path }
|
||||
}
|
||||
|
||||
pub fn dist(env: &dyn Environment, dist_args: &[String]) -> Self {
|
||||
pub fn dist(env: &Environment, dist_args: &[String]) -> Self {
|
||||
let metrics_path = env.build_root().join("build").join("metrics.json");
|
||||
let cmd = cmd(&dist_args.iter().map(|arg| arg.as_str()).collect::<Vec<_>>())
|
||||
.env("RUST_BACKTRACE", "full");
|
||||
|
|
|
|||
|
|
@ -1,17 +1,22 @@
|
|||
use crate::bolt::{bolt_optimize, with_bolt_instrumented};
|
||||
use anyhow::Context;
|
||||
use camino::{Utf8Path, Utf8PathBuf};
|
||||
use clap::Parser;
|
||||
use log::LevelFilter;
|
||||
use std::io::Cursor;
|
||||
use std::time::Duration;
|
||||
use utils::io;
|
||||
use zip::ZipArchive;
|
||||
|
||||
use crate::environment::{create_environment, Environment};
|
||||
use crate::exec::Bootstrap;
|
||||
use crate::environment::{Environment, EnvironmentBuilder};
|
||||
use crate::exec::{cmd, Bootstrap};
|
||||
use crate::tests::run_tests;
|
||||
use crate::timer::Timer;
|
||||
use crate::training::{gather_llvm_bolt_profiles, gather_llvm_profiles, gather_rustc_profiles};
|
||||
use crate::utils::io::reset_directory;
|
||||
use crate::utils::io::{copy_directory, move_directory, reset_directory};
|
||||
use crate::utils::{
|
||||
clear_llvm_files, format_env_variables, print_binary_sizes, print_free_disk_space,
|
||||
with_log_group,
|
||||
retry_action, with_log_group,
|
||||
};
|
||||
|
||||
mod bolt;
|
||||
|
|
@ -23,24 +28,104 @@
|
|||
mod training;
|
||||
mod utils;
|
||||
|
||||
#[derive(clap::Parser, Debug)]
|
||||
struct Args {
|
||||
#[clap(subcommand)]
|
||||
env: EnvironmentCmd,
|
||||
}
|
||||
|
||||
#[derive(clap::Parser, Clone, Debug)]
|
||||
struct SharedArgs {
|
||||
// Arguments passed to `x` to perform the final (dist) build.
|
||||
build_args: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(clap::Parser, Clone, Debug)]
|
||||
enum EnvironmentCmd {
|
||||
LinuxCi {
|
||||
#[clap(flatten)]
|
||||
shared: SharedArgs,
|
||||
},
|
||||
WindowsCi {
|
||||
#[clap(flatten)]
|
||||
shared: SharedArgs,
|
||||
},
|
||||
}
|
||||
|
||||
fn is_try_build() -> bool {
|
||||
std::env::var("DIST_TRY_BUILD").unwrap_or_else(|_| "0".to_string()) != "0"
|
||||
}
|
||||
|
||||
fn create_environment(args: Args) -> anyhow::Result<(Environment, Vec<String>)> {
|
||||
let (env, args) = match args.env {
|
||||
EnvironmentCmd::LinuxCi { shared } => {
|
||||
let target_triple =
|
||||
std::env::var("PGO_HOST").expect("PGO_HOST environment variable missing");
|
||||
|
||||
let checkout_dir = Utf8PathBuf::from("/checkout");
|
||||
let env = EnvironmentBuilder::default()
|
||||
.host_triple(target_triple)
|
||||
.python_binary("python3".to_string())
|
||||
.checkout_dir(checkout_dir.clone())
|
||||
.host_llvm_dir(Utf8PathBuf::from("/rustroot"))
|
||||
.artifact_dir(Utf8PathBuf::from("/tmp/tmp-multistage/opt-artifacts"))
|
||||
.build_dir(checkout_dir.join("obj"))
|
||||
.shared_llvm(true)
|
||||
.use_bolt(true)
|
||||
.skipped_tests(vec![
|
||||
// Fails because of linker errors, as of June 2023.
|
||||
"tests/ui/process/nofile-limit.rs",
|
||||
])
|
||||
.build()?;
|
||||
// /tmp/rustc-perf comes from the x64 dist Dockerfile
|
||||
with_log_group("Building rustc-perf", || {
|
||||
Ok::<(), anyhow::Error>(copy_rustc_perf(&env, Utf8Path::new("/tmp/rustc-perf"))?)
|
||||
})?;
|
||||
|
||||
(env, shared.build_args)
|
||||
}
|
||||
EnvironmentCmd::WindowsCi { shared } => {
|
||||
let target_triple =
|
||||
std::env::var("PGO_HOST").expect("PGO_HOST environment variable missing");
|
||||
|
||||
let checkout_dir: Utf8PathBuf = std::env::current_dir()?.try_into()?;
|
||||
let env = EnvironmentBuilder::default()
|
||||
.host_triple(target_triple)
|
||||
.python_binary("python".to_string())
|
||||
.checkout_dir(checkout_dir.clone())
|
||||
.host_llvm_dir(checkout_dir.join("citools").join("clang-rust"))
|
||||
.artifact_dir(checkout_dir.join("opt-artifacts"))
|
||||
.build_dir(checkout_dir)
|
||||
.shared_llvm(false)
|
||||
.use_bolt(false)
|
||||
.skipped_tests(vec![
|
||||
// Fails as of June 2023.
|
||||
"tests\\codegen\\vec-shrink-panik.rs",
|
||||
])
|
||||
.build()?;
|
||||
|
||||
with_log_group("Building rustc-perf", || {
|
||||
Ok::<(), anyhow::Error>(download_rustc_perf(&env)?)
|
||||
})?;
|
||||
|
||||
(env, shared.build_args)
|
||||
}
|
||||
};
|
||||
Ok((env, args))
|
||||
}
|
||||
|
||||
fn execute_pipeline(
|
||||
env: &dyn Environment,
|
||||
env: &Environment,
|
||||
timer: &mut Timer,
|
||||
dist_args: Vec<String>,
|
||||
) -> anyhow::Result<()> {
|
||||
reset_directory(&env.opt_artifacts())?;
|
||||
|
||||
with_log_group("Building rustc-perf", || env.prepare_rustc_perf())?;
|
||||
reset_directory(&env.artifact_dir())?;
|
||||
|
||||
// Stage 1: Build PGO instrumented rustc
|
||||
// We use a normal build of LLVM, because gathering PGO profiles for LLVM and `rustc` at the
|
||||
// same time can cause issues, because the host and in-tree LLVM versions can diverge.
|
||||
let rustc_pgo_profile = timer.section("Stage 1 (Rustc PGO)", |stage| {
|
||||
let rustc_profile_dir_root = env.opt_artifacts().join("rustc-pgo");
|
||||
let rustc_profile_dir_root = env.artifact_dir().join("rustc-pgo");
|
||||
|
||||
stage.section("Build PGO instrumented rustc and LLVM", |section| {
|
||||
let mut builder = Bootstrap::build(env).rustc_pgo_instrument(&rustc_profile_dir_root);
|
||||
|
|
@ -74,7 +159,7 @@ fn execute_pipeline(
|
|||
// Remove the previous, uninstrumented build of LLVM.
|
||||
clear_llvm_files(env)?;
|
||||
|
||||
let llvm_profile_dir_root = env.opt_artifacts().join("llvm-pgo");
|
||||
let llvm_profile_dir_root = env.artifact_dir().join("llvm-pgo");
|
||||
|
||||
stage.section("Build PGO instrumented LLVM", |section| {
|
||||
Bootstrap::build(env)
|
||||
|
|
@ -95,7 +180,7 @@ fn execute_pipeline(
|
|||
Ok(profile)
|
||||
})?;
|
||||
|
||||
let llvm_bolt_profile = if env.supports_bolt() {
|
||||
let llvm_bolt_profile = if env.use_bolt() {
|
||||
// Stage 3: Build BOLT instrumented LLVM
|
||||
// We build a PGO optimized LLVM in this step, then instrument it with BOLT and gather BOLT profiles.
|
||||
// Note that we don't remove LLVM artifacts after this step, so that they are reused in the final dist build.
|
||||
|
|
@ -171,10 +256,9 @@ fn main() -> anyhow::Result<()> {
|
|||
.parse_default_env()
|
||||
.init();
|
||||
|
||||
let target_triple = std::env::var("PGO_HOST").expect("PGO_HOST environment variable missing");
|
||||
let args = Args::parse();
|
||||
|
||||
let mut build_args: Vec<String> = std::env::args().skip(1).collect();
|
||||
println!("Running optimized build pipeline with args `{}`", build_args.join(" "));
|
||||
println!("Running optimized build pipeline with args `{:?}`", args);
|
||||
|
||||
with_log_group("Environment values", || {
|
||||
println!("Environment values\n{}", format_env_variables());
|
||||
|
|
@ -186,6 +270,8 @@ fn main() -> anyhow::Result<()> {
|
|||
}
|
||||
});
|
||||
|
||||
let (env, mut build_args) = create_environment(args).context("Cannot create environment")?;
|
||||
|
||||
// Skip components that are not needed for try builds to speed them up
|
||||
if is_try_build() {
|
||||
log::info!("Skipping building of unimportant components for a try build");
|
||||
|
|
@ -204,14 +290,56 @@ fn main() -> anyhow::Result<()> {
|
|||
}
|
||||
|
||||
let mut timer = Timer::new();
|
||||
let env = create_environment(target_triple);
|
||||
|
||||
let result = execute_pipeline(env.as_ref(), &mut timer, build_args);
|
||||
let result = execute_pipeline(&env, &mut timer, build_args);
|
||||
log::info!("Timer results\n{}", timer.format_stats());
|
||||
|
||||
print_free_disk_space()?;
|
||||
result.context("Optimized build pipeline has failed")?;
|
||||
print_binary_sizes(env.as_ref())?;
|
||||
print_binary_sizes(&env)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// Copy rustc-perf from the given path into the environment and build it.
|
||||
fn copy_rustc_perf(env: &Environment, dir: &Utf8Path) -> anyhow::Result<()> {
|
||||
copy_directory(dir, &env.rustc_perf_dir())?;
|
||||
build_rustc_perf(env)
|
||||
}
|
||||
|
||||
// Download and build rustc-perf into the given environment.
|
||||
fn download_rustc_perf(env: &Environment) -> anyhow::Result<()> {
|
||||
// FIXME: add some mechanism for synchronization of this commit SHA with
|
||||
// Linux (which builds rustc-perf in a Dockerfile)
|
||||
// rustc-perf version from 2023-05-30
|
||||
const PERF_COMMIT: &str = "8b2ac3042e1ff2c0074455a0a3618adef97156b1";
|
||||
|
||||
let url = format!("https://ci-mirrors.rust-lang.org/rustc/rustc-perf-{PERF_COMMIT}.zip");
|
||||
let client = reqwest::blocking::Client::builder()
|
||||
.timeout(Duration::from_secs(60 * 2))
|
||||
.connect_timeout(Duration::from_secs(60 * 2))
|
||||
.build()?;
|
||||
let response = retry_action(
|
||||
|| Ok(client.get(&url).send()?.error_for_status()?.bytes()?.to_vec()),
|
||||
"Download rustc-perf archive",
|
||||
5,
|
||||
)?;
|
||||
|
||||
let mut archive = ZipArchive::new(Cursor::new(response))?;
|
||||
archive.extract(env.rustc_perf_dir())?;
|
||||
move_directory(
|
||||
&env.rustc_perf_dir().join(format!("rustc-perf-{PERF_COMMIT}")),
|
||||
&env.rustc_perf_dir(),
|
||||
)?;
|
||||
|
||||
build_rustc_perf(env)
|
||||
}
|
||||
|
||||
fn build_rustc_perf(env: &Environment) -> anyhow::Result<()> {
|
||||
cmd(&[env.cargo_stage_0().as_str(), "build", "-p", "collector"])
|
||||
.workdir(&env.rustc_perf_dir())
|
||||
.env("RUSTC", &env.rustc_stage_0().into_string())
|
||||
.env("RUSTC_BOOTSTRAP", "1")
|
||||
.run()?;
|
||||
Ok(())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
use camino::{Utf8Path, Utf8PathBuf};
|
||||
|
||||
/// Run tests on optimized dist artifacts.
|
||||
pub fn run_tests(env: &dyn Environment) -> anyhow::Result<()> {
|
||||
pub fn run_tests(env: &Environment) -> anyhow::Result<()> {
|
||||
// After `dist` is executed, we extract its archived components into a sysroot directory,
|
||||
// and then use that extracted rustc as a stage0 compiler.
|
||||
// Then we run a subset of tests using that compiler, to have a basic smoke test which checks
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@
|
|||
const LLVM_BOLT_CRATES: &[&str] = LLVM_PGO_CRATES;
|
||||
|
||||
fn init_compiler_benchmarks(
|
||||
env: &dyn Environment,
|
||||
env: &Environment,
|
||||
profiles: &[&str],
|
||||
scenarios: &[&str],
|
||||
crates: &[&str],
|
||||
|
|
@ -75,7 +75,7 @@ enum LlvmProfdata {
|
|||
}
|
||||
|
||||
fn merge_llvm_profiles(
|
||||
env: &dyn Environment,
|
||||
env: &Environment,
|
||||
merged_path: &Utf8Path,
|
||||
profile_dir: &Utf8Path,
|
||||
profdata: LlvmProfdata,
|
||||
|
|
@ -116,7 +116,7 @@ fn log_profile_stats(
|
|||
pub struct LlvmPGOProfile(pub Utf8PathBuf);
|
||||
|
||||
pub fn gather_llvm_profiles(
|
||||
env: &dyn Environment,
|
||||
env: &Environment,
|
||||
profile_root: &Utf8Path,
|
||||
) -> anyhow::Result<LlvmPGOProfile> {
|
||||
log::info!("Running benchmarks with PGO instrumented LLVM");
|
||||
|
|
@ -127,7 +127,7 @@ pub fn gather_llvm_profiles(
|
|||
.context("Cannot gather LLVM PGO profiles")
|
||||
})?;
|
||||
|
||||
let merged_profile = env.opt_artifacts().join("llvm-pgo.profdata");
|
||||
let merged_profile = env.artifact_dir().join("llvm-pgo.profdata");
|
||||
log::info!("Merging LLVM PGO profiles to {merged_profile}");
|
||||
|
||||
merge_llvm_profiles(env, &merged_profile, profile_root, LlvmProfdata::Host)?;
|
||||
|
|
@ -143,7 +143,7 @@ pub fn gather_llvm_profiles(
|
|||
pub struct RustcPGOProfile(pub Utf8PathBuf);
|
||||
|
||||
pub fn gather_rustc_profiles(
|
||||
env: &dyn Environment,
|
||||
env: &Environment,
|
||||
profile_root: &Utf8Path,
|
||||
) -> anyhow::Result<RustcPGOProfile> {
|
||||
log::info!("Running benchmarks with PGO instrumented rustc");
|
||||
|
|
@ -163,7 +163,7 @@ pub fn gather_rustc_profiles(
|
|||
.context("Cannot gather rustc PGO profiles")
|
||||
})?;
|
||||
|
||||
let merged_profile = env.opt_artifacts().join("rustc-pgo.profdata");
|
||||
let merged_profile = env.artifact_dir().join("rustc-pgo.profdata");
|
||||
log::info!("Merging Rustc PGO profiles to {merged_profile}");
|
||||
|
||||
merge_llvm_profiles(env, &merged_profile, profile_root, LlvmProfdata::Target)?;
|
||||
|
|
@ -178,7 +178,7 @@ pub fn gather_rustc_profiles(
|
|||
|
||||
pub struct LlvmBoltProfile(pub Utf8PathBuf);
|
||||
|
||||
pub fn gather_llvm_bolt_profiles(env: &dyn Environment) -> anyhow::Result<LlvmBoltProfile> {
|
||||
pub fn gather_llvm_bolt_profiles(env: &Environment) -> anyhow::Result<LlvmBoltProfile> {
|
||||
log::info!("Running benchmarks with BOLT instrumented LLVM");
|
||||
|
||||
with_log_group("Running benchmarks", || {
|
||||
|
|
@ -187,7 +187,7 @@ pub fn gather_llvm_bolt_profiles(env: &dyn Environment) -> anyhow::Result<LlvmBo
|
|||
.context("Cannot gather LLVM BOLT profiles")
|
||||
})?;
|
||||
|
||||
let merged_profile = env.opt_artifacts().join("llvm-bolt.profdata");
|
||||
let merged_profile = env.artifact_dir().join("llvm-bolt.profdata");
|
||||
let profile_root = Utf8PathBuf::from("/tmp/prof.fdata");
|
||||
log::info!("Merging LLVM BOLT profiles to {merged_profile}");
|
||||
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ pub fn print_free_disk_space() -> anyhow::Result<()> {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
pub fn print_binary_sizes(env: &dyn Environment) -> anyhow::Result<()> {
|
||||
pub fn print_binary_sizes(env: &Environment) -> anyhow::Result<()> {
|
||||
use std::fmt::Write;
|
||||
|
||||
let root = env.build_artifacts().join("stage2");
|
||||
|
|
@ -48,7 +48,7 @@ pub fn print_binary_sizes(env: &dyn Environment) -> anyhow::Result<()> {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
pub fn clear_llvm_files(env: &dyn Environment) -> anyhow::Result<()> {
|
||||
pub fn clear_llvm_files(env: &Environment) -> anyhow::Result<()> {
|
||||
// Bootstrap currently doesn't support rebuilding LLVM when PGO options
|
||||
// change (or any other llvm-related options); so just clear out the relevant
|
||||
// directories ourselves.
|
||||
|
|
|
|||
Loading…
Reference in a new issue