Rollup merge of #127002 - Kobzol:bootstrap-perf-tool, r=onur-ozkan

Implement `x perf` as a separate tool

Continues work from https://github.com/rust-lang/rust/pull/126318, adds a CLI for running `rustc-perf` profiling commands through a new `rustc-perf-wrapper` tool. The CLI is in a separate tool to enable experimentation outside of `bootstrap`.

This is probably most of what we can do so far, I'll add support for benchmarking once `rustc-perf` gets a terminal output for comparing benchmark results.

r? ``@onur-ozkan``

Cargo.lock

@@ -3461,6 +3461,13 @@ dependencies = [
  "stable_mir",
 ]
 
+[[package]]
+name = "rustc-perf-wrapper"
+version = "0.1.0"
+dependencies = [
+ "clap",
+]
+
 [[package]]
 name = "rustc-rayon"
 version = "0.5.0"

Cargo.toml

@@ -44,6 +44,7 @@ members = [
   "src/tools/rustdoc-gui-test",
   "src/tools/opt-dist",
   "src/tools/coverage-dump",
+  "src/tools/rustc-perf-wrapper",
 ]
 
 exclude = [

src/bootstrap/src/core/build_steps/perf.rs

@@ -1,7 +1,5 @@
-use std::process::Command;
-
 use crate::core::build_steps::compile::{Std, Sysroot};
-use crate::core::build_steps::tool::RustcPerf;
+use crate::core::build_steps::tool::{RustcPerf, Tool};
 use crate::core::builder::Builder;
 use crate::core::config::DebuginfoLevel;
 
@@ -22,24 +20,16 @@ pub fn perf(builder: &Builder<'_>) {
     let sysroot = builder.ensure(Sysroot::new(compiler));
     let rustc = sysroot.join("bin/rustc");
 
-    let results_dir = builder.build.tempdir().join("rustc-perf");
+    let rustc_perf_dir = builder.build.tempdir().join("rustc-perf");
+    let profile_results_dir = rustc_perf_dir.join("results");
 
-    let mut cmd = Command::new(collector);
-    let cmd = cmd
-        .arg("profile_local")
-        .arg("eprintln")
-        .arg("--out-dir")
-        .arg(&results_dir)
-        .arg("--include")
-        .arg("helloworld")
-        .arg(&rustc);
+    // We need to take args passed after `--` and pass them to `rustc-perf-wrapper`
+    let args = std::env::args().skip_while(|a| a != "--").skip(1);
 
-    builder.info(&format!("Running `rustc-perf` using `{}`", rustc.display()));
-
-    // We need to set the working directory to `src/tools/perf`, so that it can find the directory
-    // with compile-time benchmarks.
-    let cmd = cmd.current_dir(builder.src.join("src/tools/rustc-perf"));
-    builder.build.run(cmd);
-
-    builder.info(&format!("You can find the results at `{}`", results_dir.display()));
+    let mut cmd = builder.tool_cmd(Tool::RustcPerfWrapper);
+    cmd.env("RUSTC_REAL", rustc)
+        .env("PERF_COLLECTOR", collector)
+        .env("PERF_RESULT_DIR", profile_results_dir)
+        .args(args);
+    builder.run(&mut cmd);
 }

src/bootstrap/src/core/build_steps/tool.rs

@@ -337,6 +337,7 @@ fn run(self, builder: &Builder<'_>) -> PathBuf {
     GenerateWindowsSys, "src/tools/generate-windows-sys", "generate-windows-sys";
     RustdocGUITest, "src/tools/rustdoc-gui-test", "rustdoc-gui-test", is_unstable_tool = true, allow_features = "test";
     CoverageDump, "src/tools/coverage-dump", "coverage-dump";
+    RustcPerfWrapper, "src/tools/rustc-perf-wrapper", "rustc-perf-wrapper";
 );
 
 #[derive(Debug, Clone, Hash, PartialEq, Eq)]

src/bootstrap/src/core/config/flags.rs

@@ -470,7 +470,9 @@ pub enum Subcommand {
         versioned_dirs: bool,
     },
     /// Perform profiling and benchmarking of the compiler using the
-    /// `rustc-perf` benchmark suite.
+    /// `rustc-perf-wrapper` tool.
+    ///
+    /// You need to pass arguments after `--`, e.g.`x perf -- cachegrind`.
     Perf {},
 }
 

src/etc/completions/x.py.fish

@@ -48,7 +48,7 @@ complete -c x.py -n "__fish_use_subcommand" -f -a "run" -d 'Run tools contained
 complete -c x.py -n "__fish_use_subcommand" -f -a "setup" -d 'Set up the environment for development'
 complete -c x.py -n "__fish_use_subcommand" -f -a "suggest" -d 'Suggest a subset of tests to run, based on modified files'
 complete -c x.py -n "__fish_use_subcommand" -f -a "vendor" -d 'Vendor dependencies'
-complete -c x.py -n "__fish_use_subcommand" -f -a "perf" -d 'Perform profiling and benchmarking of the compiler using the `rustc-perf` benchmark suite'
+complete -c x.py -n "__fish_use_subcommand" -f -a "perf" -d 'Perform profiling and benchmarking of the compiler using the `rustc-perf-wrapper` tool'
 complete -c x.py -n "__fish_seen_subcommand_from build" -l config -d 'TOML configuration file for build' -r -F
 complete -c x.py -n "__fish_seen_subcommand_from build" -l build-dir -d 'Build directory, overrides `build.build-dir` in `config.toml`' -r -f -a "(__fish_complete_directories)"
 complete -c x.py -n "__fish_seen_subcommand_from build" -l build -d 'build target of the stage0 compiler' -r -f

src/etc/completions/x.py.ps1

@@ -75,7 +75,7 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock {
             [CompletionResult]::new('setup', 'setup', [CompletionResultType]::ParameterValue, 'Set up the environment for development')
             [CompletionResult]::new('suggest', 'suggest', [CompletionResultType]::ParameterValue, 'Suggest a subset of tests to run, based on modified files')
             [CompletionResult]::new('vendor', 'vendor', [CompletionResultType]::ParameterValue, 'Vendor dependencies')
-            [CompletionResult]::new('perf', 'perf', [CompletionResultType]::ParameterValue, 'Perform profiling and benchmarking of the compiler using the `rustc-perf` benchmark suite')
+            [CompletionResult]::new('perf', 'perf', [CompletionResultType]::ParameterValue, 'Perform profiling and benchmarking of the compiler using the `rustc-perf-wrapper` tool')
             break
         }
         'x.py;build' {

src/etc/completions/x.py.zsh

@@ -856,7 +856,7 @@ _x.py_commands() {
 'setup:Set up the environment for development' \
 'suggest:Suggest a subset of tests to run, based on modified files' \
 'vendor:Vendor dependencies' \
-'perf:Perform profiling and benchmarking of the compiler using the \`rustc-perf\` benchmark suite' \
+'perf:Perform profiling and benchmarking of the compiler using the \`rustc-perf-wrapper\` tool' \
     )
     _describe -t commands 'x.py commands' commands "$@"
 }

src/tools/rustc-perf-wrapper/Cargo.toml

@@ -0,0 +1,7 @@
+[package]
+name = "rustc-perf-wrapper"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+clap = { version = "4.5.7", features = ["derive", "env"] }

src/tools/rustc-perf-wrapper/README.md

@@ -0,0 +1,3 @@
+# rustc-perf wrapper
+Utility tool for invoking [`rustc-perf`](https://github.com/rust-lang/rustc-perf) for benchmarking/profiling
+a stage1/2 compiler built by bootstrap using `x perf -- <command>`.

src/tools/rustc-perf-wrapper/src/config.rs

@@ -0,0 +1,45 @@
+use std::fmt::{Display, Formatter};
+
+#[derive(Clone, Copy, Debug, clap::ValueEnum)]
+#[value(rename_all = "PascalCase")]
+pub enum Profile {
+    Check,
+    Debug,
+    Doc,
+    Opt,
+    Clippy,
+}
+
+impl Display for Profile {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let name = match self {
+            Profile::Check => "Check",
+            Profile::Debug => "Debug",
+            Profile::Doc => "Doc",
+            Profile::Opt => "Opt",
+            Profile::Clippy => "Clippy",
+        };
+        f.write_str(name)
+    }
+}
+
+#[derive(Clone, Copy, Debug, clap::ValueEnum)]
+#[value(rename_all = "PascalCase")]
+pub enum Scenario {
+    Full,
+    IncrFull,
+    IncrUnchanged,
+    IncrPatched,
+}
+
+impl Display for Scenario {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let name = match self {
+            Scenario::Full => "Full",
+            Scenario::IncrFull => "IncrFull",
+            Scenario::IncrUnchanged => "IncrUnchanged",
+            Scenario::IncrPatched => "IncrPatched",
+        };
+        f.write_str(name)
+    }
+}

src/tools/rustc-perf-wrapper/src/main.rs

@@ -0,0 +1,130 @@
+use crate::config::{Profile, Scenario};
+use clap::Parser;
+use std::path::PathBuf;
+use std::process::Command;
+
+mod config;
+
+/// Performs profiling or benchmarking with [`rustc-perf`](https://github.com/rust-lang/rustc-perf)
+/// using a locally built compiler.
+#[derive(Debug, clap::Parser)]
+// Hide arguments from BuildContext in the default usage string.
+// Clap does not seem to have a way of disabling the usage of these arguments.
+#[clap(override_usage = "rustc-perf-wrapper [OPTIONS] <COMMAND>")]
+pub struct Args {
+    #[clap(subcommand)]
+    cmd: PerfCommand,
+
+    #[clap(flatten)]
+    opts: SharedOpts,
+
+    #[clap(flatten)]
+    ctx: BuildContext,
+}
+
+#[derive(Debug, clap::Parser)]
+enum PerfCommand {
+    /// Run `profile_local eprintln`.
+    /// This executes the compiler on the given benchmarks and stores its stderr output.
+    Eprintln,
+    /// Run `profile_local samply`
+    /// This executes the compiler on the given benchmarks and profiles it with `samply`.
+    /// You need to install `samply`, e.g. using `cargo install samply`.
+    Samply,
+    /// Run `profile_local cachegrind`.
+    /// This executes the compiler on the given benchmarks under `Cachegrind`.
+    Cachegrind,
+}
+
+impl PerfCommand {
+    fn is_profiling(&self) -> bool {
+        match self {
+            PerfCommand::Eprintln | PerfCommand::Samply | PerfCommand::Cachegrind => true,
+        }
+    }
+}
+
+#[derive(Debug, clap::Parser)]
+struct SharedOpts {
+    /// Select the benchmarks that you want to run (separated by commas).
+    /// If unspecified, all benchmarks will be executed.
+    #[clap(long, global = true, value_delimiter = ',')]
+    include: Vec<String>,
+    /// Select the scenarios that should be benchmarked.
+    #[clap(
+        long,
+        global = true,
+        value_delimiter = ',',
+        default_value = "Full,IncrFull,IncrUnchanged,IncrPatched"
+    )]
+    scenarios: Vec<Scenario>,
+    /// Select the profiles that should be benchmarked.
+    #[clap(long, global = true, value_delimiter = ',', default_value = "Check,Debug,Opt")]
+    profiles: Vec<Profile>,
+}
+
+/// These arguments are mostly designed to be passed from bootstrap, not by users
+/// directly.
+#[derive(Debug, clap::Parser)]
+struct BuildContext {
+    /// Compiler binary that will be benchmarked/profiled.
+    #[clap(long, hide = true, env = "RUSTC_REAL")]
+    compiler: PathBuf,
+    /// rustc-perf collector binary that will be used for running benchmarks/profilers.
+    #[clap(long, hide = true, env = "PERF_COLLECTOR")]
+    collector: PathBuf,
+    /// Directory where to store results.
+    #[clap(long, hide = true, env = "PERF_RESULT_DIR")]
+    results_dir: PathBuf,
+}
+
+fn main() {
+    let args = Args::parse();
+    run(args);
+}
+
+fn run(args: Args) {
+    let mut cmd = Command::new(args.ctx.collector);
+    match &args.cmd {
+        PerfCommand::Eprintln => {
+            cmd.arg("profile_local").arg("eprintln");
+        }
+        PerfCommand::Samply => {
+            cmd.arg("profile_local").arg("samply");
+        }
+        PerfCommand::Cachegrind => {
+            cmd.arg("profile_local").arg("cachegrind");
+        }
+    }
+    if args.cmd.is_profiling() {
+        cmd.arg("--out-dir").arg(&args.ctx.results_dir);
+    }
+
+    if !args.opts.include.is_empty() {
+        cmd.arg("--include").arg(args.opts.include.join(","));
+    }
+    if !args.opts.profiles.is_empty() {
+        cmd.arg("--profiles")
+            .arg(args.opts.profiles.iter().map(|p| p.to_string()).collect::<Vec<_>>().join(","));
+    }
+    if !args.opts.scenarios.is_empty() {
+        cmd.arg("--scenarios")
+            .arg(args.opts.scenarios.iter().map(|p| p.to_string()).collect::<Vec<_>>().join(","));
+    }
+    cmd.arg(&args.ctx.compiler);
+
+    println!("Running `rustc-perf` using `{}`", args.ctx.compiler.display());
+
+    const MANIFEST_DIR: &str = env!("CARGO_MANIFEST_DIR");
+
+    let rustc_perf_dir = PathBuf::from(MANIFEST_DIR).join("../rustc-perf");
+
+    // We need to set the working directory to `src/tools/perf`, so that it can find the directory
+    // with compile-time benchmarks.
+    let cmd = cmd.current_dir(rustc_perf_dir);
+    cmd.status().expect("error while running rustc-perf collector");
+
+    if args.cmd.is_profiling() {
+        println!("You can find the results at `{}`", args.ctx.results_dir.display());
+    }
+}

triagebot.toml

@@ -331,6 +331,7 @@ trigger_files = [
     "src/tools/tidy",
     "src/tools/rustdoc-gui-test",
     "src/tools/libcxx-version",
+    "src/tools/rustc-perf-wrapper",
 ]
 
 [autolabel."T-infra"]