Auto merge of #123317 - RalfJung:test-in-miri, r=m-ou-se,saethlin,onur-ozkan

Support running library tests in Miri

This adds a new bootstrap subcommand `./x.py miri` which can test libraries in Miri. This is in preparation for eventually doing that as part of bors CI, but this PR only adds the infrastructure, and doesn't enable it yet.

`@rust-lang/bootstrap` should this be `x.py test --miri library/core` or `x.py miri library/core`? The flag has the advantage that we don't have to copy all the arguments from `Subcommand::Test`. It has the disadvantage that most test steps just ignore `--miri` and still run tests the regular way. For clippy you went the route of making it a separate subcommand. ~~I went with a flag now as that seemed easier, but I can change this.~~ I made it a new subcommand. Note however that the regular cargo invocation would be `cargo miri test ...`, so `x.py` is still going to be different in that the `test` is omitted. That said, we could also make it `./x.py miri-test` to make that difference smaller -- that's in fact more consistent with the internal name of the command when bootstrap invokes cargo.

`@rust-lang/libs` ~~unfortunately this PR does some unholy things to the `lib.rs` files of our library crates.~~
`@m-ou-se` found a way that entirely avoids library-level hacks, except for some new small `lib.miri.rs` files that hopefully you will never have to touch. There's a new hack in cargo-miri but there it is in good company...
This commit is contained in:
bors 2024-04-05 13:17:09 +00:00
commit 5958f5e08f
17 changed files with 491 additions and 69 deletions

View file

@ -5,6 +5,11 @@
const BENCH_N: usize = 1000;
fn main() {
if cfg!(miri) {
// Don't benchmark Miri...
// (Due to bootstrap quirks, this gets picked up by `x.py miri library/alloc --no-doc`.)
return;
}
let a: VecDeque<i32> = (0..VECDEQUE_LEN).collect();
let b: VecDeque<i32> = (0..VECDEQUE_LEN).collect();

View file

@ -0,0 +1,4 @@
//! Grep bootstrap for `MIRI_REPLACE_LIBRS_IF_NOT_TEST` to learn what this is about.
#![no_std]
extern crate alloc as realalloc;
pub use realalloc::*;

View file

@ -0,0 +1,4 @@
//! Grep bootstrap for `MIRI_REPLACE_LIBRS_IF_NOT_TEST` to learn what this is about.
#![no_std]
extern crate core as realcore;
pub use realcore::*;

View file

@ -0,0 +1,4 @@
//! Grep bootstrap for `MIRI_REPLACE_LIBRS_IF_NOT_TEST` to learn what this is about.
#![no_std]
extern crate std as realstd;
pub use realstd::*;

View file

@ -595,7 +595,7 @@ fn run(self, builder: &Builder<'_>) {
// This is for the tests so everything is done with the target compiler.
let miri_sysroot = Miri::build_miri_sysroot(builder, target_compiler, target);
builder.ensure(compile::Std::new(target_compiler, host));
let sysroot = builder.sysroot(target_compiler);
let host_sysroot = builder.sysroot(target_compiler);
// # Run `cargo test`.
// This is with the Miri crate, so it uses the host compiler.
@ -618,7 +618,7 @@ fn run(self, builder: &Builder<'_>) {
// miri tests need to know about the stage sysroot
cargo.env("MIRI_SYSROOT", &miri_sysroot);
cargo.env("MIRI_HOST_SYSROOT", &sysroot);
cargo.env("MIRI_HOST_SYSROOT", &host_sysroot);
cargo.env("MIRI", &miri);
// Set the target.
@ -681,10 +681,6 @@ fn run(self, builder: &Builder<'_>) {
}
}
// Tell `cargo miri` where to find the sysroots.
cargo.env("MIRI_SYSROOT", &miri_sysroot);
cargo.env("MIRI_HOST_SYSROOT", sysroot);
// Finally, pass test-args and run everything.
cargo.arg("--").args(builder.config.test_args());
let mut cargo = Command::from(cargo);
@ -2540,9 +2536,14 @@ fn prepare_cargo_test(
//
// Note that to run the compiler we need to run with the *host* libraries,
// but our wrapper scripts arrange for that to be the case anyway.
let mut dylib_path = dylib_path();
dylib_path.insert(0, PathBuf::from(&*builder.sysroot_libdir(compiler, target)));
cargo.env(dylib_path_var(), env::join_paths(&dylib_path).unwrap());
//
// We skip everything on Miri as then this overwrites the libdir set up
// by `Cargo::new` and that actually makes things go wrong.
if builder.kind != Kind::Miri {
let mut dylib_path = dylib_path();
dylib_path.insert(0, PathBuf::from(&*builder.sysroot_libdir(compiler, target)));
cargo.env(dylib_path_var(), env::join_paths(&dylib_path).unwrap());
}
if builder.remote_tested(target) {
cargo.env(
@ -2598,28 +2599,62 @@ fn run(self, builder: &Builder<'_>) {
let target = self.target;
let mode = self.mode;
// Prepare sysroot
// See [field@compile::Std::force_recompile].
builder.ensure(compile::Std::force_recompile(compiler, compiler.host));
if builder.config.build != target {
builder.ensure(compile::Std::force_recompile(compiler, target));
builder.ensure(RemoteCopyLibs { compiler, target });
}
// If we're not doing a full bootstrap but we're testing a stage2
// version of libstd, then what we're actually testing is the libstd
// produced in stage1. Reflect that here by updating the compiler that
// we're working with automatically.
let compiler = builder.compiler_for(compiler.stage, compiler.host, target);
let mut cargo = builder::Cargo::new(
builder,
compiler,
mode,
SourceType::InTree,
target,
builder.kind.as_str(),
);
let mut cargo = if builder.kind == Kind::Miri {
if builder.top_stage == 0 {
eprintln!("ERROR: `x.py miri` requires stage 1 or higher");
std::process::exit(1);
}
// Build `cargo miri test` command
// (Implicitly prepares target sysroot)
let mut cargo = builder::Cargo::new(
builder,
compiler,
mode,
SourceType::InTree,
target,
"miri-test",
);
// This hack helps bootstrap run standard library tests in Miri. The issue is as
// follows: when running `cargo miri test` on libcore, cargo builds a local copy of core
// and makes it a dependency of the integration test crate. This copy duplicates all the
// lang items, so the build fails. (Regular testing avoids this because the sysroot is a
// literal copy of what `cargo build` produces, but since Miri builds its own sysroot
// this does not work for us.) So we need to make it so that the locally built libcore
// contains all the items from `core`, but does not re-define them -- we want to replace
// the entire crate but a re-export of the sysroot crate. We do this by swapping out the
// source file: if `MIRI_REPLACE_LIBRS_IF_NOT_TEST` is set and we are building a
// `lib.rs` file, and a `lib.miri.rs` file exists in the same folder, we build that
// instead. But crucially we only do that for the library, not the test builds.
cargo.env("MIRI_REPLACE_LIBRS_IF_NOT_TEST", "1");
cargo
} else {
// Also prepare a sysroot for the target.
if builder.config.build != target {
builder.ensure(compile::Std::force_recompile(compiler, target));
builder.ensure(RemoteCopyLibs { compiler, target });
}
// Build `cargo test` command
builder::Cargo::new(
builder,
compiler,
mode,
SourceType::InTree,
target,
builder.kind.as_str(),
)
};
match mode {
Mode::Std => {

View file

@ -631,6 +631,7 @@ pub enum Kind {
Format,
#[value(alias = "t")]
Test,
Miri,
Bench,
#[value(alias = "d")]
Doc,
@ -673,6 +674,7 @@ pub fn as_str(&self) -> &'static str {
Kind::Fix => "fix",
Kind::Format => "fmt",
Kind::Test => "test",
Kind::Miri => "miri",
Kind::Bench => "bench",
Kind::Doc => "doc",
Kind::Clean => "clean",
@ -822,6 +824,7 @@ macro_rules! describe {
// Run run-make last, since these won't pass without make on Windows
test::RunMake,
),
Kind::Miri => describe!(test::Crate),
Kind::Bench => describe!(test::Crate, test::CrateLibrustc),
Kind::Doc => describe!(
doc::UnstableBook,
@ -970,6 +973,7 @@ pub fn new(build: &Build) -> Builder<'_> {
Subcommand::Fix => (Kind::Fix, &paths[..]),
Subcommand::Doc { .. } => (Kind::Doc, &paths[..]),
Subcommand::Test { .. } => (Kind::Test, &paths[..]),
Subcommand::Miri { .. } => (Kind::Miri, &paths[..]),
Subcommand::Bench { .. } => (Kind::Bench, &paths[..]),
Subcommand::Dist => (Kind::Dist, &paths[..]),
Subcommand::Install => (Kind::Install, &paths[..]),
@ -1226,6 +1230,7 @@ pub fn cargo_miri_cmd(&self, run_compiler: Compiler) -> Command {
assert!(run_compiler.stage > 0, "miri can not be invoked at stage 0");
let build_compiler = self.compiler(run_compiler.stage - 1, self.build.build);
// Prepare the tools
let miri = self.ensure(tool::Miri {
compiler: build_compiler,
target: self.build.build,
@ -1236,7 +1241,7 @@ pub fn cargo_miri_cmd(&self, run_compiler: Compiler) -> Command {
target: self.build.build,
extra_features: Vec::new(),
});
// Invoke cargo-miri, make sure we can find miri and cargo.
// Invoke cargo-miri, make sure it can find miri and cargo.
let mut cmd = Command::new(cargo_miri);
cmd.env("MIRI", &miri);
cmd.env("CARGO", &self.initial_cargo);
@ -1299,7 +1304,11 @@ pub fn bare_cargo(
if cmd == "clippy" {
cargo = self.cargo_clippy_cmd(compiler);
cargo.arg(cmd);
} else if let Some(subcmd) = cmd.strip_prefix("miri-") {
} else if let Some(subcmd) = cmd.strip_prefix("miri") {
// Command must be "miri-X".
let subcmd = subcmd
.strip_prefix("-")
.unwrap_or_else(|| panic!("expected `miri-$subcommand`, but got {}", cmd));
cargo = self.cargo_miri_cmd(compiler);
cargo.arg("miri").arg(subcmd);
} else {
@ -1702,6 +1711,15 @@ fn cargo(
cargo.env("RUSTC_WRAPPER_REAL", existing_wrapper);
}
// If this is for `miri-test`, prepare the sysroots.
if cmd == "miri-test" {
self.ensure(compile::Std::new(compiler, compiler.host));
let host_sysroot = self.sysroot(compiler);
let miri_sysroot = test::Miri::build_miri_sysroot(self, compiler, target);
cargo.env("MIRI_SYSROOT", &miri_sysroot);
cargo.env("MIRI_HOST_SYSROOT", &host_sysroot);
}
cargo.env(profile_var("STRIP"), self.config.rust_strip.to_string());
if let Some(stack_protector) = &self.config.rust_stack_protector {

View file

@ -2022,7 +2022,7 @@ fn get_table(option: &str) -> Result<TomlConfig, toml::de::Error> {
Subcommand::Build { .. } => {
flags.stage.or(build_stage).unwrap_or(if download_rustc { 2 } else { 1 })
}
Subcommand::Test { .. } => {
Subcommand::Test { .. } | Subcommand::Miri { .. } => {
flags.stage.or(test_stage).unwrap_or(if download_rustc { 2 } else { 1 })
}
Subcommand::Bench { .. } => flags.stage.or(bench_stage).unwrap_or(2),
@ -2044,6 +2044,7 @@ fn get_table(option: &str) -> Result<TomlConfig, toml::de::Error> {
if flags.stage.is_none() && crate::CiEnv::current() != crate::CiEnv::None {
match config.cmd {
Subcommand::Test { .. }
| Subcommand::Miri { .. }
| Subcommand::Doc { .. }
| Subcommand::Build { .. }
| Subcommand::Bench { .. }
@ -2099,7 +2100,9 @@ pub(crate) fn git(&self) -> Command {
pub(crate) fn test_args(&self) -> Vec<&str> {
let mut test_args = match self.cmd {
Subcommand::Test { ref test_args, .. } | Subcommand::Bench { ref test_args, .. } => {
Subcommand::Test { ref test_args, .. }
| Subcommand::Bench { ref test_args, .. }
| Subcommand::Miri { ref test_args, .. } => {
test_args.iter().flat_map(|s| s.split_whitespace()).collect()
}
_ => vec![],

View file

@ -379,6 +379,25 @@ pub enum Subcommand {
/// `/<build_base>/rustfix_missing_coverage.txt`
rustfix_coverage: bool,
},
/// Build and run some test suites *in Miri*
Miri {
#[arg(long)]
/// run all tests regardless of failure
no_fail_fast: bool,
#[arg(long, value_name = "ARGS", allow_hyphen_values(true))]
/// extra arguments to be passed for the test tool being used
/// (e.g. libtest, compiletest or rustdoc)
test_args: Vec<String>,
/// extra options to pass the compiler when running tests
#[arg(long, value_name = "ARGS", allow_hyphen_values(true))]
rustc_args: Vec<String>,
#[arg(long)]
/// do not run doc tests
no_doc: bool,
#[arg(long)]
/// only run doc tests
doc: bool,
},
/// Build and run some benchmarks
Bench {
#[arg(long, allow_hyphen_values(true))]
@ -450,6 +469,7 @@ pub fn kind(&self) -> Kind {
Subcommand::Fix { .. } => Kind::Fix,
Subcommand::Format { .. } => Kind::Format,
Subcommand::Test { .. } => Kind::Test,
Subcommand::Miri { .. } => Kind::Miri,
Subcommand::Clean { .. } => Kind::Clean,
Subcommand::Dist { .. } => Kind::Dist,
Subcommand::Install { .. } => Kind::Install,
@ -461,7 +481,7 @@ pub fn kind(&self) -> Kind {
pub fn rustc_args(&self) -> Vec<&str> {
match *self {
Subcommand::Test { ref rustc_args, .. } => {
Subcommand::Test { ref rustc_args, .. } | Subcommand::Miri { ref rustc_args, .. } => {
rustc_args.iter().flat_map(|s| s.split_whitespace()).collect()
}
_ => vec![],
@ -470,14 +490,16 @@ pub fn rustc_args(&self) -> Vec<&str> {
pub fn fail_fast(&self) -> bool {
match *self {
Subcommand::Test { no_fail_fast, .. } => !no_fail_fast,
Subcommand::Test { no_fail_fast, .. } | Subcommand::Miri { no_fail_fast, .. } => {
!no_fail_fast
}
_ => false,
}
}
pub fn doc_tests(&self) -> DocTests {
match *self {
Subcommand::Test { doc, no_doc, .. } => {
Subcommand::Test { doc, no_doc, .. } | Subcommand::Miri { no_doc, doc, .. } => {
if doc {
DocTests::Only
} else if no_doc {

View file

@ -231,14 +231,16 @@ fn render_suite_outcome(&self, outcome: Outcome<'_>, suite: &SuiteOutcome) {
print!("\ntest result: ");
self.builder.colored_stdout(|stdout| outcome.write_long(stdout)).unwrap();
println!(
". {} passed; {} failed; {} ignored; {} measured; {} filtered out; \
finished in {:.2?}\n",
". {} passed; {} failed; {} ignored; {} measured; {} filtered out{time}\n",
suite.passed,
suite.failed,
suite.ignored,
suite.measured,
suite.filtered_out,
Duration::from_secs_f64(suite.exec_time)
time = match suite.exec_time {
Some(t) => format!("; finished in {:.2?}", Duration::from_secs_f64(t)),
None => format!(""),
}
);
}
@ -374,7 +376,9 @@ struct SuiteOutcome {
ignored: usize,
measured: usize,
filtered_out: usize,
exec_time: f64,
/// The time it took to execute this test suite, or `None` if time measurement was not possible
/// (e.g. due to running inside Miri).
exec_time: Option<f64>,
}
#[derive(serde_derive::Deserialize)]

View file

@ -62,3 +62,8 @@ case $HOST_TARGET in
exit 1
;;
esac
# Also smoke-test `x.py miri`. This doesn't run any actual tests (that would take too long),
# but it ensures that the crates build properly when tested with Miri.
python3 "$X_PY" miri --stage 2 library/core --test-args notest
python3 "$X_PY" miri --stage 2 library/alloc --test-args notest
python3 "$X_PY" miri --stage 2 library/std --test-args notest

View file

@ -39,6 +39,7 @@ complete -c x.py -n "__fish_use_subcommand" -f -a "fix" -d 'Run cargo fix'
complete -c x.py -n "__fish_use_subcommand" -f -a "fmt" -d 'Run rustfmt'
complete -c x.py -n "__fish_use_subcommand" -f -a "doc" -d 'Build documentation'
complete -c x.py -n "__fish_use_subcommand" -f -a "test" -d 'Build and run some test suites'
complete -c x.py -n "__fish_use_subcommand" -f -a "miri" -d 'Build and run some test suites *in Miri*'
complete -c x.py -n "__fish_use_subcommand" -f -a "bench" -d 'Build and run some benchmarks'
complete -c x.py -n "__fish_use_subcommand" -f -a "clean" -d 'Clean out build directories'
complete -c x.py -n "__fish_use_subcommand" -f -a "dist" -d 'Build distribution artifacts'
@ -308,6 +309,45 @@ complete -c x.py -n "__fish_seen_subcommand_from test" -l llvm-profile-generate
complete -c x.py -n "__fish_seen_subcommand_from test" -l enable-bolt-settings -d 'Enable BOLT link flags'
complete -c x.py -n "__fish_seen_subcommand_from test" -l skip-stage0-validation -d 'Skip stage0 compiler validation'
complete -c x.py -n "__fish_seen_subcommand_from test" -s h -l help -d 'Print help (see more with \'--help\')'
complete -c x.py -n "__fish_seen_subcommand_from miri" -l test-args -d 'extra arguments to be passed for the test tool being used (e.g. libtest, compiletest or rustdoc)' -r
complete -c x.py -n "__fish_seen_subcommand_from miri" -l rustc-args -d 'extra options to pass the compiler when running tests' -r
complete -c x.py -n "__fish_seen_subcommand_from miri" -l config -d 'TOML configuration file for build' -r -F
complete -c x.py -n "__fish_seen_subcommand_from miri" -l build-dir -d 'Build directory, overrides `build.build-dir` in `config.toml`' -r -f -a "(__fish_complete_directories)"
complete -c x.py -n "__fish_seen_subcommand_from miri" -l build -d 'build target of the stage0 compiler' -r -f
complete -c x.py -n "__fish_seen_subcommand_from miri" -l host -d 'host targets to build' -r -f
complete -c x.py -n "__fish_seen_subcommand_from miri" -l target -d 'target targets to build' -r -f
complete -c x.py -n "__fish_seen_subcommand_from miri" -l exclude -d 'build paths to exclude' -r -F
complete -c x.py -n "__fish_seen_subcommand_from miri" -l skip -d 'build paths to skip' -r -F
complete -c x.py -n "__fish_seen_subcommand_from miri" -l rustc-error-format -r -f
complete -c x.py -n "__fish_seen_subcommand_from miri" -l on-fail -d 'command to run on failure' -r -f -a "(__fish_complete_command)"
complete -c x.py -n "__fish_seen_subcommand_from miri" -l stage -d 'stage to build (indicates compiler to use/test, e.g., stage 0 uses the bootstrap compiler, stage 1 the stage 0 rustc artifacts, etc.)' -r -f
complete -c x.py -n "__fish_seen_subcommand_from miri" -l keep-stage -d 'stage(s) to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)' -r -f
complete -c x.py -n "__fish_seen_subcommand_from miri" -l keep-stage-std -d 'stage(s) of the standard library to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)' -r -f
complete -c x.py -n "__fish_seen_subcommand_from miri" -l src -d 'path to the root of the rust checkout' -r -f -a "(__fish_complete_directories)"
complete -c x.py -n "__fish_seen_subcommand_from miri" -s j -l jobs -d 'number of jobs to run in parallel' -r -f
complete -c x.py -n "__fish_seen_subcommand_from miri" -l warnings -d 'if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour' -r -f -a "{deny '',warn '',default ''}"
complete -c x.py -n "__fish_seen_subcommand_from miri" -l error-format -d 'rustc error format' -r -f
complete -c x.py -n "__fish_seen_subcommand_from miri" -l color -d 'whether to use color in cargo and rustc output' -r -f -a "{always '',never '',auto ''}"
complete -c x.py -n "__fish_seen_subcommand_from miri" -l llvm-skip-rebuild -d 'whether rebuilding llvm should be skipped, overriding `skip-rebuld` in config.toml' -r -f -a "{true '',false ''}"
complete -c x.py -n "__fish_seen_subcommand_from miri" -l rust-profile-generate -d 'generate PGO profile with rustc build' -r -F
complete -c x.py -n "__fish_seen_subcommand_from miri" -l rust-profile-use -d 'use PGO profile for rustc build' -r -F
complete -c x.py -n "__fish_seen_subcommand_from miri" -l llvm-profile-use -d 'use PGO profile for LLVM build' -r -F
complete -c x.py -n "__fish_seen_subcommand_from miri" -l reproducible-artifact -d 'Additional reproducible artifacts that should be added to the reproducible artifacts archive' -r
complete -c x.py -n "__fish_seen_subcommand_from miri" -l set -d 'override options in config.toml' -r -f
complete -c x.py -n "__fish_seen_subcommand_from miri" -l no-fail-fast -d 'run all tests regardless of failure'
complete -c x.py -n "__fish_seen_subcommand_from miri" -l no-doc -d 'do not run doc tests'
complete -c x.py -n "__fish_seen_subcommand_from miri" -l doc -d 'only run doc tests'
complete -c x.py -n "__fish_seen_subcommand_from miri" -s v -l verbose -d 'use verbose output (-vv for very verbose)'
complete -c x.py -n "__fish_seen_subcommand_from miri" -s i -l incremental -d 'use incremental compilation'
complete -c x.py -n "__fish_seen_subcommand_from miri" -l include-default-paths -d 'include default paths in addition to the provided ones'
complete -c x.py -n "__fish_seen_subcommand_from miri" -l dry-run -d 'dry run; don\'t build anything'
complete -c x.py -n "__fish_seen_subcommand_from miri" -l dump-bootstrap-shims -d 'Indicates whether to dump the work done from bootstrap shims'
complete -c x.py -n "__fish_seen_subcommand_from miri" -l json-output -d 'use message-format=json'
complete -c x.py -n "__fish_seen_subcommand_from miri" -l bypass-bootstrap-lock -d 'Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)'
complete -c x.py -n "__fish_seen_subcommand_from miri" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc'
complete -c x.py -n "__fish_seen_subcommand_from miri" -l enable-bolt-settings -d 'Enable BOLT link flags'
complete -c x.py -n "__fish_seen_subcommand_from miri" -l skip-stage0-validation -d 'Skip stage0 compiler validation'
complete -c x.py -n "__fish_seen_subcommand_from miri" -s h -l help -d 'Print help (see more with \'--help\')'
complete -c x.py -n "__fish_seen_subcommand_from bench" -l test-args -r
complete -c x.py -n "__fish_seen_subcommand_from bench" -l config -d 'TOML configuration file for build' -r -F
complete -c x.py -n "__fish_seen_subcommand_from bench" -l build-dir -d 'Build directory, overrides `build.build-dir` in `config.toml`' -r -f -a "(__fish_complete_directories)"

View file

@ -66,6 +66,7 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock {
[CompletionResult]::new('fmt', 'fmt', [CompletionResultType]::ParameterValue, 'Run rustfmt')
[CompletionResult]::new('doc', 'doc', [CompletionResultType]::ParameterValue, 'Build documentation')
[CompletionResult]::new('test', 'test', [CompletionResultType]::ParameterValue, 'Build and run some test suites')
[CompletionResult]::new('miri', 'miri', [CompletionResultType]::ParameterValue, 'Build and run some test suites *in Miri*')
[CompletionResult]::new('bench', 'bench', [CompletionResultType]::ParameterValue, 'Build and run some benchmarks')
[CompletionResult]::new('clean', 'clean', [CompletionResultType]::ParameterValue, 'Clean out build directories')
[CompletionResult]::new('dist', 'dist', [CompletionResultType]::ParameterValue, 'Build distribution artifacts')
@ -386,6 +387,52 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock {
[CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
break
}
'x.py;miri' {
[CompletionResult]::new('--test-args', 'test-args', [CompletionResultType]::ParameterName, 'extra arguments to be passed for the test tool being used (e.g. libtest, compiletest or rustdoc)')
[CompletionResult]::new('--rustc-args', 'rustc-args', [CompletionResultType]::ParameterName, 'extra options to pass the compiler when running tests')
[CompletionResult]::new('--config', 'config', [CompletionResultType]::ParameterName, 'TOML configuration file for build')
[CompletionResult]::new('--build-dir', 'build-dir', [CompletionResultType]::ParameterName, 'Build directory, overrides `build.build-dir` in `config.toml`')
[CompletionResult]::new('--build', 'build', [CompletionResultType]::ParameterName, 'build target of the stage0 compiler')
[CompletionResult]::new('--host', 'host', [CompletionResultType]::ParameterName, 'host targets to build')
[CompletionResult]::new('--target', 'target', [CompletionResultType]::ParameterName, 'target targets to build')
[CompletionResult]::new('--exclude', 'exclude', [CompletionResultType]::ParameterName, 'build paths to exclude')
[CompletionResult]::new('--skip', 'skip', [CompletionResultType]::ParameterName, 'build paths to skip')
[CompletionResult]::new('--rustc-error-format', 'rustc-error-format', [CompletionResultType]::ParameterName, 'rustc-error-format')
[CompletionResult]::new('--on-fail', 'on-fail', [CompletionResultType]::ParameterName, 'command to run on failure')
[CompletionResult]::new('--stage', 'stage', [CompletionResultType]::ParameterName, 'stage to build (indicates compiler to use/test, e.g., stage 0 uses the bootstrap compiler, stage 1 the stage 0 rustc artifacts, etc.)')
[CompletionResult]::new('--keep-stage', 'keep-stage', [CompletionResultType]::ParameterName, 'stage(s) to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)')
[CompletionResult]::new('--keep-stage-std', 'keep-stage-std', [CompletionResultType]::ParameterName, 'stage(s) of the standard library to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)')
[CompletionResult]::new('--src', 'src', [CompletionResultType]::ParameterName, 'path to the root of the rust checkout')
[CompletionResult]::new('-j', 'j', [CompletionResultType]::ParameterName, 'number of jobs to run in parallel')
[CompletionResult]::new('--jobs', 'jobs', [CompletionResultType]::ParameterName, 'number of jobs to run in parallel')
[CompletionResult]::new('--warnings', 'warnings', [CompletionResultType]::ParameterName, 'if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour')
[CompletionResult]::new('--error-format', 'error-format', [CompletionResultType]::ParameterName, 'rustc error format')
[CompletionResult]::new('--color', 'color', [CompletionResultType]::ParameterName, 'whether to use color in cargo and rustc output')
[CompletionResult]::new('--llvm-skip-rebuild', 'llvm-skip-rebuild', [CompletionResultType]::ParameterName, 'whether rebuilding llvm should be skipped, overriding `skip-rebuld` in config.toml')
[CompletionResult]::new('--rust-profile-generate', 'rust-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with rustc build')
[CompletionResult]::new('--rust-profile-use', 'rust-profile-use', [CompletionResultType]::ParameterName, 'use PGO profile for rustc build')
[CompletionResult]::new('--llvm-profile-use', 'llvm-profile-use', [CompletionResultType]::ParameterName, 'use PGO profile for LLVM build')
[CompletionResult]::new('--reproducible-artifact', 'reproducible-artifact', [CompletionResultType]::ParameterName, 'Additional reproducible artifacts that should be added to the reproducible artifacts archive')
[CompletionResult]::new('--set', 'set', [CompletionResultType]::ParameterName, 'override options in config.toml')
[CompletionResult]::new('--no-fail-fast', 'no-fail-fast', [CompletionResultType]::ParameterName, 'run all tests regardless of failure')
[CompletionResult]::new('--no-doc', 'no-doc', [CompletionResultType]::ParameterName, 'do not run doc tests')
[CompletionResult]::new('--doc', 'doc', [CompletionResultType]::ParameterName, 'only run doc tests')
[CompletionResult]::new('-v', 'v', [CompletionResultType]::ParameterName, 'use verbose output (-vv for very verbose)')
[CompletionResult]::new('--verbose', 'verbose', [CompletionResultType]::ParameterName, 'use verbose output (-vv for very verbose)')
[CompletionResult]::new('-i', 'i', [CompletionResultType]::ParameterName, 'use incremental compilation')
[CompletionResult]::new('--incremental', 'incremental', [CompletionResultType]::ParameterName, 'use incremental compilation')
[CompletionResult]::new('--include-default-paths', 'include-default-paths', [CompletionResultType]::ParameterName, 'include default paths in addition to the provided ones')
[CompletionResult]::new('--dry-run', 'dry-run', [CompletionResultType]::ParameterName, 'dry run; don''t build anything')
[CompletionResult]::new('--dump-bootstrap-shims', 'dump-bootstrap-shims', [CompletionResultType]::ParameterName, 'Indicates whether to dump the work done from bootstrap shims')
[CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json')
[CompletionResult]::new('--bypass-bootstrap-lock', 'bypass-bootstrap-lock', [CompletionResultType]::ParameterName, 'Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)')
[CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc')
[CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags')
[CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation')
[CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
[CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
break
}
'x.py;bench' {
[CompletionResult]::new('--test-args', 'test-args', [CompletionResultType]::ParameterName, 'test-args')
[CompletionResult]::new('--config', 'config', [CompletionResultType]::ParameterName, 'TOML configuration file for build')

View file

@ -42,6 +42,9 @@ _x.py() {
bootstrap,install)
cmd="bootstrap__install"
;;
bootstrap,miri)
cmd="bootstrap__miri"
;;
bootstrap,run)
cmd="bootstrap__run"
;;
@ -61,7 +64,7 @@ _x.py() {
case "${cmd}" in
x.py)
opts="-v -i -j -h --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --dump-bootstrap-shims --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --bypass-bootstrap-lock --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]... build check clippy fix fmt doc test bench clean dist install run setup suggest"
opts="-v -i -j -h --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --dump-bootstrap-shims --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --bypass-bootstrap-lock --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]... build check clippy fix fmt doc test miri bench clean dist install run setup suggest"
if [[ ${cur} == -* || ${COMP_CWORD} -eq 1 ]] ; then
COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
return 0
@ -1290,6 +1293,124 @@ _x.py() {
COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
return 0
;;
x.py__miri)
opts="-v -i -j -h --no-fail-fast --test-args --rustc-args --no-doc --doc --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --dump-bootstrap-shims --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --bypass-bootstrap-lock --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..."
if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
return 0
fi
case "${prev}" in
--test-args)
COMPREPLY=($(compgen -f "${cur}"))
return 0
;;
--rustc-args)
COMPREPLY=($(compgen -f "${cur}"))
return 0
;;
--config)
COMPREPLY=($(compgen -f "${cur}"))
return 0
;;
--build-dir)
COMPREPLY=($(compgen -f "${cur}"))
return 0
;;
--build)
COMPREPLY=("${cur}")
return 0
;;
--host)
COMPREPLY=("${cur}")
return 0
;;
--target)
COMPREPLY=("${cur}")
return 0
;;
--exclude)
COMPREPLY=($(compgen -f "${cur}"))
return 0
;;
--skip)
COMPREPLY=($(compgen -f "${cur}"))
return 0
;;
--rustc-error-format)
COMPREPLY=("${cur}")
return 0
;;
--on-fail)
COMPREPLY=($(compgen -f "${cur}"))
return 0
;;
--stage)
COMPREPLY=("${cur}")
return 0
;;
--keep-stage)
COMPREPLY=("${cur}")
return 0
;;
--keep-stage-std)
COMPREPLY=("${cur}")
return 0
;;
--src)
COMPREPLY=($(compgen -f "${cur}"))
return 0
;;
--jobs)
COMPREPLY=("${cur}")
return 0
;;
-j)
COMPREPLY=("${cur}")
return 0
;;
--warnings)
COMPREPLY=($(compgen -W "deny warn default" -- "${cur}"))
return 0
;;
--error-format)
COMPREPLY=("${cur}")
return 0
;;
--color)
COMPREPLY=($(compgen -W "always never auto" -- "${cur}"))
return 0
;;
--llvm-skip-rebuild)
COMPREPLY=($(compgen -W "true false" -- "${cur}"))
return 0
;;
--rust-profile-generate)
COMPREPLY=($(compgen -f "${cur}"))
return 0
;;
--rust-profile-use)
COMPREPLY=($(compgen -f "${cur}"))
return 0
;;
--llvm-profile-use)
COMPREPLY=($(compgen -f "${cur}"))
return 0
;;
--reproducible-artifact)
COMPREPLY=($(compgen -f "${cur}"))
return 0
;;
--set)
COMPREPLY=("${cur}")
return 0
;;
*)
COMPREPLY=()
;;
esac
COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
return 0
;;
x.py__run)
opts="-v -i -j -h --args --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --dump-bootstrap-shims --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --bypass-bootstrap-lock --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..."
if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then

View file

@ -389,6 +389,54 @@ _arguments "${_arguments_options[@]}" \
'*::paths -- paths for the subcommand:_files' \
&& ret=0
;;
(miri)
_arguments "${_arguments_options[@]}" \
'*--test-args=[extra arguments to be passed for the test tool being used (e.g. libtest, compiletest or rustdoc)]:ARGS: ' \
'*--rustc-args=[extra options to pass the compiler when running tests]:ARGS: ' \
'--config=[TOML configuration file for build]:FILE:_files' \
'--build-dir=[Build directory, overrides \`build.build-dir\` in \`config.toml\`]:DIR:_files -/' \
'--build=[build target of the stage0 compiler]:BUILD:( )' \
'--host=[host targets to build]:HOST:( )' \
'--target=[target targets to build]:TARGET:( )' \
'*--exclude=[build paths to exclude]:PATH:_files' \
'*--skip=[build paths to skip]:PATH:_files' \
'--rustc-error-format=[]:RUSTC_ERROR_FORMAT:( )' \
'--on-fail=[command to run on failure]:CMD:_cmdstring' \
'--stage=[stage to build (indicates compiler to use/test, e.g., stage 0 uses the bootstrap compiler, stage 1 the stage 0 rustc artifacts, etc.)]:N:( )' \
'*--keep-stage=[stage(s) to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)]:N:( )' \
'*--keep-stage-std=[stage(s) of the standard library to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)]:N:( )' \
'--src=[path to the root of the rust checkout]:DIR:_files -/' \
'-j+[number of jobs to run in parallel]:JOBS:( )' \
'--jobs=[number of jobs to run in parallel]:JOBS:( )' \
'--warnings=[if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour]:deny|warn:(deny warn default)' \
'--error-format=[rustc error format]:FORMAT:( )' \
'--color=[whether to use color in cargo and rustc output]:STYLE:(always never auto)' \
'--llvm-skip-rebuild=[whether rebuilding llvm should be skipped, overriding \`skip-rebuld\` in config.toml]:VALUE:(true false)' \
'--rust-profile-generate=[generate PGO profile with rustc build]:PROFILE:_files' \
'--rust-profile-use=[use PGO profile for rustc build]:PROFILE:_files' \
'--llvm-profile-use=[use PGO profile for LLVM build]:PROFILE:_files' \
'*--reproducible-artifact=[Additional reproducible artifacts that should be added to the reproducible artifacts archive]:REPRODUCIBLE_ARTIFACT: ' \
'*--set=[override options in config.toml]:section.option=value:( )' \
'--no-fail-fast[run all tests regardless of failure]' \
'--no-doc[do not run doc tests]' \
'--doc[only run doc tests]' \
'*-v[use verbose output (-vv for very verbose)]' \
'*--verbose[use verbose output (-vv for very verbose)]' \
'-i[use incremental compilation]' \
'--incremental[use incremental compilation]' \
'--include-default-paths[include default paths in addition to the provided ones]' \
'--dry-run[dry run; don'\''t build anything]' \
'--dump-bootstrap-shims[Indicates whether to dump the work done from bootstrap shims]' \
'--json-output[use message-format=json]' \
'--bypass-bootstrap-lock[Bootstrap uses this value to decide whether it should bypass locking the build process. This is rarely needed (e.g., compiling the std library for different targets in parallel)]' \
'--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
'--enable-bolt-settings[Enable BOLT link flags]' \
'--skip-stage0-validation[Skip stage0 compiler validation]' \
'-h[Print help (see more with '\''--help'\'')]' \
'--help[Print help (see more with '\''--help'\'')]' \
'*::paths -- paths for the subcommand:_files' \
&& ret=0
;;
(bench)
_arguments "${_arguments_options[@]}" \
'*--test-args=[]:TEST_ARGS: ' \
@ -710,6 +758,7 @@ _x.py_commands() {
'fmt:Run rustfmt' \
'doc:Build documentation' \
'test:Build and run some test suites' \
'miri:Build and run some test suites *in Miri*' \
'bench:Build and run some benchmarks' \
'clean:Clean out build directories' \
'dist:Build distribution artifacts' \
@ -770,6 +819,11 @@ _x.py__install_commands() {
local commands; commands=()
_describe -t commands 'x.py install commands' commands "$@"
}
(( $+functions[_x.py__miri_commands] )) ||
_x.py__miri_commands() {
local commands; commands=()
_describe -t commands 'x.py miri commands' commands "$@"
}
(( $+functions[_x.py__run_commands] )) ||
_x.py__run_commands() {
local commands; commands=()

View file

@ -507,6 +507,8 @@ binaries, and as such worth documenting:
crate currently being compiled.
* `MIRI_ORIG_RUSTDOC` is set and read by different phases of `cargo-miri` to remember the
value of `RUSTDOC` from before it was overwritten.
* `MIRI_REPLACE_LIBRS_IF_NOT_TEST` when set to any value enables a hack that helps bootstrap
run the standard library tests in Miri.
* `MIRI_VERBOSE` when set to any value tells the various `cargo-miri` phases to
perform verbose logging.
* `MIRI_HOST_SYSROOT` is set by bootstrap to tell `cargo-miri` which sysroot to use for *host*

View file

@ -3,7 +3,7 @@
use std::env;
use std::fs::{self, File};
use std::io::BufReader;
use std::path::PathBuf;
use std::path::{Path, PathBuf};
use std::process::Command;
use rustc_version::VersionMeta;
@ -412,9 +412,25 @@ fn out_filenames() -> Vec<PathBuf> {
// Arguments are treated very differently depending on whether this crate is
// for interpretation by Miri, or for use by a build script / proc macro.
if target_crate {
// Forward arguments, but remove "link" from "--emit" to make this a check-only build.
// Forward arguments, but patched.
let emit_flag = "--emit";
// This hack helps bootstrap run standard library tests in Miri. The issue is as follows:
// when running `cargo miri test` on libcore, cargo builds a local copy of core and makes it
// a dependency of the integration test crate. This copy duplicates all the lang items, so
// the build fails. (Regular testing avoids this because the sysroot is a literal copy of
// what `cargo build` produces, but since Miri builds its own sysroot this does not work for
// us.) So we need to make it so that the locally built libcore contains all the items from
// `core`, but does not re-define them -- we want to replace the entire crate but a
// re-export of the sysroot crate. We do this by swapping out the source file: if
// `MIRI_REPLACE_LIBRS_IF_NOT_TEST` is set and we are building a `lib.rs` file, and a
// `lib.miri.rs` file exists in the same folder, we build that instead. But crucially we
// only do that for the library, not the unit test crate (which would be runnable) or
// rustdoc (which would have a different `phase`).
let replace_librs = env::var_os("MIRI_REPLACE_LIBRS_IF_NOT_TEST").is_some()
&& !runnable_crate
&& phase == RustcPhase::Build;
while let Some(arg) = args.next() {
// Patch `--emit`: remove "link" from "--emit" to make this a check-only build.
if let Some(val) = arg.strip_prefix(emit_flag) {
// Patch this argument. First, extract its value.
let val =
@ -429,13 +445,36 @@ fn out_filenames() -> Vec<PathBuf> {
}
}
cmd.arg(format!("{emit_flag}={}", val.join(",")));
} else if arg == "--extern" {
// Patch `--extern` filenames, since Cargo sometimes passes stub `.rlib` files:
// https://github.com/rust-lang/miri/issues/1705
forward_patched_extern_arg(&mut args, &mut cmd);
} else {
cmd.arg(arg);
continue;
}
// Patch `--extern` filenames, since Cargo sometimes passes stub `.rlib` files:
// https://github.com/rust-lang/miri/issues/1705
if arg == "--extern" {
forward_patched_extern_arg(&mut args, &mut cmd);
continue;
}
// If the REPLACE_LIBRS hack is enabled and we are building a `lib.rs` file, and a
// `lib.miri.rs` file exists, then build that instead. We only consider relative paths
// as cargo uses those for files in the workspace; dependencies from crates.io get
// absolute paths.
if replace_librs {
let path = Path::new(&arg);
if path.is_relative()
&& path.file_name().is_some_and(|f| f == "lib.rs")
&& path.is_file()
{
let miri_rs = Path::new(&arg).with_extension("miri.rs");
if miri_rs.is_file() {
if verbose > 0 {
eprintln!("Performing REPLACE_LIBRS hack: {arg:?} -> {miri_rs:?}");
}
cmd.arg(miri_rs);
continue;
}
}
}
// Fallback: just propagate the argument.
cmd.arg(arg);
}
// During setup, patch the panic runtime for `libpanic_abort` (mirroring what bootstrap usually does).

View file

@ -105,24 +105,41 @@ fn find_children<'tcx: 'a, 'a>(
(path, None)
};
// First find the crate.
let krate =
tcx.crates(()).iter().find(|&&krate| tcx.crate_name(krate).as_str() == crate_name)?;
let mut cur_item = DefId { krate: *krate, index: CRATE_DEF_INDEX };
// Then go over the modules.
for &segment in modules {
cur_item = find_children(tcx, cur_item, segment)
.find(|item| tcx.def_kind(item) == DefKind::Mod)?;
}
// Finally, look up the desired item in this module, if any.
match item {
Some((item_name, namespace)) =>
Some(
find_children(tcx, cur_item, item_name)
.find(|item| tcx.def_kind(item).ns() == Some(namespace))?,
),
None => Some(cur_item),
// There may be more than one crate with this name. We try them all.
// (This is particularly relevant when running `std` tests as then there are two `std` crates:
// the one in the sysroot and the one locally built by `cargo test`.)
// FIXME: can we prefer the one from the sysroot?
'crates: for krate in
tcx.crates(()).iter().filter(|&&krate| tcx.crate_name(krate).as_str() == crate_name)
{
let mut cur_item = DefId { krate: *krate, index: CRATE_DEF_INDEX };
// Go over the modules.
for &segment in modules {
let Some(next_item) = find_children(tcx, cur_item, segment)
.find(|item| tcx.def_kind(item) == DefKind::Mod)
else {
continue 'crates;
};
cur_item = next_item;
}
// Finally, look up the desired item in this module, if any.
match item {
Some((item_name, namespace)) => {
let Some(item) = find_children(tcx, cur_item, item_name)
.find(|item| tcx.def_kind(item).ns() == Some(namespace))
else {
continue 'crates;
};
return Some(item);
}
None => {
// Just return the module.
return Some(cur_item);
}
}
}
// Item not found in any of the crates with the right name.
None
}
/// Convert a softfloat type to its corresponding hostfloat type.
@ -968,10 +985,6 @@ fn check_abi<'a>(&self, abi: Abi, exp_abi: Abi) -> InterpResult<'a, ()> {
fn frame_in_std(&self) -> bool {
let this = self.eval_context_ref();
let Some(start_fn) = this.tcx.lang_items().start_fn() else {
// no_std situations
return false;
};
let frame = this.frame();
// Make an attempt to get at the instance of the function this is inlined from.
let instance: Option<_> = try {
@ -982,13 +995,15 @@ fn frame_in_std(&self) -> bool {
};
// Fall back to the instance of the function itself.
let instance = instance.unwrap_or(frame.instance);
// Now check if this is in the same crate as start_fn.
// As a special exception we also allow unit tests from
// <https://github.com/rust-lang/miri-test-libstd/tree/master/std_miri_test> to call these
// shims.
// Now check the crate it is in. We could try to be clever here and e.g. check if this is
// the same crate as `start_fn`, but that would not work for running std tests in Miri, so
// we'd need some more hacks anyway. So we just check the name of the crate. If someone
// calls their crate `std` then we'll just let them keep the pieces.
let frame_crate = this.tcx.def_path(instance.def_id()).krate;
frame_crate == this.tcx.def_path(start_fn).krate
|| this.tcx.crate_name(frame_crate).as_str() == "std_miri_test"
let crate_name = this.tcx.crate_name(frame_crate);
let crate_name = crate_name.as_str();
// On miri-test-libstd, the name of the crate is different.
crate_name == "std" || crate_name == "std_miri_test"
}
/// Handler that should be called when unsupported functionality is encountered.