development/cargo
development/cargo
1
0
Fork 0
mirror of https://github.com/rust-lang/cargo synced 2024-10-02 22:13:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Change scraping strategy to embed in existing unit graph, add CompileMode::Docscrape

Browse source
This commit is contained in:
Will Crichton 2021-10-12 13:17:21 -07:00
parent 82d937e3ec
commit 70f38213da
10 changed files with 157 additions and 122 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
src/cargo/core/compiler/build_config.rs
View file

@ -149,6 +149,7 @@ pub enum CompileMode {
Doc { deps: bool },
/// A target that will be tested with `rustdoc`.
Doctest,
Docscrape,
/// A marker for Units that represent the execution of a `build.rs` script.
RunCustomBuild,
}
@ -166,6 +167,7 @@ impl ser::Serialize for CompileMode {
Bench => "bench".serialize(s),
Doc { .. } => "doc".serialize(s),
Doctest => "doctest".serialize(s),
Docscrape => "docscrape".serialize(s),
RunCustomBuild => "run-custom-build".serialize(s),
}
}
@ -187,6 +189,11 @@ impl CompileMode {
self == CompileMode::Doctest
}
/// Returns `true` if this is scraping examples for documentation.
pub fn is_doc_scrape(self) -> bool {
self == CompileMode::Docscrape
}
/// Returns `true` if this is any type of test (test, benchmark, doc test, or
/// check test).
pub fn is_any_test(self) -> bool {

4
src/cargo/core/compiler/build_context/mod.rs
View file

@ -48,6 +48,8 @@ pub struct BuildContext<'a, 'cfg> {
/// The dependency graph of units to compile.
pub unit_graph: UnitGraph,
pub scrape_units: Vec<Unit>,
/// The list of all kinds that are involved in this build
pub all_kinds: HashSet<CompileKind>,
}
@ -62,6 +64,7 @@ impl<'a, 'cfg> BuildContext<'a, 'cfg> {
target_data: RustcTargetData<'cfg>,
roots: Vec<Unit>,
unit_graph: UnitGraph,
scrape_units: Vec<Unit>,
) -> CargoResult<BuildContext<'a, 'cfg>> {
let all_kinds = unit_graph
.keys()
@ -80,6 +83,7 @@ impl<'a, 'cfg> BuildContext<'a, 'cfg> {
target_data,
roots,
unit_graph,
scrape_units,
all_kinds,
})
}

5
src/cargo/core/compiler/build_context/target_info.rs
View file

@ -461,7 +461,10 @@ impl TargetInfo {
}
}
CompileMode::Check { .. } => Ok((vec![FileType::new_rmeta()], Vec::new())),
CompileMode::Doc { .. } | CompileMode::Doctest | CompileMode::RunCustomBuild => {
CompileMode::Doc { .. }
| CompileMode::Doctest
| CompileMode::Docscrape
| CompileMode::RunCustomBuild => {
panic!("asked for rustc output for non-rustc mode")
}
}

4
src/cargo/core/compiler/context/compilation_files.rs
View file

@ -417,6 +417,10 @@ impl<'a, 'cfg: 'a> CompilationFiles<'a, 'cfg> {
// but Cargo does not know about that.
vec![]
}
CompileMode::Docscrape => {
// FIXME(wcrichto): should this include the *.examples files?
vec![]
}
CompileMode::Test
| CompileMode::Build
| CompileMode::Bench

1
src/cargo/core/compiler/job_queue.rs
View file

@ -1001,6 +1001,7 @@ impl<'cfg> DrainState<'cfg> {
let target_name = unit.target.name();
match unit.mode {
CompileMode::Doc { .. } => format!("{}(doc)", pkg_name),
CompileMode::Docscrape { .. } => format!("{}(docscrape)", pkg_name),
CompileMode::RunCustomBuild => format!("{}(build)", pkg_name),
CompileMode::Test | CompileMode::Check { test: true } => match unit.target.kind() {
TargetKind::Lib(_) => format!("{}(test)", target_name),

44
src/cargo/core/compiler/mod.rs
View file

@ -165,7 +165,7 @@ fn compile<'cfg>(
let force = exec.force_rebuild(unit) || force_rebuild;
let mut job = fingerprint::prepare_target(cx, unit, force)?;
job.before(if job.freshness() == Freshness::Dirty {
let work = if unit.mode.is_doc() {
let work = if unit.mode.is_doc() || unit.mode.is_doc_scrape() {
rustdoc(cx, unit)?
} else {
rustc(cx, unit, exec)?
@ -647,6 +647,48 @@ fn rustdoc(cx: &mut Context<'_, '_>, unit: &Unit) -> CargoResult<Work> {
rustdoc.args(args);
}
let scrape_output_path = |unit: &Unit| -> CargoResult<PathBuf> {
let layout = cx.files().layout(unit.kind);
let output_dir = layout.prepare_tmp()?;
Ok(output_dir.join(format!("{}.examples", unit.buildkey())))
};
if unit.mode.is_doc_scrape() {
rustdoc.arg("-Zunstable-options");
rustdoc
.arg("--scrape-examples-output-path")
.arg(scrape_output_path(unit)?);
for root in &cx.bcx.roots {
rustdoc
.arg("--scrape-examples-target-crate")
.arg(root.pkg.name());
}
} else if cx.bcx.scrape_units.len() > 0 {
rustdoc.arg("-Zunstable-options");
for scrape_unit in &cx.bcx.scrape_units {
rustdoc
.arg("--with-examples")
.arg(scrape_output_path(scrape_unit)?);
}
let mut all_units = cx
.bcx
.unit_graph
.values()
.map(|deps| deps.iter().map(|dep| &dep.unit))
.flatten();
let check_unit = all_units
.find(|other| {
unit.pkg == other.pkg && unit.target == other.target && other.mode.is_check()
})
.with_context(|| format!("Could not find check unit for {:?}", unit))?;
let metadata = cx.files().metadata(check_unit);
rustdoc.arg("-C").arg(format!("metadata={}", metadata));
}
build_deps_args(&mut rustdoc, cx, unit)?;
rustdoc::add_root_urls(cx, unit, &mut rustdoc)?;

1
src/cargo/core/compiler/timings.rs
View file

@ -176,6 +176,7 @@ impl<'cfg> Timings<'cfg> {
CompileMode::Bench => target.push_str(" (bench)"),
CompileMode::Doc { .. } => target.push_str(" (doc)"),
CompileMode::Doctest => target.push_str(" (doc test)"),
CompileMode::Docscrape => target.push_str(" (doc scrape)"),
CompileMode::RunCustomBuild => target.push_str(" (run)"),
}
let unit_time = UnitTime {

17
src/cargo/core/compiler/unit_dependencies.rs
View file

@ -47,6 +47,7 @@ struct State<'a, 'cfg> {
target_data: &'a RustcTargetData<'cfg>,
profiles: &'a Profiles,
interner: &'a UnitInterner,
scrape_roots: &'a [Unit],
/// A set of edges in `unit_dependencies` where (a, b) means that the
/// dependency from a to b was added purely because it was a dev-dependency.
@ -61,6 +62,7 @@ pub fn build_unit_dependencies<'a, 'cfg>(
features: &'a ResolvedFeatures,
std_resolve: Option<&'a (Resolve, ResolvedFeatures)>,
roots: &[Unit],
scrape_roots: &[Unit],
std_roots: &HashMap<CompileKind, Vec<Unit>>,
global_mode: CompileMode,
target_data: &'a RustcTargetData<'cfg>,
@ -91,12 +93,14 @@ pub fn build_unit_dependencies<'a, 'cfg>(
target_data,
profiles,
interner,
scrape_roots,
dev_dependency_edges: HashSet::new(),
};
let std_unit_deps = calc_deps_of_std(&mut state, std_roots)?;
deps_of_roots(roots, &mut state)?;
deps_of_roots(scrape_roots, &mut state)?;
super::links::validate_links(state.resolve(), &state.unit_dependencies)?;
// Hopefully there aren't any links conflicts with the standard library?
@ -477,6 +481,17 @@ fn compute_deps_doc(
if unit.target.is_bin() || unit.target.is_example() {
ret.extend(maybe_lib(unit, state, unit_for)?);
}
for scrape_unit in state.scrape_roots.iter() {
ret.push(UnitDep {
unit: scrape_unit.clone(),
unit_for: unit_for.with_dependency(scrape_unit, &scrape_unit.target),
extern_crate_name: InternedString::new(""),
public: false,
noprelude: false,
});
}
Ok(ret)
}
@ -568,7 +583,7 @@ fn dep_build_script(
/// Choose the correct mode for dependencies.
fn check_or_build_mode(mode: CompileMode, target: &Target) -> CompileMode {
match mode {
CompileMode::Check { .. } | CompileMode::Doc { .. } => {
CompileMode::Check { .. } | CompileMode::Doc { .. } | CompileMode::Docscrape => {
if target.for_host() {
// Plugin and proc macro targets should be compiled like
// normal.

2
src/cargo/core/profiles.rs
View file

@ -323,7 +323,7 @@ impl Profiles {
(InternedString::new("dev"), None)
}
}
CompileMode::Doc { .. } => (InternedString::new("doc"), None),
CompileMode::Doc { .. } | CompileMode::Docscrape => (InternedString::new("doc"), None),
}
} else {
(self.requested_profile, None)

194
src/cargo/ops/cargo_compile.rs
View file

@ -360,7 +360,7 @@ pub fn create_bcx<'a, 'cfg>(
)?;
}
}
CompileMode::Doc { .. } | CompileMode::Doctest => {
CompileMode::Doc { .. } | CompileMode::Doctest | CompileMode::Docscrape => {
if std::env::var("RUSTDOC_FLAGS").is_ok() {
config.shell().warn(
"Cargo does not read `RUSTDOC_FLAGS` environment variable. Did you mean `RUSTDOCFLAGS`?"
@ -496,6 +496,31 @@ pub fn create_bcx<'a, 'cfg>(
interner,
)?;
let mut scrape_units = match rustdoc_scrape_examples {
Some(scrape_filter) => {
let specs = Packages::All.to_package_id_specs(ws)?;
let to_build_ids = resolve.specs_to_ids(&specs)?;
let to_builds = pkg_set.get_many(to_build_ids)?;
let mode = CompileMode::Docscrape;
generate_targets(
ws,
&to_builds,
scrape_filter,
&build_config.requested_kinds,
explicit_host_kind,
mode,
&resolve,
&workspace_resolve,
&resolved_features,
&pkg_set,
&profiles,
interner,
)?
}
None => Vec::new(),
};
let std_roots = if let Some(crates) = &config.cli_unstable().build_std {
// Only build libtest if it looks like it is needed.
let mut crates = crates.clone();
@ -523,6 +548,24 @@ pub fn create_bcx<'a, 'cfg>(
Default::default()
};
let fmt_unit = |u: &Unit| format!("{} {:?} / {:?}", u.target.name(), u.target.kind(), u.mode);
let fmt_units = |v: &[Unit]| v.iter().map(|u| fmt_unit(u)).collect::<Vec<_>>().join(", ");
let fmt_graph = |g: &UnitGraph| {
g.iter()
.map(|(k, vs)| {
format!(
"{} =>\n{}",
fmt_unit(k),
vs.iter()
.map(|u| format!(" {}", fmt_unit(&u.unit)))
.collect::<Vec<_>>()
.join("\n")
)
})
.collect::<Vec<_>>()
.join("\n")
};
let mut unit_graph = build_unit_dependencies(
ws,
&pkg_set,
@ -530,12 +573,16 @@ pub fn create_bcx<'a, 'cfg>(
&resolved_features,
std_resolve_features.as_ref(),
&units,
&scrape_units,
&std_roots,
build_config.mode,
&target_data,
&profiles,
interner,
)?;
println!("SCRAPE UNITS: {}", fmt_units(&scrape_units));
println!("BEFORE ROOTS: {}", fmt_units(&units));
println!("BEFORE GRAPH: {}", fmt_graph(&unit_graph));
// TODO: In theory, Cargo should also dedupe the roots, but I'm uncertain
// what heuristics to use in that case.
@ -551,11 +598,20 @@ pub fn create_bcx<'a, 'cfg>(
// Rebuild the unit graph, replacing the explicit host targets with
// CompileKind::Host, merging any dependencies shared with build
// dependencies.
let new_graph = rebuild_unit_graph_shared(interner, unit_graph, &units, explicit_host_kind);
let new_graph = rebuild_unit_graph_shared(
interner,
unit_graph,
&units,
&scrape_units,
explicit_host_kind,
);
// This would be nicer with destructuring assignment.
units = new_graph.0;
unit_graph = new_graph.1;
scrape_units = new_graph.1;
unit_graph = new_graph.2;
}
println!("AFTER UNITS: {}", fmt_units(&units));
println!("AFTER GRAPH: {}", fmt_graph(&unit_graph));
let mut extra_compiler_args = HashMap::new();
if let Some(args) = extra_args {
@ -595,117 +651,6 @@ pub fn create_bcx<'a, 'cfg>(
}
}
if let Some(filter) = rustdoc_scrape_examples {
// Run cargo rustdoc --scrape-examples to generate calls for each file in `filter`
let paths = {
// Run in doc mode with the given `filter`
let compile_mode = CompileMode::Doc { deps: false };
let mut example_compile_opts = CompileOptions::new(ws.config(), compile_mode)?;
example_compile_opts.cli_features = options.cli_features.clone();
example_compile_opts.build_config.mode = compile_mode;
example_compile_opts.spec = Packages::All;
example_compile_opts.filter = filter.clone();
example_compile_opts.rustdoc_scrape_examples = None;
// Setup recursive Cargo context
let exec: Arc<dyn Executor> = Arc::new(DefaultExecutor);
let interner = UnitInterner::new();
let mut bcx = create_bcx(ws, &example_compile_opts, &interner)?;
// Make an output path for calls for each build unit
let paths = {
// FIXME(wcrichto): is there a better place to store these files?
let dest = bcx.profiles.get_dir_name();
let layout = Layout::new(ws, None, &dest)?;
let output_dir = layout.prepare_tmp()?;
bcx.roots
.iter()
.map(|unit| output_dir.join(format!("{}.calls", unit.buildkey())))
.collect::<Vec<_>>()
};
// Add --scrape-examples to each build unit's rustdoc args
for (path, unit) in paths.iter().zip(bcx.roots.iter()) {
let args = bcx
.extra_compiler_args
.entry(unit.clone())
.or_insert_with(Vec::new);
args.extend_from_slice(&[
"-Zunstable-options".into(),
"--scrape-examples-output-path".into(),
path.clone().into_os_string(),
]);
let crate_names = units
.iter()
.map(|unit| {
vec![
"--scrape-examples-target-crate".into(),
OsString::from(unit.pkg.name().as_str()),
]
.into_iter()
})
.flatten();
args.extend(crate_names);
}
// Find the check unit corresponding to each documented crate, then add the -C metadata=...
// flag to the doc unit for that crate
{
let mut cx = Context::new(&bcx)?;
cx.lto = lto::generate(&bcx)?;
cx.prepare_units()?;
for unit in units.iter() {
let mut root_deps = bcx
.unit_graph
.iter()
.map(|(k, v)| iter::once(k).chain(v.iter().map(|dep| &dep.unit)))
.flatten();
let check_unit = root_deps.find(|dep| {
dep.pkg == unit.pkg && dep.target == unit.target && dep.mode.is_check()
});
if let Some(check_unit) = check_unit {
let metadata = cx.files().metadata(check_unit);
extra_compiler_args
.entry(unit.clone())
.or_default()
.extend_from_slice(&[
"-C".into(),
OsString::from(format!("metadata={}", metadata)),
]);
}
}
}
// Invoke recursive Cargo
let cx = Context::new(&bcx)?;
cx.compile(&exec)?;
paths
};
// Add "--with-examples *.calls" to the current rustdoc invocation
let args = paths
.into_iter()
.map(|path| vec!["--with-examples".into(), path.into_os_string()].into_iter())
.flatten()
.chain(vec!["-Zunstable-options".into()].into_iter())
.collect::<Vec<_>>();
for unit in unit_graph.keys() {
if unit.mode.is_doc() && ws.is_member(&unit.pkg) {
extra_compiler_args
.entry(unit.clone())
.or_default()
.extend(args.clone());
}
}
}
if honor_rust_version {
// Remove any pre-release identifiers for easier comparison
let current_version = &target_data.rustc.version;
@ -745,6 +690,7 @@ pub fn create_bcx<'a, 'cfg>(
target_data,
units,
unit_graph,
scrape_units,
)?;
Ok(bcx)
@ -864,7 +810,10 @@ impl CompileFilter {
pub fn need_dev_deps(&self, mode: CompileMode) -> bool {
match mode {
CompileMode::Test | CompileMode::Doctest | CompileMode::Bench => true,
CompileMode::Test
| CompileMode::Doctest
| CompileMode::Bench
| CompileMode::Docscrape => true,
CompileMode::Check { test: true } => true,
CompileMode::Build | CompileMode::Doc { .. } | CompileMode::Check { test: false } => {
match *self {
@ -1466,7 +1415,9 @@ fn filter_default_targets(targets: &[Target], mode: CompileMode) -> Vec<&Target>
})
.collect()
}
CompileMode::Doctest | CompileMode::RunCustomBuild => panic!("Invalid mode {:?}", mode),
CompileMode::Doctest | CompileMode::Docscrape | CompileMode::RunCustomBuild => {
panic!("Invalid mode {:?}", mode)
}
}
}
@ -1578,8 +1529,9 @@ fn rebuild_unit_graph_shared(
interner: &UnitInterner,
unit_graph: UnitGraph,
roots: &[Unit],
scrape_units: &[Unit],
to_host: CompileKind,
) -> (Vec<Unit>, UnitGraph) {
) -> (Vec<Unit>, Vec<Unit>, UnitGraph) {
let mut result = UnitGraph::new();
// Map of the old unit to the new unit, used to avoid recursing into units
// that have already been computed to improve performance.
@ -1590,7 +1542,13 @@ fn rebuild_unit_graph_shared(
traverse_and_share(interner, &mut memo, &mut result, &unit_graph, root, to_host)
})
.collect();
(new_roots, result)
let new_scrape_units = scrape_units
.iter()
.map(|unit| {
traverse_and_share(interner, &mut memo, &mut result, &unit_graph, unit, to_host)
})
.collect();
(new_roots, new_scrape_units, result)
}
/// Recursive function for rebuilding the graph.