Auto merge of #9525 - willcrichton:example-analyzer, r=alexcrichton

Scrape code examples from examples/ directory for Rustdoc

Adds support for the functionality described in rust-lang/rfcs#3123

Matching changes to rustdoc are here: https://github.com/rust-lang/rust/pull/85833

src/bin/cargo/commands/doc.rs

@@ -1,6 +1,7 @@
 use crate::command_prelude::*;
 
-use cargo::ops::{self, DocOptions};
+use anyhow::anyhow;
+use cargo::ops::{self, CompileFilter, DocOptions, FilterRule, LibRule};
 
 pub fn cli() -> App {
     subcommand("doc")
@@ -19,6 +20,13 @@ pub fn cli() -> App {
         )
         .arg(opt("no-deps", "Don't build documentation for dependencies"))
         .arg(opt("document-private-items", "Document private items"))
+        .arg(
+            opt(
+                "scrape-examples",
+                "Scrape examples to include as function documentation",
+            )
+            .value_name("FLAGS"),
+        )
         .arg_jobs()
         .arg_targets_lib_bin_example(
             "Document only this package's library",
@@ -48,6 +56,33 @@ pub fn exec(config: &mut Config, args: &ArgMatches<'_>) -> CliResult {
         args.compile_options(config, mode, Some(&ws), ProfileChecking::Custom)?;
     compile_opts.rustdoc_document_private_items = args.is_present("document-private-items");
 
+    // TODO(wcrichto): move scrape example configuration into Cargo.toml before stabilization
+    // See: https://github.com/rust-lang/cargo/pull/9525#discussion_r728470927
+    compile_opts.rustdoc_scrape_examples = match args.value_of("scrape-examples") {
+        Some(s) => Some(match s {
+            "all" => CompileFilter::new_all_targets(),
+            "examples" => CompileFilter::new(
+                LibRule::False,
+                FilterRule::none(),
+                FilterRule::none(),
+                FilterRule::All,
+                FilterRule::none(),
+            ),
+            _ => {
+                return Err(CliError::from(anyhow!(
+                    r#"--scrape-examples must take "all" or "examples" as an argument"#
+                )));
+            }
+        }),
+        None => None,
+    };
+
+    if compile_opts.rustdoc_scrape_examples.is_some() {
+        config
+            .cli_unstable()
+            .fail_if_stable_opt("--scrape-examples", 9910)?;
+    }
+
     let doc_opts = DocOptions {
         open_result: args.is_present("open"),
         compile_opts,

src/cargo/core/compiler/build_config.rs

@@ -149,6 +149,8 @@ pub enum CompileMode {
     Doc { deps: bool },
     /// A target that will be tested with `rustdoc`.
     Doctest,
+    /// An example or library that will be scraped for function calls by `rustdoc`.
+    Docscrape,
     /// A marker for Units that represent the execution of a `build.rs` script.
     RunCustomBuild,
 }
@@ -166,6 +168,7 @@ impl ser::Serialize for CompileMode {
             Bench => "bench".serialize(s),
             Doc { .. } => "doc".serialize(s),
             Doctest => "doctest".serialize(s),
+            Docscrape => "docscrape".serialize(s),
             RunCustomBuild => "run-custom-build".serialize(s),
         }
     }
@@ -187,6 +190,11 @@ impl CompileMode {
         self == CompileMode::Doctest
     }
 
+    /// Returns `true` if this is scraping examples for documentation.
+    pub fn is_doc_scrape(self) -> bool {
+        self == CompileMode::Docscrape
+    }
+
     /// Returns `true` if this is any type of test (test, benchmark, doc test, or
     /// check test).
     pub fn is_any_test(self) -> bool {

src/cargo/core/compiler/build_context/mod.rs

@@ -47,6 +47,9 @@ pub struct BuildContext<'a, 'cfg> {
     /// The dependency graph of units to compile.
     pub unit_graph: UnitGraph,
 
+    /// Reverse-dependencies of documented units, used by the rustdoc --scrape-examples flag.
+    pub scrape_units: Vec<Unit>,
+
     /// The list of all kinds that are involved in this build
     pub all_kinds: HashSet<CompileKind>,
 }
@@ -61,6 +64,7 @@ impl<'a, 'cfg> BuildContext<'a, 'cfg> {
         target_data: RustcTargetData<'cfg>,
         roots: Vec<Unit>,
         unit_graph: UnitGraph,
+        scrape_units: Vec<Unit>,
     ) -> CargoResult<BuildContext<'a, 'cfg>> {
         let all_kinds = unit_graph
             .keys()
@@ -79,6 +83,7 @@ impl<'a, 'cfg> BuildContext<'a, 'cfg> {
             target_data,
             roots,
             unit_graph,
+            scrape_units,
             all_kinds,
         })
     }

src/cargo/core/compiler/build_context/target_info.rs

@@ -452,7 +452,10 @@ impl TargetInfo {
                 }
             }
             CompileMode::Check { .. } => Ok((vec![FileType::new_rmeta()], Vec::new())),
-            CompileMode::Doc { .. } | CompileMode::Doctest | CompileMode::RunCustomBuild => {
+            CompileMode::Doc { .. }
+            | CompileMode::Doctest
+            | CompileMode::Docscrape
+            | CompileMode::RunCustomBuild => {
                 panic!("asked for rustc output for non-rustc mode")
             }
         }

src/cargo/core/compiler/context/compilation_files.rs

@@ -191,7 +191,9 @@ impl<'a, 'cfg: 'a> CompilationFiles<'a, 'cfg> {
     /// Returns the directory where the artifacts for the given unit are
     /// initially created.
     pub fn out_dir(&self, unit: &Unit) -> PathBuf {
-        if unit.mode.is_doc() {
+        // Docscrape units need to have doc/ set as the out_dir so sources for reverse-dependencies
+        // will be put into doc/ and not into deps/ where the *.examples files are stored.
+        if unit.mode.is_doc() || unit.mode.is_doc_scrape() {
             self.layout(unit.kind).doc().to_path_buf()
         } else if unit.mode.is_doc_test() {
             panic!("doc tests do not have an out dir");
@@ -417,6 +419,17 @@ impl<'a, 'cfg: 'a> CompilationFiles<'a, 'cfg> {
                 // but Cargo does not know about that.
                 vec![]
             }
+            CompileMode::Docscrape => {
+                let path = self
+                    .deps_dir(unit)
+                    .join(format!("{}.examples", unit.buildkey()));
+                vec![OutputFile {
+                    path,
+                    hardlink: None,
+                    export_path: None,
+                    flavor: FileFlavor::Normal,
+                }]
+            }
             CompileMode::Test
             | CompileMode::Build
             | CompileMode::Bench

src/cargo/core/compiler/context/mod.rs

@@ -80,6 +80,10 @@ pub struct Context<'a, 'cfg> {
     /// compilation is happening (only object, only bitcode, both, etc), and is
     /// precalculated early on.
     pub lto: HashMap<Unit, Lto>,
+
+    /// Map of Doc/Docscrape units to metadata for their -Cmetadata flag.
+    /// See Context::find_metadata_units for more details.
+    pub metadata_for_doc_units: HashMap<Unit, Metadata>,
 }
 
 impl<'a, 'cfg> Context<'a, 'cfg> {
@@ -120,6 +124,7 @@ impl<'a, 'cfg> Context<'a, 'cfg> {
             rustc_clients: HashMap::new(),
             pipelining,
             lto: HashMap::new(),
+            metadata_for_doc_units: HashMap::new(),
         })
     }
 
@@ -134,6 +139,7 @@ impl<'a, 'cfg> Context<'a, 'cfg> {
         self.prepare()?;
         custom_build::build_map(&mut self)?;
         self.check_collisions()?;
+        self.compute_metadata_for_doc_units();
 
         // We need to make sure that if there were any previous docs
         // already compiled, they were compiled with the same Rustc version that we're currently
@@ -620,4 +626,40 @@ impl<'a, 'cfg> Context<'a, 'cfg> {
 
         Ok(client)
     }
+
+    /// Finds metadata for Doc/Docscrape units.
+    ///
+    /// rustdoc needs a -Cmetadata flag in order to recognize StableCrateIds that refer to
+    /// items in the crate being documented. The -Cmetadata flag used by reverse-dependencies
+    /// will be the metadata of the Cargo unit that generated the current library's rmeta file,
+    /// which should be a Check unit.
+    ///
+    /// If the current crate has reverse-dependencies, such a Check unit should exist, and so
+    /// we use that crate's metadata. If not, we use the crate's Doc unit so at least examples
+    /// scraped from the current crate can be used when documenting the current crate.
+    pub fn compute_metadata_for_doc_units(&mut self) {
+        for unit in self.bcx.unit_graph.keys() {
+            if !unit.mode.is_doc() && !unit.mode.is_doc_scrape() {
+                continue;
+            }
+
+            let matching_units = self
+                .bcx
+                .unit_graph
+                .keys()
+                .filter(|other| {
+                    unit.pkg == other.pkg
+                        && unit.target == other.target
+                        && !other.mode.is_doc_scrape()
+                })
+                .collect::<Vec<_>>();
+            let metadata_unit = matching_units
+                .iter()
+                .find(|other| other.mode.is_check())
+                .or_else(|| matching_units.iter().find(|other| other.mode.is_doc()))
+                .unwrap_or(&unit);
+            self.metadata_for_doc_units
+                .insert(unit.clone(), self.files().metadata(metadata_unit));
+        }
+    }
 }

src/cargo/core/compiler/mod.rs

@@ -165,7 +165,7 @@ fn compile<'cfg>(
         let force = exec.force_rebuild(unit) || force_rebuild;
         let mut job = fingerprint::prepare_target(cx, unit, force)?;
         job.before(if job.freshness() == Freshness::Dirty {
-            let work = if unit.mode.is_doc() {
+            let work = if unit.mode.is_doc() || unit.mode.is_doc_scrape() {
                 rustdoc(cx, unit)?
             } else {
                 rustc(cx, unit, exec)?
@@ -647,6 +647,42 @@ fn rustdoc(cx: &mut Context<'_, '_>, unit: &Unit) -> CargoResult<Work> {
         rustdoc.args(args);
     }
 
+    let metadata = cx.metadata_for_doc_units[&unit];
+    rustdoc.arg("-C").arg(format!("metadata={}", metadata));
+
+    let scrape_output_path = |unit: &Unit| -> CargoResult<PathBuf> {
+        let output_dir = cx.files().deps_dir(unit);
+        Ok(output_dir.join(format!("{}.examples", unit.buildkey())))
+    };
+
+    if unit.mode.is_doc_scrape() {
+        debug_assert!(cx.bcx.scrape_units.contains(unit));
+
+        rustdoc.arg("-Zunstable-options");
+
+        rustdoc
+            .arg("--scrape-examples-output-path")
+            .arg(scrape_output_path(unit)?);
+
+        // Only scrape example for items from crates in the workspace, to reduce generated file size
+        for pkg in cx.bcx.ws.members() {
+            rustdoc
+                .arg("--scrape-examples-target-crate")
+                .arg(pkg.name());
+        }
+    } else if cx.bcx.scrape_units.len() > 0 && cx.bcx.ws.is_member(&unit.pkg) {
+        // We only pass scraped examples to packages in the workspace
+        // since examples are only coming from reverse-dependencies of workspace packages
+
+        rustdoc.arg("-Zunstable-options");
+
+        for scrape_unit in &cx.bcx.scrape_units {
+            rustdoc
+                .arg("--with-examples")
+                .arg(scrape_output_path(scrape_unit)?);
+        }
+    }
+
     build_deps_args(&mut rustdoc, cx, unit)?;
     rustdoc::add_root_urls(cx, unit, &mut rustdoc)?;
 

src/cargo/core/compiler/timings.rs

@@ -176,6 +176,7 @@ impl<'cfg> Timings<'cfg> {
             CompileMode::Bench => target.push_str(" (bench)"),
             CompileMode::Doc { .. } => target.push_str(" (doc)"),
             CompileMode::Doctest => target.push_str(" (doc test)"),
+            CompileMode::Docscrape => target.push_str(" (doc scrape)"),
             CompileMode::RunCustomBuild => target.push_str(" (run)"),
         }
         let unit_time = UnitTime {

src/cargo/core/compiler/unit_dependencies.rs

@@ -47,6 +47,7 @@ struct State<'a, 'cfg> {
     target_data: &'a RustcTargetData<'cfg>,
     profiles: &'a Profiles,
     interner: &'a UnitInterner,
+    scrape_units: &'a [Unit],
 
     /// A set of edges in `unit_dependencies` where (a, b) means that the
     /// dependency from a to b was added purely because it was a dev-dependency.
@@ -61,6 +62,7 @@ pub fn build_unit_dependencies<'a, 'cfg>(
     features: &'a ResolvedFeatures,
     std_resolve: Option<&'a (Resolve, ResolvedFeatures)>,
     roots: &[Unit],
+    scrape_units: &[Unit],
     std_roots: &HashMap<CompileKind, Vec<Unit>>,
     global_mode: CompileMode,
     target_data: &'a RustcTargetData<'cfg>,
@@ -91,6 +93,7 @@ pub fn build_unit_dependencies<'a, 'cfg>(
         target_data,
         profiles,
         interner,
+        scrape_units,
         dev_dependency_edges: HashSet::new(),
     };
 
@@ -253,6 +256,7 @@ fn compute_deps(
         if !dep.is_transitive()
             && !unit.target.is_test()
             && !unit.target.is_example()
+            && !unit.mode.is_doc_scrape()
             && !unit.mode.is_any_test()
         {
             return false;
@@ -467,6 +471,25 @@ fn compute_deps_doc(
     if unit.target.is_bin() || unit.target.is_example() {
         ret.extend(maybe_lib(unit, state, unit_for)?);
     }
+
+    // Add all units being scraped for examples as a dependency of Doc units.
+    if state.ws.is_member(&unit.pkg) {
+        for scrape_unit in state.scrape_units.iter() {
+            // This needs to match the FeaturesFor used in cargo_compile::generate_targets.
+            let unit_for = UnitFor::new_host(scrape_unit.target.proc_macro());
+            deps_of(scrape_unit, state, unit_for)?;
+            ret.push(new_unit_dep(
+                state,
+                scrape_unit,
+                &scrape_unit.pkg,
+                &scrape_unit.target,
+                unit_for,
+                scrape_unit.kind,
+                scrape_unit.mode,
+            )?);
+        }
+    }
+
     Ok(ret)
 }
 
@@ -558,7 +581,7 @@ fn dep_build_script(
 /// Choose the correct mode for dependencies.
 fn check_or_build_mode(mode: CompileMode, target: &Target) -> CompileMode {
     match mode {
-        CompileMode::Check { .. } | CompileMode::Doc { .. } => {
+        CompileMode::Check { .. } | CompileMode::Doc { .. } | CompileMode::Docscrape => {
             if target.for_host() {
                 // Plugin and proc macro targets should be compiled like
                 // normal.
@@ -695,6 +718,14 @@ fn connect_run_custom_build_deps(state: &mut State<'_, '_>) {
                         && other.unit.target.is_linkable()
                         && other.unit.pkg.manifest().links().is_some()
                 })
+                // Avoid cycles when using the doc --scrape-examples feature:
+                // Say a workspace has crates A and B where A has a build-dependency on B.
+                // The Doc units for A and B will have a dependency on the Docscrape for both A and B.
+                // So this would add a dependency from B-build to A-build, causing a cycle:
+                //   B (build) -> A (build) -> B(build)
+                // See the test scrape_examples_avoid_build_script_cycle for a concrete example.
+                // To avoid this cycle, we filter out the B -> A (docscrape) dependency.
+                .filter(|(_parent, other)| !other.unit.mode.is_doc_scrape())
                 // Skip dependencies induced via dev-dependencies since
                 // connections between `links` and build scripts only happens
                 // via normal dependencies. Otherwise since dev-dependencies can

src/cargo/core/profiles.rs

@@ -323,7 +323,9 @@ impl Profiles {
                         (InternedString::new("dev"), None)
                     }
                 }
-                CompileMode::Doc { .. } => (InternedString::new("doc"), None),
+                CompileMode::Doc { .. } | CompileMode::Docscrape => {
+                    (InternedString::new("doc"), None)
+                }
             }
         } else {
             (self.requested_profile, None)

src/cargo/ops/cargo_compile.rs

@@ -76,6 +76,9 @@ pub struct CompileOptions {
     /// Whether the `--document-private-items` flags was specified and should
     /// be forwarded to `rustdoc`.
     pub rustdoc_document_private_items: bool,
+    /// Whether the `--scrape-examples` flag was specified and build flags for
+    /// examples should be forwarded to `rustdoc`.
+    pub rustdoc_scrape_examples: Option<CompileFilter>,
     /// Whether the build process should check the minimum Rust version
     /// defined in the cargo metadata for a crate.
     pub honor_rust_version: bool,
@@ -94,12 +97,13 @@ impl<'a> CompileOptions {
             target_rustc_args: None,
             local_rustdoc_args: None,
             rustdoc_document_private_items: false,
+            rustdoc_scrape_examples: None,
             honor_rust_version: true,
         })
     }
 }
 
-#[derive(Clone, PartialEq, Eq, Debug)]
+#[derive(PartialEq, Eq, Debug)]
 pub enum Packages {
     Default,
     All,
@@ -334,6 +338,7 @@ pub fn create_bcx<'a, 'cfg>(
         ref target_rustc_args,
         ref local_rustdoc_args,
         rustdoc_document_private_items,
+        ref rustdoc_scrape_examples,
         honor_rust_version,
     } = *options;
     let config = ws.config();
@@ -351,7 +356,7 @@ pub fn create_bcx<'a, 'cfg>(
                 )?;
             }
         }
-        CompileMode::Doc { .. } | CompileMode::Doctest => {
+        CompileMode::Doc { .. } | CompileMode::Doctest | CompileMode::Docscrape => {
             if std::env::var("RUSTDOC_FLAGS").is_ok() {
                 config.shell().warn(
                     "Cargo does not read `RUSTDOC_FLAGS` environment variable. Did you mean `RUSTDOCFLAGS`?"
@@ -363,8 +368,16 @@ pub fn create_bcx<'a, 'cfg>(
 
     let target_data = RustcTargetData::new(ws, &build_config.requested_kinds)?;
 
-    let specs = spec.to_package_id_specs(ws)?;
-    let has_dev_units = if filter.need_dev_deps(build_config.mode) {
+    let all_packages = &Packages::All;
+    let need_reverse_dependencies = rustdoc_scrape_examples.is_some();
+    let full_specs = if need_reverse_dependencies {
+        all_packages
+    } else {
+        spec
+    };
+
+    let resolve_specs = full_specs.to_package_id_specs(ws)?;
+    let has_dev_units = if filter.need_dev_deps(build_config.mode) || need_reverse_dependencies {
         HasDevUnits::Yes
     } else {
         HasDevUnits::No
@@ -374,7 +387,7 @@ pub fn create_bcx<'a, 'cfg>(
         &target_data,
         &build_config.requested_kinds,
         cli_features,
-        &specs,
+        &resolve_specs,
         has_dev_units,
         crate::core::resolver::features::ForceAllTargets::No,
     )?;
@@ -408,6 +421,11 @@ pub fn create_bcx<'a, 'cfg>(
     // Find the packages in the resolver that the user wants to build (those
     // passed in with `-p` or the defaults from the workspace), and convert
     // Vec<PackageIdSpec> to a Vec<PackageId>.
+    let specs = if need_reverse_dependencies {
+        spec.to_package_id_specs(ws)?
+    } else {
+        resolve_specs.clone()
+    };
     let to_build_ids = resolve.specs_to_ids(&specs)?;
     // Now get the `Package` for each `PackageId`. This may trigger a download
     // if the user specified `-p` for a dependency that is not downloaded.
@@ -487,6 +505,30 @@ pub fn create_bcx<'a, 'cfg>(
         interner,
     )?;
 
+    let mut scrape_units = match rustdoc_scrape_examples {
+        Some(scrape_filter) => {
+            let to_build_ids = resolve.specs_to_ids(&resolve_specs)?;
+            let to_builds = pkg_set.get_many(to_build_ids)?;
+            let mode = CompileMode::Docscrape;
+
+            generate_targets(
+                ws,
+                &to_builds,
+                scrape_filter,
+                &build_config.requested_kinds,
+                explicit_host_kind,
+                mode,
+                &resolve,
+                &workspace_resolve,
+                &resolved_features,
+                &pkg_set,
+                &profiles,
+                interner,
+            )?
+        }
+        None => Vec::new(),
+    };
+
     let std_roots = if let Some(crates) = &config.cli_unstable().build_std {
         // Only build libtest if it looks like it is needed.
         let mut crates = crates.clone();
@@ -521,6 +563,7 @@ pub fn create_bcx<'a, 'cfg>(
         &resolved_features,
         std_resolve_features.as_ref(),
         &units,
+        &scrape_units,
         &std_roots,
         build_config.mode,
         &target_data,
@@ -542,10 +585,17 @@ pub fn create_bcx<'a, 'cfg>(
         // Rebuild the unit graph, replacing the explicit host targets with
         // CompileKind::Host, merging any dependencies shared with build
         // dependencies.
-        let new_graph = rebuild_unit_graph_shared(interner, unit_graph, &units, explicit_host_kind);
+        let new_graph = rebuild_unit_graph_shared(
+            interner,
+            unit_graph,
+            &units,
+            &scrape_units,
+            explicit_host_kind,
+        );
         // This would be nicer with destructuring assignment.
         units = new_graph.0;
-        unit_graph = new_graph.1;
+        scrape_units = new_graph.1;
+        unit_graph = new_graph.2;
     }
 
     let mut extra_compiler_args = HashMap::new();
@@ -560,6 +610,7 @@ pub fn create_bcx<'a, 'cfg>(
         }
         extra_compiler_args.insert(units[0].clone(), args);
     }
+
     for unit in &units {
         if unit.mode.is_doc() || unit.mode.is_doc_test() {
             let mut extra_args = local_rustdoc_args.clone();
@@ -621,6 +672,7 @@ pub fn create_bcx<'a, 'cfg>(
         target_data,
         units,
         unit_graph,
+        scrape_units,
     )?;
 
     Ok(bcx)
@@ -742,17 +794,18 @@ impl CompileFilter {
         match mode {
             CompileMode::Test | CompileMode::Doctest | CompileMode::Bench => true,
             CompileMode::Check { test: true } => true,
-            CompileMode::Build | CompileMode::Doc { .. } | CompileMode::Check { test: false } => {
-                match *self {
-                    CompileFilter::Default { .. } => false,
-                    CompileFilter::Only {
-                        ref examples,
-                        ref tests,
-                        ref benches,
-                        ..
-                    } => examples.is_specific() || tests.is_specific() || benches.is_specific(),
-                }
-            }
+            CompileMode::Build
+            | CompileMode::Doc { .. }
+            | CompileMode::Docscrape
+            | CompileMode::Check { test: false } => match *self {
+                CompileFilter::Default { .. } => false,
+                CompileFilter::Only {
+                    ref examples,
+                    ref tests,
+                    ref benches,
+                    ..
+                } => examples.is_specific() || tests.is_specific() || benches.is_specific(),
+            },
             CompileMode::RunCustomBuild => panic!("Invalid mode"),
         }
     }
@@ -1342,7 +1395,9 @@ fn filter_default_targets(targets: &[Target], mode: CompileMode) -> Vec<&Target>
                 })
                 .collect()
         }
-        CompileMode::Doctest | CompileMode::RunCustomBuild => panic!("Invalid mode {:?}", mode),
+        CompileMode::Doctest | CompileMode::Docscrape | CompileMode::RunCustomBuild => {
+            panic!("Invalid mode {:?}", mode)
+        }
     }
 }
 
@@ -1454,8 +1509,9 @@ fn rebuild_unit_graph_shared(
     interner: &UnitInterner,
     unit_graph: UnitGraph,
     roots: &[Unit],
+    scrape_units: &[Unit],
     to_host: CompileKind,
-) -> (Vec<Unit>, UnitGraph) {
+) -> (Vec<Unit>, Vec<Unit>, UnitGraph) {
     let mut result = UnitGraph::new();
     // Map of the old unit to the new unit, used to avoid recursing into units
     // that have already been computed to improve performance.
@@ -1466,7 +1522,11 @@ fn rebuild_unit_graph_shared(
             traverse_and_share(interner, &mut memo, &mut result, &unit_graph, root, to_host)
         })
         .collect();
-    (new_roots, result)
+    let new_scrape_units = scrape_units
+        .iter()
+        .map(|unit| memo.get(unit).unwrap().clone())
+        .collect();
+    (new_roots, new_scrape_units, result)
 }
 
 /// Recursive function for rebuilding the graph.

src/cargo/ops/cargo_package.rs

@@ -765,6 +765,7 @@ fn run_verify(
             target_rustc_args: rustc_args,
             local_rustdoc_args: None,
             rustdoc_document_private_items: false,
+            rustdoc_scrape_examples: None,
             honor_rust_version: true,
         },
         &exec,

src/cargo/util/command_prelude.rs

@@ -544,6 +544,7 @@ pub trait ArgMatchesExt {
             target_rustc_args: None,
             local_rustdoc_args: None,
             rustdoc_document_private_items: false,
+            rustdoc_scrape_examples: None,
             honor_rust_version: !self._is_present("ignore-rust-version"),
         };
 

src/doc/src/reference/unstable.md

@@ -1207,7 +1207,7 @@ for the appropriate target and influenced by any other RUSTFLAGS.
 * Tracking Issue: [#9778](https://github.com/rust-lang/cargo/issues/9778)
 * PR: [#9627](https://github.com/rust-lang/cargo/pull/9627)
 
-The `different-binary-name` feature allows setting the filename of the binary without having to obey the 
+The `different-binary-name` feature allows setting the filename of the binary without having to obey the
 restrictions placed on crate names. For example, the crate name must use only `alphanumeric` characters
 or `-` or `_`, and cannot be empty.
 
@@ -1378,7 +1378,23 @@ The 2021 edition has been stabilized in the 1.56 release.
 See the [`edition` field](manifest.md#the-edition-field) for more information on setting the edition.
 See [`cargo fix --edition`](../commands/cargo-fix.md) and [The Edition Guide](../../edition-guide/index.html) for more information on migrating existing projects.
 
+
 ### Custom named profiles
 
 Custom named profiles have been stabilized in the 1.57 release. See the
 [profiles chapter](profiles.md#custom-profiles) for more information.
+
+
+### scrape-examples
+
+* RFC: [#3123](https://github.com/rust-lang/rfcs/pull/3123)
+* Tracking Issue: [#9910](https://github.com/rust-lang/cargo/issues/9910)
+
+The `--scrape-examples` argument to the `doc` command tells Rustdoc to search
+crates in the current workspace for calls to functions. Those call-sites are then
+included as documentation. The flag can take an argument of `all` or `examples`
+which configures which crate in the workspace to analyze for examples. For instance:
+
+```
+cargo doc -Z unstable-options --scrape-examples examples
+```

tests/testsuite/doc.rs

@@ -2148,3 +2148,152 @@ fn doc_fingerprint_unusual_behavior() {
     assert!(build_doc.join("somefile").exists());
     assert!(real_doc.join("somefile").exists());
 }
+
+#[cargo_test]
+fn scrape_examples_basic() {
+    if !is_nightly() {
+        // --scrape-examples is unstable
+        return;
+    }
+
+    let p = project()
+        .file(
+            "Cargo.toml",
+            r#"
+                [package]
+                name = "foo"
+                version = "0.0.1"
+                authors = []
+            "#,
+        )
+        .file("examples/ex.rs", "fn main() { foo::foo(); }")
+        .file("src/lib.rs", "pub fn foo() {}\npub fn bar() { foo(); }")
+        .build();
+
+    p.cargo("doc -Zunstable-options --scrape-examples all")
+        .masquerade_as_nightly_cargo()
+        .with_stderr(
+            "\
+[..] foo v0.0.1 ([CWD])
+[..] foo v0.0.1 ([CWD])
+[FINISHED] dev [unoptimized + debuginfo] target(s) in [..]
+",
+        )
+        .run();
+
+    let doc_html = p.read_file("target/doc/foo/fn.foo.html");
+    assert!(doc_html.contains("Examples found in repository"));
+    assert!(doc_html.contains("More examples"));
+
+    // Ensure that the reverse-dependency has its sources generated
+    assert!(p.build_dir().join("doc/src/ex/ex.rs.html").exists());
+}
+
+#[cargo_test]
+fn scrape_examples_avoid_build_script_cycle() {
+    if !is_nightly() {
+        // --scrape-examples is unstable
+        return;
+    }
+
+    let p = project()
+        // package with build dependency
+        .file(
+            "Cargo.toml",
+            r#"
+                [package]
+                name = "foo"
+                version = "0.0.1"
+                authors = []
+                links = "foo"
+
+                [workspace]
+                members = ["bar"]
+
+                [build-dependencies]
+                bar = {path = "bar"}
+            "#,
+        )
+        .file("src/lib.rs", "")
+        .file("build.rs", "fn main(){}")
+        // dependency
+        .file(
+            "bar/Cargo.toml",
+            r#"
+                [package]
+                name = "bar"
+                version = "0.0.1"
+                authors = []
+                links = "bar"
+            "#,
+        )
+        .file("bar/src/lib.rs", "")
+        .file("bar/build.rs", "fn main(){}")
+        .build();
+
+    p.cargo("doc --all -Zunstable-options --scrape-examples all")
+        .masquerade_as_nightly_cargo()
+        .run();
+}
+
+#[cargo_test]
+fn scrape_examples_complex_reverse_dependencies() {
+    if !is_nightly() {
+        // --scrape-examples is unstable
+        return;
+    }
+
+    let p = project()
+        .file(
+            "Cargo.toml",
+            r#"
+                [package]
+                name = "foo"
+                version = "0.0.1"
+                authors = []
+
+                [dev-dependencies]
+                a = {path = "a", features = ["feature"]}
+                b = {path = "b"}
+
+                [workspace]
+                members = ["b"]
+            "#,
+        )
+        .file("src/lib.rs", "")
+        .file("examples/ex.rs", "fn main() { a::f(); }")
+        .file(
+            "a/Cargo.toml",
+            r#"
+                [package]
+                name = "a"
+                version = "0.0.1"
+                authors = []
+
+                [lib]
+                proc-macro = true
+
+                [dependencies]
+                b = {path = "../b"}
+
+                [features]
+                feature = []
+            "#,
+        )
+        .file("a/src/lib.rs", "#[cfg(feature)] pub fn f();")
+        .file(
+            "b/Cargo.toml",
+            r#"
+                [package]
+                name = "b"
+                version = "0.0.1"
+                authors = []
+            "#,
+        )
+        .file("b/src/lib.rs", "")
+        .build();
+
+    p.cargo("doc -Zunstable-options --scrape-examples all")
+        .masquerade_as_nightly_cargo()
+        .run();
+}

tests/testsuite/rustdoc.rs

@@ -32,6 +32,7 @@ fn rustdoc_args() {
         -o [CWD]/target/doc \
         [..] \
         --cfg=foo \
+        -C metadata=[..] \
         -L dependency=[CWD]/target/debug/deps [..]`
 [FINISHED] dev [unoptimized + debuginfo] target(s) in [..]
 ",
@@ -83,6 +84,7 @@ fn rustdoc_foo_with_bar_dependency() {
         -o [CWD]/target/doc \
         [..] \
         --cfg=foo \
+        -C metadata=[..] \
         -L dependency=[CWD]/target/debug/deps \
         --extern [..]`
 [FINISHED] dev [unoptimized + debuginfo] target(s) in [..]
@@ -122,6 +124,7 @@ fn rustdoc_only_bar_dependency() {
         -o [CWD]/target/doc \
         [..] \
         --cfg=foo \
+        -C metadata=[..] \
         -L dependency=[CWD]/target/debug/deps [..]`
 [FINISHED] dev [unoptimized + debuginfo] target(s) in [..]
 ",
@@ -144,6 +147,7 @@ fn rustdoc_same_name_documents_lib() {
         -o [CWD]/target/doc \
         [..] \
         --cfg=foo \
+        -C metadata=[..] \
         -L dependency=[CWD]/target/debug/deps [..]`
 [FINISHED] dev [unoptimized + debuginfo] target(s) in [..]
 ",