mirror of
https://github.com/rust-lang/cargo
synced 2024-08-28 11:50:23 +00:00
Auto merge of #11230 - epage:unstable2, r=weihanglo
feat(publish): Support 'publish.timeout' config behind '-Zpublish-timeout' Originally, crates.io would block on publish requests until the publish was complete, giving `cargo publish` this behavior by extension. When crates.io switched to asynchronous publishing, this intermittently broke people's workflows when publishing multiple crates. I say interittent because it usually works until it doesn't and it is unclear why to the end user because it will be published by the time they check. In the end, callers tend to either put in timeouts (and pray), poll the server's API, or use `crates-index` crate to poll the index. This isn't sufficient because - For any new interested party, this is a pit of failure they'll fall into - crates-index has re-implemented index support incorrectly in the past, currently doesn't handle auth, doesn't support `git-cli`, etc. - None of these previous options work if we were to implement workspace-publish support (https://github.com/rust-lang/cargo/issues/1169) - The new sparse registry might increase the publish times, making the delay easier to hit manually - The new sparse registry goes through CDNs so checking the server's API might not be sufficient - Once the sparse registry is available, crates-index users will find out when the package is ready in git but it might not be ready through the sparse registry because of CDNs This introduces unstable support for blocking by setting `publish.timeout` to non-zero value. A step towards #9507
This commit is contained in:
commit
b592ba4d97
|
@ -197,6 +197,7 @@ fn substitute_macros(input: &str) -> String {
|
|||
("[MIGRATING]", " Migrating"),
|
||||
("[EXECUTABLE]", " Executable"),
|
||||
("[SKIPPING]", " Skipping"),
|
||||
("[WAITING]", " Waiting"),
|
||||
];
|
||||
let mut result = input.to_owned();
|
||||
for &(pat, subst) in ¯os {
|
||||
|
|
|
@ -686,6 +686,7 @@ unstable_cli_options!(
|
|||
rustdoc_map: bool = ("Allow passing external documentation mappings to rustdoc"),
|
||||
separate_nightlies: bool = (HIDDEN),
|
||||
terminal_width: Option<Option<usize>> = ("Provide a terminal width to rustc for error truncation"),
|
||||
publish_timeout: bool = ("Enable the `publish.timeout` key in .cargo/config.toml file"),
|
||||
unstable_options: bool = ("Allow the usage of unstable options"),
|
||||
// TODO(wcrichto): move scrape example configuration into Cargo.toml before stabilization
|
||||
// See: https://github.com/rust-lang/cargo/pull/9525#discussion_r728470927
|
||||
|
@ -930,6 +931,7 @@ impl CliUnstable {
|
|||
"jobserver-per-rustc" => self.jobserver_per_rustc = parse_empty(k, v)?,
|
||||
"host-config" => self.host_config = parse_empty(k, v)?,
|
||||
"target-applies-to-host" => self.target_applies_to_host = parse_empty(k, v)?,
|
||||
"publish-timeout" => self.publish_timeout = parse_empty(k, v)?,
|
||||
"features" => {
|
||||
// `-Z features` has been stabilized since 1.51,
|
||||
// but `-Z features=compare` is still allowed for convenience
|
||||
|
|
|
@ -18,9 +18,11 @@ use termcolor::Color::Green;
|
|||
use termcolor::ColorSpec;
|
||||
|
||||
use crate::core::dependency::DepKind;
|
||||
use crate::core::dependency::Dependency;
|
||||
use crate::core::manifest::ManifestMetadata;
|
||||
use crate::core::resolver::CliFeatures;
|
||||
use crate::core::source::Source;
|
||||
use crate::core::QueryKind;
|
||||
use crate::core::{Package, SourceId, Workspace};
|
||||
use crate::ops;
|
||||
use crate::ops::Packages;
|
||||
|
@ -183,6 +185,19 @@ pub fn publish(ws: &Workspace<'_>, opts: &PublishOpts<'_>) -> CargoResult<()> {
|
|||
reg_ids.original,
|
||||
opts.dry_run,
|
||||
)?;
|
||||
if !opts.dry_run {
|
||||
const DEFAULT_TIMEOUT: u64 = 0;
|
||||
let timeout = if opts.config.cli_unstable().publish_timeout {
|
||||
let timeout: Option<u64> = opts.config.get("publish.timeout")?;
|
||||
timeout.unwrap_or(DEFAULT_TIMEOUT)
|
||||
} else {
|
||||
DEFAULT_TIMEOUT
|
||||
};
|
||||
if 0 < timeout {
|
||||
let timeout = std::time::Duration::from_secs(timeout);
|
||||
wait_for_publish(opts.config, reg_ids.original, pkg, timeout)?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
@ -374,6 +389,72 @@ fn transmit(
|
|||
Ok(())
|
||||
}
|
||||
|
||||
fn wait_for_publish(
|
||||
config: &Config,
|
||||
registry_src: SourceId,
|
||||
pkg: &Package,
|
||||
timeout: std::time::Duration,
|
||||
) -> CargoResult<()> {
|
||||
let version_req = format!("={}", pkg.version());
|
||||
let mut source = SourceConfigMap::empty(config)?.load(registry_src, &HashSet::new())?;
|
||||
let source_description = source.describe();
|
||||
let query = Dependency::parse(pkg.name(), Some(&version_req), registry_src)?;
|
||||
|
||||
let now = std::time::Instant::now();
|
||||
let sleep_time = std::time::Duration::from_secs(1);
|
||||
let mut logged = false;
|
||||
loop {
|
||||
{
|
||||
let _lock = config.acquire_package_cache_lock()?;
|
||||
// Force re-fetching the source
|
||||
//
|
||||
// As pulling from a git source is expensive, we track when we've done it within the
|
||||
// process to only do it once, but we are one of the rare cases that needs to do it
|
||||
// multiple times
|
||||
config
|
||||
.updated_sources()
|
||||
.remove(&source.replaced_source_id());
|
||||
source.invalidate_cache();
|
||||
let summaries = loop {
|
||||
// Exact to avoid returning all for path/git
|
||||
match source.query_vec(&query, QueryKind::Exact) {
|
||||
std::task::Poll::Ready(res) => {
|
||||
break res?;
|
||||
}
|
||||
std::task::Poll::Pending => source.block_until_ready()?,
|
||||
}
|
||||
};
|
||||
if !summaries.is_empty() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if timeout < now.elapsed() {
|
||||
config.shell().warn(format!(
|
||||
"timed out waiting for `{}` to be in {}",
|
||||
pkg.name(),
|
||||
source_description
|
||||
))?;
|
||||
break;
|
||||
}
|
||||
|
||||
if !logged {
|
||||
config.shell().status(
|
||||
"Waiting",
|
||||
format!(
|
||||
"on `{}` to propagate to {} (ctrl-c to wait asynchronously)",
|
||||
pkg.name(),
|
||||
source_description
|
||||
),
|
||||
)?;
|
||||
logged = true;
|
||||
}
|
||||
std::thread::sleep(sleep_time);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Returns the index and token from the config file for the given registry.
|
||||
///
|
||||
/// `registry` is typically the registry specified on the command-line. If
|
||||
|
|
|
@ -99,6 +99,7 @@ Each new feature described below should explain how to use it.
|
|||
* [credential-process](#credential-process) — Adds support for fetching registry tokens from an external authentication program.
|
||||
* [`cargo logout`](#cargo-logout) — Adds the `logout` command to remove the currently saved registry token.
|
||||
* [sparse-registry](#sparse-registry) — Adds support for fetching from static-file HTTP registries (`sparse+`)
|
||||
* [publish-timeout](#publish-timeout) — Controls the timeout between uploading the crate and being available in the index
|
||||
|
||||
### allow-features
|
||||
|
||||
|
@ -841,6 +842,23 @@ crates, which can save significant time and bandwidth.
|
|||
|
||||
The format of the sparse index is identical to a checkout of a git-based index.
|
||||
|
||||
### publish-timeout
|
||||
* Tracking Issue: [11222](https://github.com/rust-lang/cargo/issues/11222)
|
||||
|
||||
The `publish.timeout` key in a config file can be used to control how long
|
||||
`cargo publish` waits between posting a package to the registry and it being
|
||||
available in the local index.
|
||||
|
||||
A timeout of `0` prevents any checks from occurring.
|
||||
|
||||
It requires the `-Zpublish-timeout` command-line options to be set.
|
||||
|
||||
```toml
|
||||
# config.toml
|
||||
[publish]
|
||||
timeout = 300 # in seconds
|
||||
```
|
||||
|
||||
### credential-process
|
||||
* Tracking Issue: [#8933](https://github.com/rust-lang/cargo/issues/8933)
|
||||
* RFC: [#2730](https://github.com/rust-lang/rfcs/pull/2730)
|
||||
|
|
|
@ -5,6 +5,7 @@ use cargo_test_support::paths;
|
|||
use cargo_test_support::registry::{self, Package, Response};
|
||||
use cargo_test_support::{basic_manifest, no_such_file_err_msg, project, publish};
|
||||
use std::fs;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
const CLEAN_FOO_JSON: &str = r#"
|
||||
{
|
||||
|
@ -2260,3 +2261,237 @@ fn http_api_not_noop() {
|
|||
|
||||
p.cargo("build").run();
|
||||
}
|
||||
|
||||
#[cargo_test]
|
||||
fn wait_for_publish() {
|
||||
// Counter for number of tries before the package is "published"
|
||||
let arc: Arc<Mutex<u32>> = Arc::new(Mutex::new(0));
|
||||
let arc2 = arc.clone();
|
||||
|
||||
// Registry returns an invalid response.
|
||||
let registry = registry::RegistryBuilder::new()
|
||||
.http_index()
|
||||
.http_api()
|
||||
.add_responder("/index/de/la/delay", move |req, server| {
|
||||
let mut lock = arc.lock().unwrap();
|
||||
*lock += 1;
|
||||
// if the package name contains _ or -
|
||||
if *lock <= 1 {
|
||||
server.not_found(req)
|
||||
} else {
|
||||
server.index(req)
|
||||
}
|
||||
})
|
||||
.build();
|
||||
|
||||
// The sparse-registry test server does not know how to publish on its own.
|
||||
// So let us call publish for it.
|
||||
Package::new("delay", "0.0.1")
|
||||
.file("src/lib.rs", "")
|
||||
.publish();
|
||||
|
||||
let p = project()
|
||||
.file(
|
||||
"Cargo.toml",
|
||||
r#"
|
||||
[package]
|
||||
name = "delay"
|
||||
version = "0.0.1"
|
||||
authors = []
|
||||
license = "MIT"
|
||||
description = "foo"
|
||||
|
||||
"#,
|
||||
)
|
||||
.file("src/lib.rs", "")
|
||||
.file(
|
||||
".cargo/config",
|
||||
"
|
||||
[publish]
|
||||
timeout = 60
|
||||
",
|
||||
)
|
||||
.build();
|
||||
|
||||
p.cargo("publish --no-verify -Z sparse-registry -Z publish-timeout")
|
||||
.masquerade_as_nightly_cargo(&["sparse-registry", "publish-timeout"])
|
||||
.replace_crates_io(registry.index_url())
|
||||
.with_status(0)
|
||||
.with_stderr(
|
||||
"\
|
||||
[UPDATING] `crates-io` index
|
||||
[WARNING] manifest has no documentation, [..]
|
||||
See [..]
|
||||
[PACKAGING] delay v0.0.1 ([CWD])
|
||||
[UPLOADING] delay v0.0.1 ([CWD])
|
||||
[UPDATING] `crates-io` index
|
||||
[WAITING] on `delay` to propagate to `crates-io` index (which is replacing registry `crates-io`) (ctrl-c to wait asynchronously)
|
||||
",
|
||||
)
|
||||
.run();
|
||||
|
||||
// Verify the responder has been pinged
|
||||
let lock = arc2.lock().unwrap();
|
||||
assert_eq!(*lock, 2);
|
||||
drop(lock);
|
||||
|
||||
let p = project()
|
||||
.file(
|
||||
"Cargo.toml",
|
||||
r#"
|
||||
[package]
|
||||
name = "foo"
|
||||
version = "0.0.1"
|
||||
authors = []
|
||||
[dependencies]
|
||||
delay = "0.0.1"
|
||||
"#,
|
||||
)
|
||||
.file("src/main.rs", "fn main() {}")
|
||||
.build();
|
||||
|
||||
p.cargo("build -Z sparse-registry")
|
||||
.masquerade_as_nightly_cargo(&["sparse-registry"])
|
||||
.with_status(0)
|
||||
.run();
|
||||
}
|
||||
|
||||
/// A separate test is needed for package names with - or _ as they hit
|
||||
/// the responder twice per cargo invocation. If that ever gets changed
|
||||
/// this test will need to be changed accordingly.
|
||||
#[cargo_test]
|
||||
fn wait_for_publish_underscore() {
|
||||
// Counter for number of tries before the package is "published"
|
||||
let arc: Arc<Mutex<u32>> = Arc::new(Mutex::new(0));
|
||||
let arc2 = arc.clone();
|
||||
|
||||
// Registry returns an invalid response.
|
||||
let registry = registry::RegistryBuilder::new()
|
||||
.http_index()
|
||||
.http_api()
|
||||
.add_responder("/index/de/la/delay_with_underscore", move |req, server| {
|
||||
let mut lock = arc.lock().unwrap();
|
||||
*lock += 1;
|
||||
// package names with - or _ hit the responder twice per cargo invocation
|
||||
if *lock <= 2 {
|
||||
server.not_found(req)
|
||||
} else {
|
||||
server.index(req)
|
||||
}
|
||||
})
|
||||
.build();
|
||||
|
||||
// The sparse-registry test server does not know how to publish on its own.
|
||||
// So let us call publish for it.
|
||||
Package::new("delay_with_underscore", "0.0.1")
|
||||
.file("src/lib.rs", "")
|
||||
.publish();
|
||||
|
||||
let p = project()
|
||||
.file(
|
||||
"Cargo.toml",
|
||||
r#"
|
||||
[package]
|
||||
name = "delay_with_underscore"
|
||||
version = "0.0.1"
|
||||
authors = []
|
||||
license = "MIT"
|
||||
description = "foo"
|
||||
|
||||
"#,
|
||||
)
|
||||
.file("src/lib.rs", "")
|
||||
.file(
|
||||
".cargo/config",
|
||||
"
|
||||
[publish]
|
||||
timeout = 60
|
||||
",
|
||||
)
|
||||
.build();
|
||||
|
||||
p.cargo("publish --no-verify -Z sparse-registry -Z publish-timeout")
|
||||
.masquerade_as_nightly_cargo(&["sparse-registry", "publish-timeout"])
|
||||
.replace_crates_io(registry.index_url())
|
||||
.with_status(0)
|
||||
.with_stderr(
|
||||
"\
|
||||
[UPDATING] `crates-io` index
|
||||
[WARNING] manifest has no documentation, [..]
|
||||
See [..]
|
||||
[PACKAGING] delay_with_underscore v0.0.1 ([CWD])
|
||||
[UPLOADING] delay_with_underscore v0.0.1 ([CWD])
|
||||
[UPDATING] `crates-io` index
|
||||
[WAITING] on `delay_with_underscore` to propagate to `crates-io` index (which is replacing registry `crates-io`) (ctrl-c to wait asynchronously)
|
||||
",
|
||||
)
|
||||
.run();
|
||||
|
||||
// Verify the repsponder has been pinged
|
||||
let lock = arc2.lock().unwrap();
|
||||
// NOTE: package names with - or _ hit the responder twice per cargo invocation
|
||||
assert_eq!(*lock, 3);
|
||||
drop(lock);
|
||||
|
||||
let p = project()
|
||||
.file(
|
||||
"Cargo.toml",
|
||||
r#"
|
||||
[package]
|
||||
name = "foo"
|
||||
version = "0.0.1"
|
||||
authors = []
|
||||
[dependencies]
|
||||
delay_with_underscore = "0.0.1"
|
||||
"#,
|
||||
)
|
||||
.file("src/main.rs", "fn main() {}")
|
||||
.build();
|
||||
|
||||
p.cargo("build -Z sparse-registry")
|
||||
.masquerade_as_nightly_cargo(&["sparse-registry"])
|
||||
.with_status(0)
|
||||
.run();
|
||||
}
|
||||
|
||||
#[cargo_test]
|
||||
fn skip_wait_for_publish() {
|
||||
// Intentionally using local registry so the crate never makes it to the index
|
||||
let registry = registry::init();
|
||||
|
||||
let p = project()
|
||||
.file(
|
||||
"Cargo.toml",
|
||||
r#"
|
||||
[package]
|
||||
name = "foo"
|
||||
version = "0.0.1"
|
||||
authors = []
|
||||
license = "MIT"
|
||||
description = "foo"
|
||||
"#,
|
||||
)
|
||||
.file("src/main.rs", "fn main() {}")
|
||||
.file(
|
||||
".cargo/config",
|
||||
"
|
||||
[publish]
|
||||
timeout = 0
|
||||
",
|
||||
)
|
||||
.build();
|
||||
|
||||
p.cargo("publish --no-verify -Zpublish-timeout")
|
||||
.replace_crates_io(registry.index_url())
|
||||
.masquerade_as_nightly_cargo(&["publish-timeout"])
|
||||
.with_stderr(
|
||||
"\
|
||||
[UPDATING] crates.io index
|
||||
[WARNING] manifest has no documentation, [..]
|
||||
See [..]
|
||||
[PACKAGING] foo v0.0.1 ([CWD])
|
||||
[UPLOADING] foo v0.0.1 ([CWD])
|
||||
",
|
||||
)
|
||||
.run();
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue