diff --git a/Cargo.lock b/Cargo.lock index b3f4ec3..ca2a0df 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -97,6 +97,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "arc-swap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" + [[package]] name = "async-stream" version = "0.3.6" @@ -360,6 +366,46 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "comrade" +version = "0.1.0" +source = "git+https://git.hydrar.de/jmarya/comrade#00e448ac316623bb27cc3a3b799fc201cd6302d7" +dependencies = [ + "comrade-macro", + "crossbeam", + "dashmap", + "env_logger 0.11.6", + "log", + "once_cell", + "rand 0.9.0", + "redis", + "serde", + "serde_json", + "tokio", + "uuid", +] + +[[package]] +name = "comrade-macro" +version = "0.1.0" +source = "git+https://git.hydrar.de/jmarya/comrade#00e448ac316623bb27cc3a3b799fc201cd6302d7" +dependencies = [ + "proc-macro2", + "quote", + "serde_json", + "syn", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -426,6 +472,28 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +[[package]] +name = "crossbeam" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ba6d68e24814cb8de6bb986db8222d3a027d15872cabc0d18817bc3c0e4471" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -1639,6 +1707,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-bigint-dig" version = "0.8.4" @@ -1774,6 +1852,7 @@ dependencies = [ "based", "bytesize", "chrono", + "comrade", "env_logger 0.11.6", "log", "maud", @@ -2043,6 +2122,23 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "redis" +version = "0.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8034fb926579ff49d3fe58d288d5dcb580bf11e9bccd33224b45adebf0fd0c23" +dependencies = [ + "arc-swap", + "combine", + "itoa", + "num-bigint", + "percent-encoding", + "ryu", + "sha1_smol", + "socket2", + "url", +] + [[package]] name = "redox_syscall" version = "0.5.10" @@ -2165,6 +2261,7 @@ dependencies = [ "base64 0.22.1", "bytes", "encoding_rs", + "futures-channel", "futures-core", "futures-util", "h2 0.4.8", @@ -2544,6 +2641,12 @@ dependencies = [ "digest", ] +[[package]] +name = "sha1_smol" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" + [[package]] name = "sha2" version = "0.10.8" diff --git a/Cargo.toml b/Cargo.toml index 34e93db..ec66435 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,13 +5,14 @@ edition = "2024" [dependencies] based = { git = "https://git.hydrar.de/jmarya/based" } +comrade = { git = "https://git.hydrar.de/jmarya/comrade" } bytesize = "1.3.0" chrono = "0.4.39" env_logger = "0.11.6" log = "0.4.22" maud = "0.26.0" rand = "0.9.0" -reqwest = { version = "0.12.12" } +reqwest = { version = "0.12.12", features = ["blocking"] } rocket = "0.5.1" serde = { version = "1.0.217", features = ["derive"] } serde_json = "1.0.134" diff --git a/src/pkg/mirror.rs b/src/pkg/mirror.rs index 00b5dc4..1c30e2b 100644 --- a/src/pkg/mirror.rs +++ b/src/pkg/mirror.rs @@ -1,5 +1,6 @@ use std::path::PathBuf; +use comrade::rally; use rand::seq::SliceRandom; use super::{Package, Repository, arch::Architecture}; @@ -31,33 +32,46 @@ impl MirrorRepository { arch.to_string() }; - let mut mirrorlist = mirrorlist.to_vec(); - mirrorlist.shuffle(&mut rand::rng()); + let parent = file_path.parent().unwrap(); + std::fs::create_dir_all(parent).unwrap(); - for mirror in mirrorlist { - let mirror = mirror - .replace("$repo", &self.inner.name) - .replace("$arch", &arch); + let mirrorlist_links = mirrorlist + .into_iter() + .take(5) + .map(|x| { + let mirror = x.replace("$repo", &self.inner.name).replace("$arch", &arch); - let url = format!("{mirror}/{url}"); - log::info!("Trying mirror {url}"); + ( + x.clone(), + format!("{mirror}/{url}"), + file_path.to_str().unwrap().to_string(), + ) + }) + .collect(); - let client = reqwest::Client::new(); - if let Ok(resp) = client.get(&url).send().await { + let data = rally(mirrorlist_links, |input| { + let (mirror, url, file_path) = input; + + let client = reqwest::blocking::Client::new(); + if let Ok(resp) = client.get(url).send() { if resp.status().is_success() { - let data = resp.bytes().await.unwrap().to_vec(); - let parent = file_path.parent().unwrap(); - std::fs::create_dir_all(parent).unwrap(); - + let data = resp.bytes().unwrap().to_vec(); log::info!("Downloaded {url}"); - std::fs::write(file_path, data).unwrap(); - return; + return Some(data); } else { log::warn!("Mirror {mirror} failed [{}]", resp.status().as_u16()); + return None; } } + + None + }); + + if let Some(data) = data.1 { + std::fs::write(file_path, data).unwrap(); + } else { + log::error!("Downloading {url} failed. No mirror could provide a valid response."); } - log::error!("Downloading {url} failed. No mirror could provide a valid response.") } /// Get the `.db.tar.gz` content for the repository of `arch`