jmarya/pacco
jmarya/pacco
1
0
Fork 0
Code Issues 2 Pull requests Projects Releases Packages 1 Wiki Activity

✨ parallel downloads

Browse source
This commit is contained in:
JMARyA 2025-03-08 23:30:02 +01:00
parent ac6dd1479d
commit 71fa998861
Signed by: jmarya
GPG key ID: 901B2ADDF27C2263
3 changed files with 136 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

103
Cargo.lock generated
View file

@ -97,6 +97,12 @@ dependencies = [
"windows-sys 0.59.0",
]
[[package]]
name = "arc-swap"
version = "1.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457"
[[package]]
name = "async-stream"
version = "0.3.6"
@ -360,6 +366,46 @@ version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990"
[[package]]
name = "combine"
version = "4.6.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd"
dependencies = [
"bytes",
"memchr",
]
[[package]]
name = "comrade"
version = "0.1.0"
source = "git+https://git.hydrar.de/jmarya/comrade#00e448ac316623bb27cc3a3b799fc201cd6302d7"
dependencies = [
"comrade-macro",
"crossbeam",
"dashmap",
"env_logger 0.11.6",
"log",
"once_cell",
"rand 0.9.0",
"redis",
"serde",
"serde_json",
"tokio",
"uuid",
]
[[package]]
name = "comrade-macro"
version = "0.1.0"
source = "git+https://git.hydrar.de/jmarya/comrade#00e448ac316623bb27cc3a3b799fc201cd6302d7"
dependencies = [
"proc-macro2",
"quote",
"serde_json",
"syn",
]
[[package]]
name = "concurrent-queue"
version = "2.5.0"
@ -426,6 +472,28 @@ version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5"
[[package]]
name = "crossbeam"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8"
dependencies = [
"crossbeam-channel",
"crossbeam-deque",
"crossbeam-epoch",
"crossbeam-queue",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-channel"
version = "0.5.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06ba6d68e24814cb8de6bb986db8222d3a027d15872cabc0d18817bc3c0e4471"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-deque"
version = "0.8.6"
@ -1639,6 +1707,16 @@ dependencies = [
"winapi",
]
[[package]]
name = "num-bigint"
version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9"
dependencies = [
"num-integer",
"num-traits",
]
[[package]]
name = "num-bigint-dig"
version = "0.8.4"
@ -1774,6 +1852,7 @@ dependencies = [
"based",
"bytesize",
"chrono",
"comrade",
"env_logger 0.11.6",
"log",
"maud",
@ -2043,6 +2122,23 @@ dependencies = [
"crossbeam-utils",
]
[[package]]
name = "redis"
version = "0.29.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8034fb926579ff49d3fe58d288d5dcb580bf11e9bccd33224b45adebf0fd0c23"
dependencies = [
"arc-swap",
"combine",
"itoa",
"num-bigint",
"percent-encoding",
"ryu",
"sha1_smol",
"socket2",
"url",
]
[[package]]
name = "redox_syscall"
version = "0.5.10"
@ -2165,6 +2261,7 @@ dependencies = [
"base64 0.22.1",
"bytes",
"encoding_rs",
"futures-channel",
"futures-core",
"futures-util",
"h2 0.4.8",
@ -2544,6 +2641,12 @@ dependencies = [
"digest",
]
[[package]]
name = "sha1_smol"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d"
[[package]]
name = "sha2"
version = "0.10.8"

3
Cargo.toml
View file

@ -5,13 +5,14 @@ edition = "2024"
[dependencies]
based = { git = "https://git.hydrar.de/jmarya/based" }
comrade = { git = "https://git.hydrar.de/jmarya/comrade" }
bytesize = "1.3.0"
chrono = "0.4.39"
env_logger = "0.11.6"
log = "0.4.22"
maud = "0.26.0"
rand = "0.9.0"
reqwest = { version = "0.12.12" }
reqwest = { version = "0.12.12", features = ["blocking"] }
rocket = "0.5.1"
serde = { version = "1.0.217", features = ["derive"] }
serde_json = "1.0.134"

50
src/pkg/mirror.rs
View file

@ -1,5 +1,6 @@
use std::path::PathBuf;
use comrade::rally;
use rand::seq::SliceRandom;
use super::{Package, Repository, arch::Architecture};
@ -31,33 +32,46 @@ impl MirrorRepository {
arch.to_string()
};
let mut mirrorlist = mirrorlist.to_vec();
mirrorlist.shuffle(&mut rand::rng());
for mirror in mirrorlist {
let mirror = mirror
.replace("$repo", &self.inner.name)
.replace("$arch", &arch);
let url = format!("{mirror}/{url}");
log::info!("Trying mirror {url}");
let client = reqwest::Client::new();
if let Ok(resp) = client.get(&url).send().await {
if resp.status().is_success() {
let data = resp.bytes().await.unwrap().to_vec();
let parent = file_path.parent().unwrap();
std::fs::create_dir_all(parent).unwrap();
let mirrorlist_links = mirrorlist
.into_iter()
.take(5)
.map(|x| {
let mirror = x.replace("$repo", &self.inner.name).replace("$arch", &arch);
(
x.clone(),
format!("{mirror}/{url}"),
file_path.to_str().unwrap().to_string(),
)
})
.collect();
let data = rally(mirrorlist_links, |input| {
let (mirror, url, file_path) = input;
let client = reqwest::blocking::Client::new();
if let Ok(resp) = client.get(url).send() {
if resp.status().is_success() {
let data = resp.bytes().unwrap().to_vec();
log::info!("Downloaded {url}");
std::fs::write(file_path, data).unwrap();
return;
return Some(data);
} else {
log::warn!("Mirror {mirror} failed [{}]", resp.status().as_u16());
return None;
}
}
None
});
if let Some(data) = data.1 {
std::fs::write(file_path, data).unwrap();
} else {
log::error!("Downloading {url} failed. No mirror could provide a valid response.");
}
log::error!("Downloading {url} failed. No mirror could provide a valid response.")
}
/// Get the `.db.tar.gz` content for the repository of `arch`