✨ parallel downloads

Cargo.lock

@@ -97,6 +97,12 @@ dependencies = [
  "windows-sys 0.59.0",
 ]
 
+[[package]]
+name = "arc-swap"
+version = "1.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457"
+
 [[package]]
 name = "async-stream"
 version = "0.3.6"
@@ -360,6 +366,46 @@ version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990"
 
+[[package]]
+name = "combine"
+version = "4.6.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd"
+dependencies = [
+ "bytes",
+ "memchr",
+]
+
+[[package]]
+name = "comrade"
+version = "0.1.0"
+source = "git+https://git.hydrar.de/jmarya/comrade#00e448ac316623bb27cc3a3b799fc201cd6302d7"
+dependencies = [
+ "comrade-macro",
+ "crossbeam",
+ "dashmap",
+ "env_logger 0.11.6",
+ "log",
+ "once_cell",
+ "rand 0.9.0",
+ "redis",
+ "serde",
+ "serde_json",
+ "tokio",
+ "uuid",
+]
+
+[[package]]
+name = "comrade-macro"
+version = "0.1.0"
+source = "git+https://git.hydrar.de/jmarya/comrade#00e448ac316623bb27cc3a3b799fc201cd6302d7"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "serde_json",
+ "syn",
+]
+
 [[package]]
 name = "concurrent-queue"
 version = "2.5.0"
@@ -426,6 +472,28 @@ version = "2.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5"
 
+[[package]]
+name = "crossbeam"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8"
+dependencies = [
+ "crossbeam-channel",
+ "crossbeam-deque",
+ "crossbeam-epoch",
+ "crossbeam-queue",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-channel"
+version = "0.5.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "06ba6d68e24814cb8de6bb986db8222d3a027d15872cabc0d18817bc3c0e4471"
+dependencies = [
+ "crossbeam-utils",
+]
+
 [[package]]
 name = "crossbeam-deque"
 version = "0.8.6"
@@ -1639,6 +1707,16 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "num-bigint"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9"
+dependencies = [
+ "num-integer",
+ "num-traits",
+]
+
 [[package]]
 name = "num-bigint-dig"
 version = "0.8.4"
@@ -1774,6 +1852,7 @@ dependencies = [
  "based",
  "bytesize",
  "chrono",
+ "comrade",
  "env_logger 0.11.6",
  "log",
  "maud",
@@ -2043,6 +2122,23 @@ dependencies = [
  "crossbeam-utils",
 ]
 
+[[package]]
+name = "redis"
+version = "0.29.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8034fb926579ff49d3fe58d288d5dcb580bf11e9bccd33224b45adebf0fd0c23"
+dependencies = [
+ "arc-swap",
+ "combine",
+ "itoa",
+ "num-bigint",
+ "percent-encoding",
+ "ryu",
+ "sha1_smol",
+ "socket2",
+ "url",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.5.10"
@@ -2165,6 +2261,7 @@ dependencies = [
  "base64 0.22.1",
  "bytes",
  "encoding_rs",
+ "futures-channel",
  "futures-core",
  "futures-util",
  "h2 0.4.8",
@@ -2544,6 +2641,12 @@ dependencies = [
  "digest",
 ]
 
+[[package]]
+name = "sha1_smol"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d"
+
 [[package]]
 name = "sha2"
 version = "0.10.8"

Cargo.toml

@@ -5,13 +5,14 @@ edition = "2024"
 
 [dependencies]
 based = { git = "https://git.hydrar.de/jmarya/based" }
+comrade = { git = "https://git.hydrar.de/jmarya/comrade" }
 bytesize = "1.3.0"
 chrono = "0.4.39"
 env_logger = "0.11.6"
 log = "0.4.22"
 maud = "0.26.0"
 rand = "0.9.0"
-reqwest = { version = "0.12.12" }
+reqwest = { version = "0.12.12", features = ["blocking"] }
 rocket = "0.5.1"
 serde = { version = "1.0.217", features = ["derive"] }
 serde_json = "1.0.134"

src/pkg/mirror.rs

@@ -1,5 +1,6 @@
 use std::path::PathBuf;
 
+use comrade::rally;
 use rand::seq::SliceRandom;
 
 use super::{Package, Repository, arch::Architecture};
@@ -31,33 +32,46 @@ impl MirrorRepository {
             arch.to_string()
         };
 
-        let mut mirrorlist = mirrorlist.to_vec();
+        let parent = file_path.parent().unwrap();
-        mirrorlist.shuffle(&mut rand::rng());
+        std::fs::create_dir_all(parent).unwrap();
 
-        for mirror in mirrorlist {
+        let mirrorlist_links = mirrorlist
-            let mirror = mirror
+            .into_iter()
-                .replace("$repo", &self.inner.name)
+            .take(5)
-                .replace("$arch", &arch);
+            .map(|x| {
+                let mirror = x.replace("$repo", &self.inner.name).replace("$arch", &arch);
 
-            let url = format!("{mirror}/{url}");
+                (
-            log::info!("Trying mirror {url}");
+                    x.clone(),
+                    format!("{mirror}/{url}"),
+                    file_path.to_str().unwrap().to_string(),
+                )
+            })
+            .collect();
 
-            let client = reqwest::Client::new();
+        let data = rally(mirrorlist_links, |input| {
-            if let Ok(resp) = client.get(&url).send().await {
+            let (mirror, url, file_path) = input;
+
+            let client = reqwest::blocking::Client::new();
+            if let Ok(resp) = client.get(url).send() {
                 if resp.status().is_success() {
-                    let data = resp.bytes().await.unwrap().to_vec();
+                    let data = resp.bytes().unwrap().to_vec();
-                    let parent = file_path.parent().unwrap();
-                    std::fs::create_dir_all(parent).unwrap();
-
                     log::info!("Downloaded {url}");
-                    std::fs::write(file_path, data).unwrap();
+                    return Some(data);
-                    return;
                 } else {
                     log::warn!("Mirror {mirror} failed [{}]", resp.status().as_u16());
+                    return None;
                 }
             }
+
+            None
+        });
+
+        if let Some(data) = data.1 {
+            std::fs::write(file_path, data).unwrap();
+        } else {
+            log::error!("Downloading {url} failed. No mirror could provide a valid response.");
         }
-        log::error!("Downloading {url} failed. No mirror could provide a valid response.")
     }
 
     /// Get the `.db.tar.gz` content for the repository of `arch`