add on demand download

docker-compose.yml

@@ -11,6 +11,10 @@ services:
       - "DATABASE_URL=postgres://user:pass@postgres/webarc"
       - "RUST_LOG=info"
       - "ROCKET_ADDRESS=0.0.0.0"
+      # Rewrite links to point back to the archive itself
+      - "ROUTE_INTERNAL=true"
+      # Download missing routes on demand
+      - "DOWNLOAD_ON_DEMAND=true"
 
   postgres:
     image: timescale/timescaledb:latest-pg16

src/archive.rs

@@ -1,18 +1,25 @@
-use std::{fmt::format, fs::read_to_string, path::{Path, PathBuf}};
+use std::{
+    fmt::format,
+    fs::read_to_string,
+    path::{Path, PathBuf},
+};
 
 pub struct WebsiteArchive {
-    pub dir: PathBuf
+    pub dir: PathBuf,
 }
 
 pub struct Domain {
     pub name: String,
-    dir: PathBuf
+    dir: PathBuf,
 }
 
 impl Domain {
     pub fn new(name: &str, dir: PathBuf) -> Self {
         std::fs::create_dir_all(&dir).unwrap();
-        Self { name: name.to_string(), dir }
+        Self {
+            name: name.to_string(),
+            dir,
+        }
     }
 
     pub fn path(&self, path: &str) -> Document {
@@ -23,15 +30,19 @@ impl Domain {
 pub struct Document {
     pub domain: String,
     pub path: String,
-    base_dir: PathBuf
+    base_dir: PathBuf,
 }
 
 impl Document {
     pub fn new(domain: &str, path: &str, base_dir: PathBuf) -> Self {
-        Self { domain: domain.to_string(), path: path.to_string(), base_dir }
+        Self {
+            domain: domain.to_string(),
+            path: path.to_string(),
+            base_dir,
+        }
     }
 
-    pub fn render_local(&self, version: Option<String>) -> String {
+    pub fn render_local(&self, version: Option<String>) -> Option<String> {
         let mut file_path = self.base_dir.join(&self.domain);
 
         for p in self.path.split('/') {
@@ -42,18 +53,18 @@ impl Document {
             format!("index_{version}.html")
         } else {
             let versions = Self::versions(&file_path);
-            versions.first().cloned().unwrap()
+            versions.first().cloned()?
         };
 
         file_path = file_path.join(latest_version);
 
         // TODO : Replace links with local ones
-        return std::fs::read_to_string(file_path).unwrap();
+        return std::fs::read_to_string(file_path).ok();
     }
 
     pub fn versions(path: &PathBuf) -> Vec<String> {
         let mut version_list = Vec::new();
-    
+
         if let Ok(entries) = std::fs::read_dir(path) {
             for entry in entries {
                 if let Ok(entry) = entry {
@@ -63,14 +74,16 @@ impl Document {
                 }
             }
         }
-    
+
         version_list
     }
 }
 
 impl WebsiteArchive {
     pub fn new(dir: &str) -> Self {
-        Self { dir: PathBuf::from(dir) }
+        Self {
+            dir: PathBuf::from(dir),
+        }
     }
 
     pub fn get_domain(&self, domain: &str) -> Domain {
@@ -96,13 +109,13 @@ impl WebsiteArchive {
 
         let timestamp = chrono::Utc::now().format("%Y-%m-%d").to_string();
         let filename = folder_name.join(&format!("index_{timestamp}.html"));
-        
+
         run_command(&vec![
             "monolith",
             "-I",
             "-o",
             filename.to_str().unwrap(),
-            &format!("https://{}/{}", domain, path)
+            &format!("https://{}/{}", domain, path),
         ]);
     }
 }
@@ -112,7 +125,6 @@ impl WebsiteArchive {
 // transparent auto page downloading
 // redownload after threshold
 
-
 fn run_command(cmd: &[&str]) {
     let mut cmd_setup = std::process::Command::new(cmd[0].clone());
     let cmd_setup = cmd_setup.args(cmd.into_iter().skip(1).collect::<Vec<_>>());

src/main.rs

@@ -2,8 +2,8 @@ use archive::WebsiteArchive;
 use based::get_pg;
 use rocket::routes;
 
-mod pages;
 mod archive;
+mod pages;
 
 #[rocket::launch]
 async fn launch() -> _ {
@@ -15,12 +15,6 @@ async fn launch() -> _ {
     let arc = WebsiteArchive::new("./websites");
 
     rocket::build()
-        .mount(
-            "/",
-            routes![
-                pages::index,
-                pages::render_website
-            ],
-        )
+        .mount("/", routes![pages::index, pages::render_website])
         .manage(arc)
 }

src/pages/mod.rs

@@ -13,10 +13,37 @@ pub async fn index() -> StringResponse {
 }
 
 #[get("/s/<domain>/<path..>?<time>")]
-pub async fn render_website(domain: &str, path: PathBuf, time: Option<&str>, arc: &State<WebsiteArchive>) -> StringResponse {
-    if let Some(time) = time {
-        respond_html(&arc.get_domain(domain).path(path.to_str().unwrap()).render_local(Some(time.to_string())))
+pub async fn render_website(
+    domain: &str,
+    path: PathBuf,
+    time: Option<&str>,
+    arc: &State<WebsiteArchive>,
+) -> Option<StringResponse> {
+    let document = arc.get_domain(domain).path(path.to_str().unwrap());
+
+    let content = document.render_local(if time.is_some() {
+        Some(time.unwrap().to_string())
     } else {
-        respond_html(&arc.get_domain(domain).path(path.to_str().unwrap()).render_local(None))
+        None
+    });
+
+    if let Some(content) = content {
+        return Some(respond_html(&content));
+    } else {
+        if std::env::var("DOWNLOAD_ON_DEMAND")
+            .unwrap_or("false".to_string())
+            .as_str()
+            == "true"
+        {
+            arc.archive_url(&format!("https://{domain}/{}", path.to_str().unwrap()));
+
+            return Some(respond_html(&document.render_local(if time.is_some() {
+                Some(time.unwrap().to_string())
+            } else {
+                None
+            })?));
+        }
     }
+
+    None
 }