diff --git a/Cargo.lock b/Cargo.lock index b8c2028..c276ef2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -116,6 +116,12 @@ dependencies = [ "bytemuck", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.4.0" @@ -172,8 +178,8 @@ dependencies = [ "rand", "rayon", "regex", - "reqwest", - "ring", + "reqwest 0.11.27", + "ring 0.16.20", "rocket", "rocket_cors", "serde", @@ -808,6 +814,25 @@ dependencies = [ "tracing", ] +[[package]] +name = "h2" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccae279728d634d083c00f6099cb58f01cc99c145b84b8be2f6c74618d79922e" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.2.0", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hashbrown" version = "0.14.5" @@ -917,6 +942,29 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http 1.2.0", +] + +[[package]] +name = "http-body-util" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" +dependencies = [ + "bytes", + "futures-util", + "http 1.2.0", + "http-body 1.0.1", + "pin-project-lite", +] + [[package]] name = "httparse" version = "1.9.5" @@ -945,9 +993,9 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2", + "h2 0.3.26", "http 0.2.12", - "http-body", + "http-body 0.4.6", "httparse", "httpdate", "itoa", @@ -959,6 +1007,43 @@ dependencies = [ "want", ] +[[package]] +name = "hyper" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "256fb8d4bd6413123cc9d91832d78325c48ff41677595be797d90f42969beae0" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "h2 0.4.7", + "http 1.2.0", + "http-body 1.0.1", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2" +dependencies = [ + "futures-util", + "http 1.2.0", + "hyper 1.5.2", + "hyper-util", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + [[package]] name = "hyper-tls" version = "0.5.0" @@ -966,12 +1051,47 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ "bytes", - "hyper", + "hyper 0.14.32", "native-tls", "tokio", "tokio-native-tls", ] +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper 1.5.2", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http 1.2.0", + "http-body 1.0.1", + "hyper 1.5.2", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + [[package]] name = "iana-time-zone" version = "0.1.61" @@ -1846,11 +1966,11 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2", + "h2 0.3.26", "http 0.2.12", - "http-body", - "hyper", - "hyper-tls", + "http-body 0.4.6", + "hyper 0.14.32", + "hyper-tls 0.5.0", "ipnet", "js-sys", "log", @@ -1859,12 +1979,12 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls-pemfile", + "rustls-pemfile 1.0.4", "serde", "serde_json", "serde_urlencoded", - "sync_wrapper", - "system-configuration", + "sync_wrapper 0.1.2", + "system-configuration 0.5.1", "tokio", "tokio-native-tls", "tower-service", @@ -1875,6 +1995,50 @@ dependencies = [ "winreg", ] +[[package]] +name = "reqwest" +version = "0.12.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fe060fe50f524be480214aba758c71f99f90ee8c83c5a36b5e9e1d568eb4eb3" +dependencies = [ + "base64 0.22.1", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2 0.4.7", + "http 1.2.0", + "http-body 1.0.1", + "http-body-util", + "hyper 1.5.2", + "hyper-rustls", + "hyper-tls 0.6.0", + "hyper-util", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls-pemfile 2.2.0", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper 1.0.2", + "system-configuration 0.6.1", + "tokio", + "tokio-native-tls", + "tower", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "windows-registry", +] + [[package]] name = "ring" version = "0.16.20" @@ -1885,11 +2049,26 @@ dependencies = [ "libc", "once_cell", "spin 0.5.2", - "untrusted", + "untrusted 0.7.1", "web-sys", "winapi", ] +[[package]] +name = "ring" +version = "0.17.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +dependencies = [ + "cc", + "cfg-if", + "getrandom", + "libc", + "spin 0.9.8", + "untrusted 0.9.0", + "windows-sys 0.52.0", +] + [[package]] name = "rocket" version = "0.5.1" @@ -1972,7 +2151,7 @@ dependencies = [ "either", "futures", "http 0.2.12", - "hyper", + "hyper 0.14.32", "indexmap", "log", "memchr", @@ -2028,6 +2207,19 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "rustls" +version = "0.23.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5065c3f250cbd332cd894be57c40fa52387247659b14a2d6041d121547903b1b" +dependencies = [ + "once_cell", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + [[package]] name = "rustls-pemfile" version = "1.0.4" @@ -2037,6 +2229,32 @@ dependencies = [ "base64 0.21.7", ] +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2bf47e6ff922db3825eb750c4e2ff784c6ff8fb9e13046ef6a1d1c5401b0b37" + +[[package]] +name = "rustls-webpki" +version = "0.102.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" +dependencies = [ + "ring 0.17.8", + "rustls-pki-types", + "untrusted 0.9.0", +] + [[package]] name = "rustversion" version = "1.0.19" @@ -2535,6 +2753,15 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + [[package]] name = "synstructure" version = "0.13.1" @@ -2554,7 +2781,18 @@ checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" dependencies = [ "bitflags 1.3.2", "core-foundation", - "system-configuration-sys", + "system-configuration-sys 0.5.0", +] + +[[package]] +name = "system-configuration" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +dependencies = [ + "bitflags 2.6.0", + "core-foundation", + "system-configuration-sys 0.6.0", ] [[package]] @@ -2567,6 +2805,16 @@ dependencies = [ "libc", ] +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tempfile" version = "3.14.0" @@ -2714,6 +2962,16 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37" +dependencies = [ + "rustls", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.17" @@ -2772,6 +3030,27 @@ dependencies = [ "winnow", ] +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper 1.0.2", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + [[package]] name = "tower-service" version = "0.3.3" @@ -2932,6 +3211,12 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" version = "2.5.4" @@ -3104,7 +3389,8 @@ dependencies = [ "maud", "rayon", "regex", - "ring", + "reqwest 0.12.11", + "ring 0.16.20", "rocket", "rocket_cors", "serde", @@ -3175,6 +3461,36 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-registry" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" +dependencies = [ + "windows-result", + "windows-strings", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-result" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-strings" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" +dependencies = [ + "windows-result", + "windows-targets 0.52.6", +] + [[package]] name = "windows-sys" version = "0.48.0" diff --git a/Cargo.toml b/Cargo.toml index 5dc3b24..1d1f20e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,3 +23,4 @@ sqlx = { version = "0.8", features = ["postgres", "runtime-tokio-native-tls", "d maud = "0.26.0" based = { git = "https://git.hydrar.de/jmarya/based", features = [] } url = "2.5.4" +reqwest = "0.12.11" diff --git a/migrations/0001_favicons.sql b/migrations/0001_favicons.sql new file mode 100644 index 0000000..544553f --- /dev/null +++ b/migrations/0001_favicons.sql @@ -0,0 +1,5 @@ + +CREATE TABLE site_favicon ( + domain VARCHAR(500) NOT NULL PRIMARY KEY, + favicon bytea NOT NULL +); diff --git a/src/favicon.rs b/src/favicon.rs new file mode 100644 index 0000000..6653f68 --- /dev/null +++ b/src/favicon.rs @@ -0,0 +1,30 @@ +use based::get_pg; + + +pub async fn download_favicon(domain: &str) -> Option> { + let mut favicon_url = url::Url::parse(&format!("https://{}", domain)).ok()?; + favicon_url.set_path("/favicon.ico"); + + log::info!("Fetching favicon from: {}", favicon_url); + + let response = reqwest::get(favicon_url).await.ok()?; + + if !response.status().is_success() { + return None; + } + + let favicon_data = response.bytes().await.ok()?.to_vec(); + + Some(favicon_data) +} + +pub async fn download_favicons_for_sites(sites: Vec) { + for site in sites { + if let Some(fav) = download_favicon(&site).await { + sqlx::query("INSERT INTO site_favicon VALUES ($1, $2)") + .bind(site) + .bind(fav) + .execute(get_pg!()).await.unwrap(); + } + } +} \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index 98d5fe8..28ec422 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,23 +4,27 @@ use rocket::routes; mod archive; mod pages; +mod favicon; #[rocket::launch] async fn launch() -> _ { env_logger::init(); - // let pg = get_pg!(); - // sqlx::migrate!("./migrations").run(pg).await.unwrap(); + let pg = get_pg!(); + sqlx::migrate!("./migrations").run(pg).await.unwrap(); let arc = WebsiteArchive::new("./websites"); + favicon::download_favicons_for_sites(arc.domains()).await; + rocket::build() .mount( "/", routes![ pages::index, pages::render_website, - pages::domain_info_route + pages::domain_info_route, + pages::favicon_route ], ) .manage(arc) diff --git a/src/pages/mod.rs b/src/pages/mod.rs index af1ce9e..74e6e40 100644 --- a/src/pages/mod.rs +++ b/src/pages/mod.rs @@ -1,11 +1,10 @@ use std::path::PathBuf; use based::{ - page::Shell, - request::{respond_html, RequestContext, StringResponse}, + get_pg, page::Shell, request::{assets::DataResponse, respond_html, RawResponse, RequestContext, StringResponse} }; use maud::{html, PreEscaped}; -use rocket::{get, State}; +use rocket::{get, Data, State}; use crate::archive::{PathEntry, WebsiteArchive}; @@ -25,6 +24,19 @@ pub async fn render_page(content: PreEscaped, ctx: RequestContext) -> St .await } +#[get("/favicon/")] +pub async fn favicon_route(domain: &str) -> Option { + let fav: Option<(Vec,)> = sqlx::query_as("SELECT favicon FROM site_favicon WHERE domain = $1") + .bind(domain) + .fetch_optional(get_pg!()).await.unwrap(); + + if let Some(fav_data) = fav { + return Some(DataResponse::new(fav_data.0, "image/x-icon", Some(60 * 60 * 24 * 5))); + } + + None +} + #[get("/")] pub async fn index(ctx: RequestContext, arc: &State) -> StringResponse { let websites = arc.domains();