update + favicon
Some checks failed
ci/woodpecker/push/build Pipeline failed

This commit is contained in:
JMARyA 2024-12-29 18:33:30 +01:00
parent 877ce477cc
commit 8f591b6b8c
Signed by: jmarya
GPG key ID: 901B2ADDF27C2263
6 changed files with 390 additions and 22 deletions

348
Cargo.lock generated
View file

@ -116,6 +116,12 @@ dependencies = [
"bytemuck",
]
[[package]]
name = "atomic-waker"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
[[package]]
name = "autocfg"
version = "1.4.0"
@ -172,8 +178,8 @@ dependencies = [
"rand",
"rayon",
"regex",
"reqwest",
"ring",
"reqwest 0.11.27",
"ring 0.16.20",
"rocket",
"rocket_cors",
"serde",
@ -808,6 +814,25 @@ dependencies = [
"tracing",
]
[[package]]
name = "h2"
version = "0.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ccae279728d634d083c00f6099cb58f01cc99c145b84b8be2f6c74618d79922e"
dependencies = [
"atomic-waker",
"bytes",
"fnv",
"futures-core",
"futures-sink",
"http 1.2.0",
"indexmap",
"slab",
"tokio",
"tokio-util",
"tracing",
]
[[package]]
name = "hashbrown"
version = "0.14.5"
@ -917,6 +942,29 @@ dependencies = [
"pin-project-lite",
]
[[package]]
name = "http-body"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184"
dependencies = [
"bytes",
"http 1.2.0",
]
[[package]]
name = "http-body-util"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f"
dependencies = [
"bytes",
"futures-util",
"http 1.2.0",
"http-body 1.0.1",
"pin-project-lite",
]
[[package]]
name = "httparse"
version = "1.9.5"
@ -945,9 +993,9 @@ dependencies = [
"futures-channel",
"futures-core",
"futures-util",
"h2",
"h2 0.3.26",
"http 0.2.12",
"http-body",
"http-body 0.4.6",
"httparse",
"httpdate",
"itoa",
@ -959,6 +1007,43 @@ dependencies = [
"want",
]
[[package]]
name = "hyper"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "256fb8d4bd6413123cc9d91832d78325c48ff41677595be797d90f42969beae0"
dependencies = [
"bytes",
"futures-channel",
"futures-util",
"h2 0.4.7",
"http 1.2.0",
"http-body 1.0.1",
"httparse",
"itoa",
"pin-project-lite",
"smallvec",
"tokio",
"want",
]
[[package]]
name = "hyper-rustls"
version = "0.27.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2"
dependencies = [
"futures-util",
"http 1.2.0",
"hyper 1.5.2",
"hyper-util",
"rustls",
"rustls-pki-types",
"tokio",
"tokio-rustls",
"tower-service",
]
[[package]]
name = "hyper-tls"
version = "0.5.0"
@ -966,12 +1051,47 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
dependencies = [
"bytes",
"hyper",
"hyper 0.14.32",
"native-tls",
"tokio",
"tokio-native-tls",
]
[[package]]
name = "hyper-tls"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0"
dependencies = [
"bytes",
"http-body-util",
"hyper 1.5.2",
"hyper-util",
"native-tls",
"tokio",
"tokio-native-tls",
"tower-service",
]
[[package]]
name = "hyper-util"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4"
dependencies = [
"bytes",
"futures-channel",
"futures-util",
"http 1.2.0",
"http-body 1.0.1",
"hyper 1.5.2",
"pin-project-lite",
"socket2",
"tokio",
"tower-service",
"tracing",
]
[[package]]
name = "iana-time-zone"
version = "0.1.61"
@ -1846,11 +1966,11 @@ dependencies = [
"encoding_rs",
"futures-core",
"futures-util",
"h2",
"h2 0.3.26",
"http 0.2.12",
"http-body",
"hyper",
"hyper-tls",
"http-body 0.4.6",
"hyper 0.14.32",
"hyper-tls 0.5.0",
"ipnet",
"js-sys",
"log",
@ -1859,12 +1979,12 @@ dependencies = [
"once_cell",
"percent-encoding",
"pin-project-lite",
"rustls-pemfile",
"rustls-pemfile 1.0.4",
"serde",
"serde_json",
"serde_urlencoded",
"sync_wrapper",
"system-configuration",
"sync_wrapper 0.1.2",
"system-configuration 0.5.1",
"tokio",
"tokio-native-tls",
"tower-service",
@ -1875,6 +1995,50 @@ dependencies = [
"winreg",
]
[[package]]
name = "reqwest"
version = "0.12.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fe060fe50f524be480214aba758c71f99f90ee8c83c5a36b5e9e1d568eb4eb3"
dependencies = [
"base64 0.22.1",
"bytes",
"encoding_rs",
"futures-core",
"futures-util",
"h2 0.4.7",
"http 1.2.0",
"http-body 1.0.1",
"http-body-util",
"hyper 1.5.2",
"hyper-rustls",
"hyper-tls 0.6.0",
"hyper-util",
"ipnet",
"js-sys",
"log",
"mime",
"native-tls",
"once_cell",
"percent-encoding",
"pin-project-lite",
"rustls-pemfile 2.2.0",
"serde",
"serde_json",
"serde_urlencoded",
"sync_wrapper 1.0.2",
"system-configuration 0.6.1",
"tokio",
"tokio-native-tls",
"tower",
"tower-service",
"url",
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
"windows-registry",
]
[[package]]
name = "ring"
version = "0.16.20"
@ -1885,11 +2049,26 @@ dependencies = [
"libc",
"once_cell",
"spin 0.5.2",
"untrusted",
"untrusted 0.7.1",
"web-sys",
"winapi",
]
[[package]]
name = "ring"
version = "0.17.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d"
dependencies = [
"cc",
"cfg-if",
"getrandom",
"libc",
"spin 0.9.8",
"untrusted 0.9.0",
"windows-sys 0.52.0",
]
[[package]]
name = "rocket"
version = "0.5.1"
@ -1972,7 +2151,7 @@ dependencies = [
"either",
"futures",
"http 0.2.12",
"hyper",
"hyper 0.14.32",
"indexmap",
"log",
"memchr",
@ -2028,6 +2207,19 @@ dependencies = [
"windows-sys 0.59.0",
]
[[package]]
name = "rustls"
version = "0.23.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5065c3f250cbd332cd894be57c40fa52387247659b14a2d6041d121547903b1b"
dependencies = [
"once_cell",
"rustls-pki-types",
"rustls-webpki",
"subtle",
"zeroize",
]
[[package]]
name = "rustls-pemfile"
version = "1.0.4"
@ -2037,6 +2229,32 @@ dependencies = [
"base64 0.21.7",
]
[[package]]
name = "rustls-pemfile"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50"
dependencies = [
"rustls-pki-types",
]
[[package]]
name = "rustls-pki-types"
version = "1.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2bf47e6ff922db3825eb750c4e2ff784c6ff8fb9e13046ef6a1d1c5401b0b37"
[[package]]
name = "rustls-webpki"
version = "0.102.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9"
dependencies = [
"ring 0.17.8",
"rustls-pki-types",
"untrusted 0.9.0",
]
[[package]]
name = "rustversion"
version = "1.0.19"
@ -2535,6 +2753,15 @@ version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
[[package]]
name = "sync_wrapper"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263"
dependencies = [
"futures-core",
]
[[package]]
name = "synstructure"
version = "0.13.1"
@ -2554,7 +2781,18 @@ checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7"
dependencies = [
"bitflags 1.3.2",
"core-foundation",
"system-configuration-sys",
"system-configuration-sys 0.5.0",
]
[[package]]
name = "system-configuration"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b"
dependencies = [
"bitflags 2.6.0",
"core-foundation",
"system-configuration-sys 0.6.0",
]
[[package]]
@ -2567,6 +2805,16 @@ dependencies = [
"libc",
]
[[package]]
name = "system-configuration-sys"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4"
dependencies = [
"core-foundation-sys",
"libc",
]
[[package]]
name = "tempfile"
version = "3.14.0"
@ -2714,6 +2962,16 @@ dependencies = [
"tokio",
]
[[package]]
name = "tokio-rustls"
version = "0.26.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37"
dependencies = [
"rustls",
"tokio",
]
[[package]]
name = "tokio-stream"
version = "0.1.17"
@ -2772,6 +3030,27 @@ dependencies = [
"winnow",
]
[[package]]
name = "tower"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9"
dependencies = [
"futures-core",
"futures-util",
"pin-project-lite",
"sync_wrapper 1.0.2",
"tokio",
"tower-layer",
"tower-service",
]
[[package]]
name = "tower-layer"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e"
[[package]]
name = "tower-service"
version = "0.3.3"
@ -2932,6 +3211,12 @@ version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
[[package]]
name = "untrusted"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
[[package]]
name = "url"
version = "2.5.4"
@ -3104,7 +3389,8 @@ dependencies = [
"maud",
"rayon",
"regex",
"ring",
"reqwest 0.12.11",
"ring 0.16.20",
"rocket",
"rocket_cors",
"serde",
@ -3175,6 +3461,36 @@ dependencies = [
"windows-targets 0.52.6",
]
[[package]]
name = "windows-registry"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0"
dependencies = [
"windows-result",
"windows-strings",
"windows-targets 0.52.6",
]
[[package]]
name = "windows-result"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e"
dependencies = [
"windows-targets 0.52.6",
]
[[package]]
name = "windows-strings"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10"
dependencies = [
"windows-result",
"windows-targets 0.52.6",
]
[[package]]
name = "windows-sys"
version = "0.48.0"

View file

@ -23,3 +23,4 @@ sqlx = { version = "0.8", features = ["postgres", "runtime-tokio-native-tls", "d
maud = "0.26.0"
based = { git = "https://git.hydrar.de/jmarya/based", features = [] }
url = "2.5.4"
reqwest = "0.12.11"

View file

@ -0,0 +1,5 @@
CREATE TABLE site_favicon (
domain VARCHAR(500) NOT NULL PRIMARY KEY,
favicon bytea NOT NULL
);

30
src/favicon.rs Normal file
View file

@ -0,0 +1,30 @@
use based::get_pg;
pub async fn download_favicon(domain: &str) -> Option<Vec<u8>> {
let mut favicon_url = url::Url::parse(&format!("https://{}", domain)).ok()?;
favicon_url.set_path("/favicon.ico");
log::info!("Fetching favicon from: {}", favicon_url);
let response = reqwest::get(favicon_url).await.ok()?;
if !response.status().is_success() {
return None;
}
let favicon_data = response.bytes().await.ok()?.to_vec();
Some(favicon_data)
}
pub async fn download_favicons_for_sites(sites: Vec<String>) {
for site in sites {
if let Some(fav) = download_favicon(&site).await {
sqlx::query("INSERT INTO site_favicon VALUES ($1, $2)")
.bind(site)
.bind(fav)
.execute(get_pg!()).await.unwrap();
}
}
}

View file

@ -4,23 +4,27 @@ use rocket::routes;
mod archive;
mod pages;
mod favicon;
#[rocket::launch]
async fn launch() -> _ {
env_logger::init();
// let pg = get_pg!();
// sqlx::migrate!("./migrations").run(pg).await.unwrap();
let pg = get_pg!();
sqlx::migrate!("./migrations").run(pg).await.unwrap();
let arc = WebsiteArchive::new("./websites");
favicon::download_favicons_for_sites(arc.domains()).await;
rocket::build()
.mount(
"/",
routes![
pages::index,
pages::render_website,
pages::domain_info_route
pages::domain_info_route,
pages::favicon_route
],
)
.manage(arc)

View file

@ -1,11 +1,10 @@
use std::path::PathBuf;
use based::{
page::Shell,
request::{respond_html, RequestContext, StringResponse},
get_pg, page::Shell, request::{assets::DataResponse, respond_html, RawResponse, RequestContext, StringResponse}
};
use maud::{html, PreEscaped};
use rocket::{get, State};
use rocket::{get, Data, State};
use crate::archive::{PathEntry, WebsiteArchive};
@ -25,6 +24,19 @@ pub async fn render_page(content: PreEscaped<String>, ctx: RequestContext) -> St
.await
}
#[get("/favicon/<domain>")]
pub async fn favicon_route(domain: &str) -> Option<DataResponse> {
let fav: Option<(Vec<u8>,)> = sqlx::query_as("SELECT favicon FROM site_favicon WHERE domain = $1")
.bind(domain)
.fetch_optional(get_pg!()).await.unwrap();
if let Some(fav_data) = fav {
return Some(DataResponse::new(fav_data.0, "image/x-icon", Some(60 * 60 * 24 * 5)));
}
None
}
#[get("/")]
pub async fn index(ctx: RequestContext, arc: &State<WebsiteArchive>) -> StringResponse {
let websites = arc.domains();