From 3e4e3820ff14b1f9910ff573a61660a6650a35e4 Mon Sep 17 00:00:00 2001 From: JMARyA Date: Mon, 11 Mar 2024 11:15:14 +0100 Subject: [PATCH 01/10] add tracking for new downloads --- src/db.rs | 53 ++++++++++++++++++++++++++++++++++++++++++++--- src/yt_dlp/mod.rs | 1 + 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/src/db.rs b/src/db.rs index 4040d71..cb1cce2 100644 --- a/src/db.rs +++ b/src/db.rs @@ -1,5 +1,5 @@ use jobdispatcher::{JobDispatcher, JobOrder}; -use rusqlite::Connection; +use rusqlite::{Connection, OptionalExtension}; use std::sync::{mpsc::Receiver, Arc}; pub struct DatabaseBackend { @@ -24,6 +24,18 @@ impl DatabaseBackend { ) .unwrap(); + conn.execute( + "CREATE TABLE IF NOT EXISTS item_log ( + id INTEGER PRIMARY KEY, + module TEXT NOT NULL, + name TEXT NOT NULL, + url TEXT NOT NULL, + timestamp TEXT NOT NULL + )", + [], + ) + .unwrap(); + let dispatcher = Arc::new(dispatcher); Self { file: file.to_string(), @@ -51,13 +63,39 @@ impl DatabaseBackend { job.done(Out::Ok); } Query::CheckForUrl(ref url) => { - let conn = Connection::open(&self.file).unwrap(); - let mut stmt = conn + let mut stmt = self + .conn .prepare("SELECT COUNT(*) FROM urls WHERE url = ?") .unwrap(); let count: i64 = stmt.query_row([url], |row| row.get(0)).unwrap(); job.done(Out::Bool(count > 0)); } + Query::UpdateNewDownloads(ref module, ref name, ref url) => { + let timestamp = chrono::Local::now().to_rfc3339(); + + // Check if the entry exists + let existing_timestamp: Option = self.conn.query_row( + "SELECT timestamp FROM item_log WHERE module = ? AND name = ? AND url = ?", + [module, name, url], + |row| row.get(0) + ).optional().unwrap(); + + if existing_timestamp.is_some() { + // Entry exists, update timestamp + self.conn.execute( + "UPDATE item_log SET timestamp = ? WHERE module = ? AND name = ? AND url = ?", + [×tamp, module, name, url] + ).unwrap(); + } else { + // Entry doesn't exist, insert new row + self.conn.execute( + "INSERT INTO item_log (module, name, url, timestamp) VALUES (?, ?, ?, ?)", + [module, name, url, ×tamp] + ).unwrap(); + } + + job.done(Out::Ok); + } } } } @@ -66,6 +104,7 @@ impl DatabaseBackend { pub enum Query { InsertUrl(String), CheckForUrl(String), + UpdateNewDownloads(String, String, String), } pub enum Out { @@ -94,4 +133,12 @@ impl Database { Out::Bool(b) => b, } } + + pub fn update_new_downloads(&self, module: &str, name: &str, url: &str) { + self.conn.send(Query::UpdateNewDownloads( + module.to_string(), + name.to_string(), + url.to_string(), + )); + } } diff --git a/src/yt_dlp/mod.rs b/src/yt_dlp/mod.rs index eaf68e4..8d2e687 100644 --- a/src/yt_dlp/mod.rs +++ b/src/yt_dlp/mod.rs @@ -99,6 +99,7 @@ impl Module for YtDlpModule { Ok(()) => { // mark as downloaded self.db.insert_url(&video_url); + self.db.update_new_downloads(&self.name(), item, item_url); log::info!("Downloaded \"{video_title}\""); } Err(e) => { From c2ddb4a738dfea1bf6b1e9705a84c4b6922f6f84 Mon Sep 17 00:00:00 2001 From: JMARyA Date: Thu, 14 Mar 2024 12:27:44 +0100 Subject: [PATCH 02/10] non root user --- Dockerfile | 3 ++- entrypoint.sh | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 entrypoint.sh diff --git a/Dockerfile b/Dockerfile index b824933..0c4588f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,7 +22,8 @@ RUN pacman -Sy --noconfirm archlinux-keyring && \ python-mutagen COPY --from=builder /app/target/release/hoard /hoard +COPY ./entrypoint.sh /entrypoint.sh WORKDIR / -CMD ["/hoard"] +CMD ["/bin/bash", "/entrypoint.sh"] diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100644 index 0000000..835dde2 --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +echo "Creating User ${UID:-1000}" +usermod -u "${UID:-1000}" hoard || exit 1 + +chown -R hoard /downloads +mkdir /.cache && chown -R hoard /.cache +chown -R hoard /download.db + +su hoard -c /hoard From 815345dc4fe6d4afa515045cd39dfef4ddbf3772 Mon Sep 17 00:00:00 2001 From: JMARyA Date: Thu, 14 Mar 2024 12:31:49 +0100 Subject: [PATCH 03/10] fix --- entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/entrypoint.sh b/entrypoint.sh index 835dde2..76450c1 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,7 +1,7 @@ #!/bin/bash echo "Creating User ${UID:-1000}" -usermod -u "${UID:-1000}" hoard || exit 1 +useradd -m -u "${UID:-1000}" hoard || exit 1 chown -R hoard /downloads mkdir /.cache && chown -R hoard /.cache From fc0d7c030719e66ba01525a14319a49621688c41 Mon Sep 17 00:00:00 2001 From: JMARyA Date: Thu, 14 Mar 2024 12:41:57 +0100 Subject: [PATCH 04/10] fix db --- docker-compose.yml | 2 +- entrypoint.sh | 4 ++-- src/main.rs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 5e78497..7563e30 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,5 +7,5 @@ services: TZ: Europe/Berlin volumes: - ./download:/download - - ./download.db:/download.db + - ./data:/data - ./config.toml:/config.toml diff --git a/entrypoint.sh b/entrypoint.sh index 76450c1..3e4d65c 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -3,8 +3,8 @@ echo "Creating User ${UID:-1000}" useradd -m -u "${UID:-1000}" hoard || exit 1 -chown -R hoard /downloads +chown -R hoard /download mkdir /.cache && chown -R hoard /.cache -chown -R hoard /download.db +chown -R hoard /data su hoard -c /hoard diff --git a/src/main.rs b/src/main.rs index 2e77477..5b08b11 100644 --- a/src/main.rs +++ b/src/main.rs @@ -42,7 +42,7 @@ fn main() { log::info!("Starting hoard"); - let db = db::DatabaseBackend::new("download.db"); + let db = db::DatabaseBackend::new("data/download.db"); let config: GlobalConfig = toml::from_str(&std::fs::read_to_string("config.toml").unwrap()).unwrap(); ensure_dir_exists(&config.hoard.data_dir); From ebff54a8ba4e22f049058580212c47c8667c75b8 Mon Sep 17 00:00:00 2001 From: JMARyA Date: Sun, 17 Mar 2024 00:42:36 +0100 Subject: [PATCH 05/10] add webhook support --- Cargo.lock | 821 +++++++++++++++++++++++++++++++++++++++++- Cargo.toml | 3 +- src/soundcloud/mod.rs | 3 + src/youtube/mod.rs | 3 + src/yt_dlp/mod.rs | 36 ++ 5 files changed, 852 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a817457..e4ff4ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,21 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + [[package]] name = "ahash" version = "0.8.7" @@ -79,7 +94,7 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" dependencies = [ - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -89,7 +104,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" dependencies = [ "anstyle", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -98,6 +113,33 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "backtrace" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.4.2" @@ -110,6 +152,12 @@ version = "3.15.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ea184aa71bb362a1157c896979544cc23974e08fd265f29ea96b59f0b4a555b" +[[package]] +name = "bytes" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" + [[package]] name = "cc" version = "1.0.90" @@ -133,7 +181,7 @@ dependencies = [ "js-sys", "num-traits", "wasm-bindgen", - "windows-targets", + "windows-targets 0.52.4", ] [[package]] @@ -142,12 +190,31 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +[[package]] +name = "encoding_rs" +version = "0.8.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" +dependencies = [ + "cfg-if", +] + [[package]] name = "env_filter" version = "0.1.0" @@ -177,6 +244,16 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +[[package]] +name = "errno" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "fallible-iterator" version = "0.3.0" @@ -189,6 +266,115 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" +[[package]] +name = "fastrand" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures-channel" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" + +[[package]] +name = "futures-io" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" + +[[package]] +name = "futures-sink" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" + +[[package]] +name = "futures-task" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" + +[[package]] +name = "futures-util" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +dependencies = [ + "futures-core", + "futures-io", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "gimli" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" + +[[package]] +name = "h2" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fbd2820c5e49886948654ab546d0688ff24530286bdcf8fca3cefb16d4618eb" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hashbrown" version = "0.14.3" @@ -216,18 +402,90 @@ dependencies = [ "env_logger", "jobdispatcher", "log", + "reqwest", "rusqlite", "serde", "serde_json", "toml", ] +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + [[package]] name = "humantime" version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "hyper" +version = "0.14.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "iana-time-zone" version = "0.1.60" @@ -251,6 +509,16 @@ dependencies = [ "cc", ] +[[package]] +name = "idna" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "indexmap" version = "2.2.2" @@ -261,6 +529,12 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "ipnet" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" + [[package]] name = "itoa" version = "1.0.10" @@ -281,6 +555,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + [[package]] name = "libc" version = "0.2.153" @@ -297,6 +577,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linux-raw-sys" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" + [[package]] name = "log" version = "0.4.21" @@ -309,6 +595,50 @@ version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" +dependencies = [ + "adler", +] + +[[package]] +name = "mio" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.48.0", +] + +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "num-traits" version = "0.2.18" @@ -318,12 +648,83 @@ dependencies = [ "autocfg", ] +[[package]] +name = "object" +version = "0.32.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +[[package]] +name = "openssl" +version = "0.10.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" +dependencies = [ + "bitflags 2.4.2", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dda2b0f344e78efc2facf7d195d098df0dd72151b26ab98da807afc26c198dff" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pin-project-lite" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "pkg-config" version = "0.3.29" @@ -377,13 +778,53 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" +[[package]] +name = "reqwest" +version = "0.11.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78bf93c4af7a8bb7d879d51cebe797356ff10ae8516ace542b5182d9dcac10b2" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "system-configuration", + "tokio", + "tokio-native-tls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + [[package]] name = "rusqlite" version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a78046161564f5e7cd9008aff3b2990b3850dc8e0349119b98e8f251e099f24d" dependencies = [ - "bitflags", + "bitflags 2.4.2", "fallible-iterator", "fallible-streaming-iterator", "hashlink", @@ -391,12 +832,72 @@ dependencies = [ "smallvec", ] +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + +[[package]] +name = "rustix" +version = "0.38.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" +dependencies = [ + "bitflags 2.4.2", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64", +] + [[package]] name = "ryu" version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" +[[package]] +name = "schannel" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "security-framework" +version = "2.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "serde" version = "1.0.196" @@ -437,12 +938,43 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + [[package]] name = "smallvec" version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" +[[package]] +name = "socket2" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "syn" version = "2.0.48" @@ -454,6 +986,99 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tempfile" +version = "3.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +dependencies = [ + "cfg-if", + "fastrand", + "rustix", + "windows-sys 0.52.0", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "windows-sys 0.48.0", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + [[package]] name = "toml" version = "0.8.10" @@ -488,12 +1113,69 @@ dependencies = [ "winnow", ] +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tracing" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "unicode-bidi" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" + [[package]] name = "unicode-ident" version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +[[package]] +name = "unicode-normalization" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "url" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + [[package]] name = "utf8parse" version = "0.2.1" @@ -512,6 +1194,21 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + [[package]] name = "wasm-bindgen" version = "0.2.92" @@ -537,6 +1234,18 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.92" @@ -566,13 +1275,32 @@ version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" +[[package]] +name = "web-sys" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "windows-core" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets", + "windows-targets 0.52.4", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", ] [[package]] @@ -581,7 +1309,22 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets", + "windows-targets 0.52.4", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", ] [[package]] @@ -590,51 +1333,93 @@ version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.52.4", + "windows_aarch64_msvc 0.52.4", + "windows_i686_gnu 0.52.4", + "windows_i686_msvc 0.52.4", + "windows_x86_64_gnu 0.52.4", + "windows_x86_64_gnullvm 0.52.4", + "windows_x86_64_msvc 0.52.4", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + [[package]] name = "windows_aarch64_msvc" version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + [[package]] name = "windows_i686_gnu" version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + [[package]] name = "windows_i686_msvc" version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + [[package]] name = "windows_x86_64_gnu" version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + [[package]] name = "windows_x86_64_msvc" version = "0.52.4" @@ -650,6 +1435,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + [[package]] name = "zerocopy" version = "0.7.32" diff --git a/Cargo.toml b/Cargo.toml index 8bdce57..dc8f1eb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,4 +13,5 @@ rusqlite = "0.30.0" serde = { version = "1.0.196", features = ["derive"] } serde_json = "1.0.113" toml = "0.8.10" -jobdispatcher = { git = "https://git.hydrar.de/jmarya/jobdispatcher" } \ No newline at end of file +jobdispatcher = { git = "https://git.hydrar.de/jmarya/jobdispatcher" } +reqwest = { version = "0.11.26", features = ["blocking", "json"] } diff --git a/src/soundcloud/mod.rs b/src/soundcloud/mod.rs index cb67168..37e2e00 100644 --- a/src/soundcloud/mod.rs +++ b/src/soundcloud/mod.rs @@ -42,6 +42,8 @@ pub struct SoundCloudConfig { pub format: Option, // Cookie File pub cookie: Option, + // Webhooks for notifications + pub webhooks: Option>, } #[derive(Clone)] @@ -75,6 +77,7 @@ impl SoundCloudModule { format: config.format, cookie: config.cookie, audio_only: Some(true), + webhooks: config.webhooks, }, db, root_dir, diff --git a/src/youtube/mod.rs b/src/youtube/mod.rs index 0754e4a..bb6cc9b 100644 --- a/src/youtube/mod.rs +++ b/src/youtube/mod.rs @@ -46,6 +46,8 @@ pub struct YouTubeConfig { pub format: Option, // Cookie File pub cookie: Option, + // Webhooks for notifications + pub webhooks: Option>, } #[derive(Clone)] @@ -79,6 +81,7 @@ impl YouTubeModule { format: config.format, cookie: config.cookie, audio_only: Some(false), + webhooks: config.webhooks, }, db, root_dir, diff --git a/src/yt_dlp/mod.rs b/src/yt_dlp/mod.rs index 8d2e687..5d7a81b 100644 --- a/src/yt_dlp/mod.rs +++ b/src/yt_dlp/mod.rs @@ -54,6 +54,8 @@ pub struct YtDlpConfig { pub format: Option, // Cookie File pub cookie: Option, + // Webhooks for notifications + pub webhooks: Option>, } #[derive(Clone)] @@ -101,11 +103,13 @@ impl Module for YtDlpModule { self.db.insert_url(&video_url); self.db.update_new_downloads(&self.name(), item, item_url); log::info!("Downloaded \"{video_title}\""); + self.webhook_notify(&video_url, &video_title, item, true); } Err(e) => { log::error!( "Error downloading \"{video_title}\"; Reason: {e}" ); + self.webhook_notify(&video_url, &video_title, item, false); } } } @@ -127,6 +131,38 @@ impl Module for YtDlpModule { } impl YtDlpModule { + pub fn webhook_notify(&self, video_url: &str, video_title: &str, item: &str, success: bool) { + let request = serde_json::json!({ + "module": self.name(), + "url": video_url, + "title": video_title, + "item": item, + "success": success + }); + + let client = reqwest::blocking::Client::new(); + if let Some(webhooks) = &self.config.webhooks { + for url in webhooks { + client + .post(url) + .json(&request) + .send() + .expect("Failed to send webhook request"); + } + } + } + + /// A function to get the latest entries (title and URL) for a given channel with a specified limit. + /// + /// # Arguments + /// + /// * `channel` - The name of the `YouTube` channel. + /// * `limit` - The maximum number of entries to return. + /// + /// # Returns + /// + /// A `Result` containing a vector of tuples if successful, where each tuple contains the title and URL of an entry. + /// An error message if execution of `yt-dlp` fails. fn get_latest_entries(channel: &str, limit: u64) -> Result, String> { let output = Command::new("yt-dlp") .arg("--no-warnings") From 2cfd3b4f54ca60b61a1042b16114a65643957669 Mon Sep 17 00:00:00 2001 From: JMARyA Date: Sun, 5 May 2024 16:29:20 +0200 Subject: [PATCH 06/10] add category folders for yt-dlp --- src/soundcloud/mod.rs | 2 +- src/youtube/mod.rs | 2 +- src/yt_dlp/mod.rs | 75 ++++++++++++++++++++++++++----------------- 3 files changed, 48 insertions(+), 31 deletions(-) diff --git a/src/soundcloud/mod.rs b/src/soundcloud/mod.rs index 37e2e00..d90957e 100644 --- a/src/soundcloud/mod.rs +++ b/src/soundcloud/mod.rs @@ -15,7 +15,7 @@ pub struct SoundCloudConfig { /// Amount of items to query pub limit: Option, // Items to check - pub artists: HashMap, + pub artists: HashMap, // Output Template for yt-dlp pub output_format: Option, // Download comments diff --git a/src/youtube/mod.rs b/src/youtube/mod.rs index bb6cc9b..a9a606f 100644 --- a/src/youtube/mod.rs +++ b/src/youtube/mod.rs @@ -15,7 +15,7 @@ pub struct YouTubeConfig { /// Amount of videos to query limit: Option, // Channels to check - channels: HashMap, + channels: HashMap, // Format of the Thumbnail thumbnail_format: Option, // Output Template for yt-dlp diff --git a/src/yt_dlp/mod.rs b/src/yt_dlp/mod.rs index 5d7a81b..986d198 100644 --- a/src/yt_dlp/mod.rs +++ b/src/yt_dlp/mod.rs @@ -19,7 +19,7 @@ pub struct YtDlpConfig { /// Amount of items to query pub limit: Option, // Items to check - pub items: HashMap, + pub items: HashMap, // Format of the Thumbnail pub thumbnail_format: Option, // Output Template for yt-dlp @@ -73,6 +73,36 @@ impl YtDlpModule { root_dir, } } + + fn check_item(&self, item: &str, item_url: &str, cwd: &PathBuf) { + log::info!("Fetching \"{item}\" videos"); + match Self::get_latest_entries(item_url, self.config.limit.unwrap_or(10)) { + Ok(latest_videos) => { + for (video_title, video_url) in latest_videos { + if self.db.check_for_url(&video_url) { + log::trace!("Skipping \"{video_title}\" because it was already downloaded"); + } else { + match self.download(&video_url, cwd) { + Ok(()) => { + // mark as downloaded + self.db.insert_url(&video_url); + self.db.update_new_downloads(&self.name(), item, item_url); + log::info!("Downloaded \"{video_title}\""); + self.webhook_notify(&video_url, &video_title, item, true); + } + Err(e) => { + log::error!("Error downloading \"{video_title}\"; Reason: {e}"); + self.webhook_notify(&video_url, &video_title, item, false); + } + } + } + } + } + Err(e) => { + log::error!("Could not get videos from \"{item}\". Reason: {e}"); + } + } + } } impl Module for YtDlpModule { @@ -88,36 +118,23 @@ impl Module for YtDlpModule { log::info!("Running {} Module", self.name()); log::info!("Checking {} items", self.config.items.len()); for (item, item_url) in &self.config.items { - log::info!("Fetching \"{item}\" videos"); - match Self::get_latest_entries(item_url, self.config.limit.unwrap_or(10)) { - Ok(latest_videos) => { - for (video_title, video_url) in latest_videos { - if self.db.check_for_url(&video_url) { - log::trace!( - "Skipping \"{video_title}\" because it was already downloaded" - ); - } else { - match self.download(&video_url, &self.root_dir.join(item)) { - Ok(()) => { - // mark as downloaded - self.db.insert_url(&video_url); - self.db.update_new_downloads(&self.name(), item, item_url); - log::info!("Downloaded \"{video_title}\""); - self.webhook_notify(&video_url, &video_title, item, true); - } - Err(e) => { - log::error!( - "Error downloading \"{video_title}\"; Reason: {e}" - ); - self.webhook_notify(&video_url, &video_title, item, false); - } - } - } + match item_url { + toml::Value::String(item_url) => { + self.check_item(item, item_url, &self.root_dir.join(item)); + } + toml::Value::Array(_) => todo!(), + toml::Value::Table(cat) => { + let category = item; + for (item, item_url) in cat { + let item_url = item_url.as_str().unwrap(); + self.check_item( + item, + item_url, + &self.root_dir.join(category).join(item), + ); } } - Err(e) => { - log::error!("Could not get videos from \"{item}\". Reason: {e}"); - } + _ => {} } } log::info!( From 222bf160dc4d3a583d1c2355b3393c35545d6644 Mon Sep 17 00:00:00 2001 From: JMARyA Date: Thu, 2 May 2024 13:35:13 +0200 Subject: [PATCH 07/10] fix user already exists --- entrypoint.sh | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index 3e4d65c..98a9558 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,10 +1,18 @@ #!/bin/bash -echo "Creating User ${UID:-1000}" -useradd -m -u "${UID:-1000}" hoard || exit 1 +# Check if the user already exists +if id hoard &>/dev/null; then + echo "User hoard already exists." +else + # Create the user + echo "Creating User ${UID:-1000}" + useradd -m -u "${UID:-1000}" hoard || exit 1 +fi +# Perform other setup tasks chown -R hoard /download -mkdir /.cache && chown -R hoard /.cache +mkdir -p /.cache && chown -R hoard /.cache chown -R hoard /data -su hoard -c /hoard +# Start the application as the user +su hoard -c /hoard \ No newline at end of file From 75bd4f49c1f542fa616bfe53e538810772eec3a2 Mon Sep 17 00:00:00 2001 From: JMARyA Date: Sun, 2 Jun 2024 23:04:09 +0200 Subject: [PATCH 08/10] refactor --- src/config.rs | 12 +++++----- src/db.rs | 16 +++++++++++++ src/lib.rs | 24 ++++++++++++++++++++ src/main.rs | 45 ++++++++++++------------------------ src/soundcloud/mod.rs | 34 +++++++++++++-------------- src/youtube/mod.rs | 41 ++++++++++++++++----------------- src/yt_dlp/config.rs | 52 ++++++++++++++++++++++++++++++++++++++++++ src/yt_dlp/mod.rs | 53 ++----------------------------------------- 8 files changed, 150 insertions(+), 127 deletions(-) create mode 100644 src/lib.rs create mode 100644 src/yt_dlp/config.rs diff --git a/src/config.rs b/src/config.rs index 65b7e36..d415146 100644 --- a/src/config.rs +++ b/src/config.rs @@ -2,24 +2,24 @@ use std::path::PathBuf; use serde::{Deserialize, Serialize}; -use crate::yt_dlp::YtDlpConfig; +use crate::yt_dlp::config::YtDlpConfig; /// General settings for hoard #[derive(Debug, Clone, Serialize, Deserialize)] pub struct HoardConfig { - // Top level data download directory + /// Top level data download directory pub data_dir: PathBuf, } /// Top level global config #[derive(Debug, Clone, Serialize, Deserialize)] pub struct GlobalConfig { - // Hoard Configuration + /// Hoard Configuration pub hoard: HoardConfig, - // Configuration for the YouTube Module + /// Configuration for the `YouTube` Module pub youtube: Option, - // Configuration for the SoundCloud Module + /// Configuration for the `SoundCloud` Module pub soundcloud: Option, - // Custom instances of yt-dlp + /// Custom instances of `yt-dlp` pub yt_dlp: Option>, } diff --git a/src/db.rs b/src/db.rs index cb1cce2..06f47d2 100644 --- a/src/db.rs +++ b/src/db.rs @@ -123,10 +123,24 @@ impl Database { Self { conn } } + /// Insert a URL into the database as already downloaded pub fn insert_url(&self, url: &str) { self.conn.send(Query::InsertUrl(url.to_string())); } + /// Check if a URL is already in the database + /// + /// # Return + /// Returns `true` if already present, `false` otherwise + /// + /// # Example + /// You could use this function like that: + /// + /// ```rust + /// if !db.check_for_url(some_url) { + /// // do download + /// } + /// ``` pub fn check_for_url(&self, url: &str) -> bool { match self.conn.send(Query::CheckForUrl(url.to_string())) { Out::Ok => false, @@ -134,6 +148,8 @@ impl Database { } } + /// Keep a record on when download happen. + /// This takes a `module`, `name` and `url` and saves a timestamp to the db. pub fn update_new_downloads(&self, module: &str, name: &str, url: &str) { self.conn.send(Query::UpdateNewDownloads( module.to_string(), diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..f48ac17 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,24 @@ +use std::path::PathBuf; + +pub mod config; +pub mod db; +pub mod soundcloud; +pub mod youtube; +pub mod yt_dlp; + +pub fn ensure_dir_exists(dir_path: &PathBuf) { + let path = std::path::Path::new(dir_path); + if !path.exists() { + std::fs::create_dir_all(path).unwrap(); + } +} + +/// Generic module implementation +/// +/// Each module gets it's own thread to work for itself. +pub trait Module: Send { + /// friendly name for module + fn name(&self) -> String; + /// module main loop + fn run(&self); +} diff --git a/src/main.rs b/src/main.rs index 5b08b11..ae90b50 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,30 +1,9 @@ -use std::path::PathBuf; - -mod config; -mod db; -mod soundcloud; -mod youtube; -mod yt_dlp; - -use config::GlobalConfig; - -use crate::yt_dlp::YtDlpModule; +use hoard::config::GlobalConfig; +use hoard::{ensure_dir_exists, Module}; // todo : migrate to async code? // todo : better log options -pub fn ensure_dir_exists(dir_path: &PathBuf) { - let path = std::path::Path::new(dir_path); - if !path.exists() { - std::fs::create_dir_all(path).unwrap(); - } -} - -trait Module: Send { - fn name(&self) -> String; - fn run(&self); -} - fn main() { #[cfg(debug_assertions)] { @@ -42,19 +21,23 @@ fn main() { log::info!("Starting hoard"); - let db = db::DatabaseBackend::new("data/download.db"); + let db = hoard::db::DatabaseBackend::new("data/download.db"); let config: GlobalConfig = toml::from_str(&std::fs::read_to_string("config.toml").unwrap()).unwrap(); ensure_dir_exists(&config.hoard.data_dir); - let mut modules: Vec> = vec![Box::new(youtube::YouTubeModule::new( - config.youtube.unwrap(), - db.take_db(), - config.hoard.data_dir.join("youtube"), - ))]; + let mut modules: Vec> = vec![]; + + if let Some(yt_config) = config.youtube { + modules.push(Box::new(hoard::youtube::YouTubeModule::new( + yt_config, + db.take_db(), + config.hoard.data_dir.join("youtube"), + ))); + } if let Some(sc_config) = config.soundcloud { - modules.push(Box::new(soundcloud::SoundCloudModule::new( + modules.push(Box::new(hoard::soundcloud::SoundCloudModule::new( sc_config, db.take_db(), config.hoard.data_dir.join("soundcloud"), @@ -66,7 +49,7 @@ fn main() { .name .clone() .unwrap_or_else(|| "yt_dlp".to_string()); - modules.push(Box::new(YtDlpModule::new( + modules.push(Box::new(hoard::yt_dlp::YtDlpModule::new( yt_dlp_mod, db.take_db(), config.hoard.data_dir.join(mod_name), diff --git a/src/soundcloud/mod.rs b/src/soundcloud/mod.rs index d90957e..b337e03 100644 --- a/src/soundcloud/mod.rs +++ b/src/soundcloud/mod.rs @@ -3,46 +3,46 @@ use std::{collections::HashMap, path::PathBuf}; use serde::{Deserialize, Serialize}; use crate::{ - yt_dlp::{YtDlpConfig, YtDlpModule}, + yt_dlp::{config::YtDlpConfig, YtDlpModule}, Module, }; /// Configuration for the `SoundCloud` Module #[derive(Debug, Clone, Serialize, Deserialize)] pub struct SoundCloudConfig { - // Interval in minutes between checks + /// Interval in minutes between checks pub interval: u64, /// Amount of items to query pub limit: Option, - // Items to check + /// Items to check pub artists: HashMap, - // Output Template for yt-dlp + /// Output Template for yt-dlp pub output_format: Option, - // Download comments + /// Download comments pub write_comments: Option, - // Download description + /// Download description pub write_description: Option, - // Download cover + /// Download cover pub write_cover: Option, - // Download subtitles + /// Download subtitles pub write_subs: Option, - // Audio Format + /// Audio Format pub audio_format: Option, - // Embed thumbnail + /// Embed thumbnail pub embed_thumbnail: Option, - // Embed metadata + /// Embed metadata pub embed_metadata: Option, - // Embed chapters + /// Embed chapters pub embed_chapters: Option, - // Embed info.json + /// Embed info.json pub embed_info_json: Option, - // Split by chapter + /// Split by chapter pub split_chapters: Option, - // Format Selection + /// Format Selection pub format: Option, - // Cookie File + /// Cookie File pub cookie: Option, - // Webhooks for notifications + /// Webhooks for notifications pub webhooks: Option>, } diff --git a/src/youtube/mod.rs b/src/youtube/mod.rs index a9a606f..faca786 100644 --- a/src/youtube/mod.rs +++ b/src/youtube/mod.rs @@ -2,51 +2,48 @@ use std::{collections::HashMap, path::PathBuf}; use serde::{Deserialize, Serialize}; -use crate::{ - yt_dlp::{YtDlpConfig, YtDlpModule}, - Module, -}; +use crate::{yt_dlp::config::YtDlpConfig, yt_dlp::YtDlpModule, Module}; /// Configuration for the `YouTube` Module #[derive(Debug, Clone, Serialize, Deserialize)] pub struct YouTubeConfig { - // Interval in minutes between checks + /// Interval in minutes between checks interval: u64, /// Amount of videos to query limit: Option, - // Channels to check + /// Channels to check channels: HashMap, - // Format of the Thumbnail + /// Format of the Thumbnail thumbnail_format: Option, - // Output Template for yt-dlp + /// Output Template for yt-dlp output_format: Option, - // Download description + /// Download description pub write_description: Option, - // Download info.json + /// Download info.json pub write_info_json: Option, - // Download comments + /// Download comments pub write_comments: Option, - // Download thumbnail + /// Download thumbnail pub write_thumbnail: Option, - // Download subtitles + /// Download subtitles pub write_subs: Option, - // Embed subtitles + /// Embed subtitles pub embed_subs: Option, - // Embed thumbnail + /// Embed thumbnail pub embed_thumbnail: Option, - // Embed metadata + /// Embed metadata pub embed_metadata: Option, - // Embed chapters + /// Embed chapters embed_chapters: Option, - // Embed info.json + /// Embed info.json pub embed_info_json: Option, - // Split by chapter + /// Split by chapter pub split_chapters: Option, - // Format Selection + /// Format Selection pub format: Option, - // Cookie File + /// Cookie File pub cookie: Option, - // Webhooks for notifications + /// Webhooks for notifications pub webhooks: Option>, } diff --git a/src/yt_dlp/config.rs b/src/yt_dlp/config.rs new file mode 100644 index 0000000..5893007 --- /dev/null +++ b/src/yt_dlp/config.rs @@ -0,0 +1,52 @@ +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; + +/// Configuration for the `YouTube` Module +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct YtDlpConfig { + /// Module Name + pub name: Option, + /// Interval in minutes between checks + pub interval: u64, + /// Amount of items to query + pub limit: Option, + /// Items to check + pub items: HashMap, + /// Format of the Thumbnail + pub thumbnail_format: Option, + /// Output Template for yt-dlp + pub output_format: Option, + /// Download description + pub write_description: Option, + /// Download info.json + pub write_info_json: Option, + /// Download comments + pub write_comments: Option, + /// Download thumbnail + pub write_thumbnail: Option, + /// Download subtitles + pub write_subs: Option, + /// Extract audio + pub audio_only: Option, + /// Audio Format + pub audio_format: Option, + /// Embed subtitles + pub embed_subs: Option, + /// Embed thumbnail + pub embed_thumbnail: Option, + /// Embed metadata + pub embed_metadata: Option, + /// Embed chapters + pub embed_chapters: Option, + /// Embed info.json + pub embed_info_json: Option, + /// Split by chapter + pub split_chapters: Option, + /// Format Selection + pub format: Option, + /// Cookie File + pub cookie: Option, + /// Webhooks for notifications + pub webhooks: Option>, +} diff --git a/src/yt_dlp/mod.rs b/src/yt_dlp/mod.rs index 986d198..75e7eb8 100644 --- a/src/yt_dlp/mod.rs +++ b/src/yt_dlp/mod.rs @@ -1,63 +1,14 @@ use std::{ - collections::HashMap, io::{BufRead, BufReader}, path::PathBuf, process::Command, }; -use serde::{Deserialize, Serialize}; +pub mod config; +use config::YtDlpConfig; use crate::{ensure_dir_exists, Module}; -/// Configuration for the `YouTube` Module -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct YtDlpConfig { - // Module Name - pub name: Option, - // Interval in minutes between checks - pub interval: u64, - /// Amount of items to query - pub limit: Option, - // Items to check - pub items: HashMap, - // Format of the Thumbnail - pub thumbnail_format: Option, - // Output Template for yt-dlp - pub output_format: Option, - // Download description - pub write_description: Option, - // Download info.json - pub write_info_json: Option, - // Download comments - pub write_comments: Option, - // Download thumbnail - pub write_thumbnail: Option, - // Download subtitles - pub write_subs: Option, - // Extract audio - pub audio_only: Option, - // Audio Format - pub audio_format: Option, - // Embed subtitles - pub embed_subs: Option, - // Embed thumbnail - pub embed_thumbnail: Option, - // Embed metadata - pub embed_metadata: Option, - // Embed chapters - pub embed_chapters: Option, - // Embed info.json - pub embed_info_json: Option, - // Split by chapter - pub split_chapters: Option, - // Format Selection - pub format: Option, - // Cookie File - pub cookie: Option, - // Webhooks for notifications - pub webhooks: Option>, -} - #[derive(Clone)] pub struct YtDlpModule { config: YtDlpConfig, From 260ee0b44f78fa9db8a10590ad894744efbcde21 Mon Sep 17 00:00:00 2001 From: JMARyA Date: Thu, 25 Jul 2024 23:54:19 +0200 Subject: [PATCH 09/10] update action --- .forgejo/workflows/build.yml | 35 +++++++++++++++++++++++++++++++++++ .gitea/workflows/build.yml | 25 ------------------------- 2 files changed, 35 insertions(+), 25 deletions(-) create mode 100644 .forgejo/workflows/build.yml delete mode 100644 .gitea/workflows/build.yml diff --git a/.forgejo/workflows/build.yml b/.forgejo/workflows/build.yml new file mode 100644 index 0000000..afc8b02 --- /dev/null +++ b/.forgejo/workflows/build.yml @@ -0,0 +1,35 @@ +name: deploy + +on: + push: + branches: + - main + +jobs: + deploy: + runs-on: host + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Log in to Docker Hub + uses: docker/login-action@v2 + with: + registry: git.hydrar.de + username: ${{ secrets.registry_user }} + password: ${{ secrets.registry_password }} + + - name: Build and push Docker image + uses: docker/build-push-action@v4 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: git.hydrar.de/jmarya/hoard:latest diff --git a/.gitea/workflows/build.yml b/.gitea/workflows/build.yml deleted file mode 100644 index 39b4d08..0000000 --- a/.gitea/workflows/build.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: build - -on: - push: - branches: - - main - -jobs: - deploy: - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - - name: Install Docker - run: curl -fsSL https://get.docker.com | sh - - - name: Log in to Docker registry - run: echo "${{ secrets.registry_password }}" | docker login -u "${{ secrets.registry_user }}" --password-stdin git.hydrar.de - - - name: Build and push Docker image - run: | - docker build -t git.hydrar.de/jmarya/hoard:latest . - docker push git.hydrar.de/jmarya/hoard:latest From 4090dbaac71907bbba4790eb9f9b344446339203 Mon Sep 17 00:00:00 2001 From: JMARyA Date: Sun, 28 Jul 2024 21:43:10 +0200 Subject: [PATCH 10/10] fix --- .forgejo/workflows/build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.forgejo/workflows/build.yml b/.forgejo/workflows/build.yml index afc8b02..3103821 100644 --- a/.forgejo/workflows/build.yml +++ b/.forgejo/workflows/build.yml @@ -30,6 +30,7 @@ jobs: uses: docker/build-push-action@v4 with: context: . - platforms: linux/amd64,linux/arm64 +# platforms: linux/amd64,linux/arm64 + platforms: linux/amd64 push: true tags: git.hydrar.de/jmarya/hoard:latest