From 75bd4f49c1f542fa616bfe53e538810772eec3a2 Mon Sep 17 00:00:00 2001 From: JMARyA Date: Sun, 2 Jun 2024 23:04:09 +0200 Subject: [PATCH] refactor --- src/config.rs | 12 +++++----- src/db.rs | 16 +++++++++++++ src/lib.rs | 24 ++++++++++++++++++++ src/main.rs | 45 ++++++++++++------------------------ src/soundcloud/mod.rs | 34 +++++++++++++-------------- src/youtube/mod.rs | 41 ++++++++++++++++----------------- src/yt_dlp/config.rs | 52 ++++++++++++++++++++++++++++++++++++++++++ src/yt_dlp/mod.rs | 53 ++----------------------------------------- 8 files changed, 150 insertions(+), 127 deletions(-) create mode 100644 src/lib.rs create mode 100644 src/yt_dlp/config.rs diff --git a/src/config.rs b/src/config.rs index 65b7e36..d415146 100644 --- a/src/config.rs +++ b/src/config.rs @@ -2,24 +2,24 @@ use std::path::PathBuf; use serde::{Deserialize, Serialize}; -use crate::yt_dlp::YtDlpConfig; +use crate::yt_dlp::config::YtDlpConfig; /// General settings for hoard #[derive(Debug, Clone, Serialize, Deserialize)] pub struct HoardConfig { - // Top level data download directory + /// Top level data download directory pub data_dir: PathBuf, } /// Top level global config #[derive(Debug, Clone, Serialize, Deserialize)] pub struct GlobalConfig { - // Hoard Configuration + /// Hoard Configuration pub hoard: HoardConfig, - // Configuration for the YouTube Module + /// Configuration for the `YouTube` Module pub youtube: Option, - // Configuration for the SoundCloud Module + /// Configuration for the `SoundCloud` Module pub soundcloud: Option, - // Custom instances of yt-dlp + /// Custom instances of `yt-dlp` pub yt_dlp: Option>, } diff --git a/src/db.rs b/src/db.rs index cb1cce2..06f47d2 100644 --- a/src/db.rs +++ b/src/db.rs @@ -123,10 +123,24 @@ impl Database { Self { conn } } + /// Insert a URL into the database as already downloaded pub fn insert_url(&self, url: &str) { self.conn.send(Query::InsertUrl(url.to_string())); } + /// Check if a URL is already in the database + /// + /// # Return + /// Returns `true` if already present, `false` otherwise + /// + /// # Example + /// You could use this function like that: + /// + /// ```rust + /// if !db.check_for_url(some_url) { + /// // do download + /// } + /// ``` pub fn check_for_url(&self, url: &str) -> bool { match self.conn.send(Query::CheckForUrl(url.to_string())) { Out::Ok => false, @@ -134,6 +148,8 @@ impl Database { } } + /// Keep a record on when download happen. + /// This takes a `module`, `name` and `url` and saves a timestamp to the db. pub fn update_new_downloads(&self, module: &str, name: &str, url: &str) { self.conn.send(Query::UpdateNewDownloads( module.to_string(), diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..f48ac17 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,24 @@ +use std::path::PathBuf; + +pub mod config; +pub mod db; +pub mod soundcloud; +pub mod youtube; +pub mod yt_dlp; + +pub fn ensure_dir_exists(dir_path: &PathBuf) { + let path = std::path::Path::new(dir_path); + if !path.exists() { + std::fs::create_dir_all(path).unwrap(); + } +} + +/// Generic module implementation +/// +/// Each module gets it's own thread to work for itself. +pub trait Module: Send { + /// friendly name for module + fn name(&self) -> String; + /// module main loop + fn run(&self); +} diff --git a/src/main.rs b/src/main.rs index 5b08b11..ae90b50 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,30 +1,9 @@ -use std::path::PathBuf; - -mod config; -mod db; -mod soundcloud; -mod youtube; -mod yt_dlp; - -use config::GlobalConfig; - -use crate::yt_dlp::YtDlpModule; +use hoard::config::GlobalConfig; +use hoard::{ensure_dir_exists, Module}; // todo : migrate to async code? // todo : better log options -pub fn ensure_dir_exists(dir_path: &PathBuf) { - let path = std::path::Path::new(dir_path); - if !path.exists() { - std::fs::create_dir_all(path).unwrap(); - } -} - -trait Module: Send { - fn name(&self) -> String; - fn run(&self); -} - fn main() { #[cfg(debug_assertions)] { @@ -42,19 +21,23 @@ fn main() { log::info!("Starting hoard"); - let db = db::DatabaseBackend::new("data/download.db"); + let db = hoard::db::DatabaseBackend::new("data/download.db"); let config: GlobalConfig = toml::from_str(&std::fs::read_to_string("config.toml").unwrap()).unwrap(); ensure_dir_exists(&config.hoard.data_dir); - let mut modules: Vec> = vec![Box::new(youtube::YouTubeModule::new( - config.youtube.unwrap(), - db.take_db(), - config.hoard.data_dir.join("youtube"), - ))]; + let mut modules: Vec> = vec![]; + + if let Some(yt_config) = config.youtube { + modules.push(Box::new(hoard::youtube::YouTubeModule::new( + yt_config, + db.take_db(), + config.hoard.data_dir.join("youtube"), + ))); + } if let Some(sc_config) = config.soundcloud { - modules.push(Box::new(soundcloud::SoundCloudModule::new( + modules.push(Box::new(hoard::soundcloud::SoundCloudModule::new( sc_config, db.take_db(), config.hoard.data_dir.join("soundcloud"), @@ -66,7 +49,7 @@ fn main() { .name .clone() .unwrap_or_else(|| "yt_dlp".to_string()); - modules.push(Box::new(YtDlpModule::new( + modules.push(Box::new(hoard::yt_dlp::YtDlpModule::new( yt_dlp_mod, db.take_db(), config.hoard.data_dir.join(mod_name), diff --git a/src/soundcloud/mod.rs b/src/soundcloud/mod.rs index d90957e..b337e03 100644 --- a/src/soundcloud/mod.rs +++ b/src/soundcloud/mod.rs @@ -3,46 +3,46 @@ use std::{collections::HashMap, path::PathBuf}; use serde::{Deserialize, Serialize}; use crate::{ - yt_dlp::{YtDlpConfig, YtDlpModule}, + yt_dlp::{config::YtDlpConfig, YtDlpModule}, Module, }; /// Configuration for the `SoundCloud` Module #[derive(Debug, Clone, Serialize, Deserialize)] pub struct SoundCloudConfig { - // Interval in minutes between checks + /// Interval in minutes between checks pub interval: u64, /// Amount of items to query pub limit: Option, - // Items to check + /// Items to check pub artists: HashMap, - // Output Template for yt-dlp + /// Output Template for yt-dlp pub output_format: Option, - // Download comments + /// Download comments pub write_comments: Option, - // Download description + /// Download description pub write_description: Option, - // Download cover + /// Download cover pub write_cover: Option, - // Download subtitles + /// Download subtitles pub write_subs: Option, - // Audio Format + /// Audio Format pub audio_format: Option, - // Embed thumbnail + /// Embed thumbnail pub embed_thumbnail: Option, - // Embed metadata + /// Embed metadata pub embed_metadata: Option, - // Embed chapters + /// Embed chapters pub embed_chapters: Option, - // Embed info.json + /// Embed info.json pub embed_info_json: Option, - // Split by chapter + /// Split by chapter pub split_chapters: Option, - // Format Selection + /// Format Selection pub format: Option, - // Cookie File + /// Cookie File pub cookie: Option, - // Webhooks for notifications + /// Webhooks for notifications pub webhooks: Option>, } diff --git a/src/youtube/mod.rs b/src/youtube/mod.rs index a9a606f..faca786 100644 --- a/src/youtube/mod.rs +++ b/src/youtube/mod.rs @@ -2,51 +2,48 @@ use std::{collections::HashMap, path::PathBuf}; use serde::{Deserialize, Serialize}; -use crate::{ - yt_dlp::{YtDlpConfig, YtDlpModule}, - Module, -}; +use crate::{yt_dlp::config::YtDlpConfig, yt_dlp::YtDlpModule, Module}; /// Configuration for the `YouTube` Module #[derive(Debug, Clone, Serialize, Deserialize)] pub struct YouTubeConfig { - // Interval in minutes between checks + /// Interval in minutes between checks interval: u64, /// Amount of videos to query limit: Option, - // Channels to check + /// Channels to check channels: HashMap, - // Format of the Thumbnail + /// Format of the Thumbnail thumbnail_format: Option, - // Output Template for yt-dlp + /// Output Template for yt-dlp output_format: Option, - // Download description + /// Download description pub write_description: Option, - // Download info.json + /// Download info.json pub write_info_json: Option, - // Download comments + /// Download comments pub write_comments: Option, - // Download thumbnail + /// Download thumbnail pub write_thumbnail: Option, - // Download subtitles + /// Download subtitles pub write_subs: Option, - // Embed subtitles + /// Embed subtitles pub embed_subs: Option, - // Embed thumbnail + /// Embed thumbnail pub embed_thumbnail: Option, - // Embed metadata + /// Embed metadata pub embed_metadata: Option, - // Embed chapters + /// Embed chapters embed_chapters: Option, - // Embed info.json + /// Embed info.json pub embed_info_json: Option, - // Split by chapter + /// Split by chapter pub split_chapters: Option, - // Format Selection + /// Format Selection pub format: Option, - // Cookie File + /// Cookie File pub cookie: Option, - // Webhooks for notifications + /// Webhooks for notifications pub webhooks: Option>, } diff --git a/src/yt_dlp/config.rs b/src/yt_dlp/config.rs new file mode 100644 index 0000000..5893007 --- /dev/null +++ b/src/yt_dlp/config.rs @@ -0,0 +1,52 @@ +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; + +/// Configuration for the `YouTube` Module +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct YtDlpConfig { + /// Module Name + pub name: Option, + /// Interval in minutes between checks + pub interval: u64, + /// Amount of items to query + pub limit: Option, + /// Items to check + pub items: HashMap, + /// Format of the Thumbnail + pub thumbnail_format: Option, + /// Output Template for yt-dlp + pub output_format: Option, + /// Download description + pub write_description: Option, + /// Download info.json + pub write_info_json: Option, + /// Download comments + pub write_comments: Option, + /// Download thumbnail + pub write_thumbnail: Option, + /// Download subtitles + pub write_subs: Option, + /// Extract audio + pub audio_only: Option, + /// Audio Format + pub audio_format: Option, + /// Embed subtitles + pub embed_subs: Option, + /// Embed thumbnail + pub embed_thumbnail: Option, + /// Embed metadata + pub embed_metadata: Option, + /// Embed chapters + pub embed_chapters: Option, + /// Embed info.json + pub embed_info_json: Option, + /// Split by chapter + pub split_chapters: Option, + /// Format Selection + pub format: Option, + /// Cookie File + pub cookie: Option, + /// Webhooks for notifications + pub webhooks: Option>, +} diff --git a/src/yt_dlp/mod.rs b/src/yt_dlp/mod.rs index 986d198..75e7eb8 100644 --- a/src/yt_dlp/mod.rs +++ b/src/yt_dlp/mod.rs @@ -1,63 +1,14 @@ use std::{ - collections::HashMap, io::{BufRead, BufReader}, path::PathBuf, process::Command, }; -use serde::{Deserialize, Serialize}; +pub mod config; +use config::YtDlpConfig; use crate::{ensure_dir_exists, Module}; -/// Configuration for the `YouTube` Module -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct YtDlpConfig { - // Module Name - pub name: Option, - // Interval in minutes between checks - pub interval: u64, - /// Amount of items to query - pub limit: Option, - // Items to check - pub items: HashMap, - // Format of the Thumbnail - pub thumbnail_format: Option, - // Output Template for yt-dlp - pub output_format: Option, - // Download description - pub write_description: Option, - // Download info.json - pub write_info_json: Option, - // Download comments - pub write_comments: Option, - // Download thumbnail - pub write_thumbnail: Option, - // Download subtitles - pub write_subs: Option, - // Extract audio - pub audio_only: Option, - // Audio Format - pub audio_format: Option, - // Embed subtitles - pub embed_subs: Option, - // Embed thumbnail - pub embed_thumbnail: Option, - // Embed metadata - pub embed_metadata: Option, - // Embed chapters - pub embed_chapters: Option, - // Embed info.json - pub embed_info_json: Option, - // Split by chapter - pub split_chapters: Option, - // Format Selection - pub format: Option, - // Cookie File - pub cookie: Option, - // Webhooks for notifications - pub webhooks: Option>, -} - #[derive(Clone)] pub struct YtDlpModule { config: YtDlpConfig,