refactor

src/config.rs

@@ -2,24 +2,24 @@ use std::path::PathBuf;
 
 use serde::{Deserialize, Serialize};
 
-use crate::yt_dlp::YtDlpConfig;
+use crate::yt_dlp::config::YtDlpConfig;
 
 /// General settings for hoard
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct HoardConfig {
-    // Top level data download directory
+    /// Top level data download directory
     pub data_dir: PathBuf,
 }
 
 /// Top level global config
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct GlobalConfig {
-    // Hoard Configuration
+    /// Hoard Configuration
     pub hoard: HoardConfig,
-    // Configuration for the YouTube Module
+    /// Configuration for the `YouTube` Module
     pub youtube: Option<crate::youtube::YouTubeConfig>,
-    // Configuration for the SoundCloud Module
+    /// Configuration for the `SoundCloud` Module
     pub soundcloud: Option<crate::soundcloud::SoundCloudConfig>,
-    // Custom instances of yt-dlp
+    /// Custom instances of `yt-dlp`
     pub yt_dlp: Option<Vec<YtDlpConfig>>,
 }

src/db.rs

@@ -123,10 +123,24 @@ impl Database {
         Self { conn }
     }
 
+    /// Insert a URL into the database as already downloaded
     pub fn insert_url(&self, url: &str) {
         self.conn.send(Query::InsertUrl(url.to_string()));
     }
 
+    /// Check if a URL is already in the database
+    ///
+    /// # Return
+    /// Returns `true` if already present, `false` otherwise
+    ///
+    /// # Example
+    /// You could use this function like that:
+    ///
+    /// ```rust
+    /// if !db.check_for_url(some_url) {
+    ///     // do download
+    /// }
+    /// ```
     pub fn check_for_url(&self, url: &str) -> bool {
         match self.conn.send(Query::CheckForUrl(url.to_string())) {
             Out::Ok => false,
@@ -134,6 +148,8 @@ impl Database {
         }
     }
 
+    /// Keep a record on when download happen.
+    /// This takes a `module`, `name` and `url` and saves a timestamp to the db.
     pub fn update_new_downloads(&self, module: &str, name: &str, url: &str) {
         self.conn.send(Query::UpdateNewDownloads(
             module.to_string(),

src/lib.rs

@@ -0,0 +1,24 @@
+use std::path::PathBuf;
+
+pub mod config;
+pub mod db;
+pub mod soundcloud;
+pub mod youtube;
+pub mod yt_dlp;
+
+pub fn ensure_dir_exists(dir_path: &PathBuf) {
+    let path = std::path::Path::new(dir_path);
+    if !path.exists() {
+        std::fs::create_dir_all(path).unwrap();
+    }
+}
+
+/// Generic module implementation
+///
+/// Each module gets it's own thread to work for itself.
+pub trait Module: Send {
+    /// friendly name for module
+    fn name(&self) -> String;
+    /// module main loop
+    fn run(&self);
+}

src/main.rs

@@ -1,30 +1,9 @@
-use std::path::PathBuf;
+use hoard::config::GlobalConfig;
-
+use hoard::{ensure_dir_exists, Module};
-mod config;
-mod db;
-mod soundcloud;
-mod youtube;
-mod yt_dlp;
-
-use config::GlobalConfig;
-
-use crate::yt_dlp::YtDlpModule;
 
 // todo : migrate to async code?
 // todo : better log options
 
-pub fn ensure_dir_exists(dir_path: &PathBuf) {
-    let path = std::path::Path::new(dir_path);
-    if !path.exists() {
-        std::fs::create_dir_all(path).unwrap();
-    }
-}
-
-trait Module: Send {
-    fn name(&self) -> String;
-    fn run(&self);
-}
-
 fn main() {
     #[cfg(debug_assertions)]
     {
@@ -42,19 +21,23 @@ fn main() {
 
     log::info!("Starting hoard");
 
-    let db = db::DatabaseBackend::new("data/download.db");
+    let db = hoard::db::DatabaseBackend::new("data/download.db");
     let config: GlobalConfig =
         toml::from_str(&std::fs::read_to_string("config.toml").unwrap()).unwrap();
     ensure_dir_exists(&config.hoard.data_dir);
 
-    let mut modules: Vec<Box<dyn Module>> = vec![Box::new(youtube::YouTubeModule::new(
+    let mut modules: Vec<Box<dyn Module>> = vec![];
-        config.youtube.unwrap(),
+
-        db.take_db(),
+    if let Some(yt_config) = config.youtube {
-        config.hoard.data_dir.join("youtube"),
+        modules.push(Box::new(hoard::youtube::YouTubeModule::new(
-    ))];
+            yt_config,
+            db.take_db(),
+            config.hoard.data_dir.join("youtube"),
+        )));
+    }
 
     if let Some(sc_config) = config.soundcloud {
-        modules.push(Box::new(soundcloud::SoundCloudModule::new(
+        modules.push(Box::new(hoard::soundcloud::SoundCloudModule::new(
             sc_config,
             db.take_db(),
             config.hoard.data_dir.join("soundcloud"),
@@ -66,7 +49,7 @@ fn main() {
             .name
             .clone()
             .unwrap_or_else(|| "yt_dlp".to_string());
-        modules.push(Box::new(YtDlpModule::new(
+        modules.push(Box::new(hoard::yt_dlp::YtDlpModule::new(
             yt_dlp_mod,
             db.take_db(),
             config.hoard.data_dir.join(mod_name),

src/soundcloud/mod.rs

@@ -3,46 +3,46 @@ use std::{collections::HashMap, path::PathBuf};
 use serde::{Deserialize, Serialize};
 
 use crate::{
-    yt_dlp::{YtDlpConfig, YtDlpModule},
+    yt_dlp::{config::YtDlpConfig, YtDlpModule},
     Module,
 };
 
 /// Configuration for the `SoundCloud` Module
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct SoundCloudConfig {
-    // Interval in minutes between checks
+    /// Interval in minutes between checks
     pub interval: u64,
     /// Amount of items to query
     pub limit: Option<u64>,
-    // Items to check
+    /// Items to check
     pub artists: HashMap<String, toml::Value>,
-    // Output Template for yt-dlp
+    /// Output Template for yt-dlp
     pub output_format: Option<String>,
-    // Download comments
+    /// Download comments
     pub write_comments: Option<bool>,
-    // Download description
+    /// Download description
     pub write_description: Option<bool>,
-    // Download cover
+    /// Download cover
     pub write_cover: Option<bool>,
-    // Download subtitles
+    /// Download subtitles
     pub write_subs: Option<bool>,
-    // Audio Format
+    /// Audio Format
     pub audio_format: Option<String>,
-    // Embed thumbnail
+    /// Embed thumbnail
     pub embed_thumbnail: Option<bool>,
-    // Embed metadata
+    /// Embed metadata
     pub embed_metadata: Option<bool>,
-    // Embed chapters
+    /// Embed chapters
     pub embed_chapters: Option<bool>,
-    // Embed info.json
+    /// Embed info.json
     pub embed_info_json: Option<bool>,
-    // Split by chapter
+    /// Split by chapter
     pub split_chapters: Option<bool>,
-    // Format Selection
+    /// Format Selection
     pub format: Option<String>,
-    // Cookie File
+    /// Cookie File
     pub cookie: Option<String>,
-    // Webhooks for notifications
+    /// Webhooks for notifications
     pub webhooks: Option<Vec<String>>,
 }
 

src/youtube/mod.rs

@@ -2,51 +2,48 @@ use std::{collections::HashMap, path::PathBuf};
 
 use serde::{Deserialize, Serialize};
 
-use crate::{
+use crate::{yt_dlp::config::YtDlpConfig, yt_dlp::YtDlpModule, Module};
-    yt_dlp::{YtDlpConfig, YtDlpModule},
-    Module,
-};
 
 /// Configuration for the `YouTube` Module
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct YouTubeConfig {
-    // Interval in minutes between checks
+    /// Interval in minutes between checks
     interval: u64,
     /// Amount of videos to query
     limit: Option<u64>,
-    // Channels to check
+    /// Channels to check
     channels: HashMap<String, toml::Value>,
-    // Format of the Thumbnail
+    /// Format of the Thumbnail
     thumbnail_format: Option<String>,
-    // Output Template for yt-dlp
+    /// Output Template for yt-dlp
     output_format: Option<String>,
-    // Download description
+    /// Download description
     pub write_description: Option<bool>,
-    // Download info.json
+    /// Download info.json
     pub write_info_json: Option<bool>,
-    // Download comments
+    /// Download comments
     pub write_comments: Option<bool>,
-    // Download thumbnail
+    /// Download thumbnail
     pub write_thumbnail: Option<bool>,
-    // Download subtitles
+    /// Download subtitles
     pub write_subs: Option<bool>,
-    // Embed subtitles
+    /// Embed subtitles
     pub embed_subs: Option<bool>,
-    // Embed thumbnail
+    /// Embed thumbnail
     pub embed_thumbnail: Option<bool>,
-    // Embed metadata
+    /// Embed metadata
     pub embed_metadata: Option<bool>,
-    // Embed chapters
+    /// Embed chapters
     embed_chapters: Option<bool>,
-    // Embed info.json
+    /// Embed info.json
     pub embed_info_json: Option<bool>,
-    // Split by chapter
+    /// Split by chapter
     pub split_chapters: Option<bool>,
-    // Format Selection
+    /// Format Selection
     pub format: Option<String>,
-    // Cookie File
+    /// Cookie File
     pub cookie: Option<String>,
-    // Webhooks for notifications
+    /// Webhooks for notifications
     pub webhooks: Option<Vec<String>>,
 }
 

src/yt_dlp/config.rs

@@ -0,0 +1,52 @@
+use std::collections::HashMap;
+
+use serde::{Deserialize, Serialize};
+
+/// Configuration for the `YouTube` Module
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct YtDlpConfig {
+    /// Module Name
+    pub name: Option<String>,
+    /// Interval in minutes between checks
+    pub interval: u64,
+    /// Amount of items to query
+    pub limit: Option<u64>,
+    /// Items to check
+    pub items: HashMap<String, toml::Value>,
+    /// Format of the Thumbnail
+    pub thumbnail_format: Option<String>,
+    /// Output Template for yt-dlp
+    pub output_format: Option<String>,
+    /// Download description
+    pub write_description: Option<bool>,
+    /// Download info.json
+    pub write_info_json: Option<bool>,
+    /// Download comments
+    pub write_comments: Option<bool>,
+    /// Download thumbnail
+    pub write_thumbnail: Option<bool>,
+    /// Download subtitles
+    pub write_subs: Option<bool>,
+    /// Extract audio
+    pub audio_only: Option<bool>,
+    /// Audio Format
+    pub audio_format: Option<String>,
+    /// Embed subtitles
+    pub embed_subs: Option<bool>,
+    /// Embed thumbnail
+    pub embed_thumbnail: Option<bool>,
+    /// Embed metadata
+    pub embed_metadata: Option<bool>,
+    /// Embed chapters
+    pub embed_chapters: Option<bool>,
+    /// Embed info.json
+    pub embed_info_json: Option<bool>,
+    /// Split by chapter
+    pub split_chapters: Option<bool>,
+    /// Format Selection
+    pub format: Option<String>,
+    /// Cookie File
+    pub cookie: Option<String>,
+    /// Webhooks for notifications
+    pub webhooks: Option<Vec<String>>,
+}

src/yt_dlp/mod.rs

@@ -1,63 +1,14 @@
 use std::{
-    collections::HashMap,
     io::{BufRead, BufReader},
     path::PathBuf,
     process::Command,
 };
 
-use serde::{Deserialize, Serialize};
+pub mod config;
+use config::YtDlpConfig;
 
 use crate::{ensure_dir_exists, Module};
 
-/// Configuration for the `YouTube` Module
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct YtDlpConfig {
-    // Module Name
-    pub name: Option<String>,
-    // Interval in minutes between checks
-    pub interval: u64,
-    /// Amount of items to query
-    pub limit: Option<u64>,
-    // Items to check
-    pub items: HashMap<String, toml::Value>,
-    // Format of the Thumbnail
-    pub thumbnail_format: Option<String>,
-    // Output Template for yt-dlp
-    pub output_format: Option<String>,
-    // Download description
-    pub write_description: Option<bool>,
-    // Download info.json
-    pub write_info_json: Option<bool>,
-    // Download comments
-    pub write_comments: Option<bool>,
-    // Download thumbnail
-    pub write_thumbnail: Option<bool>,
-    // Download subtitles
-    pub write_subs: Option<bool>,
-    // Extract audio
-    pub audio_only: Option<bool>,
-    // Audio Format
-    pub audio_format: Option<String>,
-    // Embed subtitles
-    pub embed_subs: Option<bool>,
-    // Embed thumbnail
-    pub embed_thumbnail: Option<bool>,
-    // Embed metadata
-    pub embed_metadata: Option<bool>,
-    // Embed chapters
-    pub embed_chapters: Option<bool>,
-    // Embed info.json
-    pub embed_info_json: Option<bool>,
-    // Split by chapter
-    pub split_chapters: Option<bool>,
-    // Format Selection
-    pub format: Option<String>,
-    // Cookie File
-    pub cookie: Option<String>,
-    // Webhooks for notifications
-    pub webhooks: Option<Vec<String>>,
-}
-
 #[derive(Clone)]
 pub struct YtDlpModule {
     config: YtDlpConfig,