mirror of
https://github.com/qarmin/czkawka
synced 2024-09-30 04:54:30 +00:00
Optimize excluded items (#1152)
This commit is contained in:
parent
306648a37d
commit
51198c2043
|
@ -12,10 +12,14 @@
|
|||
### Core
|
||||
- Using normal crossbeam channels instead of asyncio tokio channel - [#1102](https://github.com/qarmin/czkawka/pull/1102)
|
||||
- Fixed tool type when using progress of empty directories - [#1102](https://github.com/qarmin/czkawka/pull/1102)
|
||||
- Fixed missing json support in saving size and name - [#1102](https://github.com/qarmin/czkawka/pull/1102)
|
||||
- Fixed missing json support when saving size and name duplicate results - [#1102](https://github.com/qarmin/czkawka/pull/1102)
|
||||
- Fix cross-compiled debug windows build - [#1102](https://github.com/qarmin/czkawka/pull/1102)
|
||||
- Added bigger stack size by default(fixes stack overflow in some musl apps) - [#1102](https://github.com/qarmin/czkawka/pull/1102)
|
||||
- Added optional libraw dependency(better single-core performance and support more raw files) - [#1102](https://github.com/qarmin/czkawka/pull/1102)
|
||||
- Speedup checking for wildcards and fix invalid recognizing long excluded items - [#1152](https://github.com/qarmin/czkawka/pull/1152)
|
||||
- Even 10x speedup when searching for empty folders - [#1152](https://github.com/qarmin/czkawka/pull/1152)
|
||||
- Collecting files for scan can be a lot of faster due lazy file metadata gathering - [#1152](https://github.com/qarmin/czkawka/pull/1152)
|
||||
- Fixed recognizing not accessible folders as non-empty - [#1152](https://github.com/qarmin/czkawka/pull/1152)
|
||||
|
||||
## Version 6.1.0 - 15.10.2023r
|
||||
- BREAKING CHANGE - Changed cache saving method, deduplicated, optimized and simplified procedure(all files needs to be hashed again) - [#1072](https://github.com/qarmin/czkawka/pull/1072), [#1086](https://github.com/qarmin/czkawka/pull/1086)
|
||||
|
|
|
@ -419,7 +419,9 @@ impl PrintResults for BadExtensions {
|
|||
writeln!(
|
||||
writer,
|
||||
"Results of searching {:?} with excluded directories {:?} and excluded items {:?}",
|
||||
self.common_data.directories.included_directories, self.common_data.directories.excluded_directories, self.common_data.excluded_items.items
|
||||
self.common_data.directories.included_directories,
|
||||
self.common_data.directories.excluded_directories,
|
||||
self.common_data.excluded_items.get_excluded_items()
|
||||
)?;
|
||||
writeln!(writer, "Found {} files with invalid extension.\n", self.information.number_of_files_with_bad_extension)?;
|
||||
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
use std::collections::BTreeMap;
|
||||
use std::fs;
|
||||
use std::fs::{DirEntry, Metadata};
|
||||
use std::fs::DirEntry;
|
||||
use std::io::Write;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
@ -14,7 +14,7 @@ use rayon::prelude::*;
|
|||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::common::{check_folder_children, check_if_stop_received, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, split_path};
|
||||
use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData, ToolType};
|
||||
use crate::common_dir_traversal::{common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData, ToolType};
|
||||
use crate::common_tool::{CommonData, CommonToolData, DeleteMethod};
|
||||
use crate::common_traits::{DebugPrint, PrintResults};
|
||||
|
||||
|
@ -68,7 +68,7 @@ impl BigFile {
|
|||
|
||||
#[fun_time(message = "look_for_big_files", level = "debug")]
|
||||
fn look_for_big_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&Sender<ProgressData>>) -> bool {
|
||||
let mut folders_to_check: Vec<PathBuf> = Vec::with_capacity(1024 * 2); // This should be small enough too not see to big difference and big enough to store most of paths without needing to resize vector
|
||||
let mut folders_to_check: Vec<PathBuf> = Vec::with_capacity(1024 * 2);
|
||||
let mut old_map: BTreeMap<u64, Vec<FileEntry>> = Default::default();
|
||||
|
||||
// Add root folders for finding
|
||||
|
@ -99,22 +99,25 @@ impl BigFile {
|
|||
|
||||
// Check every sub folder/file/link etc.
|
||||
for entry in read_dir {
|
||||
let Some((entry_data, metadata)) = common_get_entry_data_metadata(&entry, &mut warnings, current_folder) else {
|
||||
let Ok(entry_data) = entry else {
|
||||
continue;
|
||||
};
|
||||
let Ok(file_type) = entry_data.file_type() else {
|
||||
continue;
|
||||
};
|
||||
|
||||
if metadata.is_dir() {
|
||||
if file_type.is_dir() {
|
||||
check_folder_children(
|
||||
&mut dir_result,
|
||||
&mut warnings,
|
||||
current_folder,
|
||||
entry_data,
|
||||
&entry_data,
|
||||
self.common_data.recursive_search,
|
||||
&self.common_data.directories,
|
||||
&self.common_data.excluded_items,
|
||||
);
|
||||
} else if metadata.is_file() {
|
||||
self.collect_file_entry(&atomic_counter, &metadata, entry_data, &mut fe_result, &mut warnings, current_folder);
|
||||
} else if file_type.is_file() {
|
||||
self.collect_file_entry(&atomic_counter, &entry_data, &mut fe_result, &mut warnings, current_folder);
|
||||
}
|
||||
}
|
||||
(dir_result, warnings, fe_result)
|
||||
|
@ -146,7 +149,6 @@ impl BigFile {
|
|||
pub fn collect_file_entry(
|
||||
&self,
|
||||
atomic_counter: &Arc<AtomicUsize>,
|
||||
metadata: &Metadata,
|
||||
entry_data: &DirEntry,
|
||||
fe_result: &mut Vec<(u64, FileEntry)>,
|
||||
warnings: &mut Vec<String>,
|
||||
|
@ -154,10 +156,6 @@ impl BigFile {
|
|||
) {
|
||||
atomic_counter.fetch_add(1, Ordering::Relaxed);
|
||||
|
||||
if metadata.len() == 0 {
|
||||
return;
|
||||
}
|
||||
|
||||
let Some(file_name_lowercase) = get_lowercase_name(entry_data, warnings) else {
|
||||
return;
|
||||
};
|
||||
|
@ -171,10 +169,18 @@ impl BigFile {
|
|||
return;
|
||||
}
|
||||
|
||||
let Ok(metadata) = entry_data.metadata() else {
|
||||
return;
|
||||
};
|
||||
|
||||
if metadata.len() == 0 {
|
||||
return;
|
||||
}
|
||||
|
||||
let fe: FileEntry = FileEntry {
|
||||
path: current_file_name.clone(),
|
||||
size: metadata.len(),
|
||||
modified_date: get_modified_time(metadata, warnings, ¤t_file_name, false),
|
||||
modified_date: get_modified_time(&metadata, warnings, ¤t_file_name, false),
|
||||
};
|
||||
|
||||
fe_result.push((fe.size, fe));
|
||||
|
@ -253,7 +259,9 @@ impl PrintResults for BigFile {
|
|||
writeln!(
|
||||
writer,
|
||||
"Results of searching {:?} with excluded directories {:?} and excluded items {:?}",
|
||||
self.common_data.directories.included_directories, self.common_data.directories.excluded_directories, self.common_data.excluded_items.items
|
||||
self.common_data.directories.included_directories,
|
||||
self.common_data.directories.excluded_directories,
|
||||
self.common_data.excluded_items.get_excluded_items()
|
||||
)?;
|
||||
|
||||
if self.information.number_of_real_files != 0 {
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
use std::collections::BTreeMap;
|
||||
use std::fs::{DirEntry, File, Metadata};
|
||||
use std::fs::{DirEntry, File};
|
||||
use std::io::prelude::*;
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
@ -22,7 +22,7 @@ use crate::common::{
|
|||
IMAGE_RS_BROKEN_FILES_EXTENSIONS, PDF_FILES_EXTENSIONS, ZIP_FILES_EXTENSIONS,
|
||||
};
|
||||
use crate::common_cache::{get_broken_files_cache_file, load_cache_from_file_generalized_by_path, save_cache_to_file_generalized};
|
||||
use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData, ToolType};
|
||||
use crate::common_dir_traversal::{common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData, ToolType};
|
||||
use crate::common_tool::{CommonData, CommonToolData, DeleteMethod};
|
||||
use crate::common_traits::*;
|
||||
|
||||
|
@ -108,7 +108,7 @@ impl BrokenFiles {
|
|||
|
||||
#[fun_time(message = "check_files", level = "debug")]
|
||||
fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&Sender<ProgressData>>) -> bool {
|
||||
let mut folders_to_check: Vec<PathBuf> = Vec::with_capacity(1024 * 2); // This should be small enough too not see to big difference and big enough to store most of paths without needing to resize vector
|
||||
let mut folders_to_check: Vec<PathBuf> = Vec::with_capacity(1024 * 2);
|
||||
|
||||
// Add root folders for finding
|
||||
for id in &self.common_data.directories.included_directories {
|
||||
|
@ -138,22 +138,25 @@ impl BrokenFiles {
|
|||
|
||||
// Check every sub folder/file/link etc.
|
||||
for entry in read_dir {
|
||||
let Some((entry_data, metadata)) = common_get_entry_data_metadata(&entry, &mut warnings, current_folder) else {
|
||||
let Ok(entry_data) = entry else {
|
||||
continue;
|
||||
};
|
||||
let Ok(file_type) = entry_data.file_type() else {
|
||||
continue;
|
||||
};
|
||||
|
||||
if metadata.is_dir() {
|
||||
if file_type.is_dir() {
|
||||
check_folder_children(
|
||||
&mut dir_result,
|
||||
&mut warnings,
|
||||
current_folder,
|
||||
entry_data,
|
||||
&entry_data,
|
||||
self.common_data.recursive_search,
|
||||
&self.common_data.directories,
|
||||
&self.common_data.excluded_items,
|
||||
);
|
||||
} else if metadata.is_file() {
|
||||
if let Some(file_entry) = self.get_file_entry(&metadata, &atomic_counter, entry_data, &mut warnings, current_folder) {
|
||||
} else if file_type.is_file() {
|
||||
if let Some(file_entry) = self.get_file_entry(&atomic_counter, &entry_data, &mut warnings, current_folder) {
|
||||
fe_result.push((file_entry.path.to_string_lossy().to_string(), file_entry));
|
||||
}
|
||||
}
|
||||
|
@ -180,14 +183,7 @@ impl BrokenFiles {
|
|||
true
|
||||
}
|
||||
|
||||
fn get_file_entry(
|
||||
&self,
|
||||
metadata: &Metadata,
|
||||
atomic_counter: &Arc<AtomicUsize>,
|
||||
entry_data: &DirEntry,
|
||||
warnings: &mut Vec<String>,
|
||||
current_folder: &Path,
|
||||
) -> Option<FileEntry> {
|
||||
fn get_file_entry(&self, atomic_counter: &Arc<AtomicUsize>, entry_data: &DirEntry, warnings: &mut Vec<String>, current_folder: &Path) -> Option<FileEntry> {
|
||||
atomic_counter.fetch_add(1, Ordering::Relaxed);
|
||||
|
||||
let file_name_lowercase = get_lowercase_name(entry_data, warnings)?;
|
||||
|
@ -207,9 +203,13 @@ impl BrokenFiles {
|
|||
return None;
|
||||
}
|
||||
|
||||
let Ok(metadata) = entry_data.metadata() else {
|
||||
return None;
|
||||
};
|
||||
|
||||
let fe: FileEntry = FileEntry {
|
||||
path: current_file_name.clone(),
|
||||
modified_date: get_modified_time(metadata, warnings, ¤t_file_name, false),
|
||||
modified_date: get_modified_time(&metadata, warnings, ¤t_file_name, false),
|
||||
size: metadata.len(),
|
||||
type_of_file,
|
||||
error_string: String::new(),
|
||||
|
@ -464,7 +464,9 @@ impl PrintResults for BrokenFiles {
|
|||
writeln!(
|
||||
writer,
|
||||
"Results of searching {:?} with excluded directories {:?} and excluded items {:?}",
|
||||
self.common_data.directories.included_directories, self.common_data.directories.excluded_directories, self.common_data.excluded_items.items
|
||||
self.common_data.directories.included_directories,
|
||||
self.common_data.directories.excluded_directories,
|
||||
self.common_data.excluded_items.get_excluded_items()
|
||||
)?;
|
||||
|
||||
if !self.broken_files.is_empty() {
|
||||
|
|
|
@ -29,7 +29,7 @@ use symphonia::core::conv::IntoSample;
|
|||
// use libheif_rs::LibHeif;
|
||||
use crate::common_dir_traversal::{CheckingMethod, ProgressData, ToolType};
|
||||
use crate::common_directory::Directories;
|
||||
use crate::common_items::ExcludedItems;
|
||||
use crate::common_items::{ExcludedItems, SingleExcludedItem};
|
||||
use crate::common_messages::Messages;
|
||||
use crate::common_tool::DeleteMethod;
|
||||
use crate::common_traits::ResultEntry;
|
||||
|
@ -150,8 +150,6 @@ pub const VIDEO_FILES_EXTENSIONS: &[&str] = &[
|
|||
pub const LOOP_DURATION: u32 = 20; //ms
|
||||
pub const SEND_PROGRESS_DATA_TIME_BETWEEN: u32 = 200; //ms
|
||||
|
||||
pub struct Common();
|
||||
|
||||
pub fn remove_folder_if_contains_only_empty_folders(path: impl AsRef<Path>) -> bool {
|
||||
let path = path.as_ref();
|
||||
if !path.is_dir() {
|
||||
|
@ -333,81 +331,70 @@ pub fn create_crash_message(library_name: &str, file_path: &str, home_library_ur
|
|||
format!("{library_name} library crashed when opening \"{file_path}\", please check if this is fixed with the latest version of {library_name} (e.g. with https://github.com/qarmin/crates_tester) and if it is not fixed, please report bug here - {home_library_url}")
|
||||
}
|
||||
|
||||
impl Common {
|
||||
pub fn regex_check(expression: &str, directory: impl AsRef<Path>) -> bool {
|
||||
if expression == "*" {
|
||||
return true;
|
||||
}
|
||||
|
||||
let temp_splits: Vec<&str> = expression.split('*').collect();
|
||||
let mut splits: Vec<&str> = Vec::new();
|
||||
for i in temp_splits {
|
||||
if !i.is_empty() {
|
||||
splits.push(i);
|
||||
}
|
||||
}
|
||||
if splits.is_empty() {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Get rid of non unicode characters
|
||||
let directory = directory.as_ref().to_string_lossy();
|
||||
|
||||
// Early checking if directory contains all parts needed by expression
|
||||
for split in &splits {
|
||||
if !directory.contains(split) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
let mut position_of_splits: Vec<usize> = Vec::new();
|
||||
|
||||
// `git*` shouldn't be true for `/gitsfafasfs`
|
||||
if !expression.starts_with('*') && directory.find(splits[0]).unwrap() > 0 {
|
||||
return false;
|
||||
}
|
||||
// `*home` shouldn't be true for `/homeowner`
|
||||
if !expression.ends_with('*') && !directory.ends_with(splits.last().unwrap()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// At the end we check if parts between * are correctly positioned
|
||||
position_of_splits.push(directory.find(splits[0]).unwrap());
|
||||
let mut current_index: usize;
|
||||
let mut found_index: usize;
|
||||
for i in splits[1..].iter().enumerate() {
|
||||
current_index = *position_of_splits.get(i.0).unwrap() + i.1.len();
|
||||
found_index = match directory[current_index..].find(i.1) {
|
||||
Some(t) => t,
|
||||
None => return false,
|
||||
};
|
||||
position_of_splits.push(found_index + current_index);
|
||||
}
|
||||
true
|
||||
pub fn regex_check(expression_item: &SingleExcludedItem, directory: impl AsRef<Path>) -> bool {
|
||||
if expression_item.expression == "*" {
|
||||
return true;
|
||||
}
|
||||
|
||||
pub fn normalize_windows_path(path_to_change: impl AsRef<Path>) -> PathBuf {
|
||||
let path = path_to_change.as_ref();
|
||||
if expression_item.expression_splits.is_empty() {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Don't do anything, because network path may be case intensive
|
||||
if path.to_string_lossy().starts_with('\\') {
|
||||
return path.to_path_buf();
|
||||
// Get rid of non unicode characters
|
||||
let directory_name = directory.as_ref().to_string_lossy();
|
||||
|
||||
// Early checking if directory contains all parts needed by expression
|
||||
for split in &expression_item.unique_extensions_splits {
|
||||
if !directory_name.contains(split) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
match path.to_str() {
|
||||
Some(path) if path.is_char_boundary(1) => {
|
||||
let replaced = path.replace('/', "\\");
|
||||
let mut new_path = OsString::new();
|
||||
if replaced[1..].starts_with(':') {
|
||||
new_path.push(replaced[..1].to_ascii_uppercase());
|
||||
new_path.push(replaced[1..].to_ascii_lowercase());
|
||||
} else {
|
||||
new_path.push(replaced.to_ascii_lowercase());
|
||||
}
|
||||
PathBuf::from(new_path)
|
||||
// `git*` shouldn't be true for `/gitsfafasfs`
|
||||
if !expression_item.expression.starts_with('*') && directory_name.find(&expression_item.expression_splits[0]).unwrap() > 0 {
|
||||
return false;
|
||||
}
|
||||
// `*home` shouldn't be true for `/homeowner`
|
||||
if !expression_item.expression.ends_with('*') && !directory_name.ends_with(expression_item.expression_splits.last().unwrap()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// At the end we check if parts between * are correctly positioned
|
||||
let mut last_split_point = directory_name.find(&expression_item.expression_splits[0]).unwrap();
|
||||
let mut current_index: usize = 0;
|
||||
let mut found_index: usize;
|
||||
for spl in &expression_item.expression_splits[1..] {
|
||||
found_index = match directory_name[current_index..].find(spl) {
|
||||
Some(t) => t,
|
||||
None => return false,
|
||||
};
|
||||
current_index = last_split_point + spl.len();
|
||||
last_split_point = found_index + current_index;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
pub fn normalize_windows_path(path_to_change: impl AsRef<Path>) -> PathBuf {
|
||||
let path = path_to_change.as_ref();
|
||||
|
||||
// Don't do anything, because network path may be case intensive
|
||||
if path.to_string_lossy().starts_with('\\') {
|
||||
return path.to_path_buf();
|
||||
}
|
||||
|
||||
match path.to_str() {
|
||||
Some(path) if path.is_char_boundary(1) => {
|
||||
let replaced = path.replace('/', "\\");
|
||||
let mut new_path = OsString::new();
|
||||
if replaced[1..].starts_with(':') {
|
||||
new_path.push(replaced[..1].to_ascii_uppercase());
|
||||
new_path.push(replaced[1..].to_ascii_lowercase());
|
||||
} else {
|
||||
new_path.push(replaced.to_ascii_lowercase());
|
||||
}
|
||||
_ => path.to_path_buf(),
|
||||
PathBuf::from(new_path)
|
||||
}
|
||||
_ => path.to_path_buf(),
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -624,7 +611,8 @@ mod test {
|
|||
use std::path::{Path, PathBuf};
|
||||
use tempfile::tempdir;
|
||||
|
||||
use crate::common::{remove_folder_if_contains_only_empty_folders, Common};
|
||||
use crate::common::{normalize_windows_path, regex_check, remove_folder_if_contains_only_empty_folders};
|
||||
use crate::common_items::{new_excluded_item, ExcludedItems};
|
||||
|
||||
#[test]
|
||||
fn test_remove_folder_if_contains_only_empty_folders() {
|
||||
|
@ -652,39 +640,42 @@ mod test {
|
|||
|
||||
#[test]
|
||||
fn test_regex() {
|
||||
assert!(Common::regex_check("*home*", "/home/rafal"));
|
||||
assert!(Common::regex_check("*home", "/home"));
|
||||
assert!(Common::regex_check("*home/", "/home/"));
|
||||
assert!(Common::regex_check("*home/*", "/home/"));
|
||||
assert!(Common::regex_check("*.git*", "/home/.git"));
|
||||
assert!(Common::regex_check("*/home/rafal*rafal*rafal*rafal*", "/home/rafal/rafalrafalrafal"));
|
||||
assert!(Common::regex_check("AAA", "AAA"));
|
||||
assert!(Common::regex_check("AAA*", "AAABDGG/QQPW*"));
|
||||
assert!(!Common::regex_check("*home", "/home/"));
|
||||
assert!(!Common::regex_check("*home", "/homefasfasfasfasf/"));
|
||||
assert!(!Common::regex_check("*home", "/homefasfasfasfasf"));
|
||||
assert!(!Common::regex_check("rafal*afal*fal", "rafal"));
|
||||
assert!(!Common::regex_check("rafal*a", "rafal"));
|
||||
assert!(!Common::regex_check("AAAAAAAA****", "/AAAAAAAAAAAAAAAAA"));
|
||||
assert!(!Common::regex_check("*.git/*", "/home/.git"));
|
||||
assert!(!Common::regex_check("*home/*koc", "/koc/home/"));
|
||||
assert!(!Common::regex_check("*home/", "/home"));
|
||||
assert!(!Common::regex_check("*TTT", "/GGG"));
|
||||
assert!(regex_check(&new_excluded_item("*home*"), "/home/rafal"));
|
||||
assert!(regex_check(&new_excluded_item("*home"), "/home"));
|
||||
assert!(regex_check(&new_excluded_item("*home/"), "/home/"));
|
||||
assert!(regex_check(&new_excluded_item("*home/*"), "/home/"));
|
||||
assert!(regex_check(&new_excluded_item("*.git*"), "/home/.git"));
|
||||
assert!(regex_check(&new_excluded_item("*/home/rafal*rafal*rafal*rafal*"), "/home/rafal/rafalrafalrafal"));
|
||||
assert!(regex_check(&new_excluded_item("AAA"), "AAA"));
|
||||
assert!(regex_check(&new_excluded_item("AAA*"), "AAABDGG/QQPW*"));
|
||||
assert!(!regex_check(&new_excluded_item("*home"), "/home/"));
|
||||
assert!(!regex_check(&new_excluded_item("*home"), "/homefasfasfasfasf/"));
|
||||
assert!(!regex_check(&new_excluded_item("*home"), "/homefasfasfasfasf"));
|
||||
assert!(!regex_check(&new_excluded_item("rafal*afal*fal"), "rafal"));
|
||||
assert!(!regex_check(&new_excluded_item("rafal*a"), "rafal"));
|
||||
assert!(!regex_check(&new_excluded_item("AAAAAAAA****"), "/AAAAAAAAAAAAAAAAA"));
|
||||
assert!(!regex_check(&new_excluded_item("*.git/*"), "/home/.git"));
|
||||
assert!(!regex_check(&new_excluded_item("*home/*koc"), "/koc/home/"));
|
||||
assert!(!regex_check(&new_excluded_item("*home/"), "/home"));
|
||||
assert!(!regex_check(&new_excluded_item("*TTT"), "/GGG"));
|
||||
assert!(regex_check(
|
||||
&new_excluded_item("*/home/*/.local/share/containers"),
|
||||
"/var/home/roman/.local/share/containers"
|
||||
));
|
||||
|
||||
#[cfg(target_family = "windows")]
|
||||
{
|
||||
assert!(Common::regex_check("*\\home", "C:\\home"));
|
||||
assert!(Common::regex_check("*/home", "C:\\home"));
|
||||
if cfg!(target_family = "windows") {
|
||||
assert!(regex_check(&new_excluded_item("*\\home"), "C:\\home"));
|
||||
assert!(regex_check(&new_excluded_item("*/home"), "C:\\home"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_windows_path() {
|
||||
assert_eq!(PathBuf::from("C:\\path.txt"), Common::normalize_windows_path("c:/PATH.tXt"));
|
||||
assert_eq!(PathBuf::from("H:\\reka\\weza\\roman.txt"), Common::normalize_windows_path("h:/RekA/Weza\\roMan.Txt"));
|
||||
assert_eq!(PathBuf::from("T:\\a"), Common::normalize_windows_path("T:\\A"));
|
||||
assert_eq!(PathBuf::from("\\\\aBBa"), Common::normalize_windows_path("\\\\aBBa"));
|
||||
assert_eq!(PathBuf::from("a"), Common::normalize_windows_path("a"));
|
||||
assert_eq!(PathBuf::from(""), Common::normalize_windows_path(""));
|
||||
assert_eq!(PathBuf::from("C:\\path.txt"), normalize_windows_path("c:/PATH.tXt"));
|
||||
assert_eq!(PathBuf::from("H:\\reka\\weza\\roman.txt"), normalize_windows_path("h:/RekA/Weza\\roMan.Txt"));
|
||||
assert_eq!(PathBuf::from("T:\\a"), normalize_windows_path("T:\\A"));
|
||||
assert_eq!(PathBuf::from("\\\\aBBa"), normalize_windows_path("\\\\aBBa"));
|
||||
assert_eq!(PathBuf::from("a"), normalize_windows_path("a"));
|
||||
assert_eq!(PathBuf::from(""), normalize_windows_path(""));
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
use std::collections::BTreeMap;
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
use std::fs;
|
||||
use std::fs::{DirEntry, Metadata, ReadDir};
|
||||
use std::fs::{DirEntry, FileType, Metadata, ReadDir};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::Ordering;
|
||||
use std::time::UNIX_EPOCH;
|
||||
|
@ -321,8 +321,7 @@ pub enum DirTraversalResult<T: Ord + PartialOrd> {
|
|||
Stopped,
|
||||
}
|
||||
|
||||
fn entry_type(metadata: &Metadata) -> EntryType {
|
||||
let file_type = metadata.file_type();
|
||||
fn entry_type(file_type: FileType) -> EntryType {
|
||||
if file_type.is_dir() {
|
||||
EntryType::Dir
|
||||
} else if file_type.is_symlink() {
|
||||
|
@ -345,10 +344,10 @@ where
|
|||
|
||||
let mut all_warnings = vec![];
|
||||
let mut grouped_file_entries: BTreeMap<T, Vec<FileEntry>> = BTreeMap::new();
|
||||
let mut folder_entries: BTreeMap<PathBuf, FolderEntry> = BTreeMap::new();
|
||||
let mut folder_entries: HashMap<PathBuf, FolderEntry> = HashMap::new();
|
||||
|
||||
// Add root folders into result (only for empty folder collection)
|
||||
let mut folders_to_check: Vec<PathBuf> = Vec::with_capacity(1024 * 2); // This should be small enough too not see to big difference and big enough to store most of paths without needing to resize vector
|
||||
let mut folders_to_check: Vec<PathBuf> = Vec::with_capacity(1024 * 2);
|
||||
if self.collect == Collect::EmptyFolders {
|
||||
for dir in &self.root_dirs {
|
||||
folder_entries.insert(
|
||||
|
@ -395,24 +394,25 @@ where
|
|||
let mut folder_entries_list = vec![];
|
||||
|
||||
let Some(read_dir) = common_read_dir(current_folder, &mut warnings) else {
|
||||
set_as_not_empty_folder_list.push(current_folder.clone());
|
||||
return (dir_result, warnings, fe_result, set_as_not_empty_folder_list, folder_entries_list);
|
||||
};
|
||||
|
||||
let mut counter = 0;
|
||||
// Check every sub folder/file/link etc.
|
||||
'dir: for entry in read_dir {
|
||||
let Some((entry_data, metadata)) = common_get_entry_data_metadata(&entry, &mut warnings, current_folder) else {
|
||||
let Some(entry_data) = common_get_entry_data(&entry, &mut warnings, current_folder) else {
|
||||
continue;
|
||||
};
|
||||
let Ok(file_type) = entry_data.file_type() else { continue };
|
||||
|
||||
match (entry_type(&metadata), collect) {
|
||||
match (entry_type(file_type), collect) {
|
||||
(EntryType::Dir, Collect::Files | Collect::InvalidSymlinks) => {
|
||||
process_dir_in_file_symlink_mode(recursive_search, current_folder, entry_data, &directories, &mut dir_result, &mut warnings, &excluded_items);
|
||||
}
|
||||
(EntryType::Dir, Collect::EmptyFolders) => {
|
||||
counter += 1;
|
||||
process_dir_in_dir_mode(
|
||||
&metadata,
|
||||
current_folder,
|
||||
entry_data,
|
||||
&directories,
|
||||
|
@ -426,7 +426,6 @@ where
|
|||
(EntryType::File, Collect::Files) => {
|
||||
counter += 1;
|
||||
process_file_in_file_mode(
|
||||
&metadata,
|
||||
entry_data,
|
||||
&mut warnings,
|
||||
&mut fe_result,
|
||||
|
@ -456,7 +455,6 @@ where
|
|||
(EntryType::Symlink, Collect::InvalidSymlinks) => {
|
||||
counter += 1;
|
||||
process_symlink_in_symlink_mode(
|
||||
&metadata,
|
||||
entry_data,
|
||||
&mut warnings,
|
||||
&mut fe_result,
|
||||
|
@ -513,7 +511,7 @@ where
|
|||
warnings: all_warnings,
|
||||
},
|
||||
Collect::EmptyFolders => DirTraversalResult::SuccessFolders {
|
||||
folder_entries,
|
||||
folder_entries: folder_entries.into_iter().collect(),
|
||||
warnings: all_warnings,
|
||||
},
|
||||
}
|
||||
|
@ -521,7 +519,6 @@ where
|
|||
}
|
||||
|
||||
fn process_file_in_file_mode(
|
||||
metadata: &Metadata,
|
||||
entry_data: &DirEntry,
|
||||
warnings: &mut Vec<String>,
|
||||
fe_result: &mut Vec<FileEntry>,
|
||||
|
@ -540,26 +537,30 @@ fn process_file_in_file_mode(
|
|||
return;
|
||||
}
|
||||
|
||||
let current_file_name = current_folder.join(entry_data.file_name());
|
||||
if excluded_items.is_excluded(¤t_file_name) {
|
||||
return;
|
||||
}
|
||||
|
||||
#[cfg(target_family = "unix")]
|
||||
if directories.exclude_other_filesystems() {
|
||||
match directories.is_on_other_filesystems(¤t_file_name) {
|
||||
Ok(true) => return,
|
||||
Err(e) => warnings.push(e),
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
|
||||
let Some(metadata) = common_get_metadata_dir(entry_data, warnings, current_folder) else {
|
||||
return;
|
||||
};
|
||||
|
||||
if (minimal_file_size..=maximal_file_size).contains(&metadata.len()) {
|
||||
let current_file_name = current_folder.join(entry_data.file_name());
|
||||
if excluded_items.is_excluded(¤t_file_name) {
|
||||
return;
|
||||
}
|
||||
|
||||
#[cfg(target_family = "unix")]
|
||||
if directories.exclude_other_filesystems() {
|
||||
match directories.is_on_other_filesystems(¤t_file_name) {
|
||||
Ok(true) => return,
|
||||
Err(e) => warnings.push(e),
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
|
||||
// Creating new file entry
|
||||
let fe: FileEntry = FileEntry {
|
||||
path: current_file_name.clone(),
|
||||
size: metadata.len(),
|
||||
modified_date: get_modified_time(metadata, warnings, ¤t_file_name, false),
|
||||
modified_date: get_modified_time(&metadata, warnings, ¤t_file_name, false),
|
||||
hash: String::new(),
|
||||
symlink_info: None,
|
||||
};
|
||||
|
@ -569,7 +570,6 @@ fn process_file_in_file_mode(
|
|||
}
|
||||
|
||||
fn process_dir_in_dir_mode(
|
||||
metadata: &Metadata,
|
||||
current_folder: &Path,
|
||||
entry_data: &DirEntry,
|
||||
directories: &Directories,
|
||||
|
@ -594,13 +594,18 @@ fn process_dir_in_dir_mode(
|
|||
}
|
||||
}
|
||||
|
||||
let Some(metadata) = common_get_metadata_dir(entry_data, warnings, current_folder) else {
|
||||
set_as_not_empty_folder_list.push(current_folder.to_path_buf());
|
||||
return;
|
||||
};
|
||||
|
||||
dir_result.push(next_folder.clone());
|
||||
folder_entries_list.push((
|
||||
next_folder,
|
||||
FolderEntry {
|
||||
parent_path: Some(current_folder.to_path_buf()),
|
||||
is_empty: FolderEmptiness::Maybe,
|
||||
modified_date: get_modified_time(metadata, warnings, current_folder, true),
|
||||
modified_date: get_modified_time(&metadata, warnings, current_folder, true),
|
||||
},
|
||||
));
|
||||
}
|
||||
|
@ -640,7 +645,6 @@ fn process_dir_in_file_symlink_mode(
|
|||
}
|
||||
|
||||
fn process_symlink_in_symlink_mode(
|
||||
metadata: &Metadata,
|
||||
entry_data: &DirEntry,
|
||||
warnings: &mut Vec<String>,
|
||||
fe_result: &mut Vec<FileEntry>,
|
||||
|
@ -671,6 +675,10 @@ fn process_symlink_in_symlink_mode(
|
|||
}
|
||||
}
|
||||
|
||||
let Some(metadata) = common_get_metadata_dir(entry_data, warnings, current_folder) else {
|
||||
return;
|
||||
};
|
||||
|
||||
let mut destination_path = PathBuf::new();
|
||||
let type_of_error;
|
||||
|
||||
|
@ -709,7 +717,7 @@ fn process_symlink_in_symlink_mode(
|
|||
// Creating new file entry
|
||||
let fe: FileEntry = FileEntry {
|
||||
path: current_file_name.clone(),
|
||||
modified_date: get_modified_time(metadata, warnings, ¤t_file_name, false),
|
||||
modified_date: get_modified_time(&metadata, warnings, ¤t_file_name, false),
|
||||
size: 0,
|
||||
hash: String::new(),
|
||||
symlink_info: Some(SymlinkInfo { destination_path, type_of_error }),
|
||||
|
@ -731,6 +739,32 @@ pub fn common_read_dir(current_folder: &Path, warnings: &mut Vec<String>) -> Opt
|
|||
}
|
||||
}
|
||||
}
|
||||
pub fn common_get_entry_data<'a>(entry: &'a Result<DirEntry, std::io::Error>, warnings: &mut Vec<String>, current_folder: &Path) -> Option<&'a DirEntry> {
|
||||
let entry_data = match entry {
|
||||
Ok(t) => t,
|
||||
Err(e) => {
|
||||
warnings.push(flc!(
|
||||
"core_cannot_read_entry_dir",
|
||||
generate_translation_hashmap(vec![("dir", current_folder.display().to_string()), ("reason", e.to_string())])
|
||||
));
|
||||
return None;
|
||||
}
|
||||
};
|
||||
Some(entry_data)
|
||||
}
|
||||
pub fn common_get_metadata_dir(entry_data: &DirEntry, warnings: &mut Vec<String>, current_folder: &Path) -> Option<Metadata> {
|
||||
let metadata: Metadata = match entry_data.metadata() {
|
||||
Ok(t) => t,
|
||||
Err(e) => {
|
||||
warnings.push(flc!(
|
||||
"core_cannot_read_metadata_dir",
|
||||
generate_translation_hashmap(vec![("dir", current_folder.display().to_string()), ("reason", e.to_string())])
|
||||
));
|
||||
return None;
|
||||
}
|
||||
};
|
||||
Some(metadata)
|
||||
}
|
||||
|
||||
pub fn common_get_entry_data_metadata<'a>(entry: &'a Result<DirEntry, std::io::Error>, warnings: &mut Vec<String>, current_folder: &Path) -> Option<(&'a DirEntry, Metadata)> {
|
||||
let entry_data = match entry {
|
||||
|
@ -797,16 +831,17 @@ pub fn get_lowercase_name(entry_data: &DirEntry, warnings: &mut Vec<String>) ->
|
|||
Some(name)
|
||||
}
|
||||
|
||||
fn set_as_not_empty_folder(folder_entries: &mut BTreeMap<PathBuf, FolderEntry>, current_folder: &Path) {
|
||||
fn set_as_not_empty_folder(folder_entries: &mut HashMap<PathBuf, FolderEntry>, current_folder: &Path) {
|
||||
let mut d = folder_entries.get_mut(current_folder).unwrap();
|
||||
// Not folder so it may be a file or symbolic link so it isn't empty
|
||||
d.is_empty = FolderEmptiness::No;
|
||||
// Loop to recursively set as non empty this and all his parent folders
|
||||
loop {
|
||||
d.is_empty = FolderEmptiness::No;
|
||||
if d.parent_path.is_some() {
|
||||
let cf = d.parent_path.clone().unwrap();
|
||||
d = folder_entries.get_mut(&cf).unwrap();
|
||||
if d.is_empty == FolderEmptiness::No {
|
||||
break; // Already set as non empty, so one of child already set it to non empty
|
||||
}
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
|
|
|
@ -1,8 +1,8 @@
|
|||
use crate::common::normalize_windows_path;
|
||||
use std::path::{Path, PathBuf};
|
||||
#[cfg(target_family = "unix")]
|
||||
use std::{fs, os::unix::fs::MetadataExt};
|
||||
|
||||
use crate::common::Common;
|
||||
use crate::common_messages::Messages;
|
||||
use crate::flc;
|
||||
use crate::localizer_core::generate_translation_hashmap;
|
||||
|
@ -154,9 +154,9 @@ impl Directories {
|
|||
let mut optimized_excluded: Vec<PathBuf> = Vec::new();
|
||||
|
||||
if cfg!(target_family = "windows") {
|
||||
self.included_directories = self.included_directories.iter().map(Common::normalize_windows_path).collect();
|
||||
self.excluded_directories = self.excluded_directories.iter().map(Common::normalize_windows_path).collect();
|
||||
self.reference_directories = self.reference_directories.iter().map(Common::normalize_windows_path).collect();
|
||||
self.included_directories = self.included_directories.iter().map(normalize_windows_path).collect();
|
||||
self.excluded_directories = self.excluded_directories.iter().map(normalize_windows_path).collect();
|
||||
self.reference_directories = self.reference_directories.iter().map(normalize_windows_path).collect();
|
||||
}
|
||||
|
||||
// Remove duplicated entries like: "/", "/"
|
||||
|
@ -309,7 +309,7 @@ impl Directories {
|
|||
pub fn is_excluded(&self, path: impl AsRef<Path>) -> bool {
|
||||
let path = path.as_ref();
|
||||
#[cfg(target_family = "windows")]
|
||||
let path = Common::normalize_windows_path(path);
|
||||
let path = normalize_windows_path(path);
|
||||
// We're assuming that `excluded_directories` are already normalized
|
||||
self.excluded_directories.iter().any(|p| p.as_path() == path)
|
||||
}
|
||||
|
|
|
@ -1,6 +1,9 @@
|
|||
#[cfg(not(target_family = "unix"))]
|
||||
use crate::common::normalize_windows_path;
|
||||
use crate::common::regex_check;
|
||||
use std::path::Path;
|
||||
|
||||
use crate::common::Common;
|
||||
use crate::common_messages::Messages;
|
||||
|
||||
#[cfg(target_family = "unix")]
|
||||
pub const DEFAULT_EXCLUDED_DIRECTORIES: &[&str] = &["/proc", "/dev", "/sys", "/run", "/snap"];
|
||||
|
@ -14,7 +17,15 @@ pub const DEFAULT_EXCLUDED_ITEMS: &str = "*\\.git\\*,*\\node_modules\\*,*\\lost+
|
|||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct ExcludedItems {
|
||||
pub items: Vec<String>,
|
||||
expressions: Vec<String>,
|
||||
connected_expressions: Vec<SingleExcludedItem>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct SingleExcludedItem {
|
||||
pub expression: String,
|
||||
pub expression_splits: Vec<String>,
|
||||
pub unique_extensions_splits: Vec<String>,
|
||||
}
|
||||
|
||||
impl ExcludedItems {
|
||||
|
@ -22,12 +33,16 @@ impl ExcludedItems {
|
|||
Default::default()
|
||||
}
|
||||
|
||||
pub fn set_excluded_items(&mut self, excluded_items: Vec<String>) -> (Vec<String>, Vec<String>, Vec<String>) {
|
||||
let messages: Vec<String> = Vec::new();
|
||||
pub fn new_from(excluded_items: Vec<String>) -> Self {
|
||||
let mut s = Self::new();
|
||||
s.set_excluded_items(excluded_items);
|
||||
s
|
||||
}
|
||||
|
||||
pub fn set_excluded_items(&mut self, excluded_items: Vec<String>) -> Messages {
|
||||
let mut warnings: Vec<String> = Vec::new();
|
||||
let errors: Vec<String> = Vec::new();
|
||||
if excluded_items.is_empty() {
|
||||
return (messages, warnings, errors);
|
||||
return Messages::new();
|
||||
}
|
||||
|
||||
let expressions: Vec<String> = excluded_items;
|
||||
|
@ -54,19 +69,47 @@ impl ExcludedItems {
|
|||
|
||||
checked_expressions.push(expression);
|
||||
}
|
||||
self.items = checked_expressions;
|
||||
(messages, warnings, errors)
|
||||
|
||||
for checked_expression in &checked_expressions {
|
||||
let item = new_excluded_item(checked_expression);
|
||||
self.expressions.push(item.expression.clone());
|
||||
self.connected_expressions.push(item);
|
||||
}
|
||||
Messages {
|
||||
messages: vec![],
|
||||
warnings,
|
||||
errors: vec![],
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_excluded_items(&self) -> &Vec<String> {
|
||||
&self.expressions
|
||||
}
|
||||
pub fn is_excluded(&self, path: impl AsRef<Path>) -> bool {
|
||||
if self.connected_expressions.is_empty() {
|
||||
return false;
|
||||
}
|
||||
#[cfg(target_family = "windows")]
|
||||
let path = Common::normalize_windows_path(path);
|
||||
let path = normalize_windows_path(path);
|
||||
|
||||
for expression in &self.items {
|
||||
if Common::regex_check(expression, &path) {
|
||||
for expression in &self.connected_expressions {
|
||||
if regex_check(expression, &path) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
pub fn new_excluded_item(expression: &str) -> SingleExcludedItem {
|
||||
let expression = expression.trim().to_string();
|
||||
let expression_splits: Vec<String> = expression.split('*').filter_map(|e| if e.is_empty() { None } else { Some(e.to_string()) }).collect();
|
||||
let mut unique_extensions_splits = expression_splits.clone();
|
||||
unique_extensions_splits.sort();
|
||||
unique_extensions_splits.dedup();
|
||||
SingleExcludedItem {
|
||||
expression,
|
||||
expression_splits,
|
||||
unique_extensions_splits,
|
||||
}
|
||||
}
|
||||
|
|
|
@ -173,10 +173,8 @@ pub trait CommonData {
|
|||
}
|
||||
|
||||
fn set_excluded_items(&mut self, excluded_items: Vec<String>) {
|
||||
let (messages, warnings, errors) = self.get_cd_mut().excluded_items.set_excluded_items(excluded_items);
|
||||
self.get_cd_mut().text_messages.messages.extend(messages);
|
||||
self.get_cd_mut().text_messages.warnings.extend(warnings);
|
||||
self.get_cd_mut().text_messages.errors.extend(errors);
|
||||
let messages = self.get_cd_mut().excluded_items.set_excluded_items(excluded_items);
|
||||
self.get_cd_mut().text_messages.extend_with_another_messages(messages);
|
||||
}
|
||||
|
||||
fn optimize_dirs_before_start(&mut self) {
|
||||
|
|
|
@ -982,7 +982,9 @@ impl PrintResults for DuplicateFinder {
|
|||
writeln!(
|
||||
writer,
|
||||
"Results of searching {:?} with excluded directories {:?} and excluded items {:?}",
|
||||
self.common_data.directories.included_directories, self.common_data.directories.excluded_directories, self.common_data.excluded_items.items
|
||||
self.common_data.directories.included_directories,
|
||||
self.common_data.directories.excluded_directories,
|
||||
self.common_data.excluded_items.get_excluded_items()
|
||||
)?;
|
||||
|
||||
match self.check_method {
|
||||
|
|
|
@ -127,7 +127,9 @@ impl PrintResults for EmptyFiles {
|
|||
writeln!(
|
||||
writer,
|
||||
"Results of searching {:?} with excluded directories {:?} and excluded items {:?}",
|
||||
self.common_data.directories.included_directories, self.common_data.directories.excluded_directories, self.common_data.excluded_items.items
|
||||
self.common_data.directories.included_directories,
|
||||
self.common_data.directories.excluded_directories,
|
||||
self.common_data.excluded_items.get_excluded_items()
|
||||
)?;
|
||||
|
||||
if !self.empty_files.is_empty() {
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
|
||||
use std::fs::{DirEntry, Metadata};
|
||||
use std::fs::DirEntry;
|
||||
use std::io::Write;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::Ordering;
|
||||
|
@ -24,7 +24,7 @@ use crate::common::{
|
|||
send_info_and_wait_for_ending_all_threads, HEIC_EXTENSIONS, IMAGE_RS_SIMILAR_IMAGES_EXTENSIONS, RAW_IMAGE_EXTENSIONS,
|
||||
};
|
||||
use crate::common_cache::{get_similar_images_cache_file, load_cache_from_file_generalized_by_path, save_cache_to_file_generalized};
|
||||
use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData, ToolType};
|
||||
use crate::common_dir_traversal::{common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData, ToolType};
|
||||
use crate::common_tool::{CommonData, CommonToolData, DeleteMethod};
|
||||
use crate::common_traits::{DebugPrint, PrintResults, ResultEntry};
|
||||
use crate::flc;
|
||||
|
@ -146,7 +146,7 @@ impl SimilarImages {
|
|||
|
||||
#[fun_time(message = "check_for_similar_images", level = "debug")]
|
||||
fn check_for_similar_images(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&Sender<ProgressData>>) -> bool {
|
||||
let mut folders_to_check: Vec<PathBuf> = Vec::with_capacity(1024 * 2); // This should be small enough too not see to big difference and big enough to store most of paths without needing to resize vector
|
||||
let mut folders_to_check: Vec<PathBuf> = Vec::with_capacity(1024 * 2);
|
||||
|
||||
if !self.common_data.allowed_extensions.using_custom_extensions() {
|
||||
self.common_data.allowed_extensions.extend_allowed_extensions(IMAGE_RS_SIMILAR_IMAGES_EXTENSIONS);
|
||||
|
@ -188,23 +188,26 @@ impl SimilarImages {
|
|||
};
|
||||
|
||||
for entry in read_dir {
|
||||
let Some((entry_data, metadata)) = common_get_entry_data_metadata(&entry, &mut warnings, current_folder) else {
|
||||
let Ok(entry_data) = entry else {
|
||||
continue;
|
||||
};
|
||||
let Ok(file_type) = entry_data.file_type() else {
|
||||
continue;
|
||||
};
|
||||
|
||||
if metadata.is_dir() {
|
||||
if file_type.is_dir() {
|
||||
check_folder_children(
|
||||
&mut dir_result,
|
||||
&mut warnings,
|
||||
current_folder,
|
||||
entry_data,
|
||||
&entry_data,
|
||||
self.common_data.recursive_search,
|
||||
&self.common_data.directories,
|
||||
&self.common_data.excluded_items,
|
||||
);
|
||||
} else if metadata.is_file() {
|
||||
} else if file_type.is_file() {
|
||||
atomic_counter.fetch_add(1, Ordering::Relaxed);
|
||||
self.add_file_entry(&metadata, current_folder, entry_data, &mut fe_result, &mut warnings);
|
||||
self.add_file_entry(current_folder, &entry_data, &mut fe_result, &mut warnings);
|
||||
}
|
||||
}
|
||||
(dir_result, warnings, fe_result)
|
||||
|
@ -229,7 +232,7 @@ impl SimilarImages {
|
|||
true
|
||||
}
|
||||
|
||||
fn add_file_entry(&self, metadata: &Metadata, current_folder: &Path, entry_data: &DirEntry, fe_result: &mut Vec<(String, FileEntry)>, warnings: &mut Vec<String>) {
|
||||
fn add_file_entry(&self, current_folder: &Path, entry_data: &DirEntry, fe_result: &mut Vec<(String, FileEntry)>, warnings: &mut Vec<String>) {
|
||||
let Some(file_name_lowercase) = get_lowercase_name(entry_data, warnings) else {
|
||||
return;
|
||||
};
|
||||
|
@ -238,18 +241,22 @@ impl SimilarImages {
|
|||
return;
|
||||
}
|
||||
|
||||
let current_file_name = current_folder.join(entry_data.file_name());
|
||||
if self.common_data.excluded_items.is_excluded(¤t_file_name) {
|
||||
return;
|
||||
}
|
||||
|
||||
let Ok(metadata) = entry_data.metadata() else {
|
||||
return;
|
||||
};
|
||||
|
||||
// Checking files
|
||||
if (self.common_data.minimal_file_size..=self.common_data.maximal_file_size).contains(&metadata.len()) {
|
||||
let current_file_name = current_folder.join(entry_data.file_name());
|
||||
if self.common_data.excluded_items.is_excluded(¤t_file_name) {
|
||||
return;
|
||||
}
|
||||
|
||||
let fe: FileEntry = FileEntry {
|
||||
path: current_file_name.clone(),
|
||||
size: metadata.len(),
|
||||
dimensions: String::new(),
|
||||
modified_date: get_modified_time(metadata, warnings, ¤t_file_name, false),
|
||||
modified_date: get_modified_time(&metadata, warnings, ¤t_file_name, false),
|
||||
hash: Vec::new(),
|
||||
similarity: 0,
|
||||
};
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
use std::collections::{BTreeMap, BTreeSet, HashMap};
|
||||
use std::fs::{DirEntry, Metadata};
|
||||
use std::fs::DirEntry;
|
||||
use std::io::Write;
|
||||
use std::mem;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
@ -18,7 +18,7 @@ use crate::common::{
|
|||
check_folder_children, check_if_stop_received, delete_files_custom, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, VIDEO_FILES_EXTENSIONS,
|
||||
};
|
||||
use crate::common_cache::{get_similar_videos_cache_file, load_cache_from_file_generalized_by_path, save_cache_to_file_generalized};
|
||||
use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData, ToolType};
|
||||
use crate::common_dir_traversal::{common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData, ToolType};
|
||||
use crate::common_tool::{CommonData, CommonToolData, DeleteMethod};
|
||||
use crate::common_traits::{DebugPrint, PrintResults, ResultEntry};
|
||||
use crate::flc;
|
||||
|
@ -130,7 +130,7 @@ impl SimilarVideos {
|
|||
|
||||
#[fun_time(message = "check_for_similar_videos", level = "debug")]
|
||||
fn check_for_similar_videos(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&Sender<ProgressData>>) -> bool {
|
||||
let mut folders_to_check: Vec<PathBuf> = Vec::with_capacity(1024 * 2); // This should be small enough too not see to big difference and big enough to store most of paths without needing to resize vector
|
||||
let mut folders_to_check: Vec<PathBuf> = Vec::with_capacity(1024 * 2);
|
||||
|
||||
if !self.common_data.allowed_extensions.using_custom_extensions() {
|
||||
self.common_data.allowed_extensions.extend_allowed_extensions(VIDEO_FILES_EXTENSIONS);
|
||||
|
@ -168,23 +168,26 @@ impl SimilarVideos {
|
|||
|
||||
// Check every sub folder/file/link etc.
|
||||
for entry in read_dir {
|
||||
let Some((entry_data, metadata)) = common_get_entry_data_metadata(&entry, &mut warnings, current_folder) else {
|
||||
let Ok(entry_data) = entry else {
|
||||
continue;
|
||||
};
|
||||
let Ok(file_type) = entry_data.file_type() else {
|
||||
continue;
|
||||
};
|
||||
|
||||
if metadata.is_dir() {
|
||||
if file_type.is_dir() {
|
||||
check_folder_children(
|
||||
&mut dir_result,
|
||||
&mut warnings,
|
||||
current_folder,
|
||||
entry_data,
|
||||
&entry_data,
|
||||
self.common_data.recursive_search,
|
||||
&self.common_data.directories,
|
||||
&self.common_data.excluded_items,
|
||||
);
|
||||
} else if metadata.is_file() {
|
||||
} else if file_type.is_file() {
|
||||
atomic_counter.fetch_add(1, Ordering::Relaxed);
|
||||
self.add_video_file_entry(&metadata, entry_data, &mut fe_result, &mut warnings, current_folder);
|
||||
self.add_video_file_entry(&entry_data, &mut fe_result, &mut warnings, current_folder);
|
||||
}
|
||||
}
|
||||
(dir_result, warnings, fe_result)
|
||||
|
@ -209,7 +212,7 @@ impl SimilarVideos {
|
|||
true
|
||||
}
|
||||
|
||||
fn add_video_file_entry(&self, metadata: &Metadata, entry_data: &DirEntry, fe_result: &mut Vec<(String, FileEntry)>, warnings: &mut Vec<String>, current_folder: &Path) {
|
||||
fn add_video_file_entry(&self, entry_data: &DirEntry, fe_result: &mut Vec<(String, FileEntry)>, warnings: &mut Vec<String>, current_folder: &Path) {
|
||||
let Some(file_name_lowercase) = get_lowercase_name(entry_data, warnings) else {
|
||||
return;
|
||||
};
|
||||
|
@ -218,18 +221,22 @@ impl SimilarVideos {
|
|||
return;
|
||||
}
|
||||
|
||||
let current_file_name = current_folder.join(entry_data.file_name());
|
||||
if self.common_data.excluded_items.is_excluded(¤t_file_name) {
|
||||
return;
|
||||
}
|
||||
let current_file_name_str = current_file_name.to_string_lossy().to_string();
|
||||
|
||||
let Ok(metadata) = entry_data.metadata() else {
|
||||
return;
|
||||
};
|
||||
|
||||
// Checking files
|
||||
if (self.common_data.minimal_file_size..=self.common_data.maximal_file_size).contains(&metadata.len()) {
|
||||
let current_file_name = current_folder.join(entry_data.file_name());
|
||||
if self.common_data.excluded_items.is_excluded(¤t_file_name) {
|
||||
return;
|
||||
}
|
||||
let current_file_name_str = current_file_name.to_string_lossy().to_string();
|
||||
|
||||
let fe: FileEntry = FileEntry {
|
||||
path: current_file_name.clone(),
|
||||
size: metadata.len(),
|
||||
modified_date: get_modified_time(metadata, warnings, ¤t_file_name, false),
|
||||
modified_date: get_modified_time(&metadata, warnings, ¤t_file_name, false),
|
||||
vhash: Default::default(),
|
||||
error: String::new(),
|
||||
};
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
use std::fs;
|
||||
use std::fs::{DirEntry, Metadata};
|
||||
use std::fs::DirEntry;
|
||||
use std::io::prelude::*;
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
@ -12,7 +12,7 @@ use rayon::prelude::*;
|
|||
use serde::Serialize;
|
||||
|
||||
use crate::common::{check_folder_children, check_if_stop_received, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads};
|
||||
use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData, ToolType};
|
||||
use crate::common_dir_traversal::{common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData, ToolType};
|
||||
use crate::common_tool::{CommonData, CommonToolData, DeleteMethod};
|
||||
use crate::common_traits::*;
|
||||
|
||||
|
@ -71,7 +71,7 @@ impl Temporary {
|
|||
|
||||
#[fun_time(message = "check_files", level = "debug")]
|
||||
fn check_files(&mut self, stop_receiver: Option<&Receiver<()>>, progress_sender: Option<&Sender<ProgressData>>) -> bool {
|
||||
let mut folders_to_check: Vec<PathBuf> = Vec::with_capacity(1024 * 2); // This should be small enough too not see to big difference and big enough to store most of paths without needing to resize vector
|
||||
let mut folders_to_check: Vec<PathBuf> = Vec::with_capacity(1024 * 2);
|
||||
|
||||
// Add root folders for finding
|
||||
for id in &self.common_data.directories.included_directories {
|
||||
|
@ -100,22 +100,25 @@ impl Temporary {
|
|||
|
||||
// Check every sub folder/file/link etc.
|
||||
for entry in read_dir {
|
||||
let Some((entry_data, metadata)) = common_get_entry_data_metadata(&entry, &mut warnings, current_folder) else {
|
||||
let Ok(entry_data) = entry else {
|
||||
continue;
|
||||
};
|
||||
let Ok(file_type) = entry_data.file_type() else {
|
||||
continue;
|
||||
};
|
||||
|
||||
if metadata.is_dir() {
|
||||
if file_type.is_dir() {
|
||||
check_folder_children(
|
||||
&mut dir_result,
|
||||
&mut warnings,
|
||||
current_folder,
|
||||
entry_data,
|
||||
&entry_data,
|
||||
self.common_data.recursive_search,
|
||||
&self.common_data.directories,
|
||||
&self.common_data.excluded_items,
|
||||
);
|
||||
} else if metadata.is_file() {
|
||||
if let Some(file_entry) = self.get_file_entry(&metadata, &atomic_counter, entry_data, &mut warnings, current_folder) {
|
||||
} else if file_type.is_file() {
|
||||
if let Some(file_entry) = self.get_file_entry(&atomic_counter, &entry_data, &mut warnings, current_folder) {
|
||||
fe_result.push(file_entry);
|
||||
}
|
||||
}
|
||||
|
@ -142,14 +145,7 @@ impl Temporary {
|
|||
|
||||
true
|
||||
}
|
||||
pub fn get_file_entry(
|
||||
&self,
|
||||
metadata: &Metadata,
|
||||
atomic_counter: &Arc<AtomicUsize>,
|
||||
entry_data: &DirEntry,
|
||||
warnings: &mut Vec<String>,
|
||||
current_folder: &Path,
|
||||
) -> Option<FileEntry> {
|
||||
pub fn get_file_entry(&self, atomic_counter: &Arc<AtomicUsize>, entry_data: &DirEntry, warnings: &mut Vec<String>, current_folder: &Path) -> Option<FileEntry> {
|
||||
atomic_counter.fetch_add(1, Ordering::Relaxed);
|
||||
|
||||
let file_name_lowercase = get_lowercase_name(entry_data, warnings)?;
|
||||
|
@ -162,10 +158,14 @@ impl Temporary {
|
|||
return None;
|
||||
}
|
||||
|
||||
let Ok(metadata) = entry_data.metadata() else {
|
||||
return None;
|
||||
};
|
||||
|
||||
// Creating new file entry
|
||||
Some(FileEntry {
|
||||
path: current_file_name.clone(),
|
||||
modified_date: get_modified_time(metadata, warnings, ¤t_file_name, false),
|
||||
modified_date: get_modified_time(&metadata, warnings, ¤t_file_name, false),
|
||||
})
|
||||
}
|
||||
|
||||
|
@ -194,7 +194,9 @@ impl PrintResults for Temporary {
|
|||
writeln!(
|
||||
writer,
|
||||
"Results of searching {:?} with excluded directories {:?} and excluded items {:?}",
|
||||
self.common_data.directories.included_directories, self.common_data.directories.excluded_directories, self.common_data.excluded_items.items
|
||||
self.common_data.directories.included_directories,
|
||||
self.common_data.directories.excluded_directories,
|
||||
self.common_data.excluded_items.get_excluded_items()
|
||||
)?;
|
||||
writeln!(writer, "Found {} temporary files.\n", self.information.number_of_temporary_files)?;
|
||||
|
||||
|
|
|
@ -31,7 +31,7 @@ pacman -S mingw-w64-x86_64-czkawka-gui
|
|||
and you can create shortcut to `C:\msys64\mingw64\bin\czkawka_gui.exe`
|
||||
|
||||
## Compilation
|
||||
Compilation of gui is harder that compilation cli or core, because uses gtk4 which is written in C and also requires a lot build and runtime dependencies.
|
||||
Compilation of gui is harder than compilation cli or core, because uses gtk4 which is written in C and also requires a lot build and runtime dependencies.
|
||||
|
||||
### Requirements
|
||||
| Program | Minimal version |
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
use std::collections::BTreeMap;
|
||||
use std::fs;
|
||||
use std::fs::Metadata;
|
||||
use std::fs::FileType;
|
||||
|
||||
use gtk4::prelude::*;
|
||||
use gtk4::{Align, CheckButton, Dialog, Orientation, ResponseType, TextView};
|
||||
|
@ -325,14 +325,14 @@ pub fn empty_folder_remover(
|
|||
break 'dir;
|
||||
}
|
||||
};
|
||||
let metadata: Metadata = match entry_data.metadata() {
|
||||
let file_type: FileType = match entry_data.file_type() {
|
||||
Ok(t) => t,
|
||||
Err(_inspected) => {
|
||||
error_happened = true;
|
||||
break 'dir;
|
||||
}
|
||||
};
|
||||
if metadata.is_dir() {
|
||||
if file_type.is_dir() {
|
||||
next_folder = String::new()
|
||||
+ ¤t_folder
|
||||
+ "/"
|
||||
|
|
|
@ -1,9 +1,9 @@
|
|||
use czkawka_core::common::regex_check;
|
||||
use czkawka_core::common_items::new_excluded_item;
|
||||
use gtk4::prelude::*;
|
||||
use gtk4::{ResponseType, TreeIter, Window};
|
||||
use regex::Regex;
|
||||
|
||||
use czkawka_core::common::Common;
|
||||
|
||||
use crate::flg;
|
||||
use crate::gui_structs::gui_data::GuiData;
|
||||
use crate::help_functions::*;
|
||||
|
@ -369,6 +369,11 @@ fn popover_custom_select_unselect(
|
|||
#[cfg(target_family = "windows")]
|
||||
let path_wildcard = path_wildcard.replace("/", "\\");
|
||||
|
||||
let name_wildcard_excluded = new_excluded_item(&name_wildcard);
|
||||
let name_wildcard_lowercase_excluded = new_excluded_item(&name_wildcard.to_lowercase());
|
||||
let path_wildcard_excluded = new_excluded_item(&path_wildcard);
|
||||
let path_wildcard_lowercase_excluded = new_excluded_item(&path_wildcard.to_lowercase());
|
||||
|
||||
if response_type == ResponseType::Ok {
|
||||
let check_path = check_button_path.is_active();
|
||||
let check_name = check_button_name.is_active();
|
||||
|
@ -441,22 +446,22 @@ fn popover_custom_select_unselect(
|
|||
} else {
|
||||
if check_name {
|
||||
if case_sensitive {
|
||||
if Common::regex_check(&name_wildcard, &name) {
|
||||
if regex_check(&name_wildcard_excluded, &name) {
|
||||
need_to_change_thing = true;
|
||||
}
|
||||
} else {
|
||||
if Common::regex_check(&name_wildcard.to_lowercase(), name.to_lowercase()) {
|
||||
if regex_check(&name_wildcard_lowercase_excluded, name.to_lowercase()) {
|
||||
need_to_change_thing = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
if check_path {
|
||||
if case_sensitive {
|
||||
if Common::regex_check(&path_wildcard, &path) {
|
||||
if regex_check(&path_wildcard_excluded, &path) {
|
||||
need_to_change_thing = true;
|
||||
}
|
||||
} else {
|
||||
if Common::regex_check(&path_wildcard.to_lowercase(), path.to_lowercase()) {
|
||||
if regex_check(&path_wildcard_lowercase_excluded, path.to_lowercase()) {
|
||||
need_to_change_thing = true;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -6,7 +6,7 @@ use gtk4::prelude::*;
|
|||
use gtk4::{DropTarget, FileChooserNative, Notebook, Orientation, ResponseType, TreeView, Window};
|
||||
|
||||
#[cfg(target_family = "windows")]
|
||||
use czkawka_core::common::Common;
|
||||
use czkawka_core::common::normalize_windows_path;
|
||||
|
||||
use crate::flg;
|
||||
use crate::gui_structs::gui_data::GuiData;
|
||||
|
@ -204,7 +204,7 @@ fn add_manually_directories(window_main: &Window, tree_view: &TreeView, excluded
|
|||
for text in entry.text().split(';') {
|
||||
let mut text = text.trim().to_string();
|
||||
#[cfg(target_family = "windows")]
|
||||
let mut text = Common::normalize_windows_path(text).to_string_lossy().to_string();
|
||||
let mut text = normalize_windows_path(text).to_string_lossy().to_string();
|
||||
|
||||
remove_ending_slashes(&mut text);
|
||||
|
||||
|
|
|
@ -1,7 +1,6 @@
|
|||
use crate::localizer_krokiet::LANGUAGE_LOADER_GUI;
|
||||
use crate::{Callabler, MainWindow};
|
||||
use slint::ComponentHandle;
|
||||
use slint::Model;
|
||||
use slint::{ComponentHandle, Model};
|
||||
use std::collections::HashMap;
|
||||
|
||||
pub fn connect_translations(app: &MainWindow) {
|
||||
|
|
|
@ -2,9 +2,7 @@ use czkawka_core::common::get_available_threads;
|
|||
use slint::{ComponentHandle, SharedString, VecModel};
|
||||
|
||||
use crate::settings::{ALLOWED_HASH_SIZE_VALUES, ALLOWED_HASH_TYPE_VALUES, ALLOWED_RESIZE_ALGORITHM_VALUES};
|
||||
use crate::GuiState;
|
||||
use crate::MainWindow;
|
||||
use crate::Settings;
|
||||
use crate::{GuiState, MainWindow, Settings};
|
||||
|
||||
// Some info needs to be send to gui at the start like available thread number in OS.
|
||||
//
|
||||
|
|
|
@ -13,8 +13,7 @@ use czkawka_core::common::{get_available_threads, set_number_of_threads};
|
|||
use czkawka_core::common_items::{DEFAULT_EXCLUDED_DIRECTORIES, DEFAULT_EXCLUDED_ITEMS};
|
||||
|
||||
use crate::common::{create_string_standard_list_view_from_pathbuf, create_vec_model_from_vec_string};
|
||||
use crate::{Callabler, MainWindow};
|
||||
use crate::{GuiState, Settings};
|
||||
use crate::{Callabler, GuiState, MainWindow, Settings};
|
||||
|
||||
pub const DEFAULT_MINIMUM_SIZE_KB: i32 = 16;
|
||||
pub const DEFAULT_MAXIMUM_SIZE_KB: i32 = i32::MAX / 1024;
|
||||
|
|
Loading…
Reference in a new issue