This commit is contained in:
epi 2023-02-26 06:49:48 -06:00
parent ad38b56473
commit 729d88a724
11 changed files with 23 additions and 84 deletions

View File

@ -248,7 +248,7 @@ impl TermOutHandler {
.unwrap()
.filters
.data
.should_filter_response(&resp, self.handles.as_ref().unwrap().stats.tx.clone());
.should_filter_response(&resp);
let contains_sentry = if !self.config.filter_status.is_empty() {
// -C was used, meaning -s was not and we should ignore the defaults

View File

@ -148,7 +148,7 @@ impl<'a> Extractor<'a> {
.handles
.filters
.data
.should_filter_response(&resp, self.handles.stats.tx.clone())
.should_filter_response(&resp)
{
continue;
}

View File

@ -3,10 +3,7 @@ use std::sync::RwLock;
use anyhow::Result;
use serde::{ser::SerializeSeq, Serialize, Serializer};
use crate::{
event_handlers::Command::AddToUsizeField, response::FeroxResponse,
statistics::StatField::WildcardsFiltered, CommandSender,
};
use crate::response::FeroxResponse;
use super::{
FeroxFilter, LinesFilter, RegexFilter, SimilarityFilter, SizeFilter, StatusCodeFilter,
@ -67,11 +64,7 @@ impl FeroxFilters {
/// Simple helper to stay DRY; determines whether or not a given `FeroxResponse` should be reported
/// to the user or not.
pub fn should_filter_response(
&self,
response: &FeroxResponse,
tx_stats: CommandSender,
) -> bool {
pub fn should_filter_response(&self, response: &FeroxResponse) -> bool {
if let Ok(filters) = self.filters.read() {
for filter in filters.iter() {
// wildcard.should_filter goes here

View File

@ -4,7 +4,7 @@ use std::any::Any;
use std::fmt::Debug;
use crate::response::FeroxResponse;
use crate::traits::{FeroxFilter, FeroxSerialize};
use crate::traits::{FeroxFilter};
pub use self::container::FeroxFilters;
pub(crate) use self::empty::EmptyFilter;
@ -17,7 +17,6 @@ pub use self::status_code::StatusCodeFilter;
pub(crate) use self::utils::{create_similarity_filter, filter_lookup};
pub use self::words::WordsFilter;
mod wildcard;
mod status_code;
mod words;
mod lines;

View File

@ -32,7 +32,7 @@ impl FeroxFilter for SimilarityFilter {
/// --filter-similar-to
fn should_filter_response(&self, response: &FeroxResponse) -> bool {
let other = SIM_HASHER.create_signature(preprocess(response.text()).iter());
return self.hash.hamming_distance(&other) <= MAX_HAMMING_DISTANCE;
self.hash.hamming_distance(&other) <= MAX_HAMMING_DISTANCE
}
/// Compare one SimilarityFilter to another

View File

@ -1,5 +1,4 @@
use super::*;
use ::fuzzyhash::FuzzyHash;
use ::regex::Regex;
@ -112,8 +111,7 @@ fn similarity_filter_is_accurate() {
resp.set_text("sitting");
let mut filter = SimilarityFilter {
hash: HashValueType::String(FuzzyHash::new("kitten").to_string()),
threshold: 95,
hash: 1,
original_url: "".to_string(),
};
@ -121,15 +119,13 @@ fn similarity_filter_is_accurate() {
assert!(!filter.should_filter_response(&resp));
resp.set_text("");
filter.hash = HashValueType::String(String::new());
filter.threshold = 100;
filter.hash = 1;
// two empty strings are the same, however ssdeep doesn't accept empty strings, expect false
assert!(!filter.should_filter_response(&resp));
resp.set_text("some data to hash for the purposes of running a test");
filter.hash = HashValueType::String(FuzzyHash::new("some data to hash for the purposes of running a te").to_string());
filter.threshold = 17;
filter.hash = 1;
assert!(filter.should_filter_response(&resp));
}
@ -138,14 +134,12 @@ fn similarity_filter_is_accurate() {
/// just a simple test to increase code coverage by hitting as_any and the inner value
fn similarity_filter_as_any() {
let filter = SimilarityFilter {
hash: HashValueType::String(String::from("stuff")),
threshold: 95,
hash: 1,
original_url: "".to_string(),
};
let filter2 = SimilarityFilter {
hash: HashValueType::String(String::from("stuff")),
threshold: 95,
hash: 1,
original_url: "".to_string(),
};

View File

@ -192,7 +192,7 @@ mod tests {
assert_eq!(
filter,
SimilarityFilter {
hash: HashValueType::String("3:YKEpn:Yfp".to_string()),
hash: 1,
original_url: srv.url("/")
}
);

View File

@ -20,9 +20,6 @@ use crate::{
DEFAULT_METHOD,
};
/// length of a standard UUID, used when determining wildcard responses
const UUID_LENGTH: u64 = 32;
/// enum representing the different servers that `parse_html` can detect when directory listing is
/// enabled
#[derive(Copy, Debug, Clone)]
@ -288,7 +285,7 @@ impl HeuristicTests {
// - http://localhost/.htaccessa005a2131e68449aa26e99029c914c09
// - http://localhost/adminf1d2541e73c44dcb9d1fb7d93334b280
let response =
logged_request(&nonexistent_url, &method, data, self.handles.clone()).await;
logged_request(&nonexistent_url, method, data, self.handles.clone()).await;
req_counter += 1;
@ -311,7 +308,7 @@ impl HeuristicTests {
let ferox_response = FeroxResponse::from(
response,
&ferox_url.target,
&method,
method,
self.handles.config.output_level,
)
.await;
@ -378,7 +375,7 @@ impl HeuristicTests {
log::trace!("exit: detect_404_like_responses");
return Ok(req_counter);
Ok(req_counter)
}
/// for all responses, examine chars/words/lines

View File

@ -1,7 +1,7 @@
use super::*;
use crate::filters::{
FeroxFilters, LinesFilter, RegexFilter, SimilarityFilter, SizeFilter, StatusCodeFilter,
WordsFilter,HashValueType
FeroxFilters, LinesFilter, RegexFilter, SimilarityFilter, SizeFilter,
StatusCodeFilter, WordsFilter,
};
use crate::{
config::{Configuration, OutputLevel},
@ -10,7 +10,7 @@ use crate::{
scanner::RESPONSES,
statistics::Stats,
traits::FeroxSerialize,
SIMILARITY_THRESHOLD, SLEEP_DURATION, VERSION,
SLEEP_DURATION, VERSION,
};
use indicatif::ProgressBar;
use predicates::prelude::*;
@ -399,8 +399,7 @@ fn feroxstates_feroxserialize_implementation() {
.unwrap();
filters
.push(Box::new(SimilarityFilter {
hash: HashValueType::String("3:YKEpn:Yfp".to_string()),
threshold: SIMILARITY_THRESHOLD,
hash: 1,
original_url: "http://localhost:12345/".to_string(),
}))
.unwrap();
@ -499,7 +498,7 @@ fn feroxstates_feroxserialize_implementation() {
r#""collect_extensions":true"#,
r#""collect_backups":false"#,
r#""collect_words":false"#,
r#""filters":[{"filter_code":100},{"word_count":200},{"content_length":300},{"line_count":400},{"compiled":".*","raw_string":".*"},{"hash":"3:YKEpn:Yfp","threshold":95,"original_url":"http://localhost:12345/"}]"#,
r#""filters":[{"filter_code":100},{"word_count":200},{"content_length":300},{"line_count":400},{"compiled":".*","raw_string":".*"},{"hash":1,"original_url":"http://localhost:12345/"}]"#,
r#""collected_extensions":["php"]"#,
r#""dont_collect":["tif","tiff","ico","cur","bmp","webp","svg","png","jpg","jpeg","jfif","gif","avif","apng","pjpeg","pjp","mov","wav","mpg","mpeg","mp3","mp4","m4a","m4p","m4v","ogg","webm","ogv","oga","flac","aac","3gp","css","zip","xls","xml","gz","tgz"]"#,
]
@ -636,9 +635,7 @@ fn menu_print_header_and_footer() {
let menu_cmd_2 = MenuCmd::Cancel(vec![0], false);
let menu_cmd_res_1 = MenuCmdResult::Url(String::from("http://localhost"));
let menu_cmd_res_2 = MenuCmdResult::NumCancelled(2);
println!(
"{menu_cmd_1:?}{menu_cmd_2:?}{menu_cmd_res_1:?}{menu_cmd_res_2:?}"
);
println!("{menu_cmd_1:?}{menu_cmd_2:?}{menu_cmd_res_1:?}{menu_cmd_res_2:?}");
menu.clear_screen();
menu.print_header();
menu.print_footer();

View File

@ -437,7 +437,7 @@ impl Requester {
.handles
.filters
.data
.should_filter_response(&ferox_response, self.handles.stats.tx.clone())
.should_filter_response(&ferox_response)
{
continue;
}

View File

@ -2,7 +2,7 @@ use crate::{event_handlers::Handles, statistics::StatError::UrlFormat, Command::
use anyhow::{anyhow, bail, Result};
use reqwest::Url;
use std::collections::HashSet;
use std::{convert::TryInto, fmt, sync::Arc};
use std::{fmt, sync::Arc};
/// abstraction around target urls; collects all Url related shenanigans in one place
#[derive(Debug)]
@ -157,47 +157,6 @@ impl FeroxUrl {
}
}
/// Gets the length of a url's path
pub fn path_length(&self) -> Result<u64> {
let parsed = Url::parse(&self.target)?;
Ok(FeroxUrl::path_length_of_url(&parsed))
}
/// Gets the length of a url's path
///
/// example: http://localhost/stuff -> 5
pub fn path_length_of_url(url: &Url) -> u64 {
log::trace!("enter: get_path_length({})", url);
let path = url.path();
let segments = if let Some(split) = path.strip_prefix('/') {
split.split_terminator('/')
} else {
log::trace!("exit: get_path_length -> 0");
return 0;
};
if let Some(last) = segments.last() {
// failure on conversion should be very unlikely. While a usize can absolutely overflow a
// u64, the generally accepted maximum for the length of a url is ~2000. so the value we're
// putting into the u64 should never realistically be anywhere close to producing an
// overflow.
// usize max: 18,446,744,073,709,551,615
// u64 max: 9,223,372,036,854,775,807
let url_len: u64 = last
.len()
.try_into()
.expect("Failed usize -> u64 conversion");
log::trace!("exit: get_path_length -> {}", url_len);
return url_len;
}
log::trace!("exit: get_path_length -> 0");
0
}
/// Simple helper to abstract away adding a forward-slash to a url if not present
///
/// used mostly for deduplication purposes and url state tracking