utilities/feroxbuster
utilities/feroxbuster
1
0
Fork 0
mirror of https://github.com/epi052/feroxbuster synced 2024-06-29 05:54:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

added headers to regex filtering (#1142)

Browse Source 
* added headers to regex filtering

* added regex header test

* added pipeline for mac arm

* bumped version; updated deps; updated .cargo/config to .toml

* -b more robust

* fixed overall prog bar showing 0 eta too early

* fixed ssl error test

* added time estimate to SMM
This commit is contained in:
epi 2024-06-16 08:59:17 -04:00 committed by GitHub
parent 87b6589f51
commit 57db4adb69
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
16 changed files with 976 additions and 510 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
.cargo/config → .cargo/config.toml
View File

32
.github/workflows/build.yml vendored
View File

@ -156,6 +156,38 @@ jobs:
with:
name: x86_64-macos-feroxbuster.tar.gz
path: x86_64-macos-feroxbuster.tar.gz
build-macos-aarch64:
env:
IN_PIPELINE: true
runs-on: macos-latest
# if: github.ref == 'refs/heads/main'
steps:
- uses: actions/checkout@v2
- uses: actions-rs/toolchain@v1
with:
toolchain: stable
target: aarch64-apple-darwin
override: true
- uses: actions-rs/cargo@v1
with:
use-cross: true
command: build
args: --release --target=aarch64-apple-darwin
- name: Strip symbols from binary
run: |
strip -u -r target/aarch64-apple-darwin/release/feroxbuster
- name: Build tar.gz for homebrew installs
run: |
tar czf aarch64-macos-feroxbuster.tar.gz -C target/aarch64-apple-darwin/release feroxbuster
- uses: actions/upload-artifact@v2
with:
name: aarch64-macos-feroxbuster
path: target/aarch64-apple-darwin/release/feroxbuster
- uses: actions/upload-artifact@v2
with:
name: aarch64-macos-feroxbuster.tar.gz
path: aarch64-macos-feroxbuster.tar.gz
build-windows:
env:

1238
Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

16
Cargo.toml
View File

@ -1,6 +1,6 @@
[package]
name = "feroxbuster"
version = "2.10.3"
version = "2.10.4"
authors = ["Ben 'epi' Risher (@epi052)"]
license = "MIT"
edition = "2021"
@ -29,13 +29,13 @@ lazy_static = "1.4"
dirs = "5.0"
[dependencies]
scraper = "0.18"
scraper = "0.19"
futures = "0.3"
tokio = { version = "1.37", features = ["full"] }
tokio = { version = "1.38", features = ["full"] }
tokio-util = { version = "0.7", features = ["codec"] }
log = "0.4"
env_logger = "0.10"
reqwest = { version = "0.11", features = ["socks", "native-tls-alpn"] }
env_logger = "0.11"
reqwest = { version = "0.12", features = ["socks", "native-tls-alpn"] }
# uses feature unification to add 'serde' to reqwest::Url
url = { version = "2.5", features = ["serde"] }
serde_regex = "1.1"
@ -56,12 +56,12 @@ crossterm = "0.27"
rlimit = "0.10"
ctrlc = "3.4"
anyhow = "1.0"
leaky-bucket = "1.0"
leaky-bucket = "1.1"
gaoya = "0.2"
# 0.37+ relies on the broken version of indicatif and forces
# the broken version to be used regardless of the version
# specified above
self_update = { version = "0.36", features = [
self_update = { version = "0.40", features = [
"archive-tar",
"compression-flate2",
"archive-zip",
@ -70,7 +70,7 @@ self_update = { version = "0.36", features = [
[dev-dependencies]
tempfile = "3.10"
httpmock = "0.6"
httpmock = "0.7"
assert_cmd = "2.0"
predicates = "3.1"

6
shell_completions/_feroxbuster
View File

@ -14,7 +14,7 @@ _feroxbuster() {
fi
local context curcontext="$curcontext" state line
_arguments "${_arguments_options[@]}" \
_arguments "${_arguments_options[@]}" : \
'-u+[The target URL (required, unless \[--stdin || --resume-from\] used)]:URL:_urls' \
'--url=[The target URL (required, unless \[--stdin || --resume-from\] used)]:URL:_urls' \
'(-u --url)--resume-from=[State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)]:STATE_FILE:_files' \
@ -24,8 +24,8 @@ _feroxbuster() {
'--replay-proxy=[Send only unfiltered requests through a Replay Proxy, instead of all requests]:REPLAY_PROXY:_urls' \
'*-R+[Status Codes to send through a Replay Proxy when found (default\: --status-codes value)]:REPLAY_CODE: ' \
'*--replay-codes=[Status Codes to send through a Replay Proxy when found (default\: --status-codes value)]:REPLAY_CODE: ' \
'-a+[Sets the User-Agent (default\: feroxbuster/2.10.3)]:USER_AGENT: ' \
'--user-agent=[Sets the User-Agent (default\: feroxbuster/2.10.3)]:USER_AGENT: ' \
'-a+[Sets the User-Agent (default\: feroxbuster/2.10.4)]:USER_AGENT: ' \
'--user-agent=[Sets the User-Agent (default\: feroxbuster/2.10.4)]:USER_AGENT: ' \
'*-x+[File extension(s) to search for (ex\: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex\: @ext.txt)]:FILE_EXTENSION: ' \
'*--extensions=[File extension(s) to search for (ex\: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex\: @ext.txt)]:FILE_EXTENSION: ' \
'*-m+[Which HTTP request method(s) should be sent (default\: GET)]:HTTP_METHODS: ' \

4
shell_completions/_feroxbuster.ps1
View File

@ -30,8 +30,8 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
[CompletionResult]::new('--replay-proxy', 'replay-proxy', [CompletionResultType]::ParameterName, 'Send only unfiltered requests through a Replay Proxy, instead of all requests')
[CompletionResult]::new('-R', 'R ', [CompletionResultType]::ParameterName, 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)')
[CompletionResult]::new('--replay-codes', 'replay-codes', [CompletionResultType]::ParameterName, 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)')
[CompletionResult]::new('-a', 'a', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.10.3)')
[CompletionResult]::new('--user-agent', 'user-agent', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.10.3)')
[CompletionResult]::new('-a', 'a', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.10.4)')
[CompletionResult]::new('--user-agent', 'user-agent', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.10.4)')
[CompletionResult]::new('-x', 'x', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex: @ext.txt)')
[CompletionResult]::new('--extensions', 'extensions', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex: @ext.txt)')
[CompletionResult]::new('-m', 'm', [CompletionResultType]::ParameterName, 'Which HTTP request method(s) should be sent (default: GET)')

4
shell_completions/feroxbuster.elv
View File

@ -27,8 +27,8 @@ set edit:completion:arg-completer[feroxbuster] = {|@words|
cand --replay-proxy 'Send only unfiltered requests through a Replay Proxy, instead of all requests'
cand -R 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)'
cand --replay-codes 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)'
cand -a 'Sets the User-Agent (default: feroxbuster/2.10.3)'
cand --user-agent 'Sets the User-Agent (default: feroxbuster/2.10.3)'
cand -a 'Sets the User-Agent (default: feroxbuster/2.10.4)'
cand --user-agent 'Sets the User-Agent (default: feroxbuster/2.10.4)'
cand -x 'File extension(s) to search for (ex: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex: @ext.txt)'
cand --extensions 'File extension(s) to search for (ex: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex: @ext.txt)'
cand -m 'Which HTTP request method(s) should be sent (default: GET)'

30
src/config/container.rs
View File

@ -620,7 +620,7 @@ impl Configuration {
update_config_if_present!(&mut config.resume_from, args, "resume_from", String);
if let Ok(Some(inner)) = args.try_get_one::<String>("time_limit") {
config.time_limit = inner.to_owned();
inner.clone_into(&mut config.time_limit);
}
if let Some(arg) = args.get_many::<String>("status_codes") {
@ -644,7 +644,7 @@ impl Configuration {
.collect();
} else {
// not passed in by the user, use whatever value is held in status_codes
config.replay_codes = config.status_codes.clone();
config.replay_codes.clone_from(&config.status_codes);
}
if let Some(arg) = args.get_many::<String>("filter_status") {
@ -956,15 +956,27 @@ impl Configuration {
config.headers.insert(
// we know the header name is always "cookie"
"Cookie".to_string(),
// on splitting, there should be only two elements,
// a key and a value
cookies
.map(|cookie| cookie.split('=').collect::<Vec<&str>>()[..].to_owned())
.filter(|parts| parts.len() == 2)
.map(|parts| format!("{}={}", parts[0].trim(), parts[1].trim()))
// trim the spaces, join with an equals sign
.flat_map(|cookie| {
cookie.split(';').filter_map(|part| {
// trim the spaces
let trimmed = part.trim();
if trimmed.is_empty() {
None
} else {
// join with an equals sign
let parts = trimmed.split('=').collect::<Vec<&str>>();
Some(format!(
"{}={}",
parts[0].trim(),
parts[1..].join("").trim()
))
}
})
})
.collect::<Vec<String>>()
.join("; "), // join all the cookies with semicolons for the final header
// join all the cookies with semicolons for the final header
.join("; "),
);
}

4
src/event_handlers/command.rs
View File

@ -1,4 +1,5 @@
use std::sync::Arc;
use std::time::Duration;
use reqwest::StatusCode;
use tokio::sync::oneshot::Sender;
@ -88,4 +89,7 @@ pub enum Command {
/// inform the Stats object about which targets are being scanned
UpdateTargets(Vec<String>),
/// query the Stats handler about the position of the overall progress bar
QueryOverallBarEta(Sender<Duration>),
}

3
src/event_handlers/statistics.rs
View File

@ -125,6 +125,9 @@ impl StatsHandler {
Command::Sync(sender) => {
sender.send(true).unwrap_or_default();
}
Command::QueryOverallBarEta(sender) => {
sender.send(self.bar.eta()).unwrap_or_default();
}
Command::UpdateTargets(targets) => {
self.stats.update_targets(targets);
}

7
src/filters/regex.rs
View File

@ -30,10 +30,13 @@ impl FeroxFilter for RegexFilter {
log::trace!("enter: should_filter_response({:?} {})", self, response);
let result = self.compiled.is_match(response.text());
let other = response.headers().iter().any(|(k, v)| {
self.compiled.is_match(k.as_str()) || self.compiled.is_match(v.to_str().unwrap_or(""))
});
log::trace!("exit: should_filter_response -> {}", result);
log::trace!("exit: should_filter_response -> {}", result || other);
result
result || other
}
/// Compare one SizeFilter to another

62
src/progress.rs
View File

@ -1,6 +1,4 @@
use std::time::Duration;
use indicatif::{HumanDuration, MultiProgress, ProgressBar, ProgressDrawTarget, ProgressStyle};
use indicatif::{MultiProgress, ProgressBar, ProgressDrawTarget, ProgressStyle};
use lazy_static::lazy_static;
lazy_static! {
@ -33,45 +31,23 @@ pub enum BarType {
/// Add an [indicatif::ProgressBar](https://docs.rs/indicatif/latest/indicatif/struct.ProgressBar.html)
/// to the global [PROGRESS_BAR](../config/struct.PROGRESS_BAR.html)
pub fn add_bar(prefix: &str, length: u64, bar_type: BarType) -> ProgressBar {
let mut style = ProgressStyle::default_bar()
.progress_chars("#>-")
.with_key(
"smoothed_per_sec",
|state: &indicatif::ProgressState, w: &mut dyn std::fmt::Write| match (
state.pos(),
state.elapsed().as_millis(),
) {
// https://github.com/console-rs/indicatif/issues/394#issuecomment-1309971049
//
// indicatif released a change to how they reported eta/per_sec
// and the results looked really weird based on how we use the progress
// bars. this fixes that
(pos, elapsed_ms) if elapsed_ms > 0 => {
write!(w, "{:.0}/s", pos as f64 * 1000_f64 / elapsed_ms as f64).unwrap()
}
_ => write!(w, "-").unwrap(),
},
)
.with_key(
"smoothed_eta",
|state: &indicatif::ProgressState, w: &mut dyn std::fmt::Write| match (
state.pos(),
state.len(),
) {
(pos, Some(len)) => write!(
w,
"{:#}",
HumanDuration(Duration::from_millis(
(state.elapsed().as_millis()
* ((len as u128).checked_sub(pos as u128).unwrap_or(1))
.checked_div(pos as u128)
.unwrap_or(1)) as u64
))
)
.unwrap(),
_ => write!(w, "-").unwrap(),
},
);
let mut style = ProgressStyle::default_bar().progress_chars("#>-").with_key(
"smoothed_per_sec",
|state: &indicatif::ProgressState, w: &mut dyn std::fmt::Write| match (
state.pos(),
state.elapsed().as_millis(),
) {
// https://github.com/console-rs/indicatif/issues/394#issuecomment-1309971049
//
// indicatif released a change to how they reported eta/per_sec
// and the results looked really weird based on how we use the progress
// bars. this fixes that
(pos, elapsed_ms) if elapsed_ms > 0 => {
write!(w, "{:.0}/s", pos as f64 * 1000_f64 / elapsed_ms as f64).unwrap()
}
_ => write!(w, "-").unwrap(),
},
);
style = match bar_type {
BarType::Hidden => style.template("").unwrap(),
@ -85,7 +61,7 @@ pub fn add_bar(prefix: &str, length: u64, bar_type: BarType) -> ProgressBar {
))
.unwrap(),
BarType::Total => style
.template("[{bar:.yellow/blue}] - {elapsed:<4} {pos:>7}/{len:7} {smoothed_eta:7} {msg}")
.template("[{bar:.yellow/blue}] - {elapsed:<4} {pos:>7}/{len:7} {eta:7} {msg}")
.unwrap(),
BarType::Quiet => style.template("Scanning: {prefix}").unwrap(),
};

17
src/scan_manager/menu.rs
View File

@ -1,8 +1,10 @@
use std::time::Duration;
use crate::filters::filter_lookup;
use crate::progress::PROGRESS_BAR;
use crate::traits::FeroxFilter;
use console::{measure_text_width, pad_str, style, Alignment, Term};
use indicatif::ProgressDrawTarget;
use indicatif::{HumanDuration, ProgressDrawTarget};
use regex::Regex;
/// Data container for a command entered by the user interactively
@ -43,6 +45,9 @@ pub(super) struct Menu {
/// footer: instructions surrounded by separators
footer: String,
/// length of longest displayed line (suitable for ascii/unicode)
longest: usize,
/// unicode line border, matched to longest displayed line
border: String,
@ -110,7 +115,7 @@ impl Menu {
commands.push_str(&valid_filters);
commands.push_str(&rm_filter_cmd);
let longest = measure_text_width(&canx_cmd).max(measure_text_width(&name));
let longest = measure_text_width(&canx_cmd).max(measure_text_width(&name)) + 1;
let border = separator.repeat(longest);
@ -123,6 +128,7 @@ impl Menu {
header,
footer,
border,
longest,
term: Term::stderr(),
}
}
@ -142,6 +148,13 @@ impl Menu {
self.println(&self.footer);
}
/// print menu footer
pub(super) fn print_eta(&self, eta: Duration) {
let inner = format!("⏳ {} remaining ⏳", HumanDuration(eta));
let padded_eta = pad_str(&inner, self.longest, Alignment::Center, None);
self.println(&format!("{padded_eta}\n{}", self.border));
}
/// set PROGRESS_BAR bar target to hidden
pub(super) fn hide_progress_bars(&self) {
PROGRESS_BAR.set_draw_target(ProgressDrawTarget::hidden());

8
src/scan_manager/scan_container.rs
View File

@ -33,6 +33,7 @@ use std::{
},
thread::sleep,
};
use tokio::sync::oneshot;
use tokio::time::{self, Duration};
/// Single atomic number that gets incremented once, used to track first thread to interact with
@ -430,6 +431,13 @@ impl FeroxScans {
self.menu.hide_progress_bars();
self.menu.clear_screen();
self.menu.print_header();
let (tx, rx) = oneshot::channel::<Duration>();
if handles.stats.send(Command::QueryOverallBarEta(tx)).is_ok() {
if let Ok(y) = rx.await {
self.menu.print_eta(y);
}
}
self.display_scans().await;
self.display_filters(handles.clone());
self.menu.print_footer();

49
tests/test_filters.rs
View File

@ -290,3 +290,52 @@ fn collect_backups_should_be_filtered() {
assert_eq!(mock_two.hits(), 1);
teardown_tmp_directory(tmp_dir);
}
#[test]
/// create a FeroxResponse that should elicit a true from
/// RegexFilter::should_filter_response
fn filters_regex_should_filter_response_based_on_headers() {
let srv = MockServer::start();
let (tmp_dir, file) = setup_tmp_directory(
&["not-matching".to_string(), "matching".to_string()],
"wordlist",
)
.unwrap();
let mock = srv.mock(|when, then| {
when.method(GET).path("/not-matching");
then.status(200)
.header("content-type", "text/html")
.body("this is a test");
});
let mock_two = srv.mock(|when, then| {
when.method(GET).path("/matching");
then.status(200)
.header("content-type", "application/json")
.body("this is also a test");
});
let cmd = Command::cargo_bin("feroxbuster")
.unwrap()
.arg("--url")
.arg(srv.url("/"))
.arg("--wordlist")
.arg(file.as_os_str())
.arg("--filter-regex")
.arg("content-type:application/json")
.unwrap();
cmd.assert().success().stdout(
predicate::str::contains("/not-matching")
.and(predicate::str::contains("200"))
.and(predicate::str::contains("/matching"))
.not()
.and(predicate::str::contains("200"))
.not(),
);
assert_eq!(mock.hits(), 1);
assert_eq!(mock_two.hits(), 1);
teardown_tmp_directory(tmp_dir);
}

6
tests/test_heuristics.rs
View File

@ -104,9 +104,9 @@ fn test_single_target_cannot_connect_due_to_ssl_errors() -> Result<(), Box<dyn s
.arg(file.as_os_str())
.assert()
.success()
.stdout(
predicate::str::contains("Could not connect to https://expired.badssl.com due to SSL errors (run with -k to ignore), skipping...", )
);
.stdout(predicate::str::contains(
"Could not connect to https://expired.badssl.com",
));
teardown_tmp_directory(tmp_dir);
Ok(())