mirror of
https://github.com/epi052/feroxbuster
synced 2024-07-01 06:54:24 +00:00
added headers to regex filtering (#1142)
* added headers to regex filtering * added regex header test * added pipeline for mac arm * bumped version; updated deps; updated .cargo/config to .toml * -b more robust * fixed overall prog bar showing 0 eta too early * fixed ssl error test * added time estimate to SMM
This commit is contained in:
parent
87b6589f51
commit
57db4adb69
32
.github/workflows/build.yml
vendored
32
.github/workflows/build.yml
vendored
|
@ -157,6 +157,38 @@ jobs:
|
|||
name: x86_64-macos-feroxbuster.tar.gz
|
||||
path: x86_64-macos-feroxbuster.tar.gz
|
||||
|
||||
build-macos-aarch64:
|
||||
env:
|
||||
IN_PIPELINE: true
|
||||
runs-on: macos-latest
|
||||
# if: github.ref == 'refs/heads/main'
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions-rs/toolchain@v1
|
||||
with:
|
||||
toolchain: stable
|
||||
target: aarch64-apple-darwin
|
||||
override: true
|
||||
- uses: actions-rs/cargo@v1
|
||||
with:
|
||||
use-cross: true
|
||||
command: build
|
||||
args: --release --target=aarch64-apple-darwin
|
||||
- name: Strip symbols from binary
|
||||
run: |
|
||||
strip -u -r target/aarch64-apple-darwin/release/feroxbuster
|
||||
- name: Build tar.gz for homebrew installs
|
||||
run: |
|
||||
tar czf aarch64-macos-feroxbuster.tar.gz -C target/aarch64-apple-darwin/release feroxbuster
|
||||
- uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: aarch64-macos-feroxbuster
|
||||
path: target/aarch64-apple-darwin/release/feroxbuster
|
||||
- uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: aarch64-macos-feroxbuster.tar.gz
|
||||
path: aarch64-macos-feroxbuster.tar.gz
|
||||
|
||||
build-windows:
|
||||
env:
|
||||
IN_PIPELINE: true
|
||||
|
|
1238
Cargo.lock
generated
1238
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
16
Cargo.toml
16
Cargo.toml
|
@ -1,6 +1,6 @@
|
|||
[package]
|
||||
name = "feroxbuster"
|
||||
version = "2.10.3"
|
||||
version = "2.10.4"
|
||||
authors = ["Ben 'epi' Risher (@epi052)"]
|
||||
license = "MIT"
|
||||
edition = "2021"
|
||||
|
@ -29,13 +29,13 @@ lazy_static = "1.4"
|
|||
dirs = "5.0"
|
||||
|
||||
[dependencies]
|
||||
scraper = "0.18"
|
||||
scraper = "0.19"
|
||||
futures = "0.3"
|
||||
tokio = { version = "1.37", features = ["full"] }
|
||||
tokio = { version = "1.38", features = ["full"] }
|
||||
tokio-util = { version = "0.7", features = ["codec"] }
|
||||
log = "0.4"
|
||||
env_logger = "0.10"
|
||||
reqwest = { version = "0.11", features = ["socks", "native-tls-alpn"] }
|
||||
env_logger = "0.11"
|
||||
reqwest = { version = "0.12", features = ["socks", "native-tls-alpn"] }
|
||||
# uses feature unification to add 'serde' to reqwest::Url
|
||||
url = { version = "2.5", features = ["serde"] }
|
||||
serde_regex = "1.1"
|
||||
|
@ -56,12 +56,12 @@ crossterm = "0.27"
|
|||
rlimit = "0.10"
|
||||
ctrlc = "3.4"
|
||||
anyhow = "1.0"
|
||||
leaky-bucket = "1.0"
|
||||
leaky-bucket = "1.1"
|
||||
gaoya = "0.2"
|
||||
# 0.37+ relies on the broken version of indicatif and forces
|
||||
# the broken version to be used regardless of the version
|
||||
# specified above
|
||||
self_update = { version = "0.36", features = [
|
||||
self_update = { version = "0.40", features = [
|
||||
"archive-tar",
|
||||
"compression-flate2",
|
||||
"archive-zip",
|
||||
|
@ -70,7 +70,7 @@ self_update = { version = "0.36", features = [
|
|||
|
||||
[dev-dependencies]
|
||||
tempfile = "3.10"
|
||||
httpmock = "0.6"
|
||||
httpmock = "0.7"
|
||||
assert_cmd = "2.0"
|
||||
predicates = "3.1"
|
||||
|
||||
|
|
|
@ -14,7 +14,7 @@ _feroxbuster() {
|
|||
fi
|
||||
|
||||
local context curcontext="$curcontext" state line
|
||||
_arguments "${_arguments_options[@]}" \
|
||||
_arguments "${_arguments_options[@]}" : \
|
||||
'-u+[The target URL (required, unless \[--stdin || --resume-from\] used)]:URL:_urls' \
|
||||
'--url=[The target URL (required, unless \[--stdin || --resume-from\] used)]:URL:_urls' \
|
||||
'(-u --url)--resume-from=[State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)]:STATE_FILE:_files' \
|
||||
|
@ -24,8 +24,8 @@ _feroxbuster() {
|
|||
'--replay-proxy=[Send only unfiltered requests through a Replay Proxy, instead of all requests]:REPLAY_PROXY:_urls' \
|
||||
'*-R+[Status Codes to send through a Replay Proxy when found (default\: --status-codes value)]:REPLAY_CODE: ' \
|
||||
'*--replay-codes=[Status Codes to send through a Replay Proxy when found (default\: --status-codes value)]:REPLAY_CODE: ' \
|
||||
'-a+[Sets the User-Agent (default\: feroxbuster/2.10.3)]:USER_AGENT: ' \
|
||||
'--user-agent=[Sets the User-Agent (default\: feroxbuster/2.10.3)]:USER_AGENT: ' \
|
||||
'-a+[Sets the User-Agent (default\: feroxbuster/2.10.4)]:USER_AGENT: ' \
|
||||
'--user-agent=[Sets the User-Agent (default\: feroxbuster/2.10.4)]:USER_AGENT: ' \
|
||||
'*-x+[File extension(s) to search for (ex\: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex\: @ext.txt)]:FILE_EXTENSION: ' \
|
||||
'*--extensions=[File extension(s) to search for (ex\: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex\: @ext.txt)]:FILE_EXTENSION: ' \
|
||||
'*-m+[Which HTTP request method(s) should be sent (default\: GET)]:HTTP_METHODS: ' \
|
||||
|
|
|
@ -30,8 +30,8 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
|
|||
[CompletionResult]::new('--replay-proxy', 'replay-proxy', [CompletionResultType]::ParameterName, 'Send only unfiltered requests through a Replay Proxy, instead of all requests')
|
||||
[CompletionResult]::new('-R', 'R ', [CompletionResultType]::ParameterName, 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)')
|
||||
[CompletionResult]::new('--replay-codes', 'replay-codes', [CompletionResultType]::ParameterName, 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)')
|
||||
[CompletionResult]::new('-a', 'a', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.10.3)')
|
||||
[CompletionResult]::new('--user-agent', 'user-agent', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.10.3)')
|
||||
[CompletionResult]::new('-a', 'a', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.10.4)')
|
||||
[CompletionResult]::new('--user-agent', 'user-agent', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.10.4)')
|
||||
[CompletionResult]::new('-x', 'x', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex: @ext.txt)')
|
||||
[CompletionResult]::new('--extensions', 'extensions', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex: @ext.txt)')
|
||||
[CompletionResult]::new('-m', 'm', [CompletionResultType]::ParameterName, 'Which HTTP request method(s) should be sent (default: GET)')
|
||||
|
|
|
@ -27,8 +27,8 @@ set edit:completion:arg-completer[feroxbuster] = {|@words|
|
|||
cand --replay-proxy 'Send only unfiltered requests through a Replay Proxy, instead of all requests'
|
||||
cand -R 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)'
|
||||
cand --replay-codes 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)'
|
||||
cand -a 'Sets the User-Agent (default: feroxbuster/2.10.3)'
|
||||
cand --user-agent 'Sets the User-Agent (default: feroxbuster/2.10.3)'
|
||||
cand -a 'Sets the User-Agent (default: feroxbuster/2.10.4)'
|
||||
cand --user-agent 'Sets the User-Agent (default: feroxbuster/2.10.4)'
|
||||
cand -x 'File extension(s) to search for (ex: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex: @ext.txt)'
|
||||
cand --extensions 'File extension(s) to search for (ex: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex: @ext.txt)'
|
||||
cand -m 'Which HTTP request method(s) should be sent (default: GET)'
|
||||
|
|
|
@ -620,7 +620,7 @@ impl Configuration {
|
|||
update_config_if_present!(&mut config.resume_from, args, "resume_from", String);
|
||||
|
||||
if let Ok(Some(inner)) = args.try_get_one::<String>("time_limit") {
|
||||
config.time_limit = inner.to_owned();
|
||||
inner.clone_into(&mut config.time_limit);
|
||||
}
|
||||
|
||||
if let Some(arg) = args.get_many::<String>("status_codes") {
|
||||
|
@ -644,7 +644,7 @@ impl Configuration {
|
|||
.collect();
|
||||
} else {
|
||||
// not passed in by the user, use whatever value is held in status_codes
|
||||
config.replay_codes = config.status_codes.clone();
|
||||
config.replay_codes.clone_from(&config.status_codes);
|
||||
}
|
||||
|
||||
if let Some(arg) = args.get_many::<String>("filter_status") {
|
||||
|
@ -956,15 +956,27 @@ impl Configuration {
|
|||
config.headers.insert(
|
||||
// we know the header name is always "cookie"
|
||||
"Cookie".to_string(),
|
||||
// on splitting, there should be only two elements,
|
||||
// a key and a value
|
||||
cookies
|
||||
.map(|cookie| cookie.split('=').collect::<Vec<&str>>()[..].to_owned())
|
||||
.filter(|parts| parts.len() == 2)
|
||||
.map(|parts| format!("{}={}", parts[0].trim(), parts[1].trim()))
|
||||
// trim the spaces, join with an equals sign
|
||||
.flat_map(|cookie| {
|
||||
cookie.split(';').filter_map(|part| {
|
||||
// trim the spaces
|
||||
let trimmed = part.trim();
|
||||
if trimmed.is_empty() {
|
||||
None
|
||||
} else {
|
||||
// join with an equals sign
|
||||
let parts = trimmed.split('=').collect::<Vec<&str>>();
|
||||
Some(format!(
|
||||
"{}={}",
|
||||
parts[0].trim(),
|
||||
parts[1..].join("").trim()
|
||||
))
|
||||
}
|
||||
})
|
||||
})
|
||||
.collect::<Vec<String>>()
|
||||
.join("; "), // join all the cookies with semicolons for the final header
|
||||
// join all the cookies with semicolons for the final header
|
||||
.join("; "),
|
||||
);
|
||||
}
|
||||
|
||||
|
|
|
@ -1,4 +1,5 @@
|
|||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use reqwest::StatusCode;
|
||||
use tokio::sync::oneshot::Sender;
|
||||
|
@ -88,4 +89,7 @@ pub enum Command {
|
|||
|
||||
/// inform the Stats object about which targets are being scanned
|
||||
UpdateTargets(Vec<String>),
|
||||
|
||||
/// query the Stats handler about the position of the overall progress bar
|
||||
QueryOverallBarEta(Sender<Duration>),
|
||||
}
|
||||
|
|
|
@ -125,6 +125,9 @@ impl StatsHandler {
|
|||
Command::Sync(sender) => {
|
||||
sender.send(true).unwrap_or_default();
|
||||
}
|
||||
Command::QueryOverallBarEta(sender) => {
|
||||
sender.send(self.bar.eta()).unwrap_or_default();
|
||||
}
|
||||
Command::UpdateTargets(targets) => {
|
||||
self.stats.update_targets(targets);
|
||||
}
|
||||
|
|
|
@ -30,10 +30,13 @@ impl FeroxFilter for RegexFilter {
|
|||
log::trace!("enter: should_filter_response({:?} {})", self, response);
|
||||
|
||||
let result = self.compiled.is_match(response.text());
|
||||
let other = response.headers().iter().any(|(k, v)| {
|
||||
self.compiled.is_match(k.as_str()) || self.compiled.is_match(v.to_str().unwrap_or(""))
|
||||
});
|
||||
|
||||
log::trace!("exit: should_filter_response -> {}", result);
|
||||
log::trace!("exit: should_filter_response -> {}", result || other);
|
||||
|
||||
result
|
||||
result || other
|
||||
}
|
||||
|
||||
/// Compare one SizeFilter to another
|
||||
|
|
|
@ -1,6 +1,4 @@
|
|||
use std::time::Duration;
|
||||
|
||||
use indicatif::{HumanDuration, MultiProgress, ProgressBar, ProgressDrawTarget, ProgressStyle};
|
||||
use indicatif::{MultiProgress, ProgressBar, ProgressDrawTarget, ProgressStyle};
|
||||
use lazy_static::lazy_static;
|
||||
|
||||
lazy_static! {
|
||||
|
@ -33,45 +31,23 @@ pub enum BarType {
|
|||
/// Add an [indicatif::ProgressBar](https://docs.rs/indicatif/latest/indicatif/struct.ProgressBar.html)
|
||||
/// to the global [PROGRESS_BAR](../config/struct.PROGRESS_BAR.html)
|
||||
pub fn add_bar(prefix: &str, length: u64, bar_type: BarType) -> ProgressBar {
|
||||
let mut style = ProgressStyle::default_bar()
|
||||
.progress_chars("#>-")
|
||||
.with_key(
|
||||
"smoothed_per_sec",
|
||||
|state: &indicatif::ProgressState, w: &mut dyn std::fmt::Write| match (
|
||||
state.pos(),
|
||||
state.elapsed().as_millis(),
|
||||
) {
|
||||
// https://github.com/console-rs/indicatif/issues/394#issuecomment-1309971049
|
||||
//
|
||||
// indicatif released a change to how they reported eta/per_sec
|
||||
// and the results looked really weird based on how we use the progress
|
||||
// bars. this fixes that
|
||||
(pos, elapsed_ms) if elapsed_ms > 0 => {
|
||||
write!(w, "{:.0}/s", pos as f64 * 1000_f64 / elapsed_ms as f64).unwrap()
|
||||
}
|
||||
_ => write!(w, "-").unwrap(),
|
||||
},
|
||||
)
|
||||
.with_key(
|
||||
"smoothed_eta",
|
||||
|state: &indicatif::ProgressState, w: &mut dyn std::fmt::Write| match (
|
||||
state.pos(),
|
||||
state.len(),
|
||||
) {
|
||||
(pos, Some(len)) => write!(
|
||||
w,
|
||||
"{:#}",
|
||||
HumanDuration(Duration::from_millis(
|
||||
(state.elapsed().as_millis()
|
||||
* ((len as u128).checked_sub(pos as u128).unwrap_or(1))
|
||||
.checked_div(pos as u128)
|
||||
.unwrap_or(1)) as u64
|
||||
))
|
||||
)
|
||||
.unwrap(),
|
||||
_ => write!(w, "-").unwrap(),
|
||||
},
|
||||
);
|
||||
let mut style = ProgressStyle::default_bar().progress_chars("#>-").with_key(
|
||||
"smoothed_per_sec",
|
||||
|state: &indicatif::ProgressState, w: &mut dyn std::fmt::Write| match (
|
||||
state.pos(),
|
||||
state.elapsed().as_millis(),
|
||||
) {
|
||||
// https://github.com/console-rs/indicatif/issues/394#issuecomment-1309971049
|
||||
//
|
||||
// indicatif released a change to how they reported eta/per_sec
|
||||
// and the results looked really weird based on how we use the progress
|
||||
// bars. this fixes that
|
||||
(pos, elapsed_ms) if elapsed_ms > 0 => {
|
||||
write!(w, "{:.0}/s", pos as f64 * 1000_f64 / elapsed_ms as f64).unwrap()
|
||||
}
|
||||
_ => write!(w, "-").unwrap(),
|
||||
},
|
||||
);
|
||||
|
||||
style = match bar_type {
|
||||
BarType::Hidden => style.template("").unwrap(),
|
||||
|
@ -85,7 +61,7 @@ pub fn add_bar(prefix: &str, length: u64, bar_type: BarType) -> ProgressBar {
|
|||
))
|
||||
.unwrap(),
|
||||
BarType::Total => style
|
||||
.template("[{bar:.yellow/blue}] - {elapsed:<4} {pos:>7}/{len:7} {smoothed_eta:7} {msg}")
|
||||
.template("[{bar:.yellow/blue}] - {elapsed:<4} {pos:>7}/{len:7} {eta:7} {msg}")
|
||||
.unwrap(),
|
||||
BarType::Quiet => style.template("Scanning: {prefix}").unwrap(),
|
||||
};
|
||||
|
|
|
@ -1,8 +1,10 @@
|
|||
use std::time::Duration;
|
||||
|
||||
use crate::filters::filter_lookup;
|
||||
use crate::progress::PROGRESS_BAR;
|
||||
use crate::traits::FeroxFilter;
|
||||
use console::{measure_text_width, pad_str, style, Alignment, Term};
|
||||
use indicatif::ProgressDrawTarget;
|
||||
use indicatif::{HumanDuration, ProgressDrawTarget};
|
||||
use regex::Regex;
|
||||
|
||||
/// Data container for a command entered by the user interactively
|
||||
|
@ -43,6 +45,9 @@ pub(super) struct Menu {
|
|||
/// footer: instructions surrounded by separators
|
||||
footer: String,
|
||||
|
||||
/// length of longest displayed line (suitable for ascii/unicode)
|
||||
longest: usize,
|
||||
|
||||
/// unicode line border, matched to longest displayed line
|
||||
border: String,
|
||||
|
||||
|
@ -110,7 +115,7 @@ impl Menu {
|
|||
commands.push_str(&valid_filters);
|
||||
commands.push_str(&rm_filter_cmd);
|
||||
|
||||
let longest = measure_text_width(&canx_cmd).max(measure_text_width(&name));
|
||||
let longest = measure_text_width(&canx_cmd).max(measure_text_width(&name)) + 1;
|
||||
|
||||
let border = separator.repeat(longest);
|
||||
|
||||
|
@ -123,6 +128,7 @@ impl Menu {
|
|||
header,
|
||||
footer,
|
||||
border,
|
||||
longest,
|
||||
term: Term::stderr(),
|
||||
}
|
||||
}
|
||||
|
@ -142,6 +148,13 @@ impl Menu {
|
|||
self.println(&self.footer);
|
||||
}
|
||||
|
||||
/// print menu footer
|
||||
pub(super) fn print_eta(&self, eta: Duration) {
|
||||
let inner = format!("⏳ {} remaining ⏳", HumanDuration(eta));
|
||||
let padded_eta = pad_str(&inner, self.longest, Alignment::Center, None);
|
||||
self.println(&format!("{padded_eta}\n{}", self.border));
|
||||
}
|
||||
|
||||
/// set PROGRESS_BAR bar target to hidden
|
||||
pub(super) fn hide_progress_bars(&self) {
|
||||
PROGRESS_BAR.set_draw_target(ProgressDrawTarget::hidden());
|
||||
|
|
|
@ -33,6 +33,7 @@ use std::{
|
|||
},
|
||||
thread::sleep,
|
||||
};
|
||||
use tokio::sync::oneshot;
|
||||
use tokio::time::{self, Duration};
|
||||
|
||||
/// Single atomic number that gets incremented once, used to track first thread to interact with
|
||||
|
@ -430,6 +431,13 @@ impl FeroxScans {
|
|||
self.menu.hide_progress_bars();
|
||||
self.menu.clear_screen();
|
||||
self.menu.print_header();
|
||||
let (tx, rx) = oneshot::channel::<Duration>();
|
||||
if handles.stats.send(Command::QueryOverallBarEta(tx)).is_ok() {
|
||||
if let Ok(y) = rx.await {
|
||||
self.menu.print_eta(y);
|
||||
}
|
||||
}
|
||||
|
||||
self.display_scans().await;
|
||||
self.display_filters(handles.clone());
|
||||
self.menu.print_footer();
|
||||
|
|
|
@ -290,3 +290,52 @@ fn collect_backups_should_be_filtered() {
|
|||
assert_eq!(mock_two.hits(), 1);
|
||||
teardown_tmp_directory(tmp_dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
/// create a FeroxResponse that should elicit a true from
|
||||
/// RegexFilter::should_filter_response
|
||||
fn filters_regex_should_filter_response_based_on_headers() {
|
||||
let srv = MockServer::start();
|
||||
let (tmp_dir, file) = setup_tmp_directory(
|
||||
&["not-matching".to_string(), "matching".to_string()],
|
||||
"wordlist",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let mock = srv.mock(|when, then| {
|
||||
when.method(GET).path("/not-matching");
|
||||
then.status(200)
|
||||
.header("content-type", "text/html")
|
||||
.body("this is a test");
|
||||
});
|
||||
|
||||
let mock_two = srv.mock(|when, then| {
|
||||
when.method(GET).path("/matching");
|
||||
then.status(200)
|
||||
.header("content-type", "application/json")
|
||||
.body("this is also a test");
|
||||
});
|
||||
|
||||
let cmd = Command::cargo_bin("feroxbuster")
|
||||
.unwrap()
|
||||
.arg("--url")
|
||||
.arg(srv.url("/"))
|
||||
.arg("--wordlist")
|
||||
.arg(file.as_os_str())
|
||||
.arg("--filter-regex")
|
||||
.arg("content-type:application/json")
|
||||
.unwrap();
|
||||
|
||||
cmd.assert().success().stdout(
|
||||
predicate::str::contains("/not-matching")
|
||||
.and(predicate::str::contains("200"))
|
||||
.and(predicate::str::contains("/matching"))
|
||||
.not()
|
||||
.and(predicate::str::contains("200"))
|
||||
.not(),
|
||||
);
|
||||
|
||||
assert_eq!(mock.hits(), 1);
|
||||
assert_eq!(mock_two.hits(), 1);
|
||||
teardown_tmp_directory(tmp_dir);
|
||||
}
|
||||
|
|
|
@ -104,9 +104,9 @@ fn test_single_target_cannot_connect_due_to_ssl_errors() -> Result<(), Box<dyn s
|
|||
.arg(file.as_os_str())
|
||||
.assert()
|
||||
.success()
|
||||
.stdout(
|
||||
predicate::str::contains("Could not connect to https://expired.badssl.com due to SSL errors (run with -k to ignore), skipping...", )
|
||||
);
|
||||
.stdout(predicate::str::contains(
|
||||
"Could not connect to https://expired.badssl.com",
|
||||
));
|
||||
|
||||
teardown_tmp_directory(tmp_dir);
|
||||
Ok(())
|
||||
|
|
Loading…
Reference in New Issue
Block a user