mirror of
https://github.com/epi052/feroxbuster
synced 2024-06-29 05:54:32 +00:00
1105 - improve json logs for post processing (#1114)
* added timestamp field to responses * added targets field to stats object * clippy * write config to output file * --data implies post request if no other method provided * fixed issue with multiple leading spaces * ignoring flaky test * upgraded deps * bumped version to 2.10.3
This commit is contained in:
epi
GitHub
parent
f3d6d185cd
commit
8eb41f40a0
607
Cargo.lock
generated
607
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
24
Cargo.toml
24
Cargo.toml
|
|
@ -1,6 +1,6 @@
|
|||
[package]
|
||||
name = "feroxbuster"
|
||||
version = "2.10.2"
|
||||
version = "2.10.3"
|
||||
authors = ["Ben 'epi' Risher (@epi052)"]
|
||||
license = "MIT"
|
||||
edition = "2021"
|
||||
|
|
@ -22,36 +22,36 @@ build = "build.rs"
|
|||
maintenance = { status = "actively-developed" }
|
||||
|
||||
[build-dependencies]
|
||||
clap = { version = "4.3", features = ["wrap_help", "cargo"] }
|
||||
clap_complete = "4.3"
|
||||
regex = "1.9"
|
||||
clap = { version = "4.5", features = ["wrap_help", "cargo"] }
|
||||
clap_complete = "4.5"
|
||||
regex = "1.10"
|
||||
lazy_static = "1.4"
|
||||
dirs = "5.0"
|
||||
|
||||
[dependencies]
|
||||
scraper = "0.18"
|
||||
futures = "0.3"
|
||||
tokio = { version = "1.29", features = ["full"] }
|
||||
tokio = { version = "1.37", features = ["full"] }
|
||||
tokio-util = { version = "0.7", features = ["codec"] }
|
||||
log = "0.4"
|
||||
env_logger = "0.10"
|
||||
reqwest = { version = "0.11", features = ["socks", "native-tls-alpn"] }
|
||||
# uses feature unification to add 'serde' to reqwest::Url
|
||||
url = { version = "2.4", features = ["serde"] }
|
||||
url = { version = "2.5", features = ["serde"] }
|
||||
serde_regex = "1.1"
|
||||
clap = { version = "4.3", features = ["wrap_help", "cargo"] }
|
||||
clap = { version = "4.5", features = ["wrap_help", "cargo"] }
|
||||
lazy_static = "1.4"
|
||||
toml = "0.8"
|
||||
serde = { version = "1.0", features = ["derive", "rc"] }
|
||||
serde_json = "1.0"
|
||||
uuid = { version = "1.4", features = ["v4"] }
|
||||
uuid = { version = "1.8", features = ["v4"] }
|
||||
# last known working version of indicatif; 0.17.5 has a bug that causes the
|
||||
# scan menu to fail spectacularly
|
||||
indicatif = { version = "0.17.3" }
|
||||
indicatif = { version = "0.17.8" }
|
||||
console = "0.15"
|
||||
openssl = { version = "0.10", features = ["vendored"] }
|
||||
dirs = "5.0"
|
||||
regex = "1.9"
|
||||
regex = "1.10"
|
||||
crossterm = "0.27"
|
||||
rlimit = "0.10"
|
||||
ctrlc = "3.4"
|
||||
|
|
@ -69,10 +69,10 @@ self_update = { version = "0.36", features = [
|
|||
] }
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3.6"
|
||||
tempfile = "3.10"
|
||||
httpmock = "0.6"
|
||||
assert_cmd = "2.0"
|
||||
predicates = "3.0"
|
||||
predicates = "3.1"
|
||||
|
||||
[profile.release]
|
||||
lto = true
|
||||
|
|
|
|||
|
|
@ -2,6 +2,9 @@
|
|||
[tasks.upgrade]
|
||||
dependencies = ["upgrade-deps", "update"]
|
||||
|
||||
[tasks.check]
|
||||
dependencies = ["fmt", "clippy", "test"]
|
||||
|
||||
# cleaning
|
||||
[tasks.clean-state]
|
||||
script = """
|
||||
|
|
@ -24,6 +27,12 @@ script = """
|
|||
cargo clippy --all-targets --all-features -- -D warnings
|
||||
"""
|
||||
|
||||
[tasks.fmt]
|
||||
clear = true
|
||||
script = """
|
||||
cargo fmt --all
|
||||
"""
|
||||
|
||||
# tests
|
||||
[tasks.test]
|
||||
clear = true
|
||||
|
|
|
|||
|
|
@ -24,8 +24,8 @@ _feroxbuster() {
|
|||
'--replay-proxy=[Send only unfiltered requests through a Replay Proxy, instead of all requests]:REPLAY_PROXY:_urls' \
|
||||
'*-R+[Status Codes to send through a Replay Proxy when found (default\: --status-codes value)]:REPLAY_CODE: ' \
|
||||
'*--replay-codes=[Status Codes to send through a Replay Proxy when found (default\: --status-codes value)]:REPLAY_CODE: ' \
|
||||
'-a+[Sets the User-Agent (default\: feroxbuster/2.10.2)]:USER_AGENT: ' \
|
||||
'--user-agent=[Sets the User-Agent (default\: feroxbuster/2.10.2)]:USER_AGENT: ' \
|
||||
'-a+[Sets the User-Agent (default\: feroxbuster/2.10.3)]:USER_AGENT: ' \
|
||||
'--user-agent=[Sets the User-Agent (default\: feroxbuster/2.10.3)]:USER_AGENT: ' \
|
||||
'*-x+[File extension(s) to search for (ex\: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex\: @ext.txt)]:FILE_EXTENSION: ' \
|
||||
'*--extensions=[File extension(s) to search for (ex\: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex\: @ext.txt)]:FILE_EXTENSION: ' \
|
||||
'*-m+[Which HTTP request method(s) should be sent (default\: GET)]:HTTP_METHODS: ' \
|
||||
|
|
|
|||
|
|
@ -30,8 +30,8 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
|
|||
[CompletionResult]::new('--replay-proxy', 'replay-proxy', [CompletionResultType]::ParameterName, 'Send only unfiltered requests through a Replay Proxy, instead of all requests')
|
||||
[CompletionResult]::new('-R', 'R ', [CompletionResultType]::ParameterName, 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)')
|
||||
[CompletionResult]::new('--replay-codes', 'replay-codes', [CompletionResultType]::ParameterName, 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)')
|
||||
[CompletionResult]::new('-a', 'a', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.10.2)')
|
||||
[CompletionResult]::new('--user-agent', 'user-agent', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.10.2)')
|
||||
[CompletionResult]::new('-a', 'a', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.10.3)')
|
||||
[CompletionResult]::new('--user-agent', 'user-agent', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.10.3)')
|
||||
[CompletionResult]::new('-x', 'x', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex: @ext.txt)')
|
||||
[CompletionResult]::new('--extensions', 'extensions', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex: @ext.txt)')
|
||||
[CompletionResult]::new('-m', 'm', [CompletionResultType]::ParameterName, 'Which HTTP request method(s) should be sent (default: GET)')
|
||||
|
|
|
|||
|
|
@ -35,12 +35,12 @@ _feroxbuster() {
|
|||
;;
|
||||
--resume-from)
|
||||
local oldifs
|
||||
if [[ -v IFS ]]; then
|
||||
if [ -n "${IFS+x}" ]; then
|
||||
oldifs="$IFS"
|
||||
fi
|
||||
IFS=$'\n'
|
||||
COMPREPLY=($(compgen -f "${cur}"))
|
||||
if [[ -v oldifs ]]; then
|
||||
if [ -n "${oldifs+x}" ]; then
|
||||
IFS="$oldifs"
|
||||
fi
|
||||
if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
|
||||
|
|
@ -190,12 +190,12 @@ _feroxbuster() {
|
|||
;;
|
||||
--server-certs)
|
||||
local oldifs
|
||||
if [[ -v IFS ]]; then
|
||||
if [ -n "${IFS+x}" ]; then
|
||||
oldifs="$IFS"
|
||||
fi
|
||||
IFS=$'\n'
|
||||
COMPREPLY=($(compgen -f "${cur}"))
|
||||
if [[ -v oldifs ]]; then
|
||||
if [ -n "${oldifs+x}" ]; then
|
||||
IFS="$oldifs"
|
||||
fi
|
||||
if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
|
||||
|
|
@ -205,12 +205,12 @@ _feroxbuster() {
|
|||
;;
|
||||
--client-cert)
|
||||
local oldifs
|
||||
if [[ -v IFS ]]; then
|
||||
if [ -n "${IFS+x}" ]; then
|
||||
oldifs="$IFS"
|
||||
fi
|
||||
IFS=$'\n'
|
||||
COMPREPLY=($(compgen -f "${cur}"))
|
||||
if [[ -v oldifs ]]; then
|
||||
if [ -n "${oldifs+x}" ]; then
|
||||
IFS="$oldifs"
|
||||
fi
|
||||
if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
|
||||
|
|
@ -220,12 +220,12 @@ _feroxbuster() {
|
|||
;;
|
||||
--client-key)
|
||||
local oldifs
|
||||
if [[ -v IFS ]]; then
|
||||
if [ -n "${IFS+x}" ]; then
|
||||
oldifs="$IFS"
|
||||
fi
|
||||
IFS=$'\n'
|
||||
COMPREPLY=($(compgen -f "${cur}"))
|
||||
if [[ -v oldifs ]]; then
|
||||
if [ -n "${oldifs+x}" ]; then
|
||||
IFS="$oldifs"
|
||||
fi
|
||||
if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
|
||||
|
|
@ -271,12 +271,12 @@ _feroxbuster() {
|
|||
;;
|
||||
--wordlist)
|
||||
local oldifs
|
||||
if [[ -v IFS ]]; then
|
||||
if [ -n "${IFS+x}" ]; then
|
||||
oldifs="$IFS"
|
||||
fi
|
||||
IFS=$'\n'
|
||||
COMPREPLY=($(compgen -f "${cur}"))
|
||||
if [[ -v oldifs ]]; then
|
||||
if [ -n "${oldifs+x}" ]; then
|
||||
IFS="$oldifs"
|
||||
fi
|
||||
if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
|
||||
|
|
@ -286,12 +286,12 @@ _feroxbuster() {
|
|||
;;
|
||||
-w)
|
||||
local oldifs
|
||||
if [[ -v IFS ]]; then
|
||||
if [ -n "${IFS+x}" ]; then
|
||||
oldifs="$IFS"
|
||||
fi
|
||||
IFS=$'\n'
|
||||
COMPREPLY=($(compgen -f "${cur}"))
|
||||
if [[ -v oldifs ]]; then
|
||||
if [ -n "${oldifs+x}" ]; then
|
||||
IFS="$oldifs"
|
||||
fi
|
||||
if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
|
||||
|
|
@ -317,12 +317,12 @@ _feroxbuster() {
|
|||
;;
|
||||
--output)
|
||||
local oldifs
|
||||
if [[ -v IFS ]]; then
|
||||
if [ -n "${IFS+x}" ]; then
|
||||
oldifs="$IFS"
|
||||
fi
|
||||
IFS=$'\n'
|
||||
COMPREPLY=($(compgen -f "${cur}"))
|
||||
if [[ -v oldifs ]]; then
|
||||
if [ -n "${oldifs+x}" ]; then
|
||||
IFS="$oldifs"
|
||||
fi
|
||||
if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
|
||||
|
|
@ -332,12 +332,12 @@ _feroxbuster() {
|
|||
;;
|
||||
-o)
|
||||
local oldifs
|
||||
if [[ -v IFS ]]; then
|
||||
if [ -n "${IFS+x}" ]; then
|
||||
oldifs="$IFS"
|
||||
fi
|
||||
IFS=$'\n'
|
||||
COMPREPLY=($(compgen -f "${cur}"))
|
||||
if [[ -v oldifs ]]; then
|
||||
if [ -n "${oldifs+x}" ]; then
|
||||
IFS="$oldifs"
|
||||
fi
|
||||
if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
|
||||
|
|
@ -347,12 +347,12 @@ _feroxbuster() {
|
|||
;;
|
||||
--debug-log)
|
||||
local oldifs
|
||||
if [[ -v IFS ]]; then
|
||||
if [ -n "${IFS+x}" ]; then
|
||||
oldifs="$IFS"
|
||||
fi
|
||||
IFS=$'\n'
|
||||
COMPREPLY=($(compgen -f "${cur}"))
|
||||
if [[ -v oldifs ]]; then
|
||||
if [ -n "${oldifs+x}" ]; then
|
||||
IFS="$oldifs"
|
||||
fi
|
||||
if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
|
||||
|
|
|
|||
|
|
@ -27,8 +27,8 @@ set edit:completion:arg-completer[feroxbuster] = {|@words|
|
|||
cand --replay-proxy 'Send only unfiltered requests through a Replay Proxy, instead of all requests'
|
||||
cand -R 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)'
|
||||
cand --replay-codes 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)'
|
||||
cand -a 'Sets the User-Agent (default: feroxbuster/2.10.2)'
|
||||
cand --user-agent 'Sets the User-Agent (default: feroxbuster/2.10.2)'
|
||||
cand -a 'Sets the User-Agent (default: feroxbuster/2.10.3)'
|
||||
cand --user-agent 'Sets the User-Agent (default: feroxbuster/2.10.3)'
|
||||
cand -x 'File extension(s) to search for (ex: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex: @ext.txt)'
|
||||
cand --extensions 'File extension(s) to search for (ex: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex: @ext.txt)'
|
||||
cand -m 'Which HTTP request method(s) should be sent (default: GET)'
|
||||
|
|
|
|||
|
|
@ -704,6 +704,11 @@ impl Configuration {
|
|||
} else {
|
||||
config.data = arg.as_bytes().to_vec();
|
||||
}
|
||||
|
||||
if config.methods == methods() {
|
||||
// if the user didn't specify a method, we're going to assume they meant to use POST
|
||||
config.methods = vec![Method::POST.as_str().to_string()];
|
||||
}
|
||||
}
|
||||
|
||||
if came_from_cli!(args, "stdin") {
|
||||
|
|
@ -935,7 +940,15 @@ impl Configuration {
|
|||
// all other items in the iterator returned by split, when combined with the
|
||||
// original split deliminator (:), make up the header's final value
|
||||
let value = split_val.collect::<Vec<&str>>().join(":");
|
||||
config.headers.insert(name.to_string(), value.to_string());
|
||||
|
||||
if value.starts_with(' ') && !value.starts_with(" ") {
|
||||
// first character is a space and the second character isn't
|
||||
// we can trim the leading space
|
||||
let trimmed = value.trim_start();
|
||||
config.headers.insert(name.to_string(), trimmed.to_string());
|
||||
} else {
|
||||
config.headers.insert(name.to_string(), value.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -85,4 +85,7 @@ pub enum Command {
|
|||
/// Give a handler access to an Arc<Handles> instance after the handler has
|
||||
/// already been initialized
|
||||
AddHandles(Arc<Handles>),
|
||||
|
||||
/// inform the Stats object about which targets are being scanned
|
||||
UpdateTargets(Vec<String>),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -98,6 +98,8 @@ impl FileOutHandler {
|
|||
|
||||
log::info!("Writing scan results to {}", self.config.output);
|
||||
|
||||
write_to(&*self.config, &mut file, self.config.json)?;
|
||||
|
||||
while let Some(command) = self.receiver.recv().await {
|
||||
match command {
|
||||
Command::Report(response) => {
|
||||
|
|
|
|||
|
|
@ -125,6 +125,9 @@ impl StatsHandler {
|
|||
Command::Sync(sender) => {
|
||||
sender.send(true).unwrap_or_default();
|
||||
}
|
||||
Command::UpdateTargets(targets) => {
|
||||
self.stats.update_targets(targets);
|
||||
}
|
||||
Command::Exit => break,
|
||||
_ => {} // no more commands needed
|
||||
}
|
||||
|
|
|
|||
|
|
@ -271,7 +271,7 @@ fn remove_function_works_as_expected() {
|
|||
|
||||
assert_eq!(data.filters.read().unwrap().len(), 5);
|
||||
|
||||
let expected = vec![
|
||||
let expected = [
|
||||
WordsFilter { word_count: 1 },
|
||||
WordsFilter { word_count: 3 },
|
||||
WordsFilter { word_count: 5 },
|
||||
|
|
|
|||
11
src/main.rs
11
src/main.rs
|
|
@ -25,7 +25,8 @@ use feroxbuster::{
|
|||
config::{Configuration, OutputLevel},
|
||||
event_handlers::{
|
||||
Command::{
|
||||
AddHandles, CreateBar, Exit, JoinTasks, LoadStats, ScanInitialUrls, UpdateWordlist,
|
||||
AddHandles, CreateBar, Exit, JoinTasks, LoadStats, ScanInitialUrls, UpdateTargets,
|
||||
UpdateWordlist,
|
||||
},
|
||||
FiltersHandler, Handles, ScanHandler, StatsHandler, Tasks, TermInputHandler,
|
||||
TermOutHandler, SCAN_COMPLETE,
|
||||
|
|
@ -507,6 +508,14 @@ async fn wrapped_main(config: Arc<Configuration>) -> Result<()> {
|
|||
return Ok(());
|
||||
}
|
||||
|
||||
// in order for the Stats object to know about which targets are being scanned, we need to
|
||||
// wait until the parallel branch has been handled before sending the UpdateTargets command
|
||||
// this ensures that only the targets being scanned are sent to the Stats object
|
||||
//
|
||||
// if sent before the parallel branch is handled, the Stats object will have duplicate
|
||||
// targets
|
||||
handles.stats.send(UpdateTargets(targets.clone()))?;
|
||||
|
||||
if matches!(config.output_level, OutputLevel::Default) {
|
||||
// only print banner if output level is default (no banner on --quiet|--silent)
|
||||
let std_stderr = stderr(); // std::io::stderr
|
||||
|
|
|
|||
|
|
@ -45,9 +45,7 @@ impl Document {
|
|||
|
||||
let html = Html::parse_document(raw_html);
|
||||
|
||||
let Some(element) = html.select(&selector).next() else {
|
||||
return None;
|
||||
};
|
||||
let element = html.select(&selector).next()?;
|
||||
|
||||
let text = element
|
||||
.descendants()
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ use crate::{
|
|||
event_handlers::{Command, Handles},
|
||||
traits::FeroxSerialize,
|
||||
url::FeroxUrl,
|
||||
utils::{self, fmt_err, parse_url_with_raw_path, status_colorizer},
|
||||
utils::{self, fmt_err, parse_url_with_raw_path, status_colorizer, timestamp},
|
||||
CommandSender,
|
||||
};
|
||||
|
||||
|
|
@ -63,6 +63,9 @@ pub struct FeroxResponse {
|
|||
|
||||
/// Url's file extension, if one exists
|
||||
pub(crate) extension: Option<String>,
|
||||
|
||||
/// Timestamp of when this response was received
|
||||
timestamp: f64,
|
||||
}
|
||||
|
||||
/// implement Default trait for FeroxResponse
|
||||
|
|
@ -82,6 +85,7 @@ impl Default for FeroxResponse {
|
|||
wildcard: false,
|
||||
output_level: Default::default(),
|
||||
extension: None,
|
||||
timestamp: timestamp(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -138,6 +142,11 @@ impl FeroxResponse {
|
|||
self.content_length
|
||||
}
|
||||
|
||||
/// Get the timestamp of this response
|
||||
pub fn timestamp(&self) -> f64 {
|
||||
self.timestamp
|
||||
}
|
||||
|
||||
/// Set `FeroxResponse`'s `url` attribute, has no affect if an error occurs
|
||||
pub fn set_url(&mut self, url: &str) {
|
||||
match parse_url_with_raw_path(url) {
|
||||
|
|
@ -216,6 +225,7 @@ impl FeroxResponse {
|
|||
let status = response.status();
|
||||
let headers = response.headers().clone();
|
||||
let content_length = response.content_length().unwrap_or(0);
|
||||
let timestamp = timestamp();
|
||||
|
||||
// .text() consumes the response, must be called last
|
||||
let text = response
|
||||
|
|
@ -248,6 +258,7 @@ impl FeroxResponse {
|
|||
output_level,
|
||||
wildcard: false,
|
||||
extension: None,
|
||||
timestamp,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -574,6 +585,7 @@ impl Serialize for FeroxResponse {
|
|||
"extension",
|
||||
self.extension.as_ref().unwrap_or(&String::new()),
|
||||
)?;
|
||||
state.serialize_field("timestamp", &self.timestamp)?;
|
||||
|
||||
state.end()
|
||||
}
|
||||
|
|
@ -599,6 +611,7 @@ impl<'de> Deserialize<'de> for FeroxResponse {
|
|||
line_count: 0,
|
||||
word_count: 0,
|
||||
extension: None,
|
||||
timestamp: timestamp(),
|
||||
};
|
||||
|
||||
let map: HashMap<String, Value> = HashMap::deserialize(deserializer)?;
|
||||
|
|
@ -672,6 +685,11 @@ impl<'de> Deserialize<'de> for FeroxResponse {
|
|||
response.extension = Some(result.to_string());
|
||||
}
|
||||
}
|
||||
"timestamp" => {
|
||||
if let Some(result) = value.as_f64() {
|
||||
response.timestamp = result;
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -314,7 +314,7 @@ fn ferox_scans_serialize() {
|
|||
#[test]
|
||||
/// given a FeroxResponses, test that it serializes into the proper JSON entry
|
||||
fn ferox_responses_serialize() {
|
||||
let json_response = r#"{"type":"response","url":"https://nerdcore.com/css","original_url":"https://nerdcore.com","path":"/css","wildcard":true,"status":301,"method":"GET","content_length":173,"line_count":10,"word_count":16,"headers":{"server":"nginx/1.16.1"},"extension":""}"#;
|
||||
let json_response = r#"{"type":"response","url":"https://nerdcore.com/css","original_url":"https://nerdcore.com","path":"/css","wildcard":true,"status":301,"method":"GET","content_length":173,"line_count":10,"word_count":16,"headers":{"server":"nginx/1.16.1"},"extension":"","timestamp":1711796681.3455093}"#;
|
||||
let response: FeroxResponse = serde_json::from_str(json_response).unwrap();
|
||||
|
||||
let responses = FeroxResponses::default();
|
||||
|
|
@ -332,7 +332,7 @@ fn ferox_responses_serialize() {
|
|||
/// given a FeroxResponse, test that it serializes into the proper JSON entry
|
||||
fn ferox_response_serialize_and_deserialize() {
|
||||
// deserialize
|
||||
let json_response = r#"{"type":"response","url":"https://nerdcore.com/css","original_url":"https://nerdcore.com","path":"/css","wildcard":true,"status":301,"method":"GET","content_length":173,"line_count":10,"word_count":16,"headers":{"server":"nginx/1.16.1"},"extension":""}"#;
|
||||
let json_response = r#"{"type":"response","url":"https://nerdcore.com/css","original_url":"https://nerdcore.com","path":"/css","wildcard":true,"status":301,"method":"GET","content_length":173,"line_count":10,"word_count":16,"headers":{"server":"nginx/1.16.1"},"extension":"","timestamp":1711796681.3455093}"#;
|
||||
let response: FeroxResponse = serde_json::from_str(json_response).unwrap();
|
||||
|
||||
assert_eq!(response.url().as_str(), "https://nerdcore.com/css");
|
||||
|
|
@ -343,6 +343,7 @@ fn ferox_response_serialize_and_deserialize() {
|
|||
assert_eq!(response.line_count(), 10);
|
||||
assert_eq!(response.word_count(), 16);
|
||||
assert_eq!(response.headers().get("server").unwrap(), "nginx/1.16.1");
|
||||
assert_eq!(response.timestamp(), 1711796681.3455093);
|
||||
|
||||
// serialize, however, this can fail when headers are out of order
|
||||
let new_json = serde_json::to_string(&response).unwrap();
|
||||
|
|
|
|||
|
|
@ -132,6 +132,9 @@ pub struct Stats {
|
|||
|
||||
/// tracker for whether to use json during serialization or not
|
||||
json: bool,
|
||||
|
||||
/// tracker for the initial targets that were passed in to the scan
|
||||
targets: Mutex<Vec<String>>,
|
||||
}
|
||||
|
||||
/// FeroxSerialize implementation for Stats
|
||||
|
|
@ -196,6 +199,7 @@ impl Serialize for Stats {
|
|||
state.serialize_field("request_errors", &atomic_load!(self.request_errors))?;
|
||||
state.serialize_field("directory_scan_times", &self.directory_scan_times)?;
|
||||
state.serialize_field("total_runtime", &self.total_runtime)?;
|
||||
state.serialize_field("targets", &self.targets)?;
|
||||
|
||||
state.end()
|
||||
}
|
||||
|
|
@ -446,6 +450,17 @@ impl<'a> Deserialize<'a> for Stats {
|
|||
}
|
||||
}
|
||||
}
|
||||
"targets" => {
|
||||
if let Some(arr) = value.as_array() {
|
||||
for val in arr {
|
||||
if let Some(parsed) = val.as_str() {
|
||||
if let Ok(mut guard) = stats.targets.lock() {
|
||||
guard.push(parsed.to_string())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
|
@ -514,6 +529,13 @@ impl Stats {
|
|||
}
|
||||
}
|
||||
|
||||
/// update targets with the given vector of strings
|
||||
pub fn update_targets(&self, targets: Vec<String>) {
|
||||
if let Ok(mut locked_targets) = self.targets.lock() {
|
||||
*locked_targets = targets;
|
||||
}
|
||||
}
|
||||
|
||||
/// save an instance of `Stats` to disk after updating the total runtime for the scan
|
||||
pub fn save(&self, seconds: f64, location: &str) -> Result<()> {
|
||||
let mut file = open_file(location)?;
|
||||
|
|
|
|||
|
|
@ -270,7 +270,7 @@ mod tests {
|
|||
let pdf = Url::parse("http://localhost/turbo.pdf").unwrap();
|
||||
let tar = Url::parse("http://localhost/turbo.tar.gz").unwrap();
|
||||
|
||||
let expected = vec![
|
||||
let expected = [
|
||||
vec![base.clone(), js.clone()],
|
||||
vec![base.clone(), js.clone(), php.clone()],
|
||||
vec![base.clone(), js.clone(), php.clone(), pdf.clone()],
|
||||
|
|
|
|||
14
src/utils.rs
14
src/utils.rs
|
|
@ -68,6 +68,20 @@ pub fn fmt_err(msg: &str) -> String {
|
|||
format!("{}: {}", status_colorizer("ERROR"), msg)
|
||||
}
|
||||
|
||||
/// simple wrapper to get the current system time as
|
||||
/// time elapsed from unix epoch
|
||||
pub fn timestamp() -> f64 {
|
||||
let since_the_epoch = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_else(|_| Duration::from_secs(0));
|
||||
|
||||
let secs = since_the_epoch.as_secs() as f64;
|
||||
let nanos = since_the_epoch.subsec_nanos() as f64;
|
||||
|
||||
// Convert nanoseconds to fractional seconds and add to secs
|
||||
secs + (nanos / 1_000_000_000.0)
|
||||
}
|
||||
|
||||
/// given a FeroxResponse, send a TryRecursion command
|
||||
///
|
||||
/// moved to utils to allow for calls from extractor and scanner
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ use utils::{setup_tmp_directory, teardown_tmp_directory};
|
|||
// these words will be used along with pattern matching to trigger different policies
|
||||
|
||||
#[test]
|
||||
#[ignore]
|
||||
/// --auto-bail should cancel a scan with spurious errors
|
||||
fn auto_bail_cancels_scan_with_timeouts() {
|
||||
let srv = MockServer::start();
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user