utilities/feroxbuster
utilities/feroxbuster
1
0
Fork 0
mirror of https://github.com/epi052/feroxbuster synced 2024-07-08 19:45:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactored a few functions; added CI; clippy'd codebase

Browse Source
This commit is contained in:
epi 2020-08-30 07:40:58 -05:00
parent e0fa0de6d2
commit 04af97d8d6
4 changed files with 203 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
.github/workflows/rust-ci.yml vendored Normal file
View File

@ -0,0 +1,64 @@
name: CI Pipeline
on: [push]
jobs:
check:
name: Check
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions-rs/toolchain@v1
with:
profile: minimal
toolchain: stable
override: true
- uses: actions-rs/cargo@v1
with:
command: check
test:
name: Test Suite
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions-rs/toolchain@v1
with:
profile: minimal
toolchain: stable
override: true
- uses: actions-rs/cargo@v1
with:
command: test
fmt:
name: Rust fmt
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions-rs/toolchain@v1
with:
profile: minimal
toolchain: stable
override: true
- run: rustup component add rustfmt
- uses: actions-rs/cargo@v1
with:
command: fmt
args: --all -- --check
clippy:
name: Clippy
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions-rs/toolchain@v1
with:
profile: minimal
toolchain: stable
override: true
- run: rustup component add clippy
- uses: actions-rs/cargo@v1
with:
command: clippy
args: -- -D clippy::all -D warnings -A clippy::unnecessary_unwrap

52
src/config.rs
View File

@ -1,12 +1,12 @@
use clap::{value_t, App, Arg, ArgMatches};
use reqwest::{redirect::Policy, Client, StatusCode, Proxy};
use std::time::Duration;
use lazy_static::lazy_static;
use std::fs::read_to_string;
use reqwest::{redirect::Policy, Client, Proxy, StatusCode};
use serde::Deserialize;
use std::fs::read_to_string;
use std::time::Duration;
/// Version pulled from Cargo.toml at compile time
const VERSION: &'static str = env!("CARGO_PKG_VERSION");
const VERSION: &str = env!("CARGO_PKG_VERSION");
/// Default wordlist to use when `-w|--wordlist` isn't specified and not `wordlist` isn't set
/// in a [feroxbuster.toml](constant.DEFAULT_CONFIG_PATH.html) config file.
@ -73,12 +73,14 @@ pub struct Configuration {
pub timeout: u64,
#[serde(default)]
pub verbosity: u8,
}
fn seven() -> u64 {7}
fn fifty() -> usize {50}
fn seven() -> u64 {
7
}
fn fifty() -> usize {
50
}
impl Default for Configuration {
fn default() -> Self {
@ -105,7 +107,7 @@ impl Configuration {
///
/// - timeout: 5 seconds
/// - follow redirects: false
/// - wordlist: [DEFAULT_WORDLIST](constant.DEFAULT_WORDLIST.html)
/// - wordlist: [`DEFAULT_WORDLIST`](constant.DEFAULT_WORDLIST.html)
/// - threads: 50
/// - timeout: 7
/// - verbosity: 0 (no logging enabled)
@ -131,6 +133,7 @@ impl Configuration {
config.wordlist = settings.wordlist;
config.extensions = settings.extensions;
config.proxy = settings.proxy;
config.verbosity = settings.verbosity;
}
let args = Self::parse_args();
@ -184,7 +187,7 @@ impl Configuration {
.long("wordlist")
.value_name("FILE")
.help("Path to the wordlist")
.takes_value(true)
.takes_value(true),
)
.arg(
Arg::with_name("url")
@ -214,14 +217,17 @@ impl Configuration {
.long("verbosity")
.takes_value(false)
.multiple(true)
.help("Increase verbosity level (use -vv or more for greater effect)"))
.help("Increase verbosity level (use -vv or more for greater effect)"),
)
.arg(
Arg::with_name("proxy")
.short("p")
.long("proxy")
.takes_value(true)
.help("Proxy to use for requests (ex: http(s)://host:port, socks5://host:port)"))
.help(
"Proxy to use for requests (ex: http(s)://host:port, socks5://host:port)",
),
)
.get_matches()
}
@ -240,30 +246,32 @@ impl Configuration {
}
fn create_client(timeout: u64, proxy: Option<&str>) -> Client {
let client = Client::builder()
let client = Client::builder()
.timeout(Duration::new(timeout, 0))
.redirect(Policy::none());
// add optional proxy to config
let client = if proxy.is_some() && !proxy.unwrap().is_empty() {
match Proxy::all(proxy.unwrap()) {
Ok(proxy_obj) => {
client.proxy(proxy_obj)
}
match Proxy::all(proxy.unwrap()) {
Ok(proxy_obj) => client.proxy(proxy_obj),
Err(e) => {
log::error!("Could not add proxy ({:?}) to Client configuration: {}", proxy, e);
eprintln!(
"[!] Could not add proxy ({:?}) to Client configuration: {}",
proxy, e
);
client
}
}
} else {
log::debug!("proxy ({:?}) not added to Client configuration", proxy);
eprintln!("[!] proxy ({:?}) not added to Client configuration", proxy);
client
};
match client.build() {
Ok(client) => client,
Err(e) => {
eprintln!("Could not create a Client with the given configuration, exiting.");
eprintln!("[!] Could not create a Client with the given configuration, exiting.");
panic!("Client::build: {}", e);
}
}
@ -285,4 +293,4 @@ mod tests {
assert_eq!(config.timeout, 7);
assert_eq!(config.verbosity, 0);
}
}
}

12
src/logger.rs
View File

@ -1,9 +1,19 @@
use env_logger::Builder;
use std::env;
use std::io::Write;
use std::time::Instant;
/// Create an instance of an `env_logger` with an added time offset
pub fn init_logger() {
pub fn initialize(verbosity: u8) {
// use occurrences of -v on commandline to or verbosity = N in feroxconfig.toml to set
// log level for the application; respects already specified RUST_LOG environment variable
match verbosity {
0 => (),
1 => env::set_var("RUST_LOG", "warn"),
2 => env::set_var("RUST_LOG", "info"),
_ => env::set_var("RUST_LOG", "debug"),
}
let start = Instant::now();
let mut builder = Builder::from_default_env();

188
src/main.rs
View File

@ -1,37 +1,30 @@
use feroxbuster::logger::init_logger;
use feroxbuster::logger;
use feroxbuster::FeroxResult;
mod config;
use crate::config::{Configuration, DEFAULT_RESPONSE_CODES, CONFIGURATION};
use crate::config::{Configuration, CONFIGURATION, DEFAULT_RESPONSE_CODES};
use futures::stream;
use futures::StreamExt;
use log;
use reqwest::{Response, Url, Client};
use std::fs::File;
use std::io::{BufRead, BufReader, Lines};
use tokio::task;
use reqwest::{Client, Response, Url};
use std::collections::HashSet;
use std::env;
use std::fs::File;
use std::io::{BufRead, BufReader};
use tokio::task;
/// Simple helper to generate a `reqwest::Url`
///
/// If an error occurs during parsing `url` or joining `word`, None is returned
pub fn format_url(word: &str, url: &str) -> Option<Url> {
let base_url = match reqwest::Url::parse(&url) {
Ok(url) => url,
Err(e) => {
log::warn!("Could not convert {} into a URL: {}", url, e);
return None;
}
};
/// Errors during parsing `url` or joining `word` are propagated up the call stack
pub fn format_url(word: &str, url: &str) -> FeroxResult<Url> {
let base_url = reqwest::Url::parse(&url)?;
if let Ok(req) = base_url.join(word) {
log::debug!("Requested URL: {}", req);
Some(req)
}
else {
log::warn!("Could not join {} with {}", word, base_url);
None
match base_url.join(word) {
Ok(request) => {
log::debug!("Requested URL: {}", request);
Ok(request)
}
Err(e) => {
log::warn!("Could not join {} with {}", word, base_url);
Err(Box::new(e))
}
}
}
@ -40,89 +33,92 @@ async fn make_request(client: &Client, url: Url) -> Response {
client.get(url).send().await.unwrap()
}
fn process_wordlist(config: &Configuration) -> Lines<BufReader<File>> {
// todo: remove unwrap
let file = File::open(&config.wordlist).unwrap();
/// Creates a Set of Strings from the given wordlist
fn get_unique_words_from_wordlist(config: &Configuration) -> FeroxResult<HashSet<String>> {
//todo: stupid function name
let file = match File::open(&config.wordlist) {
Ok(f) => f,
Err(e) => {
log::error!("Could not open wordlist: {}", e);
return Err(Box::new(e));
}
};
let reader = BufReader::new(file);
let words = reader.lines();
let mut words = HashSet::new();
words
for line in reader.lines() {
match line {
Ok(word) => {
words.insert(word);
}
Err(e) => {
log::warn!("Could not parse current line from wordlist : {}", e);
}
}
}
Ok(words)
}
async fn bust_dirs(urls: &HashSet<Url>, client: &'static Client, threads: usize) -> FeroxResult<()> {
let mut buffered_futures = stream::iter(urls.to_owned())
.map(move |url| {
let future = task::spawn(async move { make_request(&client, url).await });
future
async fn bust_dir(
base_url: &'static str,
words: &HashSet<String>,
client: &'static Client,
threads: usize,
) -> FeroxResult<()> {
let mut buffered_futures = stream::iter(words.to_owned())
.map(move |directory| {
// todo: can i remove the unwrap? map_err or something?
let url = format_url(&directory, &base_url).unwrap();
task::spawn(async move { make_request(&client, url).await })
})
.buffer_unordered(threads);
log::debug!("{:#?}", buffered_futures);
log::debug!("{:?}", buffered_futures);
while let Some(item) = buffered_futures.next().await {
match item {
Ok(response) => {
let response_code = &response.status();
for code in DEFAULT_RESPONSE_CODES.iter() {
if response_code == code {
println!("[{}] - {}", response.status(), response.url());
break;
}
while let Some(item) = buffered_futures.next().await {
match item {
Ok(response) => {
let response_code = &response.status();
for code in DEFAULT_RESPONSE_CODES.iter() {
if response_code == code {
println!(
"[{}] - {} - [{} bytes]",
response.status(),
response.url(),
response.content_length().unwrap_or(0)
);
break;
}
}
Err(e) => {
println!("Err: {}", e);
}
}
Err(e) => {
println!("Err: {}", e);
}
}
}
Ok(())
Ok(())
}
async fn app() -> FeroxResult<()> {
let words = task::spawn_blocking(move || process_wordlist(&CONFIGURATION)).await?;
let words = task::spawn(async move { get_unique_words_from_wordlist(&CONFIGURATION) }).await??;
// urls is a Set of urls, invalid UTF-8 words result in the base url being accumulated
// as this is a set, the base url will only be in the set once, even if there are multiple
// invalid UTF-8 results from the wordlist
let base_url = CONFIGURATION.target_url.to_owned();
bust_dir(
&CONFIGURATION.target_url,
&words,
&CONFIGURATION.client,
CONFIGURATION.threads,
)
.await?;
let urls: HashSet<Url> = task::spawn_blocking(move || {
words.map(move |word| {
match word {
Ok(w) => {
// todo: remove unwrap here
format_url(&w, &base_url).unwrap()
}
Err(e) => {
log::warn!("get_urls: {}", e);
// todo: remove unwrap here
format_url(&"", &base_url).unwrap()
}
}
})
})
.await?.collect();
log::debug!("{:#?}", urls);
bust_dirs(&urls, &CONFIGURATION.client, CONFIGURATION.threads).await?;
Ok(())
}
fn main() {
// use occurrences of -v on commandline to or verbosity = N in feroxconfig.toml to set
// log level for the application; respects already specified RUST_LOG environment variable
match CONFIGURATION.verbosity {
0 => (),
1 => env::set_var("RUST_LOG", "warn"),
2 => env::set_var("RUST_LOG", "info"),
_ => env::set_var("RUST_LOG", "debug"),
}
init_logger();
logger::initialize(CONFIGURATION.verbosity);
log::debug!("{:#?}", *CONFIGURATION);
@ -140,12 +136,18 @@ mod tests {
#[test]
fn test_format_url_normal() {
assert_eq!(format_url("stuff", "http://localhost").unwrap(), reqwest::Url::parse("http://localhost/stuff").unwrap());
assert_eq!(
format_url("stuff", "http://localhost").unwrap(),
reqwest::Url::parse("http://localhost/stuff").unwrap()
);
}
#[test]
fn test_format_url_no_word() {
assert_eq!(format_url("", "http://localhost").unwrap(), reqwest::Url::parse("http://localhost").unwrap());
assert_eq!(
format_url("", "http://localhost").unwrap(),
reqwest::Url::parse("http://localhost").unwrap()
);
}
#[test]
@ -156,11 +158,17 @@ mod tests {
#[test]
fn test_format_url_word_with_preslash() {
assert_eq!(format_url("/stuff", "http://localhost").unwrap(), reqwest::Url::parse("http://localhost/stuff").unwrap());
assert_eq!(
format_url("/stuff", "http://localhost").unwrap(),
reqwest::Url::parse("http://localhost/stuff").unwrap()
);
}
#[test]
fn test_format_url_word_with_postslash() {
assert_eq!(format_url("stuff/", "http://localhost").unwrap(), reqwest::Url::parse("http://localhost/stuff/").unwrap());
assert_eq!(
format_url("stuff/", "http://localhost").unwrap(),
reqwest::Url::parse("http://localhost/stuff/").unwrap()
);
}
}
}