From 2efe3bc5b677a825bae42624d2fc58b13902b951 Mon Sep 17 00:00:00 2001 From: epi Date: Tue, 12 Jan 2021 14:35:31 -0600 Subject: [PATCH 01/13] fixed shared lib error --- src/filters/mod.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/filters/mod.rs b/src/filters/mod.rs index 02d4bd5..30f8510 100644 --- a/src/filters/mod.rs +++ b/src/filters/mod.rs @@ -2,7 +2,6 @@ mod traits; mod wildcard; mod status_code; -mod shared; mod words; mod lines; mod size; @@ -23,5 +22,3 @@ pub use self::words::WordsFilter; use crate::{config::CONFIGURATION, utils::get_url_path_length, FeroxResponse, FeroxSerialize}; use std::any::Any; use std::fmt::Debug; - -// try using pub(in self) or w/e From 1f57f823585a105a20ca59e01e4a4a7c1c4c701e Mon Sep 17 00:00:00 2001 From: epi Date: Tue, 12 Jan 2021 14:56:27 -0600 Subject: [PATCH 02/13] added 403 as a valid recursion target --- src/scanner.rs | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/scanner.rs b/src/scanner.rs index 1295ead..44e8dd2 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -22,7 +22,7 @@ use futures::{ use fuzzyhash::FuzzyHash; use lazy_static::lazy_static; use regex::Regex; -use reqwest::Url; +use reqwest::{StatusCode, Url}; #[cfg(not(test))] use std::process::exit; use std::{ @@ -260,8 +260,9 @@ fn response_is_directory(response: &FeroxResponse) -> bool { return false; } } - } else if response.status().is_success() { - // status code is 2xx, need to check if it ends in / + } else if response.status().is_success() || matches!(response.status(), &StatusCode::FORBIDDEN) + { + // status code is 2xx or 403, need to check if it ends in / if response.url().as_str().ends_with('/') { log::debug!("{} is directory suitable for recursion", response.url()); @@ -464,10 +465,14 @@ async fn make_requests( if !CONFIGURATION.no_recursion { log::debug!("Recursive extraction: {}", new_ferox_response); - if new_ferox_response.status().is_success() - && !new_ferox_response.url().as_str().ends_with('/') + if !new_ferox_response.url().as_str().ends_with('/') + && (new_ferox_response.status().is_success() + || matches!(new_ferox_response.status(), &StatusCode::FORBIDDEN)) { - // since all of these are 2xx, recursion is only attempted if the + // if the url doesn't end with a / + // and the response code is either a 2xx or 403 + + // since all of these are 2xx or 403, recursion is only attempted if the // url ends in a /. I am actually ok with adding the slash and not // adding it, as both have merit. Leaving it in for now to see how // things turn out (current as of: v1.1.0) @@ -540,8 +545,12 @@ async fn scan_robots_txt( } else if !CONFIGURATION.no_recursion { log::debug!("Directory extracted from robots.txt: {}", ferox_response); // todo this code is essentially the same as another piece around ~467 of this file - if ferox_response.status().is_success() && !ferox_response.url().as_str().ends_with('/') + if !ferox_response.url().as_str().ends_with('/') + && (ferox_response.status().is_success() + || matches!(ferox_response.status(), &StatusCode::FORBIDDEN)) { + // if the url doesn't end with a / + // and the response code is either a 2xx or 403 ferox_response.set_url(&format!("{}/", ferox_response.url())); } From d22d8aea517567701a70957e10e925169d9f47a9 Mon Sep 17 00:00:00 2001 From: epi Date: Tue, 12 Jan 2021 19:23:30 -0600 Subject: [PATCH 03/13] added test for 403 recursion --- tests/test_extractor.rs | 51 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/tests/test_extractor.rs b/tests/test_extractor.rs index b02667d..4d872d6 100644 --- a/tests/test_extractor.rs +++ b/tests/test_extractor.rs @@ -295,3 +295,54 @@ fn extractor_finds_robots_txt_links_and_displays_files_or_scans_directories() { assert_eq!(mock_scanned_file.hits(), 1); teardown_tmp_directory(tmp_dir); } + +#[test] +/// send a request to a page that contains a link that contains a directory that returns a 403 +/// --extract-links should find the link and make recurse into the 403 directory, finding LICENSE +fn extractor_recurses_into_403_directories() -> Result<(), Box> { + let srv = MockServer::start(); + let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?; + + let mock = srv.mock(|when, then| { + when.method(GET).path("/LICENSE"); + then.status(200) + .body(&srv.url("'/homepage/assets/img/icons/handshake.svg'")); + }); + + let mock_two = srv.mock(|when, then| { + when.method(GET).path("/homepage/assets/img/icons/LICENSE"); + then.status(200).body("that's just like, your opinion man"); + }); + + let forbidden_dir = srv.mock(|when, then| { + when.method(GET).path("/homepage/assets/img/icons/"); + then.status(403); + }); + + let cmd = Command::cargo_bin("feroxbuster") + .unwrap() + .arg("--url") + .arg(srv.url("/")) + .arg("--wordlist") + .arg(file.as_os_str()) + .arg("--extract-links") + .arg("--depth") // need to go past default 4 directories + .arg("0") + .unwrap(); + + cmd.assert().success().stdout( + predicate::str::contains("/LICENSE") + .count(2) + .and(predicate::str::contains("1w")) // link in /LICENSE + .and(predicate::str::contains("34c")) // recursed LICENSE + .and(predicate::str::contains( + "/homepage/assets/img/icons/LICENSE", + )), + ); + + assert_eq!(mock.hits(), 1); + assert_eq!(mock_two.hits(), 1); + assert_eq!(forbidden_dir.hits(), 1); + teardown_tmp_directory(tmp_dir); + Ok(()) +} From dfc0c2ba7f647b5dc6e103a4caa8e9d09d2a554a Mon Sep 17 00:00:00 2001 From: epi Date: Tue, 12 Jan 2021 20:06:42 -0600 Subject: [PATCH 04/13] added another test for 403 recursion --- tests/test_scanner.rs | 50 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/tests/test_scanner.rs b/tests/test_scanner.rs index 6fd7723..6dc3c55 100644 --- a/tests/test_scanner.rs +++ b/tests/test_scanner.rs @@ -546,3 +546,53 @@ fn scanner_single_request_scan_with_regex_filtered_result() { assert_eq!(filtered_mock.hits(), 1); teardown_tmp_directory(tmp_dir); } + +#[test] +/// send a request to a 403 directory, expect recursion to work into the 403 +fn scanner_recursion_works_with_403_directories() { + let srv = MockServer::start(); + let (tmp_dir, file) = + setup_tmp_directory(&["LICENSE".to_string(), "ignored/".to_string()], "wordlist").unwrap(); + + let mock = srv.mock(|when, then| { + when.method(GET).path("/LICENSE"); + then.status(200).body("this is a test"); + }); + + let forbidden_dir = srv.mock(|when, then| { + when.method(GET).path("/ignored/"); + then.status(403); + }); + + let found_anyway = srv.mock(|when, then| { + when.method(GET).path("/ignored/LICENSE"); + then.status(200) + .body("this is a test\nThat rug really tied the room together"); + }); + + let cmd = Command::cargo_bin("feroxbuster") + .unwrap() + .arg("--url") + .arg(srv.url("/")) + .arg("--wordlist") + .arg(file.as_os_str()) + .unwrap(); + + cmd.assert().success().stdout( + predicate::str::contains("/LICENSE") + .count(2) + .and(predicate::str::contains("200").count(2)) + .and(predicate::str::contains("403")) + .and(predicate::str::contains("53c")) + .and(predicate::str::contains("14c")) + .and(predicate::str::contains("0c")) + .and(predicate::str::contains("ignored").count(2)) + .and(predicate::str::contains("/ignored/LICENSE")), + ); + + assert_eq!(mock.hits(), 1); + assert_eq!(found_anyway.hits(), 1); + assert_eq!(forbidden_dir.hits(), 1); + + teardown_tmp_directory(tmp_dir); +} From 5054f6673e0f2dfe04a880121a84933d49cd5a6b Mon Sep 17 00:00:00 2001 From: epi Date: Tue, 12 Jan 2021 20:08:56 -0600 Subject: [PATCH 05/13] bumped version to 1.12.1 --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 8d1ec61..b7ef492 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "feroxbuster" -version = "1.12.0" +version = "1.12.1" authors = ["Ben 'epi' Risher "] license = "MIT" edition = "2018" From 3bda77b21b28b3189a0101ed2a971db5338c5625 Mon Sep 17 00:00:00 2001 From: epi Date: Tue, 12 Jan 2021 20:17:27 -0600 Subject: [PATCH 06/13] fixed regression with stats bar --- src/main.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main.rs b/src/main.rs index 2d5c73a..e46d44d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -409,8 +409,6 @@ async fn clean_up( stats_handle, save_output ); - update_stat!(tx_stats, StatCommand::Exit); // send exit command and await the end of the future - drop(tx_term); log::trace!("dropped terminal output handler's transmitter"); @@ -442,6 +440,7 @@ async fn clean_up( log::trace!("done awaiting file output handler's receiver"); } + update_stat!(tx_stats, StatCommand::Exit); // send exit command and await the end of the future stats_handle.await.unwrap_or_default(); // mark all scans complete so the terminal input handler will exit cleanly From 6b05fba068e4f9d5d5829e41ada873919cf5cdef Mon Sep 17 00:00:00 2001 From: epi Date: Wed, 13 Jan 2021 07:27:37 -0600 Subject: [PATCH 07/13] restructured statistics; created event_handlers module --- src/event_handlers/mod.rs | 2 + src/event_handlers/statistics.rs | 112 +++++++++ src/lib.rs | 1 + src/statistics/command.rs | 33 +++ src/{statistics.rs => statistics/data.rs} | 288 +--------------------- src/statistics/error.rs | 24 ++ src/statistics/field.rs | 33 +++ src/statistics/init.rs | 29 +++ src/statistics/macros.rs | 23 ++ src/statistics/mod.rs | 17 ++ src/statistics/tests.rs | 52 ++++ 11 files changed, 338 insertions(+), 276 deletions(-) create mode 100644 src/event_handlers/mod.rs create mode 100644 src/event_handlers/statistics.rs create mode 100644 src/statistics/command.rs rename src/{statistics.rs => statistics/data.rs} (71%) create mode 100644 src/statistics/error.rs create mode 100644 src/statistics/field.rs create mode 100644 src/statistics/init.rs create mode 100644 src/statistics/macros.rs create mode 100644 src/statistics/mod.rs create mode 100644 src/statistics/tests.rs diff --git a/src/event_handlers/mod.rs b/src/event_handlers/mod.rs new file mode 100644 index 0000000..867d818 --- /dev/null +++ b/src/event_handlers/mod.rs @@ -0,0 +1,2 @@ +mod statistics; +pub use statistics::StatsHandler; diff --git a/src/event_handlers/statistics.rs b/src/event_handlers/statistics.rs new file mode 100644 index 0000000..34ed1ca --- /dev/null +++ b/src/event_handlers/statistics.rs @@ -0,0 +1,112 @@ +use crate::{ + atomic_load, + config::CONFIGURATION, + progress::{add_bar, BarType}, + statistics::{StatCommand, StatField, Stats}, +}; +use console::style; +use indicatif::ProgressBar; +use std::{ + sync::{atomic::Ordering, Arc}, + time::Instant, +}; +use tokio::sync::mpsc::UnboundedReceiver; + +/// event handler struct for updating statistics +#[derive(Debug)] +pub struct StatsHandler { + /// overall scan's progress bar + bar: ProgressBar, + + /// Receiver half of mpsc from which `StatCommand`s are processed + receiver: UnboundedReceiver, + + /// data class that stores all statistics updates + stats: Arc, +} + +/// implementation of event handler for statistics +impl StatsHandler { + /// create new event handler builder + pub fn new(stats: Arc, rx_stats: UnboundedReceiver) -> Self { + // will be updated later via StatCommand; delay is for banner to print first + let bar = ProgressBar::hidden(); + + Self { + bar, + stats, + receiver: rx_stats, + } + } + + /// Start a single consumer task (sc side of mpsc) + /// + /// The consumer simply receives `StatCommands` and updates the given `Stats` object as appropriate + pub async fn start(&mut self) { + log::trace!("enter: statistics::start({:?})", self); + + let start = Instant::now(); + + while let Some(command) = self.receiver.recv().await { + match command as StatCommand { + StatCommand::AddError(err) => { + self.stats.add_error(err); + self.increment_bar(); + } + StatCommand::AddStatus(status) => { + self.stats.add_status_code(status); + self.increment_bar(); + } + StatCommand::AddRequest => { + self.stats.add_request(); + self.increment_bar(); + } + StatCommand::Save => self + .stats + .save(start.elapsed().as_secs_f64(), &CONFIGURATION.output), + StatCommand::UpdateUsizeField(field, value) => { + let update_len = matches!(field, StatField::TotalScans); + self.stats.update_usize_field(field, value); + + if update_len { + self.bar + .set_length(atomic_load!(self.stats.total_expected) as u64) + } + } + StatCommand::UpdateF64Field(field, value) => { + self.stats.update_f64_field(field, value) + } + StatCommand::CreateBar => { + self.bar = add_bar( + "", + atomic_load!(self.stats.total_expected) as u64, + BarType::Total, + ); + } + StatCommand::LoadStats(filename) => { + self.stats.merge_from(&filename); + } + StatCommand::Exit => break, + } + } + + self.bar.finish(); + + log::debug!("{:#?}", *self.stats); + log::trace!("exit: statistics::start") + } + + /// Wrapper around incrementing the overall scan's progress bar + fn increment_bar(&self) { + let msg = format!( + "{}:{:<7} {}:{:<7}", + style("found").green(), + atomic_load!(self.stats.resources_discovered), + style("errors").red(), + atomic_load!(self.stats.errors), + ); + + self.bar.set_message(&msg); + self.bar.inc(1); + } +} diff --git a/src/lib.rs b/src/lib.rs index 6e083d4..de41da5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -12,6 +12,7 @@ pub mod reporter; pub mod scan_manager; pub mod scanner; pub mod statistics; +mod event_handlers; use crate::utils::{get_url_path_length, status_colorizer}; use console::{style, Color}; diff --git a/src/statistics/command.rs b/src/statistics/command.rs new file mode 100644 index 0000000..0d6ac1d --- /dev/null +++ b/src/statistics/command.rs @@ -0,0 +1,33 @@ +use super::{error::StatError, field::StatField}; +use reqwest::StatusCode; + +/// Protocol definition for updating a Stats object via mpsc +#[derive(Debug)] +pub enum StatCommand { + /// Add one to the total number of requests + AddRequest, + + /// Add one to the proper field(s) based on the given `StatError` + AddError(StatError), + + /// Add one to the proper field(s) based on the given `StatusCode` + AddStatus(StatusCode), + + /// Create the progress bar (`BarType::Total`) that is updated from the stats thread + CreateBar, + + /// Update a `Stats` field that corresponds to the given `StatField` by the given `usize` value + UpdateUsizeField(StatField, usize), + + /// Update a `Stats` field that corresponds to the given `StatField` by the given `f64` value + UpdateF64Field(StatField, f64), + + /// Save a `Stats` object to disk using `reporter::get_cached_file_handle` + Save, + + /// Load a `Stats` object from disk + LoadStats(String), + + /// Break out of the (infinite) mpsc receive loop + Exit, +} diff --git a/src/statistics.rs b/src/statistics/data.rs similarity index 71% rename from src/statistics.rs rename to src/statistics/data.rs index d142007..401cc32 100644 --- a/src/statistics.rs +++ b/src/statistics/data.rs @@ -1,11 +1,9 @@ +use super::{error::StatError, field::StatField}; use crate::{ config::CONFIGURATION, - progress::{add_bar, BarType}, reporter::{get_cached_file_handle, safe_file_write}, - FeroxChannel, FeroxSerialize, + FeroxSerialize, }; -use console::style; -use indicatif::ProgressBar; use reqwest::StatusCode; use serde::{Deserialize, Serialize}; use std::{ @@ -13,34 +11,9 @@ use std::{ io::BufReader, sync::{ atomic::{AtomicUsize, Ordering}, - Arc, Mutex, + Mutex, }, - time::Instant, }; -use tokio::{ - sync::mpsc::{self, UnboundedReceiver, UnboundedSender}, - task::JoinHandle, -}; - -/// Wrapper `Atomic*.fetch_add` to save me from writing Ordering::Relaxed a bajillion times -/// -/// default is to increment by 1, second arg can be used to increment by a different value -macro_rules! atomic_increment { - ($metric:expr) => { - $metric.fetch_add(1, Ordering::Relaxed); - }; - - ($metric:expr, $value:expr) => { - $metric.fetch_add($value, Ordering::Relaxed); - }; -} - -/// Wrapper around `Atomic*.load` to save me from writing Ordering::Relaxed a bajillion times -macro_rules! atomic_load { - ($metric:expr) => { - $metric.load(Ordering::Relaxed); - }; -} /// Data collection of statistics related to a scan #[derive(Default, Deserialize, Debug, Serialize)] @@ -63,10 +36,10 @@ pub struct Stats { /// tracker for accumulating total number of requests expected (i.e. as a new scan is started /// this value should increase by `expected_requests` - total_expected: AtomicUsize, + pub total_expected: AtomicUsize, /// tracker for total number of errors encountered by the client - errors: AtomicUsize, + pub errors: AtomicUsize, /// tracker for overall number of 2xx status codes seen by the client successes: AtomicUsize, @@ -128,7 +101,7 @@ pub struct Stats { responses_filtered: AtomicUsize, /// tracker for number of files found - resources_discovered: AtomicUsize, + pub resources_discovered: AtomicUsize, /// tracker for number of errors triggered during URL formatting url_format_errors: AtomicUsize, @@ -176,7 +149,7 @@ impl Stats { } /// increment `requests` field by one - fn add_request(&self) { + pub fn add_request(&self) { atomic_increment!(self.requests); } @@ -188,7 +161,7 @@ impl Stats { } /// save an instance of `Stats` to disk after updating the total runtime for the scan - fn save(&self, seconds: f64, location: &str) { + pub fn save(&self, seconds: f64, location: &str) { let buffered_file = match get_cached_file_handle(location) { Some(file) => file, None => { @@ -242,7 +215,7 @@ impl Stats { /// - requests /// - status_403s (when code is 403) /// - errors (when code is [45]xx) - fn add_status_code(&self, status: StatusCode) { + pub fn add_status_code(&self, status: StatusCode) { self.add_request(); if status.is_success() { @@ -291,7 +264,7 @@ impl Stats { } /// Update a `Stats` field of type f64 - fn update_f64_field(&self, field: StatField, value: f64) { + pub fn update_f64_field(&self, field: StatField, value: f64) { if let StatField::DirScanTimes = field { if let Ok(mut locked_times) = self.directory_scan_times.lock() { locked_times.push(value); @@ -300,7 +273,7 @@ impl Stats { } /// Update a `Stats` field of type usize - fn update_usize_field(&self, field: StatField, value: usize) { + pub fn update_usize_field(&self, field: StatField, value: usize) { match field { StatField::ExpectedPerScan => { atomic_increment!(self.expected_per_scan, value); @@ -402,228 +375,13 @@ impl Stats { } } -#[derive(Debug)] -/// Enum variants used to inform the `StatCommand` protocol what `Stats` fields should be updated -pub enum StatError { - /// Represents a 403 response code - Status403, - - /// Represents a timeout error - Timeout, - - /// Represents a URL formatting error - UrlFormat, - - /// Represents an error encountered during redirection - Redirection, - - /// Represents an error encountered during connection - Connection, - - /// Represents an error resulting from the client's request - Request, - - /// Represents any other error not explicitly defined above - Other, -} - -/// Protocol definition for updating a Stats object via mpsc -#[derive(Debug)] -pub enum StatCommand { - /// Add one to the total number of requests - AddRequest, - - /// Add one to the proper field(s) based on the given `StatError` - AddError(StatError), - - /// Add one to the proper field(s) based on the given `StatusCode` - AddStatus(StatusCode), - - /// Create the progress bar (`BarType::Total`) that is updated from the stats thread - CreateBar, - - /// Update a `Stats` field that corresponds to the given `StatField` by the given `usize` value - UpdateUsizeField(StatField, usize), - - /// Update a `Stats` field that corresponds to the given `StatField` by the given `f64` value - UpdateF64Field(StatField, f64), - - /// Save a `Stats` object to disk using `reporter::get_cached_file_handle` - Save, - - /// Load a `Stats` object from disk - LoadStats(String), - - /// Break out of the (infinite) mpsc receive loop - Exit, -} - -/// Enum representing fields whose updates need to be performed in batches instead of one at -/// a time -#[derive(Debug)] -pub enum StatField { - /// Due to the necessary order of events, the number of requests expected to be sent isn't - /// known until after `statistics::initialize` is called. This command allows for updating - /// the `expected_per_scan` field after initialization - ExpectedPerScan, - - /// Translates to `total_scans` - TotalScans, - - /// Translates to `links_extracted` - LinksExtracted, - - /// Translates to `total_expected` - TotalExpected, - - /// Translates to `wildcards_filtered` - WildcardsFiltered, - - /// Translates to `responses_filtered` - ResponsesFiltered, - - /// Translates to `resources_discovered` - ResourcesDiscovered, - - /// Translates to `initial_targets` - InitialTargets, - - /// Translates to `directory_scan_times`; assumes a single append to the vector - DirScanTimes, -} - -/// Spawn a single consumer task (sc side of mpsc) -/// -/// The consumer simply receives `StatCommands` and updates the given `Stats` object as appropriate -pub async fn spawn_statistics_handler( - mut rx_stats: UnboundedReceiver, - stats: Arc, - tx_stats: UnboundedSender, -) { - log::trace!( - "enter: spawn_statistics_handler({:?}, {:?}, {:?})", - rx_stats, - stats, - tx_stats - ); - - // will be updated later via StatCommand; delay is for banner to print first - let mut bar = ProgressBar::hidden(); - - let start = Instant::now(); - - while let Some(command) = rx_stats.recv().await { - match command as StatCommand { - StatCommand::AddError(err) => { - stats.add_error(err); - increment_bar(&bar, stats.clone()); - } - StatCommand::AddStatus(status) => { - stats.add_status_code(status); - increment_bar(&bar, stats.clone()); - } - StatCommand::AddRequest => { - stats.add_request(); - increment_bar(&bar, stats.clone()); - } - StatCommand::Save => stats.save(start.elapsed().as_secs_f64(), &CONFIGURATION.output), - StatCommand::UpdateUsizeField(field, value) => { - let update_len = matches!(field, StatField::TotalScans); - stats.update_usize_field(field, value); - - if update_len { - bar.set_length(atomic_load!(stats.total_expected) as u64) - } - } - StatCommand::UpdateF64Field(field, value) => stats.update_f64_field(field, value), - StatCommand::CreateBar => { - bar = add_bar( - "", - atomic_load!(stats.total_expected) as u64, - BarType::Total, - ); - } - StatCommand::LoadStats(filename) => { - stats.merge_from(&filename); - } - StatCommand::Exit => break, - } - } - - bar.finish(); - - log::debug!("{:#?}", *stats); - log::trace!("exit: spawn_statistics_handler") -} - -/// Wrapper around incrementing the overall scan's progress bar -fn increment_bar(bar: &ProgressBar, stats: Arc) { - let msg = format!( - "{}:{:<7} {}:{:<7}", - style("found").green(), - atomic_load!(stats.resources_discovered), - style("errors").red(), - atomic_load!(stats.errors), - ); - - bar.set_message(&msg); - bar.inc(1); -} - -/// Initialize new `Stats` object and the sc side of an mpsc channel that is responsible for -/// updates to the aforementioned object. -pub fn initialize() -> (Arc, UnboundedSender, JoinHandle<()>) { - log::trace!("enter: initialize"); - - let stats_tracker = Arc::new(Stats::new()); - let stats_cloned = stats_tracker.clone(); - let (tx_stats, rx_stats): FeroxChannel = mpsc::unbounded_channel(); - let tx_stats_cloned = tx_stats.clone(); - let stats_thread = tokio::spawn(async move { - spawn_statistics_handler(rx_stats, stats_cloned, tx_stats_cloned).await - }); - - log::trace!( - "exit: initialize -> ({:?}, {:?}, {:?})", - stats_tracker, - tx_stats, - stats_thread - ); - - (stats_tracker, tx_stats, stats_thread) -} - #[cfg(test)] mod tests { + use super::super::*; use super::*; use std::fs::write; use tempfile::NamedTempFile; - /// simple helper to reduce code reuse - fn setup_stats_test() -> (Arc, UnboundedSender, JoinHandle<()>) { - initialize() - } - - /// another helper to stay DRY; must be called after any sent commands and before any checks - /// performed against the Stats object - async fn teardown_stats_test(sender: UnboundedSender, handle: JoinHandle<()>) { - // send exit and await, once the await completes, stats should be updated - sender.send(StatCommand::Exit).unwrap_or_default(); - handle.await.unwrap(); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 1)] - /// when sent StatCommand::Exit, function should exit its while loop (runs forever otherwise) - async fn statistics_handler_exits() { - let (_, sender, handle) = setup_stats_test(); - - sender.send(StatCommand::Exit).unwrap_or_default(); - - handle.await.unwrap(); // blocks on the handler's while loop - - // if we've made it here, the test has succeeded - } - #[tokio::test(flavor = "multi_thread", worker_threads = 1)] /// when sent StatCommand::AddRequest, stats object should reflect the change async fn statistics_handler_increments_requests() { @@ -808,26 +566,4 @@ mod tests { stats.update_runtime(20.2); assert!((stats.total_runtime.lock().unwrap()[0] - 20.2).abs() < f64::EPSILON); } - - #[test] - /// Stats::save should write contents of Stats to disk - fn save_writes_stats_object_to_disk() { - let stats = Stats::new(); - stats.add_request(); - stats.add_request(); - stats.add_request(); - stats.add_request(); - stats.add_error(StatError::Timeout); - stats.add_error(StatError::Timeout); - stats.add_error(StatError::Timeout); - stats.add_error(StatError::Timeout); - stats.add_status_code(StatusCode::OK); - stats.add_status_code(StatusCode::OK); - stats.add_status_code(StatusCode::OK); - let outfile = "/tmp/stuff"; - stats.save(174.33, outfile); - assert!(stats.as_json().contains("statistics")); - assert!(stats.as_json().contains("11")); // requests made - assert!(stats.as_str().is_empty()); - } } diff --git a/src/statistics/error.rs b/src/statistics/error.rs new file mode 100644 index 0000000..048d728 --- /dev/null +++ b/src/statistics/error.rs @@ -0,0 +1,24 @@ +#[derive(Debug)] +/// Enum variants used to inform the `StatCommand` protocol what `Stats` fields should be updated +pub enum StatError { + /// Represents a 403 response code + Status403, + + /// Represents a timeout error + Timeout, + + /// Represents a URL formatting error + UrlFormat, + + /// Represents an error encountered during redirection + Redirection, + + /// Represents an error encountered during connection + Connection, + + /// Represents an error resulting from the client's request + Request, + + /// Represents any other error not explicitly defined above + Other, +} diff --git a/src/statistics/field.rs b/src/statistics/field.rs new file mode 100644 index 0000000..c2774eb --- /dev/null +++ b/src/statistics/field.rs @@ -0,0 +1,33 @@ +/// Enum representing fields whose updates need to be performed in batches instead of one at +/// a time +#[derive(Debug)] +pub enum StatField { + /// Due to the necessary order of events, the number of requests expected to be sent isn't + /// known until after `statistics::initialize` is called. This command allows for updating + /// the `expected_per_scan` field after initialization + ExpectedPerScan, + + /// Translates to `total_scans` + TotalScans, + + /// Translates to `links_extracted` + LinksExtracted, + + /// Translates to `total_expected` + TotalExpected, + + /// Translates to `wildcards_filtered` + WildcardsFiltered, + + /// Translates to `responses_filtered` + ResponsesFiltered, + + /// Translates to `resources_discovered` + ResourcesDiscovered, + + /// Translates to `initial_targets` + InitialTargets, + + /// Translates to `directory_scan_times`; assumes a single append to the vector + DirScanTimes, +} diff --git a/src/statistics/init.rs b/src/statistics/init.rs new file mode 100644 index 0000000..949db37 --- /dev/null +++ b/src/statistics/init.rs @@ -0,0 +1,29 @@ +use super::{command::StatCommand, data::Stats}; +use crate::{event_handlers::StatsHandler, FeroxChannel}; +use std::sync::Arc; +use tokio::{ + sync::mpsc::{self, UnboundedSender}, + task::JoinHandle, +}; + +/// Initialize new `Stats` object and the sc side of an mpsc channel that is responsible for +/// updates to the aforementioned object. +pub fn initialize() -> (Arc, UnboundedSender, JoinHandle<()>) { + log::trace!("enter: initialize"); + + let stats_tracker = Arc::new(Stats::new()); + let (tx_stats, rx_stats): FeroxChannel = mpsc::unbounded_channel(); + + let mut handler = StatsHandler::new(stats_tracker.clone(), rx_stats); + + let stats_thread = tokio::spawn(async move { handler.start().await }); + + log::trace!( + "exit: initialize -> ({:?}, {:?}, {:?})", + stats_tracker, + tx_stats, + stats_thread + ); + + (stats_tracker, tx_stats, stats_thread) +} diff --git a/src/statistics/macros.rs b/src/statistics/macros.rs new file mode 100644 index 0000000..9125806 --- /dev/null +++ b/src/statistics/macros.rs @@ -0,0 +1,23 @@ +#![macro_use] + +/// Wrapper `Atomic*.fetch_add` to save me from writing Ordering::Relaxed a bajillion times +/// +/// default is to increment by 1, second arg can be used to increment by a different value +#[macro_export] +macro_rules! atomic_increment { + ($metric:expr) => { + $metric.fetch_add(1, Ordering::Relaxed); + }; + + ($metric:expr, $value:expr) => { + $metric.fetch_add($value, Ordering::Relaxed); + }; +} + +/// Wrapper around `Atomic*.load` to save me from writing Ordering::Relaxed a bajillion times +#[macro_export] +macro_rules! atomic_load { + ($metric:expr) => { + $metric.load(Ordering::Relaxed); + }; +} diff --git a/src/statistics/mod.rs b/src/statistics/mod.rs new file mode 100644 index 0000000..d6edafc --- /dev/null +++ b/src/statistics/mod.rs @@ -0,0 +1,17 @@ +mod error; +mod macros; +mod data; +mod command; +mod field; +mod init; +#[cfg(test)] +mod tests; + +pub use self::command::StatCommand; +pub use self::data::Stats; +pub use self::error::StatError; +pub use self::field::StatField; +pub use self::init::initialize; + +#[cfg(test)] +use self::tests::{setup_stats_test, teardown_stats_test}; diff --git a/src/statistics/tests.rs b/src/statistics/tests.rs new file mode 100644 index 0000000..002799b --- /dev/null +++ b/src/statistics/tests.rs @@ -0,0 +1,52 @@ +use super::*; +use crate::FeroxSerialize; +use reqwest::StatusCode; +use std::sync::Arc; +use tokio::{sync::mpsc::UnboundedSender, task::JoinHandle}; + +/// simple helper to reduce code reuse +pub fn setup_stats_test() -> (Arc, UnboundedSender, JoinHandle<()>) { + initialize() +} + +/// another helper to stay DRY; must be called after any sent commands and before any checks +/// performed against the Stats object +pub async fn teardown_stats_test(sender: UnboundedSender, handle: JoinHandle<()>) { + // send exit and await, once the await completes, stats should be updated + sender.send(StatCommand::Exit).unwrap_or_default(); + handle.await.unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +/// when sent StatCommand::Exit, function should exit its while loop (runs forever otherwise) +async fn statistics_handler_exits() { + let (_, sender, handle) = setup_stats_test(); + + sender.send(StatCommand::Exit).unwrap_or_default(); + + handle.await.unwrap(); // blocks on the handler's while loop + + // if we've made it here, the test has succeeded +} + +#[test] +/// Stats::save should write contents of Stats to disk +fn save_writes_stats_object_to_disk() { + let stats = Stats::new(); + stats.add_request(); + stats.add_request(); + stats.add_request(); + stats.add_request(); + stats.add_error(StatError::Timeout); + stats.add_error(StatError::Timeout); + stats.add_error(StatError::Timeout); + stats.add_error(StatError::Timeout); + stats.add_status_code(StatusCode::OK); + stats.add_status_code(StatusCode::OK); + stats.add_status_code(StatusCode::OK); + let outfile = "/tmp/stuff"; + stats.save(174.33, outfile); + assert!(stats.as_json().contains("statistics")); + assert!(stats.as_json().contains("11")); // requests made + assert!(stats.as_str().is_empty()); +} From 6832cbcdd8f480dade1d357950f385bf2d341c0b Mon Sep 17 00:00:00 2001 From: epi Date: Wed, 13 Jan 2021 15:52:24 -0600 Subject: [PATCH 08/13] nitpickery and started incorporating anyhow to error handling --- Cargo.toml | 1 + src/config.rs | 17 ++++--- src/event_handlers/statistics.rs | 34 ++++++------- src/lib.rs | 61 ++++++++++++------------ src/main.rs | 53 ++++++++++---------- src/reporter.rs | 2 +- src/scan_manager.rs | 11 +++-- src/scanner.rs | 2 +- src/statistics/{data.rs => container.rs} | 54 +++++++++++++++------ src/statistics/init.rs | 9 +++- src/statistics/mod.rs | 4 +- src/statistics/tests.rs | 29 +++++++---- src/utils.rs | 28 +++++------ tests/test_banner.rs | 4 +- 14 files changed, 178 insertions(+), 131 deletions(-) rename src/statistics/{data.rs => container.rs} (94%) diff --git a/Cargo.toml b/Cargo.toml index 3c86a07..4d24c6a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,6 +42,7 @@ crossterm = "0.19" rlimit = "0.5" ctrlc = "3.1" fuzzyhash = "0.2" +anyhow = "1.0" [dev-dependencies] tempfile = "3.1" diff --git a/src/config.rs b/src/config.rs index ab08318..0042b20 100644 --- a/src/config.rs +++ b/src/config.rs @@ -2,9 +2,10 @@ use crate::{ client, parser, progress::{add_bar, BarType}, scan_manager::resume_scan, - utils::{module_colorizer, status_colorizer}, + utils::{fmt_err, module_colorizer, status_colorizer}, FeroxSerialize, DEFAULT_CONFIG_NAME, DEFAULT_STATUS_CODES, DEFAULT_WORDLIST, VERSION, }; +use anyhow::{Context, Result}; use clap::{value_t, ArgMatches}; use indicatif::{MultiProgress, ProgressBar, ProgressDrawTarget}; use lazy_static::lazy_static; @@ -875,13 +876,11 @@ impl FeroxSerialize for Configuration { /// ], /// ... /// }\n - fn as_json(&self) -> String { - if let Ok(mut json) = serde_json::to_string(&self) { - json.push('\n'); - json - } else { - String::from("{\"error\":\"could not Configuration convert to json\"}") - } + fn as_json(&self) -> Result { + let mut json = serde_json::to_string(&self) + .with_context(|| fmt_err("Could not convert Configuration to JSON"))?; + json.push('\n'); + Ok(json) } } @@ -1238,7 +1237,7 @@ mod tests { let mut config = Configuration::new(); config.timeout = 12; config.depth = 2; - let config_str = config.as_json(); + let config_str = config.as_json().unwrap(); let json: Configuration = serde_json::from_str(&config_str).unwrap(); assert_eq!(json.config, config.config); assert_eq!(json.wordlist, config.wordlist); diff --git a/src/event_handlers/statistics.rs b/src/event_handlers/statistics.rs index 34ed1ca..300ef16 100644 --- a/src/event_handlers/statistics.rs +++ b/src/event_handlers/statistics.rs @@ -1,15 +1,12 @@ use crate::{ - atomic_load, config::CONFIGURATION, progress::{add_bar, BarType}, statistics::{StatCommand, StatField, Stats}, }; +use anyhow::Result; use console::style; use indicatif::ProgressBar; -use std::{ - sync::{atomic::Ordering, Arc}, - time::Instant, -}; +use std::{sync::Arc, time::Instant}; use tokio::sync::mpsc::UnboundedReceiver; /// event handler struct for updating statistics @@ -42,8 +39,8 @@ impl StatsHandler { /// Start a single consumer task (sc side of mpsc) /// /// The consumer simply receives `StatCommands` and updates the given `Stats` object as appropriate - pub async fn start(&mut self) { - log::trace!("enter: statistics::start({:?})", self); + pub async fn start(&mut self) -> Result<()> { + log::trace!("enter: start({:?})", self); let start = Instant::now(); @@ -61,27 +58,23 @@ impl StatsHandler { self.stats.add_request(); self.increment_bar(); } - StatCommand::Save => self - .stats - .save(start.elapsed().as_secs_f64(), &CONFIGURATION.output), + StatCommand::Save => { + self.stats + .save(start.elapsed().as_secs_f64(), &CONFIGURATION.output)?; + } StatCommand::UpdateUsizeField(field, value) => { let update_len = matches!(field, StatField::TotalScans); self.stats.update_usize_field(field, value); if update_len { - self.bar - .set_length(atomic_load!(self.stats.total_expected) as u64) + self.bar.set_length(self.stats.total_expected() as u64) } } StatCommand::UpdateF64Field(field, value) => { self.stats.update_f64_field(field, value) } StatCommand::CreateBar => { - self.bar = add_bar( - "", - atomic_load!(self.stats.total_expected) as u64, - BarType::Total, - ); + self.bar = add_bar("", self.stats.total_expected() as u64, BarType::Total); } StatCommand::LoadStats(filename) => { self.stats.merge_from(&filename); @@ -93,7 +86,8 @@ impl StatsHandler { self.bar.finish(); log::debug!("{:#?}", *self.stats); - log::trace!("exit: statistics::start") + log::trace!("exit: start"); + Ok(()) } /// Wrapper around incrementing the overall scan's progress bar @@ -101,9 +95,9 @@ impl StatsHandler { let msg = format!( "{}:{:<7} {}:{:<7}", style("found").green(), - atomic_load!(self.stats.resources_discovered), + self.stats.resources_discovered(), style("errors").red(), - atomic_load!(self.stats.errors), + self.stats.errors(), ); self.bar.set_message(&msg); diff --git a/src/lib.rs b/src/lib.rs index de41da5..b14c5ab 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -14,7 +14,8 @@ pub mod scanner; pub mod statistics; mod event_handlers; -use crate::utils::{get_url_path_length, status_colorizer}; +use crate::utils::{fmt_err, get_url_path_length, status_colorizer}; +use anyhow::{Context, Result}; use console::{style, Color}; use reqwest::header::{HeaderName, HeaderValue}; use reqwest::{header::HeaderMap, Response, StatusCode, Url}; @@ -103,7 +104,7 @@ pub trait FeroxSerialize: Serialize { fn as_str(&self) -> String; /// Return an NDJSON representation of the object - fn as_json(&self) -> String; + fn as_json(&self) -> Result; } /// A `FeroxResponse`, derived from a `Response` to a submitted `Request` @@ -343,13 +344,11 @@ impl FeroxSerialize for FeroxResponse { /// "access-control-allow-origin":"https://localhost.com" /// } /// }\n - fn as_json(&self) -> String { - if let Ok(mut json) = serde_json::to_string(&self) { - json.push('\n'); - json - } else { - format!("{{\"error\":\"could not convert {} to json\"}}", self.url()) - } + fn as_json(&self) -> Result { + let mut json = serde_json::to_string(&self) + .with_context(|| fmt_err(&format!("Could not convert {} to JSON", self.url())))?; + json.push('\n'); + Ok(json) } } @@ -488,26 +487,6 @@ pub struct FeroxMessage { /// Implementation of FeroxMessage impl FeroxSerialize for FeroxMessage { - /// Create an NDJSON representation of the log message - /// - /// (expanded for clarity) - /// ex: - /// { - /// "type": "log", - /// "message": "Sent https://localhost/api to file handler", - /// "level": "DEBUG", - /// "time_offset": 0.86333454, - /// "module": "feroxbuster::reporter" - /// }\n - fn as_json(&self) -> String { - if let Ok(mut json) = serde_json::to_string(&self) { - json.push('\n'); - json - } else { - String::from("{\"error\":\"could not convert to json\"}") - } - } - /// Create a string representation of the log message /// /// ex: 301 10l 16w 173c https://localhost/api @@ -530,6 +509,28 @@ impl FeroxSerialize for FeroxMessage { style(&self.message).dim(), ) } + + /// Create an NDJSON representation of the log message + /// + /// (expanded for clarity) + /// ex: + /// { + /// "type": "log", + /// "message": "Sent https://localhost/api to file handler", + /// "level": "DEBUG", + /// "time_offset": 0.86333454, + /// "module": "feroxbuster::reporter" + /// }\n + fn as_json(&self) -> Result { + let mut json = serde_json::to_string(&self).with_context(|| { + fmt_err(&format!( + "Could not convert {}:{} to JSON", + self.level, self.message + )) + })?; + json.push('\n'); + Ok(json) + } } #[cfg(test)] @@ -587,7 +588,7 @@ mod tests { kind: "log".to_string(), }; - let message_str = message.as_json(); + let message_str = message.as_json().unwrap(); let error_margin = f32::EPSILON; diff --git a/src/main.rs b/src/main.rs index e46d44d..8dd8ba2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,3 +1,4 @@ +use anyhow::{bail, Context, Result}; use crossterm::event::{self, Event, KeyCode}; use feroxbuster::{ banner, @@ -14,7 +15,7 @@ use feroxbuster::{ Stats, }, update_stat, - utils::{ferox_print, get_current_depth, module_colorizer, status_colorizer}, + utils::{ferox_print, fmt_err, get_current_depth, status_colorizer}, FeroxError, FeroxResponse, FeroxResult, SLEEP_DURATION, VERSION, }; #[cfg(not(target_os = "windows"))] @@ -245,7 +246,7 @@ async fn get_targets() -> FeroxResult> { /// async main called from real main, broken out in this way to allow for some synchronous code /// to be executed before bringing the tokio runtime online -async fn wrapped_main() { +async fn wrapped_main() -> Result<()> { // join can only be called once, otherwise it causes the thread to panic tokio::task::spawn_blocking(move || { // ok, lazy_static! uses (unsurprisingly in retrospect) a lazy loading model where the @@ -295,7 +296,6 @@ async fn wrapped_main() { Ok(t) => t, Err(e) => { // should only happen in the event that there was an error reading from stdin - log::error!("{} {}", module_colorizer("main::get_targets"), e); clean_up( tx_term, term_handle, @@ -305,8 +305,8 @@ async fn wrapped_main() { stats_handle, save_output, ) - .await; - return; + .await?; + bail!("Could not get determine initial targets: {}", e); } }; @@ -338,8 +338,8 @@ async fn wrapped_main() { stats_handle, save_output, ) - .await; - return; + .await?; + bail!(fmt_err("Could not find any live targets to scan")); } // kick off a scan against any targets determined to be responsive @@ -356,6 +356,7 @@ async fn wrapped_main() { log::info!("All scans complete!"); } Err(e) => { + // todo status colorizer here and print is likely not needed ferox_print( &format!("{} while scanning: {}", status_colorizer("Error"), e), &PROGRESS_PRINTER, @@ -369,7 +370,8 @@ async fn wrapped_main() { stats_handle, save_output, ) - .await; + .await?; + // todo bail? process::exit(1); } }; @@ -383,9 +385,10 @@ async fn wrapped_main() { stats_handle, save_output, ) - .await; + .await?; log::trace!("exit: wrapped_main"); + Ok(()) } /// Single cleanup function that handles all the necessary drops/finishes etc required to gracefully @@ -396,9 +399,9 @@ async fn clean_up( tx_file: UnboundedSender, file_handle: Option>, tx_stats: UnboundedSender, - stats_handle: JoinHandle<()>, + stats_handle: JoinHandle>, save_output: bool, -) { +) -> Result<()> { log::trace!( "enter: clean_up({:?}, {:?}, {:?}, {:?}, {:?}, {:?}, {})", tx_term, @@ -414,12 +417,9 @@ async fn clean_up( log::trace!("awaiting terminal output handler's receiver"); // after dropping tx, we can await the future where rx lived - match term_handle.await { - Ok(_) => {} - Err(e) => { - log::error!("error awaiting terminal output handler's receiver: {}", e); - } - } + term_handle + .await + .with_context(|| fmt_err("Could not await terminal output handler's receiver"))?; log::trace!("done awaiting terminal output handler's receiver"); log::trace!("tx_file: {:?}", tx_file); @@ -431,17 +431,17 @@ async fn clean_up( if save_output { // but we only await if -o was specified log::trace!("awaiting file output handler's receiver"); - match file_handle.unwrap().await { - Ok(_) => {} - Err(e) => { - log::error!("error awaiting file output handler's receiver: {}", e); - } - } + file_handle + .unwrap() + .await + .with_context(|| fmt_err("Could not await file output handler's receiver"))?; log::trace!("done awaiting file output handler's receiver"); } update_stat!(tx_stats, StatCommand::Exit); // send exit command and await the end of the future - stats_handle.await.unwrap_or_default(); + stats_handle + .await? + .with_context(|| fmt_err("Could not await statistics handler's receiver"))?; // mark all scans complete so the terminal input handler will exit cleanly SCAN_COMPLETE.store(true, Ordering::Relaxed); @@ -453,6 +453,7 @@ async fn clean_up( drop(tx_stats); log::trace!("exit: clean_up"); + Ok(()) } fn main() { @@ -468,7 +469,9 @@ fn main() { .build() { let future = wrapped_main(); - runtime.block_on(future); + if let Err(e) = runtime.block_on(future) { + eprintln!("{}", e); + }; } log::trace!("exit: main"); diff --git a/src/reporter.rs b/src/reporter.rs index dfd29f6..85193cd 100644 --- a/src/reporter.rs +++ b/src/reporter.rs @@ -220,7 +220,7 @@ pub fn safe_file_write( // the second log entry being injected into the first. let contents = if convert_to_json { - value.as_json() + value.as_json().unwrap_or_default() // todo this fn should return result } else { value.as_str() }; diff --git a/src/scan_manager.rs b/src/scan_manager.rs index 5bbbfb8..6614e4e 100644 --- a/src/scan_manager.rs +++ b/src/scan_manager.rs @@ -1,3 +1,4 @@ +use crate::utils::fmt_err; use crate::{ config::{Configuration, CONFIGURATION, PROGRESS_BAR, PROGRESS_PRINTER}, parser::TIMESPEC_REGEX, @@ -8,6 +9,7 @@ use crate::{ utils::open_file, FeroxResponse, FeroxSerialize, SLEEP_DURATION, }; +use anyhow::{Context, Result}; use console::{measure_text_width, pad_str, style, Alignment, Term}; use indicatif::{ProgressBar, ProgressDrawTarget}; use serde::{ @@ -652,7 +654,7 @@ impl FeroxScans { scan_type: ScanType, stats: Arc, ) -> (bool, Arc>) { - let num_requests = stats.expected_per_scan.load(Ordering::Relaxed) as u64; + let num_requests = stats.expected_per_scan() as u64; let bar = match scan_type { ScanType::Directory => { @@ -782,8 +784,9 @@ impl FeroxSerialize for FeroxState { } /// Simple call to produce a JSON string using the given FeroxState - fn as_json(&self) -> String { - serde_json::to_string(&self).unwrap_or_default() + fn as_json(&self) -> Result { + Ok(serde_json::to_string(&self) + .with_context(|| fmt_err("Could not convert scan's running state to JSON"))?) } } @@ -1245,7 +1248,7 @@ mod tests { assert!(expected_strs.eval(&ferox_state.as_str())); - let json_state = ferox_state.as_json(); + let json_state = ferox_state.as_json().unwrap(); let expected = format!( r#"{{"scans":[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","status":"NotStarted","num_requests":0}}],"config":{{"type":"configuration","wordlist":"/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt","config":"","proxy":"","replay_proxy":"","target_url":"","status_codes":[200,204,301,302,307,308,401,403,405],"replay_codes":[200,204,301,302,307,308,401,403,405],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"quiet":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/{}","redirects":false,"insecure":false,"extensions":[],"headers":{{}},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":false,"depth":4,"scan_limit":0,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false,"resumed":false,"resume_from":"","save_state":false,"time_limit":"","filter_similar":[]}},"responses":[{{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{{"server":"nginx/1.16.1"}}}}]"#, saved_id, VERSION diff --git a/src/scanner.rs b/src/scanner.rs index 44e8dd2..5aede35 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -589,7 +589,7 @@ pub async fn scan_url( let (tx_dir, rx_dir): FeroxChannel = mpsc::unbounded_channel(); - if CALL_COUNT.load(Ordering::Relaxed) < stats.initial_targets.load(Ordering::Relaxed) { + if CALL_COUNT.load(Ordering::Relaxed) < stats.initial_targets() { CALL_COUNT.fetch_add(1, Ordering::Relaxed); if CONFIGURATION.extract_links { diff --git a/src/statistics/data.rs b/src/statistics/container.rs similarity index 94% rename from src/statistics/data.rs rename to src/statistics/container.rs index 401cc32..4140173 100644 --- a/src/statistics/data.rs +++ b/src/statistics/container.rs @@ -2,8 +2,10 @@ use super::{error::StatError, field::StatField}; use crate::{ config::CONFIGURATION, reporter::{get_cached_file_handle, safe_file_write}, + utils::status_colorizer, FeroxSerialize, }; +use anyhow::{Context, Result}; use reqwest::StatusCode; use serde::{Deserialize, Serialize}; use std::{ @@ -32,14 +34,14 @@ pub struct Stats { /// /// Note: this is a per-scan expectation; `expected_requests * current # of scans` would be /// indicative of the current expectation at any given time, but is a moving target. - pub expected_per_scan: AtomicUsize, + expected_per_scan: AtomicUsize, /// tracker for accumulating total number of requests expected (i.e. as a new scan is started /// this value should increase by `expected_requests` - pub total_expected: AtomicUsize, + total_expected: AtomicUsize, /// tracker for total number of errors encountered by the client - pub errors: AtomicUsize, + errors: AtomicUsize, /// tracker for overall number of 2xx status codes seen by the client successes: AtomicUsize, @@ -58,7 +60,7 @@ pub struct Stats { total_scans: AtomicUsize, /// tracker for initial number of requested targets - pub initial_targets: AtomicUsize, + initial_targets: AtomicUsize, /// tracker for number of links extracted when `--extract-links` is used; sources are /// response bodies and robots.txt as of v1.11.0 @@ -101,7 +103,7 @@ pub struct Stats { responses_filtered: AtomicUsize, /// tracker for number of files found - pub resources_discovered: AtomicUsize, + resources_discovered: AtomicUsize, /// tracker for number of errors triggered during URL formatting url_format_errors: AtomicUsize, @@ -131,8 +133,8 @@ impl FeroxSerialize for Stats { } /// Simple call to produce a JSON string using the given Stats object - fn as_json(&self) -> String { - serde_json::to_string(&self).unwrap_or_default() + fn as_json(&self) -> Result { + Ok(serde_json::to_string(&self)?) } } @@ -148,6 +150,31 @@ impl Stats { } } + /// public getter for expected_per_scan + pub fn expected_per_scan(&self) -> usize { + atomic_load!(self.expected_per_scan) + } + + /// public getter for resources_discovered + pub fn resources_discovered(&self) -> usize { + atomic_load!(self.resources_discovered) + } + + /// public getter for errors + pub fn errors(&self) -> usize { + atomic_load!(self.errors) + } + + /// public getter for total_expected + pub fn total_expected(&self) -> usize { + atomic_load!(self.total_expected) + } + + /// public getter for initial_targets + pub fn initial_targets(&self) -> usize { + atomic_load!(self.initial_targets) + } + /// increment `requests` field by one pub fn add_request(&self) { atomic_increment!(self.requests); @@ -161,17 +188,16 @@ impl Stats { } /// save an instance of `Stats` to disk after updating the total runtime for the scan - pub fn save(&self, seconds: f64, location: &str) { - let buffered_file = match get_cached_file_handle(location) { - Some(file) => file, - None => { - return; - } - }; + pub fn save(&self, seconds: f64, location: &str) -> Result<()> { + let buffered_file = get_cached_file_handle(location).with_context(|| { + format!("{}: Could not open {}", status_colorizer("ERROR"), location) + })?; self.update_runtime(seconds); safe_file_write(self, buffered_file, CONFIGURATION.json); + + Ok(()) } /// Inspect the given `StatError` and increment the appropriate fields diff --git a/src/statistics/init.rs b/src/statistics/init.rs index 949db37..dbb60e0 100644 --- a/src/statistics/init.rs +++ b/src/statistics/init.rs @@ -1,5 +1,6 @@ -use super::{command::StatCommand, data::Stats}; +use super::{command::StatCommand, container::Stats}; use crate::{event_handlers::StatsHandler, FeroxChannel}; +use anyhow::Result; use std::sync::Arc; use tokio::{ sync::mpsc::{self, UnboundedSender}, @@ -8,7 +9,11 @@ use tokio::{ /// Initialize new `Stats` object and the sc side of an mpsc channel that is responsible for /// updates to the aforementioned object. -pub fn initialize() -> (Arc, UnboundedSender, JoinHandle<()>) { +pub fn initialize() -> ( + Arc, + UnboundedSender, + JoinHandle>, +) { log::trace!("enter: initialize"); let stats_tracker = Arc::new(Stats::new()); diff --git a/src/statistics/mod.rs b/src/statistics/mod.rs index d6edafc..a54823c 100644 --- a/src/statistics/mod.rs +++ b/src/statistics/mod.rs @@ -1,6 +1,6 @@ mod error; mod macros; -mod data; +mod container; mod command; mod field; mod init; @@ -8,7 +8,7 @@ mod init; mod tests; pub use self::command::StatCommand; -pub use self::data::Stats; +pub use self::container::Stats; pub use self::error::StatError; pub use self::field::StatField; pub use self::init::initialize; diff --git a/src/statistics/tests.rs b/src/statistics/tests.rs index 002799b..419ae65 100644 --- a/src/statistics/tests.rs +++ b/src/statistics/tests.rs @@ -1,20 +1,29 @@ use super::*; use crate::FeroxSerialize; +use anyhow::Result; use reqwest::StatusCode; use std::sync::Arc; +use tempfile::NamedTempFile; use tokio::{sync::mpsc::UnboundedSender, task::JoinHandle}; /// simple helper to reduce code reuse -pub fn setup_stats_test() -> (Arc, UnboundedSender, JoinHandle<()>) { +pub fn setup_stats_test() -> ( + Arc, + UnboundedSender, + JoinHandle>, +) { initialize() } /// another helper to stay DRY; must be called after any sent commands and before any checks /// performed against the Stats object -pub async fn teardown_stats_test(sender: UnboundedSender, handle: JoinHandle<()>) { +pub async fn teardown_stats_test( + sender: UnboundedSender, + handle: JoinHandle>, +) { // send exit and await, once the await completes, stats should be updated sender.send(StatCommand::Exit).unwrap_or_default(); - handle.await.unwrap(); + handle.await.unwrap().unwrap(); } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] @@ -24,7 +33,7 @@ async fn statistics_handler_exits() { sender.send(StatCommand::Exit).unwrap_or_default(); - handle.await.unwrap(); // blocks on the handler's while loop + handle.await.unwrap().unwrap(); // blocks on the handler's while loop // if we've made it here, the test has succeeded } @@ -44,9 +53,13 @@ fn save_writes_stats_object_to_disk() { stats.add_status_code(StatusCode::OK); stats.add_status_code(StatusCode::OK); stats.add_status_code(StatusCode::OK); - let outfile = "/tmp/stuff"; - stats.save(174.33, outfile); - assert!(stats.as_json().contains("statistics")); - assert!(stats.as_json().contains("11")); // requests made + let outfile = NamedTempFile::new().unwrap(); + if stats + .save(174.33, &outfile.path().to_str().unwrap()) + .is_ok() + {} + + assert!(stats.as_json().unwrap().contains("statistics")); + assert!(stats.as_json().unwrap().contains("11")); // requests made assert!(stats.as_str().is_empty()); } diff --git a/src/utils.rs b/src/utils.rs index b66a032..a77c57a 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -7,6 +7,7 @@ use crate::{ }, FeroxError, FeroxResult, }; +use anyhow::Context; use console::{strip_ansi_codes, style, user_attended}; use indicatif::ProgressBar; use reqwest::{Client, Response, Url}; @@ -22,25 +23,19 @@ use tokio::sync::mpsc::UnboundedSender; pub fn open_file(filename: &str) -> Option>>> { log::trace!("enter: open_file({})", filename); - match fs::OpenOptions::new() // std fs + let file = fs::OpenOptions::new() // std fs .create(true) .append(true) .open(filename) - { - Ok(file) => { - let writer = io::BufWriter::new(file); // std io + .with_context(|| fmt_err(&format!("Could not open {}", filename))) + .ok()?; - let locked_file = Some(Arc::new(RwLock::new(writer))); + let writer = io::BufWriter::new(file); // std io - log::trace!("exit: open_file -> {:?}", locked_file); - locked_file - } - Err(e) => { - log::error!("{}", e); - log::trace!("exit: open_file -> None"); - None - } - } + let locked_file = Arc::new(RwLock::new(writer)); + + log::trace!("exit: open_file -> {:?}", locked_file); + Some(locked_file) } /// Helper function that determines the current depth of a given url @@ -105,6 +100,11 @@ pub fn status_colorizer(status: &str) -> String { } } +/// simple wrapper to stay DRY +pub fn fmt_err(msg: &str) -> String { + format!("{}: {}", status_colorizer("ERROR"), msg) +} + /// Takes in a string and colors it using console::style /// /// mainly putting this here in case i want to change the color later, making any changes easy diff --git a/tests/test_banner.rs b/tests/test_banner.rs index ef14a89..cc52028 100644 --- a/tests/test_banner.rs +++ b/tests/test_banner.rs @@ -615,7 +615,9 @@ fn banner_doesnt_print() -> Result<(), Box> { .arg("-q") .assert() .success() - .stderr(predicate::str::is_empty()); + .stderr(predicate::str::contains( + "Could not find any live targets to scan", + )); Ok(()) } From 4b1f1afabcc2ef2d9ef52b2da8b2cb7ae43cafd5 Mon Sep 17 00:00:00 2001 From: epi Date: Wed, 13 Jan 2021 15:58:39 -0600 Subject: [PATCH 09/13] more error handling for statistics --- src/statistics/container.rs | 109 ++++++++++++++++++------------------ 1 file changed, 55 insertions(+), 54 deletions(-) diff --git a/src/statistics/container.rs b/src/statistics/container.rs index 4140173..9fe4fd4 100644 --- a/src/statistics/container.rs +++ b/src/statistics/container.rs @@ -1,4 +1,5 @@ use super::{error::StatError, field::StatField}; +use crate::utils::fmt_err; use crate::{ config::CONFIGURATION, reporter::{get_cached_file_handle, safe_file_write}, @@ -339,65 +340,65 @@ impl Stats { /// Merge a given `Stats` object from a json entry written to disk when handling a Ctrl+c /// /// This is only ever called when resuming a scan from disk - pub fn merge_from(&self, filename: &str) { - if let Ok(file) = File::open(filename) { - let reader = BufReader::new(file); - let state: serde_json::Value = serde_json::from_reader(reader).unwrap(); + pub fn merge_from(&self, filename: &str) -> Result<()> { + let file = + File::open(filename).with_context(fmt_err(&format!("Could not open {}", filename)))?; + let reader = BufReader::new(file); + let state: serde_json::Value = serde_json::from_reader(reader)?; - if let Some(state_stats) = state.get("statistics") { - if let Ok(d_stats) = serde_json::from_value::(state_stats.clone()) { - atomic_increment!(self.successes, atomic_load!(d_stats.successes)); - atomic_increment!(self.timeouts, atomic_load!(d_stats.timeouts)); - atomic_increment!(self.requests, atomic_load!(d_stats.requests)); - atomic_increment!(self.errors, atomic_load!(d_stats.errors)); - atomic_increment!(self.redirects, atomic_load!(d_stats.redirects)); - atomic_increment!(self.client_errors, atomic_load!(d_stats.client_errors)); - atomic_increment!(self.server_errors, atomic_load!(d_stats.server_errors)); - atomic_increment!(self.links_extracted, atomic_load!(d_stats.links_extracted)); - atomic_increment!(self.status_200s, atomic_load!(d_stats.status_200s)); - atomic_increment!(self.status_301s, atomic_load!(d_stats.status_301s)); - atomic_increment!(self.status_302s, atomic_load!(d_stats.status_302s)); - atomic_increment!(self.status_401s, atomic_load!(d_stats.status_401s)); - atomic_increment!(self.status_403s, atomic_load!(d_stats.status_403s)); - atomic_increment!(self.status_429s, atomic_load!(d_stats.status_429s)); - atomic_increment!(self.status_500s, atomic_load!(d_stats.status_500s)); - atomic_increment!(self.status_503s, atomic_load!(d_stats.status_503s)); - atomic_increment!(self.status_504s, atomic_load!(d_stats.status_504s)); - atomic_increment!(self.status_508s, atomic_load!(d_stats.status_508s)); - atomic_increment!( - self.wildcards_filtered, - atomic_load!(d_stats.wildcards_filtered) - ); - atomic_increment!( - self.responses_filtered, - atomic_load!(d_stats.responses_filtered) - ); - atomic_increment!( - self.resources_discovered, - atomic_load!(d_stats.resources_discovered) - ); - atomic_increment!( - self.url_format_errors, - atomic_load!(d_stats.url_format_errors) - ); - atomic_increment!( - self.connection_errors, - atomic_load!(d_stats.connection_errors) - ); - atomic_increment!( - self.redirection_errors, - atomic_load!(d_stats.redirection_errors) - ); - atomic_increment!(self.request_errors, atomic_load!(d_stats.request_errors)); + if let Some(state_stats) = state.get("statistics") { + let d_stats = serde_json::from_value::(state_stats.clone())?; + atomic_increment!(self.successes, atomic_load!(d_stats.successes)); + atomic_increment!(self.timeouts, atomic_load!(d_stats.timeouts)); + atomic_increment!(self.requests, atomic_load!(d_stats.requests)); + atomic_increment!(self.errors, atomic_load!(d_stats.errors)); + atomic_increment!(self.redirects, atomic_load!(d_stats.redirects)); + atomic_increment!(self.client_errors, atomic_load!(d_stats.client_errors)); + atomic_increment!(self.server_errors, atomic_load!(d_stats.server_errors)); + atomic_increment!(self.links_extracted, atomic_load!(d_stats.links_extracted)); + atomic_increment!(self.status_200s, atomic_load!(d_stats.status_200s)); + atomic_increment!(self.status_301s, atomic_load!(d_stats.status_301s)); + atomic_increment!(self.status_302s, atomic_load!(d_stats.status_302s)); + atomic_increment!(self.status_401s, atomic_load!(d_stats.status_401s)); + atomic_increment!(self.status_403s, atomic_load!(d_stats.status_403s)); + atomic_increment!(self.status_429s, atomic_load!(d_stats.status_429s)); + atomic_increment!(self.status_500s, atomic_load!(d_stats.status_500s)); + atomic_increment!(self.status_503s, atomic_load!(d_stats.status_503s)); + atomic_increment!(self.status_504s, atomic_load!(d_stats.status_504s)); + atomic_increment!(self.status_508s, atomic_load!(d_stats.status_508s)); + atomic_increment!( + self.wildcards_filtered, + atomic_load!(d_stats.wildcards_filtered) + ); + atomic_increment!( + self.responses_filtered, + atomic_load!(d_stats.responses_filtered) + ); + atomic_increment!( + self.resources_discovered, + atomic_load!(d_stats.resources_discovered) + ); + atomic_increment!( + self.url_format_errors, + atomic_load!(d_stats.url_format_errors) + ); + atomic_increment!( + self.connection_errors, + atomic_load!(d_stats.connection_errors) + ); + atomic_increment!( + self.redirection_errors, + atomic_load!(d_stats.redirection_errors) + ); + atomic_increment!(self.request_errors, atomic_load!(d_stats.request_errors)); - if let Ok(scan_times) = d_stats.directory_scan_times.lock() { - for scan_time in scan_times.iter() { - self.update_f64_field(StatField::DirScanTimes, *scan_time); - } - } + if let Ok(scan_times) = d_stats.directory_scan_times.lock() { + for scan_time in scan_times.iter() { + self.update_f64_field(StatField::DirScanTimes, *scan_time); } } } + Ok(()) } } From 218be60bc26dbc94a6c31aafec0ed47e56eb1638 Mon Sep 17 00:00:00 2001 From: epi Date: Wed, 13 Jan 2021 19:03:35 -0600 Subject: [PATCH 10/13] fixed closure error --- src/event_handlers/statistics.rs | 2 +- src/statistics/container.rs | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/event_handlers/statistics.rs b/src/event_handlers/statistics.rs index 300ef16..f0f7f77 100644 --- a/src/event_handlers/statistics.rs +++ b/src/event_handlers/statistics.rs @@ -77,7 +77,7 @@ impl StatsHandler { self.bar = add_bar("", self.stats.total_expected() as u64, BarType::Total); } StatCommand::LoadStats(filename) => { - self.stats.merge_from(&filename); + self.stats.merge_from(&filename)?; } StatCommand::Exit => break, } diff --git a/src/statistics/container.rs b/src/statistics/container.rs index 9fe4fd4..587e25a 100644 --- a/src/statistics/container.rs +++ b/src/statistics/container.rs @@ -341,8 +341,8 @@ impl Stats { /// /// This is only ever called when resuming a scan from disk pub fn merge_from(&self, filename: &str) -> Result<()> { - let file = - File::open(filename).with_context(fmt_err(&format!("Could not open {}", filename)))?; + let file = File::open(filename) + .with_context(|| fmt_err(&format!("Could not open {}", filename)))?; let reader = BufReader::new(file); let state: serde_json::Value = serde_json::from_reader(reader)?; @@ -396,7 +396,7 @@ impl Stats { for scan_time in scan_times.iter() { self.update_f64_field(StatField::DirScanTimes, *scan_time); } - } + }; } Ok(()) } @@ -543,7 +543,7 @@ mod tests { let tfile = NamedTempFile::new().unwrap(); write(&tfile, contents).unwrap(); - stats.merge_from(tfile.path().to_str().unwrap()); + stats.merge_from(tfile.path().to_str().unwrap()).unwrap(); // as of 1.11.1; all Stats fields are accounted for whether they're updated in merge_from // or not From 70a5eed2ee33943254dce5d6009f60c4f08e714e Mon Sep 17 00:00:00 2001 From: epi Date: Wed, 13 Jan 2021 21:02:11 -0600 Subject: [PATCH 11/13] tidied up banner a lot --- src/banner.rs | 502 +++++++++++++++++++------------------------------- src/main.rs | 2 +- 2 files changed, 186 insertions(+), 318 deletions(-) diff --git a/src/banner.rs b/src/banner.rs index 542dbb8..e4d9977 100644 --- a/src/banner.rs +++ b/src/banner.rs @@ -3,51 +3,20 @@ use crate::{ statistics::StatCommand, utils::{make_request, status_colorizer}, }; +use anyhow::Result; use console::{style, Emoji}; use reqwest::{Client, Url}; +use serde::export::Formatter; use serde_json::Value; +use std::fmt::{self, Display}; use std::io::Write; use tokio::sync::mpsc::UnboundedSender; -/// macro helper to abstract away repetitive string formatting -macro_rules! format_banner_entry_helper { - // \u{0020} -> unicode space - // \u{2502} -> vertical box drawing character, i.e. │ - ($rune:expr, $name:expr, $value:expr, $indent:expr, $col_width:expr) => { - format!( - "\u{0020}{:\u{0020} { - format!( - "\u{0020}{:\u{0020} unicode emoji padding width - // 22 -> column width (when unicode rune is 4 bytes wide, 23 when it's 3) - // hardcoded since macros don't allow let statements - ($rune:expr, $name:expr, $value:expr) => { - format_banner_entry_helper!($rune, $name, $value, 3, 22) - }; - ($rune:expr, $name:expr, $value1:expr, $value2:expr) => { - format_banner_entry_helper!($rune, $name, $value1, $value2, 3, 22) - }; -} +/// Column width used in formatting banner entries +const COL_WIDTH: usize = 22; /// Url used to query github's api; specifically used to look for the latest tagged release name const UPDATE_URL: &str = "https://api.github.com/repos/epi052/feroxbuster/releases/latest"; @@ -65,6 +34,55 @@ enum UpdateStatus { Unknown, } +/// Represents a single line on the banner +#[derive(Default)] +struct BannerEntry { + /// emoji used in the banner entry + emoji: String, + + /// title used in the banner entry + title: String, + + /// value passed in via config/cli/defaults + value: String, +} + +/// implementation of a banner entry +impl BannerEntry { + /// Create a new banner entry from given fields + pub fn new(emoji: &str, title: &str, value: &str) -> Self { + BannerEntry { + emoji: emoji.to_string(), + title: title.to_string(), + value: value.to_string(), + } + } + + /// Simple wrapper for emoji or fallback when terminal doesn't support emoji + fn format_emoji(&self) -> String { + let width = console::measure_text_width(&self.emoji); + let pad_len = width * width; + let pad = format!("{:) -> fmt::Result { + write!( + f, + "\u{0020}{:\u{0020} String { - let width = console::measure_text_width(emoji); - let pad_len = width * width; - let pad = format!("{:( version: &str, mut writer: W, tx_stats: UnboundedSender, -) where +) -> Result<()> +where W: Write, { let artwork = format!( @@ -170,17 +181,13 @@ by Ben "epi" Risher {} ver: {}"#, let addl_section = "──────────────────────────────────────────────────"; let bottom = "───────────────────────────┴──────────────────────"; - writeln!(&mut writer, "{}", artwork).unwrap_or_default(); - writeln!(&mut writer, "{}", top).unwrap_or_default(); + writeln!(&mut writer, "{}", artwork)?; + writeln!(&mut writer, "{}", top)?; // begin with always printed items for target in targets { - writeln!( - &mut writer, - "{}", - format_banner_entry!(format_emoji("🎯"), "Target Url", target) - ) - .unwrap_or_default(); // 🎯 + let tgt = BannerEntry::new("🎯", "Target Url", target); + writeln!(&mut writer, "{}", tgt)?; } let mut codes = vec![]; @@ -189,30 +196,14 @@ by Ben "epi" Risher {} ver: {}"#, codes.push(status_colorizer(&code.to_string())) } - writeln!( - &mut writer, - "{}", - format_banner_entry!(format_emoji("🚀"), "Threads", config.threads) - ) - .unwrap_or_default(); // 🚀 + let threads = BannerEntry::new("🚀", "Threads", &config.threads.to_string()); + writeln!(&mut writer, "{}", threads)?; - writeln!( - &mut writer, - "{}", - format_banner_entry!(format_emoji("📖"), "Wordlist", config.wordlist) - ) - .unwrap_or_default(); // 📖 + let words = BannerEntry::new("📖", "Wordlist", &config.wordlist); + writeln!(&mut writer, "{}", words)?; - writeln!( - &mut writer, - "{}", - format_banner_entry!( - format_emoji("🆗"), - "Status Codes", - format!("[{}]", codes.join(", ")) - ) - ) - .unwrap_or_default(); // 🆗 + let status_codes = BannerEntry::new("👌", "Status Codes", &format!("[{}]", codes.join(", "))); + writeln!(&mut writer, "{}", status_codes)?; if !config.filter_status.is_empty() { // exception here for optional print due to me wanting the allows and denys to be printed @@ -223,49 +214,30 @@ by Ben "epi" Risher {} ver: {}"#, code_filters.push(status_colorizer(&code.to_string())) } - writeln!( - &mut writer, - "{}", - format_banner_entry!( - format_emoji("🗑"), - "Status Code Filters", - format!("[{}]", code_filters.join(", ")) - ) - ) - .unwrap_or_default(); // 🗑 + let banner_cfs = BannerEntry::new( + "🗑", + "Status Code Filters", + &format!("[{}]", code_filters.join(", ")), + ); + + writeln!(&mut writer, "{}", banner_cfs)?; } - writeln!( - &mut writer, - "{}", - format_banner_entry!(format_emoji("💥"), "Timeout (secs)", config.timeout) - ) - .unwrap_or_default(); // 💥 + let timeout = BannerEntry::new("💥", "Timeout (secs)", &config.timeout.to_string()); + writeln!(&mut writer, "{}", timeout)?; - writeln!( - &mut writer, - "{}", - format_banner_entry!(format_emoji("🦡"), "User-Agent", config.user_agent) - ) - .unwrap_or_default(); // 🦡 + let user_agent = BannerEntry::new("🦡", "User-Agent", &config.user_agent); + writeln!(&mut writer, "{}", user_agent)?; // followed by the maybe printed or variably displayed values if !config.config.is_empty() { - writeln!( - &mut writer, - "{}", - format_banner_entry!(format_emoji("💉"), "Config File", config.config) - ) - .unwrap_or_default(); // 💉 + let banner_cfg = BannerEntry::new("💉", "Config File", &config.config); + writeln!(&mut writer, "{}", banner_cfg)?; } if !config.proxy.is_empty() { - writeln!( - &mut writer, - "{}", - format_banner_entry!(format_emoji("💎"), "Proxy", config.proxy) - ) - .unwrap_or_default(); // 💎 + let proxy = BannerEntry::new("💎", "Proxy", &config.proxy); + writeln!(&mut writer, "{}", proxy)?; } if !config.replay_proxy.is_empty() { @@ -274,276 +246,166 @@ by Ben "epi" Risher {} ver: {}"#, let mut replay_codes = vec![]; - writeln!( - &mut writer, - "{}", - format_banner_entry!(format_emoji("🎥"), "Replay Proxy", config.replay_proxy) - ) - .unwrap_or_default(); // 🎥 - for code in &config.replay_codes { replay_codes.push(status_colorizer(&code.to_string())) } - writeln!( - &mut writer, - "{}", - format_banner_entry!( - format_emoji("📼"), - "Replay Proxy Codes", - format!("[{}]", replay_codes.join(", ")) - ) - ) - .unwrap_or_default(); // 📼 + let banner_rcs = BannerEntry::new( + "📼", + "Replay Proxy Codes", + &format!("[{}]", replay_codes.join(", ")), + ); + + let rproxy = BannerEntry::new("🎥", "Replay Proxy", &config.replay_proxy); + + writeln!(&mut writer, "{}", rproxy)?; + writeln!(&mut writer, "{}", banner_rcs)?; } - if !config.headers.is_empty() { - for (name, value) in &config.headers { - writeln!( - &mut writer, - "{}", - format_banner_entry!(format_emoji("🤯"), "Header", name, value) - ) - .unwrap_or_default(); // 🤯 - } + for (name, value) in &config.headers { + let header = BannerEntry::new("🤯", "Header", &format!("{}: {}", name, value)); + writeln!(&mut writer, "{}", header)?; } - if !config.filter_size.is_empty() { - for filter in &config.filter_size { - writeln!( - &mut writer, - "{}", - format_banner_entry!(format_emoji("💢"), "Size Filter", filter) - ) - .unwrap_or_default(); // 💢 - } + for filter in &config.filter_size { + let sz_filter = BannerEntry::new("💢", "Size Filter", &filter.to_string()); + writeln!(&mut writer, "{}", sz_filter)?; } - if !config.filter_similar.is_empty() { - for filter in &config.filter_similar { - writeln!( - &mut writer, - "{}", - format_banner_entry!(format_emoji("💢"), "Similarity Filter", filter) - ) - .unwrap_or_default(); // 💢 - } + for filter in &config.filter_similar { + let sim_filter = BannerEntry::new("💢", "Similarity Filter", filter); + writeln!(&mut writer, "{}", sim_filter)?; } for filter in &config.filter_word_count { - writeln!( - &mut writer, - "{}", - format_banner_entry!(format_emoji("💢"), "Word Count Filter", filter) - ) - .unwrap_or_default(); // 💢 + let wc_filter = BannerEntry::new("💢", "Word Count Filter", &filter.to_string()); + writeln!(&mut writer, "{}", wc_filter)?; } for filter in &config.filter_line_count { - writeln!( - &mut writer, - "{}", - format_banner_entry!(format_emoji("💢"), "Line Count Filter", filter) - ) - .unwrap_or_default(); // 💢 + let lc_filter = BannerEntry::new("💢", "Line Count Filter", &filter.to_string()); + writeln!(&mut writer, "{}", lc_filter)?; } for filter in &config.filter_regex { - writeln!( - &mut writer, - "{}", - format_banner_entry!(format_emoji("💢"), "Regex Filter", filter) - ) - .unwrap_or_default(); // 💢 + let reg_filter = BannerEntry::new("💢", "Regex Filter", filter); + writeln!(&mut writer, "{}", reg_filter)?; } if config.extract_links { - writeln!( - &mut writer, - "{}", - format_banner_entry!(format_emoji("🔎"), "Extract Links", config.extract_links) - ) - .unwrap_or_default(); // 🔎 + let ext_links = BannerEntry::new("🔎", "Extract Links", &config.extract_links.to_string()); + writeln!(&mut writer, "{}", ext_links)?; } if config.json { - writeln!( - &mut writer, - "{}", - format_banner_entry!(format_emoji("🧔"), "JSON Output", config.json) - ) - .unwrap_or_default(); // 🧔 + let json = BannerEntry::new("🧔", "JSON Output", &config.json.to_string()); + writeln!(&mut writer, "{}", json)?; } - if !config.queries.is_empty() { - for query in &config.queries { - writeln!( - &mut writer, - "{}", - format_banner_entry!( - format_emoji("🤔"), - "Query Parameter", - format!("{}={}", query.0, query.1) - ) - ) - .unwrap_or_default(); // 🤔 - } + for query in &config.queries { + let query = BannerEntry::new("🤔", "Query Parameter", &format!("{}={}", query.0, query.1)); + writeln!(&mut writer, "{}", query)?; } if !config.output.is_empty() { - writeln!( - &mut writer, - "{}", - format_banner_entry!(format_emoji("💾"), "Output File", config.output) - ) - .unwrap_or_default(); // 💾 + let out = BannerEntry::new("💾", "Output File", &config.output); + writeln!(&mut writer, "{}", out)?; } if !config.debug_log.is_empty() { - writeln!( - &mut writer, - "{}", - format_banner_entry!(format_emoji("🪲"), "Debugging Log", config.debug_log) - ) - .unwrap_or_default(); // 🪲 + let debug_log = BannerEntry::new("🪲", "Debugging Log", &config.debug_log); + writeln!(&mut writer, "{}", debug_log)?; } if !config.extensions.is_empty() { - writeln!( - &mut writer, - "{}", - format_banner_entry!( - format_emoji("💲"), - "Extensions", - format!("[{}]", config.extensions.join(", ")) - ) - ) - .unwrap_or_default(); // 💲 + let b_exts = BannerEntry::new( + "💲", + "Extensions", + &format!("[{}]", config.extensions.join(", ")), + ); + writeln!(&mut writer, "{}", b_exts)?; } if config.insecure { - writeln!( - &mut writer, - "{}", - format_banner_entry!(format_emoji("🔓"), "Insecure", config.insecure) - ) - .unwrap_or_default(); // 🔓 + let b_insec = BannerEntry::new("🔓", "Insecure", &config.insecure.to_string()); + writeln!(&mut writer, "{}", b_insec)?; } if config.redirects { - writeln!( - &mut writer, - "{}", - format_banner_entry!(format_emoji("📍"), "Follow Redirects", config.redirects) - ) - .unwrap_or_default(); // 📍 + let b_follow = BannerEntry::new("📍", "Follow Redirects", &config.redirects.to_string()); + writeln!(&mut writer, "{}", b_follow)?; } if config.dont_filter { - writeln!( - &mut writer, - "{}", - format_banner_entry!(format_emoji("🤪"), "Filter Wildcards", !config.dont_filter) - ) - .unwrap_or_default(); // 🤪 + let b_wild = BannerEntry::new("🤪", "Filter Wildcards", &(!config.dont_filter).to_string()); + writeln!(&mut writer, "{}", b_wild)?; } let volume = ["🔈", "🔉", "🔊", "📢"]; if let 1..=4 = config.verbosity { //speaker medium volume (increasing with verbosity to loudspeaker) - writeln!( - &mut writer, - "{}", - format_banner_entry!( - format_emoji(volume[config.verbosity as usize - 1]), - "Verbosity", - config.verbosity - ) - ) - .unwrap_or_default(); + let vol = BannerEntry::new( + volume[config.verbosity as usize - 1], + "Verbosity", + &config.verbosity.to_string(), + ); + writeln!(&mut writer, "{}", vol)?; } if config.add_slash { - writeln!( - &mut writer, - "{}", - format_banner_entry!(format_emoji("🪓"), "Add Slash", config.add_slash) - ) - .unwrap_or_default(); // 🪓 + let add = BannerEntry::new("🪓", "Add Slash", &config.add_slash.to_string()); + writeln!(&mut writer, "{}", add)?; } - if !config.no_recursion { - if config.depth == 0 { - writeln!( - &mut writer, - "{}", - format_banner_entry!(format_emoji("🔃"), "Recursion Depth", "INFINITE") - ) - .unwrap_or_default(); // 🔃 + let b_recurse = if !config.no_recursion { + let depth = if config.depth == 0 { + "INFINITE".to_string() } else { - writeln!( - &mut writer, - "{}", - format_banner_entry!(format_emoji("🔃"), "Recursion Depth", config.depth) - ) - .unwrap_or_default(); // 🔃 - } + config.depth.to_string() + }; + + BannerEntry::new("🔃", "Recursion Depth", &depth) } else { - writeln!( - &mut writer, - "{}", - format_banner_entry!(format_emoji("🚫"), "Do Not Recurse", config.no_recursion) - ) - .unwrap_or_default(); // 🚫 + BannerEntry::new("🚫", "Do Not Recurse", &config.no_recursion.to_string()) + }; + + writeln!(&mut writer, "{}", b_recurse)?; + + if config.scan_limit > 0 { + let s_lim = BannerEntry::new( + "🦥", + "Concurrent Scan Limit", + &config.scan_limit.to_string(), + ); + writeln!(&mut writer, "{}", s_lim)?; } - if CONFIGURATION.scan_limit > 0 { - writeln!( - &mut writer, - "{}", - format_banner_entry!( - format_emoji("🦥"), - "Concurrent Scan Limit", - config.scan_limit - ) - ) - .unwrap_or_default(); // 🦥 - } - - if !CONFIGURATION.time_limit.is_empty() { - writeln!( - &mut writer, - "{}", - format_banner_entry!(format_emoji("🕖"), "Time Limit", config.time_limit) - ) - .unwrap_or_default(); // 🕖 + if !config.time_limit.is_empty() { + let t_lim = BannerEntry::new("🕖", "Time Limit", &config.time_limit); + writeln!(&mut writer, "{}", t_lim)?; } if matches!(status, UpdateStatus::OutOfDate) { - writeln!( - &mut writer, - "{}", - format_banner_entry!( - format_emoji("🎉"), - "New Version Available", - "https://github.com/epi052/feroxbuster/releases/latest" - ) - ) - .unwrap_or_default(); // 🎉 + let update = BannerEntry::new( + "🎉", + "New Version Available", + "https://github.com/epi052/feroxbuster/releases/latest", + ); + writeln!(&mut writer, "{}", update)?; } - writeln!(&mut writer, "{}", bottom).unwrap_or_default(); - // ⏯ + writeln!(&mut writer, "{}", bottom)?; + writeln!( &mut writer, - " {} Press [{}] to use the {}™", - format_emoji("🏁"), + " 🏁 Press [{}] to use the {}™", style("ENTER").yellow(), style("Scan Cancel Menu").bright().yellow(), - ) - .unwrap_or_default(); + )?; - writeln!(&mut writer, "{}", addl_section).unwrap_or_default(); + writeln!(&mut writer, "{}", addl_section)?; + + Ok(()) } #[cfg(test)] @@ -564,7 +426,9 @@ mod tests { let config = Configuration::default(); let (tx, _): FeroxChannel = mpsc::unbounded_channel(); - initialize(&[], &config, VERSION, stderr(), tx).await; + initialize(&[], &config, VERSION, stderr(), tx) + .await + .unwrap(); } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] @@ -584,7 +448,8 @@ mod tests { stderr(), tx, ) - .await; + .await + .unwrap(); } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] @@ -604,7 +469,8 @@ mod tests { stderr(), tx, ) - .await; + .await + .unwrap(); } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] @@ -624,7 +490,8 @@ mod tests { stderr(), tx, ) - .await; + .await + .unwrap(); } #[ignore] @@ -642,7 +509,8 @@ mod tests { &file, tx, ) - .await; + .await + .unwrap(); let contents = read_to_string(file.path()).unwrap(); println!("contents: {}", contents); assert!(contents.contains("New Version Available")); diff --git a/src/main.rs b/src/main.rs index 8dd8ba2..4d0c4d4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -322,7 +322,7 @@ async fn wrapped_main() -> Result<()> { std_stderr, tx_stats.clone(), ) - .await; + .await?; } // discard non-responsive targets From c301d54083ba8151aec3b040a496a5aad229cebc Mon Sep 17 00:00:00 2001 From: epi Date: Thu, 14 Jan 2021 11:28:15 -0600 Subject: [PATCH 12/13] cleaned up banner code --- src/banner.rs | 876 +++++++++++++++++++++++++++++--------------------- src/main.rs | 16 +- src/utils.rs | 6 +- 3 files changed, 518 insertions(+), 380 deletions(-) diff --git a/src/banner.rs b/src/banner.rs index e4d9977..7bb04b0 100644 --- a/src/banner.rs +++ b/src/banner.rs @@ -1,9 +1,10 @@ use crate::{ - config::{Configuration, CONFIGURATION}, + config::Configuration, statistics::StatCommand, utils::{make_request, status_colorizer}, + VERSION, }; -use anyhow::Result; +use anyhow::{bail, Result}; use console::{style, Emoji}; use reqwest::{Client, Url}; use serde::export::Formatter; @@ -19,7 +20,7 @@ const INDENT: usize = 3; const COL_WIDTH: usize = 22; /// Url used to query github's api; specifically used to look for the latest tagged release name -const UPDATE_URL: &str = "https://api.github.com/repos/epi052/feroxbuster/releases/latest"; +pub const UPDATE_URL: &str = "https://api.github.com/repos/epi052/feroxbuster/releases/latest"; /// Simple enum to hold three different update states #[derive(Debug)] @@ -83,57 +84,349 @@ impl Display for BannerEntry { } } -/// Makes a request to the given url, expecting to receive a JSON response that contains a field -/// named `tag_name` that holds a value representing the latest tagged release of this tool. -/// -/// ex: v1.1.0 -/// -/// Returns `UpdateStatus` -async fn needs_update( - client: &Client, - url: &str, - bin_version: &str, - tx_stats: UnboundedSender, -) -> UpdateStatus { - log::trace!("enter: needs_update({:?}, {}, {:?})", client, url, tx_stats); +/// Banner object, contains multiple BannerEntry's and knows how to display itself +pub struct Banner { + /// all live targets + targets: Vec, - let unknown = UpdateStatus::Unknown; + /// represents Configuration.status_codes + status_codes: BannerEntry, - let api_url = match Url::parse(url) { - Ok(url) => url, - Err(e) => { - log::error!("{}", e); - log::trace!("exit: needs_update -> {:?}", unknown); - return unknown; + /// represents Configuration.filter_status + filter_status: BannerEntry, + + /// represents Configuration.threads + threads: BannerEntry, + + /// represents Configuration.wordlist + wordlist: BannerEntry, + + /// represents Configuration.timeout + timeout: BannerEntry, + + /// represents Configuration.user_agent + user_agent: BannerEntry, + + /// represents Configuration.config + config: BannerEntry, + + /// represents Configuration.proxy + proxy: BannerEntry, + + /// represents Configuration.replay_proxy + replay_proxy: BannerEntry, + + /// represents Configuration.replay_codes + replay_codes: BannerEntry, + + /// represents Configuration.headers + headers: Vec, + + /// represents Configuration.filter_size + filter_size: Vec, + + /// represents Configuration.filter_similar + filter_similar: Vec, + + /// represents Configuration.filter_word_count + filter_word_count: Vec, + + /// represents Configuration.filter_line_count + filter_line_count: Vec, + + /// represents Configuration.filter_regex + filter_regex: Vec, + + /// represents Configuration.extract_links + extract_links: BannerEntry, + + /// represents Configuration.json + json: BannerEntry, + + /// represents Configuration.output + output: BannerEntry, + + /// represents Configuration.debug_log + debug_log: BannerEntry, + + /// represents Configuration.extensions + extensions: BannerEntry, + + /// represents Configuration.insecure + insecure: BannerEntry, + + /// represents Configuration.redirects + redirects: BannerEntry, + + /// represents Configuration.dont_filter + dont_filter: BannerEntry, + + /// represents Configuration.queries + queries: Vec, + + /// represents Configuration.verbosity + verbosity: BannerEntry, + + /// represents Configuration.add_slash + add_slash: BannerEntry, + + /// represents Configuration.no_recursion + no_recursion: BannerEntry, + + /// represents Configuration.scan_limit + scan_limit: BannerEntry, + + /// represents Configuration.time_limit + time_limit: BannerEntry, + + /// current version of feroxbuster + version: String, + + /// whether or not there is a known new version + update_status: UpdateStatus, +} + +/// implementation of Banner +impl Banner { + /// Create a new Banner from a Configuration and live targets + pub fn new(tgts: &[String], config: &Configuration) -> Self { + let mut targets = Vec::new(); + let mut code_filters = Vec::new(); + let mut replay_codes = Vec::new(); + let mut headers = Vec::new(); + let mut filter_size = Vec::new(); + let mut filter_similar = Vec::new(); + let mut filter_word_count = Vec::new(); + let mut filter_line_count = Vec::new(); + let mut filter_regex = Vec::new(); + let mut queries = Vec::new(); + + for target in tgts { + targets.push(BannerEntry::new("🎯", "Target Url", target)); } - }; - if let Ok(response) = make_request(&client, &api_url, tx_stats.clone()).await { - let body = response.text().await.unwrap_or_default(); - - let json_response: Value = serde_json::from_str(&body).unwrap_or_default(); - - if json_response.is_null() { - // unwrap_or_default above should result in a null value for the json_response variable - log::error!("Could not parse JSON from response body"); - log::trace!("exit: needs_update -> {:?}", unknown); - return unknown; + let mut codes = vec![]; + for code in &config.status_codes { + codes.push(status_colorizer(&code.to_string())) } + let status_codes = + BannerEntry::new("👌", "Status Codes", &format!("[{}]", codes.join(", "))); + + for code in &config.filter_status { + code_filters.push(status_colorizer(&code.to_string())) + } + let filter_status = BannerEntry::new( + "🗑", + "Status Code Filters", + &format!("[{}]", code_filters.join(", ")), + ); + + for code in &config.replay_codes { + replay_codes.push(status_colorizer(&code.to_string())) + } + let replay_codes = BannerEntry::new( + "📼", + "Replay Proxy Codes", + &format!("[{}]", replay_codes.join(", ")), + ); + + for (name, value) in &config.headers { + headers.push(BannerEntry::new( + "🤯", + "Header", + &format!("{}: {}", name, value), + )); + } + + for filter in &config.filter_size { + filter_size.push(BannerEntry::new("💢", "Size Filter", &filter.to_string())); + } + + for filter in &config.filter_similar { + filter_similar.push(BannerEntry::new("💢", "Similarity Filter", filter)); + } + + for filter in &config.filter_word_count { + filter_word_count.push(BannerEntry::new( + "💢", + "Word Count Filter", + &filter.to_string(), + )); + } + + for filter in &config.filter_line_count { + filter_line_count.push(BannerEntry::new( + "💢", + "Line Count Filter", + &filter.to_string(), + )); + } + + for filter in &config.filter_regex { + filter_regex.push(BannerEntry::new("💢", "Regex Filter", filter)); + } + + for query in &config.queries { + queries.push(BannerEntry::new( + "🤔", + "Query Parameter", + &format!("{}={}", query.0, query.1), + )); + } + + let volume = ["🔈", "🔉", "🔊", "📢"]; + let verbosity = if let 1..=4 = config.verbosity { + //speaker medium volume (increasing with verbosity to loudspeaker) + BannerEntry::new( + volume[config.verbosity as usize - 1], + "Verbosity", + &config.verbosity.to_string(), + ) + } else { + BannerEntry::default() + }; + + let no_recursion = if !config.no_recursion { + let depth = if config.depth == 0 { + "INFINITE".to_string() + } else { + config.depth.to_string() + }; + + BannerEntry::new("🔃", "Recursion Depth", &depth) + } else { + BannerEntry::new("🚫", "Do Not Recurse", &config.no_recursion.to_string()) + }; + + let scan_limit = BannerEntry::new( + "🦥", + "Concurrent Scan Limit", + &config.scan_limit.to_string(), + ); + + let replay_proxy = BannerEntry::new("🎥", "Replay Proxy", &config.replay_proxy); + let cfg = BannerEntry::new("💉", "Config File", &config.config); + let proxy = BannerEntry::new("💎", "Proxy", &config.proxy); + let threads = BannerEntry::new("🚀", "Threads", &config.threads.to_string()); + let wordlist = BannerEntry::new("📖", "Wordlist", &config.wordlist); + let timeout = BannerEntry::new("💥", "Timeout (secs)", &config.timeout.to_string()); + let user_agent = BannerEntry::new("🦡", "User-Agent", &config.user_agent); + let extract_links = + BannerEntry::new("🔎", "Extract Links", &config.extract_links.to_string()); + let json = BannerEntry::new("🧔", "JSON Output", &config.json.to_string()); + let output = BannerEntry::new("💾", "Output File", &config.output); + let debug_log = BannerEntry::new("🪲", "Debugging Log", &config.debug_log); + let extensions = BannerEntry::new( + "💲", + "Extensions", + &format!("[{}]", config.extensions.join(", ")), + ); + let insecure = BannerEntry::new("🔓", "Insecure", &config.insecure.to_string()); + let redirects = BannerEntry::new("📍", "Follow Redirects", &config.redirects.to_string()); + let dont_filter = + BannerEntry::new("🤪", "Filter Wildcards", &(!config.dont_filter).to_string()); + let add_slash = BannerEntry::new("🪓", "Add Slash", &config.add_slash.to_string()); + let time_limit = BannerEntry::new("🕖", "Time Limit", &config.time_limit); + + Self { + targets, + status_codes, + threads, + wordlist, + filter_status, + timeout, + user_agent, + proxy, + replay_codes, + replay_proxy, + headers, + filter_size, + filter_similar, + filter_word_count, + filter_line_count, + filter_regex, + extract_links, + json, + queries, + output, + debug_log, + extensions, + insecure, + dont_filter, + redirects, + verbosity, + add_slash, + no_recursion, + scan_limit, + time_limit, + config: cfg, + version: VERSION.to_string(), + update_status: UpdateStatus::Unknown, + } + } + + /// get a fancy header for the banner + fn header(&self) -> String { + let artwork = format!( + r#" + ___ ___ __ __ __ __ __ ___ +|__ |__ |__) |__) | / ` / \ \_/ | | \ |__ +| |___ | \ | \ | \__, \__/ / \ | |__/ |___ +by Ben "epi" Risher {} ver: {}"#, + Emoji("🤓", &format!("{:<2}", "\u{0020}")), + self.version + ); + + let top = "───────────────────────────┬──────────────────────"; + + format!("{}\n{}", artwork, top) + } + + /// get a fancy footer for the banner + fn footer(&self) -> String { + let addl_section = "──────────────────────────────────────────────────"; + let bottom = "───────────────────────────┴──────────────────────"; + + let instructions = format!( + " 🏁 Press [{}] to use the {}™", + style("ENTER").yellow(), + style("Scan Cancel Menu").bright().yellow(), + ); + + format!("{}\n{}\n{}", bottom, instructions, addl_section) + } + + /// Makes a request to the given url, expecting to receive a JSON response that contains a field + /// named `tag_name` that holds a value representing the latest tagged release of this tool. + /// + /// ex: v1.1.0 + pub async fn check_for_updates( + &mut self, + client: &Client, + url: &str, + tx_stats: UnboundedSender, + ) -> Result<()> { + log::trace!("enter: needs_update({:?}, {}, {:?})", client, url, tx_stats); + + let api_url = Url::parse(url)?; + + let response = make_request(&client, &api_url, tx_stats.clone()).await?; + + let body = response.text().await?; + + let json_response: Value = serde_json::from_str(&body)?; let latest_version = match json_response["tag_name"].as_str() { Some(tag) => tag.trim_start_matches('v'), None => { - log::error!("Could not get version field from JSON response"); - log::debug!("{}", json_response); - log::trace!("exit: needs_update -> {:?}", unknown); - return unknown; + bail!("JSON has no tag_name: {}", json_response); } }; // if we've gotten this far, we have a string in the form of X.X.X where X is a number // all that's left is to compare the current version with the version found above - return if latest_version == bin_version { + return if latest_version == self.version { // there's really only two possible outcomes if we accept that the tag conforms to // the X.X.X pattern: // 1. the version strings match, meaning we're up to date @@ -141,294 +434,163 @@ async fn needs_update( // // except for developers working on this code, nobody should ever be in a situation // where they have a version greater than the latest tagged release - log::trace!("exit: needs_update -> UpdateStatus::UpToDate"); - UpdateStatus::UpToDate + self.update_status = UpdateStatus::UpToDate; + Ok(()) } else { - log::trace!("exit: needs_update -> UpdateStatus::OutOfDate"); - UpdateStatus::OutOfDate + self.update_status = UpdateStatus::OutOfDate; + Ok(()) }; } - log::trace!("exit: needs_update -> {:?}", unknown); - unknown -} + /// display the banner on Write writer + pub fn print_to(&self, mut writer: W, config: &Configuration) -> Result<()> + where + W: Write, + { + writeln!(&mut writer, "{}", self.header())?; -/// Prints the banner to stdout. -/// -/// Only prints those settings which are either always present, or passed in by the user. -pub async fn initialize( - targets: &[String], - config: &Configuration, - version: &str, - mut writer: W, - tx_stats: UnboundedSender, -) -> Result<()> -where - W: Write, -{ - let artwork = format!( - r#" - ___ ___ __ __ __ __ __ ___ -|__ |__ |__) |__) | / ` / \ \_/ | | \ |__ -| |___ | \ | \ | \__, \__/ / \ | |__/ |___ -by Ben "epi" Risher {} ver: {}"#, - Emoji("🤓", &format!("{:<2}", "\u{0020}")), - version - ); - let status = needs_update(&CONFIGURATION.client, UPDATE_URL, version, tx_stats).await; - - let top = "───────────────────────────┬──────────────────────"; - let addl_section = "──────────────────────────────────────────────────"; - let bottom = "───────────────────────────┴──────────────────────"; - - writeln!(&mut writer, "{}", artwork)?; - writeln!(&mut writer, "{}", top)?; - - // begin with always printed items - for target in targets { - let tgt = BannerEntry::new("🎯", "Target Url", target); - writeln!(&mut writer, "{}", tgt)?; - } - - let mut codes = vec![]; - - for code in &config.status_codes { - codes.push(status_colorizer(&code.to_string())) - } - - let threads = BannerEntry::new("🚀", "Threads", &config.threads.to_string()); - writeln!(&mut writer, "{}", threads)?; - - let words = BannerEntry::new("📖", "Wordlist", &config.wordlist); - writeln!(&mut writer, "{}", words)?; - - let status_codes = BannerEntry::new("👌", "Status Codes", &format!("[{}]", codes.join(", "))); - writeln!(&mut writer, "{}", status_codes)?; - - if !config.filter_status.is_empty() { - // exception here for optional print due to me wanting the allows and denys to be printed - // one after the other - let mut code_filters = vec![]; - - for code in &config.filter_status { - code_filters.push(status_colorizer(&code.to_string())) + // begin with always printed items + for target in &self.targets { + writeln!(&mut writer, "{}", target)?; } - let banner_cfs = BannerEntry::new( - "🗑", - "Status Code Filters", - &format!("[{}]", code_filters.join(", ")), - ); + writeln!(&mut writer, "{}", self.threads)?; + writeln!(&mut writer, "{}", self.wordlist)?; + writeln!(&mut writer, "{}", self.status_codes)?; - writeln!(&mut writer, "{}", banner_cfs)?; - } - - let timeout = BannerEntry::new("💥", "Timeout (secs)", &config.timeout.to_string()); - writeln!(&mut writer, "{}", timeout)?; - - let user_agent = BannerEntry::new("🦡", "User-Agent", &config.user_agent); - writeln!(&mut writer, "{}", user_agent)?; - - // followed by the maybe printed or variably displayed values - if !config.config.is_empty() { - let banner_cfg = BannerEntry::new("💉", "Config File", &config.config); - writeln!(&mut writer, "{}", banner_cfg)?; - } - - if !config.proxy.is_empty() { - let proxy = BannerEntry::new("💎", "Proxy", &config.proxy); - writeln!(&mut writer, "{}", proxy)?; - } - - if !config.replay_proxy.is_empty() { - // i include replay codes logic here because in config.rs, replay codes are set to the - // value in status codes, meaning it's never empty - - let mut replay_codes = vec![]; - - for code in &config.replay_codes { - replay_codes.push(status_colorizer(&code.to_string())) + if !config.filter_status.is_empty() { + // exception here for an optional print in the middle of always printed values is due + // to me wanting the allows and denys to be printed one after the other + writeln!(&mut writer, "{}", self.filter_status)?; } - let banner_rcs = BannerEntry::new( - "📼", - "Replay Proxy Codes", - &format!("[{}]", replay_codes.join(", ")), - ); + writeln!(&mut writer, "{}", self.timeout)?; + writeln!(&mut writer, "{}", self.user_agent)?; - let rproxy = BannerEntry::new("🎥", "Replay Proxy", &config.replay_proxy); + // followed by the maybe printed or variably displayed values + if !config.config.is_empty() { + writeln!(&mut writer, "{}", self.config)?; + } - writeln!(&mut writer, "{}", rproxy)?; - writeln!(&mut writer, "{}", banner_rcs)?; + if !config.proxy.is_empty() { + writeln!(&mut writer, "{}", self.proxy)?; + } + + if !config.replay_proxy.is_empty() { + // i include replay codes logic here because in config.rs, replay codes are set to the + // value in status codes, meaning it's never empty + writeln!(&mut writer, "{}", self.replay_proxy)?; + writeln!(&mut writer, "{}", self.replay_codes)?; + } + + for header in &self.headers { + writeln!(&mut writer, "{}", header)?; + } + + for filter in &self.filter_size { + writeln!(&mut writer, "{}", filter)?; + } + + for filter in &self.filter_similar { + writeln!(&mut writer, "{}", filter)?; + } + + for filter in &self.filter_word_count { + writeln!(&mut writer, "{}", filter)?; + } + + for filter in &self.filter_line_count { + writeln!(&mut writer, "{}", filter)?; + } + + for filter in &self.filter_regex { + writeln!(&mut writer, "{}", filter)?; + } + + if config.extract_links { + writeln!(&mut writer, "{}", self.extract_links)?; + } + + if config.json { + writeln!(&mut writer, "{}", self.json)?; + } + + for query in &self.queries { + writeln!(&mut writer, "{}", query)?; + } + + if !config.output.is_empty() { + writeln!(&mut writer, "{}", self.output)?; + } + + if !config.debug_log.is_empty() { + writeln!(&mut writer, "{}", self.debug_log)?; + } + + if !config.extensions.is_empty() { + writeln!(&mut writer, "{}", self.extensions)?; + } + + if config.insecure { + writeln!(&mut writer, "{}", self.insecure)?; + } + + if config.redirects { + writeln!(&mut writer, "{}", self.redirects)?; + } + + if config.dont_filter { + writeln!(&mut writer, "{}", self.dont_filter)?; + } + + if let 1..=4 = config.verbosity { + writeln!(&mut writer, "{}", self.verbosity)?; + } + + if config.add_slash { + writeln!(&mut writer, "{}", self.add_slash)?; + } + + writeln!(&mut writer, "{}", self.no_recursion)?; + + if config.scan_limit > 0 { + writeln!(&mut writer, "{}", self.scan_limit)?; + } + if !config.time_limit.is_empty() { + writeln!(&mut writer, "{}", self.time_limit)?; + } + + if matches!(self.update_status, UpdateStatus::OutOfDate) { + let update = BannerEntry::new( + "🎉", + "New Version Available", + "https://github.com/epi052/feroxbuster/releases/latest", + ); + writeln!(&mut writer, "{}", update)?; + } + + writeln!(&mut writer, "{}", self.footer())?; + + Ok(()) } - - for (name, value) in &config.headers { - let header = BannerEntry::new("🤯", "Header", &format!("{}: {}", name, value)); - writeln!(&mut writer, "{}", header)?; - } - - for filter in &config.filter_size { - let sz_filter = BannerEntry::new("💢", "Size Filter", &filter.to_string()); - writeln!(&mut writer, "{}", sz_filter)?; - } - - for filter in &config.filter_similar { - let sim_filter = BannerEntry::new("💢", "Similarity Filter", filter); - writeln!(&mut writer, "{}", sim_filter)?; - } - - for filter in &config.filter_word_count { - let wc_filter = BannerEntry::new("💢", "Word Count Filter", &filter.to_string()); - writeln!(&mut writer, "{}", wc_filter)?; - } - - for filter in &config.filter_line_count { - let lc_filter = BannerEntry::new("💢", "Line Count Filter", &filter.to_string()); - writeln!(&mut writer, "{}", lc_filter)?; - } - - for filter in &config.filter_regex { - let reg_filter = BannerEntry::new("💢", "Regex Filter", filter); - writeln!(&mut writer, "{}", reg_filter)?; - } - - if config.extract_links { - let ext_links = BannerEntry::new("🔎", "Extract Links", &config.extract_links.to_string()); - writeln!(&mut writer, "{}", ext_links)?; - } - - if config.json { - let json = BannerEntry::new("🧔", "JSON Output", &config.json.to_string()); - writeln!(&mut writer, "{}", json)?; - } - - for query in &config.queries { - let query = BannerEntry::new("🤔", "Query Parameter", &format!("{}={}", query.0, query.1)); - writeln!(&mut writer, "{}", query)?; - } - - if !config.output.is_empty() { - let out = BannerEntry::new("💾", "Output File", &config.output); - writeln!(&mut writer, "{}", out)?; - } - - if !config.debug_log.is_empty() { - let debug_log = BannerEntry::new("🪲", "Debugging Log", &config.debug_log); - writeln!(&mut writer, "{}", debug_log)?; - } - - if !config.extensions.is_empty() { - let b_exts = BannerEntry::new( - "💲", - "Extensions", - &format!("[{}]", config.extensions.join(", ")), - ); - writeln!(&mut writer, "{}", b_exts)?; - } - - if config.insecure { - let b_insec = BannerEntry::new("🔓", "Insecure", &config.insecure.to_string()); - writeln!(&mut writer, "{}", b_insec)?; - } - - if config.redirects { - let b_follow = BannerEntry::new("📍", "Follow Redirects", &config.redirects.to_string()); - writeln!(&mut writer, "{}", b_follow)?; - } - - if config.dont_filter { - let b_wild = BannerEntry::new("🤪", "Filter Wildcards", &(!config.dont_filter).to_string()); - writeln!(&mut writer, "{}", b_wild)?; - } - - let volume = ["🔈", "🔉", "🔊", "📢"]; - if let 1..=4 = config.verbosity { - //speaker medium volume (increasing with verbosity to loudspeaker) - let vol = BannerEntry::new( - volume[config.verbosity as usize - 1], - "Verbosity", - &config.verbosity.to_string(), - ); - writeln!(&mut writer, "{}", vol)?; - } - - if config.add_slash { - let add = BannerEntry::new("🪓", "Add Slash", &config.add_slash.to_string()); - writeln!(&mut writer, "{}", add)?; - } - - let b_recurse = if !config.no_recursion { - let depth = if config.depth == 0 { - "INFINITE".to_string() - } else { - config.depth.to_string() - }; - - BannerEntry::new("🔃", "Recursion Depth", &depth) - } else { - BannerEntry::new("🚫", "Do Not Recurse", &config.no_recursion.to_string()) - }; - - writeln!(&mut writer, "{}", b_recurse)?; - - if config.scan_limit > 0 { - let s_lim = BannerEntry::new( - "🦥", - "Concurrent Scan Limit", - &config.scan_limit.to_string(), - ); - writeln!(&mut writer, "{}", s_lim)?; - } - - if !config.time_limit.is_empty() { - let t_lim = BannerEntry::new("🕖", "Time Limit", &config.time_limit); - writeln!(&mut writer, "{}", t_lim)?; - } - - if matches!(status, UpdateStatus::OutOfDate) { - let update = BannerEntry::new( - "🎉", - "New Version Available", - "https://github.com/epi052/feroxbuster/releases/latest", - ); - writeln!(&mut writer, "{}", update)?; - } - - writeln!(&mut writer, "{}", bottom)?; - - writeln!( - &mut writer, - " 🏁 Press [{}] to use the {}™", - style("ENTER").yellow(), - style("Scan Cancel Menu").bright().yellow(), - )?; - - writeln!(&mut writer, "{}", addl_section)?; - - Ok(()) } #[cfg(test)] mod tests { use super::*; - use crate::{FeroxChannel, VERSION}; + use crate::{config::CONFIGURATION, FeroxChannel}; use httpmock::Method::GET; use httpmock::MockServer; - use std::fs::read_to_string; use std::io::stderr; use std::time::Duration; - use tempfile::NamedTempFile; use tokio::sync::mpsc; #[tokio::test(flavor = "multi_thread", worker_threads = 1)] /// test to hit no execution of targets for loop in banner async fn banner_intialize_without_targets() { let config = Configuration::default(); - let (tx, _): FeroxChannel = mpsc::unbounded_channel(); - - initialize(&[], &config, VERSION, stderr(), tx) - .await - .unwrap(); + let banner = Banner::new(&[], &config); + banner.print_to(stderr(), &config).unwrap(); } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] @@ -439,17 +601,8 @@ mod tests { ..Default::default() }; - let (tx, _): FeroxChannel = mpsc::unbounded_channel(); - - initialize( - &[String::from("http://localhost")], - &config, - VERSION, - stderr(), - tx, - ) - .await - .unwrap(); + let banner = Banner::new(&[String::from("http://localhost")], &config); + banner.print_to(stderr(), &config).unwrap(); } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] @@ -460,61 +613,20 @@ mod tests { ..Default::default() }; - let (tx, _): FeroxChannel = mpsc::unbounded_channel(); - - initialize( - &[String::from("http://localhost")], - &config, - VERSION, - stderr(), - tx, - ) - .await - .unwrap(); + let banner = Banner::new(&[String::from("http://localhost")], &config); + banner.print_to(stderr(), &config).unwrap(); } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] - /// test to hit an empty config file + /// test to hit an empty queries async fn banner_intialize_without_queries() { let config = Configuration { queries: vec![(String::new(), String::new())], ..Default::default() }; - let (tx, _): FeroxChannel = mpsc::unbounded_channel(); - - initialize( - &[String::from("http://localhost")], - &config, - VERSION, - stderr(), - tx, - ) - .await - .unwrap(); - } - - #[ignore] - #[tokio::test(flavor = "multi_thread", worker_threads = 1)] - /// test to show that a new version is available for download - async fn banner_intialize_with_mismatched_version() { - let config = Configuration::default(); - let file = NamedTempFile::new().unwrap(); - let (tx, _): FeroxChannel = mpsc::unbounded_channel(); - - initialize( - &[String::from("http://localhost")], - &config, - "mismatched-version", - &file, - tx, - ) - .await - .unwrap(); - let contents = read_to_string(file.path()).unwrap(); - println!("contents: {}", contents); - assert!(contents.contains("New Version Available")); - assert!(contents.contains("https://github.com/epi052/feroxbuster/releases/latest")); + let banner = Banner::new(&[String::from("http://localhost")], &config); + banner.print_to(stderr(), &config).unwrap(); } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] @@ -522,8 +634,13 @@ mod tests { async fn banner_needs_update_returns_unknown_with_bad_url() { let (tx, _): FeroxChannel = mpsc::unbounded_channel(); - let result = needs_update(&CONFIGURATION.client, &"", VERSION, tx).await; - assert!(matches!(result, UpdateStatus::Unknown)); + let mut banner = Banner::new(&[String::from("http://localhost")], &CONFIGURATION); + + let _ = banner + .check_for_updates(&CONFIGURATION.client, &"", tx) + .await; + + assert!(matches!(banner.update_status, UpdateStatus::Unknown)); } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] @@ -538,10 +655,15 @@ mod tests { let (tx, _): FeroxChannel = mpsc::unbounded_channel(); - let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.1.0", tx).await; + let mut banner = Banner::new(&[srv.url("")], &CONFIGURATION); + banner.version = String::from("1.1.0"); + + let _ = banner + .check_for_updates(&CONFIGURATION.client, &srv.url("/latest"), tx) + .await; assert_eq!(mock.hits(), 1); - assert!(matches!(result, UpdateStatus::UpToDate)); + assert!(matches!(banner.update_status, UpdateStatus::UpToDate)); } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] @@ -556,10 +678,15 @@ mod tests { let (tx, _): FeroxChannel = mpsc::unbounded_channel(); - let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1", tx).await; + let mut banner = Banner::new(&[srv.url("")], &CONFIGURATION); + banner.version = String::from("1.0.1"); + + let _ = banner + .check_for_updates(&CONFIGURATION.client, &srv.url("/latest"), tx) + .await; assert_eq!(mock.hits(), 1); - assert!(matches!(result, UpdateStatus::OutOfDate)); + assert!(matches!(banner.update_status, UpdateStatus::OutOfDate)); } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] @@ -576,10 +703,14 @@ mod tests { let (tx, _): FeroxChannel = mpsc::unbounded_channel(); - let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1", tx).await; + let mut banner = Banner::new(&[srv.url("")], &CONFIGURATION); + + let _ = banner + .check_for_updates(&CONFIGURATION.client, &srv.url("/latest"), tx) + .await; assert_eq!(mock.hits(), 1); - assert!(matches!(result, UpdateStatus::Unknown)); + assert!(matches!(banner.update_status, UpdateStatus::Unknown)); } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] @@ -594,10 +725,14 @@ mod tests { let (tx, _): FeroxChannel = mpsc::unbounded_channel(); - let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1", tx).await; + let mut banner = Banner::new(&[srv.url("")], &CONFIGURATION); + + let _ = banner + .check_for_updates(&CONFIGURATION.client, &srv.url("/latest"), tx) + .await; assert_eq!(mock.hits(), 1); - assert!(matches!(result, UpdateStatus::Unknown)); + assert!(matches!(banner.update_status, UpdateStatus::Unknown)); } #[tokio::test(flavor = "multi_thread", worker_threads = 1)] @@ -613,9 +748,14 @@ mod tests { let (tx, _): FeroxChannel = mpsc::unbounded_channel(); - let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1", tx).await; + let mut banner = Banner::new(&[srv.url("")], &CONFIGURATION); + banner.version = String::from("1.0.1"); + + let _ = banner + .check_for_updates(&CONFIGURATION.client, &srv.url("/latest"), tx) + .await; assert_eq!(mock.hits(), 1); - assert!(matches!(result, UpdateStatus::Unknown)); + assert!(matches!(banner.update_status, UpdateStatus::Unknown)); } } diff --git a/src/main.rs b/src/main.rs index 4d0c4d4..230fd26 100644 --- a/src/main.rs +++ b/src/main.rs @@ -16,7 +16,7 @@ use feroxbuster::{ }, update_stat, utils::{ferox_print, fmt_err, get_current_depth, status_colorizer}, - FeroxError, FeroxResponse, FeroxResult, SLEEP_DURATION, VERSION, + FeroxError, FeroxResponse, FeroxResult, SLEEP_DURATION, }; #[cfg(not(target_os = "windows"))] use feroxbuster::{utils::set_open_file_limit, DEFAULT_OPEN_FILE_LIMIT}; @@ -315,14 +315,12 @@ async fn wrapped_main() -> Result<()> { if !CONFIGURATION.quiet { // only print banner if -q isn't used let std_stderr = stderr(); // std::io::stderr - banner::initialize( - &targets, - &CONFIGURATION, - &VERSION, - std_stderr, - tx_stats.clone(), - ) - .await?; + let mut bnr = banner::Banner::new(&targets, &CONFIGURATION); + // only interested in the side-effect + let _ = bnr + .check_for_updates(&CONFIGURATION.client, banner::UPDATE_URL, tx_stats.clone()) + .await; + bnr.print_to(std_stderr, &CONFIGURATION)?; } // discard non-responsive targets diff --git a/src/utils.rs b/src/utils.rs index a77c57a..b8f8976 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -7,7 +7,7 @@ use crate::{ }, FeroxError, FeroxResult, }; -use anyhow::Context; +use anyhow::{bail, Context, Result}; use console::{strip_ansi_codes, style, user_attended}; use indicatif::ProgressBar; use reqwest::{Client, Response, Url}; @@ -285,7 +285,7 @@ pub async fn make_request( client: &Client, url: &Url, tx_stats: UnboundedSender, -) -> FeroxResult { +) -> Result { log::trace!( "enter: make_request(CONFIGURATION.Client, {}, {:?})", url, @@ -331,7 +331,7 @@ pub async fn make_request( log::warn!("Error while making request: {}", e); } - Err(Box::new(e)) + bail!("{}", e) } Ok(resp) => { log::trace!("exit: make_request -> {:?}", resp); From e7b3c9f7c0e80b35045a04c440aba2fce791b4c5 Mon Sep 17 00:00:00 2001 From: epi Date: Thu, 14 Jan 2021 11:36:03 -0600 Subject: [PATCH 13/13] fixed Formatter issue --- src/banner.rs | 3 +-- src/main.rs | 15 +++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/banner.rs b/src/banner.rs index 7bb04b0..2107eba 100644 --- a/src/banner.rs +++ b/src/banner.rs @@ -7,7 +7,6 @@ use crate::{ use anyhow::{bail, Result}; use console::{style, Emoji}; use reqwest::{Client, Url}; -use serde::export::Formatter; use serde_json::Value; use std::fmt::{self, Display}; use std::io::Write; @@ -71,7 +70,7 @@ impl BannerEntry { /// Display implementation for a banner entry impl Display for BannerEntry { /// Display formatter for the given banner entry - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!( f, "\u{0020}{:\u{0020} Result<()> { if !CONFIGURATION.quiet { // only print banner if -q isn't used let std_stderr = stderr(); // std::io::stderr - let mut bnr = banner::Banner::new(&targets, &CONFIGURATION); - // only interested in the side-effect - let _ = bnr - .check_for_updates(&CONFIGURATION.client, banner::UPDATE_URL, tx_stats.clone()) + + let mut banner = Banner::new(&targets, &CONFIGURATION); + + // only interested in the side-effect that sets banner.update_status + let _ = banner + .check_for_updates(&CONFIGURATION.client, UPDATE_URL, tx_stats.clone()) .await; - bnr.print_to(std_stderr, &CONFIGURATION)?; + + banner.print_to(std_stderr, &CONFIGURATION)?; } // discard non-responsive targets