From 584fc940cd97af2ef104694ace2f98afba31366f Mon Sep 17 00:00:00 2001 From: epi Date: Mon, 6 Mar 2023 20:44:14 -0600 Subject: [PATCH 01/10] implemented fix for wildcard directories --- Cargo.lock | 2 +- Cargo.toml | 2 +- shell_completions/_feroxbuster | 4 +- shell_completions/_feroxbuster.ps1 | 4 +- shell_completions/feroxbuster.elv | 4 +- src/event_handlers/outputs.rs | 2 +- src/event_handlers/scans.rs | 54 ++++++++- src/extractor/container.rs | 2 +- src/filters/container.rs | 22 +++- src/filters/mod.rs | 2 + src/filters/wildcard.rs | 180 +++++++++++++++++++++++++++++ src/heuristics.rs | 179 +++++++++++++++++++--------- src/scan_manager/scan_container.rs | 6 +- src/scanner/ferox_scanner.rs | 17 ++- src/scanner/requester.rs | 2 +- src/traits.rs | 5 +- 16 files changed, 412 insertions(+), 75 deletions(-) create mode 100644 src/filters/wildcard.rs diff --git a/Cargo.lock b/Cargo.lock index 977b5c0..be63ad4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -761,7 +761,7 @@ dependencies = [ [[package]] name = "feroxbuster" -version = "2.8.0" +version = "2.8.1" dependencies = [ "anyhow", "assert_cmd", diff --git a/Cargo.toml b/Cargo.toml index 0a4c9fd..6455a1f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "feroxbuster" -version = "2.8.0" +version = "2.8.1" authors = ["Ben 'epi' Risher (@epi052)"] license = "MIT" edition = "2021" diff --git a/shell_completions/_feroxbuster b/shell_completions/_feroxbuster index 7049945..fb86f81 100644 --- a/shell_completions/_feroxbuster +++ b/shell_completions/_feroxbuster @@ -24,8 +24,8 @@ _feroxbuster() { '--replay-proxy=[Send only unfiltered requests through a Replay Proxy, instead of all requests]:REPLAY_PROXY:_urls' \ '*-R+[Status Codes to send through a Replay Proxy when found (default: --status-codes value)]:REPLAY_CODE: ' \ '*--replay-codes=[Status Codes to send through a Replay Proxy when found (default: --status-codes value)]:REPLAY_CODE: ' \ -'-a+[Sets the User-Agent (default: feroxbuster/2.8.0)]:USER_AGENT: ' \ -'--user-agent=[Sets the User-Agent (default: feroxbuster/2.8.0)]:USER_AGENT: ' \ +'-a+[Sets the User-Agent (default: feroxbuster/2.8.1)]:USER_AGENT: ' \ +'--user-agent=[Sets the User-Agent (default: feroxbuster/2.8.1)]:USER_AGENT: ' \ '*-x+[File extension(s) to search for (ex: -x php -x pdf js)]:FILE_EXTENSION: ' \ '*--extensions=[File extension(s) to search for (ex: -x php -x pdf js)]:FILE_EXTENSION: ' \ '*-m+[Which HTTP request method(s) should be sent (default: GET)]:HTTP_METHODS: ' \ diff --git a/shell_completions/_feroxbuster.ps1 b/shell_completions/_feroxbuster.ps1 index 9c52c88..f461d70 100644 --- a/shell_completions/_feroxbuster.ps1 +++ b/shell_completions/_feroxbuster.ps1 @@ -30,8 +30,8 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock { [CompletionResult]::new('--replay-proxy', 'replay-proxy', [CompletionResultType]::ParameterName, 'Send only unfiltered requests through a Replay Proxy, instead of all requests') [CompletionResult]::new('-R', 'R', [CompletionResultType]::ParameterName, 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)') [CompletionResult]::new('--replay-codes', 'replay-codes', [CompletionResultType]::ParameterName, 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)') - [CompletionResult]::new('-a', 'a', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.8.0)') - [CompletionResult]::new('--user-agent', 'user-agent', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.8.0)') + [CompletionResult]::new('-a', 'a', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.8.1)') + [CompletionResult]::new('--user-agent', 'user-agent', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.8.1)') [CompletionResult]::new('-x', 'x', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js)') [CompletionResult]::new('--extensions', 'extensions', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js)') [CompletionResult]::new('-m', 'm', [CompletionResultType]::ParameterName, 'Which HTTP request method(s) should be sent (default: GET)') diff --git a/shell_completions/feroxbuster.elv b/shell_completions/feroxbuster.elv index 7d67ebb..e569124 100644 --- a/shell_completions/feroxbuster.elv +++ b/shell_completions/feroxbuster.elv @@ -27,8 +27,8 @@ set edit:completion:arg-completer[feroxbuster] = {|@words| cand --replay-proxy 'Send only unfiltered requests through a Replay Proxy, instead of all requests' cand -R 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)' cand --replay-codes 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)' - cand -a 'Sets the User-Agent (default: feroxbuster/2.8.0)' - cand --user-agent 'Sets the User-Agent (default: feroxbuster/2.8.0)' + cand -a 'Sets the User-Agent (default: feroxbuster/2.8.1)' + cand --user-agent 'Sets the User-Agent (default: feroxbuster/2.8.1)' cand -x 'File extension(s) to search for (ex: -x php -x pdf js)' cand --extensions 'File extension(s) to search for (ex: -x php -x pdf js)' cand -m 'Which HTTP request method(s) should be sent (default: GET)' diff --git a/src/event_handlers/outputs.rs b/src/event_handlers/outputs.rs index 0c58b6e..e6f19ef 100644 --- a/src/event_handlers/outputs.rs +++ b/src/event_handlers/outputs.rs @@ -248,7 +248,7 @@ impl TermOutHandler { .unwrap() .filters .data - .should_filter_response(&resp); + .should_filter_response(&resp, tx_stats.clone()); let contains_sentry = if !self.config.filter_status.is_empty() { // -C was used, meaning -s was not and we should ignore the defaults diff --git a/src/event_handlers/scans.rs b/src/event_handlers/scans.rs index 8329c20..9b18fe4 100644 --- a/src/event_handlers/scans.rs +++ b/src/event_handlers/scans.rs @@ -6,7 +6,7 @@ use tokio::sync::{mpsc, Semaphore}; use crate::{ response::FeroxResponse, scan_manager::{FeroxScan, FeroxScans, ScanOrder}, - scanner::FeroxScanner, + scanner::{FeroxScanner, RESPONSES}, statistics::StatField::TotalScans, url::FeroxUrl, utils::should_deny_url, @@ -395,6 +395,58 @@ impl ScanHandler { return Ok(()); } + if let Ok(responses) = RESPONSES.responses.read() { + for maybe_wild in responses.iter() { + if !maybe_wild.wildcard() || !maybe_wild.is_directory() { + // if the stored response isn't a wildcard, skip it + // if the stored response isn't a directory, skip it + // we're only interested in preventing recursion into wildcard directories + continue; + } + + if maybe_wild.method() != response.method() { + // methods don't match, skip it + continue; + } + + // methods match and is a directory wildcard + // need to check the wildcard's parent directory + // for equality with the incoming response's parent directory + // + // if the parent directories match, we need to prevent recursion + // into the wildcard directory + + match ( + maybe_wild.url().path_segments(), + response.url().path_segments(), + ) { + // both urls must have path segments + (Some(mut maybe_wild_segments), Some(mut response_segments)) => { + match ( + maybe_wild_segments.nth_back(1), + response_segments.nth_back(1), + ) { + // both urls must have at least 2 path segments, the next to last being the parent + (Some(maybe_wild_parent), Some(response_parent)) => { + if maybe_wild_parent == response_parent { + // the parent directories match, so we need to prevent recursion + return Ok(()); + } + } + _ => { + // we couldn't get the parent directory, so we'll skip this + continue; + } + } + } + _ => { + // we couldn't get the path segments, so we'll skip this + continue; + } + } + } + } + let targets = vec![response.url().to_string()]; self.ordered_scan_url(targets, ScanOrder::Latest).await?; diff --git a/src/extractor/container.rs b/src/extractor/container.rs index e7f5e1f..291bd2e 100644 --- a/src/extractor/container.rs +++ b/src/extractor/container.rs @@ -144,7 +144,7 @@ impl<'a> Extractor<'a> { }; // filter if necessary - if self.handles.filters.data.should_filter_response(&resp) { + if self.handles.filters.data.should_filter_response(&resp, self.handles.stats.tx.clone()) { continue; } diff --git a/src/filters/container.rs b/src/filters/container.rs index 7788bf8..f28e16d 100644 --- a/src/filters/container.rs +++ b/src/filters/container.rs @@ -7,9 +7,12 @@ use crate::response::FeroxResponse; use super::{ FeroxFilter, LinesFilter, RegexFilter, SimilarityFilter, SizeFilter, StatusCodeFilter, - WordsFilter, + WildcardFilter, WordsFilter, +}; +use crate::{ + event_handlers::Command::AddToUsizeField, statistics::StatField::WildcardsFiltered, + CommandSender, }; - /// Container around a collection of `FeroxFilters`s #[derive(Debug, Default)] pub struct FeroxFilters { @@ -64,12 +67,21 @@ impl FeroxFilters { /// Simple helper to stay DRY; determines whether or not a given `FeroxResponse` should be reported /// to the user or not. - pub fn should_filter_response(&self, response: &FeroxResponse) -> bool { + pub fn should_filter_response( + &self, + response: &FeroxResponse, + tx_stats: CommandSender, + ) -> bool { if let Ok(filters) = self.filters.read() { for filter in filters.iter() { // wildcard.should_filter goes here if filter.should_filter_response(response) { log::debug!("filtering response due to: {:?}", filter); + if filter.as_any().downcast_ref::().is_some() { + tx_stats + .send(AddToUsizeField(WildcardsFiltered, 1)) + .unwrap_or_default(); + } return true; } } @@ -93,6 +105,10 @@ impl Serialize for FeroxFilters { seq.serialize_element(word_filter).unwrap_or_default(); } else if let Some(size_filter) = filter.as_any().downcast_ref::() { seq.serialize_element(size_filter).unwrap_or_default(); + } else if let Some(wildcard_filter) = + filter.as_any().downcast_ref::() + { + seq.serialize_element(wildcard_filter).unwrap_or_default(); } else if let Some(status_filter) = filter.as_any().downcast_ref::() { diff --git a/src/filters/mod.rs b/src/filters/mod.rs index 44fa127..d6582d0 100644 --- a/src/filters/mod.rs +++ b/src/filters/mod.rs @@ -15,6 +15,7 @@ pub use self::similarity::{SimilarityFilter, SIM_HASHER}; pub use self::size::SizeFilter; pub use self::status_code::StatusCodeFilter; pub(crate) use self::utils::{create_similarity_filter, filter_lookup}; +pub use self::wildcard::WildcardFilter; pub use self::words::WordsFilter; mod status_code; @@ -28,4 +29,5 @@ mod container; mod tests; mod init; mod utils; +mod wildcard; mod empty; diff --git a/src/filters/wildcard.rs b/src/filters/wildcard.rs new file mode 100644 index 0000000..7a54d16 --- /dev/null +++ b/src/filters/wildcard.rs @@ -0,0 +1,180 @@ +use console::style; + +use super::*; +use crate::utils::create_report_string; +use crate::{config::OutputLevel, DEFAULT_METHOD}; + +/// Data holder for all relevant data needed when auto-filtering out wildcard responses +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct WildcardFilter { + /// The content-length of this response, if known + pub content_length: Option, + + /// The number of lines contained in the body of this response, if known + pub line_count: Option, + + /// The number of words contained in the body of this response, if known + pub word_count: Option, + + /// method used in request that should be included with filters passed via runtime configuration + pub method: String, + + /// the status code returned in the response + pub status_code: u16, + + /// whether or not the user passed -D on the command line + pub dont_filter: bool, +} + +/// implementation of WildcardFilter +impl WildcardFilter { + /// given a boolean representing whether -D was used or not, create a new WildcardFilter + pub fn new(dont_filter: bool) -> Self { + Self { + dont_filter, + ..Default::default() + } + } +} + +/// implement default that populates `method` with its default value +impl Default for WildcardFilter { + fn default() -> Self { + Self { + content_length: None, + line_count: None, + word_count: None, + method: DEFAULT_METHOD.to_string(), + status_code: 0, + dont_filter: false, + } + } +} + +/// implementation of FeroxFilter for WildcardFilter +impl FeroxFilter for WildcardFilter { + /// Examine size/words/lines and method to determine whether or not the response received + /// is a wildcard response and therefore should be filtered out + fn should_filter_response(&self, response: &FeroxResponse) -> bool { + log::trace!("enter: should_filter_response({:?} {})", self, response); + + // quick return if dont_filter is set + if self.dont_filter { + // --dont-filter applies specifically to wildcard filters, it is not a 100% catch all + // for not filtering anything. As such, it should live in the implementation of + // a wildcard filter + return false; + } + + if self.method != response.method().as_str() { + // method's don't match, so this response should not be filtered out + log::trace!("exit: should_filter_response -> false"); + return false; + } + + if self.status_code != response.status().as_u16() { + // status codes don't match, so this response should not be filtered out + log::trace!("exit: should_filter_response -> false"); + return false; + } + + // methods and status codes match at this point, just need to check the other fields + + match (self.content_length, self.word_count, self.line_count) { + (Some(cl), Some(wc), Some(lc)) => { + if cl == response.content_length() + && wc == response.word_count() + && lc == response.line_count() + { + log::debug!("filtered out {}", response.url()); + log::trace!("exit: should_filter_response -> true"); + return true; + } + } + (Some(cl), Some(wc), None) => { + if cl == response.content_length() && wc == response.word_count() { + log::debug!("filtered out {}", response.url()); + log::trace!("exit: should_filter_response -> true"); + return true; + } + } + (Some(cl), None, Some(lc)) => { + if cl == response.content_length() && lc == response.line_count() { + log::debug!("filtered out {}", response.url()); + log::trace!("exit: should_filter_response -> true"); + return true; + } + } + (None, Some(wc), Some(lc)) => { + if wc == response.word_count() && lc == response.line_count() { + log::debug!("filtered out {}", response.url()); + log::trace!("exit: should_filter_response -> true"); + return true; + } + } + (Some(cl), None, None) => { + if cl == response.content_length() { + log::debug!("filtered out {}", response.url()); + log::trace!("exit: should_filter_response -> true"); + return true; + } + } + (None, Some(wc), None) => { + if wc == response.word_count() { + log::debug!("filtered out {}", response.url()); + log::trace!("exit: should_filter_response -> true"); + return true; + } + } + (None, None, Some(lc)) => { + if lc == response.line_count() { + log::debug!("filtered out {}", response.url()); + log::trace!("exit: should_filter_response -> true"); + return true; + } + } + (None, None, None) => { + unreachable!("wildcard filter without any filters set"); + } + } + + log::trace!("exit: should_filter_response -> false"); + false + } + + /// Compare one WildcardFilter to another + fn box_eq(&self, other: &dyn Any) -> bool { + other.downcast_ref::().map_or(false, |a| self == a) + } + + /// Return self as Any for dynamic dispatch purposes + fn as_any(&self) -> &dyn Any { + self + } +} + +impl std::fmt::Display for WildcardFilter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let msg = create_report_string( + self.status_code.to_string().as_str(), + self.method.as_str(), + &self + .line_count + .map_or_else(|| "-".to_string(), |x| x.to_string()), + &self + .word_count + .map_or_else(|| "-".to_string(), |x| x.to_string()), + &self + .content_length + .map_or_else(|| "-".to_string(), |x| x.to_string()), + &format!( + "{} found {}-like response and created new filter; toggle off with {}", + style("Auto-filtering").bright().green(), + style("404").red(), + style("--dont-filter").yellow() + ), + OutputLevel::Default, + ); + write!(f, "{}", msg) + } +} diff --git a/src/heuristics.rs b/src/heuristics.rs index 66d6275..ee23c47 100644 --- a/src/heuristics.rs +++ b/src/heuristics.rs @@ -1,22 +1,21 @@ use std::sync::Arc; use anyhow::{bail, Result}; -use console::style; use scraper::{Html, Selector}; use uuid::Uuid; -use crate::filters::{SimilarityFilter, SIM_HASHER}; +use crate::filters::{SimilarityFilter, WildcardFilter, SIM_HASHER}; use crate::message::FeroxMessage; use crate::nlp::preprocess; +use crate::scanner::RESPONSES; use crate::{ config::OutputLevel, event_handlers::{Command, Handles}, - filters::{LinesFilter, SizeFilter, WordsFilter}, progress::PROGRESS_PRINTER, response::FeroxResponse, skip_fail, url::FeroxUrl, - utils::{ferox_print, fmt_err, logged_request, status_colorizer}, + utils::{ferox_print, fmt_err, logged_request}, DEFAULT_METHOD, }; @@ -50,6 +49,16 @@ pub struct DirListingResult { pub response: FeroxResponse, } +/// wrapper around the results of running a wildcard detection against a target web page +#[derive(Copy, Debug, Clone)] +pub enum WildcardResult { + /// variant that represents a wildcard directory + WildcardDirectory(usize), + + /// variant that represents the presence of a 404-like response + FourOhFourLike(usize), +} + /// container for heuristics related info pub struct HeuristicTests { /// Handles object for event handler interaction @@ -240,13 +249,16 @@ impl HeuristicTests { /// given a target's base url, attempt to automatically detect its 404 response /// pattern(s), and then set filters that will exclude those patterns from future /// responses - pub async fn detect_404_like_responses(&self, target_url: &str) -> Result { + pub async fn detect_404_like_responses( + &self, + target_url: &str, + ) -> Result> { log::trace!("enter: detect_404_like_responses({:?})", target_url); if self.handles.config.dont_filter { // early return, dont_filter scans don't need tested log::trace!("exit: detect_404_like_responses -> dont_filter is true"); - return Ok(0); + return Ok(None); } let mut req_counter = 0; @@ -323,28 +335,41 @@ impl HeuristicTests { } // Command::AddFilter, &str (bytes/words/lines), usize (i.e. length associated with the type) - let Some((command, filter_type, filter_length)) = self.examine_404_like_responses(&responses) else { + let Some(filter) = self.examine_404_like_responses(&responses) else { // no match was found during analysis of responses responses.clear(); continue; }; - // check whether we already know about this filter - match command { - Command::AddFilter(ref filter) => { - if let Ok(guard) = self.handles.filters.data.filters.read() { - if guard.contains(filter) { - // match was found, but already known; clear the vec and continue to the next - responses.clear(); - continue; + // report to the user, if appropriate + if matches!( + self.handles.config.output_level, + OutputLevel::Default | OutputLevel::Quiet + ) { + // sentry value to control whether or not to print the filter + // used because we only want to print the same filter once + let mut print_sentry = true; + + if let Ok(filters) = self.handles.filters.data.filters.read() { + for other in filters.iter() { + if let Some(other_wildcard) = + other.as_any().downcast_ref::() + { + if &*filter == other_wildcard { + print_sentry = false; + break; + } } } } - _ => unreachable!(), + + if print_sentry { + ferox_print(&format!("{}", filter), &PROGRESS_PRINTER); + } } // create the new filter - self.handles.filters.send(command)?; + self.handles.filters.send(Command::AddFilter(filter))?; // if we're here, we've detected a 404-like response pattern, and we're already filtering for size/word/line // @@ -360,22 +385,35 @@ impl HeuristicTests { .filters .send(Command::AddFilter(Box::new(sim_filter)))?; + if responses[0].is_directory() { + // response is either a 3XX with a Location header that matches url + '/' + // or it's a 2XX that ends with a '/' + // or it's a 403 that ends with a '/' + + // set the wildcard flag to true, so we can check it when preventing + // recursion in event_handlers/scans.rs + responses[0].set_wildcard(true); + + // add the response to the global list of responses + RESPONSES.insert(responses[0].clone()); + + // function-internal magic number, indicates that we've detected a wildcard directory + req_counter += 100; + } + // reset the responses for the next method, if it exists responses.clear(); - - // report to the user, if appropriate - if matches!( - self.handles.config.output_level, - OutputLevel::Default | OutputLevel::Quiet - ) { - let msg = format!("{} {:>8} {:>9} {:>9} {:>9} {} => {} {}-like response ({} {}); toggle this behavior by using {}\n", status_colorizer("WLD"), "-", "-", "-", "-", style(target_url).cyan(), style("auto-filtering").bright().green(), style("404").red(), style(filter_length).cyan(), filter_type, style("--dont-filter").yellow()); - ferox_print(&msg, &PROGRESS_PRINTER); - } } log::trace!("exit: detect_404_like_responses"); - Ok(req_counter) + let retval = if req_counter > 100 { + WildcardResult::WildcardDirectory(req_counter) + } else { + WildcardResult::FourOhFourLike(req_counter) + }; + + Ok(Some(retval)) } /// for all responses, examine chars/words/lines @@ -386,11 +424,13 @@ impl HeuristicTests { fn examine_404_like_responses( &self, responses: &[FeroxResponse], - ) -> Option<(Command, &'static str, usize)> { + ) -> Option> { let mut size_sentry = true; let mut word_sentry = true; let mut line_sentry = true; + let method = responses[0].method(); + let status_code = responses[0].status(); let content_length = responses[0].content_length(); let word_count = responses[0].word_count(); let line_count = responses[0].line_count(); @@ -411,36 +451,61 @@ impl HeuristicTests { } } - // the if/else-if/else nature of the block means that we'll get the most - // specific match, if one is to be had - // - // each block returns the information needed to send the filter away and - // display a message to the user - if size_sentry { - // - command to send to the filters handler - // - the unit-type we're filtering on (bytes/words/lines) - // - the value associated with the unit-type on which we're filtering - Some(( - Command::AddFilter(Box::new(SizeFilter { content_length })), - "bytes", - content_length as usize, - )) - } else if word_sentry { - Some(( - Command::AddFilter(Box::new(WordsFilter { word_count })), - "words", - word_count, - )) - } else if line_sentry { - Some(( - Command::AddFilter(Box::new(LinesFilter { line_count })), - "lines", - line_count, - )) - } else { - // no match was found; clear the vec and continue to the next - None + if !size_sentry && !word_sentry && !line_sentry { + // none of the response lengths match, so we can't filter on any of them + return None; } + + let mut wildcard = WildcardFilter { + content_length: None, + line_count: None, + word_count: None, + method: method.to_string(), + status_code: status_code.as_u16(), + dont_filter: self.handles.config.dont_filter, + }; + + match (size_sentry, word_sentry, line_sentry) { + (true, true, true) => { + // all three types of length match, so we can't filter on any of them + wildcard.content_length = Some(content_length); + wildcard.word_count = Some(word_count); + wildcard.line_count = Some(line_count); + } + (true, true, false) => { + // content length and word count match, so we can filter on either + wildcard.content_length = Some(content_length); + wildcard.word_count = Some(word_count); + } + (true, false, true) => { + // content length and line count match, so we can filter on either + wildcard.content_length = Some(content_length); + wildcard.line_count = Some(line_count); + } + (false, true, true) => { + // word count and line count match, so we can filter on either + wildcard.word_count = Some(word_count); + wildcard.line_count = Some(line_count); + } + (true, false, false) => { + // content length matches, so we can filter on that + wildcard.content_length = Some(content_length); + } + (false, true, false) => { + // word count matches, so we can filter on that + wildcard.word_count = Some(word_count); + } + (false, false, true) => { + // line count matches, so we can filter on that + wildcard.line_count = Some(line_count); + } + (false, false, false) => { + // none of the length types match, so we can't filter on any of them + unreachable!("no wildcard size matches; handled by the if statement above"); + } + }; + + Some(Box::new(wildcard)) } } diff --git a/src/scan_manager/scan_container.rs b/src/scan_manager/scan_container.rs index 290e22c..d3118bb 100644 --- a/src/scan_manager/scan_container.rs +++ b/src/scan_manager/scan_container.rs @@ -3,7 +3,7 @@ use super::*; use crate::event_handlers::Handles; use crate::filters::{ EmptyFilter, LinesFilter, RegexFilter, SimilarityFilter, SizeFilter, StatusCodeFilter, - WordsFilter, + WildcardFilter, WordsFilter, }; use crate::traits::FeroxFilter; use crate::Command::AddFilter; @@ -182,6 +182,10 @@ impl FeroxScans { serde_json::from_value::(filter.clone()) { Box::new(deserialized) + } else if let Ok(deserialized) = + serde_json::from_value::(filter.clone()) + { + Box::new(deserialized) } else if let Ok(deserialized) = serde_json::from_value::(filter.clone()) { diff --git a/src/scanner/ferox_scanner.rs b/src/scanner/ferox_scanner.rs index 5e884e6..3c68479 100644 --- a/src/scanner/ferox_scanner.rs +++ b/src/scanner/ferox_scanner.rs @@ -10,6 +10,7 @@ use lazy_static::lazy_static; use tokio::sync::Semaphore; use crate::filters::{create_similarity_filter, EmptyFilter, SimilarityFilter}; +use crate::heuristics::WildcardResult; use crate::Command::AddFilter; use crate::{ event_handlers::{ @@ -303,7 +304,21 @@ impl FeroxScanner { // wildcard test let num_reqs_made = test.detect_404_like_responses(&self.target_url).await?; - progress_bar.inc(num_reqs_made); + match num_reqs_made { + Some(WildcardResult::WildcardDirectory(num_reqs)) => { + let message = format!( + "=> {} dir! {} recursion", + style("Wildcard").blue().bright(), + style("stopped").red() + ); + progress_bar.set_message(&message); + progress_bar.inc(num_reqs as u64); + } + Some(WildcardResult::FourOhFourLike(num_reqs)) => { + progress_bar.inc(num_reqs as u64); + } + _ => {} + } } // Arc clones to be passed around to the various scans diff --git a/src/scanner/requester.rs b/src/scanner/requester.rs index bc69a8c..3822a11 100644 --- a/src/scanner/requester.rs +++ b/src/scanner/requester.rs @@ -437,7 +437,7 @@ impl Requester { .handles .filters .data - .should_filter_response(&ferox_response) + .should_filter_response(&ferox_response, self.handles.stats.tx.clone()) { continue; } diff --git a/src/traits.rs b/src/traits.rs index c742778..e285931 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -1,6 +1,7 @@ //! collection of all traits used use crate::filters::{ - LinesFilter, RegexFilter, SimilarityFilter, SizeFilter, StatusCodeFilter, WordsFilter, + LinesFilter, RegexFilter, SimilarityFilter, SizeFilter, StatusCodeFilter, WildcardFilter, + WordsFilter, }; use crate::response::FeroxResponse; use anyhow::Result; @@ -36,6 +37,8 @@ impl Display for dyn FeroxFilter { write!(f, "Response size: {}", style(filter.content_length).cyan()) } else if let Some(filter) = self.as_any().downcast_ref::() { write!(f, "Regex: {}", style(&filter.raw_string).cyan()) + } else if let Some(filter) = self.as_any().downcast_ref::() { + write!(f, "{}", filter) // real Display in the wildcard module } else if let Some(filter) = self.as_any().downcast_ref::() { write!(f, "Status code: {}", style(filter.filter_code).cyan()) } else if let Some(filter) = self.as_any().downcast_ref::() { From 31cdba64e413a118afdf2a81a3368c853153ea8f Mon Sep 17 00:00:00 2001 From: epi Date: Mon, 6 Mar 2023 20:44:24 -0600 Subject: [PATCH 02/10] fmt --- src/extractor/container.rs | 7 ++++++- src/heuristics.rs | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/extractor/container.rs b/src/extractor/container.rs index 291bd2e..a1b123c 100644 --- a/src/extractor/container.rs +++ b/src/extractor/container.rs @@ -144,7 +144,12 @@ impl<'a> Extractor<'a> { }; // filter if necessary - if self.handles.filters.data.should_filter_response(&resp, self.handles.stats.tx.clone()) { + if self + .handles + .filters + .data + .should_filter_response(&resp, self.handles.stats.tx.clone()) + { continue; } diff --git a/src/heuristics.rs b/src/heuristics.rs index ee23c47..ca7958c 100644 --- a/src/heuristics.rs +++ b/src/heuristics.rs @@ -397,7 +397,7 @@ impl HeuristicTests { // add the response to the global list of responses RESPONSES.insert(responses[0].clone()); - // function-internal magic number, indicates that we've detected a wildcard directory + // function-internal magic number, indicates that we've detected a wildcard directory req_counter += 100; } From ad0df8ccd3fa63dcb2bbe9b0c9176ada5982eece Mon Sep 17 00:00:00 2001 From: epi Date: Tue, 7 Mar 2023 06:00:00 -0600 Subject: [PATCH 03/10] updated deps --- Cargo.lock | 161 +++++++++++++++------------------ Cargo.toml | 12 +-- shell_completions/_feroxbuster | 6 +- 3 files changed, 84 insertions(+), 95 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index be63ad4..76c3cec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -123,12 +123,11 @@ dependencies = [ [[package]] name = "async-lock" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8101efe8695a6c17e02911402145357e718ac92d3ff88ae8419e84b1707b685" +checksum = "fa24f727524730b077666307f2734b4a1a1c57acb79193127dcc8914d5242dd7" dependencies = [ "event-listener", - "futures-lite", ] [[package]] @@ -193,9 +192,9 @@ checksum = "7a40729d2133846d9ed0ea60a8b9541bccddab49cd30f0715a1da672fe9a2524" [[package]] name = "async-trait" -version = "0.1.64" +version = "0.1.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd7fce9ba8c3c042128ce72d8b2ddbf3a05747efb67ea0313c635e10bda47a2" +checksum = "b84f9ebcc6c1f5b8cb160f6990096a5c127f423fcb6e1ccc46c370cbdfb75dfc" dependencies = [ "proc-macro2", "quote", @@ -342,9 +341,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clap" -version = "4.1.6" +version = "4.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0b0588d44d4d63a87dbd75c136c166bbfd9a86a31cb89e09906521c7d3f5e3" +checksum = "c3d7ae14b20b94cb02149ed21a86c423859cbe18dc7ed69845cace50e52b40a5" dependencies = [ "bitflags", "clap_lex", @@ -357,9 +356,9 @@ dependencies = [ [[package]] name = "clap_complete" -version = "4.1.3" +version = "4.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0012995dc3a54314f4710f5631d74767e73c534b8757221708303e48eef7a19b" +checksum = "501ff0a401473ea1d4c3b125ff95506b62c5bc5768d818634195fbb7c4ad5ff4" dependencies = [ "clap", ] @@ -428,9 +427,9 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.6" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" +checksum = "cf2b3e8478797446514c91ef04bafcb59faba183e621ad488df88983cc14128c" dependencies = [ "cfg-if", "crossbeam-utils", @@ -438,9 +437,9 @@ dependencies = [ [[package]] name = "crossbeam-deque" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" dependencies = [ "cfg-if", "crossbeam-epoch", @@ -449,9 +448,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.13" +version = "0.9.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a" +checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695" dependencies = [ "autocfg", "cfg-if", @@ -462,9 +461,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.14" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f" +checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" dependencies = [ "cfg-if", ] @@ -502,15 +501,15 @@ checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" [[package]] name = "cssparser" -version = "0.27.2" +version = "0.29.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "754b69d351cdc2d8ee09ae203db831e005560fc6030da058f86ad60c92a9cb0a" +checksum = "f93d03419cb5950ccfd3daf3ff1c7a36ace64609a1a8746d493df1ca0afde0fa" dependencies = [ "cssparser-macros", "dtoa-short", - "itoa 0.4.8", + "itoa", "matches", - "phf 0.8.0", + "phf 0.10.1", "proc-macro2", "quote", "smallvec", @@ -1047,9 +1046,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.15" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f9f29bc9dda355256b2916cf526ab02ce0aeaaaf2bad60d65ef3f12f11dd0f4" +checksum = "5be7b54589b581f624f566bf5d8eb2bab1db736c51528720b6bd36b96b55924d" dependencies = [ "bytes", "fnv", @@ -1116,7 +1115,7 @@ checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" dependencies = [ "bytes", "fnv", - "itoa 1.0.5", + "itoa", ] [[package]] @@ -1191,7 +1190,7 @@ dependencies = [ "http-body", "httparse", "httpdate", - "itoa 1.0.5", + "itoa", "pin-project-lite", "socket2", "tokio", @@ -1256,9 +1255,9 @@ dependencies = [ [[package]] name = "io-lifetimes" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1abeb7a0dd0f8181267ff8adc397075586500b81b28a73e8a0208b00fc170fb3" +checksum = "cfa919a82ea574332e2de6e74b4c36e74d41982b335080fa59d4ef31be20fdf3" dependencies = [ "libc", "windows-sys 0.45.0", @@ -1320,15 +1319,9 @@ dependencies = [ [[package]] name = "itoa" -version = "0.4.8" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" - -[[package]] -name = "itoa" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" +checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" [[package]] name = "js-sys" @@ -1491,9 +1484,9 @@ checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "memoffset" -version = "0.7.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" +checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" dependencies = [ "autocfg", ] @@ -1635,9 +1628,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-src" -version = "111.25.0+1.1.1t" +version = "111.25.1+1.1.1t" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3173cd3626c43e3854b1b727422a276e568d9ec5fe8cec197822cf52cfb743d6" +checksum = "1ef9a9cc6ea7d9d5e7c4a913dc4b48d0e359eddf01af1dfec96ba7064b4aba10" dependencies = [ "cc", ] @@ -1713,9 +1706,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12" dependencies = [ - "phf_macros", "phf_shared 0.8.0", - "proc-macro-hack", ] [[package]] @@ -1724,7 +1715,9 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fabbf1ead8a5bcbc20f5f8b939ee3f5b0f6f281b6ad3468b84656b658b455259" dependencies = [ + "phf_macros", "phf_shared 0.10.0", + "proc-macro-hack", ] [[package]] @@ -1769,12 +1762,12 @@ dependencies = [ [[package]] name = "phf_macros" -version = "0.8.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f6fde18ff429ffc8fe78e2bf7f8b7a5a5a6e2a8b58bc5a9ac69198bbda9189c" +checksum = "58fdf3184dd560f160dd73922bea2d5cd6e8f064bf4b13110abd81b03697b4e0" dependencies = [ - "phf_generator 0.8.0", - "phf_shared 0.8.0", + "phf_generator 0.10.0", + "phf_shared 0.10.0", "proc-macro-hack", "proc-macro2", "quote", @@ -2062,9 +2055,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7" +checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" dependencies = [ "either", "rayon-core", @@ -2072,9 +2065,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.10.2" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "356a0625f1954f730c0201cdab48611198dc6ce21f4acff55089b5a78e6e835b" +checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" dependencies = [ "crossbeam-channel", "crossbeam-deque", @@ -2192,9 +2185,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.36.8" +version = "0.36.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43abb88211988493c1abb44a70efa56ff0ce98f233b7b276146f1f3f7ba9644" +checksum = "fd5c6ff11fecd55b40746d1995a02f2eb375bf8c00d192d521ee09f42bef37bc" dependencies = [ "bitflags", "errno", @@ -2206,15 +2199,15 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5583e89e108996506031660fe09baa5011b9dd0341b89029313006d1fb508d70" +checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" [[package]] name = "ryu" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" +checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" [[package]] name = "schannel" @@ -2233,9 +2226,9 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "scraper" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7cb4dae083699a22a65aa9d2699c27f525e35dffaec38b10801e958ed4cf27" +checksum = "4c557a9a03db98b0b298b497f0e16cd35a04a1fa9ee1130a6889c0714e0b73df" dependencies = [ "cssparser", "ego-tree", @@ -2278,22 +2271,20 @@ dependencies = [ [[package]] name = "selectors" -version = "0.22.0" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df320f1889ac4ba6bc0cdc9c9af7af4bd64bb927bccdf32d81140dc1f9be12fe" +checksum = "0c37578180969d00692904465fb7f6b3d50b9a2b952b87c23d0e2e5cb5013416" dependencies = [ "bitflags", "cssparser", "derive_more", "fxhash", "log", - "matches", "phf 0.8.0", "phf_codegen 0.8.0", "precomputed-hash", "servo_arc", "smallvec", - "thin-slice", ] [[package]] @@ -2324,11 +2315,11 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.93" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cad406b69c91885b5107daf2c29572f6c8cdb3c66826821e286c533490c0bc76" +checksum = "1c533a59c9d8a93a09c6ab31f0fd5e5f4dd1b8fc9434804029839884765d04ea" dependencies = [ - "itoa 1.0.5", + "itoa", "ryu", "serde", ] @@ -2359,16 +2350,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" dependencies = [ "form_urlencoded", - "itoa 1.0.5", + "itoa", "ryu", "serde", ] [[package]] name = "servo_arc" -version = "0.1.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d98238b800e0d1576d8b6e3de32827c2d74bee68bb97748dcf5071fb53965432" +checksum = "d52aa42f8fdf0fed91e5ce7f23d8138441002fa31dca008acf47e6fd4721f741" dependencies = [ "nodrop", "stable_deref_trait", @@ -2463,9 +2454,9 @@ checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" [[package]] name = "socket2" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" dependencies = [ "libc", "winapi", @@ -2485,9 +2476,9 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "string_cache" -version = "0.8.4" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213494b7a2b503146286049378ce02b482200519accc31872ee8be91fa820a08" +checksum = "7d69e88b23f23030bf4d0e9ca7b07434f70e1c1f4d3ca7e93ce958b373654d9f" dependencies = [ "new_debug_unreachable", "once_cell", @@ -2586,26 +2577,20 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95059e91184749cb66be6dc994f67f182b6d897cb3df74a5bf66b5e709295fd8" -[[package]] -name = "thin-slice" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eaa81235c7058867fa8c0e7314f33dcce9c215f535d1913822a2b3f5e289f3c" - [[package]] name = "thiserror" -version = "1.0.38" +version = "1.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0" +checksum = "a5ab016db510546d856297882807df8da66a16fb8c4101cb8b30054b0d5b2d9c" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.38" +version = "1.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" +checksum = "5420d42e90af0c38c3290abcca25b9b3bdf379fc9f55c528f53a269d9c9a267e" dependencies = [ "proc-macro2", "quote", @@ -2638,9 +2623,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.25.0" +version = "1.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e00990ebabbe4c14c08aca901caed183ecd5c09562a12c824bb53d3c3fd3af" +checksum = "03201d01c3c27a29c8a5cee5b55a93ddae1ccf6f08f65365c2c918f8c1b76f64" dependencies = [ "autocfg", "bytes", @@ -2653,7 +2638,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys 0.42.0", + "windows-sys 0.45.0", ] [[package]] @@ -2816,9 +2801,9 @@ checksum = "d54675592c1dbefd78cbd98db9bacd89886e1ca50692a0692baefffdeb92dd58" [[package]] name = "unicode-ident" -version = "1.0.6" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" +checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" [[package]] name = "unicode-normalization" @@ -3126,9 +3111,9 @@ checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" [[package]] name = "winnow" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faf09497b8f8b5ac5d3bb4d05c0a99be20f26fd3d5f2db7b0716e946d5103658" +checksum = "ee7b2c67f962bf5042bfd8b6a916178df33a26eec343ae064cb8e069f638fa6f" dependencies = [ "memchr", ] diff --git a/Cargo.toml b/Cargo.toml index 6455a1f..9b3dbdb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,16 +22,16 @@ build = "build.rs" maintenance = { status = "actively-developed" } [build-dependencies] -clap = { version = "4.1.6", features = ["wrap_help", "cargo"] } -clap_complete = "4.1.3" +clap = { version = "4.1.8", features = ["wrap_help", "cargo"] } +clap_complete = "4.1.4" regex = "1.5.5" lazy_static = "1.4.0" dirs = "4.0.0" [dependencies] -scraper = "0.14.0" +scraper = "0.15.0" futures = "0.3.26" -tokio = { version = "1.25.0", features = ["full"] } +tokio = { version = "1.26.0", features = ["full"] } tokio-util = { version = "0.7.7", features = ["codec"] } log = "0.4.17" env_logger = "0.10.0" @@ -39,11 +39,11 @@ reqwest = { version = "0.11.10", features = ["socks"] } # uses feature unification to add 'serde' to reqwest::Url url = { version = "2.2.2", features = ["serde"] } serde_regex = "1.1.0" -clap = { version = "4.1.6", features = ["wrap_help", "cargo"] } +clap = { version = "4.1.8", features = ["wrap_help", "cargo"] } lazy_static = "1.4.0" toml = "0.7.2" serde = { version = "1.0.137", features = ["derive", "rc"] } -serde_json = "1.0.93" +serde_json = "1.0.94" uuid = { version = "1.3.0", features = ["v4"] } indicatif = "0.15" console = "0.15.2" diff --git a/shell_completions/_feroxbuster b/shell_completions/_feroxbuster index fb86f81..0b40efb 100644 --- a/shell_completions/_feroxbuster +++ b/shell_completions/_feroxbuster @@ -117,4 +117,8 @@ _feroxbuster_commands() { _describe -t commands 'feroxbuster commands' commands "$@" } -_feroxbuster "$@" +if [ "$funcstack[1]" = "_feroxbuster" ]; then + _feroxbuster "$@" +else + compdef _feroxbuster feroxbuster +fi From 8dd8871ae5ba74771b3e7de565c3507254f7bf09 Mon Sep 17 00:00:00 2001 From: epi Date: Tue, 7 Mar 2023 06:27:24 -0600 Subject: [PATCH 04/10] old tests pass --- tests/test_heuristics.rs | 20 ++++++++++++-------- tests/test_scanner.rs | 7 ++++--- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/tests/test_heuristics.rs b/tests/test_heuristics.rs index fa35dd6..6696a03 100644 --- a/tests/test_heuristics.rs +++ b/tests/test_heuristics.rs @@ -180,9 +180,12 @@ fn test_static_wildcard_request_found() -> Result<(), Box teardown_tmp_directory(tmp_dir); cmd.assert().success().stdout( - predicate::str::contains("WLD").and(predicate::str::contains( - "auto-filtering 404-like response (1 lines);", - )), + predicate::str::contains("GET") + .and(predicate::str::contains( + "Auto-filtering found 404-like response and created new filter", + )) + .and(predicate::str::contains("200")) + .and(predicate::str::contains("1l")), ); assert_eq!(mock.hits(), 1); @@ -273,14 +276,14 @@ fn heuristics_wildcard_test_with_two_static_wildcards_with_silent_enabled( let mock = srv.mock(|when, then| { when.method(GET) - .path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap()); + .path_matches(Regex::new("/.?[a-zA-Z0-9]{32,}").unwrap()); then.status(200) .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"); }); let mock2 = srv.mock(|when, then| { when.method(GET) - .path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap()); + .path_matches(Regex::new("/LICENSE").unwrap()); then.status(200) .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"); }); @@ -291,7 +294,6 @@ fn heuristics_wildcard_test_with_two_static_wildcards_with_silent_enabled( .arg(srv.url("/")) .arg("--wordlist") .arg(file.as_os_str()) - .arg("--add-slash") .arg("--silent") .arg("--threads") .arg("1") @@ -299,9 +301,11 @@ fn heuristics_wildcard_test_with_two_static_wildcards_with_silent_enabled( teardown_tmp_directory(tmp_dir); - cmd.assert().success().stdout(predicate::str::is_empty()); + cmd.assert() + .success() + .stdout(predicate::str::contains(srv.url("/"))); - assert_eq!(mock.hits(), 1); + assert_eq!(mock.hits(), 4); assert_eq!(mock2.hits(), 1); Ok(()) } diff --git a/tests/test_scanner.rs b/tests/test_scanner.rs index 4fcd5c1..626838c 100644 --- a/tests/test_scanner.rs +++ b/tests/test_scanner.rs @@ -573,7 +573,7 @@ fn scanner_recursion_works_with_403_directories() { let found_anyway = srv.mock(|when, then| { when.method(GET).path("/ignored/LICENSE"); then.status(200) - .body("this is a test\nThat rugf really tied the room together"); + .body("this is a test\nThat rug really tied the room together"); }); let cmd = Command::cargo_bin("feroxbuster") @@ -588,9 +588,10 @@ fn scanner_recursion_works_with_403_directories() { predicate::str::contains("/LICENSE") .count(2) .and(predicate::str::contains("200")) - .and(predicate::str::contains("WLD")) + .and(predicate::str::contains("404")) + .and(predicate::str::contains("53c Auto-filtering")) .and(predicate::str::contains( - "auto-filtering 404-like response (53 bytes);", + "Auto-filtering found 404-like response and created new filter;", )) .and(predicate::str::contains("14c")) .and(predicate::str::contains("0c")) From 94aafccf8a0e7e287da0ef969a44d82b6fe218d0 Mon Sep 17 00:00:00 2001 From: epi Date: Tue, 7 Mar 2023 06:30:53 -0600 Subject: [PATCH 05/10] bumped version --- Cargo.lock | 2 +- Cargo.toml | 2 +- shell_completions/_feroxbuster | 4 ++-- shell_completions/_feroxbuster.ps1 | 4 ++-- shell_completions/feroxbuster.elv | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 76c3cec..228d4c5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -760,7 +760,7 @@ dependencies = [ [[package]] name = "feroxbuster" -version = "2.8.1" +version = "2.9.0" dependencies = [ "anyhow", "assert_cmd", diff --git a/Cargo.toml b/Cargo.toml index 9b3dbdb..8d540e4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "feroxbuster" -version = "2.8.1" +version = "2.9.0" authors = ["Ben 'epi' Risher (@epi052)"] license = "MIT" edition = "2021" diff --git a/shell_completions/_feroxbuster b/shell_completions/_feroxbuster index 0b40efb..5abc70d 100644 --- a/shell_completions/_feroxbuster +++ b/shell_completions/_feroxbuster @@ -24,8 +24,8 @@ _feroxbuster() { '--replay-proxy=[Send only unfiltered requests through a Replay Proxy, instead of all requests]:REPLAY_PROXY:_urls' \ '*-R+[Status Codes to send through a Replay Proxy when found (default: --status-codes value)]:REPLAY_CODE: ' \ '*--replay-codes=[Status Codes to send through a Replay Proxy when found (default: --status-codes value)]:REPLAY_CODE: ' \ -'-a+[Sets the User-Agent (default: feroxbuster/2.8.1)]:USER_AGENT: ' \ -'--user-agent=[Sets the User-Agent (default: feroxbuster/2.8.1)]:USER_AGENT: ' \ +'-a+[Sets the User-Agent (default: feroxbuster/2.9.0)]:USER_AGENT: ' \ +'--user-agent=[Sets the User-Agent (default: feroxbuster/2.9.0)]:USER_AGENT: ' \ '*-x+[File extension(s) to search for (ex: -x php -x pdf js)]:FILE_EXTENSION: ' \ '*--extensions=[File extension(s) to search for (ex: -x php -x pdf js)]:FILE_EXTENSION: ' \ '*-m+[Which HTTP request method(s) should be sent (default: GET)]:HTTP_METHODS: ' \ diff --git a/shell_completions/_feroxbuster.ps1 b/shell_completions/_feroxbuster.ps1 index f461d70..26191be 100644 --- a/shell_completions/_feroxbuster.ps1 +++ b/shell_completions/_feroxbuster.ps1 @@ -30,8 +30,8 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock { [CompletionResult]::new('--replay-proxy', 'replay-proxy', [CompletionResultType]::ParameterName, 'Send only unfiltered requests through a Replay Proxy, instead of all requests') [CompletionResult]::new('-R', 'R', [CompletionResultType]::ParameterName, 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)') [CompletionResult]::new('--replay-codes', 'replay-codes', [CompletionResultType]::ParameterName, 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)') - [CompletionResult]::new('-a', 'a', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.8.1)') - [CompletionResult]::new('--user-agent', 'user-agent', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.8.1)') + [CompletionResult]::new('-a', 'a', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.9.0)') + [CompletionResult]::new('--user-agent', 'user-agent', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.9.0)') [CompletionResult]::new('-x', 'x', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js)') [CompletionResult]::new('--extensions', 'extensions', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js)') [CompletionResult]::new('-m', 'm', [CompletionResultType]::ParameterName, 'Which HTTP request method(s) should be sent (default: GET)') diff --git a/shell_completions/feroxbuster.elv b/shell_completions/feroxbuster.elv index e569124..3ed01c3 100644 --- a/shell_completions/feroxbuster.elv +++ b/shell_completions/feroxbuster.elv @@ -27,8 +27,8 @@ set edit:completion:arg-completer[feroxbuster] = {|@words| cand --replay-proxy 'Send only unfiltered requests through a Replay Proxy, instead of all requests' cand -R 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)' cand --replay-codes 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)' - cand -a 'Sets the User-Agent (default: feroxbuster/2.8.1)' - cand --user-agent 'Sets the User-Agent (default: feroxbuster/2.8.1)' + cand -a 'Sets the User-Agent (default: feroxbuster/2.9.0)' + cand --user-agent 'Sets the User-Agent (default: feroxbuster/2.9.0)' cand -x 'File extension(s) to search for (ex: -x php -x pdf js)' cand --extensions 'File extension(s) to search for (ex: -x php -x pdf js)' cand -m 'Which HTTP request method(s) should be sent (default: GET)' From 6d01bc8ec4a3311bf9b91e100100485615238383 Mon Sep 17 00:00:00 2001 From: epi Date: Tue, 7 Mar 2023 18:38:10 -0600 Subject: [PATCH 06/10] added a few tests taht were removed previously --- src/filters/tests.rs | 81 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/src/filters/tests.rs b/src/filters/tests.rs index 4c5eb03..1c82c69 100644 --- a/src/filters/tests.rs +++ b/src/filters/tests.rs @@ -1,7 +1,41 @@ use super::*; use crate::nlp::preprocess; +use crate::DEFAULT_METHOD; use ::regex::Regex; +#[test] +/// simply test the default values for wildcardfilter +fn wildcard_filter_default() { + let wcf = WildcardFilter::default(); + assert_eq!(wcf.content_length, None); + assert_eq!(wcf.line_count, None); + assert_eq!(wcf.word_count, None); + assert_eq!(wcf.method, DEFAULT_METHOD.to_string()); + assert_eq!(wcf.status_code, 0); + assert_eq!(wcf.dont_filter, false); +} + +#[test] +/// just a simple test to increase code coverage by hitting as_any and the inner value +fn wildcard_filter_as_any() { + let mut filter = WildcardFilter::default(); + let filter2 = WildcardFilter::default(); + + assert!(filter.box_eq(filter2.as_any())); + + assert_eq!( + *filter.as_any().downcast_ref::().unwrap(), + filter2 + ); + + filter.content_length = Some(1); + + assert_ne!( + *filter.as_any().downcast_ref::().unwrap(), + filter2 + ); +} + #[test] /// just a simple test to increase code coverage by hitting as_any and the inner value fn lines_filter_as_any() { @@ -86,6 +120,53 @@ fn regex_filter_as_any() { ); } +#[test] +/// test should_filter on WilcardFilter where static logic matches +fn wildcard_should_filter_when_static_wildcard_found() { + let body = + "pellentesque diam volutpat commodo sed egestas egestas fringilla phasellus faucibus"; + + let mut resp = FeroxResponse::default(); + resp.set_wildcard(true); + resp.set_url("http://localhost"); + resp.set_text(body); + + let mut filter = WildcardFilter::default(); + filter.content_length = Some(body.len() as u64); + filter.status_code = 200; + + assert!(filter.should_filter_response(&resp)); +} + +#[test] +/// test should_filter on WilcardFilter where static logic matches but response length is 0 +fn wildcard_should_filter_when_static_wildcard_len_is_zero() { + let mut resp = FeroxResponse::default(); + resp.set_wildcard(true); + resp.set_url("http://localhost"); + + // default WildcardFilter is used in the code that executes when response.content_length() == 0 + let mut filter = WildcardFilter::default(); + filter.content_length = Some(0); + filter.status_code = 200; + + assert!(filter.should_filter_response(&resp)); +} + +#[test] +/// test should_filter on WilcardFilter where dynamic logic matches +fn wildcard_should_filter_when_dynamic_wildcard_found() { + let mut resp = FeroxResponse::default(); + resp.set_wildcard(true); + resp.set_url("http://localhost/stuff"); + resp.set_text("pellentesque diam volutpat commodo sed egestas egestas fringilla"); + + let mut filter = WildcardFilter::default(); + filter.word_count = Some(8); + filter.status_code = 200; + + assert!(filter.should_filter_response(&resp)); +} #[test] /// test should_filter on RegexFilter where regex matches body fn regexfilter_should_filter_when_regex_matches_on_response_body() { From 291ccedba3de7bdf227ab44e63e2b9cb9dadda8e Mon Sep 17 00:00:00 2001 From: epi Date: Tue, 7 Mar 2023 18:54:32 -0600 Subject: [PATCH 07/10] clippy --- src/filters/tests.rs | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/src/filters/tests.rs b/src/filters/tests.rs index 1c82c69..01bb12e 100644 --- a/src/filters/tests.rs +++ b/src/filters/tests.rs @@ -12,7 +12,7 @@ fn wildcard_filter_default() { assert_eq!(wcf.word_count, None); assert_eq!(wcf.method, DEFAULT_METHOD.to_string()); assert_eq!(wcf.status_code, 0); - assert_eq!(wcf.dont_filter, false); + assert!(!wcf.dont_filter); } #[test] @@ -131,9 +131,14 @@ fn wildcard_should_filter_when_static_wildcard_found() { resp.set_url("http://localhost"); resp.set_text(body); - let mut filter = WildcardFilter::default(); - filter.content_length = Some(body.len() as u64); - filter.status_code = 200; + let filter = WildcardFilter { + content_length: Some(body.len() as u64), + line_count: Some(1), + word_count: Some(8), + method: DEFAULT_METHOD.to_string(), + status_code: 200, + dont_filter: false, + }; assert!(filter.should_filter_response(&resp)); } @@ -146,9 +151,14 @@ fn wildcard_should_filter_when_static_wildcard_len_is_zero() { resp.set_url("http://localhost"); // default WildcardFilter is used in the code that executes when response.content_length() == 0 - let mut filter = WildcardFilter::default(); - filter.content_length = Some(0); - filter.status_code = 200; + let filter = WildcardFilter { + content_length: Some(0), + line_count: Some(0), + word_count: Some(0), + method: DEFAULT_METHOD.to_string(), + status_code: 200, + dont_filter: false, + }; assert!(filter.should_filter_response(&resp)); } @@ -161,9 +171,14 @@ fn wildcard_should_filter_when_dynamic_wildcard_found() { resp.set_url("http://localhost/stuff"); resp.set_text("pellentesque diam volutpat commodo sed egestas egestas fringilla"); - let mut filter = WildcardFilter::default(); - filter.word_count = Some(8); - filter.status_code = 200; + let filter = WildcardFilter { + content_length: None, + line_count: None, + word_count: Some(8), + method: DEFAULT_METHOD.to_string(), + status_code: 200, + dont_filter: false, + }; assert!(filter.should_filter_response(&resp)); } From ca43a767d22b9ce2bba87520ea9b22f06f1d65c1 Mon Sep 17 00:00:00 2001 From: epi Date: Tue, 7 Mar 2023 20:15:10 -0600 Subject: [PATCH 08/10] fixed failing test --- Makefile.toml | 7 +++++++ src/filters/tests.rs | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Makefile.toml b/Makefile.toml index 60b23f9..d89245e 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -23,3 +23,10 @@ clear = true script = """ cargo clippy --all-targets --all-features -- -D warnings """ + +# tests +[tasks.test] +clear = true +script = """ +cargo nextest run --all-features --all-targets --retries 10 +""" diff --git a/src/filters/tests.rs b/src/filters/tests.rs index 01bb12e..9c5847a 100644 --- a/src/filters/tests.rs +++ b/src/filters/tests.rs @@ -134,7 +134,7 @@ fn wildcard_should_filter_when_static_wildcard_found() { let filter = WildcardFilter { content_length: Some(body.len() as u64), line_count: Some(1), - word_count: Some(8), + word_count: Some(10), method: DEFAULT_METHOD.to_string(), status_code: 200, dont_filter: false, From 8392f6d26b24bf0b53d1cd2fb861987a4ed8ddab Mon Sep 17 00:00:00 2001 From: epi Date: Tue, 7 Mar 2023 21:14:20 -0600 Subject: [PATCH 09/10] fixed menu filter display; fixed wildcard filter comparison --- src/filters/similarity.rs | 4 +++- src/traits.rs | 39 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/src/filters/similarity.rs b/src/filters/similarity.rs index ae931ee..1b3013e 100644 --- a/src/filters/similarity.rs +++ b/src/filters/similarity.rs @@ -37,7 +37,9 @@ impl FeroxFilter for SimilarityFilter { /// Compare one SimilarityFilter to another fn box_eq(&self, other: &dyn Any) -> bool { - other.downcast_ref::().map_or(false, |a| self == a) + other + .downcast_ref::() + .map_or(false, |a| self.hash == a.hash) } /// Return self as Any for dynamic dispatch purposes diff --git a/src/traits.rs b/src/traits.rs index e285931..77513c5 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -4,6 +4,7 @@ use crate::filters::{ WordsFilter, }; use crate::response::FeroxResponse; +use crate::utils::status_colorizer; use anyhow::Result; use crossterm::style::{style, Stylize}; use serde::Serialize; @@ -38,7 +39,43 @@ impl Display for dyn FeroxFilter { } else if let Some(filter) = self.as_any().downcast_ref::() { write!(f, "Regex: {}", style(&filter.raw_string).cyan()) } else if let Some(filter) = self.as_any().downcast_ref::() { - write!(f, "{}", filter) // real Display in the wildcard module + let mut msg = format!( + "{} requests with {} responses ", + style(&filter.method).cyan(), + status_colorizer(&filter.status_code.to_string()) + ); + + match (filter.content_length, filter.word_count, filter.line_count) { + (None, None, None) => { + unreachable!("wildcard filter without any filters set"); + } + (None, None, Some(lc)) => { + msg.push_str(&format!("containing {} lines", lc)); + } + (None, Some(wc), None) => { + msg.push_str(&format!("containing {} words", wc)); + } + (None, Some(wc), Some(lc)) => { + msg.push_str(&format!("containing {} words and {} lines", wc, lc)); + } + (Some(cl), None, None) => { + msg.push_str(&format!("containing {} bytes", cl)); + } + (Some(cl), None, Some(lc)) => { + msg.push_str(&format!("containing {} bytes and {} lines", cl, lc)); + } + (Some(cl), Some(wc), None) => { + msg.push_str(&format!("containing {} bytes and {} words", cl, wc)); + } + (Some(cl), Some(wc), Some(lc)) => { + msg.push_str(&format!( + "containing {} bytes, {} words, and {} lines", + cl, wc, lc + )); + } + } + + write!(f, "{}", msg) } else if let Some(filter) = self.as_any().downcast_ref::() { write!(f, "Status code: {}", style(filter.filter_code).cyan()) } else if let Some(filter) = self.as_any().downcast_ref::() { From 5e93da0a6512ad16c17b6ee8a50e05eef367bff7 Mon Sep 17 00:00:00 2001 From: epi Date: Wed, 8 Mar 2023 05:29:30 -0600 Subject: [PATCH 10/10] fixed #809; thorough/smart bypassed mutual exclusion --- shell_completions/_feroxbuster | 4 ++-- src/parser.rs | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/shell_completions/_feroxbuster b/shell_completions/_feroxbuster index 5abc70d..b742df4 100644 --- a/shell_completions/_feroxbuster +++ b/shell_completions/_feroxbuster @@ -72,8 +72,8 @@ _feroxbuster() { '(-u --url)--stdin[Read url(s) from STDIN]' \ '(-p --proxy -k --insecure --burp-replay)--burp[Set --proxy to http://127.0.0.1:8080 and set --insecure to true]' \ '(-P --replay-proxy -k --insecure)--burp-replay[Set --replay-proxy to http://127.0.0.1:8080 and set --insecure to true]' \ -'--smart[Set --extract-links, --auto-tune, --collect-words, and --collect-backups to true]' \ -'--thorough[Use the same settings as --smart and set --collect-extensions to true]' \ +'(--rate-limit --auto-bail)--smart[Set --extract-links, --auto-tune, --collect-words, and --collect-backups to true]' \ +'(--rate-limit --auto-bail)--thorough[Use the same settings as --smart and set --collect-extensions to true]' \ '-A[Use a random User-Agent]' \ '--random-agent[Use a random User-Agent]' \ '-f[Append / to each request'\''s URL]' \ diff --git a/src/parser.rs b/src/parser.rs index 2cc15ec..b610528 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -91,12 +91,14 @@ pub fn initialize() -> Command { .long("smart") .num_args(0) .help_heading("Composite settings") + .conflicts_with_all(["rate_limit", "auto_bail"]) .help("Set --extract-links, --auto-tune, --collect-words, and --collect-backups to true"), ).arg( Arg::new("thorough") .long("thorough") .num_args(0) .help_heading("Composite settings") + .conflicts_with_all(["rate_limit", "auto_bail"]) .help("Use the same settings as --smart and set --collect-extensions to true"), );