diff --git a/src/heuristics.rs b/src/heuristics.rs index c29b942..be407bc 100644 --- a/src/heuristics.rs +++ b/src/heuristics.rs @@ -1,63 +1,109 @@ use crate::{ - config::{CONFIGURATION, PROGRESS_PRINTER}, + config::{Configuration, CONFIGURATION, PROGRESS_PRINTER}, event_handlers::{Command, Handles}, filters::WildcardFilter, + skip_fail, utils::{ferox_print, format_url, get_url_path_length, make_request, status_colorizer}, FeroxResponse, }; -use anyhow::Result; +use anyhow::{bail, Result}; use console::style; -use indicatif::ProgressBar; +use reqwest::Client; use std::sync::Arc; -use tokio::sync::mpsc::UnboundedSender; use uuid::Uuid; /// length of a standard UUID, used when determining wildcard responses const UUID_LENGTH: u64 = 32; -/// Simple helper to return a uuid, formatted as lowercase without hyphens -/// -/// `length` determines the number of uuids to string together. Each uuid -/// is 32 characters long. So, a length of 1 return a 32 character string, -/// a length of 2 returns a 64 character string, and so on... -fn unique_string(length: usize) -> String { - log::trace!("enter: unique_string({})", length); - let mut ids = vec![]; - - for _ in 0..length { - ids.push(Uuid::new_v4().to_simple().to_string()); - } - - let unique_id = ids.join(""); - - log::trace!("exit: unique_string -> {}", unique_id); - unique_id +/// wrapper around ugly string formatting +macro_rules! format_template { + ($template:expr, $length:expr) => { + format!( + $template, + status_colorizer("WLD"), + "-", + "-", + "-", + style("auto-filtering").yellow(), + style($length).cyan(), + style("--dont-filter").yellow() + ) + }; } -/// Tests the given url to see if it issues a wildcard response -/// -/// In the event that url returns a wildcard response, a -/// [WildcardFilter](struct.WildcardFilter.html) is created and returned to the caller. -pub async fn wildcard_test( - target_url: &str, - bar: ProgressBar, +/// container for heuristics related info +pub struct HeuristicTests<'a> { + /// Handles object for event handler interaction handles: Arc, -) -> Result<()> { - log::trace!( - "enter: wildcard_test({:?}, {:?}, {:?})", - target_url, - bar, - handles, - ); - if CONFIGURATION.dont_filter { - // early return, dont_filter scans don't need tested - log::trace!("exit: wildcard_test -> None"); - return Ok(()); + dont_filter: bool, + + quiet: bool, + + add_slash: bool, + + client: &'a Client, + + queries: &'a Vec<(String, String)>, +} + +/// HeuristicTests implementation +impl<'a> HeuristicTests<'a> { + /// create a new HeuristicTests struct + pub fn new(handles: Arc, config: &'a Configuration) -> Self { + Self { + handles, + dont_filter: config.dont_filter, + quiet: config.quiet, + add_slash: config.add_slash, + client: &config.client, + queries: &config.queries, + } } - if let Some(ferox_response) = make_wildcard_request(&target_url, 1, handles.clone()).await { - bar.inc(1); + /// Simple helper to return a uuid, formatted as lowercase without hyphens + /// + /// `length` determines the number of uuids to string together. Each uuid + /// is 32 characters long. So, a length of 1 return a 32 character string, + /// a length of 2 returns a 64 character string, and so on... + fn unique_string(&self, length: usize) -> String { + log::trace!("enter: unique_string({})", length); + let mut ids = vec![]; + + for _ in 0..length { + ids.push(Uuid::new_v4().to_simple().to_string()); + } + + let unique_id = ids.join(""); + + log::trace!("exit: unique_string -> {}", unique_id); + unique_id + } + + /// wrapper for sending a filter to the filters event handler + fn send_filter(&self, filter: WildcardFilter) -> Result<()> { + self.handles + .filters + .send(Command::AddFilter(Box::new(filter))) + } + + /// Tests the given url to see if it issues a wildcard response + /// + /// In the event that url returns a wildcard response, a + /// [WildcardFilter](struct.WildcardFilter.html) is created and sent to the filters event + /// handler. + /// + /// Returns the number of times to increment the caller's progress bar + pub async fn wildcard(&self, target_url: &str) -> Result { + log::trace!("enter: wildcard_test({:?})", target_url); + + if self.dont_filter { + // early return, dont_filter scans don't need tested + log::trace!("exit: wildcard_test -> 0"); + return Ok(0); + } + + let ferox_response = self.make_wildcard_request(&target_url, 1).await?; // found a wildcard response let mut wildcard = WildcardFilter::default(); @@ -65,212 +111,149 @@ pub async fn wildcard_test( let wc_length = ferox_response.content_length(); if wc_length == 0 { - log::trace!("exit: wildcard_test -> Some({:?})", wildcard); - handles - .filters - .send(Command::AddFilter(Box::new(wildcard)))?; - return Ok(()); + log::trace!("exit: wildcard_test -> 1"); + self.send_filter(wildcard)?; + return Ok(1); } // content length of wildcard is non-zero, perform additional tests: // make a second request, with a known-sized (64) longer request - if let Some(resp_two) = make_wildcard_request(&target_url, 3, handles.clone()).await { - bar.inc(1); + let resp_two = self.make_wildcard_request(&target_url, 3).await?; - let wc2_length = resp_two.content_length(); + let wc2_length = resp_two.content_length(); - if wc2_length == wc_length + (UUID_LENGTH * 2) { - // second length is what we'd expect to see if the requested url is - // reflected in the response along with some static content; aka custom 404 - let url_len = get_url_path_length(&ferox_response.url()); + if wc2_length == wc_length + (UUID_LENGTH * 2) { + // second length is what we'd expect to see if the requested url is + // reflected in the response along with some static content; aka custom 404 + let url_len = get_url_path_length(&ferox_response.url()); - wildcard.dynamic = wc_length - url_len; + wildcard.dynamic = wc_length - url_len; - if !CONFIGURATION.quiet { - let msg = format!( - "{} {:>9} {:>9} {:>9} Wildcard response is dynamic; {} ({} + url length) responses; toggle this behavior by using {}\n", - status_colorizer("WLD"), - "-", - "-", - "-", - style("auto-filtering").yellow(), - style(wc_length - url_len).cyan(), - style("--dont-filter").yellow() - ); - - ferox_print(&msg, &PROGRESS_PRINTER); - } - } else if wc_length == wc2_length { - wildcard.size = wc_length; - - if !CONFIGURATION.quiet { - let msg = format!( - "{} {:>9} {:>9} {:>9} Wildcard response is static; {} {} responses; toggle this behavior by using {}\n", - status_colorizer("WLD"), - "-", - "-", - "-", - style("auto-filtering").yellow(), - style(wc_length).cyan(), - style("--dont-filter").yellow() - ); - - ferox_print(&msg, &PROGRESS_PRINTER); - } + if !self.quiet { + let msg = format_template!("{} {:>9} {:>9} {:>9} Wildcard response is dynamic; {} ({} + url length) responses; toggle this behavior by using {}\n", wildcard.dynamic); + ferox_print(&msg, &PROGRESS_PRINTER); } - } else { - bar.inc(2); - } + } else if wc_length == wc2_length { + wildcard.size = wc_length; - log::trace!("exit: wildcard_test -> Some({:?})", wildcard); - handles - .filters - .send(Command::AddFilter(Box::new(wildcard)))?; - return Ok(()); - } - - log::trace!("exit: wildcard_test -> None"); - Ok(()) -} - -/// Generates a uuid and appends it to the given target url. The reasoning is that the randomly -/// generated unique string should not exist on and be served by the target web server. -/// -/// Once the unique url is created, the request is sent to the server. If the server responds -/// back with a valid status code, the response is considered to be a wildcard response. If that -/// wildcard response has a 3xx status code, that redirection location is displayed to the user. -async fn make_wildcard_request( - target_url: &str, - length: usize, - handles: Arc, -) -> Option { - log::trace!( - "enter: make_wildcard_request({}, {}, {:?})", - target_url, - length, - handles - ); - - let unique_str = unique_string(length); - - let nonexistent = match format_url( - target_url, - &unique_str, - CONFIGURATION.add_slash, - &CONFIGURATION.queries, - None, - handles.stats.tx.clone(), - ) { - Ok(url) => url, - Err(e) => { - log::error!("{}", e); - log::trace!("exit: make_wildcard_request -> None"); - return None; - } - }; - - match make_request( - &CONFIGURATION.client, - &nonexistent.to_owned(), - handles.stats.tx.clone(), - ) - .await - { - Ok(response) => { - if CONFIGURATION - .status_codes - .contains(&response.status().as_u16()) - { - // found a wildcard response - let mut ferox_response = FeroxResponse::from(response, true).await; - ferox_response.wildcard = true; - - if !CONFIGURATION.quiet - && !handles - .filters - .data - .should_filter_response(&ferox_response, handles.stats.tx.clone()) - && handles - .output - .send(Command::Report(Box::new(ferox_response.clone()))) - .is_err() - { - // abusing short-circuiting to protect the terminal send behind - // not quiet and shouldn't filter out the response - return None; - } - - log::trace!("exit: make_wildcard_request -> {}", ferox_response); - return Some(ferox_response); + if !self.quiet { + let msg = format_template!("{} {:>9} {:>9} {:>9} Wildcard response is static; {} {} responses; toggle this behavior by using {}\n", wildcard.size); + ferox_print(&msg, &PROGRESS_PRINTER); } } - Err(e) => { - log::warn!("{}", e); - log::trace!("exit: make_wildcard_request -> None"); - return None; - } + + self.send_filter(wildcard)?; + + log::trace!("exit: wildcard_test"); + Ok(2) } - log::trace!("exit: make_wildcard_request -> None"); - None -} + /// Generates a uuid and appends it to the given target url. The reasoning is that the randomly + /// generated unique string should not exist on and be served by the target web server. + /// + /// Once the unique url is created, the request is sent to the server. If the server responds + /// back with a valid status code, the response is considered to be a wildcard response. If that + /// wildcard response has a 3xx status code, that redirection location is displayed to the user. + async fn make_wildcard_request( + &self, + target_url: &str, + length: usize, + ) -> Result { + log::trace!("enter: make_wildcard_request({}, {})", target_url, length); -/// Simply tries to connect to all given sites before starting to scan -/// -/// In the event that no sites can be reached, the program will exit. -/// -/// Any urls that are found to be alive are returned to the caller. -pub async fn connectivity_test( - target_urls: &[String], - tx_stats: UnboundedSender, -) -> Vec { - log::trace!( - "enter: connectivity_test({:?}, {:?})", - target_urls, - tx_stats - ); + let unique_str = self.unique_string(length); - let mut good_urls = vec![]; - - for target_url in target_urls { - let request = match format_url( + let nonexistent_url = format_url( target_url, - "", - CONFIGURATION.add_slash, - &CONFIGURATION.queries, + &unique_str, + self.add_slash, + &self.queries, None, - tx_stats.clone(), - ) { - Ok(url) => url, - Err(e) => { - log::error!("{}", e); - continue; - } - }; + self.handles.stats.tx.clone(), + )?; - match make_request(&CONFIGURATION.client, &request, tx_stats.clone()).await { - Ok(_) => { - good_urls.push(target_url.to_owned()); + let response = make_request( + &self.client, + &nonexistent_url.to_owned(), + self.handles.stats.tx.clone(), + ) + .await?; + + if CONFIGURATION + .status_codes + .contains(&response.status().as_u16()) + { + // found a wildcard response + let mut ferox_response = FeroxResponse::from(response, true).await; + ferox_response.wildcard = true; + + if self + .handles + .filters + .data + .should_filter_response(&ferox_response, self.handles.stats.tx.clone()) + { + bail!("filtered response") } - Err(e) => { - if !CONFIGURATION.quiet { - ferox_print( - &format!("Could not connect to {}, skipping...", target_url), - &PROGRESS_PRINTER, - ); + + if !self.quiet { + let boxed = Box::new(ferox_response.clone()); + self.handles.output.send(Command::Report(boxed))?; + } + + log::trace!("exit: make_wildcard_request -> {}", ferox_response); + return Ok(ferox_response); + } + + log::trace!("exit: make_wildcard_request -> Err"); + bail!("uninteresting status code") + } + + /// Simply tries to connect to all given sites before starting to scan + /// + /// In the event that no sites can be reached, the program will exit. + /// + /// Any urls that are found to be alive are returned to the caller. + pub async fn connectivity(&self, target_urls: &[String]) -> Result> { + log::trace!("enter: connectivity_test({:?})", target_urls); + + let mut good_urls = vec![]; + + for target_url in target_urls { + let request = skip_fail!(format_url( + target_url, + "", + self.add_slash, + &self.queries, + None, + self.handles.stats.tx.clone(), + )); + + let result = make_request(&self.client, &request, self.handles.stats.tx.clone()).await; + + match result { + Ok(_) => { + good_urls.push(target_url.to_owned()); + } + Err(e) => { + if !CONFIGURATION.quiet { + ferox_print( + &format!("Could not connect to {}, skipping...", target_url), + &PROGRESS_PRINTER, + ); + } + log::error!("{}", e); } - log::error!("{}", e); } } + + if good_urls.is_empty() { + bail!("Could not connect to any target provided"); + } + + log::trace!("exit: connectivity_test -> {:?}", good_urls); + Ok(good_urls) } - - if good_urls.is_empty() { - log::error!("Could not connect to any target provided, exiting."); - } - - log::trace!("exit: connectivity_test -> {:?}", good_urls); - - good_urls } #[cfg(test)] @@ -280,8 +263,10 @@ mod tests { #[test] /// request a unique string of 32bytes * a value returns correct result fn heuristics_unique_string_returns_correct_length() { + let (handles, _) = Handles::for_testing(None); + let tester = HeuristicTests::new(Arc::new(handles), &CONFIGURATION); for i in 0..10 { - assert_eq!(unique_string(i).len(), i * 32); + assert_eq!(tester.unique_string(i).len(), i * 32); } } } diff --git a/src/main.rs b/src/main.rs index 2cafc78..1ac856c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -286,7 +286,15 @@ async fn wrapped_main() -> Result<()> { } // discard non-responsive targets - let live_targets = heuristics::connectivity_test(&targets, handles.stats.tx.clone()).await; + let live_targets = { + let test = heuristics::HeuristicTests::new(handles.clone(), &CONFIGURATION); + let result = test.connectivity(&targets).await; + if result.is_err() { + clean_up(handles, tasks).await?; + bail!(fmt_err(&result.unwrap_err().to_string())); + } + result? + }; if live_targets.is_empty() { clean_up(handles, tasks).await?; diff --git a/src/scanner.rs b/src/scanner.rs index b598581..6fb6f31 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -220,10 +220,14 @@ pub async fn scan_url( let permit = SCAN_LIMITER.acquire().await; // Arc clones to be passed around to the various scans - let wildcard_bar = progress_bar.clone(); let looping_words = wordlist.clone(); - heuristics::wildcard_test(&target_url, wildcard_bar, handles.clone()).await?; + { + let test = heuristics::HeuristicTests::new(handles.clone(), &CONFIGURATION); + if let Ok(num_reqs) = test.wildcard(&target_url).await { + progress_bar.inc(num_reqs); + } + } // producer tasks (mp of mpsc); responsible for making requests let producers = stream::iter(looping_words.deref().to_owned()) diff --git a/src/utils.rs b/src/utils.rs index 1fbfb81..da98cb6 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -173,6 +173,19 @@ macro_rules! send_command { }; } +#[macro_export] +macro_rules! skip_fail { + ($res:expr) => { + match $res { + Ok(val) => val, + Err(e) => { + log::warn!("An error: {}; skipped.", e); + continue; + } + } + }; +} + /// Simple helper to generate a `Url` /// /// Errors during parsing `url` or joining `word` are propagated up the call stack diff --git a/tests/test_banner.rs b/tests/test_banner.rs index 2dc0a3e..6c2b390 100644 --- a/tests/test_banner.rs +++ b/tests/test_banner.rs @@ -619,7 +619,7 @@ fn banner_doesnt_print() -> Result<(), Box> { .assert() .success() .stderr(predicate::str::contains( - "Could not find any live targets to scan", + "Could not connect to any target provided", )); Ok(()) }