diff --git a/img/demo.gif b/img/demo.gif index 058928f..1b89c62 100644 Binary files a/img/demo.gif and b/img/demo.gif differ diff --git a/src/banner.rs b/src/banner.rs index 39f8e9f..7078c42 100644 --- a/src/banner.rs +++ b/src/banner.rs @@ -545,9 +545,6 @@ by Ben "epi" Risher {} ver: {}"#, .unwrap_or_default(); writeln!(&mut writer, "{}", addl_section).unwrap_or_default(); - // todo: this isn't printing properly anymore, feels like the totals bar is overwriting it - // writeln!(&mut writer, "{}", addl_section).unwrap_or_default(); - // writeln!(&mut writer, "{}", addl_section).unwrap_or_default(); } #[cfg(test)] diff --git a/src/extractor.rs b/src/extractor.rs index 6c3608d..71e3056 100644 --- a/src/extractor.rs +++ b/src/extractor.rs @@ -204,6 +204,7 @@ pub async fn request_feroxresponse_from_new_link( CONFIGURATION.add_slash, &CONFIGURATION.queries, None, + tx_stats.clone(), ) { Ok(url) => url, Err(_) => { diff --git a/src/heuristics.rs b/src/heuristics.rs index 75e71a7..5ef3991 100644 --- a/src/heuristics.rs +++ b/src/heuristics.rs @@ -2,10 +2,7 @@ use crate::{ config::{CONFIGURATION, PROGRESS_PRINTER}, filters::WildcardFilter, scanner::should_filter_response, - statistics::{ - StatCommand::{self, AddError}, - StatError::UrlFormat, - }, + statistics::StatCommand, utils::{ferox_print, format_url, get_url_path_length, make_request, status_colorizer}, FeroxResponse, }; @@ -168,6 +165,7 @@ async fn make_wildcard_request( CONFIGURATION.add_slash, &CONFIGURATION.queries, None, + tx_stats.clone(), ) { Ok(url) => url, Err(e) => { @@ -239,11 +237,10 @@ pub async fn connectivity_test( CONFIGURATION.add_slash, &CONFIGURATION.queries, None, + tx_stats.clone(), ) { Ok(url) => url, Err(e) => { - // todo this probably makes more sense inside format_url, similar to make_request - update_stat!(tx_stats, AddError(UrlFormat)); log::error!("{}", e); continue; } diff --git a/src/main.rs b/src/main.rs index cda1924..4842b77 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,15 +1,19 @@ use crossterm::event::{self, Event, KeyCode}; -use feroxbuster::progress::BarType; use feroxbuster::{ banner, config::{CONFIGURATION, PROGRESS_BAR, PROGRESS_PRINTER}, extractor::{extract_robots_txt, request_feroxresponse_from_new_link}, heuristics, logger, - progress::add_bar, + progress::{add_bar, BarType}, reporter, scan_manager::{self, PAUSE_SCAN}, scanner::{self, scan_url, send_report, RESPONSES, SCANNED_URLS}, - statistics::{self, StatCommand}, + statistics::{ + self, + StatCommand::{self, UpdateUsizeField}, + StatField::InitialTargets, + Stats, + }, update_stat, utils::{ferox_print, get_current_depth, module_colorizer, status_colorizer}, FeroxError, FeroxResponse, FeroxResult, FeroxSerialize, SLEEP_DURATION, VERSION, @@ -103,13 +107,15 @@ fn get_unique_words_from_wordlist(path: &str) -> FeroxResult /// Determine whether it's a single url scan or urls are coming from stdin, then scan as needed async fn scan( mut targets: Vec, + stats: Arc, tx_term: UnboundedSender, tx_file: UnboundedSender, tx_stats: UnboundedSender, ) -> FeroxResult<()> { log::trace!( - "enter: scan({:?}, {:?}, {:?}, {:?})", + "enter: scan({:?}, {:?}, {:?}, {:?}, {:?})", targets, + stats, tx_term, tx_file, tx_stats @@ -182,10 +188,10 @@ async fn scan( }; if ferox_response.is_file() { - SCANNED_URLS.add_file_scan(&robot_link); + SCANNED_URLS.add_file_scan(&robot_link, stats.clone()); send_report(tx_term.clone(), ferox_response); } else { - let (unknown, _) = SCANNED_URLS.add_directory_scan(&robot_link); + let (unknown, _) = SCANNED_URLS.add_directory_scan(&robot_link, stats.clone()); if !unknown { // known directory; can skip (unlikely) @@ -200,13 +206,13 @@ async fn scan( } let mut tasks = vec![]; - let num_targets = targets.len(); for target in targets { let word_clone = words.clone(); let term_clone = tx_term.clone(); let file_clone = tx_file.clone(); - let stats_clone = tx_stats.clone(); + let tx_stats_clone = tx_stats.clone(); + let stats_clone = stats.clone(); let task = tokio::spawn(async move { let base_depth = get_current_depth(&target); @@ -214,10 +220,10 @@ async fn scan( &target, word_clone, base_depth, - num_targets, + stats_clone, term_clone, file_clone, - stats_clone, + tx_stats_clone, ) .await; }); @@ -339,6 +345,8 @@ async fn wrapped_main() { } }; + update_stat!(tx_stats, UpdateUsizeField(InitialTargets, targets.len())); + if !CONFIGURATION.quiet { // only print banner if -q isn't used let std_stderr = stderr(); // std::io::stderr @@ -372,6 +380,7 @@ async fn wrapped_main() { // kick off a scan against any targets determined to be responsive match scan( live_targets, + stats, tx_term.clone(), tx_file.clone(), tx_stats.clone(), diff --git a/src/progress.rs b/src/progress.rs index 4fe5fd7..a3014e6 100644 --- a/src/progress.rs +++ b/src/progress.rs @@ -58,14 +58,16 @@ mod tests { let p1 = add_bar("prefix", 2, BarType::Hidden); // hidden let p2 = add_bar("prefix", 2, BarType::Message); // no per second field let p3 = add_bar("prefix", 2, BarType::Default); // normal bar - // todo add Total bartype to test + let p4 = add_bar("prefix", 2, BarType::Total); // totals bar p1.finish(); p2.finish(); p3.finish(); + p4.finish(); assert!(p1.is_finished()); assert!(p2.is_finished()); assert!(p3.is_finished()); + assert!(p4.is_finished()); } } diff --git a/src/reporter.rs b/src/reporter.rs index a6d90a2..bf23462 100644 --- a/src/reporter.rs +++ b/src/reporter.rs @@ -198,7 +198,6 @@ async fn spawn_file_reporter( safe_file_write(&response, buffered_file.clone(), CONFIGURATION.json); } - // todo if --summary was used, do this, else pass update_stat!(tx_stats, StatCommand::Save); log::trace!("exit: spawn_file_reporter"); diff --git a/src/scan_manager.rs b/src/scan_manager.rs index 97e1c8f..27c94b8 100644 --- a/src/scan_manager.rs +++ b/src/scan_manager.rs @@ -3,7 +3,7 @@ use crate::{ parser::TIMESPEC_REGEX, progress::{add_bar, BarType}, reporter::safe_file_write, - scanner::{NUMBER_OF_REQUESTS, RESPONSES, SCANNED_URLS}, + scanner::{RESPONSES, SCANNED_URLS}, statistics::Stats, utils::open_file, FeroxResponse, FeroxSerialize, SLEEP_DURATION, @@ -73,6 +73,9 @@ pub struct FeroxScan { /// The type of scan pub scan_type: ScanType, + /// Number of requests to populate the progress bar with + num_requests: u64, + /// Whether or not this scan has completed pub complete: bool, @@ -93,6 +96,7 @@ impl Default for FeroxScan { id: new_id, task: None, complete: false, + num_requests: 0, url: String::new(), progress_bar: None, scan_type: ScanType::File, @@ -123,8 +127,7 @@ impl FeroxScan { if let Some(pb) = &self.progress_bar { pb.clone() } else { - let num_requests = NUMBER_OF_REQUESTS.load(Ordering::Relaxed); - let pb = add_bar(&self.url, num_requests, BarType::Default); + let pb = add_bar(&self.url, self.num_requests, BarType::Default); pb.reset_elapsed(); @@ -135,10 +138,16 @@ impl FeroxScan { } /// Given a URL and ProgressBar, create a new FeroxScan, wrap it in an Arc and return it - pub fn new(url: &str, scan_type: ScanType, pb: Option) -> Arc> { + pub fn new( + url: &str, + scan_type: ScanType, + num_requests: u64, + pb: Option, + ) -> Arc> { Arc::new(Mutex::new(Self { url: url.to_string(), scan_type, + num_requests, progress_bar: pb, ..Default::default() })) @@ -435,12 +444,17 @@ impl FeroxScans { /// If `FeroxScans` did not already contain the scan, return true; otherwise return false /// /// Also return a reference to the new `FeroxScan` - fn add_scan(&self, url: &str, scan_type: ScanType) -> (bool, Arc>) { + fn add_scan( + &self, + url: &str, + scan_type: ScanType, + stats: Arc, + ) -> (bool, Arc>) { let bar = match scan_type { ScanType::Directory => { let progress_bar = add_bar( &url, - NUMBER_OF_REQUESTS.load(Ordering::Relaxed), + stats.expected_per_scan.load(Ordering::Relaxed) as u64, BarType::Default, ); @@ -451,7 +465,9 @@ impl FeroxScans { ScanType::File => None, }; - let ferox_scan = FeroxScan::new(&url, scan_type, bar); + let num_requests = stats.expected_per_scan.load(Ordering::Relaxed) as u64; + + let ferox_scan = FeroxScan::new(&url, scan_type, num_requests, bar); // If the set did not contain the scan, true is returned. // If the set did contain the scan, false is returned. @@ -465,8 +481,12 @@ impl FeroxScans { /// If `FeroxScans` did not already contain the scan, return true; otherwise return false /// /// Also return a reference to the new `FeroxScan` - pub fn add_directory_scan(&self, url: &str) -> (bool, Arc>) { - self.add_scan(&url, ScanType::Directory) + pub fn add_directory_scan( + &self, + url: &str, + stats: Arc, + ) -> (bool, Arc>) { + self.add_scan(&url, ScanType::Directory, stats) } /// Given a url, create a new `FeroxScan` and add it to `FeroxScans` as a File Scan @@ -474,8 +494,8 @@ impl FeroxScans { /// If `FeroxScans` did not already contain the scan, return true; otherwise return false /// /// Also return a reference to the new `FeroxScan` - pub fn add_file_scan(&self, url: &str) -> (bool, Arc>) { - self.add_scan(&url, ScanType::File) + pub fn add_file_scan(&self, url: &str, stats: Arc) -> (bool, Arc>) { + self.add_scan(&url, ScanType::File, stats) } } @@ -796,8 +816,9 @@ mod tests { /// add an unknown url to the hashset, expect true fn add_url_to_list_of_scanned_urls_with_unknown_url() { let urls = FeroxScans::default(); + let stats = Arc::new(Stats::new()); let url = "http://unknown_url"; - let (result, _scan) = urls.add_scan(url, ScanType::Directory); + let (result, _scan) = urls.add_scan(url, ScanType::Directory, stats); assert_eq!(result, true); } @@ -807,11 +828,13 @@ mod tests { let urls = FeroxScans::default(); let pb = ProgressBar::new(1); let url = "http://unknown_url/"; - let scan = FeroxScan::new(url, ScanType::Directory, Some(pb)); + let stats = Arc::new(Stats::new()); + + let scan = FeroxScan::new(url, ScanType::Directory, pb.length(), Some(pb)); assert_eq!(urls.insert(scan), true); - let (result, _scan) = urls.add_scan(url, ScanType::Directory); + let (result, _scan) = urls.add_scan(url, ScanType::Directory, stats); assert_eq!(result, false); } @@ -821,7 +844,8 @@ mod tests { fn abort_stops_progress_bar() { let pb = ProgressBar::new(1); let url = "http://unknown_url/"; - let scan = FeroxScan::new(url, ScanType::Directory, Some(pb)); + + let scan = FeroxScan::new(url, ScanType::Directory, pb.length(), Some(pb)); assert_eq!( scan.lock() @@ -851,11 +875,13 @@ mod tests { fn add_url_to_list_of_scanned_urls_with_known_url_without_slash() { let urls = FeroxScans::default(); let url = "http://unknown_url"; - let scan = FeroxScan::new(url, ScanType::File, None); + let stats = Arc::new(Stats::new()); + + let scan = FeroxScan::new(url, ScanType::File, 0, None); assert_eq!(urls.insert(scan), true); - let (result, _scan) = urls.add_scan(url, ScanType::File); + let (result, _scan) = urls.add_scan(url, ScanType::File, stats); assert_eq!(result, false); } @@ -868,8 +894,8 @@ mod tests { let pb_two = ProgressBar::new(2); let url = "http://unknown_url/"; let url_two = "http://unknown_url/fa"; - let scan = FeroxScan::new(url, ScanType::Directory, Some(pb)); - let scan_two = FeroxScan::new(url_two, ScanType::Directory, Some(pb_two)); + let scan = FeroxScan::new(url, ScanType::Directory, pb.length(), Some(pb)); + let scan_two = FeroxScan::new(url_two, ScanType::Directory, pb_two.length(), Some(pb_two)); scan_two.lock().unwrap().finish(); // one complete, one incomplete @@ -882,8 +908,8 @@ mod tests { /// ensure that PartialEq compares FeroxScan.id fields fn partial_eq_compares_the_id_field() { let url = "http://unknown_url/"; - let scan = FeroxScan::new(url, ScanType::Directory, None); - let scan_two = FeroxScan::new(url, ScanType::Directory, None); + let scan = FeroxScan::new(url, ScanType::Directory, 0, None); + let scan_two = FeroxScan::new(url, ScanType::Directory, 0, None); assert!(!scan.lock().unwrap().eq(&scan_two.lock().unwrap())); @@ -942,7 +968,7 @@ mod tests { #[test] /// given a FeroxScan, test that it serializes into the proper JSON entry fn ferox_scan_serialize() { - let fs = FeroxScan::new("https://spiritanimal.com", ScanType::Directory, None); + let fs = FeroxScan::new("https://spiritanimal.com", ScanType::Directory, 0, None); let fs_json = format!( r#"{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","complete":false}}"#, fs.lock().unwrap().id @@ -956,7 +982,7 @@ mod tests { #[test] /// given a FeroxScans, test that it serializes into the proper JSON entry fn ferox_scans_serialize() { - let ferox_scan = FeroxScan::new("https://spiritanimal.com", ScanType::Directory, None); + let ferox_scan = FeroxScan::new("https://spiritanimal.com", ScanType::Directory, 0, None); let ferox_scans = FeroxScans::default(); let ferox_scans_json = format!( r#"[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","complete":false}}]"#, @@ -1010,7 +1036,7 @@ mod tests { #[test] /// test FeroxSerialize implementation of FeroxState fn feroxstates_feroxserialize_implementation() { - let ferox_scan = FeroxScan::new("https://spiritanimal.com", ScanType::Directory, None); + let ferox_scan = FeroxScan::new("https://spiritanimal.com", ScanType::Directory, 0, None); let saved_id = ferox_scan.lock().unwrap().id.clone(); SCANNED_URLS.insert(ferox_scan); diff --git a/src/scanner.rs b/src/scanner.rs index ec0934b..08fb2bc 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -1,4 +1,4 @@ -use crate::statistics::StatCommand::UpdateF64Field; +use crate::statistics::Stats; use crate::{ config::{Configuration, CONFIGURATION}, extractor::{get_links, request_feroxresponse_from_new_link}, @@ -9,7 +9,7 @@ use crate::{ heuristics, scan_manager::{FeroxResponses, FeroxScans, PAUSE_SCAN}, statistics::{ - StatCommand::{self, UpdateUsizeField}, + StatCommand::{self, UpdateF64Field, UpdateUsizeField}, StatField::{DirScanTimes, ExpectedPerScan, TotalScans, WildcardsFiltered}, }, utils::{format_url, get_current_depth, make_request}, @@ -29,8 +29,9 @@ use std::{ collections::HashSet, convert::TryInto, ops::Deref, - sync::atomic::{AtomicU64, AtomicUsize, Ordering}, + sync::atomic::{AtomicUsize, Ordering}, sync::{Arc, RwLock}, + time::Instant, }; use tokio::{ sync::{ @@ -47,9 +48,6 @@ use tokio::{ /// --stdin means this will be incremented by the number of targets passed via STDIN static CALL_COUNT: AtomicUsize = AtomicUsize::new(0); -/// Single atomic number that gets holds the number of requests to be sent per directory scanned -pub static NUMBER_OF_REQUESTS: AtomicU64 = AtomicU64::new(0); // todo move to stats - lazy_static! { /// Set of urls that have been sent to [scan_url](fn.scan_url.html), used for deduplication pub static ref SCANNED_URLS: FeroxScans = FeroxScans::default(); @@ -107,17 +105,17 @@ fn spawn_recursion_handler( mut recursion_channel: UnboundedReceiver, wordlist: Arc>, base_depth: usize, - num_targets: usize, + stats: Arc, tx_term: UnboundedSender, tx_file: UnboundedSender, tx_stats: UnboundedSender, ) -> BoxFuture<'static, Vec>> { log::trace!( - "enter: spawn_recursion_handler({:?}, wordlist[{} words...], {}, {}, {:?}, {:?}, {:?})", + "enter: spawn_recursion_handler({:?}, wordlist[{} words...], {}, {:?}, {:?}, {:?}, {:?})", recursion_channel, wordlist.len(), base_depth, - num_targets, + stats, tx_term, tx_file, tx_stats @@ -127,7 +125,7 @@ fn spawn_recursion_handler( let mut scans = vec![]; while let Some(resp) = recursion_channel.recv().await { - let (unknown, _) = SCANNED_URLS.add_directory_scan(&resp); + let (unknown, _) = SCANNED_URLS.add_directory_scan(&resp, stats.clone()); if !unknown { // not unknown, i.e. we've seen the url before and don't need to scan again @@ -140,7 +138,8 @@ fn spawn_recursion_handler( let term_clone = tx_term.clone(); let file_clone = tx_file.clone(); - let stats_clone = tx_stats.clone(); + let tx_stats_clone = tx_stats.clone(); + let stats_clone = stats.clone(); let resp_clone = resp.clone(); let list_clone = wordlist.clone(); @@ -149,10 +148,10 @@ fn spawn_recursion_handler( resp_clone.to_owned().as_str(), list_clone, base_depth, - num_targets, + stats_clone, term_clone, file_clone, - stats_clone, + tx_stats_clone, ) .await }); @@ -173,12 +172,18 @@ fn spawn_recursion_handler( /// /// If any extensions were passed to the program, each extension will add a /// (base_url + word + ext) Url to the vector -fn create_urls(target_url: &str, word: &str, extensions: &[String]) -> Vec { +fn create_urls( + target_url: &str, + word: &str, + extensions: &[String], + tx_stats: UnboundedSender, +) -> Vec { log::trace!( - "enter: create_urls({}, {}, {:?})", + "enter: create_urls({}, {}, {:?}, {:?})", target_url, word, - extensions + extensions, + tx_stats ); let mut urls = vec![]; @@ -189,6 +194,7 @@ fn create_urls(target_url: &str, word: &str, extensions: &[String]) -> Vec CONFIGURATION.add_slash, &CONFIGURATION.queries, None, + tx_stats.clone(), ) { urls.push(url); // default request, i.e. no extension } @@ -200,6 +206,7 @@ fn create_urls(target_url: &str, word: &str, extensions: &[String]) -> Vec CONFIGURATION.add_slash, &CONFIGURATION.queries, Some(ext), + tx_stats.clone(), ) { urls.push(url); // any extensions passed in } @@ -375,21 +382,28 @@ async fn make_requests( target_url: &str, word: &str, base_depth: usize, + stats: Arc, dir_chan: UnboundedSender, report_chan: UnboundedSender, tx_stats: UnboundedSender, ) { log::trace!( - "enter: make_requests({}, {}, {}, {:?}, {:?}, {:?})", + "enter: make_requests({}, {}, {}, {:?}, {:?}, {:?}, {:?})", target_url, word, base_depth, + stats, dir_chan, report_chan, tx_stats ); - let urls = create_urls(&target_url, &word, &CONFIGURATION.extensions); + let urls = create_urls( + &target_url, + &word, + &CONFIGURATION.extensions, + tx_stats.clone(), + ); for url in urls { if let Ok(response) = make_request(&CONFIGURATION.client, &url, tx_stats.clone()).await { @@ -431,7 +445,8 @@ async fn make_requests( // very likely a file, simply request and report log::debug!("Singular extraction: {}", new_ferox_response); - SCANNED_URLS.add_file_scan(&new_ferox_response.url().to_string()); + SCANNED_URLS + .add_file_scan(&new_ferox_response.url().to_string(), stats.clone()); send_report(report_chan.clone(), new_ferox_response); @@ -484,17 +499,17 @@ pub async fn scan_url( target_url: &str, wordlist: Arc>, base_depth: usize, - num_targets: usize, + stats: Arc, tx_term: UnboundedSender, tx_file: UnboundedSender, tx_stats: UnboundedSender, ) { log::trace!( - "enter: scan_url({:?}, wordlist[{} words...], {}, {}, {:?}, {:?}, {:?})", + "enter: scan_url({:?}, wordlist[{} words...], {}, {:?}, {:?}, {:?}, {:?})", target_url, wordlist.len(), base_depth, - num_targets, + stats, tx_term, tx_file, tx_stats @@ -502,19 +517,18 @@ pub async fn scan_url( log::info!("Starting scan against: {}", target_url); - // todo import - let scan_timer = std::time::Instant::now(); + let scan_timer = Instant::now(); let (tx_dir, rx_dir): FeroxChannel = mpsc::unbounded_channel(); - if CALL_COUNT.load(Ordering::Relaxed) < num_targets { + if CALL_COUNT.load(Ordering::Relaxed) < stats.initial_targets.load(Ordering::Relaxed) { CALL_COUNT.fetch_add(1, Ordering::Relaxed); update_stat!(tx_stats, UpdateUsizeField(TotalScans, 1)); // this protection allows us to add the first scanned url to SCANNED_URLS // from within the scan_url function instead of the recursion handler - SCANNED_URLS.add_directory_scan(&target_url); + SCANNED_URLS.add_directory_scan(&target_url, stats.clone()); } let ferox_scan = match SCANNED_URLS.get_scan_by_url(&target_url) { @@ -551,13 +565,14 @@ pub async fn scan_url( let recurser_stats_clone = tx_stats.clone(); let recurser_words = wordlist.clone(); let looping_words = wordlist.clone(); + let looping_stats = stats.clone(); let recurser = tokio::spawn(async move { spawn_recursion_handler( rx_dir, recurser_words, base_depth, - num_targets, + stats.clone(), recurser_term_clone, recurser_file_clone, recurser_stats_clone, @@ -588,6 +603,7 @@ pub async fn scan_url( let txs = tx_stats.clone(); let pb = progress_bar.clone(); // progress bar is an Arc around internal state let tgt = target_url.to_string(); // done to satisfy 'static lifetime below + let lst = looping_stats.clone(); ( tokio::spawn(async move { if PAUSE_SCAN.load(Ordering::Acquire) { @@ -598,7 +614,7 @@ pub async fn scan_url( // todo change to true when issue #107 is resolved SCANNED_URLS.pause(false).await; } - make_requests(&tgt, &word, base_depth, txd, txr, txs).await + make_requests(&tgt, &word, base_depth, lst, txd, txr, txs).await }), pb, ) @@ -665,8 +681,6 @@ pub async fn initialize( total.try_into().unwrap() }; - NUMBER_OF_REQUESTS.store(num_reqs_expected, Ordering::Relaxed); - // tell Stats object about the number of expected requests update_stat!( tx_stats, @@ -734,7 +748,14 @@ pub async fn initialize( // add any similarity filters to `FILTERS` (--filter-similar-to) for similarity_filter in &config.filter_similar { // url as-is based on input, ignores user-specified url manipulation options (add-slash etc) - if let Ok(url) = format_url(&similarity_filter, &"", false, &Vec::new(), None) { + if let Ok(url) = format_url( + &similarity_filter, + &"", + false, + &Vec::new(), + None, + tx_stats.clone(), + ) { // attempt to request the given url if let Ok(resp) = make_request(&CONFIGURATION.client, &url, tx_stats.clone()).await { // if successful, create a filter based on the response's body @@ -771,14 +792,16 @@ mod tests { #[test] /// sending url + word without any extensions should get back one url with the joined word fn create_urls_no_extension_returns_base_url_with_word() { - let urls = create_urls("http://localhost", "turbo", &[]); + let (tx, _): FeroxChannel = mpsc::unbounded_channel(); + let urls = create_urls("http://localhost", "turbo", &[], tx); assert_eq!(urls, [Url::parse("http://localhost/turbo").unwrap()]) } #[test] /// sending url + word + 1 extension should get back two urls, one base and one with extension fn create_urls_one_extension_returns_two_urls() { - let urls = create_urls("http://localhost", "turbo", &[String::from("js")]); + let (tx, _): FeroxChannel = mpsc::unbounded_channel(); + let urls = create_urls("http://localhost", "turbo", &[String::from("js")], tx); assert_eq!( urls, [ @@ -816,8 +839,10 @@ mod tests { vec![base, js, php, pdf, tar], ]; + let (tx, _): FeroxChannel = mpsc::unbounded_channel(); + for (i, ext_set) in ext_vec.into_iter().enumerate() { - let urls = create_urls("http://localhost", "turbo", &ext_set); + let urls = create_urls("http://localhost", "turbo", &ext_set, tx.clone()); assert_eq!(urls, expected[i]); } } diff --git a/src/statistics.rs b/src/statistics.rs index a745f27..fb1e67f 100644 --- a/src/statistics.rs +++ b/src/statistics.rs @@ -70,7 +70,7 @@ pub struct Stats { /// /// Note: this is a per-scan expectation; `expected_requests * current # of scans` would be /// indicative of the current expectation at any given time, but is a moving target. - expected_per_scan: AtomicUsize, + pub expected_per_scan: AtomicUsize, /// tracker for accumulating total number of requests expected (i.e. as a new scan is started /// this value should increase by `expected_requests` @@ -95,6 +95,9 @@ pub struct Stats { /// recursed into and affects the total number of expected requests total_scans: AtomicUsize, + /// tracker for initial number of requested targets + pub initial_targets: AtomicUsize, + /// tracker for number of links extracted when `--extract-links` is used; sources are /// response bodies and robots.txt as of v1.11.0 links_extracted: AtomicUsize, @@ -299,6 +302,9 @@ impl Stats { StatField::ResourcesDiscovered => { atomic_increment!(self.resources_discovered, value); } + StatField::InitialTargets => { + atomic_increment!(self.initial_targets, value); + } _ => {} // f64 fields } } @@ -463,6 +469,9 @@ pub enum StatField { /// Translates to `resources_discovered` ResourcesDiscovered, + /// Translates to `initial_targets` + InitialTargets, + /// Translates to `directory_scan_times`; assumes a single append to the vector DirScanTimes, } diff --git a/src/utils.rs b/src/utils.rs index d091b76..b66a032 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,10 +1,10 @@ #![macro_use] -use crate::statistics::{ - StatCommand::{self, AddError, AddStatus}, - StatError::{Connection, Other, Redirection, Request, Timeout}, -}; use crate::{ config::{CONFIGURATION, PROGRESS_PRINTER}, + statistics::{ + StatCommand::{self, AddError, AddStatus}, + StatError::{Connection, Other, Redirection, Request, Timeout, UrlFormat}, + }, FeroxError, FeroxResult, }; use console::{strip_ansi_codes, style, user_attended}; @@ -166,6 +166,14 @@ pub fn ferox_print(msg: &str, bar: &ProgressBar) { } } +#[macro_export] +/// wrapper to improve code readability +macro_rules! update_stat { + ($tx:expr, $value:expr) => { + $tx.send($value).unwrap_or_default(); + }; +} + /// Simple helper to generate a `Url` /// /// Errors during parsing `url` or joining `word` are propagated up the call stack @@ -175,14 +183,16 @@ pub fn format_url( add_slash: bool, queries: &[(String, String)], extension: Option<&str>, + tx_stats: UnboundedSender, ) -> FeroxResult { log::trace!( - "enter: format_url({}, {}, {}, {:?} {:?})", + "enter: format_url({}, {}, {}, {:?} {:?}, {:?})", url, word, add_slash, queries, - extension + extension, + tx_stats ); if Url::parse(&word).is_ok() { @@ -201,6 +211,8 @@ pub fn format_url( let err = FeroxError { message }; + update_stat!(tx_stats, AddError(UrlFormat)); + log::trace!("exit: format_url -> {}", err); return Err(Box::new(err)); } @@ -260,6 +272,7 @@ pub fn format_url( } } Err(e) => { + update_stat!(tx_stats, AddError(UrlFormat)); log::trace!("exit: format_url -> {}", e); log::error!("Could not join {} with {}", word, base_url); Err(Box::new(e)) @@ -267,14 +280,6 @@ pub fn format_url( } } -#[macro_export] -/// wrapper to improve code readability -macro_rules! update_stat { - ($tx:expr, $value:expr) => { - $tx.send($value).unwrap_or_default(); - }; -} - /// Initiate request to the given `Url` using `Client` pub async fn make_request( client: &Client, @@ -430,6 +435,8 @@ pub fn normalize_url(url: &str) -> String { #[cfg(test)] mod tests { use super::*; + use crate::FeroxChannel; + use tokio::sync::mpsc; #[test] /// set_open_file_limit with a low requested limit succeeds @@ -497,8 +504,9 @@ mod tests { #[test] /// base url + 1 word + no slash + no extension fn format_url_normal() { + let (tx, _): FeroxChannel = mpsc::unbounded_channel(); assert_eq!( - format_url("http://localhost", "stuff", false, &Vec::new(), None).unwrap(), + format_url("http://localhost", "stuff", false, &Vec::new(), None, tx).unwrap(), reqwest::Url::parse("http://localhost/stuff").unwrap() ); } @@ -506,8 +514,9 @@ mod tests { #[test] /// base url + no word + no slash + no extension fn format_url_no_word() { + let (tx, _): FeroxChannel = mpsc::unbounded_channel(); assert_eq!( - format_url("http://localhost", "", false, &Vec::new(), None).unwrap(), + format_url("http://localhost", "", false, &Vec::new(), None, tx).unwrap(), reqwest::Url::parse("http://localhost").unwrap() ); } @@ -515,13 +524,15 @@ mod tests { #[test] /// base url + word + no slash + no extension + queries fn format_url_joins_queries() { + let (tx, _): FeroxChannel = mpsc::unbounded_channel(); assert_eq!( format_url( "http://localhost", "lazer", false, &[(String::from("stuff"), String::from("things"))], - None + None, + tx ) .unwrap(), reqwest::Url::parse("http://localhost/lazer?stuff=things").unwrap() @@ -531,13 +542,15 @@ mod tests { #[test] /// base url + no word + no slash + no extension + queries fn format_url_without_word_joins_queries() { + let (tx, _): FeroxChannel = mpsc::unbounded_channel(); assert_eq!( format_url( "http://localhost", "", false, &[(String::from("stuff"), String::from("things"))], - None + None, + tx ) .unwrap(), reqwest::Url::parse("http://localhost/?stuff=things").unwrap() @@ -548,14 +561,16 @@ mod tests { #[should_panic] /// no base url is an error fn format_url_no_url() { - format_url("", "stuff", false, &Vec::new(), None).unwrap(); + let (tx, _): FeroxChannel = mpsc::unbounded_channel(); + format_url("", "stuff", false, &Vec::new(), None, tx).unwrap(); } #[test] /// word prepended with slash is adjusted for correctness fn format_url_word_with_preslash() { + let (tx, _): FeroxChannel = mpsc::unbounded_channel(); assert_eq!( - format_url("http://localhost", "/stuff", false, &Vec::new(), None).unwrap(), + format_url("http://localhost", "/stuff", false, &Vec::new(), None, tx).unwrap(), reqwest::Url::parse("http://localhost/stuff").unwrap() ); } @@ -563,8 +578,9 @@ mod tests { #[test] /// word with appended slash allows the slash to persist fn format_url_word_with_postslash() { + let (tx, _): FeroxChannel = mpsc::unbounded_channel(); assert_eq!( - format_url("http://localhost", "stuff/", false, &Vec::new(), None).unwrap(), + format_url("http://localhost", "stuff/", false, &Vec::new(), None, tx).unwrap(), reqwest::Url::parse("http://localhost/stuff/").unwrap() ); } @@ -572,12 +588,14 @@ mod tests { #[test] /// word that is a fully formed url, should return an error fn format_url_word_that_is_a_url() { + let (tx, _): FeroxChannel = mpsc::unbounded_channel(); let url = format_url( "http://localhost", "http://schmocalhost", false, &Vec::new(), None, + tx, ); assert!(url.is_err()); }