removed errant log statements

Add statistics tracking

.rustfmt.toml

@@ -0,0 +1 @@
+reorder_modules = false

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "feroxbuster"
-version = "1.11.0"
+version = "1.11.1"
 authors = ["Ben 'epi' Risher <epibar052@gmail.com>"]
 license = "MIT"
 edition = "2018"
@@ -30,15 +30,15 @@ reqwest = { version = "0.10", features = ["socks"] }
 clap = "2.33"
 lazy_static = "1.4"
 toml = "0.5"
-serde = { version = "1.0", features = ["derive"] }
+serde = { version = "1.0", features = ["derive", "rc"] }
 serde_json = "1.0"
 uuid = { version = "0.8", features = ["v4"] }
 indicatif = "0.15"
-console = "0.13"
+console = "0.14"
 openssl = { version = "0.10", features = ["vendored"] }
 dirs = "3.0"
 regex = "1"
-crossterm = "0.18"
+crossterm = "0.19"
 rlimit = "0.5"
 ctrlc = "3.1"
 fuzzyhash = "0.2"

README.md

@@ -78,6 +78,10 @@ Enumeration.
     - [Threads and Connection Limits At A High-Level](#threads-and-connection-limits-at-a-high-level)
     - [ferox-config.toml](#ferox-configtoml)
     - [Command Line Parsing](#command-line-parsing)
+- [Scan's Display Explained](#-scans-display-explained)
+    - [Discovered Resource](#discovered-resource)
+    - [Overall Scan Progress Bar](#overall-scan-progress-bar)
+    - [Directory Scan Progress Bar](#directory-scan-progress-bar)
 - [Example Usage](#-example-usage)
     - [Multiple Values](#multiple-values)
     - [Include Headers](#include-headers)
@@ -460,6 +464,31 @@ OPTIONS:
     -w, --wordlist <FILE>                   Path to the wordlist
 ```
 
+## 📊 Scan's Display Explained
+
+`feroxbuster` attempts to be intuitive and easy to understand, however, if you are wondering about any of the scan's
+output and what it means, this is the section for you!  
+
+### Discovered Resource
+
+When `feroxbuster` finds a response that you haven't filtered out, it's reported above the progress bars and looks similar to what's pictured below.  
+
+The number of lines, words, and bytes shown here can be used to [filter those responses](#filter-response-by-word-count--line-count--new-in-v160)
+
+![response-bar-explained](img/response-bar-explained.png)
+
+### Overall Scan Progress Bar
+
+The top progress bar, colored yellow, tracks the overall scan status.  Its fields are described in the image below.
+
+![total-bar-explained](img/total-bar-explained.png)
+
+### Directory Scan Progress Bar
+
+All other progress bars, colored cyan, represent a scan of one particular directory and will look similar to what's below.   
+
+![dir-scan-bar-explained](img/dir-scan-bar-explained.png)
+
 ## 🧰 Example Usage
 
 ### Multiple Values

src/banner.rs

@@ -1,9 +1,13 @@
-use crate::config::{Configuration, CONFIGURATION};
-use crate::utils::{make_request, status_colorizer};
+use crate::{
+    config::{Configuration, CONFIGURATION},
+    statistics::StatCommand,
+    utils::{make_request, status_colorizer},
+};
 use console::{style, Emoji};
 use reqwest::{Client, Url};
 use serde_json::Value;
 use std::io::Write;
+use tokio::sync::mpsc::UnboundedSender;
 
 /// macro helper to abstract away repetitive string formatting
 macro_rules! format_banner_entry_helper {
@@ -67,8 +71,13 @@ enum UpdateStatus {
 /// ex: v1.1.0
 ///
 /// Returns `UpdateStatus`
-async fn needs_update(client: &Client, url: &str, bin_version: &str) -> UpdateStatus {
-    log::trace!("enter: needs_update({:?}, {})", client, url);
+async fn needs_update(
+    client: &Client,
+    url: &str,
+    bin_version: &str,
+    tx_stats: UnboundedSender<StatCommand>,
+) -> UpdateStatus {
+    log::trace!("enter: needs_update({:?}, {}, {:?})", client, url, tx_stats);
 
     let unknown = UpdateStatus::Unknown;
 
@@ -81,7 +90,7 @@ async fn needs_update(client: &Client, url: &str, bin_version: &str) -> UpdateSt
         }
     };
 
-    if let Ok(response) = make_request(&client, &api_url).await {
+    if let Ok(response) = make_request(&client, &api_url, tx_stats.clone()).await {
         let body = response.text().await.unwrap_or_default();
 
         let json_response: Value = serde_json::from_str(&body).unwrap_or_default();
@@ -137,8 +146,13 @@ fn format_emoji(emoji: &str) -> String {
 /// Prints the banner to stdout.
 ///
 /// Only prints those settings which are either always present, or passed in by the user.
-pub async fn initialize<W>(targets: &[String], config: &Configuration, version: &str, mut writer: W)
-where
+pub async fn initialize<W>(
+    targets: &[String],
+    config: &Configuration,
+    version: &str,
+    mut writer: W,
+    tx_stats: UnboundedSender<StatCommand>,
+) where
     W: Write,
 {
     let artwork = format!(
@@ -150,7 +164,7 @@ by Ben "epi" Risher {}                 ver: {}"#,
         Emoji("🤓", &format!("{:<2}", "\u{0020}")),
         version
     );
-    let status = needs_update(&CONFIGURATION.client, UPDATE_URL, version).await;
+    let status = needs_update(&CONFIGURATION.client, UPDATE_URL, version, tx_stats).await;
 
     let top = "───────────────────────────┬──────────────────────";
     let addl_section = "──────────────────────────────────────────────────";
@@ -536,31 +550,40 @@ by Ben "epi" Risher {}                 ver: {}"#,
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::VERSION;
+    use crate::{FeroxChannel, VERSION};
     use httpmock::Method::GET;
     use httpmock::MockServer;
     use std::fs::read_to_string;
     use std::io::stderr;
     use std::time::Duration;
     use tempfile::NamedTempFile;
+    use tokio::sync::mpsc;
 
     #[tokio::test(core_threads = 1)]
     /// test to hit no execution of targets for loop in banner
     async fn banner_intialize_without_targets() {
         let config = Configuration::default();
-        initialize(&[], &config, VERSION, stderr()).await;
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
+        initialize(&[], &config, VERSION, stderr(), tx).await;
     }
 
     #[tokio::test(core_threads = 1)]
     /// test to hit no execution of statuscode for loop in banner
     async fn banner_intialize_without_status_codes() {
-        let mut config = Configuration::default();
-        config.status_codes = vec![];
+        let config = Configuration {
+            status_codes: vec![],
+            ..Default::default()
+        };
+
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
         initialize(
             &[String::from("http://localhost")],
             &config,
             VERSION,
             stderr(),
+            tx,
         )
         .await;
     }
@@ -568,13 +591,19 @@ mod tests {
     #[tokio::test(core_threads = 1)]
     /// test to hit an empty config file
     async fn banner_intialize_without_config_file() {
-        let mut config = Configuration::default();
-        config.config = String::new();
+        let config = Configuration {
+            config: String::new(),
+            ..Default::default()
+        };
+
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
         initialize(
             &[String::from("http://localhost")],
             &config,
             VERSION,
             stderr(),
+            tx,
         )
         .await;
     }
@@ -582,13 +611,19 @@ mod tests {
     #[tokio::test(core_threads = 1)]
     /// test to hit an empty config file
     async fn banner_intialize_without_queries() {
-        let mut config = Configuration::default();
-        config.queries = vec![(String::new(), String::new())];
+        let config = Configuration {
+            queries: vec![(String::new(), String::new())],
+            ..Default::default()
+        };
+
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
         initialize(
             &[String::from("http://localhost")],
             &config,
             VERSION,
             stderr(),
+            tx,
         )
         .await;
     }
@@ -598,11 +633,14 @@ mod tests {
     async fn banner_intialize_with_mismatched_version() {
         let config = Configuration::default();
         let file = NamedTempFile::new().unwrap();
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
         initialize(
             &[String::from("http://localhost")],
             &config,
             "mismatched-version",
             &file,
+            tx,
         )
         .await;
         let contents = read_to_string(file.path()).unwrap();
@@ -614,7 +652,9 @@ mod tests {
     #[tokio::test(core_threads = 1)]
     /// test that
     async fn banner_needs_update_returns_unknown_with_bad_url() {
-        let result = needs_update(&CONFIGURATION.client, &"", VERSION).await;
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
+        let result = needs_update(&CONFIGURATION.client, &"", VERSION, tx).await;
         assert!(matches!(result, UpdateStatus::Unknown));
     }
 
@@ -628,7 +668,9 @@ mod tests {
             then.status(200).body("{\"tag_name\":\"v1.1.0\"}");
         });
 
-        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.1.0").await;
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
+        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.1.0", tx).await;
 
         assert_eq!(mock.hits(), 1);
         assert!(matches!(result, UpdateStatus::UpToDate));
@@ -644,7 +686,9 @@ mod tests {
             then.status(200).body("{\"tag_name\":\"v1.1.0\"}");
         });
 
-        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1").await;
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
+        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1", tx).await;
 
         assert_eq!(mock.hits(), 1);
         assert!(matches!(result, UpdateStatus::OutOfDate));
@@ -662,7 +706,9 @@ mod tests {
                 .delay(Duration::from_secs(8));
         });
 
-        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1").await;
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
+        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1", tx).await;
 
         assert_eq!(mock.hits(), 1);
         assert!(matches!(result, UpdateStatus::Unknown));
@@ -678,7 +724,9 @@ mod tests {
             then.status(200).body("not json");
         });
 
-        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1").await;
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
+        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1", tx).await;
 
         assert_eq!(mock.hits(), 1);
         assert!(matches!(result, UpdateStatus::Unknown));
@@ -695,7 +743,9 @@ mod tests {
                 .body("{\"no tag_name\": \"doesn't exist\"}");
         });
 
-        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1").await;
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
+        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1", tx).await;
 
         assert_eq!(mock.hits(), 1);
         assert!(matches!(result, UpdateStatus::Unknown));

src/config.rs

@@ -1,18 +1,23 @@
-use crate::scan_manager::resume_scan;
-use crate::utils::{module_colorizer, status_colorizer};
-use crate::{client, parser, progress};
-use crate::{FeroxSerialize, DEFAULT_CONFIG_NAME, DEFAULT_STATUS_CODES, DEFAULT_WORDLIST, VERSION};
+use crate::{
+    client, parser,
+    progress::{add_bar, BarType},
+    scan_manager::resume_scan,
+    utils::{module_colorizer, status_colorizer},
+    FeroxSerialize, DEFAULT_CONFIG_NAME, DEFAULT_STATUS_CODES, DEFAULT_WORDLIST, VERSION,
+};
 use clap::{value_t, ArgMatches};
 use indicatif::{MultiProgress, ProgressBar, ProgressDrawTarget};
 use lazy_static::lazy_static;
 use reqwest::{Client, StatusCode};
 use serde::{Deserialize, Serialize};
-use std::collections::HashMap;
-use std::env::{current_dir, current_exe};
-use std::fs::read_to_string;
-use std::path::PathBuf;
 #[cfg(not(test))]
 use std::process::exit;
+use std::{
+    collections::HashMap,
+    env::{current_dir, current_exe},
+    fs::read_to_string,
+    path::PathBuf,
+};
 
 lazy_static! {
     /// Global configuration state
@@ -22,7 +27,7 @@ lazy_static! {
     pub static ref PROGRESS_BAR: MultiProgress = MultiProgress::with_draw_target(ProgressDrawTarget::stdout());
 
     /// Global progress bar that is only used for printing messages that don't jack up other bars
-    pub static ref PROGRESS_PRINTER: ProgressBar = progress::add_bar("", 0, true, false);
+    pub static ref PROGRESS_PRINTER: ProgressBar = add_bar("", 0, BarType::Hidden);
 }
 
 /// macro helper to abstract away repetitive configuration updates
@@ -80,7 +85,7 @@ fn report_and_exit(err: &str) -> ! {
 pub struct Configuration {
     #[serde(rename = "type", default = "serialized_type")]
     /// Name of this type of struct, used for serialization, i.e. `{"type":"configuration"}`
-    kind: String,
+    pub kind: String,
 
     /// Path to the wordlist
     #[serde(default = "wordlist")]
@@ -223,6 +228,10 @@ pub struct Configuration {
     #[serde(default)]
     pub resumed: bool,
 
+    /// Resume scan from this file
+    #[serde(default)]
+    pub resume_from: String,
+
     /// Whether or not a scan's current state should be saved when user presses Ctrl+C
     ///
     /// Not configurable from CLI; can only be set from a config file
@@ -324,6 +333,7 @@ impl Default for Configuration {
             debug_log: String::new(),
             target_url: String::new(),
             time_limit: String::new(),
+            resume_from: String::new(),
             replay_proxy: String::new(),
             queries: Vec::new(),
             extensions: Vec::new(),
@@ -401,8 +411,10 @@ impl Configuration {
     pub fn new() -> Self {
         // when compiling for test, we want to eliminate the runtime dependency of the parser
         if cfg!(test) {
-            let mut test_config = Configuration::default();
-            test_config.save_state = false; // don't clutter up junk when testing
+            let test_config = Configuration {
+                save_state: false, // don't clutter up junk when testing
+                ..Default::default()
+            };
             return test_config;
         }
 
@@ -515,6 +527,7 @@ impl Configuration {
         update_config_if_present!(&mut config.output, args, "output", String);
         update_config_if_present!(&mut config.debug_log, args, "debug_log", String);
         update_config_if_present!(&mut config.time_limit, args, "time_limit", String);
+        update_config_if_present!(&mut config.resume_from, args, "resume_from", String);
 
         if let Some(arg) = args.values_of("status_codes") {
             config.status_codes = arg
@@ -794,6 +807,7 @@ impl Configuration {
         update_if_not_default!(&mut conf.scan_limit, new.scan_limit, 0);
         update_if_not_default!(&mut conf.replay_proxy, new.replay_proxy, "");
         update_if_not_default!(&mut conf.debug_log, new.debug_log, "");
+        update_if_not_default!(&mut conf.resume_from, new.resume_from, "");
         update_if_not_default!(&mut conf.json, new.json, false);
 
         update_if_not_default!(&mut conf.timeout, new.timeout, timeout());
@@ -893,6 +907,7 @@ mod tests {
             time_limit = "10m"
             output = "/some/otherpath"
             debug_log = "/yet/anotherpath"
+            resume_from = "/some/state/file"
             redirects = true
             insecure = true
             extensions = ["html", "php", "js"]
@@ -927,6 +942,7 @@ mod tests {
         assert_eq!(config.proxy, String::new());
         assert_eq!(config.target_url, String::new());
         assert_eq!(config.time_limit, String::new());
+        assert_eq!(config.resume_from, String::new());
         assert_eq!(config.debug_log, String::new());
         assert_eq!(config.config, String::new());
         assert_eq!(config.replay_proxy, String::new());
@@ -1169,6 +1185,13 @@ mod tests {
         assert_eq!(config.time_limit, "10m");
     }
 
+    #[test]
+    /// parse the test config and see that the value parsed is correct
+    fn config_reads_resume_from() {
+        let config = setup_config_test();
+        assert_eq!(config.resume_from, "/some/state/file");
+    }
+
     #[test]
     /// parse the test config and see that the values parsed are correct
     fn config_reads_headers() {

src/extractor.rs

@@ -2,6 +2,10 @@ use crate::{
     client,
     config::{Configuration, CONFIGURATION},
     scanner::SCANNED_URLS,
+    statistics::{
+        StatCommand::{self, UpdateUsizeField},
+        StatField::{LinksExtracted, TotalExpected},
+    },
     utils::{format_url, make_request},
     FeroxResponse,
 };
@@ -9,6 +13,7 @@ use lazy_static::lazy_static;
 use regex::Regex;
 use reqwest::Url;
 use std::collections::HashSet;
+use tokio::sync::mpsc::UnboundedSender;
 
 /// Regular expression used in [LinkFinder](https://github.com/GerbenJavado/LinkFinder)
 ///
@@ -98,8 +103,15 @@ fn add_link_to_set_of_links(link: &str, url: &Url, links: &mut HashSet<String>)
 ///         - homepage/assets/img/
 ///         - homepage/assets/
 ///         - homepage/
-pub async fn get_links(response: &FeroxResponse) -> HashSet<String> {
-    log::trace!("enter: get_links({})", response.url().as_str());
+pub async fn get_links(
+    response: &FeroxResponse,
+    tx_stats: UnboundedSender<StatCommand>,
+) -> HashSet<String> {
+    log::trace!(
+        "enter: get_links({}, {:?})",
+        response.url().as_str(),
+        tx_stats
+    );
 
     let mut links = HashSet::<String>::new();
 
@@ -136,6 +148,14 @@ pub async fn get_links(response: &FeroxResponse) -> HashSet<String> {
         }
     }
 
+    let multiplier = CONFIGURATION.extensions.len().max(1);
+
+    update_stat!(tx_stats, UpdateUsizeField(LinksExtracted, links.len()));
+    update_stat!(
+        tx_stats,
+        UpdateUsizeField(TotalExpected, links.len() * multiplier)
+    );
+
     log::trace!("exit: get_links -> {:?}", links);
 
     links
@@ -172,8 +192,15 @@ fn add_all_sub_paths(url_path: &str, response: &FeroxResponse, mut links: &mut H
 ///   - create a new Url object based on cli options/args
 ///   - check if the new Url has already been seen/scanned -> None
 ///   - make a request to the new Url ? -> Some(response) : None
-pub async fn request_feroxresponse_from_new_link(url: &str) -> Option<FeroxResponse> {
-    log::trace!("enter: request_feroxresponse_from_new_link({})", url);
+pub async fn request_feroxresponse_from_new_link(
+    url: &str,
+    tx_stats: UnboundedSender<StatCommand>,
+) -> Option<FeroxResponse> {
+    log::trace!(
+        "enter: request_feroxresponse_from_new_link({}, {:?})",
+        url,
+        tx_stats
+    );
 
     // create a url based on the given command line options, return None on error
     let new_url = match format_url(
@@ -182,6 +209,7 @@ pub async fn request_feroxresponse_from_new_link(url: &str) -> Option<FeroxRespo
         CONFIGURATION.add_slash,
         &CONFIGURATION.queries,
         None,
+        tx_stats.clone(),
     ) {
         Ok(url) => url,
         Err(_) => {
@@ -197,7 +225,7 @@ pub async fn request_feroxresponse_from_new_link(url: &str) -> Option<FeroxRespo
     }
 
     // make the request and store the response
-    let new_response = match make_request(&CONFIGURATION.client, &new_url).await {
+    let new_response = match make_request(&CONFIGURATION.client, &new_url, tx_stats).await {
         Ok(resp) => resp,
         Err(_) => {
             log::trace!("exit: request_feroxresponse_from_new_link -> None");
@@ -221,8 +249,16 @@ pub async fn request_feroxresponse_from_new_link(url: &str) -> Option<FeroxRespo
 ///     
 /// The length of the given path has no effect on what's requested; it's always
 /// base url + /robots.txt
-pub async fn request_robots_txt(base_url: &str, config: &Configuration) -> Option<FeroxResponse> {
-    log::trace!("enter: get_robots_file({})", base_url);
+pub async fn request_robots_txt(
+    base_url: &str,
+    config: &Configuration,
+    tx_stats: UnboundedSender<StatCommand>,
+) -> Option<FeroxResponse> {
+    log::trace!(
+        "enter: get_robots_file({}, CONFIGURATION, {:?})",
+        base_url,
+        tx_stats
+    );
 
     // more often than not, domain/robots.txt will redirect to www.domain/robots.txt or something
     // similar; to account for that, create a client that will follow redirects, regardless of
@@ -248,7 +284,7 @@ pub async fn request_robots_txt(base_url: &str, config: &Configuration) -> Optio
     if let Ok(mut url) = Url::parse(base_url) {
         url.set_path("/robots.txt"); // overwrite existing path with /robots.txt
 
-        if let Ok(response) = make_request(&client, &url).await {
+        if let Ok(response) = make_request(&client, &url, tx_stats).await {
             let ferox_response = FeroxResponse::from(response, true).await;
 
             log::trace!("exit: get_robots_file -> {}", ferox_response);
@@ -267,11 +303,19 @@ pub async fn request_robots_txt(base_url: &str, config: &Configuration) -> Optio
 ///     http://localhost/stuff/things
 /// this function requests:
 ///     http://localhost/robots.txt
-pub async fn extract_robots_txt(base_url: &str, config: &Configuration) -> HashSet<String> {
-    log::trace!("enter: extract_robots_txt({}, CONFIGURATION)", base_url);
+pub async fn extract_robots_txt(
+    base_url: &str,
+    config: &Configuration,
+    tx_stats: UnboundedSender<StatCommand>,
+) -> HashSet<String> {
+    log::trace!(
+        "enter: extract_robots_txt({}, CONFIGURATION, {:?})",
+        base_url,
+        tx_stats
+    );
     let mut links = HashSet::new();
 
-    if let Some(response) = request_robots_txt(&base_url, &config).await {
+    if let Some(response) = request_robots_txt(&base_url, &config, tx_stats.clone()).await {
         for capture in ROBOTS_REGEX.captures_iter(response.text.as_str()) {
             if let Some(new_path) = capture.name("url_path") {
                 if let Ok(mut new_url) = Url::parse(base_url) {
@@ -282,6 +326,14 @@ pub async fn extract_robots_txt(base_url: &str, config: &Configuration) -> HashS
         }
     }
 
+    let multiplier = CONFIGURATION.extensions.len().max(1);
+
+    update_stat!(tx_stats, UpdateUsizeField(LinksExtracted, links.len()));
+    update_stat!(
+        tx_stats,
+        UpdateUsizeField(TotalExpected, links.len() * multiplier)
+    );
+
     log::trace!("exit: extract_robots_txt -> {:?}", links);
     links
 }
@@ -290,9 +342,11 @@ pub async fn extract_robots_txt(base_url: &str, config: &Configuration) -> HashS
 mod tests {
     use super::*;
     use crate::utils::make_request;
+    use crate::FeroxChannel;
     use httpmock::Method::GET;
     use httpmock::MockServer;
     use reqwest::Client;
+    use tokio::sync::mpsc;
 
     #[test]
     /// extract sub paths from the given url fragment; expect 4 sub paths and that all are
@@ -402,12 +456,13 @@ mod tests {
 
         let client = Client::new();
         let url = Url::parse(&srv.url("/some-path")).unwrap();
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
 
-        let response = make_request(&client, &url).await.unwrap();
+        let response = make_request(&client, &url, tx.clone()).await.unwrap();
 
         let ferox_response = FeroxResponse::from(response, true).await;
 
-        let links = get_links(&ferox_response).await;
+        let links = get_links(&ferox_response, tx).await;
 
         assert!(links.is_empty());
 
@@ -427,13 +482,15 @@ mod tests {
 
         let mut config = Configuration::default();
 
-        request_robots_txt(&srv.url("/api/users/stuff/things"), &config).await;
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
+        request_robots_txt(&srv.url("/api/users/stuff/things"), &config, tx.clone()).await;
 
         // note: the proxy doesn't actually do anything other than hit a different code branch
         // in this unit test; it would however have an effect on an integration test
         config.proxy = srv.url("/ima-proxy");
 
-        request_robots_txt(&srv.url("/api/different/path"), &config).await;
+        request_robots_txt(&srv.url("/api/different/path"), &config, tx).await;
 
         assert_eq!(mock.hits(), 2);
     }

src/heuristics.rs

@@ -2,6 +2,7 @@ use crate::{
     config::{CONFIGURATION, PROGRESS_PRINTER},
     filters::WildcardFilter,
     scanner::should_filter_response,
+    statistics::StatCommand,
     utils::{ferox_print, format_url, get_url_path_length, make_request, status_colorizer},
     FeroxResponse,
 };
@@ -40,12 +41,14 @@ pub async fn wildcard_test(
     target_url: &str,
     bar: ProgressBar,
     tx_term: UnboundedSender<FeroxResponse>,
+    tx_stats: UnboundedSender<StatCommand>,
 ) -> Option<WildcardFilter> {
     log::trace!(
-        "enter: wildcard_test({:?}, {:?}, {:?})",
+        "enter: wildcard_test({:?}, {:?}, {:?}, {:?})",
         target_url,
         bar,
-        tx_term
+        tx_term,
+        tx_stats
     );
 
     if CONFIGURATION.dont_filter {
@@ -54,10 +57,14 @@ pub async fn wildcard_test(
         return None;
     }
 
-    let tx_clone_one = tx_term.clone();
-    let tx_clone_two = tx_term.clone();
+    let tx_term_mwcr1 = tx_term.clone();
+    let tx_term_mwcr2 = tx_term.clone();
+    let tx_stats_mwcr1 = tx_stats.clone();
+    let tx_stats_mwcr2 = tx_stats.clone();
 
-    if let Some(ferox_response) = make_wildcard_request(&target_url, 1, tx_clone_one).await {
+    if let Some(ferox_response) =
+        make_wildcard_request(&target_url, 1, tx_term_mwcr1, tx_stats_mwcr1).await
+    {
         bar.inc(1);
 
         // found a wildcard response
@@ -72,7 +79,9 @@ pub async fn wildcard_test(
 
         // content length of wildcard is non-zero, perform additional tests:
         //   make a second request, with a known-sized (64) longer request
-        if let Some(resp_two) = make_wildcard_request(&target_url, 3, tx_clone_two).await {
+        if let Some(resp_two) =
+            make_wildcard_request(&target_url, 3, tx_term_mwcr2, tx_stats_mwcr2).await
+        {
             bar.inc(1);
 
             let wc2_length = resp_two.content_length();
@@ -138,12 +147,14 @@ async fn make_wildcard_request(
     target_url: &str,
     length: usize,
     tx_file: UnboundedSender<FeroxResponse>,
+    tx_stats: UnboundedSender<StatCommand>,
 ) -> Option<FeroxResponse> {
     log::trace!(
-        "enter: make_wildcard_request({}, {}, {:?})",
+        "enter: make_wildcard_request({}, {}, {:?}, {:?})",
         target_url,
         length,
-        tx_file
+        tx_file,
+        tx_stats,
     );
 
     let unique_str = unique_string(length);
@@ -154,6 +165,7 @@ async fn make_wildcard_request(
         CONFIGURATION.add_slash,
         &CONFIGURATION.queries,
         None,
+        tx_stats.clone(),
     ) {
         Ok(url) => url,
         Err(e) => {
@@ -163,7 +175,13 @@ async fn make_wildcard_request(
         }
     };
 
-    match make_request(&CONFIGURATION.client, &nonexistent.to_owned()).await {
+    match make_request(
+        &CONFIGURATION.client,
+        &nonexistent.to_owned(),
+        tx_stats.clone(),
+    )
+    .await
+    {
         Ok(response) => {
             if CONFIGURATION
                 .status_codes
@@ -174,7 +192,7 @@ async fn make_wildcard_request(
                 ferox_response.wildcard = true;
 
                 if !CONFIGURATION.quiet
-                    && !should_filter_response(&ferox_response)
+                    && !should_filter_response(&ferox_response, tx_stats.clone())
                     && tx_file.send(ferox_response.clone()).is_err()
                 {
                     return None;
@@ -190,6 +208,7 @@ async fn make_wildcard_request(
             return None;
         }
     }
+
     log::trace!("exit: make_wildcard_request -> None");
     None
 }
@@ -199,8 +218,15 @@ async fn make_wildcard_request(
 /// In the event that no sites can be reached, the program will exit.
 ///
 /// Any urls that are found to be alive are returned to the caller.
-pub async fn connectivity_test(target_urls: &[String]) -> Vec<String> {
-    log::trace!("enter: connectivity_test({:?})", target_urls);
+pub async fn connectivity_test(
+    target_urls: &[String],
+    tx_stats: UnboundedSender<StatCommand>,
+) -> Vec<String> {
+    log::trace!(
+        "enter: connectivity_test({:?}, {:?})",
+        target_urls,
+        tx_stats
+    );
 
     let mut good_urls = vec![];
 
@@ -211,6 +237,7 @@ pub async fn connectivity_test(target_urls: &[String]) -> Vec<String> {
             CONFIGURATION.add_slash,
             &CONFIGURATION.queries,
             None,
+            tx_stats.clone(),
         ) {
             Ok(url) => url,
             Err(e) => {
@@ -219,7 +246,7 @@ pub async fn connectivity_test(target_urls: &[String]) -> Vec<String> {
             }
         };
 
-        match make_request(&CONFIGURATION.client, &request).await {
+        match make_request(&CONFIGURATION.client, &request, tx_stats.clone()).await {
             Ok(_) => {
                 good_urls.push(target_url.to_owned());
             }

src/lib.rs

@@ -1,3 +1,4 @@
+pub mod utils;
 pub mod banner;
 pub mod client;
 pub mod config;
@@ -10,7 +11,7 @@ pub mod progress;
 pub mod reporter;
 pub mod scan_manager;
 pub mod scanner;
-pub mod utils;
+pub mod statistics;
 
 use crate::utils::{get_url_path_length, status_colorizer};
 use console::{style, Color};

src/main.rs

@@ -4,19 +4,26 @@ use feroxbuster::{
     config::{CONFIGURATION, PROGRESS_BAR, PROGRESS_PRINTER},
     extractor::{extract_robots_txt, request_feroxresponse_from_new_link},
     heuristics, logger,
-    progress::add_bar,
+    progress::{add_bar, BarType},
     reporter,
     scan_manager::{self, PAUSE_SCAN},
     scanner::{self, scan_url, send_report, RESPONSES, SCANNED_URLS},
+    statistics::{
+        self,
+        StatCommand::{self, CreateBar, LoadStats, UpdateUsizeField},
+        StatField::InitialTargets,
+        Stats,
+    },
+    update_stat,
     utils::{ferox_print, get_current_depth, module_colorizer, status_colorizer},
     FeroxError, FeroxResponse, FeroxResult, FeroxSerialize, SLEEP_DURATION, VERSION,
 };
 #[cfg(not(target_os = "windows"))]
 use feroxbuster::{utils::set_open_file_limit, DEFAULT_OPEN_FILE_LIMIT};
 use futures::StreamExt;
-use std::convert::TryInto;
 use std::{
     collections::HashSet,
+    convert::TryInto,
     fs::File,
     io::{stderr, BufRead, BufReader},
     process,
@@ -100,10 +107,19 @@ fn get_unique_words_from_wordlist(path: &str) -> FeroxResult<Arc<HashSet<String>
 /// Determine whether it's a single url scan or urls are coming from stdin, then scan as needed
 async fn scan(
     mut targets: Vec<String>,
+    stats: Arc<Stats>,
     tx_term: UnboundedSender<FeroxResponse>,
     tx_file: UnboundedSender<FeroxResponse>,
+    tx_stats: UnboundedSender<StatCommand>,
 ) -> FeroxResult<()> {
-    log::trace!("enter: scan({:?}, {:?}, {:?})", targets, tx_term, tx_file);
+    log::trace!(
+        "enter: scan({:?}, {:?}, {:?}, {:?}, {:?})",
+        targets,
+        stats,
+        tx_term,
+        tx_file,
+        tx_stats
+    );
     // cloning an Arc is cheap (it's basically a pointer into the heap)
     // so that will allow for cheap/safe sharing of a single wordlist across multi-target scans
     // as well as additional directories found as part of recursion
@@ -112,14 +128,32 @@ async fn scan(
             .await??;
 
     if words.len() == 0 {
-        let mut err = FeroxError::default();
-        err.message = format!("Did not find any words in {}", CONFIGURATION.wordlist);
+        let err = FeroxError {
+            message: format!("Did not find any words in {}", CONFIGURATION.wordlist),
+        };
+
         return Err(Box::new(err));
     }
 
-    scanner::initialize(words.len(), &CONFIGURATION).await;
+    scanner::initialize(words.len(), &CONFIGURATION, tx_stats.clone()).await;
+
+    // at this point, the stat thread's progress bar can be created; things that needed to happen
+    // first:
+    // - banner gets printed
+    // - scanner initialized (this sent expected requests per directory to the stats thread, which
+    //   having been set, makes it so the progress bar doesn't flash as full before anything has
+    //   even happened
+    update_stat!(tx_stats, CreateBar);
 
     if CONFIGURATION.resumed {
+        update_stat!(tx_stats, LoadStats(CONFIGURATION.resume_from.clone()));
+
+        if let Ok(responses) = RESPONSES.responses.read() {
+            for response in responses.iter() {
+                PROGRESS_PRINTER.println(response.as_str());
+            }
+        }
+
         if let Ok(scans) = SCANNED_URLS.scans.lock() {
             for scan in scans.iter() {
                 if let Ok(locked_scan) = scan.lock() {
@@ -128,40 +162,38 @@ async fn scan(
                         let pb = add_bar(
                             &locked_scan.url,
                             words.len().try_into().unwrap_or_default(),
-                            false,
-                            true,
+                            BarType::Message,
                         );
                         pb.finish();
                     }
                 }
             }
         }
-
-        if let Ok(responses) = RESPONSES.responses.read() {
-            for response in responses.iter() {
-                PROGRESS_PRINTER.println(response.as_str());
-            }
-        }
     }
 
     if CONFIGURATION.extract_links {
         for target in targets.clone() {
             // modifying the targets vector, so we can't have a reference to it while we borrow
             // it as mutable; thus the clone
-            let robots_links = extract_robots_txt(&target, &CONFIGURATION).await;
+            let robots_links = extract_robots_txt(&target, &CONFIGURATION, tx_stats.clone()).await;
 
             for robot_link in robots_links {
                 // create a url based on the given command line options, continue on error
-                let ferox_response = match request_feroxresponse_from_new_link(&robot_link).await {
+                let ferox_response = match request_feroxresponse_from_new_link(
+                    &robot_link,
+                    tx_stats.clone(),
+                )
+                .await
+                {
                     Some(resp) => resp,
                     None => continue,
                 };
 
                 if ferox_response.is_file() {
-                    SCANNED_URLS.add_file_scan(&robot_link);
+                    SCANNED_URLS.add_file_scan(&robot_link, stats.clone());
                     send_report(tx_term.clone(), ferox_response);
                 } else {
-                    let (unknown, _) = SCANNED_URLS.add_directory_scan(&robot_link);
+                    let (unknown, _) = SCANNED_URLS.add_directory_scan(&robot_link, stats.clone());
 
                     if !unknown {
                         // known directory; can skip (unlikely)
@@ -176,12 +208,13 @@ async fn scan(
     }
 
     let mut tasks = vec![];
-    let num_targets = targets.len();
 
     for target in targets {
         let word_clone = words.clone();
         let term_clone = tx_term.clone();
         let file_clone = tx_file.clone();
+        let tx_stats_clone = tx_stats.clone();
+        let stats_clone = stats.clone();
 
         let task = tokio::spawn(async move {
             let base_depth = get_current_depth(&target);
@@ -189,9 +222,10 @@ async fn scan(
                 &target,
                 word_clone,
                 base_depth,
-                num_targets,
+                stats_clone,
                 term_clone,
                 file_clone,
+                tx_stats_clone,
             )
             .await;
         });
@@ -262,11 +296,16 @@ async fn wrapped_main() {
         PROGRESS_BAR.join().unwrap();
     });
 
+    let (stats, tx_stats, stats_handle) = statistics::initialize();
+
     if !CONFIGURATION.time_limit.is_empty() {
         // --time-limit value not an empty string, need to kick off the thread that enforces
         // the limit
+
+        let max_time_stats = stats.clone();
+
         tokio::spawn(async move {
-            scan_manager::start_max_time_thread(&CONFIGURATION.time_limit).await
+            scan_manager::start_max_time_thread(&CONFIGURATION.time_limit, max_time_stats).await
         });
     }
 
@@ -280,8 +319,13 @@ async fn wrapped_main() {
 
     let save_output = !CONFIGURATION.output.is_empty(); // was -o used?
 
+    if CONFIGURATION.save_state {
+        // start the ctrl+c handler
+        scan_manager::initialize(stats.clone());
+    }
+
     let (tx_term, tx_file, term_handle, file_handle) =
-        reporter::initialize(&CONFIGURATION.output, save_output);
+        reporter::initialize(&CONFIGURATION.output, save_output, tx_stats.clone());
 
     // get targets from command line or stdin
     let targets = match get_targets().await {
@@ -289,27 +333,62 @@ async fn wrapped_main() {
         Err(e) => {
             // should only happen in the event that there was an error reading from stdin
             log::error!("{} {}", module_colorizer("main::get_targets"), e);
-            clean_up(tx_term, term_handle, tx_file, file_handle, save_output).await;
+            clean_up(
+                tx_term,
+                term_handle,
+                tx_file,
+                file_handle,
+                tx_stats,
+                stats_handle,
+                save_output,
+            )
+            .await;
             return;
         }
     };
 
+    update_stat!(tx_stats, UpdateUsizeField(InitialTargets, targets.len()));
+
     if !CONFIGURATION.quiet {
         // only print banner if -q isn't used
         let std_stderr = stderr(); // std::io::stderr
-        banner::initialize(&targets, &CONFIGURATION, &VERSION, std_stderr).await;
+        banner::initialize(
+            &targets,
+            &CONFIGURATION,
+            &VERSION,
+            std_stderr,
+            tx_stats.clone(),
+        )
+        .await;
     }
 
     // discard non-responsive targets
-    let live_targets = heuristics::connectivity_test(&targets).await;
+    let live_targets = heuristics::connectivity_test(&targets, tx_stats.clone()).await;
 
     if live_targets.is_empty() {
-        clean_up(tx_term, term_handle, tx_file, file_handle, save_output).await;
+        clean_up(
+            tx_term,
+            term_handle,
+            tx_file,
+            file_handle,
+            tx_stats,
+            stats_handle,
+            save_output,
+        )
+        .await;
         return;
     }
 
     // kick off a scan against any targets determined to be responsive
-    match scan(live_targets, tx_term.clone(), tx_file.clone()).await {
+    match scan(
+        live_targets,
+        stats,
+        tx_term.clone(),
+        tx_file.clone(),
+        tx_stats.clone(),
+    )
+    .await
+    {
         Ok(_) => {
             log::info!("All scans complete!");
         }
@@ -318,12 +397,30 @@ async fn wrapped_main() {
                 &format!("{} while scanning: {}", status_colorizer("Error"), e),
                 &PROGRESS_PRINTER,
             );
-            clean_up(tx_term, term_handle, tx_file, file_handle, save_output).await;
+            clean_up(
+                tx_term,
+                term_handle,
+                tx_file,
+                file_handle,
+                tx_stats,
+                stats_handle,
+                save_output,
+            )
+            .await;
             process::exit(1);
         }
     };
 
-    clean_up(tx_term, term_handle, tx_file, file_handle, save_output).await;
+    clean_up(
+        tx_term,
+        term_handle,
+        tx_file,
+        file_handle,
+        tx_stats,
+        stats_handle,
+        save_output,
+    )
+    .await;
 
     log::trace!("exit: main");
 }
@@ -335,14 +432,18 @@ async fn clean_up(
     term_handle: JoinHandle<()>,
     tx_file: UnboundedSender<FeroxResponse>,
     file_handle: Option<JoinHandle<()>>,
+    tx_stats: UnboundedSender<StatCommand>,
+    stats_handle: JoinHandle<()>,
     save_output: bool,
 ) {
     log::trace!(
-        "enter: clean_up({:?}, {:?}, {:?}, {:?}, {})",
+        "enter: clean_up({:?}, {:?}, {:?}, {:?}, {:?}, {:?}, {})",
         tx_term,
         term_handle,
         tx_file,
         file_handle,
+        tx_stats,
+        stats_handle,
         save_output
     );
 
@@ -377,6 +478,10 @@ async fn clean_up(
         log::trace!("done awaiting file output handler's receiver");
     }
 
+    log::trace!("tx_stats: {:?}", tx_stats);
+    update_stat!(tx_stats, StatCommand::Exit); // send exit command and await the end of the future
+    stats_handle.await.unwrap_or_default();
+
     // mark all scans complete so the terminal input handler will exit cleanly
     SCAN_COMPLETE.store(true, Ordering::Relaxed);
 
@@ -391,11 +496,6 @@ fn main() {
     // setup logging based on the number of -v's used
     logger::initialize(CONFIGURATION.verbosity);
 
-    if CONFIGURATION.save_state {
-        // start the ctrl+c handler
-        scan_manager::initialize();
-    }
-
     // this function uses rlimit, which is not supported on windows
     #[cfg(not(target_os = "windows"))]
     set_open_file_limit(DEFAULT_OPEN_FILE_LIMIT);

src/progress.rs

@@ -1,22 +1,42 @@
 use crate::config::{CONFIGURATION, PROGRESS_BAR};
 use indicatif::{ProgressBar, ProgressStyle};
 
+/// Types of ProgressBars that can be added to `PROGRESS_BAR`
+pub enum BarType {
+    /// no template used / not visible
+    Hidden,
+
+    /// normal directory status bar (reqs/sec shown)
+    Default,
+
+    /// similar to `Default`, except `-` is used in place of line/word/char count
+    Message,
+
+    /// bar used to show overall scan metrics
+    Total,
+}
+
 /// Add an [indicatif::ProgressBar](https://docs.rs/indicatif/latest/indicatif/struct.ProgressBar.html)
 /// to the global [PROGRESS_BAR](../config/struct.PROGRESS_BAR.html)
-pub fn add_bar(prefix: &str, length: u64, hidden: bool, hide_per_sec: bool) -> ProgressBar {
-    let style = if hidden || CONFIGURATION.quiet {
-        ProgressStyle::default_bar().template("")
-    } else if hide_per_sec {
-        ProgressStyle::default_bar()
-            .template(&format!(
+pub fn add_bar(prefix: &str, length: u64, bar_type: BarType) -> ProgressBar {
+    let mut style = ProgressStyle::default_bar().progress_chars("#>-");
+
+    style = if CONFIGURATION.quiet {
+        style.template("")
+    } else {
+        match bar_type {
+            BarType::Hidden => style.template(""),
+            BarType::Default => style.template(
+                "[{bar:.cyan/blue}] - {elapsed:<4} {pos:>7}/{len:7} {per_sec:7} {prefix}",
+            ),
+            BarType::Message => style.template(&format!(
                 "[{{bar:.cyan/blue}}] - {{elapsed:<4}} {{pos:>7}}/{{len:7}} {:7} {{prefix}}",
                 "-"
-            ))
-            .progress_chars("#>-")
-    } else {
-        ProgressStyle::default_bar()
-            .template("[{bar:.cyan/blue}] - {elapsed:<4} {pos:>7}/{len:7} {per_sec:7} {prefix}")
-            .progress_chars("#>-")
+            )),
+            BarType::Total => {
+                style.template("[{bar:.yellow/blue}] - {elapsed:<4} {pos:>7}/{len:7} {eta:7} {msg}")
+            }
+        }
     };
 
     let progress_bar = PROGRESS_BAR.add(ProgressBar::new(length));
@@ -35,16 +55,19 @@ mod tests {
     #[test]
     /// hit all code branches for add_bar
     fn add_bar_with_all_configurations() {
-        let p1 = add_bar("prefix", 2, true, false); // hidden
-        let p2 = add_bar("prefix", 2, false, true); // no per second field
-        let p3 = add_bar("prefix", 2, false, false); // normal bar
+        let p1 = add_bar("prefix", 2, BarType::Hidden); // hidden
+        let p2 = add_bar("prefix", 2, BarType::Message); // no per second field
+        let p3 = add_bar("prefix", 2, BarType::Default); // normal bar
+        let p4 = add_bar("prefix", 2, BarType::Total); // totals bar
 
         p1.finish();
         p2.finish();
         p3.finish();
+        p4.finish();
 
         assert!(p1.is_finished());
         assert!(p2.is_finished());
         assert!(p3.is_finished());
+        assert!(p4.is_finished());
     }
 }

src/reporter.rs

@@ -1,13 +1,19 @@
 use crate::{
     config::{CONFIGURATION, PROGRESS_PRINTER},
     scanner::RESPONSES,
+    statistics::{
+        StatCommand::{self, UpdateUsizeField},
+        StatField::ResourcesDiscovered,
+    },
     utils::{ferox_print, make_request, open_file},
     FeroxChannel, FeroxResponse, FeroxSerialize,
 };
 use console::strip_ansi_codes;
-use std::io::Write;
-use std::sync::{Arc, Once, RwLock};
-use std::{fs, io};
+use std::{
+    fs, io,
+    io::Write,
+    sync::{Arc, Once, RwLock},
+};
 use tokio::sync::mpsc::{self, UnboundedReceiver, UnboundedSender};
 use tokio::task::JoinHandle;
 
@@ -42,27 +48,35 @@ pub fn get_cached_file_handle(filename: &str) -> Option<Arc<RwLock<io::BufWriter
 pub fn initialize(
     output_file: &str,
     save_output: bool,
+    tx_stats: UnboundedSender<StatCommand>,
 ) -> (
     UnboundedSender<FeroxResponse>,
     UnboundedSender<FeroxResponse>,
     JoinHandle<()>,
     Option<JoinHandle<()>>,
 ) {
-    log::trace!("enter: initialize({}, {})", output_file, save_output);
+    log::trace!(
+        "enter: initialize({}, {}, {:?})",
+        output_file,
+        save_output,
+        tx_stats
+    );
 
     let (tx_rpt, rx_rpt): FeroxChannel<FeroxResponse> = mpsc::unbounded_channel();
     let (tx_file, rx_file): FeroxChannel<FeroxResponse> = mpsc::unbounded_channel();
 
     let file_clone = tx_file.clone();
+    let stats_clone = tx_stats.clone();
 
-    let term_reporter =
-        tokio::spawn(async move { spawn_terminal_reporter(rx_rpt, file_clone, save_output).await });
+    let term_reporter = tokio::spawn(async move {
+        spawn_terminal_reporter(rx_rpt, file_clone, stats_clone, save_output).await
+    });
 
     let file_reporter = if save_output {
         // -o used, need to spawn the thread for writing to disk
         let file_clone = output_file.to_string();
         Some(tokio::spawn(async move {
-            spawn_file_reporter(rx_file, &file_clone).await
+            spawn_file_reporter(rx_file, tx_stats, &file_clone).await
         }))
     } else {
         None
@@ -85,12 +99,14 @@ pub fn initialize(
 async fn spawn_terminal_reporter(
     mut resp_chan: UnboundedReceiver<FeroxResponse>,
     file_chan: UnboundedSender<FeroxResponse>,
+    tx_stats: UnboundedSender<StatCommand>,
     save_output: bool,
 ) {
     log::trace!(
-        "enter: spawn_terminal_reporter({:?}, {:?}, {})",
+        "enter: spawn_terminal_reporter({:?}, {:?}, {:?}, {})",
         resp_chan,
         file_chan,
+        tx_stats,
         save_output
     );
 
@@ -105,6 +121,8 @@ async fn spawn_terminal_reporter(
             // print to stdout
             ferox_print(&resp.as_str(), &PROGRESS_PRINTER);
 
+            update_stat!(tx_stats, UpdateUsizeField(ResourcesDiscovered, 1));
+
             if save_output {
                 // -o used, need to send the report to be written out to disk
                 match file_chan.send(resp.clone()) {
@@ -122,7 +140,13 @@ async fn spawn_terminal_reporter(
         if CONFIGURATION.replay_client.is_some() && should_process_response {
             // replay proxy specified/client created and this response's status code is one that
             // should be replayed
-            match make_request(CONFIGURATION.replay_client.as_ref().unwrap(), &resp.url()).await {
+            match make_request(
+                CONFIGURATION.replay_client.as_ref().unwrap(),
+                &resp.url(),
+                tx_stats.clone(),
+            )
+            .await
+            {
                 Ok(_) => {}
                 Err(e) => {
                     log::error!("{}", e);
@@ -151,6 +175,7 @@ async fn spawn_terminal_reporter(
 /// the given reporting criteria
 async fn spawn_file_reporter(
     mut report_channel: UnboundedReceiver<FeroxResponse>,
+    tx_stats: UnboundedSender<StatCommand>,
     output_file: &str,
 ) {
     let buffered_file = match get_cached_file_handle(&CONFIGURATION.output) {
@@ -173,6 +198,8 @@ async fn spawn_file_reporter(
         safe_file_write(&response, buffered_file.clone(), CONFIGURATION.json);
     }
 
+    update_stat!(tx_stats, StatCommand::Save);
+
     log::trace!("exit: spawn_file_reporter");
 }
 

src/scan_manager.rs

@@ -1,11 +1,11 @@
-use crate::config::Configuration;
-use crate::reporter::safe_file_write;
-use crate::utils::open_file;
 use crate::{
-    config::{CONFIGURATION, PROGRESS_PRINTER},
+    config::{Configuration, CONFIGURATION, PROGRESS_PRINTER},
     parser::TIMESPEC_REGEX,
-    progress,
-    scanner::{NUMBER_OF_REQUESTS, RESPONSES, SCANNED_URLS},
+    progress::{add_bar, BarType},
+    reporter::safe_file_write,
+    scanner::{RESPONSES, SCANNED_URLS},
+    statistics::Stats,
+    utils::open_file,
     FeroxResponse, FeroxSerialize, SLEEP_DURATION,
 };
 use console::style;
@@ -16,18 +16,16 @@ use serde::{
     Deserialize, Deserializer, Serialize, Serializer,
 };
 use serde_json::Value;
-use std::collections::HashMap;
 use std::{
     cmp::PartialEq,
+    collections::HashMap,
     fmt,
     fs::File,
     io::BufReader,
-    sync::{Arc, Mutex, RwLock},
-    time::{SystemTime, UNIX_EPOCH},
-};
-use std::{
     io::{stderr, Write},
     sync::atomic::{AtomicBool, AtomicUsize, Ordering},
+    sync::{Arc, Mutex, RwLock},
+    time::{SystemTime, UNIX_EPOCH},
 };
 use tokio::{task::JoinHandle, time};
 use uuid::Uuid;
@@ -75,6 +73,9 @@ pub struct FeroxScan {
     /// The type of scan
     pub scan_type: ScanType,
 
+    /// Number of requests to populate the progress bar with
+    pub num_requests: u64,
+
     /// Whether or not this scan has completed
     pub complete: bool,
 
@@ -95,6 +96,7 @@ impl Default for FeroxScan {
             id: new_id,
             task: None,
             complete: false,
+            num_requests: 0,
             url: String::new(),
             progress_bar: None,
             scan_type: ScanType::File,
@@ -125,8 +127,7 @@ impl FeroxScan {
         if let Some(pb) = &self.progress_bar {
             pb.clone()
         } else {
-            let num_requests = NUMBER_OF_REQUESTS.load(Ordering::Relaxed);
-            let pb = progress::add_bar(&self.url, num_requests, false, false);
+            let pb = add_bar(&self.url, self.num_requests, BarType::Default);
 
             pb.reset_elapsed();
 
@@ -137,14 +138,19 @@ impl FeroxScan {
     }
 
     /// Given a URL and ProgressBar, create a new FeroxScan, wrap it in an Arc and return it
-    pub fn new(url: &str, scan_type: ScanType, pb: Option<ProgressBar>) -> Arc<Mutex<Self>> {
-        let mut me = Self::default();
-
-        me.url = url.to_string();
-        me.scan_type = scan_type;
-        me.progress_bar = pb;
-
-        Arc::new(Mutex::new(me))
+    pub fn new(
+        url: &str,
+        scan_type: ScanType,
+        num_requests: u64,
+        pb: Option<ProgressBar>,
+    ) -> Arc<Mutex<Self>> {
+        Arc::new(Mutex::new(Self {
+            url: url.to_string(),
+            scan_type,
+            num_requests,
+            progress_bar: pb,
+            ..Default::default()
+        }))
     }
 
     /// Mark the scan as complete and stop the scan's progress bar
@@ -187,6 +193,7 @@ impl Serialize for FeroxScan {
         state.serialize_field("url", &self.url)?;
         state.serialize_field("scan_type", &self.scan_type)?;
         state.serialize_field("complete", &self.complete)?;
+        state.serialize_field("num_requests", &self.num_requests)?;
 
         state.end()
     }
@@ -229,6 +236,11 @@ impl<'de> Deserialize<'de> for FeroxScan {
                         scan.url = url.to_string();
                     }
                 }
+                "num_requests" => {
+                    if let Some(num_requests) = value.as_u64() {
+                        scan.num_requests = num_requests;
+                    }
+                }
                 _ => {}
             }
         }
@@ -438,15 +450,17 @@ impl FeroxScans {
     /// If `FeroxScans` did not already contain the scan, return true; otherwise return false
     ///
     /// Also return a reference to the new `FeroxScan`
-    fn add_scan(&self, url: &str, scan_type: ScanType) -> (bool, Arc<Mutex<FeroxScan>>) {
+    fn add_scan(
+        &self,
+        url: &str,
+        scan_type: ScanType,
+        stats: Arc<Stats>,
+    ) -> (bool, Arc<Mutex<FeroxScan>>) {
+        let num_requests = stats.expected_per_scan.load(Ordering::Relaxed) as u64;
+
         let bar = match scan_type {
             ScanType::Directory => {
-                let progress_bar = progress::add_bar(
-                    &url,
-                    NUMBER_OF_REQUESTS.load(Ordering::Relaxed),
-                    false,
-                    false,
-                );
+                let progress_bar = add_bar(&url, num_requests, BarType::Default);
 
                 progress_bar.reset_elapsed();
 
@@ -455,7 +469,7 @@ impl FeroxScans {
             ScanType::File => None,
         };
 
-        let ferox_scan = FeroxScan::new(&url, scan_type, bar);
+        let ferox_scan = FeroxScan::new(&url, scan_type, num_requests, bar);
 
         // If the set did not contain the scan, true is returned.
         // If the set did contain the scan, false is returned.
@@ -469,8 +483,12 @@ impl FeroxScans {
     /// If `FeroxScans` did not already contain the scan, return true; otherwise return false
     ///
     /// Also return a reference to the new `FeroxScan`
-    pub fn add_directory_scan(&self, url: &str) -> (bool, Arc<Mutex<FeroxScan>>) {
-        self.add_scan(&url, ScanType::Directory)
+    pub fn add_directory_scan(
+        &self,
+        url: &str,
+        stats: Arc<Stats>,
+    ) -> (bool, Arc<Mutex<FeroxScan>>) {
+        self.add_scan(&url, ScanType::Directory, stats)
     }
 
     /// Given a url, create a new `FeroxScan` and add it to `FeroxScans` as a File Scan
@@ -478,8 +496,8 @@ impl FeroxScans {
     /// If `FeroxScans` did not already contain the scan, return true; otherwise return false
     ///
     /// Also return a reference to the new `FeroxScan`
-    pub fn add_file_scan(&self, url: &str) -> (bool, Arc<Mutex<FeroxScan>>) {
-        self.add_scan(&url, ScanType::File)
+    pub fn add_file_scan(&self, url: &str, stats: Arc<Stats>) -> (bool, Arc<Mutex<FeroxScan>>) {
+        self.add_scan(&url, ScanType::File, stats)
     }
 }
 
@@ -575,6 +593,9 @@ pub struct FeroxState {
 
     /// Known responses
     responses: &'static FeroxResponses,
+
+    /// Gathered statistics
+    statistics: Arc<Stats>,
 }
 
 /// FeroxSerialize implementation for FeroxState
@@ -594,7 +615,7 @@ impl FeroxSerialize for FeroxState {
 /// that representation to seconds and then wait for those seconds to elapse.  Once that period
 /// of time has elapsed, kill all currently running scans and dump a state file to disk that can
 /// be used to resume any unfinished scan.
-pub async fn start_max_time_thread(time_spec: &str) {
+pub async fn start_max_time_thread(time_spec: &str, stats: Arc<Stats>) {
     log::trace!("enter: start_max_time_thread({})", time_spec);
 
     // as this function has already made it through the parser, which calls is_match on
@@ -624,9 +645,9 @@ pub async fn start_max_time_thread(time_spec: &str) {
         log::trace!("exit: start_max_time_thread");
 
         #[cfg(test)]
-        panic!();
+        panic!(stats);
         #[cfg(not(test))]
-        sigint_handler();
+        sigint_handler(stats);
     }
 
     log::error!(
@@ -636,8 +657,8 @@ pub async fn start_max_time_thread(time_spec: &str) {
 }
 
 /// Writes the current state of the program to disk (if save_state is true) and then exits
-fn sigint_handler() {
-    log::trace!("enter: sigint_handler");
+fn sigint_handler(stats: Arc<Stats>) {
+    log::trace!("enter: sigint_handler({:?})", stats);
 
     let ts = SystemTime::now()
         .duration_since(UNIX_EPOCH)
@@ -669,6 +690,7 @@ fn sigint_handler() {
         config: &CONFIGURATION,
         scans: &SCANNED_URLS,
         responses: &RESPONSES,
+        statistics: stats,
     };
 
     let state_file = open_file(&filename);
@@ -682,10 +704,12 @@ fn sigint_handler() {
 }
 
 /// Initialize the ctrl+c handler that saves scan state to disk
-pub fn initialize() {
-    log::trace!("enter: initialize");
+pub fn initialize(stats: Arc<Stats>) {
+    log::trace!("enter: initialize({:?})", stats);
 
-    let result = ctrlc::set_handler(sigint_handler);
+    let result = ctrlc::set_handler(move || {
+        sigint_handler(stats.clone());
+    });
 
     if result.is_err() {
         log::error!("Could not set Ctrl+c handler");
@@ -795,8 +819,9 @@ mod tests {
     /// add an unknown url to the hashset, expect true
     fn add_url_to_list_of_scanned_urls_with_unknown_url() {
         let urls = FeroxScans::default();
+        let stats = Arc::new(Stats::new());
         let url = "http://unknown_url";
-        let (result, _scan) = urls.add_scan(url, ScanType::Directory);
+        let (result, _scan) = urls.add_scan(url, ScanType::Directory, stats);
         assert_eq!(result, true);
     }
 
@@ -806,11 +831,13 @@ mod tests {
         let urls = FeroxScans::default();
         let pb = ProgressBar::new(1);
         let url = "http://unknown_url/";
-        let scan = FeroxScan::new(url, ScanType::Directory, Some(pb));
+        let stats = Arc::new(Stats::new());
+
+        let scan = FeroxScan::new(url, ScanType::Directory, pb.length(), Some(pb));
 
         assert_eq!(urls.insert(scan), true);
 
-        let (result, _scan) = urls.add_scan(url, ScanType::Directory);
+        let (result, _scan) = urls.add_scan(url, ScanType::Directory, stats);
 
         assert_eq!(result, false);
     }
@@ -820,7 +847,8 @@ mod tests {
     fn abort_stops_progress_bar() {
         let pb = ProgressBar::new(1);
         let url = "http://unknown_url/";
-        let scan = FeroxScan::new(url, ScanType::Directory, Some(pb));
+
+        let scan = FeroxScan::new(url, ScanType::Directory, pb.length(), Some(pb));
 
         assert_eq!(
             scan.lock()
@@ -850,11 +878,13 @@ mod tests {
     fn add_url_to_list_of_scanned_urls_with_known_url_without_slash() {
         let urls = FeroxScans::default();
         let url = "http://unknown_url";
-        let scan = FeroxScan::new(url, ScanType::File, None);
+        let stats = Arc::new(Stats::new());
+
+        let scan = FeroxScan::new(url, ScanType::File, 0, None);
 
         assert_eq!(urls.insert(scan), true);
 
-        let (result, _scan) = urls.add_scan(url, ScanType::File);
+        let (result, _scan) = urls.add_scan(url, ScanType::File, stats);
 
         assert_eq!(result, false);
     }
@@ -867,8 +897,8 @@ mod tests {
         let pb_two = ProgressBar::new(2);
         let url = "http://unknown_url/";
         let url_two = "http://unknown_url/fa";
-        let scan = FeroxScan::new(url, ScanType::Directory, Some(pb));
-        let scan_two = FeroxScan::new(url_two, ScanType::Directory, Some(pb_two));
+        let scan = FeroxScan::new(url, ScanType::Directory, pb.length(), Some(pb));
+        let scan_two = FeroxScan::new(url_two, ScanType::Directory, pb_two.length(), Some(pb_two));
 
         scan_two.lock().unwrap().finish(); // one complete, one incomplete
 
@@ -881,8 +911,8 @@ mod tests {
     /// ensure that PartialEq compares FeroxScan.id fields
     fn partial_eq_compares_the_id_field() {
         let url = "http://unknown_url/";
-        let scan = FeroxScan::new(url, ScanType::Directory, None);
-        let scan_two = FeroxScan::new(url, ScanType::Directory, None);
+        let scan = FeroxScan::new(url, ScanType::Directory, 0, None);
+        let scan_two = FeroxScan::new(url, ScanType::Directory, 0, None);
 
         assert!(!scan.lock().unwrap().eq(&scan_two.lock().unwrap()));
 
@@ -941,9 +971,9 @@ mod tests {
     #[test]
     /// given a FeroxScan, test that it serializes into the proper JSON entry
     fn ferox_scan_serialize() {
-        let fs = FeroxScan::new("https://spiritanimal.com", ScanType::Directory, None);
+        let fs = FeroxScan::new("https://spiritanimal.com", ScanType::Directory, 0, None);
         let fs_json = format!(
-            r#"{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","complete":false}}"#,
+            r#"{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","complete":false,"num_requests":0}}"#,
             fs.lock().unwrap().id
         );
         assert_eq!(
@@ -955,10 +985,10 @@ mod tests {
     #[test]
     /// given a FeroxScans, test that it serializes into the proper JSON entry
     fn ferox_scans_serialize() {
-        let ferox_scan = FeroxScan::new("https://spiritanimal.com", ScanType::Directory, None);
+        let ferox_scan = FeroxScan::new("https://spiritanimal.com", ScanType::Directory, 0, None);
         let ferox_scans = FeroxScans::default();
         let ferox_scans_json = format!(
-            r#"[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","complete":false}}]"#,
+            r#"[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","complete":false,"num_requests":0}}]"#,
             ferox_scan.lock().unwrap().id
         );
         ferox_scans.scans.lock().unwrap().push(ferox_scan);
@@ -1009,10 +1039,12 @@ mod tests {
     #[test]
     /// test FeroxSerialize implementation of FeroxState
     fn feroxstates_feroxserialize_implementation() {
-        let ferox_scan = FeroxScan::new("https://spiritanimal.com", ScanType::Directory, None);
+        let ferox_scan = FeroxScan::new("https://spiritanimal.com", ScanType::Directory, 0, None);
         let saved_id = ferox_scan.lock().unwrap().id.clone();
         SCANNED_URLS.insert(ferox_scan);
 
+        let stats = Arc::new(Stats::new());
+
         let json_response = r#"{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{"server":"nginx/1.16.1"}}"#;
         let response: FeroxResponse = serde_json::from_str(json_response).unwrap();
         RESPONSES.insert(response);
@@ -1021,6 +1053,7 @@ mod tests {
             scans: &SCANNED_URLS,
             responses: &RESPONSES,
             config: &CONFIGURATION,
+            statistics: stats,
         };
 
         let expected_strs = predicates::str::contains("scans: FeroxScans").and(
@@ -1035,11 +1068,11 @@ mod tests {
 
         let json_state = ferox_state.as_json();
         let expected = format!(
-            r#"{{"scans":[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","complete":false}}],"config":{{"type":"configuration","wordlist":"/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt","config":"","proxy":"","replay_proxy":"","target_url":"","status_codes":[200,204,301,302,307,308,401,403,405],"replay_codes":[200,204,301,302,307,308,401,403,405],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"quiet":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/{}","redirects":false,"insecure":false,"extensions":[],"headers":{{}},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":false,"depth":4,"scan_limit":0,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false,"resumed":false,"save_state":false,"time_limit":"","filter_similar":[]}},"responses":[{{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{{"server":"nginx/1.16.1"}}}}]}}"#,
+            r#"{{"scans":[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","complete":false,"num_requests":0}}],"config":{{"type":"configuration","wordlist":"/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt","config":"","proxy":"","replay_proxy":"","target_url":"","status_codes":[200,204,301,302,307,308,401,403,405],"replay_codes":[200,204,301,302,307,308,401,403,405],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"quiet":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/{}","redirects":false,"insecure":false,"extensions":[],"headers":{{}},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":false,"depth":4,"scan_limit":0,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false,"resumed":false,"resume_from":"","save_state":false,"time_limit":"","filter_similar":[]}},"responses":[{{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{{"server":"nginx/1.16.1"}}}}]"#,
             saved_id, VERSION
         );
         println!("{}\n{}", expected, json_state);
-        assert!(predicates::str::similar(expected).eval(&json_state));
+        assert!(predicates::str::contains(expected).eval(&json_state));
     }
 
     #[should_panic]
@@ -1049,8 +1082,9 @@ mod tests {
     async fn start_max_time_thread_panics_after_delay() {
         let now = time::Instant::now();
         let delay = time::Duration::new(3, 0);
+        let stats = Arc::new(Stats::new());
 
-        start_max_time_thread("3s").await;
+        start_max_time_thread("3s", stats).await;
 
         assert!(now.elapsed() > delay);
     }
@@ -1061,9 +1095,10 @@ mod tests {
     async fn start_max_time_thread_returns_immediately_with_too_large_input() {
         let now = time::Instant::now();
         let delay = time::Duration::new(1, 0);
+        let stats = Arc::new(Stats::new());
 
         // pub const MAX: usize = usize::MAX; // 18_446_744_073_709_551_615usize
-        start_max_time_thread("18446744073709551616m").await; // can't fit in dest u64
+        start_max_time_thread("18446744073709551616m", stats).await; // can't fit in dest u64
 
         assert!(now.elapsed() < delay); // assuming function call will take less than 1second
     }

src/scanner.rs

@@ -1,3 +1,4 @@
+use crate::statistics::Stats;
 use crate::{
     config::{Configuration, CONFIGURATION},
     extractor::{get_links, request_feroxresponse_from_new_link},
@@ -7,6 +8,10 @@ use crate::{
     },
     heuristics,
     scan_manager::{FeroxResponses, FeroxScans, PAUSE_SCAN},
+    statistics::{
+        StatCommand::{self, UpdateF64Field, UpdateUsizeField},
+        StatField::{DirScanTimes, ExpectedPerScan, TotalScans, WildcardsFiltered},
+    },
     utils::{format_url, get_current_depth, make_request},
     FeroxChannel, FeroxResponse, SIMILARITY_THRESHOLD,
 };
@@ -24,8 +29,9 @@ use std::{
     collections::HashSet,
     convert::TryInto,
     ops::Deref,
-    sync::atomic::{AtomicU64, AtomicUsize, Ordering},
+    sync::atomic::{AtomicUsize, Ordering},
     sync::{Arc, RwLock},
+    time::Instant,
 };
 use tokio::{
     sync::{
@@ -35,12 +41,13 @@ use tokio::{
     task::JoinHandle,
 };
 
-/// Single atomic number that gets incremented once, used to track first scan vs. all others
+/// Single atomic number that gets incremented at least once, used to track first scan(s) vs. all
+/// others found during recursion
+///
+/// -u means this will be incremented once
+/// --stdin means this will be incremented by the number of targets passed via STDIN
 static CALL_COUNT: AtomicUsize = AtomicUsize::new(0);
 
-/// Single atomic number that gets holds the number of requests to be sent per directory scanned
-pub static NUMBER_OF_REQUESTS: AtomicU64 = AtomicU64::new(0);
-
 lazy_static! {
     /// Set of urls that have been sent to [scan_url](fn.scan_url.html), used for deduplication
     pub static ref SCANNED_URLS: FeroxScans = FeroxScans::default();
@@ -98,35 +105,41 @@ fn spawn_recursion_handler(
     mut recursion_channel: UnboundedReceiver<String>,
     wordlist: Arc<HashSet<String>>,
     base_depth: usize,
-    num_targets: usize,
+    stats: Arc<Stats>,
     tx_term: UnboundedSender<FeroxResponse>,
     tx_file: UnboundedSender<FeroxResponse>,
+    tx_stats: UnboundedSender<StatCommand>,
 ) -> BoxFuture<'static, Vec<JoinHandle<()>>> {
     log::trace!(
-        "enter: spawn_recursion_handler({:?}, wordlist[{} words...], {}, {}, {:?}, {:?})",
+        "enter: spawn_recursion_handler({:?}, wordlist[{} words...], {}, {:?}, {:?}, {:?}, {:?})",
         recursion_channel,
         wordlist.len(),
         base_depth,
-        num_targets,
+        stats,
         tx_term,
-        tx_file
+        tx_file,
+        tx_stats
     );
 
     let boxed_future = async move {
         let mut scans = vec![];
 
         while let Some(resp) = recursion_channel.recv().await {
-            let (unknown, _) = SCANNED_URLS.add_directory_scan(&resp);
+            let (unknown, _) = SCANNED_URLS.add_directory_scan(&resp, stats.clone());
 
             if !unknown {
                 // not unknown, i.e. we've seen the url before and don't need to scan again
                 continue;
             }
 
+            update_stat!(tx_stats, UpdateUsizeField(TotalScans, 1));
+
             log::info!("received {} on recursion channel", resp);
 
             let term_clone = tx_term.clone();
             let file_clone = tx_file.clone();
+            let tx_stats_clone = tx_stats.clone();
+            let stats_clone = stats.clone();
             let resp_clone = resp.clone();
             let list_clone = wordlist.clone();
 
@@ -135,9 +148,10 @@ fn spawn_recursion_handler(
                     resp_clone.to_owned().as_str(),
                     list_clone,
                     base_depth,
-                    num_targets,
+                    stats_clone,
                     term_clone,
                     file_clone,
+                    tx_stats_clone,
                 )
                 .await
             });
@@ -158,12 +172,18 @@ fn spawn_recursion_handler(
 ///
 /// If any extensions were passed to the program, each extension will add a
 /// (base_url + word + ext) Url to the vector
-fn create_urls(target_url: &str, word: &str, extensions: &[String]) -> Vec<Url> {
+fn create_urls(
+    target_url: &str,
+    word: &str,
+    extensions: &[String],
+    tx_stats: UnboundedSender<StatCommand>,
+) -> Vec<Url> {
     log::trace!(
-        "enter: create_urls({}, {}, {:?})",
+        "enter: create_urls({}, {}, {:?}, {:?})",
         target_url,
         word,
-        extensions
+        extensions,
+        tx_stats
     );
 
     let mut urls = vec![];
@@ -174,6 +194,7 @@ fn create_urls(target_url: &str, word: &str, extensions: &[String]) -> Vec<Url>
         CONFIGURATION.add_slash,
         &CONFIGURATION.queries,
         None,
+        tx_stats.clone(),
     ) {
         urls.push(url); // default request, i.e. no extension
     }
@@ -185,6 +206,7 @@ fn create_urls(target_url: &str, word: &str, extensions: &[String]) -> Vec<Url>
             CONFIGURATION.add_slash,
             &CONFIGURATION.queries,
             Some(ext),
+            tx_stats.clone(),
         ) {
             urls.push(url); // any extensions passed in
         }
@@ -284,7 +306,7 @@ async fn try_recursion(
         "enter: try_recursion({}, {}, {:?})",
         response,
         base_depth,
-        transmitter
+        transmitter,
     );
 
     if !reached_max_depth(response.url(), base_depth, CONFIGURATION.depth)
@@ -328,12 +350,18 @@ async fn try_recursion(
 
 /// Simple helper to stay DRY; determines whether or not a given `FeroxResponse` should be reported
 /// to the user or not.
-pub fn should_filter_response(response: &FeroxResponse) -> bool {
+pub fn should_filter_response(
+    response: &FeroxResponse,
+    tx_stats: UnboundedSender<StatCommand>,
+) -> bool {
     match FILTERS.read() {
         Ok(filters) => {
             for filter in filters.iter() {
                 // wildcard.should_filter goes here
                 if filter.should_filter_response(&response) {
+                    if filter.as_any().downcast_ref::<WildcardFilter>().is_some() {
+                        update_stat!(tx_stats, UpdateUsizeField(WildcardsFiltered, 1))
+                    }
                     return true;
                 }
             }
@@ -354,22 +382,31 @@ async fn make_requests(
     target_url: &str,
     word: &str,
     base_depth: usize,
+    stats: Arc<Stats>,
     dir_chan: UnboundedSender<String>,
     report_chan: UnboundedSender<FeroxResponse>,
+    tx_stats: UnboundedSender<StatCommand>,
 ) {
     log::trace!(
-        "enter: make_requests({}, {}, {}, {:?}, {:?})",
+        "enter: make_requests({}, {}, {}, {:?}, {:?}, {:?}, {:?})",
         target_url,
         word,
         base_depth,
+        stats,
         dir_chan,
-        report_chan
+        report_chan,
+        tx_stats
     );
 
-    let urls = create_urls(&target_url, &word, &CONFIGURATION.extensions);
+    let urls = create_urls(
+        &target_url,
+        &word,
+        &CONFIGURATION.extensions,
+        tx_stats.clone(),
+    );
 
     for url in urls {
-        if let Ok(response) = make_request(&CONFIGURATION.client, &url).await {
+        if let Ok(response) = make_request(&CONFIGURATION.client, &url, tx_stats.clone()).await {
             // response came back without error, convert it to FeroxResponse
             let ferox_response = FeroxResponse::from(response, true).await;
 
@@ -381,22 +418,26 @@ async fn make_requests(
             // purposefully doing recursion before filtering. the thought process is that
             // even though this particular url is filtered, subsequent urls may not
 
-            if should_filter_response(&ferox_response) {
+            if should_filter_response(&ferox_response, tx_stats.clone()) {
                 continue;
             }
 
             if CONFIGURATION.extract_links && !ferox_response.status().is_redirection() {
-                let new_links = get_links(&ferox_response).await;
+                let new_links = get_links(&ferox_response, tx_stats.clone()).await;
 
                 for new_link in new_links {
-                    let mut new_ferox_response =
-                        match request_feroxresponse_from_new_link(&new_link).await {
-                            Some(resp) => resp,
-                            None => continue,
-                        };
+                    let mut new_ferox_response = match request_feroxresponse_from_new_link(
+                        &new_link,
+                        tx_stats.clone(),
+                    )
+                    .await
+                    {
+                        Some(resp) => resp,
+                        None => continue,
+                    };
 
                     // filter if necessary
-                    if should_filter_response(&new_ferox_response) {
+                    if should_filter_response(&new_ferox_response, tx_stats.clone()) {
                         continue;
                     }
 
@@ -404,7 +445,8 @@ async fn make_requests(
                         // very likely a file, simply request and report
                         log::debug!("Singular extraction: {}", new_ferox_response);
 
-                        SCANNED_URLS.add_file_scan(&new_ferox_response.url().to_string());
+                        SCANNED_URLS
+                            .add_file_scan(&new_ferox_response.url().to_string(), stats.clone());
 
                         send_report(report_chan.clone(), new_ferox_response);
 
@@ -457,30 +499,36 @@ pub async fn scan_url(
     target_url: &str,
     wordlist: Arc<HashSet<String>>,
     base_depth: usize,
-    num_targets: usize,
+    stats: Arc<Stats>,
     tx_term: UnboundedSender<FeroxResponse>,
     tx_file: UnboundedSender<FeroxResponse>,
+    tx_stats: UnboundedSender<StatCommand>,
 ) {
     log::trace!(
-        "enter: scan_url({:?}, wordlist[{} words...], {}, {}, {:?}, {:?})",
+        "enter: scan_url({:?}, wordlist[{} words...], {}, {:?}, {:?}, {:?}, {:?})",
         target_url,
         wordlist.len(),
         base_depth,
-        num_targets,
+        stats,
         tx_term,
-        tx_file
+        tx_file,
+        tx_stats
     );
 
     log::info!("Starting scan against: {}", target_url);
 
+    let scan_timer = Instant::now();
+
     let (tx_dir, rx_dir): FeroxChannel<String> = mpsc::unbounded_channel();
 
-    if CALL_COUNT.load(Ordering::Relaxed) < num_targets {
+    if CALL_COUNT.load(Ordering::Relaxed) < stats.initial_targets.load(Ordering::Relaxed) {
         CALL_COUNT.fetch_add(1, Ordering::Relaxed);
 
+        update_stat!(tx_stats, UpdateUsizeField(TotalScans, 1));
+
         // this protection allows us to add the first scanned url to SCANNED_URLS
         // from within the scan_url function instead of the recursion handler
-        SCANNED_URLS.add_directory_scan(&target_url);
+        SCANNED_URLS.add_directory_scan(&target_url, stats.clone());
     }
 
     let ferox_scan = match SCANNED_URLS.get_scan_by_url(&target_url) {
@@ -511,29 +559,39 @@ pub async fn scan_url(
     // Arc clones to be passed around to the various scans
     let wildcard_bar = progress_bar.clone();
     let heuristics_term_clone = tx_term.clone();
+    let heuristics_stats_clone = tx_stats.clone();
     let recurser_term_clone = tx_term.clone();
     let recurser_file_clone = tx_file.clone();
+    let recurser_stats_clone = tx_stats.clone();
     let recurser_words = wordlist.clone();
     let looping_words = wordlist.clone();
+    let looping_stats = stats.clone();
 
     let recurser = tokio::spawn(async move {
         spawn_recursion_handler(
             rx_dir,
             recurser_words,
             base_depth,
-            num_targets,
+            stats.clone(),
             recurser_term_clone,
             recurser_file_clone,
+            recurser_stats_clone,
         )
         .await
     });
 
     // add any wildcard filters to `FILTERS`
-    let filter =
-        match heuristics::wildcard_test(&target_url, wildcard_bar, heuristics_term_clone).await {
-            Some(f) => Box::new(f),
-            None => Box::new(WildcardFilter::default()),
-        };
+    let filter = match heuristics::wildcard_test(
+        &target_url,
+        wildcard_bar,
+        heuristics_term_clone,
+        heuristics_stats_clone,
+    )
+    .await
+    {
+        Some(f) => Box::new(f),
+        None => Box::new(WildcardFilter::default()),
+    };
 
     add_filter_to_list_of_ferox_filters(filter, FILTERS.clone());
 
@@ -542,8 +600,10 @@ pub async fn scan_url(
         .map(|word| {
             let txd = tx_dir.clone();
             let txr = tx_term.clone();
+            let txs = tx_stats.clone();
             let pb = progress_bar.clone(); // progress bar is an Arc around internal state
             let tgt = target_url.to_string(); // done to satisfy 'static lifetime below
+            let lst = looping_stats.clone();
             (
                 tokio::spawn(async move {
                     if PAUSE_SCAN.load(Ordering::Acquire) {
@@ -554,7 +614,7 @@ pub async fn scan_url(
                         // todo change to true when issue #107 is resolved
                         SCANNED_URLS.pause(false).await;
                     }
-                    make_requests(&tgt, &word, base_depth, txd, txr).await
+                    make_requests(&tgt, &word, base_depth, lst, txd, txr, txs).await
                 }),
                 pb,
             )
@@ -575,6 +635,11 @@ pub async fn scan_url(
     producers.await;
     log::trace!("done awaiting scan producers");
 
+    update_stat!(
+        tx_stats,
+        UpdateF64Field(DirScanTimes, scan_timer.elapsed().as_secs_f64())
+    );
+
     // drop the current permit so the semaphore will allow another scan to proceed
     drop(permit);
 
@@ -596,8 +661,17 @@ pub async fn scan_url(
 
 /// Perform steps necessary to run scans that only need to be performed once (warming up the
 /// engine, as it were)
-pub async fn initialize(num_words: usize, config: &Configuration) {
-    log::trace!("enter: initialize({}, {:?})", num_words, config,);
+pub async fn initialize(
+    num_words: usize,
+    config: &Configuration,
+    tx_stats: UnboundedSender<StatCommand>,
+) {
+    log::trace!(
+        "enter: initialize({}, {:?}, {:?})",
+        num_words,
+        config,
+        tx_stats
+    );
 
     // number of requests only needs to be calculated once, and then can be reused
     let num_reqs_expected: u64 = if config.extensions.is_empty() {
@@ -607,7 +681,11 @@ pub async fn initialize(num_words: usize, config: &Configuration) {
         total.try_into().unwrap()
     };
 
-    NUMBER_OF_REQUESTS.store(num_reqs_expected, Ordering::Relaxed);
+    // tell Stats object about the number of expected requests
+    update_stat!(
+        tx_stats,
+        UpdateUsizeField(ExpectedPerScan, num_reqs_expected as usize)
+    );
 
     // add any status code filters to `FILTERS` (-C|--filter-status)
     for code_filter in &config.filter_status {
@@ -670,9 +748,16 @@ pub async fn initialize(num_words: usize, config: &Configuration) {
     // add any similarity filters to `FILTERS` (--filter-similar-to)
     for similarity_filter in &config.filter_similar {
         // url as-is based on input, ignores user-specified url manipulation options (add-slash etc)
-        if let Ok(url) = format_url(&similarity_filter, &"", false, &Vec::new(), None) {
+        if let Ok(url) = format_url(
+            &similarity_filter,
+            &"",
+            false,
+            &Vec::new(),
+            None,
+            tx_stats.clone(),
+        ) {
             // attempt to request the given url
-            if let Ok(resp) = make_request(&CONFIGURATION.client, &url).await {
+            if let Ok(resp) = make_request(&CONFIGURATION.client, &url, tx_stats.clone()).await {
                 // if successful, create a filter based on the response's body
                 let fr = FeroxResponse::from(resp, true).await;
 
@@ -707,14 +792,16 @@ mod tests {
     #[test]
     /// sending url + word without any extensions should get back one url with the joined word
     fn create_urls_no_extension_returns_base_url_with_word() {
-        let urls = create_urls("http://localhost", "turbo", &[]);
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+        let urls = create_urls("http://localhost", "turbo", &[], tx);
         assert_eq!(urls, [Url::parse("http://localhost/turbo").unwrap()])
     }
 
     #[test]
     /// sending url + word + 1 extension should get back two urls, one base and one with extension
     fn create_urls_one_extension_returns_two_urls() {
-        let urls = create_urls("http://localhost", "turbo", &[String::from("js")]);
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+        let urls = create_urls("http://localhost", "turbo", &[String::from("js")], tx);
         assert_eq!(
             urls,
             [
@@ -752,8 +839,10 @@ mod tests {
             vec![base, js, php, pdf, tar],
         ];
 
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
         for (i, ext_set) in ext_vec.into_iter().enumerate() {
-            let urls = create_urls("http://localhost", "turbo", &ext_set);
+            let urls = create_urls("http://localhost", "turbo", &ext_set, tx.clone());
             assert_eq!(urls, expected[i]);
         }
     }
@@ -802,8 +891,11 @@ mod tests {
     #[should_panic]
     /// call initialize with a bad regex, triggering a panic
     async fn initialize_panics_on_bad_regex() {
-        let mut config = Configuration::default();
-        config.filter_regex = vec![r"(".to_string()];
-        initialize(1, &config).await;
+        let config = Configuration {
+            filter_regex: vec![r"(".to_string()],
+            ..Default::default()
+        };
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+        initialize(1, &config, tx).await;
     }
 }

src/statistics.rs

@@ -0,0 +1,793 @@
+use crate::{
+    config::CONFIGURATION,
+    progress::{add_bar, BarType},
+    reporter::{get_cached_file_handle, safe_file_write},
+    FeroxChannel, FeroxSerialize,
+};
+use console::style;
+use indicatif::ProgressBar;
+use reqwest::StatusCode;
+use serde::{Deserialize, Serialize};
+use std::{
+    fs::File,
+    io::BufReader,
+    sync::{
+        atomic::{AtomicUsize, Ordering},
+        Arc, Mutex,
+    },
+    time::Instant,
+};
+use tokio::{
+    sync::mpsc::{self, UnboundedReceiver, UnboundedSender},
+    task::JoinHandle,
+};
+
+/// Wrapper `Atomic*.fetch_add` to save me from writing Ordering::Relaxed a bajillion times
+///
+/// default is to increment by 1, second arg can be used to increment by a different value
+macro_rules! atomic_increment {
+    ($metric:expr) => {
+        $metric.fetch_add(1, Ordering::Relaxed);
+    };
+
+    ($metric:expr, $value:expr) => {
+        $metric.fetch_add($value, Ordering::Relaxed);
+    };
+}
+
+/// Wrapper around `Atomic*.load` to save me from writing Ordering::Relaxed a bajillion times
+macro_rules! atomic_load {
+    ($metric:expr) => {
+        $metric.load(Ordering::Relaxed);
+    };
+}
+
+/// Data collection of statistics related to a scan
+#[derive(Default, Deserialize, Debug, Serialize)]
+pub struct Stats {
+    #[serde(rename = "type")]
+    /// Name of this type of struct, used for serialization, i.e. `{"type":"statistics"}`
+    kind: String,
+
+    /// tracker for number of timeouts seen by the client
+    timeouts: AtomicUsize,
+
+    /// tracker for total number of requests sent by the client
+    requests: AtomicUsize,
+
+    /// tracker for total number of requests expected to send if the scan runs to completion
+    ///
+    /// Note: this is a per-scan expectation; `expected_requests * current # of scans` would be
+    /// indicative of the current expectation at any given time, but is a moving target.  
+    pub expected_per_scan: AtomicUsize,
+
+    /// tracker for accumulating total number of requests expected (i.e. as a new scan is started
+    /// this value should increase by `expected_requests`
+    total_expected: AtomicUsize,
+
+    /// tracker for total number of errors encountered by the client
+    errors: AtomicUsize,
+
+    /// tracker for overall number of 2xx status codes seen by the client
+    successes: AtomicUsize,
+
+    /// tracker for overall number of 3xx status codes seen by the client
+    redirects: AtomicUsize,
+
+    /// tracker for overall number of 4xx status codes seen by the client
+    client_errors: AtomicUsize,
+
+    /// tracker for overall number of 5xx status codes seen by the client
+    server_errors: AtomicUsize,
+
+    /// tracker for number of scans performed, this directly equates to number of directories
+    /// recursed into and affects the total number of expected requests
+    total_scans: AtomicUsize,
+
+    /// tracker for initial number of requested targets
+    pub initial_targets: AtomicUsize,
+
+    /// tracker for number of links extracted when `--extract-links` is used; sources are
+    /// response bodies and robots.txt as of v1.11.0
+    links_extracted: AtomicUsize,
+
+    /// tracker for overall number of 200s seen by the client
+    status_200s: AtomicUsize,
+
+    /// tracker for overall number of 301s seen by the client
+    status_301s: AtomicUsize,
+
+    /// tracker for overall number of 302s seen by the client
+    status_302s: AtomicUsize,
+
+    /// tracker for overall number of 401s seen by the client
+    status_401s: AtomicUsize,
+
+    /// tracker for overall number of 403s seen by the client
+    status_403s: AtomicUsize,
+
+    /// tracker for overall number of 429s seen by the client
+    status_429s: AtomicUsize,
+
+    /// tracker for overall number of 500s seen by the client
+    status_500s: AtomicUsize,
+
+    /// tracker for overall number of 503s seen by the client
+    status_503s: AtomicUsize,
+
+    /// tracker for overall number of 504s seen by the client
+    status_504s: AtomicUsize,
+
+    /// tracker for overall number of 508s seen by the client
+    status_508s: AtomicUsize,
+
+    /// tracker for overall number of wildcard urls filtered out by the client
+    wildcards_filtered: AtomicUsize,
+
+    /// tracker for overall number of all filtered responses
+    responses_filtered: AtomicUsize,
+
+    /// tracker for number of files found
+    resources_discovered: AtomicUsize,
+
+    /// tracker for number of errors triggered during URL formatting
+    url_format_errors: AtomicUsize,
+
+    /// tracker for number of errors triggered by the `reqwest::RedirectPolicy`
+    redirection_errors: AtomicUsize,
+
+    /// tracker for number of errors related to the connecting
+    connection_errors: AtomicUsize,
+
+    /// tracker for number of errors related to the request used
+    request_errors: AtomicUsize,
+
+    /// tracker for each directory's total scan time in seconds as a float
+    directory_scan_times: Mutex<Vec<f64>>,
+
+    /// tracker for total runtime
+    total_runtime: Mutex<Vec<f64>>,
+}
+
+/// FeroxSerialize implementation for Stats
+impl FeroxSerialize for Stats {
+    /// Simply return debug format of Stats to satisfy as_str
+    fn as_str(&self) -> String {
+        String::new()
+    }
+
+    /// Simple call to produce a JSON string using the given Stats object
+    fn as_json(&self) -> String {
+        serde_json::to_string(&self).unwrap_or_default()
+    }
+}
+
+/// implementation of statistics data collection struct
+impl Stats {
+    /// Small wrapper for default to set `kind` to "statistics" and `total_runtime` to have at least
+    /// one value
+    pub fn new() -> Self {
+        Self {
+            kind: String::from("statistics"),
+            total_runtime: Mutex::new(vec![0.0]),
+            ..Default::default()
+        }
+    }
+
+    /// increment `requests` field by one
+    fn add_request(&self) {
+        atomic_increment!(self.requests);
+    }
+
+    /// given an `Instant` update total runtime
+    fn update_runtime(&self, seconds: f64) {
+        if let Ok(mut runtime) = self.total_runtime.lock() {
+            runtime[0] = seconds;
+        }
+    }
+
+    /// save an instance of `Stats` to disk after updating the total runtime for the scan
+    fn save(&self, seconds: f64) {
+        let buffered_file = match get_cached_file_handle(&CONFIGURATION.output) {
+            Some(file) => file,
+            None => {
+                return;
+            }
+        };
+
+        self.update_runtime(seconds);
+
+        safe_file_write(self, buffered_file, CONFIGURATION.json);
+    }
+
+    /// Inspect the given `StatError` and increment the appropriate fields
+    ///
+    /// Implies incrementing:
+    ///     - requests
+    ///     - errors
+    pub fn add_error(&self, error: StatError) {
+        self.add_request();
+        atomic_increment!(self.errors);
+
+        match error {
+            StatError::Timeout => {
+                atomic_increment!(self.timeouts);
+            }
+            StatError::Status403 => {
+                atomic_increment!(self.status_403s);
+                atomic_increment!(self.client_errors);
+            }
+            StatError::UrlFormat => {
+                atomic_increment!(self.url_format_errors);
+            }
+            StatError::Redirection => {
+                atomic_increment!(self.redirection_errors);
+            }
+            StatError::Connection => {
+                atomic_increment!(self.connection_errors);
+            }
+            StatError::Request => {
+                atomic_increment!(self.request_errors);
+            }
+            StatError::Other => {
+                atomic_increment!(self.errors);
+            }
+        }
+    }
+
+    /// Inspect the given `StatusCode` and increment the appropriate fields
+    ///
+    /// Implies incrementing:
+    ///     - requests
+    ///     - status_403s (when code is 403)
+    ///     - errors (when code is [45]xx)
+    fn add_status_code(&self, status: StatusCode) {
+        self.add_request();
+
+        if status.is_success() {
+            atomic_increment!(self.successes);
+        } else if status.is_redirection() {
+            atomic_increment!(self.redirects);
+        } else if status.is_client_error() {
+            atomic_increment!(self.client_errors);
+        } else if status.is_server_error() {
+            atomic_increment!(self.server_errors);
+        }
+
+        match status {
+            StatusCode::FORBIDDEN => {
+                atomic_increment!(self.status_403s);
+            }
+            StatusCode::OK => {
+                atomic_increment!(self.status_200s);
+            }
+            StatusCode::MOVED_PERMANENTLY => {
+                atomic_increment!(self.status_301s);
+            }
+            StatusCode::FOUND => {
+                atomic_increment!(self.status_302s);
+            }
+            StatusCode::UNAUTHORIZED => {
+                atomic_increment!(self.status_401s);
+            }
+            StatusCode::TOO_MANY_REQUESTS => {
+                atomic_increment!(self.status_429s);
+            }
+            StatusCode::INTERNAL_SERVER_ERROR => {
+                atomic_increment!(self.status_500s);
+            }
+            StatusCode::SERVICE_UNAVAILABLE => {
+                atomic_increment!(self.status_503s);
+            }
+            StatusCode::GATEWAY_TIMEOUT => {
+                atomic_increment!(self.status_504s);
+            }
+            StatusCode::LOOP_DETECTED => {
+                atomic_increment!(self.status_508s);
+            }
+            _ => {} // other status codes ignored for stat gathering
+        }
+    }
+
+    /// Update a `Stats` field of type f64
+    fn update_f64_field(&self, field: StatField, value: f64) {
+        if let StatField::DirScanTimes = field {
+            if let Ok(mut locked_times) = self.directory_scan_times.lock() {
+                locked_times.push(value);
+            }
+        }
+    }
+
+    /// Update a `Stats` field of type usize
+    fn update_usize_field(&self, field: StatField, value: usize) {
+        match field {
+            StatField::ExpectedPerScan => {
+                atomic_increment!(self.expected_per_scan, value);
+            }
+            StatField::TotalScans => {
+                let multiplier = CONFIGURATION.extensions.len().max(1);
+
+                atomic_increment!(self.total_scans, value);
+                atomic_increment!(
+                    self.total_expected,
+                    value * self.expected_per_scan.load(Ordering::Relaxed) * multiplier
+                );
+            }
+            StatField::TotalExpected => {
+                atomic_increment!(self.total_expected, value);
+            }
+            StatField::LinksExtracted => {
+                atomic_increment!(self.links_extracted, value);
+            }
+            StatField::WildcardsFiltered => {
+                atomic_increment!(self.wildcards_filtered, value);
+                atomic_increment!(self.responses_filtered, value);
+            }
+            StatField::ResponsesFiltered => {
+                atomic_increment!(self.responses_filtered, value);
+            }
+            StatField::ResourcesDiscovered => {
+                atomic_increment!(self.resources_discovered, value);
+            }
+            StatField::InitialTargets => {
+                atomic_increment!(self.initial_targets, value);
+            }
+            _ => {} // f64 fields
+        }
+    }
+
+    /// Merge a given `Stats` object from a json entry written to disk when handling a Ctrl+c
+    ///
+    /// This is only ever called when resuming a scan from disk
+    pub fn merge_from(&self, filename: &str) {
+        if let Ok(file) = File::open(filename) {
+            let reader = BufReader::new(file);
+            let state: serde_json::Value = serde_json::from_reader(reader).unwrap();
+
+            if let Some(state_stats) = state.get("statistics") {
+                if let Ok(d_stats) = serde_json::from_value::<Stats>(state_stats.clone()) {
+                    atomic_increment!(self.successes, atomic_load!(d_stats.successes));
+                    atomic_increment!(self.timeouts, atomic_load!(d_stats.timeouts));
+                    atomic_increment!(self.requests, atomic_load!(d_stats.requests));
+                    atomic_increment!(self.errors, atomic_load!(d_stats.errors));
+                    atomic_increment!(self.redirects, atomic_load!(d_stats.redirects));
+                    atomic_increment!(self.client_errors, atomic_load!(d_stats.client_errors));
+                    atomic_increment!(self.server_errors, atomic_load!(d_stats.server_errors));
+                    atomic_increment!(self.links_extracted, atomic_load!(d_stats.links_extracted));
+                    atomic_increment!(self.status_200s, atomic_load!(d_stats.status_200s));
+                    atomic_increment!(self.status_301s, atomic_load!(d_stats.status_301s));
+                    atomic_increment!(self.status_302s, atomic_load!(d_stats.status_302s));
+                    atomic_increment!(self.status_401s, atomic_load!(d_stats.status_401s));
+                    atomic_increment!(self.status_403s, atomic_load!(d_stats.status_403s));
+                    atomic_increment!(self.status_429s, atomic_load!(d_stats.status_429s));
+                    atomic_increment!(self.status_500s, atomic_load!(d_stats.status_500s));
+                    atomic_increment!(self.status_503s, atomic_load!(d_stats.status_503s));
+                    atomic_increment!(self.status_504s, atomic_load!(d_stats.status_504s));
+                    atomic_increment!(self.status_508s, atomic_load!(d_stats.status_508s));
+                    atomic_increment!(
+                        self.wildcards_filtered,
+                        atomic_load!(d_stats.wildcards_filtered)
+                    );
+                    atomic_increment!(
+                        self.responses_filtered,
+                        atomic_load!(d_stats.responses_filtered)
+                    );
+                    atomic_increment!(
+                        self.resources_discovered,
+                        atomic_load!(d_stats.resources_discovered)
+                    );
+                    atomic_increment!(
+                        self.url_format_errors,
+                        atomic_load!(d_stats.url_format_errors)
+                    );
+                    atomic_increment!(
+                        self.connection_errors,
+                        atomic_load!(d_stats.connection_errors)
+                    );
+                    atomic_increment!(
+                        self.redirection_errors,
+                        atomic_load!(d_stats.redirection_errors)
+                    );
+                    atomic_increment!(self.request_errors, atomic_load!(d_stats.request_errors));
+
+                    if let Ok(scan_times) = d_stats.directory_scan_times.lock() {
+                        for scan_time in scan_times.iter() {
+                            self.update_f64_field(StatField::DirScanTimes, *scan_time);
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+#[derive(Debug)]
+/// Enum variants used to inform the `StatCommand` protocol what `Stats` fields should be updated
+pub enum StatError {
+    /// Represents a 403 response code
+    Status403,
+
+    /// Represents a timeout error
+    Timeout,
+
+    /// Represents a URL formatting error
+    UrlFormat,
+
+    /// Represents an error encountered during redirection
+    Redirection,
+
+    /// Represents an error encountered during connection
+    Connection,
+
+    /// Represents an error resulting from the client's request
+    Request,
+
+    /// Represents any other error not explicitly defined above
+    Other,
+}
+
+/// Protocol definition for updating a Stats object via mpsc
+#[derive(Debug)]
+pub enum StatCommand {
+    /// Add one to the total number of requests
+    AddRequest,
+
+    /// Add one to the proper field(s) based on the given `StatError`
+    AddError(StatError),
+
+    /// Add one to the proper field(s) based on the given `StatusCode`
+    AddStatus(StatusCode),
+
+    /// Create the progress bar (`BarType::Total`) that is updated from the stats thread
+    CreateBar,
+
+    /// Update a `Stats` field that corresponds to the given `StatField` by the given `usize` value
+    UpdateUsizeField(StatField, usize),
+
+    /// Update a `Stats` field that corresponds to the given `StatField` by the given `f64` value
+    UpdateF64Field(StatField, f64),
+
+    /// Save a `Stats` object to disk using `reporter::get_cached_file_handle`
+    Save,
+
+    /// Load a `Stats` object from disk
+    LoadStats(String),
+
+    /// Break out of the (infinite) mpsc receive loop
+    Exit,
+}
+
+/// Enum representing fields whose updates need to be performed in batches instead of one at
+/// a time
+#[derive(Debug)]
+pub enum StatField {
+    /// Due to the necessary order of events, the number of requests expected to be sent isn't
+    /// known until after `statistics::initialize` is called. This command allows for updating
+    /// the `expected_per_scan` field after initialization
+    ExpectedPerScan,
+
+    /// Translates to `total_scans`
+    TotalScans,
+
+    /// Translates to `links_extracted`
+    LinksExtracted,
+
+    /// Translates to `total_expected`
+    TotalExpected,
+
+    /// Translates to `wildcards_filtered`
+    WildcardsFiltered,
+
+    /// Translates to `responses_filtered`
+    ResponsesFiltered,
+
+    /// Translates to `resources_discovered`
+    ResourcesDiscovered,
+
+    /// Translates to `initial_targets`
+    InitialTargets,
+
+    /// Translates to `directory_scan_times`; assumes a single append to the vector
+    DirScanTimes,
+}
+
+/// Spawn a single consumer task (sc side of mpsc)
+///
+/// The consumer simply receives `StatCommands` and updates the given `Stats` object as appropriate
+pub async fn spawn_statistics_handler(
+    mut rx_stats: UnboundedReceiver<StatCommand>,
+    stats: Arc<Stats>,
+    tx_stats: UnboundedSender<StatCommand>,
+) {
+    log::trace!(
+        "enter: spawn_statistics_handler({:?}, {:?}, {:?})",
+        rx_stats,
+        stats,
+        tx_stats
+    );
+
+    // will be updated later via StatCommand; delay is for banner to print first
+    let mut bar = ProgressBar::hidden();
+
+    let start = Instant::now();
+
+    while let Some(command) = rx_stats.recv().await {
+        match command as StatCommand {
+            StatCommand::AddError(err) => {
+                stats.add_error(err);
+            }
+            StatCommand::AddStatus(status) => {
+                stats.add_status_code(status);
+            }
+            StatCommand::AddRequest => stats.add_request(),
+            StatCommand::Save => stats.save(start.elapsed().as_secs_f64()),
+            StatCommand::UpdateUsizeField(field, value) => {
+                let update_len = matches!(field, StatField::TotalScans);
+                stats.update_usize_field(field, value);
+
+                if update_len {
+                    bar.set_length(atomic_load!(stats.total_expected) as u64)
+                }
+            }
+            StatCommand::UpdateF64Field(field, value) => stats.update_f64_field(field, value),
+            StatCommand::CreateBar => {
+                bar = add_bar(
+                    "",
+                    atomic_load!(stats.total_expected) as u64,
+                    BarType::Total,
+                );
+            }
+            StatCommand::LoadStats(filename) => {
+                stats.merge_from(&filename);
+            }
+            StatCommand::Exit => break,
+        }
+
+        let msg = format!(
+            "{}:{:<7} {}:{:<7}",
+            style("found").green(),
+            atomic_load!(stats.resources_discovered),
+            style("errors").red(),
+            atomic_load!(stats.errors),
+        );
+
+        bar.set_message(&msg);
+        bar.inc(1);
+    }
+
+    bar.finish();
+
+    log::debug!("{:#?}", *stats);
+    log::trace!("exit: spawn_statistics_handler")
+}
+
+/// Initialize new `Stats` object and the sc side of an mpsc channel that is responsible for
+/// updates to the aforementioned object.
+pub fn initialize() -> (Arc<Stats>, UnboundedSender<StatCommand>, JoinHandle<()>) {
+    log::trace!("enter: initialize");
+
+    let stats_tracker = Arc::new(Stats::new());
+    let stats_cloned = stats_tracker.clone();
+    let (tx_stats, rx_stats): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+    let tx_stats_cloned = tx_stats.clone();
+    let stats_thread = tokio::spawn(async move {
+        spawn_statistics_handler(rx_stats, stats_cloned, tx_stats_cloned).await
+    });
+
+    log::trace!(
+        "exit: initialize -> ({:?}, {:?}, {:?})",
+        stats_tracker,
+        tx_stats,
+        stats_thread
+    );
+
+    (stats_tracker, tx_stats, stats_thread)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::fs::write;
+    use tempfile::NamedTempFile;
+
+    /// simple helper to reduce code reuse
+    fn setup_stats_test() -> (Arc<Stats>, UnboundedSender<StatCommand>, JoinHandle<()>) {
+        initialize()
+    }
+
+    /// another helper to stay DRY; must be called after any sent commands and before any checks
+    /// performed against the Stats object
+    async fn teardown_stats_test(sender: UnboundedSender<StatCommand>, handle: JoinHandle<()>) {
+        // send exit and await, once the await completes, stats should be updated
+        sender.send(StatCommand::Exit).unwrap_or_default();
+        handle.await.unwrap();
+    }
+
+    #[tokio::test(core_threads = 1)]
+    /// when sent StatCommand::Exit, function should exit its while loop (runs forever otherwise)
+    async fn statistics_handler_exits() {
+        let (_, sender, handle) = setup_stats_test();
+
+        sender.send(StatCommand::Exit).unwrap_or_default();
+
+        handle.await.unwrap(); // blocks on the handler's while loop
+
+        // if we've made it here, the test has succeeded
+    }
+
+    #[tokio::test(core_threads = 1)]
+    /// when sent StatCommand::AddRequest, stats object should reflect the change
+    async fn statistics_handler_increments_requests() {
+        let (stats, tx, handle) = setup_stats_test();
+
+        tx.send(StatCommand::AddRequest).unwrap_or_default();
+        tx.send(StatCommand::AddRequest).unwrap_or_default();
+        tx.send(StatCommand::AddRequest).unwrap_or_default();
+
+        teardown_stats_test(tx, handle).await;
+
+        assert_eq!(stats.requests.load(Ordering::Relaxed), 3);
+    }
+
+    #[tokio::test(core_threads = 1)]
+    /// when sent StatCommand::AddRequest, stats object should reflect the change
+    ///
+    /// incrementing a 403 (tracked in status_403s) should also increment:
+    ///     - errors
+    ///     - requests
+    ///     - client_errors
+    async fn statistics_handler_increments_403() {
+        let (stats, tx, handle) = setup_stats_test();
+
+        let err = StatCommand::AddError(StatError::Status403);
+        let err2 = StatCommand::AddError(StatError::Status403);
+
+        tx.send(err).unwrap_or_default();
+        tx.send(err2).unwrap_or_default();
+
+        teardown_stats_test(tx, handle).await;
+
+        assert_eq!(stats.errors.load(Ordering::Relaxed), 2);
+        assert_eq!(stats.requests.load(Ordering::Relaxed), 2);
+        assert_eq!(stats.status_403s.load(Ordering::Relaxed), 2);
+        assert_eq!(stats.client_errors.load(Ordering::Relaxed), 2);
+    }
+
+    #[tokio::test(core_threads = 1)]
+    /// when sent StatCommand::AddRequest, stats object should reflect the change
+    ///
+    /// incrementing a 403 (tracked in status_403s) should also increment:
+    ///     - requests
+    ///     - client_errors
+    async fn statistics_handler_increments_403_via_status_code() {
+        let (stats, tx, handle) = setup_stats_test();
+
+        let err = StatCommand::AddStatus(reqwest::StatusCode::FORBIDDEN);
+        let err2 = StatCommand::AddStatus(reqwest::StatusCode::FORBIDDEN);
+
+        tx.send(err).unwrap_or_default();
+        tx.send(err2).unwrap_or_default();
+
+        teardown_stats_test(tx, handle).await;
+
+        assert_eq!(stats.requests.load(Ordering::Relaxed), 2);
+        assert_eq!(stats.status_403s.load(Ordering::Relaxed), 2);
+        assert_eq!(stats.client_errors.load(Ordering::Relaxed), 2);
+    }
+
+    #[tokio::test(core_threads = 1)]
+    /// when sent StatCommand::AddStatus, stats object should reflect the change
+    ///
+    /// incrementing a 500 (tracked in server_errors) should also increment:
+    ///     - requests
+    async fn statistics_handler_increments_500_via_status_code() {
+        let (stats, tx, handle) = setup_stats_test();
+
+        let err = StatCommand::AddStatus(reqwest::StatusCode::INTERNAL_SERVER_ERROR);
+        let err2 = StatCommand::AddStatus(reqwest::StatusCode::INTERNAL_SERVER_ERROR);
+
+        tx.send(err).unwrap_or_default();
+        tx.send(err2).unwrap_or_default();
+
+        teardown_stats_test(tx, handle).await;
+
+        assert_eq!(stats.requests.load(Ordering::Relaxed), 2);
+        assert_eq!(stats.server_errors.load(Ordering::Relaxed), 2);
+    }
+
+    #[test]
+    /// when Stats::add_error receives StatError::Timeout, it should increment the following:
+    ///     - timeouts
+    ///     - requests
+    ///     - errors
+    fn stats_increments_timeouts() {
+        let stats = Stats::new();
+        stats.add_error(StatError::Timeout);
+        stats.add_error(StatError::Timeout);
+        stats.add_error(StatError::Timeout);
+        stats.add_error(StatError::Timeout);
+
+        assert_eq!(stats.errors.load(Ordering::Relaxed), 4);
+        assert_eq!(stats.requests.load(Ordering::Relaxed), 4);
+        assert_eq!(stats.timeouts.load(Ordering::Relaxed), 4);
+    }
+
+    #[test]
+    /// when Stats::update_usize_field receives StatField::WildcardsFiltered, it should increment
+    /// the following:
+    ///     - responses_filtered
+    fn stats_increments_wildcards() {
+        let stats = Stats::new();
+        assert_eq!(stats.responses_filtered.load(Ordering::Relaxed), 0);
+        assert_eq!(stats.wildcards_filtered.load(Ordering::Relaxed), 0);
+
+        stats.update_usize_field(StatField::WildcardsFiltered, 1);
+        stats.update_usize_field(StatField::WildcardsFiltered, 1);
+
+        assert_eq!(stats.responses_filtered.load(Ordering::Relaxed), 2);
+        assert_eq!(stats.wildcards_filtered.load(Ordering::Relaxed), 2);
+    }
+
+    #[test]
+    /// when Stats::update_usize_field receives StatField::ResponsesFiltered, it should increment
+    fn stats_increments_responses_filtered() {
+        let stats = Stats::new();
+        assert_eq!(stats.responses_filtered.load(Ordering::Relaxed), 0);
+
+        stats.update_usize_field(StatField::ResponsesFiltered, 1);
+        stats.update_usize_field(StatField::ResponsesFiltered, 1);
+        stats.update_usize_field(StatField::ResponsesFiltered, 1);
+
+        assert_eq!(stats.responses_filtered.load(Ordering::Relaxed), 3);
+    }
+
+    #[test]
+    /// Stats::merge_from should properly incrememnt expected fields and ignore others
+    fn stats_merge_from_alters_correct_fields() {
+        let contents = r#"{"statistics":{"type":"statistics","timeouts":1,"requests":9207,"expected_per_scan":707,"total_expected":9191,"errors":3,"successes":720,"redirects":13,"client_errors":8474,"server_errors":2,"total_scans":13,"initial_targets":1,"links_extracted":51,"status_403s":3,"status_200s":720,"status_301s":12,"status_302s":1,"status_401s":4,"status_429s":2,"status_500s":5,"status_503s":9,"status_504s":6,"status_508s":7,"wildcards_filtered":707,"responses_filtered":707,"resources_discovered":27,"directory_scan_times":[2.211973078,1.989015505,1.898675839,3.9714468910000003,4.938152838,5.256073528,6.021986595,6.065740734,6.42633762,7.095142125,7.336982137,5.319785619,4.843649778],"total_runtime":[11.556575456000001],"url_format_errors":17,"redirection_errors":12,"connection_errors":21,"request_errors":4}}"#;
+        let stats = Stats::new();
+        let tfile = NamedTempFile::new().unwrap();
+        write(&tfile, contents).unwrap();
+
+        stats.merge_from(tfile.path().to_str().unwrap());
+
+        // as of 1.11.1; all Stats fields are accounted for whether they're updated in merge_from
+        // or not
+        assert_eq!(atomic_load!(stats.timeouts), 1);
+        assert_eq!(atomic_load!(stats.requests), 9207);
+        assert_eq!(atomic_load!(stats.expected_per_scan), 0); // not updated in merge_from
+        assert_eq!(atomic_load!(stats.total_expected), 0); // not updated in merge_from
+        assert_eq!(atomic_load!(stats.errors), 3);
+        assert_eq!(atomic_load!(stats.successes), 720);
+        assert_eq!(atomic_load!(stats.redirects), 13);
+        assert_eq!(atomic_load!(stats.client_errors), 8474);
+        assert_eq!(atomic_load!(stats.server_errors), 2);
+        assert_eq!(atomic_load!(stats.total_scans), 0); // not updated in merge_from
+        assert_eq!(atomic_load!(stats.initial_targets), 0); // not updated in merge_from
+        assert_eq!(atomic_load!(stats.links_extracted), 51);
+        assert_eq!(atomic_load!(stats.status_200s), 720);
+        assert_eq!(atomic_load!(stats.status_301s), 12);
+        assert_eq!(atomic_load!(stats.status_302s), 1);
+        assert_eq!(atomic_load!(stats.status_401s), 4);
+        assert_eq!(atomic_load!(stats.status_403s), 3);
+        assert_eq!(atomic_load!(stats.status_429s), 2);
+        assert_eq!(atomic_load!(stats.status_500s), 5);
+        assert_eq!(atomic_load!(stats.status_503s), 9);
+        assert_eq!(atomic_load!(stats.status_504s), 6);
+        assert_eq!(atomic_load!(stats.status_508s), 7);
+        assert_eq!(atomic_load!(stats.wildcards_filtered), 707);
+        assert_eq!(atomic_load!(stats.responses_filtered), 707);
+        assert_eq!(atomic_load!(stats.resources_discovered), 27);
+        assert_eq!(atomic_load!(stats.url_format_errors), 17);
+        assert_eq!(atomic_load!(stats.redirection_errors), 12);
+        assert_eq!(atomic_load!(stats.connection_errors), 21);
+        assert_eq!(atomic_load!(stats.request_errors), 4);
+        assert_eq!(stats.directory_scan_times.lock().unwrap().len(), 13);
+        for scan in stats.directory_scan_times.lock().unwrap().iter() {
+            assert!(scan.max(0.0) > 0.0); // all scans are non-zero
+        }
+        // total_runtime not updated in merge_from
+        assert_eq!(stats.total_runtime.lock().unwrap().len(), 1);
+        assert!((stats.total_runtime.lock().unwrap()[0] - 0.0).abs() < f64::EPSILON);
+    }
+}

src/utils.rs

@@ -1,5 +1,10 @@
+#![macro_use]
 use crate::{
     config::{CONFIGURATION, PROGRESS_PRINTER},
+    statistics::{
+        StatCommand::{self, AddError, AddStatus},
+        StatError::{Connection, Other, Redirection, Request, Timeout, UrlFormat},
+    },
     FeroxError, FeroxResult,
 };
 use console::{strip_ansi_codes, style, user_attended};
@@ -10,6 +15,7 @@ use rlimit::{getrlimit, setrlimit, Resource, Rlim};
 use std::convert::TryInto;
 use std::sync::{Arc, RwLock};
 use std::{fs, io};
+use tokio::sync::mpsc::UnboundedSender;
 
 /// Given the path to a file, open the file in append mode (create it if it doesn't exist) and
 /// return a reference to the file that is buffered and locked
@@ -160,6 +166,14 @@ pub fn ferox_print(msg: &str, bar: &ProgressBar) {
     }
 }
 
+#[macro_export]
+/// wrapper to improve code readability
+macro_rules! update_stat {
+    ($tx:expr, $value:expr) => {
+        $tx.send($value).unwrap_or_default();
+    };
+}
+
 /// Simple helper to generate a `Url`
 ///
 /// Errors during parsing `url` or joining `word` are propagated up the call stack
@@ -169,14 +183,16 @@ pub fn format_url(
     add_slash: bool,
     queries: &[(String, String)],
     extension: Option<&str>,
+    tx_stats: UnboundedSender<StatCommand>,
 ) -> FeroxResult<Url> {
     log::trace!(
-        "enter: format_url({}, {}, {}, {:?} {:?})",
+        "enter: format_url({}, {}, {}, {:?} {:?}, {:?})",
         url,
         word,
         add_slash,
         queries,
-        extension
+        extension,
+        tx_stats
     );
 
     if Url::parse(&word).is_ok() {
@@ -193,8 +209,9 @@ pub fn format_url(
         );
         log::warn!("{}", message);
 
-        let mut err = FeroxError::default();
-        err.message = message;
+        let err = FeroxError { message };
+
+        update_stat!(tx_stats, AddError(UrlFormat));
 
         log::trace!("exit: format_url -> {}", err);
         return Err(Box::new(err));
@@ -208,7 +225,7 @@ pub fn format_url(
     // the transforms that occur here will need to keep this in mind, i.e. add a slash to preserve
     // the current directory sent as part of the url
     let url = if word.is_empty() {
-        // v1.0.6: added during --extract-links feature inplementation to support creating urls
+        // v1.0.6: added during --extract-links feature implementation to support creating urls
         // that were extracted from response bodies, i.e. http://localhost/some/path/js/main.js
         url.to_string()
     } else if !url.ends_with('/') {
@@ -255,6 +272,7 @@ pub fn format_url(
             }
         }
         Err(e) => {
+            update_stat!(tx_stats, AddError(UrlFormat));
             log::trace!("exit: format_url -> {}", e);
             log::error!("Could not join {} with {}", word, base_url);
             Err(Box::new(e))
@@ -263,37 +281,63 @@ pub fn format_url(
 }
 
 /// Initiate request to the given `Url` using `Client`
-pub async fn make_request(client: &Client, url: &Url) -> FeroxResult<Response> {
-    log::trace!("enter: make_request(CONFIGURATION.Client, {})", url);
+pub async fn make_request(
+    client: &Client,
+    url: &Url,
+    tx_stats: UnboundedSender<StatCommand>,
+) -> FeroxResult<Response> {
+    log::trace!(
+        "enter: make_request(CONFIGURATION.Client, {}, {:?})",
+        url,
+        tx_stats
+    );
 
     match client.get(url.to_owned()).send().await {
-        Ok(resp) => {
-            log::trace!("exit: make_request -> {:?}", resp);
-            Ok(resp)
-        }
         Err(e) => {
+            let mut log_level = log::Level::Error;
+
             log::trace!("exit: make_request -> {}", e);
-            if e.to_string().contains("operation timed out") {
+            if e.is_timeout() {
                 // only warn for timeouts, while actual errors are still left as errors
-                log::warn!("Error while making request: {}", e);
+                log_level = log::Level::Warn;
+                update_stat!(tx_stats, AddError(Timeout));
             } else if e.is_redirect() {
                 if let Some(last_redirect) = e.url() {
                     // get where we were headed (last_redirect) and where we came from (url)
                     let fancy_message = format!("{} !=> {}", url, last_redirect);
 
                     let report = if let Some(msg_status) = e.status() {
+                        update_stat!(tx_stats, AddStatus(msg_status));
                         create_report_string(msg_status.as_str(), "-1", "-1", "-1", &fancy_message)
                     } else {
                         create_report_string("UNK", "-1", "-1", "-1", &fancy_message)
                     };
 
+                    update_stat!(tx_stats, AddError(Redirection));
+
                     ferox_print(&report, &PROGRESS_PRINTER)
                 };
+            } else if e.is_connect() {
+                update_stat!(tx_stats, AddError(Connection));
+            } else if e.is_request() {
+                update_stat!(tx_stats, AddError(Request));
             } else {
-                log::error!("Error while making request: {}", e);
+                update_stat!(tx_stats, AddError(Other));
             }
+
+            if matches!(log_level, log::Level::Error) {
+                log::error!("Error while making request: {}", e);
+            } else {
+                log::warn!("Error while making request: {}", e);
+            }
+
             Err(Box::new(e))
         }
+        Ok(resp) => {
+            log::trace!("exit: make_request -> {:?}", resp);
+            update_stat!(tx_stats, AddStatus(resp.status()));
+            Ok(resp)
+        }
     }
 }
 
@@ -391,6 +435,8 @@ pub fn normalize_url(url: &str) -> String {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::FeroxChannel;
+    use tokio::sync::mpsc;
 
     #[test]
     /// set_open_file_limit with a low requested limit succeeds
@@ -458,8 +504,9 @@ mod tests {
     #[test]
     /// base url + 1 word + no slash + no extension
     fn format_url_normal() {
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
         assert_eq!(
-            format_url("http://localhost", "stuff", false, &Vec::new(), None).unwrap(),
+            format_url("http://localhost", "stuff", false, &Vec::new(), None, tx).unwrap(),
             reqwest::Url::parse("http://localhost/stuff").unwrap()
         );
     }
@@ -467,8 +514,9 @@ mod tests {
     #[test]
     /// base url + no word + no slash + no extension
     fn format_url_no_word() {
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
         assert_eq!(
-            format_url("http://localhost", "", false, &Vec::new(), None).unwrap(),
+            format_url("http://localhost", "", false, &Vec::new(), None, tx).unwrap(),
             reqwest::Url::parse("http://localhost").unwrap()
         );
     }
@@ -476,13 +524,15 @@ mod tests {
     #[test]
     /// base url + word + no slash + no extension + queries
     fn format_url_joins_queries() {
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
         assert_eq!(
             format_url(
                 "http://localhost",
                 "lazer",
                 false,
                 &[(String::from("stuff"), String::from("things"))],
-                None
+                None,
+                tx
             )
             .unwrap(),
             reqwest::Url::parse("http://localhost/lazer?stuff=things").unwrap()
@@ -492,13 +542,15 @@ mod tests {
     #[test]
     /// base url + no word + no slash + no extension + queries
     fn format_url_without_word_joins_queries() {
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
         assert_eq!(
             format_url(
                 "http://localhost",
                 "",
                 false,
                 &[(String::from("stuff"), String::from("things"))],
-                None
+                None,
+                tx
             )
             .unwrap(),
             reqwest::Url::parse("http://localhost/?stuff=things").unwrap()
@@ -509,14 +561,16 @@ mod tests {
     #[should_panic]
     /// no base url is an error
     fn format_url_no_url() {
-        format_url("", "stuff", false, &Vec::new(), None).unwrap();
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+        format_url("", "stuff", false, &Vec::new(), None, tx).unwrap();
     }
 
     #[test]
     /// word prepended with slash is adjusted for correctness
     fn format_url_word_with_preslash() {
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
         assert_eq!(
-            format_url("http://localhost", "/stuff", false, &Vec::new(), None).unwrap(),
+            format_url("http://localhost", "/stuff", false, &Vec::new(), None, tx).unwrap(),
             reqwest::Url::parse("http://localhost/stuff").unwrap()
         );
     }
@@ -524,8 +578,9 @@ mod tests {
     #[test]
     /// word with appended slash allows the slash to persist
     fn format_url_word_with_postslash() {
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
         assert_eq!(
-            format_url("http://localhost", "stuff/", false, &Vec::new(), None).unwrap(),
+            format_url("http://localhost", "stuff/", false, &Vec::new(), None, tx).unwrap(),
             reqwest::Url::parse("http://localhost/stuff/").unwrap()
         );
     }
@@ -533,12 +588,14 @@ mod tests {
     #[test]
     /// word that is a fully formed url, should return an error
     fn format_url_word_that_is_a_url() {
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
         let url = format_url(
             "http://localhost",
             "http://schmocalhost",
             false,
             &Vec::new(),
             None,
+            tx,
         );
         assert!(url.is_err());
     }

tests/test_extractor.rs

@@ -264,6 +264,7 @@ fn extractor_finds_robots_txt_links_and_displays_files_or_scans_directories() {
         .arg("--wordlist")
         .arg(file.as_os_str())
         .arg("--extract-links")
+        .arg("-vvvv")
         .unwrap();
 
     cmd.assert().success().stdout(