Merge pull request #185 from epi052/2.0-restructure-filters

Restructure filters

.rustfmt.toml

@@ -0,0 +1 @@
+reorder_modules = false

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "feroxbuster"
-version = "1.11.0"
+version = "1.12.1"
 authors = ["Ben 'epi' Risher <epibar052@gmail.com>"]
 license = "MIT"
 edition = "2018"
@@ -22,23 +22,23 @@ lazy_static = "1.4"
 
 [dependencies]
 futures = { version = "0.3"}
-tokio = { version = "0.2", features = ["full"] }
-tokio-util = {version = "0.3", features = ["codec"]}
+tokio = { version = "1.0", features = ["full"] }
+tokio-util = {version = "0.6", features = ["codec"]}
 log = "0.4"
 env_logger = "0.8"
-reqwest = { version = "0.10", features = ["socks"] }
+reqwest = { version = "0.11", features = ["socks"] }
 clap = "2.33"
 lazy_static = "1.4"
 toml = "0.5"
-serde = { version = "1.0", features = ["derive"] }
+serde = { version = "1.0", features = ["derive", "rc"] }
 serde_json = "1.0"
 uuid = { version = "0.8", features = ["v4"] }
 indicatif = "0.15"
-console = "0.13"
+console = "0.14"
 openssl = { version = "0.10", features = ["vendored"] }
 dirs = "3.0"
 regex = "1"
-crossterm = "0.18"
+crossterm = "0.19"
 rlimit = "0.5"
 ctrlc = "3.1"
 fuzzyhash = "0.2"

README.md

@@ -78,6 +78,10 @@ Enumeration.
     - [Threads and Connection Limits At A High-Level](#threads-and-connection-limits-at-a-high-level)
     - [ferox-config.toml](#ferox-configtoml)
     - [Command Line Parsing](#command-line-parsing)
+- [Scan's Display Explained](#-scans-display-explained)
+    - [Discovered Resource](#discovered-resource)
+    - [Overall Scan Progress Bar](#overall-scan-progress-bar)
+    - [Directory Scan Progress Bar](#directory-scan-progress-bar)
 - [Example Usage](#-example-usage)
     - [Multiple Values](#multiple-values)
     - [Include Headers](#include-headers)
@@ -97,6 +101,7 @@ Enumeration.
     - [Enforce a Time Limit on Your Scan (new in `v1.10.0`)](#enforce-a-time-limit-on-your-scan-new-in-v1100)
     - [Extract Links from robots.txt (New in `v1.10.2`)](#extract-links-from-robotstxt-new-in-v1102)
     - [Filter Response by Similarity to A Given Page (fuzzy filter) (new in `v1.11.0`)](#filter-response-by-similarity-to-a-given-page-fuzzy-filter-new-in-v1110)
+    - [Cancel a Recursive Scan Interactively (new in `v1.12.0`)](#cancel-a-recursive-scan-interactively-new-in-v1120)
 - [Comparison w/ Similar Tools](#-comparison-w-similar-tools)
 - [Common Problems/Issues (FAQ)](#-common-problemsissues-faq)
     - [No file descriptors available](#no-file-descriptors-available)
@@ -460,6 +465,31 @@ OPTIONS:
     -w, --wordlist <FILE>                   Path to the wordlist
 ```
 
+## 📊 Scan's Display Explained
+
+`feroxbuster` attempts to be intuitive and easy to understand, however, if you are wondering about any of the scan's
+output and what it means, this is the section for you!  
+
+### Discovered Resource
+
+When `feroxbuster` finds a response that you haven't filtered out, it's reported above the progress bars and looks similar to what's pictured below.  
+
+The number of lines, words, and bytes shown here can be used to [filter those responses](#filter-response-by-word-count--line-count--new-in-v160)
+
+![response-bar-explained](img/response-bar-explained.png)
+
+### Overall Scan Progress Bar
+
+The top progress bar, colored yellow, tracks the overall scan status.  Its fields are described in the image below.
+
+![total-bar-explained](img/total-bar-explained.png)
+
+### Directory Scan Progress Bar
+
+All other progress bars, colored cyan, represent a scan of one particular directory and will look similar to what's below.   
+
+![dir-scan-bar-explained](img/dir-scan-bar-explained.png)
+
 ## 🧰 Example Usage
 
 ### Multiple Values
@@ -567,9 +597,11 @@ is checked against a list of known filters and either displayed or not according
 
 ### Pause an Active Scan (new in `v1.4.0`)
 
-Scans can be paused and resumed by pressing the ENTER key (shown below)
+**NOTE**: [v1.12.0](#cancel-a-recursive-scan-interactively-new-in-v1120) added an interactive menu to the pause/resume
+functionality.  Active scans can still be paused, however, now you're presented with the option to cancel a scan instead
+of simply seeing a spinner.
 
-![pause-resume-demo](img/pause-resume-demo.gif)
+Scans can be paused and resumed by pressing the ENTER key (~~shown below~~, please see [v1.12.0](#cancel-a-recursive-scan-interactively-new-in-v1120)'s entry for the latest visual representation)
 
 ### Replay Responses to a Proxy based on Status Code (new in `v1.5.0`)
 
@@ -739,6 +771,27 @@ magnitude slower on requests/second).
   - The lack of accuracy with very small responses is considered a fair trade-off for not negatively impacting performance
 - Using a bunch of `--filter-similar-to` values **may** negatively impact performance
 
+### Cancel a Recursive Scan Interactively (new in `v1.12.0`)
+
+Version 1.12.0 expanded the pause/resume functionality introduced in [v1.4.0](#pause-an-active-scan-new-in-v140) by 
+adding an interactive menu from which currently running recursive scans can be cancelled, without affecting the overall scan.  Scans can still be paused indefinitely by pressing `ENTER`, however, the   
+
+Scans that are started via `-u` or passed in through `--stdin` cannot be cancelled, only scans found via `--extract-links` or recursion are eligible.
+
+Below is an example of the Scan Cancel Menu™.
+
+![cancel-menu](img/cancel-menu.png)
+
+Using the menu is pretty simple:
+- Press `ENTER` to view the menu
+- Choose a scan to cancel by entering its scan index (`1`)
+  - more than one scan can be selected by using a comma-separated list (`1,2,3` ... etc)
+- Confirm selections, after which all non-cancelled scans will resume
+
+Here is a short demonstration of cancelling two in-progress scans found via recursion.
+
+![cancel-scan](img/cancel-scan.gif)
+
 ## 🧐 Comparison w/ Similar Tools
 
 There are quite a few similar tools for forced browsing/content discovery. Burp Suite Pro, Dirb, Dirbuster, etc...
@@ -784,6 +837,7 @@ few of the use-cases in which feroxbuster may be a better fit:
 | maximum run time limit (`v1.10.0`)                                           | ✔ |   | ✔ |
 | use robots.txt to increase scan coverage (`v1.10.2`)                         | ✔ |   |   |
 | use example page's response to fuzzily filter similar pages  (`v1.11.0`)     | ✔ |   |   |
+| cancel a recursive scan interactively (`v1.12.0`)                            | ✔ |   |   |
 | **huge** number of other options                                             |   |   | ✔ |
 
 Of note, there's another written-in-rust content discovery tool, [rustbuster](https://github.com/phra/rustbuster). I

install-nix.sh

@@ -36,18 +36,23 @@ elif [[ "$(expr substr $(uname -s) 1 5)" == "Linux" ]]; then
         rm "${LIN64_ZIP}"
     fi
 
-    echo "[=] Installing Noto Emoji Font"
-    mkdir -p ~/.fonts
-    pushd ~/.fonts 2>&1 >/dev/null
+    if [[ -e ~/.fonts/NotoColorEmoji.ttf ]]; then
+       echo "[=] Found Noto Emoji Font, skipping install" 
+    else 
+        echo "[=] Installing Noto Emoji Font"
+        mkdir -p ~/.fonts
+        pushd ~/.fonts 2>&1 >/dev/null
 
-    curl -sLO "${EMOJI_URL}"
+        curl -sLO "${EMOJI_URL}"
 
-    fc-cache -f -v >/dev/null
+        fc-cache -f -v >/dev/null
 
-    popd 2>&1 >/dev/null
-    echo "[+] Noto Emoji Font installed"
+        popd 2>&1 >/dev/null
+        echo "[+] Noto Emoji Font installed"
+    fi
 fi
 
+
 chmod +x ./feroxbuster
 
 echo "[+] Installed feroxbuster version $(./feroxbuster -V)"

src/banner.rs

@@ -1,9 +1,13 @@
-use crate::config::{Configuration, CONFIGURATION};
-use crate::utils::{make_request, status_colorizer};
+use crate::{
+    config::{Configuration, CONFIGURATION},
+    statistics::StatCommand,
+    utils::{make_request, status_colorizer},
+};
 use console::{style, Emoji};
 use reqwest::{Client, Url};
 use serde_json::Value;
 use std::io::Write;
+use tokio::sync::mpsc::UnboundedSender;
 
 /// macro helper to abstract away repetitive string formatting
 macro_rules! format_banner_entry_helper {
@@ -67,8 +71,13 @@ enum UpdateStatus {
 /// ex: v1.1.0
 ///
 /// Returns `UpdateStatus`
-async fn needs_update(client: &Client, url: &str, bin_version: &str) -> UpdateStatus {
-    log::trace!("enter: needs_update({:?}, {})", client, url);
+async fn needs_update(
+    client: &Client,
+    url: &str,
+    bin_version: &str,
+    tx_stats: UnboundedSender<StatCommand>,
+) -> UpdateStatus {
+    log::trace!("enter: needs_update({:?}, {}, {:?})", client, url, tx_stats);
 
     let unknown = UpdateStatus::Unknown;
 
@@ -81,7 +90,7 @@ async fn needs_update(client: &Client, url: &str, bin_version: &str) -> UpdateSt
         }
     };
 
-    if let Ok(response) = make_request(&client, &api_url).await {
+    if let Ok(response) = make_request(&client, &api_url, tx_stats.clone()).await {
         let body = response.text().await.unwrap_or_default();
 
         let json_response: Value = serde_json::from_str(&body).unwrap_or_default();
@@ -137,8 +146,13 @@ fn format_emoji(emoji: &str) -> String {
 /// Prints the banner to stdout.
 ///
 /// Only prints those settings which are either always present, or passed in by the user.
-pub async fn initialize<W>(targets: &[String], config: &Configuration, version: &str, mut writer: W)
-where
+pub async fn initialize<W>(
+    targets: &[String],
+    config: &Configuration,
+    version: &str,
+    mut writer: W,
+    tx_stats: UnboundedSender<StatCommand>,
+) where
     W: Write,
 {
     let artwork = format!(
@@ -150,7 +164,7 @@ by Ben "epi" Risher {}                 ver: {}"#,
         Emoji("🤓", &format!("{:<2}", "\u{0020}")),
         version
     );
-    let status = needs_update(&CONFIGURATION.client, UPDATE_URL, version).await;
+    let status = needs_update(&CONFIGURATION.client, UPDATE_URL, version, tx_stats).await;
 
     let top = "───────────────────────────┬──────────────────────";
     let addl_section = "──────────────────────────────────────────────────";
@@ -522,11 +536,10 @@ by Ben "epi" Risher {}                 ver: {}"#,
     // ⏯
     writeln!(
         &mut writer,
-        " {}   Press [{}] to {}|{} your scan",
-        format_emoji("⏯"),
+        " {}  Press [{}] to use the {}™",
+        format_emoji("🏁"),
         style("ENTER").yellow(),
-        style("pause").red(),
-        style("resume").green()
+        style("Scan Cancel Menu").bright().yellow(),
     )
     .unwrap_or_default();
 
@@ -536,73 +549,98 @@ by Ben "epi" Risher {}                 ver: {}"#,
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::VERSION;
+    use crate::{FeroxChannel, VERSION};
     use httpmock::Method::GET;
     use httpmock::MockServer;
     use std::fs::read_to_string;
     use std::io::stderr;
     use std::time::Duration;
     use tempfile::NamedTempFile;
+    use tokio::sync::mpsc;
 
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// test to hit no execution of targets for loop in banner
     async fn banner_intialize_without_targets() {
         let config = Configuration::default();
-        initialize(&[], &config, VERSION, stderr()).await;
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
+        initialize(&[], &config, VERSION, stderr(), tx).await;
     }
 
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// test to hit no execution of statuscode for loop in banner
     async fn banner_intialize_without_status_codes() {
-        let mut config = Configuration::default();
-        config.status_codes = vec![];
+        let config = Configuration {
+            status_codes: vec![],
+            ..Default::default()
+        };
+
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
         initialize(
             &[String::from("http://localhost")],
             &config,
             VERSION,
             stderr(),
+            tx,
         )
         .await;
     }
 
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// test to hit an empty config file
     async fn banner_intialize_without_config_file() {
-        let mut config = Configuration::default();
-        config.config = String::new();
+        let config = Configuration {
+            config: String::new(),
+            ..Default::default()
+        };
+
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
         initialize(
             &[String::from("http://localhost")],
             &config,
             VERSION,
             stderr(),
+            tx,
         )
         .await;
     }
 
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// test to hit an empty config file
     async fn banner_intialize_without_queries() {
-        let mut config = Configuration::default();
-        config.queries = vec![(String::new(), String::new())];
+        let config = Configuration {
+            queries: vec![(String::new(), String::new())],
+            ..Default::default()
+        };
+
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
         initialize(
             &[String::from("http://localhost")],
             &config,
             VERSION,
             stderr(),
+            tx,
         )
         .await;
     }
 
-    #[tokio::test(core_threads = 1)]
+    #[ignore]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// test to show that a new version is available for download
     async fn banner_intialize_with_mismatched_version() {
         let config = Configuration::default();
         let file = NamedTempFile::new().unwrap();
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
         initialize(
             &[String::from("http://localhost")],
             &config,
             "mismatched-version",
             &file,
+            tx,
         )
         .await;
         let contents = read_to_string(file.path()).unwrap();
@@ -611,14 +649,16 @@ mod tests {
         assert!(contents.contains("https://github.com/epi052/feroxbuster/releases/latest"));
     }
 
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// test that
     async fn banner_needs_update_returns_unknown_with_bad_url() {
-        let result = needs_update(&CONFIGURATION.client, &"", VERSION).await;
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
+        let result = needs_update(&CONFIGURATION.client, &"", VERSION, tx).await;
         assert!(matches!(result, UpdateStatus::Unknown));
     }
 
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// test return value of good url to needs_update
     async fn banner_needs_update_returns_up_to_date() {
         let srv = MockServer::start();
@@ -628,13 +668,15 @@ mod tests {
             then.status(200).body("{\"tag_name\":\"v1.1.0\"}");
         });
 
-        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.1.0").await;
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
+        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.1.0", tx).await;
 
         assert_eq!(mock.hits(), 1);
         assert!(matches!(result, UpdateStatus::UpToDate));
     }
 
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// test return value of good url to needs_update that returns a newer version than current
     async fn banner_needs_update_returns_out_of_date() {
         let srv = MockServer::start();
@@ -644,13 +686,15 @@ mod tests {
             then.status(200).body("{\"tag_name\":\"v1.1.0\"}");
         });
 
-        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1").await;
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
+        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1", tx).await;
 
         assert_eq!(mock.hits(), 1);
         assert!(matches!(result, UpdateStatus::OutOfDate));
     }
 
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// test return value of good url that times out
     async fn banner_needs_update_returns_unknown_on_timeout() {
         let srv = MockServer::start();
@@ -662,13 +706,15 @@ mod tests {
                 .delay(Duration::from_secs(8));
         });
 
-        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1").await;
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
+        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1", tx).await;
 
         assert_eq!(mock.hits(), 1);
         assert!(matches!(result, UpdateStatus::Unknown));
     }
 
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// test return value of good url with bad json response
     async fn banner_needs_update_returns_unknown_on_bad_json_response() {
         let srv = MockServer::start();
@@ -678,13 +724,15 @@ mod tests {
             then.status(200).body("not json");
         });
 
-        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1").await;
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
+        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1", tx).await;
 
         assert_eq!(mock.hits(), 1);
         assert!(matches!(result, UpdateStatus::Unknown));
     }
 
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// test return value of good url with json response that lacks the tag_name field
     async fn banner_needs_update_returns_unknown_on_json_without_correct_tag() {
         let srv = MockServer::start();
@@ -695,7 +743,9 @@ mod tests {
                 .body("{\"no tag_name\": \"doesn't exist\"}");
         });
 
-        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1").await;
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
+        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1", tx).await;
 
         assert_eq!(mock.hits(), 1);
         assert!(matches!(result, UpdateStatus::Unknown));

src/config.rs

@@ -1,18 +1,23 @@
-use crate::scan_manager::resume_scan;
-use crate::utils::{module_colorizer, status_colorizer};
-use crate::{client, parser, progress};
-use crate::{FeroxSerialize, DEFAULT_CONFIG_NAME, DEFAULT_STATUS_CODES, DEFAULT_WORDLIST, VERSION};
+use crate::{
+    client, parser,
+    progress::{add_bar, BarType},
+    scan_manager::resume_scan,
+    utils::{module_colorizer, status_colorizer},
+    FeroxSerialize, DEFAULT_CONFIG_NAME, DEFAULT_STATUS_CODES, DEFAULT_WORDLIST, VERSION,
+};
 use clap::{value_t, ArgMatches};
 use indicatif::{MultiProgress, ProgressBar, ProgressDrawTarget};
 use lazy_static::lazy_static;
 use reqwest::{Client, StatusCode};
 use serde::{Deserialize, Serialize};
-use std::collections::HashMap;
-use std::env::{current_dir, current_exe};
-use std::fs::read_to_string;
-use std::path::PathBuf;
 #[cfg(not(test))]
 use std::process::exit;
+use std::{
+    collections::HashMap,
+    env::{current_dir, current_exe},
+    fs::read_to_string,
+    path::PathBuf,
+};
 
 lazy_static! {
     /// Global configuration state
@@ -22,7 +27,7 @@ lazy_static! {
     pub static ref PROGRESS_BAR: MultiProgress = MultiProgress::with_draw_target(ProgressDrawTarget::stdout());
 
     /// Global progress bar that is only used for printing messages that don't jack up other bars
-    pub static ref PROGRESS_PRINTER: ProgressBar = progress::add_bar("", 0, true, false);
+    pub static ref PROGRESS_PRINTER: ProgressBar = add_bar("", 0, BarType::Hidden);
 }
 
 /// macro helper to abstract away repetitive configuration updates
@@ -80,7 +85,7 @@ fn report_and_exit(err: &str) -> ! {
 pub struct Configuration {
     #[serde(rename = "type", default = "serialized_type")]
     /// Name of this type of struct, used for serialization, i.e. `{"type":"configuration"}`
-    kind: String,
+    pub kind: String,
 
     /// Path to the wordlist
     #[serde(default = "wordlist")]
@@ -223,6 +228,10 @@ pub struct Configuration {
     #[serde(default)]
     pub resumed: bool,
 
+    /// Resume scan from this file
+    #[serde(default)]
+    pub resume_from: String,
+
     /// Whether or not a scan's current state should be saved when user presses Ctrl+C
     ///
     /// Not configurable from CLI; can only be set from a config file
@@ -324,6 +333,7 @@ impl Default for Configuration {
             debug_log: String::new(),
             target_url: String::new(),
             time_limit: String::new(),
+            resume_from: String::new(),
             replay_proxy: String::new(),
             queries: Vec::new(),
             extensions: Vec::new(),
@@ -401,8 +411,10 @@ impl Configuration {
     pub fn new() -> Self {
         // when compiling for test, we want to eliminate the runtime dependency of the parser
         if cfg!(test) {
-            let mut test_config = Configuration::default();
-            test_config.save_state = false; // don't clutter up junk when testing
+            let test_config = Configuration {
+                save_state: false, // don't clutter up junk when testing
+                ..Default::default()
+            };
             return test_config;
         }
 
@@ -515,6 +527,7 @@ impl Configuration {
         update_config_if_present!(&mut config.output, args, "output", String);
         update_config_if_present!(&mut config.debug_log, args, "debug_log", String);
         update_config_if_present!(&mut config.time_limit, args, "time_limit", String);
+        update_config_if_present!(&mut config.resume_from, args, "resume_from", String);
 
         if let Some(arg) = args.values_of("status_codes") {
             config.status_codes = arg
@@ -794,6 +807,7 @@ impl Configuration {
         update_if_not_default!(&mut conf.scan_limit, new.scan_limit, 0);
         update_if_not_default!(&mut conf.replay_proxy, new.replay_proxy, "");
         update_if_not_default!(&mut conf.debug_log, new.debug_log, "");
+        update_if_not_default!(&mut conf.resume_from, new.resume_from, "");
         update_if_not_default!(&mut conf.json, new.json, false);
 
         update_if_not_default!(&mut conf.timeout, new.timeout, timeout());
@@ -893,6 +907,7 @@ mod tests {
             time_limit = "10m"
             output = "/some/otherpath"
             debug_log = "/yet/anotherpath"
+            resume_from = "/some/state/file"
             redirects = true
             insecure = true
             extensions = ["html", "php", "js"]
@@ -927,6 +942,7 @@ mod tests {
         assert_eq!(config.proxy, String::new());
         assert_eq!(config.target_url, String::new());
         assert_eq!(config.time_limit, String::new());
+        assert_eq!(config.resume_from, String::new());
         assert_eq!(config.debug_log, String::new());
         assert_eq!(config.config, String::new());
         assert_eq!(config.replay_proxy, String::new());
@@ -1169,6 +1185,13 @@ mod tests {
         assert_eq!(config.time_limit, "10m");
     }
 
+    #[test]
+    /// parse the test config and see that the value parsed is correct
+    fn config_reads_resume_from() {
+        let config = setup_config_test();
+        assert_eq!(config.resume_from, "/some/state/file");
+    }
+
     #[test]
     /// parse the test config and see that the values parsed are correct
     fn config_reads_headers() {

src/extractor.rs

@@ -2,6 +2,10 @@ use crate::{
     client,
     config::{Configuration, CONFIGURATION},
     scanner::SCANNED_URLS,
+    statistics::{
+        StatCommand::{self, UpdateUsizeField},
+        StatField::{LinksExtracted, TotalExpected},
+    },
     utils::{format_url, make_request},
     FeroxResponse,
 };
@@ -9,6 +13,7 @@ use lazy_static::lazy_static;
 use regex::Regex;
 use reqwest::Url;
 use std::collections::HashSet;
+use tokio::sync::mpsc::UnboundedSender;
 
 /// Regular expression used in [LinkFinder](https://github.com/GerbenJavado/LinkFinder)
 ///
@@ -47,20 +52,27 @@ fn get_sub_paths_from_path(path: &str) -> Vec<String> {
 
     let length = parts.len();
 
-    for _ in 0..length {
+    for i in 0..length {
         // iterate over all parts of the path
         if parts.is_empty() {
             // pop left us with an empty vector, we're done
             break;
         }
 
-        let possible_path = parts.join("/");
+        let mut possible_path = parts.join("/");
 
         if possible_path.is_empty() {
             // .join can result in an empty string, which we don't need, ignore
             continue;
         }
 
+        if i > 0 {
+            // this isn't the last index of the parts array
+            // ex: /buried/misc/stupidfile.php
+            // this block skips the file but sees all parent folders
+            possible_path = format!("{}/", possible_path);
+        }
+
         paths.push(possible_path); // good sub-path found
         parts.pop(); // use .pop() to remove the last part of the path and continue iteration
     }
@@ -98,8 +110,15 @@ fn add_link_to_set_of_links(link: &str, url: &Url, links: &mut HashSet<String>)
 ///         - homepage/assets/img/
 ///         - homepage/assets/
 ///         - homepage/
-pub async fn get_links(response: &FeroxResponse) -> HashSet<String> {
-    log::trace!("enter: get_links({})", response.url().as_str());
+pub async fn get_links(
+    response: &FeroxResponse,
+    tx_stats: UnboundedSender<StatCommand>,
+) -> HashSet<String> {
+    log::trace!(
+        "enter: get_links({}, {:?})",
+        response.url().as_str(),
+        tx_stats
+    );
 
     let mut links = HashSet::<String>::new();
 
@@ -136,6 +155,14 @@ pub async fn get_links(response: &FeroxResponse) -> HashSet<String> {
         }
     }
 
+    let multiplier = CONFIGURATION.extensions.len().max(1);
+
+    update_stat!(tx_stats, UpdateUsizeField(LinksExtracted, links.len()));
+    update_stat!(
+        tx_stats,
+        UpdateUsizeField(TotalExpected, links.len() * multiplier)
+    );
+
     log::trace!("exit: get_links -> {:?}", links);
 
     links
@@ -172,8 +199,15 @@ fn add_all_sub_paths(url_path: &str, response: &FeroxResponse, mut links: &mut H
 ///   - create a new Url object based on cli options/args
 ///   - check if the new Url has already been seen/scanned -> None
 ///   - make a request to the new Url ? -> Some(response) : None
-pub async fn request_feroxresponse_from_new_link(url: &str) -> Option<FeroxResponse> {
-    log::trace!("enter: request_feroxresponse_from_new_link({})", url);
+pub async fn request_feroxresponse_from_new_link(
+    url: &str,
+    tx_stats: UnboundedSender<StatCommand>,
+) -> Option<FeroxResponse> {
+    log::trace!(
+        "enter: request_feroxresponse_from_new_link({}, {:?})",
+        url,
+        tx_stats
+    );
 
     // create a url based on the given command line options, return None on error
     let new_url = match format_url(
@@ -182,6 +216,7 @@ pub async fn request_feroxresponse_from_new_link(url: &str) -> Option<FeroxRespo
         CONFIGURATION.add_slash,
         &CONFIGURATION.queries,
         None,
+        tx_stats.clone(),
     ) {
         Ok(url) => url,
         Err(_) => {
@@ -197,7 +232,7 @@ pub async fn request_feroxresponse_from_new_link(url: &str) -> Option<FeroxRespo
     }
 
     // make the request and store the response
-    let new_response = match make_request(&CONFIGURATION.client, &new_url).await {
+    let new_response = match make_request(&CONFIGURATION.client, &new_url, tx_stats).await {
         Ok(resp) => resp,
         Err(_) => {
             log::trace!("exit: request_feroxresponse_from_new_link -> None");
@@ -221,8 +256,16 @@ pub async fn request_feroxresponse_from_new_link(url: &str) -> Option<FeroxRespo
 ///     
 /// The length of the given path has no effect on what's requested; it's always
 /// base url + /robots.txt
-pub async fn request_robots_txt(base_url: &str, config: &Configuration) -> Option<FeroxResponse> {
-    log::trace!("enter: get_robots_file({})", base_url);
+pub async fn request_robots_txt(
+    base_url: &str,
+    config: &Configuration,
+    tx_stats: UnboundedSender<StatCommand>,
+) -> Option<FeroxResponse> {
+    log::trace!(
+        "enter: get_robots_file({}, CONFIGURATION, {:?})",
+        base_url,
+        tx_stats
+    );
 
     // more often than not, domain/robots.txt will redirect to www.domain/robots.txt or something
     // similar; to account for that, create a client that will follow redirects, regardless of
@@ -248,7 +291,7 @@ pub async fn request_robots_txt(base_url: &str, config: &Configuration) -> Optio
     if let Ok(mut url) = Url::parse(base_url) {
         url.set_path("/robots.txt"); // overwrite existing path with /robots.txt
 
-        if let Ok(response) = make_request(&client, &url).await {
+        if let Ok(response) = make_request(&client, &url, tx_stats).await {
             let ferox_response = FeroxResponse::from(response, true).await;
 
             log::trace!("exit: get_robots_file -> {}", ferox_response);
@@ -267,11 +310,19 @@ pub async fn request_robots_txt(base_url: &str, config: &Configuration) -> Optio
 ///     http://localhost/stuff/things
 /// this function requests:
 ///     http://localhost/robots.txt
-pub async fn extract_robots_txt(base_url: &str, config: &Configuration) -> HashSet<String> {
-    log::trace!("enter: extract_robots_txt({}, CONFIGURATION)", base_url);
+pub async fn extract_robots_txt(
+    base_url: &str,
+    config: &Configuration,
+    tx_stats: UnboundedSender<StatCommand>,
+) -> HashSet<String> {
+    log::trace!(
+        "enter: extract_robots_txt({}, CONFIGURATION, {:?})",
+        base_url,
+        tx_stats
+    );
     let mut links = HashSet::new();
 
-    if let Some(response) = request_robots_txt(&base_url, &config).await {
+    if let Some(response) = request_robots_txt(&base_url, &config, tx_stats.clone()).await {
         for capture in ROBOTS_REGEX.captures_iter(response.text.as_str()) {
             if let Some(new_path) = capture.name("url_path") {
                 if let Ok(mut new_url) = Url::parse(base_url) {
@@ -282,6 +333,14 @@ pub async fn extract_robots_txt(base_url: &str, config: &Configuration) -> HashS
         }
     }
 
+    let multiplier = CONFIGURATION.extensions.len().max(1);
+
+    update_stat!(tx_stats, UpdateUsizeField(LinksExtracted, links.len()));
+    update_stat!(
+        tx_stats,
+        UpdateUsizeField(TotalExpected, links.len() * multiplier)
+    );
+
     log::trace!("exit: extract_robots_txt -> {:?}", links);
     links
 }
@@ -290,9 +349,11 @@ pub async fn extract_robots_txt(base_url: &str, config: &Configuration) -> HashS
 mod tests {
     use super::*;
     use crate::utils::make_request;
+    use crate::FeroxChannel;
     use httpmock::Method::GET;
     use httpmock::MockServer;
     use reqwest::Client;
+    use tokio::sync::mpsc;
 
     #[test]
     /// extract sub paths from the given url fragment; expect 4 sub paths and that all are
@@ -301,10 +362,10 @@ mod tests {
         let path = "homepage/assets/img/icons/handshake.svg";
         let paths = get_sub_paths_from_path(&path);
         let expected = vec![
-            "homepage",
-            "homepage/assets",
-            "homepage/assets/img",
-            "homepage/assets/img/icons",
+            "homepage/",
+            "homepage/assets/",
+            "homepage/assets/img/",
+            "homepage/assets/img/icons/",
             "homepage/assets/img/icons/handshake.svg",
         ];
 
@@ -321,7 +382,7 @@ mod tests {
     fn extractor_get_sub_paths_from_path_with_enclosing_slashes() {
         let path = "/homepage/assets/";
         let paths = get_sub_paths_from_path(&path);
-        let expected = vec!["homepage", "homepage/assets"];
+        let expected = vec!["homepage/", "homepage/assets"];
 
         assert_eq!(paths.len(), expected.len());
         for expected_path in expected {
@@ -385,7 +446,7 @@ mod tests {
         assert!(links.is_empty());
     }
 
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// use make_request to generate a Response, and use the Response to test get_links;
     /// the response will contain an absolute path to a domain that is not part of the scanned
     /// domain; expect an empty set returned
@@ -402,12 +463,13 @@ mod tests {
 
         let client = Client::new();
         let url = Url::parse(&srv.url("/some-path")).unwrap();
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
 
-        let response = make_request(&client, &url).await.unwrap();
+        let response = make_request(&client, &url, tx.clone()).await.unwrap();
 
         let ferox_response = FeroxResponse::from(response, true).await;
 
-        let links = get_links(&ferox_response).await;
+        let links = get_links(&ferox_response, tx).await;
 
         assert!(links.is_empty());
 
@@ -415,7 +477,7 @@ mod tests {
         Ok(())
     }
 
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// test that /robots.txt is correctly requested given a base url (happy path)
     async fn request_robots_txt_with_and_without_proxy() {
         let srv = MockServer::start();
@@ -427,13 +489,15 @@ mod tests {
 
         let mut config = Configuration::default();
 
-        request_robots_txt(&srv.url("/api/users/stuff/things"), &config).await;
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
+        request_robots_txt(&srv.url("/api/users/stuff/things"), &config, tx.clone()).await;
 
         // note: the proxy doesn't actually do anything other than hit a different code branch
         // in this unit test; it would however have an effect on an integration test
         config.proxy = srv.url("/ima-proxy");
 
-        request_robots_txt(&srv.url("/api/different/path"), &config).await;
+        request_robots_txt(&srv.url("/api/different/path"), &config, tx).await;
 
         assert_eq!(mock.hits(), 2);
     }

src/filters.rs

@@ -1,513 +0,0 @@
-use crate::config::CONFIGURATION;
-use crate::utils::get_url_path_length;
-use crate::{FeroxResponse, FeroxSerialize};
-use fuzzyhash::FuzzyHash;
-use regex::Regex;
-use std::any::Any;
-use std::fmt::Debug;
-
-// references:
-//   https://dev.to/magnusstrale/rust-trait-objects-in-a-vector-non-trivial-4co5
-//   https://stackoverflow.com/questions/25339603/how-to-test-for-equality-between-trait-objects
-
-/// FeroxFilter trait; represents different types of possible filters that can be applied to
-/// responses
-pub trait FeroxFilter: Debug + Send + Sync {
-    /// Determine whether or not this particular filter should be applied or not
-    fn should_filter_response(&self, response: &FeroxResponse) -> bool;
-
-    /// delegates to the FeroxFilter-implementing type which gives us the actual type of self
-    fn box_eq(&self, other: &dyn Any) -> bool;
-
-    /// gives us `other` as Any in box_eq
-    fn as_any(&self) -> &dyn Any;
-}
-
-/// implementation of PartialEq, necessary long-form due to "trait cannot be made into an object"
-/// error when attempting to derive PartialEq on the trait itself
-impl PartialEq for Box<dyn FeroxFilter> {
-    /// Perform a comparison of two implementors of the FeroxFilter trait
-    fn eq(&self, other: &Box<dyn FeroxFilter>) -> bool {
-        self.box_eq(other.as_any())
-    }
-}
-
-/// Data holder for two pieces of data needed when auto-filtering out wildcard responses
-///
-/// `dynamic` is the size of the response that will later be combined with the length
-/// of the path of the url requested and used to determine interesting pages from custom
-/// 404s where the requested url is reflected back in the response
-///
-/// `size` is size of the response that should be included with filters passed via runtime
-/// configuration and any static wildcard lengths.
-#[derive(Debug, Default, Clone, PartialEq)]
-pub struct WildcardFilter {
-    /// size of the response that will later be combined with the length of the path of the url
-    /// requested
-    pub dynamic: u64,
-
-    /// size of the response that should be included with filters passed via runtime configuration
-    pub size: u64,
-}
-
-/// implementation of FeroxFilter for WildcardFilter
-impl FeroxFilter for WildcardFilter {
-    /// Examine size, dynamic, and content_len to determine whether or not the response received
-    /// is a wildcard response and therefore should be filtered out
-    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
-        log::trace!("enter: should_filter_response({:?} {})", self, response);
-
-        // quick return if dont_filter is set
-        if CONFIGURATION.dont_filter {
-            // --dont-filter applies specifically to wildcard filters, it is not a 100% catch all
-            // for not filtering anything.  As such, it should live in the implementation of
-            // a wildcard filter
-            return false;
-        }
-
-        if self.size > 0 && self.size == response.content_length() {
-            // static wildcard size found during testing
-            // size isn't default, size equals response length, and auto-filter is on
-            log::debug!("static wildcard: filtered out {}", response.url());
-            log::trace!("exit: should_filter_response -> true");
-            return true;
-        }
-
-        if self.dynamic > 0 {
-            // dynamic wildcard offset found during testing
-
-            // I'm about to manually split this url path instead of using reqwest::Url's
-            // builtin parsing. The reason is that they call .split() on the url path
-            // except that I don't want an empty string taking up the last index in the
-            // event that the url ends with a forward slash.  It's ugly enough to be split
-            // into its own function for readability.
-            let url_len = get_url_path_length(&response.url());
-
-            if url_len + self.dynamic == response.content_length() {
-                log::debug!("dynamic wildcard: filtered out {}", response.url());
-                log::trace!("exit: should_filter_response -> true");
-                return true;
-            }
-        }
-        log::trace!("exit: should_filter_response -> false");
-        false
-    }
-
-    /// Compare one WildcardFilter to another
-    fn box_eq(&self, other: &dyn Any) -> bool {
-        other.downcast_ref::<Self>().map_or(false, |a| self == a)
-    }
-
-    /// Return self as Any for dynamic dispatch purposes
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-}
-
-/// Simple implementor of FeroxFilter; used to filter out status codes specified using
-/// -C|--filter-status
-#[derive(Default, Debug, PartialEq)]
-pub struct StatusCodeFilter {
-    /// Status code that should not be displayed to the user
-    pub filter_code: u16,
-}
-
-/// implementation of FeroxFilter for StatusCodeFilter
-impl FeroxFilter for StatusCodeFilter {
-    /// Check `filter_code` against what was passed in via -C|--filter-status
-    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
-        log::trace!("enter: should_filter_response({:?} {})", self, response);
-
-        if response.status().as_u16() == self.filter_code {
-            log::debug!(
-                "filtered out {} based on --filter-status of {}",
-                response.url(),
-                self.filter_code
-            );
-            log::trace!("exit: should_filter_response -> true");
-            return true;
-        }
-
-        log::trace!("exit: should_filter_response -> false");
-        false
-    }
-
-    /// Compare one StatusCodeFilter to another
-    fn box_eq(&self, other: &dyn Any) -> bool {
-        other.downcast_ref::<Self>().map_or(false, |a| self == a)
-    }
-
-    /// Return self as Any for dynamic dispatch purposes
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-}
-
-/// Simple implementor of FeroxFilter; used to filter out responses based on the number of lines
-/// in a Response body; specified using -N|--filter-lines
-#[derive(Default, Debug, PartialEq)]
-pub struct LinesFilter {
-    /// Number of lines in a Response's body that should be filtered
-    pub line_count: usize,
-}
-
-/// implementation of FeroxFilter for LinesFilter
-impl FeroxFilter for LinesFilter {
-    /// Check `line_count` against what was passed in via -N|--filter-lines
-    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
-        log::trace!("enter: should_filter_response({:?} {})", self, response);
-
-        let result = response.line_count() == self.line_count;
-
-        log::trace!("exit: should_filter_response -> {}", result);
-
-        result
-    }
-
-    /// Compare one LinesFilter to another
-    fn box_eq(&self, other: &dyn Any) -> bool {
-        other.downcast_ref::<Self>().map_or(false, |a| self == a)
-    }
-
-    /// Return self as Any for dynamic dispatch purposes
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-}
-
-/// Simple implementor of FeroxFilter; used to filter out responses based on the number of words
-/// in a Response body; specified using -W|--filter-words
-#[derive(Default, Debug, PartialEq)]
-pub struct WordsFilter {
-    /// Number of words in a Response's body that should be filtered
-    pub word_count: usize,
-}
-
-/// implementation of FeroxFilter for WordsFilter
-impl FeroxFilter for WordsFilter {
-    /// Check `word_count` against what was passed in via -W|--filter-words
-    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
-        log::trace!("enter: should_filter_response({:?} {})", self, response);
-
-        let result = response.word_count() == self.word_count;
-
-        log::trace!("exit: should_filter_response -> {}", result);
-
-        result
-    }
-
-    /// Compare one WordsFilter to another
-    fn box_eq(&self, other: &dyn Any) -> bool {
-        other.downcast_ref::<Self>().map_or(false, |a| self == a)
-    }
-
-    /// Return self as Any for dynamic dispatch purposes
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-}
-
-/// Simple implementor of FeroxFilter; used to filter out responses based on the length of a
-/// Response body; specified using -S|--filter-size
-#[derive(Default, Debug, PartialEq)]
-pub struct SizeFilter {
-    /// Overall length of a Response's body that should be filtered
-    pub content_length: u64,
-}
-
-/// implementation of FeroxFilter for SizeFilter
-impl FeroxFilter for SizeFilter {
-    /// Check `content_length` against what was passed in via -S|--filter-size
-    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
-        log::trace!("enter: should_filter_response({:?} {})", self, response);
-
-        let result = response.content_length() == self.content_length;
-
-        log::trace!("exit: should_filter_response -> {}", result);
-
-        result
-    }
-
-    /// Compare one SizeFilter to another
-    fn box_eq(&self, other: &dyn Any) -> bool {
-        other.downcast_ref::<Self>().map_or(false, |a| self == a)
-    }
-
-    /// Return self as Any for dynamic dispatch purposes
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-}
-
-/// Simple implementor of FeroxFilter; used to filter out responses based on a given regular
-/// expression; specified using -X|--filter-regex
-#[derive(Debug)]
-pub struct RegexFilter {
-    /// Regular expression to be applied to the response body for filtering, compiled
-    pub compiled: Regex,
-
-    /// Regular expression as passed in on the command line, not compiled
-    pub raw_string: String,
-}
-
-/// implementation of FeroxFilter for RegexFilter
-impl FeroxFilter for RegexFilter {
-    /// Check `expression` against the response body, if the expression matches, the response
-    /// should be filtered out
-    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
-        log::trace!("enter: should_filter_response({:?} {})", self, response);
-
-        let result = self.compiled.is_match(response.text());
-
-        log::trace!("exit: should_filter_response -> {}", result);
-
-        result
-    }
-
-    /// Compare one SizeFilter to another
-    fn box_eq(&self, other: &dyn Any) -> bool {
-        other.downcast_ref::<Self>().map_or(false, |a| self == a)
-    }
-
-    /// Return self as Any for dynamic dispatch purposes
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-}
-
-/// PartialEq implementation for RegexFilter
-impl PartialEq for RegexFilter {
-    /// Simple comparison of the raw string passed in via the command line
-    fn eq(&self, other: &RegexFilter) -> bool {
-        self.raw_string == other.raw_string
-    }
-}
-
-/// Simple implementor of FeroxFilter; used to filter out responses based on the similarity of a
-/// Response body with a known response; specified using --filter-similar-to
-#[derive(Default, Debug, PartialEq)]
-pub struct SimilarityFilter {
-    /// Response's body to be used for comparison for similarity
-    pub text: String,
-
-    /// Percentage of similarity at which a page is determined to be a near-duplicate of another
-    pub threshold: u32,
-}
-
-/// implementation of FeroxFilter for SimilarityFilter
-impl FeroxFilter for SimilarityFilter {
-    /// Check `FeroxResponse::text` against what was requested from the site passed in via
-    /// --filter-similar-to
-    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
-        let other = FuzzyHash::new(&response.text);
-
-        if let Ok(result) = FuzzyHash::compare(&self.text, &other.to_string()) {
-            return result >= self.threshold;
-        }
-
-        // couldn't hash the response, don't filter
-        log::warn!("Could not hash body from {}", response.as_str());
-        false
-    }
-
-    /// Compare one SimilarityFilter to another
-    fn box_eq(&self, other: &dyn Any) -> bool {
-        other.downcast_ref::<Self>().map_or(false, |a| self == a)
-    }
-
-    /// Return self as Any for dynamic dispatch purposes
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use reqwest::Url;
-
-    #[test]
-    /// just a simple test to increase code coverage by hitting as_any and the inner value
-    fn lines_filter_as_any() {
-        let filter = LinesFilter { line_count: 1 };
-
-        assert_eq!(filter.line_count, 1);
-        assert_eq!(
-            *filter.as_any().downcast_ref::<LinesFilter>().unwrap(),
-            filter
-        );
-    }
-
-    #[test]
-    /// just a simple test to increase code coverage by hitting as_any and the inner value
-    fn words_filter_as_any() {
-        let filter = WordsFilter { word_count: 1 };
-
-        assert_eq!(filter.word_count, 1);
-        assert_eq!(
-            *filter.as_any().downcast_ref::<WordsFilter>().unwrap(),
-            filter
-        );
-    }
-
-    #[test]
-    /// just a simple test to increase code coverage by hitting as_any and the inner value
-    fn size_filter_as_any() {
-        let filter = SizeFilter { content_length: 1 };
-
-        assert_eq!(filter.content_length, 1);
-        assert_eq!(
-            *filter.as_any().downcast_ref::<SizeFilter>().unwrap(),
-            filter
-        );
-    }
-
-    #[test]
-    /// just a simple test to increase code coverage by hitting as_any and the inner value
-    fn status_code_filter_as_any() {
-        let filter = StatusCodeFilter { filter_code: 200 };
-
-        assert_eq!(filter.filter_code, 200);
-        assert_eq!(
-            *filter.as_any().downcast_ref::<StatusCodeFilter>().unwrap(),
-            filter
-        );
-    }
-
-    #[test]
-    /// just a simple test to increase code coverage by hitting as_any and the inner value
-    fn regex_filter_as_any() {
-        let raw = r".*\.txt$";
-        let compiled = Regex::new(raw).unwrap();
-        let filter = RegexFilter {
-            compiled,
-            raw_string: raw.to_string(),
-        };
-
-        assert_eq!(filter.raw_string, r".*\.txt$");
-        assert_eq!(
-            *filter.as_any().downcast_ref::<RegexFilter>().unwrap(),
-            filter
-        );
-    }
-
-    #[test]
-    /// test should_filter on WilcardFilter where static logic matches
-    fn wildcard_should_filter_when_static_wildcard_found() {
-        let resp = FeroxResponse {
-            text: String::new(),
-            wildcard: true,
-            url: Url::parse("http://localhost").unwrap(),
-            content_length: 100,
-            word_count: 50,
-            line_count: 25,
-            headers: reqwest::header::HeaderMap::new(),
-            status: reqwest::StatusCode::OK,
-        };
-
-        let filter = WildcardFilter {
-            size: 100,
-            dynamic: 0,
-        };
-
-        assert!(filter.should_filter_response(&resp));
-    }
-
-    #[test]
-    /// test should_filter on WilcardFilter where dynamic logic matches
-    fn wildcard_should_filter_when_dynamic_wildcard_found() {
-        let resp = FeroxResponse {
-            text: String::new(),
-            wildcard: true,
-            url: Url::parse("http://localhost/stuff").unwrap(),
-            content_length: 100,
-            word_count: 50,
-            line_count: 25,
-            headers: reqwest::header::HeaderMap::new(),
-            status: reqwest::StatusCode::OK,
-        };
-
-        let filter = WildcardFilter {
-            size: 0,
-            dynamic: 95,
-        };
-
-        assert!(filter.should_filter_response(&resp));
-    }
-
-    #[test]
-    /// test should_filter on RegexFilter where regex matches body
-    fn regexfilter_should_filter_when_regex_matches_on_response_body() {
-        let resp = FeroxResponse {
-            text: String::from("im a body response hurr durr!"),
-            wildcard: false,
-            url: Url::parse("http://localhost/stuff").unwrap(),
-            content_length: 100,
-            word_count: 50,
-            line_count: 25,
-            headers: reqwest::header::HeaderMap::new(),
-            status: reqwest::StatusCode::OK,
-        };
-
-        let raw = r"response...rr";
-
-        let filter = RegexFilter {
-            raw_string: raw.to_string(),
-            compiled: Regex::new(raw).unwrap(),
-        };
-
-        assert!(filter.should_filter_response(&resp));
-    }
-
-    #[test]
-    /// a few simple tests for similarity filter
-    fn similarity_filter_is_accurate() {
-        let mut resp = FeroxResponse {
-            text: String::from("sitting"),
-            wildcard: false,
-            url: Url::parse("http://localhost/stuff").unwrap(),
-            content_length: 100,
-            word_count: 50,
-            line_count: 25,
-            headers: reqwest::header::HeaderMap::new(),
-            status: reqwest::StatusCode::OK,
-        };
-
-        let mut filter = SimilarityFilter {
-            text: FuzzyHash::new("kitten").to_string(),
-            threshold: 95,
-        };
-
-        // kitten/sitting is 57% similar, so a threshold of 95 should not be filtered
-        assert!(!filter.should_filter_response(&resp));
-
-        resp.text = String::new();
-        filter.text = String::new();
-        filter.threshold = 100;
-
-        // two empty strings are the same, however ssdeep doesn't accept empty strings, expect false
-        assert!(!filter.should_filter_response(&resp));
-
-        resp.text = String::from("some data to hash for the purposes of running a test");
-        filter.text =
-            FuzzyHash::new("some data to hash for the purposes of running a te").to_string();
-        filter.threshold = 17;
-
-        assert!(filter.should_filter_response(&resp));
-    }
-
-    #[test]
-    /// just a simple test to increase code coverage by hitting as_any and the inner value
-    fn similarity_filter_as_any() {
-        let filter = SimilarityFilter {
-            text: String::from("stuff"),
-            threshold: 95,
-        };
-
-        assert_eq!(filter.text, "stuff");
-        assert_eq!(
-            *filter.as_any().downcast_ref::<SimilarityFilter>().unwrap(),
-            filter
-        );
-    }
-}

src/filters/lines.rs

@@ -0,0 +1,33 @@
+use super::*;
+
+/// Simple implementor of FeroxFilter; used to filter out responses based on the number of lines
+/// in a Response body; specified using -N|--filter-lines
+#[derive(Default, Debug, PartialEq)]
+pub struct LinesFilter {
+    /// Number of lines in a Response's body that should be filtered
+    pub line_count: usize,
+}
+
+/// implementation of FeroxFilter for LinesFilter
+impl FeroxFilter for LinesFilter {
+    /// Check `line_count` against what was passed in via -N|--filter-lines
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        let result = response.line_count() == self.line_count;
+
+        log::trace!("exit: should_filter_response -> {}", result);
+
+        result
+    }
+
+    /// Compare one LinesFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/filters/mod.rs

@@ -0,0 +1,24 @@
+//! module containing all of feroxbuster's filters
+mod traits;
+mod wildcard;
+mod status_code;
+mod words;
+mod lines;
+mod size;
+mod regex;
+mod similarity;
+#[cfg(test)]
+mod tests;
+
+pub use self::lines::LinesFilter;
+pub use self::regex::RegexFilter;
+pub use self::similarity::SimilarityFilter;
+pub use self::size::SizeFilter;
+pub use self::status_code::StatusCodeFilter;
+pub use self::traits::FeroxFilter;
+pub use self::wildcard::WildcardFilter;
+pub use self::words::WordsFilter;
+
+use crate::{config::CONFIGURATION, utils::get_url_path_length, FeroxResponse, FeroxSerialize};
+use std::any::Any;
+use std::fmt::Debug;

src/filters/regex.rs

@@ -0,0 +1,46 @@
+use super::*;
+use ::regex::Regex;
+
+/// Simple implementor of FeroxFilter; used to filter out responses based on a given regular
+/// expression; specified using -X|--filter-regex
+#[derive(Debug)]
+pub struct RegexFilter {
+    /// Regular expression to be applied to the response body for filtering, compiled
+    pub compiled: Regex,
+
+    /// Regular expression as passed in on the command line, not compiled
+    pub raw_string: String,
+}
+
+/// implementation of FeroxFilter for RegexFilter
+impl FeroxFilter for RegexFilter {
+    /// Check `expression` against the response body, if the expression matches, the response
+    /// should be filtered out
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        let result = self.compiled.is_match(response.text());
+
+        log::trace!("exit: should_filter_response -> {}", result);
+
+        result
+    }
+
+    /// Compare one SizeFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
+
+/// PartialEq implementation for RegexFilter
+impl PartialEq for RegexFilter {
+    /// Simple comparison of the raw string passed in via the command line
+    fn eq(&self, other: &RegexFilter) -> bool {
+        self.raw_string == other.raw_string
+    }
+}

src/filters/similarity.rs

@@ -0,0 +1,40 @@
+use super::*;
+use fuzzyhash::FuzzyHash;
+
+/// Simple implementor of FeroxFilter; used to filter out responses based on the similarity of a
+/// Response body with a known response; specified using --filter-similar-to
+#[derive(Default, Debug, PartialEq)]
+pub struct SimilarityFilter {
+    /// Response's body to be used for comparison for similarity
+    pub text: String,
+
+    /// Percentage of similarity at which a page is determined to be a near-duplicate of another
+    pub threshold: u32,
+}
+
+/// implementation of FeroxFilter for SimilarityFilter
+impl FeroxFilter for SimilarityFilter {
+    /// Check `FeroxResponse::text` against what was requested from the site passed in via
+    /// --filter-similar-to
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        let other = FuzzyHash::new(&response.text);
+
+        if let Ok(result) = FuzzyHash::compare(&self.text, &other.to_string()) {
+            return result >= self.threshold;
+        }
+
+        // couldn't hash the response, don't filter
+        log::warn!("Could not hash body from {}", response.as_str());
+        false
+    }
+
+    /// Compare one SimilarityFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/filters/size.rs

@@ -0,0 +1,33 @@
+use super::*;
+
+/// Simple implementor of FeroxFilter; used to filter out responses based on the length of a
+/// Response body; specified using -S|--filter-size
+#[derive(Default, Debug, PartialEq)]
+pub struct SizeFilter {
+    /// Overall length of a Response's body that should be filtered
+    pub content_length: u64,
+}
+
+/// implementation of FeroxFilter for SizeFilter
+impl FeroxFilter for SizeFilter {
+    /// Check `content_length` against what was passed in via -S|--filter-size
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        let result = response.content_length() == self.content_length;
+
+        log::trace!("exit: should_filter_response -> {}", result);
+
+        result
+    }
+
+    /// Compare one SizeFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/filters/status_code.rs

@@ -0,0 +1,40 @@
+use super::*;
+
+/// Simple implementor of FeroxFilter; used to filter out status codes specified using
+/// -C|--filter-status
+#[derive(Default, Debug, PartialEq)]
+pub struct StatusCodeFilter {
+    /// Status code that should not be displayed to the user
+    pub filter_code: u16,
+}
+
+/// implementation of FeroxFilter for StatusCodeFilter
+impl FeroxFilter for StatusCodeFilter {
+    /// Check `filter_code` against what was passed in via -C|--filter-status
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        if response.status().as_u16() == self.filter_code {
+            log::debug!(
+                "filtered out {} based on --filter-status of {}",
+                response.url(),
+                self.filter_code
+            );
+            log::trace!("exit: should_filter_response -> true");
+            return true;
+        }
+
+        log::trace!("exit: should_filter_response -> false");
+        false
+    }
+
+    /// Compare one StatusCodeFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/filters/tests.rs

@@ -0,0 +1,188 @@
+use super::*;
+use ::fuzzyhash::FuzzyHash;
+use ::regex::Regex;
+use reqwest::Url;
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn lines_filter_as_any() {
+    let filter = LinesFilter { line_count: 1 };
+
+    assert_eq!(filter.line_count, 1);
+    assert_eq!(
+        *filter.as_any().downcast_ref::<LinesFilter>().unwrap(),
+        filter
+    );
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn words_filter_as_any() {
+    let filter = WordsFilter { word_count: 1 };
+
+    assert_eq!(filter.word_count, 1);
+    assert_eq!(
+        *filter.as_any().downcast_ref::<WordsFilter>().unwrap(),
+        filter
+    );
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn size_filter_as_any() {
+    let filter = SizeFilter { content_length: 1 };
+
+    assert_eq!(filter.content_length, 1);
+    assert_eq!(
+        *filter.as_any().downcast_ref::<SizeFilter>().unwrap(),
+        filter
+    );
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn status_code_filter_as_any() {
+    let filter = StatusCodeFilter { filter_code: 200 };
+
+    assert_eq!(filter.filter_code, 200);
+    assert_eq!(
+        *filter.as_any().downcast_ref::<StatusCodeFilter>().unwrap(),
+        filter
+    );
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn regex_filter_as_any() {
+    let raw = r".*\.txt$";
+    let compiled = Regex::new(raw).unwrap();
+    let filter = RegexFilter {
+        compiled,
+        raw_string: raw.to_string(),
+    };
+
+    assert_eq!(filter.raw_string, r".*\.txt$");
+    assert_eq!(
+        *filter.as_any().downcast_ref::<RegexFilter>().unwrap(),
+        filter
+    );
+}
+
+#[test]
+/// test should_filter on WilcardFilter where static logic matches
+fn wildcard_should_filter_when_static_wildcard_found() {
+    let resp = FeroxResponse {
+        text: String::new(),
+        wildcard: true,
+        url: Url::parse("http://localhost").unwrap(),
+        content_length: 100,
+        word_count: 50,
+        line_count: 25,
+        headers: reqwest::header::HeaderMap::new(),
+        status: reqwest::StatusCode::OK,
+    };
+
+    let filter = WildcardFilter {
+        size: 100,
+        dynamic: 0,
+    };
+
+    assert!(filter.should_filter_response(&resp));
+}
+
+#[test]
+/// test should_filter on WilcardFilter where dynamic logic matches
+fn wildcard_should_filter_when_dynamic_wildcard_found() {
+    let resp = FeroxResponse {
+        text: String::new(),
+        wildcard: true,
+        url: Url::parse("http://localhost/stuff").unwrap(),
+        content_length: 100,
+        word_count: 50,
+        line_count: 25,
+        headers: reqwest::header::HeaderMap::new(),
+        status: reqwest::StatusCode::OK,
+    };
+
+    let filter = WildcardFilter {
+        size: 0,
+        dynamic: 95,
+    };
+
+    assert!(filter.should_filter_response(&resp));
+}
+
+#[test]
+/// test should_filter on RegexFilter where regex matches body
+fn regexfilter_should_filter_when_regex_matches_on_response_body() {
+    let resp = FeroxResponse {
+        text: String::from("im a body response hurr durr!"),
+        wildcard: false,
+        url: Url::parse("http://localhost/stuff").unwrap(),
+        content_length: 100,
+        word_count: 50,
+        line_count: 25,
+        headers: reqwest::header::HeaderMap::new(),
+        status: reqwest::StatusCode::OK,
+    };
+
+    let raw = r"response...rr";
+
+    let filter = RegexFilter {
+        raw_string: raw.to_string(),
+        compiled: Regex::new(raw).unwrap(),
+    };
+
+    assert!(filter.should_filter_response(&resp));
+}
+
+#[test]
+/// a few simple tests for similarity filter
+fn similarity_filter_is_accurate() {
+    let mut resp = FeroxResponse {
+        text: String::from("sitting"),
+        wildcard: false,
+        url: Url::parse("http://localhost/stuff").unwrap(),
+        content_length: 100,
+        word_count: 50,
+        line_count: 25,
+        headers: reqwest::header::HeaderMap::new(),
+        status: reqwest::StatusCode::OK,
+    };
+
+    let mut filter = SimilarityFilter {
+        text: FuzzyHash::new("kitten").to_string(),
+        threshold: 95,
+    };
+
+    // kitten/sitting is 57% similar, so a threshold of 95 should not be filtered
+    assert!(!filter.should_filter_response(&resp));
+
+    resp.text = String::new();
+    filter.text = String::new();
+    filter.threshold = 100;
+
+    // two empty strings are the same, however ssdeep doesn't accept empty strings, expect false
+    assert!(!filter.should_filter_response(&resp));
+
+    resp.text = String::from("some data to hash for the purposes of running a test");
+    filter.text = FuzzyHash::new("some data to hash for the purposes of running a te").to_string();
+    filter.threshold = 17;
+
+    assert!(filter.should_filter_response(&resp));
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn similarity_filter_as_any() {
+    let filter = SimilarityFilter {
+        text: String::from("stuff"),
+        threshold: 95,
+    };
+
+    assert_eq!(filter.text, "stuff");
+    assert_eq!(
+        *filter.as_any().downcast_ref::<SimilarityFilter>().unwrap(),
+        filter
+    );
+}

src/filters/traits.rs

@@ -0,0 +1,27 @@
+use super::*;
+
+// references:
+//   https://dev.to/magnusstrale/rust-trait-objects-in-a-vector-non-trivial-4co5
+//   https://stackoverflow.com/questions/25339603/how-to-test-for-equality-between-trait-objects
+
+/// FeroxFilter trait; represents different types of possible filters that can be applied to
+/// responses
+pub trait FeroxFilter: Debug + Send + Sync {
+    /// Determine whether or not this particular filter should be applied or not
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool;
+
+    /// delegates to the FeroxFilter-implementing type which gives us the actual type of self
+    fn box_eq(&self, other: &dyn Any) -> bool;
+
+    /// gives us `other` as Any in box_eq
+    fn as_any(&self) -> &dyn Any;
+}
+
+/// implementation of PartialEq, necessary long-form due to "trait cannot be made into an object"
+/// error when attempting to derive PartialEq on the trait itself
+impl PartialEq for Box<dyn FeroxFilter> {
+    /// Perform a comparison of two implementors of the FeroxFilter trait
+    fn eq(&self, other: &Box<dyn FeroxFilter>) -> bool {
+        self.box_eq(other.as_any())
+    }
+}

src/filters/wildcard.rs

@@ -0,0 +1,73 @@
+use super::*;
+
+/// Data holder for two pieces of data needed when auto-filtering out wildcard responses
+///
+/// `dynamic` is the size of the response that will later be combined with the length
+/// of the path of the url requested and used to determine interesting pages from custom
+/// 404s where the requested url is reflected back in the response
+///
+/// `size` is size of the response that should be included with filters passed via runtime
+/// configuration and any static wildcard lengths.
+#[derive(Debug, Default, Clone, PartialEq)]
+pub struct WildcardFilter {
+    /// size of the response that will later be combined with the length of the path of the url
+    /// requested
+    pub dynamic: u64,
+
+    /// size of the response that should be included with filters passed via runtime configuration
+    pub size: u64,
+}
+
+/// implementation of FeroxFilter for WildcardFilter
+impl FeroxFilter for WildcardFilter {
+    /// Examine size, dynamic, and content_len to determine whether or not the response received
+    /// is a wildcard response and therefore should be filtered out
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        // quick return if dont_filter is set
+        if CONFIGURATION.dont_filter {
+            // --dont-filter applies specifically to wildcard filters, it is not a 100% catch all
+            // for not filtering anything.  As such, it should live in the implementation of
+            // a wildcard filter
+            return false;
+        }
+
+        if self.size > 0 && self.size == response.content_length() {
+            // static wildcard size found during testing
+            // size isn't default, size equals response length, and auto-filter is on
+            log::debug!("static wildcard: filtered out {}", response.url());
+            log::trace!("exit: should_filter_response -> true");
+            return true;
+        }
+
+        if self.dynamic > 0 {
+            // dynamic wildcard offset found during testing
+
+            // I'm about to manually split this url path instead of using reqwest::Url's
+            // builtin parsing. The reason is that they call .split() on the url path
+            // except that I don't want an empty string taking up the last index in the
+            // event that the url ends with a forward slash.  It's ugly enough to be split
+            // into its own function for readability.
+            let url_len = get_url_path_length(&response.url());
+
+            if url_len + self.dynamic == response.content_length() {
+                log::debug!("dynamic wildcard: filtered out {}", response.url());
+                log::trace!("exit: should_filter_response -> true");
+                return true;
+            }
+        }
+        log::trace!("exit: should_filter_response -> false");
+        false
+    }
+
+    /// Compare one WildcardFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/filters/words.rs

@@ -0,0 +1,33 @@
+use super::*;
+
+/// Simple implementor of FeroxFilter; used to filter out responses based on the number of words
+/// in a Response body; specified using -W|--filter-words
+#[derive(Default, Debug, PartialEq)]
+pub struct WordsFilter {
+    /// Number of words in a Response's body that should be filtered
+    pub word_count: usize,
+}
+
+/// implementation of FeroxFilter for WordsFilter
+impl FeroxFilter for WordsFilter {
+    /// Check `word_count` against what was passed in via -W|--filter-words
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        let result = response.word_count() == self.word_count;
+
+        log::trace!("exit: should_filter_response -> {}", result);
+
+        result
+    }
+
+    /// Compare one WordsFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/heuristics.rs

@@ -2,6 +2,7 @@ use crate::{
     config::{CONFIGURATION, PROGRESS_PRINTER},
     filters::WildcardFilter,
     scanner::should_filter_response,
+    statistics::StatCommand,
     utils::{ferox_print, format_url, get_url_path_length, make_request, status_colorizer},
     FeroxResponse,
 };
@@ -40,12 +41,14 @@ pub async fn wildcard_test(
     target_url: &str,
     bar: ProgressBar,
     tx_term: UnboundedSender<FeroxResponse>,
+    tx_stats: UnboundedSender<StatCommand>,
 ) -> Option<WildcardFilter> {
     log::trace!(
-        "enter: wildcard_test({:?}, {:?}, {:?})",
+        "enter: wildcard_test({:?}, {:?}, {:?}, {:?})",
         target_url,
         bar,
-        tx_term
+        tx_term,
+        tx_stats
     );
 
     if CONFIGURATION.dont_filter {
@@ -54,10 +57,14 @@ pub async fn wildcard_test(
         return None;
     }
 
-    let tx_clone_one = tx_term.clone();
-    let tx_clone_two = tx_term.clone();
+    let tx_term_mwcr1 = tx_term.clone();
+    let tx_term_mwcr2 = tx_term.clone();
+    let tx_stats_mwcr1 = tx_stats.clone();
+    let tx_stats_mwcr2 = tx_stats.clone();
 
-    if let Some(ferox_response) = make_wildcard_request(&target_url, 1, tx_clone_one).await {
+    if let Some(ferox_response) =
+        make_wildcard_request(&target_url, 1, tx_term_mwcr1, tx_stats_mwcr1).await
+    {
         bar.inc(1);
 
         // found a wildcard response
@@ -72,7 +79,9 @@ pub async fn wildcard_test(
 
         // content length of wildcard is non-zero, perform additional tests:
         //   make a second request, with a known-sized (64) longer request
-        if let Some(resp_two) = make_wildcard_request(&target_url, 3, tx_clone_two).await {
+        if let Some(resp_two) =
+            make_wildcard_request(&target_url, 3, tx_term_mwcr2, tx_stats_mwcr2).await
+        {
             bar.inc(1);
 
             let wc2_length = resp_two.content_length();
@@ -138,12 +147,14 @@ async fn make_wildcard_request(
     target_url: &str,
     length: usize,
     tx_file: UnboundedSender<FeroxResponse>,
+    tx_stats: UnboundedSender<StatCommand>,
 ) -> Option<FeroxResponse> {
     log::trace!(
-        "enter: make_wildcard_request({}, {}, {:?})",
+        "enter: make_wildcard_request({}, {}, {:?}, {:?})",
         target_url,
         length,
-        tx_file
+        tx_file,
+        tx_stats,
     );
 
     let unique_str = unique_string(length);
@@ -154,6 +165,7 @@ async fn make_wildcard_request(
         CONFIGURATION.add_slash,
         &CONFIGURATION.queries,
         None,
+        tx_stats.clone(),
     ) {
         Ok(url) => url,
         Err(e) => {
@@ -163,7 +175,13 @@ async fn make_wildcard_request(
         }
     };
 
-    match make_request(&CONFIGURATION.client, &nonexistent.to_owned()).await {
+    match make_request(
+        &CONFIGURATION.client,
+        &nonexistent.to_owned(),
+        tx_stats.clone(),
+    )
+    .await
+    {
         Ok(response) => {
             if CONFIGURATION
                 .status_codes
@@ -174,7 +192,7 @@ async fn make_wildcard_request(
                 ferox_response.wildcard = true;
 
                 if !CONFIGURATION.quiet
-                    && !should_filter_response(&ferox_response)
+                    && !should_filter_response(&ferox_response, tx_stats.clone())
                     && tx_file.send(ferox_response.clone()).is_err()
                 {
                     return None;
@@ -190,6 +208,7 @@ async fn make_wildcard_request(
             return None;
         }
     }
+
     log::trace!("exit: make_wildcard_request -> None");
     None
 }
@@ -199,8 +218,15 @@ async fn make_wildcard_request(
 /// In the event that no sites can be reached, the program will exit.
 ///
 /// Any urls that are found to be alive are returned to the caller.
-pub async fn connectivity_test(target_urls: &[String]) -> Vec<String> {
-    log::trace!("enter: connectivity_test({:?})", target_urls);
+pub async fn connectivity_test(
+    target_urls: &[String],
+    tx_stats: UnboundedSender<StatCommand>,
+) -> Vec<String> {
+    log::trace!(
+        "enter: connectivity_test({:?}, {:?})",
+        target_urls,
+        tx_stats
+    );
 
     let mut good_urls = vec![];
 
@@ -211,6 +237,7 @@ pub async fn connectivity_test(target_urls: &[String]) -> Vec<String> {
             CONFIGURATION.add_slash,
             &CONFIGURATION.queries,
             None,
+            tx_stats.clone(),
         ) {
             Ok(url) => url,
             Err(e) => {
@@ -219,7 +246,7 @@ pub async fn connectivity_test(target_urls: &[String]) -> Vec<String> {
             }
         };
 
-        match make_request(&CONFIGURATION.client, &request).await {
+        match make_request(&CONFIGURATION.client, &request, tx_stats.clone()).await {
             Ok(_) => {
                 good_urls.push(target_url.to_owned());
             }

src/lib.rs

@@ -1,3 +1,4 @@
+pub mod utils;
 pub mod banner;
 pub mod client;
 pub mod config;
@@ -10,7 +11,7 @@ pub mod progress;
 pub mod reporter;
 pub mod scan_manager;
 pub mod scanner;
-pub mod utils;
+pub mod statistics;
 
 use crate::utils::{get_url_path_length, status_colorizer};
 use console::{style, Color};

src/main.rs

@@ -2,21 +2,27 @@ use crossterm::event::{self, Event, KeyCode};
 use feroxbuster::{
     banner,
     config::{CONFIGURATION, PROGRESS_BAR, PROGRESS_PRINTER},
-    extractor::{extract_robots_txt, request_feroxresponse_from_new_link},
     heuristics, logger,
-    progress::add_bar,
+    progress::{add_bar, BarType},
     reporter,
-    scan_manager::{self, PAUSE_SCAN},
-    scanner::{self, scan_url, send_report, RESPONSES, SCANNED_URLS},
+    scan_manager::{self, ScanStatus, PAUSE_SCAN},
+    scanner::{self, scan_url, SCANNED_URLS},
+    statistics::{
+        self,
+        StatCommand::{self, CreateBar, LoadStats, UpdateUsizeField},
+        StatField::InitialTargets,
+        Stats,
+    },
+    update_stat,
     utils::{ferox_print, get_current_depth, module_colorizer, status_colorizer},
-    FeroxError, FeroxResponse, FeroxResult, FeroxSerialize, SLEEP_DURATION, VERSION,
+    FeroxError, FeroxResponse, FeroxResult, SLEEP_DURATION, VERSION,
 };
 #[cfg(not(target_os = "windows"))]
 use feroxbuster::{utils::set_open_file_limit, DEFAULT_OPEN_FILE_LIMIT};
 use futures::StreamExt;
-use std::convert::TryInto;
 use std::{
     collections::HashSet,
+    convert::TryInto,
     fs::File,
     io::{stderr, BufRead, BufReader},
     process,
@@ -24,6 +30,7 @@ use std::{
         atomic::{AtomicBool, Ordering},
         Arc,
     },
+    thread::sleep,
     time::Duration,
 };
 use tokio::{io, sync::mpsc::UnboundedSender, task::JoinHandle};
@@ -37,17 +44,20 @@ fn terminal_input_handler() {
     log::trace!("enter: terminal_input_handler");
 
     loop {
-        if event::poll(Duration::from_millis(SLEEP_DURATION)).unwrap_or(false) {
+        if PAUSE_SCAN.load(Ordering::Relaxed) {
+            // if the scan is already paused, we don't want this event poller fighting the user
+            // over stdin
+            sleep(Duration::from_millis(SLEEP_DURATION));
+        } else if event::poll(Duration::from_millis(SLEEP_DURATION)).unwrap_or(false) {
             // It's guaranteed that the `read()` won't block when the `poll()`
             // function returns `true`
 
             if let Ok(key_pressed) = event::read() {
+                // ignore any other keys
                 if key_pressed == Event::Key(KeyCode::Enter.into()) {
-                    // if the user presses Enter, toggle the value stored in PAUSE_SCAN
-                    // ignore any other keys
-                    let current = PAUSE_SCAN.load(Ordering::Acquire);
-
-                    PAUSE_SCAN.store(!current, Ordering::Release);
+                    // if the user presses Enter, set PAUSE_SCAN to true. The interactive menu
+                    // will be triggered and will handle setting PAUSE_SCAN to false
+                    PAUSE_SCAN.store(true, Ordering::Release);
                 }
             }
         } else {
@@ -99,11 +109,20 @@ fn get_unique_words_from_wordlist(path: &str) -> FeroxResult<Arc<HashSet<String>
 
 /// Determine whether it's a single url scan or urls are coming from stdin, then scan as needed
 async fn scan(
-    mut targets: Vec<String>,
+    targets: Vec<String>,
+    stats: Arc<Stats>,
     tx_term: UnboundedSender<FeroxResponse>,
     tx_file: UnboundedSender<FeroxResponse>,
+    tx_stats: UnboundedSender<StatCommand>,
 ) -> FeroxResult<()> {
-    log::trace!("enter: scan({:?}, {:?}, {:?})", targets, tx_term, tx_file);
+    log::trace!(
+        "enter: scan({:?}, {:?}, {:?}, {:?}, {:?})",
+        targets,
+        stats,
+        tx_term,
+        tx_file,
+        tx_stats
+    );
     // cloning an Arc is cheap (it's basically a pointer into the heap)
     // so that will allow for cheap/safe sharing of a single wordlist across multi-target scans
     // as well as additional directories found as part of recursion
@@ -112,76 +131,53 @@ async fn scan(
             .await??;
 
     if words.len() == 0 {
-        let mut err = FeroxError::default();
-        err.message = format!("Did not find any words in {}", CONFIGURATION.wordlist);
+        let err = FeroxError {
+            message: format!("Did not find any words in {}", CONFIGURATION.wordlist),
+        };
+
         return Err(Box::new(err));
     }
 
-    scanner::initialize(words.len(), &CONFIGURATION).await;
+    scanner::initialize(words.len(), &CONFIGURATION, tx_stats.clone()).await;
+
+    // at this point, the stat thread's progress bar can be created; things that needed to happen
+    // first:
+    // - banner gets printed
+    // - scanner initialized (this sent expected requests per directory to the stats thread, which
+    //   having been set, makes it so the progress bar doesn't flash as full before anything has
+    //   even happened
+    update_stat!(tx_stats, CreateBar);
 
     if CONFIGURATION.resumed {
+        update_stat!(tx_stats, LoadStats(CONFIGURATION.resume_from.clone()));
+
+        SCANNED_URLS.print_known_responses();
+
         if let Ok(scans) = SCANNED_URLS.scans.lock() {
             for scan in scans.iter() {
                 if let Ok(locked_scan) = scan.lock() {
-                    if locked_scan.complete {
+                    if matches!(locked_scan.status, ScanStatus::Complete) {
                         // these scans are complete, and just need to be shown to the user
                         let pb = add_bar(
                             &locked_scan.url,
                             words.len().try_into().unwrap_or_default(),
-                            false,
-                            true,
+                            BarType::Message,
                         );
                         pb.finish();
                     }
                 }
             }
         }
-
-        if let Ok(responses) = RESPONSES.responses.read() {
-            for response in responses.iter() {
-                PROGRESS_PRINTER.println(response.as_str());
-            }
-        }
-    }
-
-    if CONFIGURATION.extract_links {
-        for target in targets.clone() {
-            // modifying the targets vector, so we can't have a reference to it while we borrow
-            // it as mutable; thus the clone
-            let robots_links = extract_robots_txt(&target, &CONFIGURATION).await;
-
-            for robot_link in robots_links {
-                // create a url based on the given command line options, continue on error
-                let ferox_response = match request_feroxresponse_from_new_link(&robot_link).await {
-                    Some(resp) => resp,
-                    None => continue,
-                };
-
-                if ferox_response.is_file() {
-                    SCANNED_URLS.add_file_scan(&robot_link);
-                    send_report(tx_term.clone(), ferox_response);
-                } else {
-                    let (unknown, _) = SCANNED_URLS.add_directory_scan(&robot_link);
-
-                    if !unknown {
-                        // known directory; can skip (unlikely)
-                        continue;
-                    }
-
-                    // unknown directory; add to targets for scanning
-                    targets.push(robot_link);
-                }
-            }
-        }
     }
 
     let mut tasks = vec![];
-    let num_targets = targets.len();
 
     for target in targets {
         let word_clone = words.clone();
         let term_clone = tx_term.clone();
         let file_clone = tx_file.clone();
+        let tx_stats_clone = tx_stats.clone();
+        let stats_clone = stats.clone();
 
         let task = tokio::spawn(async move {
             let base_depth = get_current_depth(&target);
@@ -189,9 +185,10 @@ async fn scan(
                 &target,
                 word_clone,
                 base_depth,
-                num_targets,
+                stats_clone,
                 term_clone,
                 file_clone,
+                tx_stats_clone,
             )
             .await;
         });
@@ -229,7 +226,7 @@ async fn get_targets() -> FeroxResult<Vec<String>> {
                 // SCANNED_URLS gets deserialized scans added to it at program start if --resume-from
                 // is used, so scans that aren't marked complete still need to be scanned
                 if let Ok(locked_scan) = scan.lock() {
-                    if locked_scan.complete {
+                    if matches!(locked_scan.status, ScanStatus::Complete) {
                         // this one's already done, ignore it
                         continue;
                     }
@@ -262,11 +259,16 @@ async fn wrapped_main() {
         PROGRESS_BAR.join().unwrap();
     });
 
+    let (stats, tx_stats, stats_handle) = statistics::initialize();
+
     if !CONFIGURATION.time_limit.is_empty() {
         // --time-limit value not an empty string, need to kick off the thread that enforces
         // the limit
+
+        let max_time_stats = stats.clone();
+
         tokio::spawn(async move {
-            scan_manager::start_max_time_thread(&CONFIGURATION.time_limit).await
+            scan_manager::start_max_time_thread(&CONFIGURATION.time_limit, max_time_stats).await
         });
     }
 
@@ -280,8 +282,13 @@ async fn wrapped_main() {
 
     let save_output = !CONFIGURATION.output.is_empty(); // was -o used?
 
+    if CONFIGURATION.save_state {
+        // start the ctrl+c handler
+        scan_manager::initialize(stats.clone());
+    }
+
     let (tx_term, tx_file, term_handle, file_handle) =
-        reporter::initialize(&CONFIGURATION.output, save_output);
+        reporter::initialize(&CONFIGURATION.output, save_output, tx_stats.clone());
 
     // get targets from command line or stdin
     let targets = match get_targets().await {
@@ -289,27 +296,62 @@ async fn wrapped_main() {
         Err(e) => {
             // should only happen in the event that there was an error reading from stdin
             log::error!("{} {}", module_colorizer("main::get_targets"), e);
-            clean_up(tx_term, term_handle, tx_file, file_handle, save_output).await;
+            clean_up(
+                tx_term,
+                term_handle,
+                tx_file,
+                file_handle,
+                tx_stats,
+                stats_handle,
+                save_output,
+            )
+            .await;
             return;
         }
     };
 
+    update_stat!(tx_stats, UpdateUsizeField(InitialTargets, targets.len()));
+
     if !CONFIGURATION.quiet {
         // only print banner if -q isn't used
         let std_stderr = stderr(); // std::io::stderr
-        banner::initialize(&targets, &CONFIGURATION, &VERSION, std_stderr).await;
+        banner::initialize(
+            &targets,
+            &CONFIGURATION,
+            &VERSION,
+            std_stderr,
+            tx_stats.clone(),
+        )
+        .await;
     }
 
     // discard non-responsive targets
-    let live_targets = heuristics::connectivity_test(&targets).await;
+    let live_targets = heuristics::connectivity_test(&targets, tx_stats.clone()).await;
 
     if live_targets.is_empty() {
-        clean_up(tx_term, term_handle, tx_file, file_handle, save_output).await;
+        clean_up(
+            tx_term,
+            term_handle,
+            tx_file,
+            file_handle,
+            tx_stats,
+            stats_handle,
+            save_output,
+        )
+        .await;
         return;
     }
 
     // kick off a scan against any targets determined to be responsive
-    match scan(live_targets, tx_term.clone(), tx_file.clone()).await {
+    match scan(
+        live_targets,
+        stats,
+        tx_term.clone(),
+        tx_file.clone(),
+        tx_stats.clone(),
+    )
+    .await
+    {
         Ok(_) => {
             log::info!("All scans complete!");
         }
@@ -318,14 +360,32 @@ async fn wrapped_main() {
                 &format!("{} while scanning: {}", status_colorizer("Error"), e),
                 &PROGRESS_PRINTER,
             );
-            clean_up(tx_term, term_handle, tx_file, file_handle, save_output).await;
+            clean_up(
+                tx_term,
+                term_handle,
+                tx_file,
+                file_handle,
+                tx_stats,
+                stats_handle,
+                save_output,
+            )
+            .await;
             process::exit(1);
         }
     };
 
-    clean_up(tx_term, term_handle, tx_file, file_handle, save_output).await;
+    clean_up(
+        tx_term,
+        term_handle,
+        tx_file,
+        file_handle,
+        tx_stats,
+        stats_handle,
+        save_output,
+    )
+    .await;
 
-    log::trace!("exit: main");
+    log::trace!("exit: wrapped_main");
 }
 
 /// Single cleanup function that handles all the necessary drops/finishes etc required to gracefully
@@ -335,17 +395,20 @@ async fn clean_up(
     term_handle: JoinHandle<()>,
     tx_file: UnboundedSender<FeroxResponse>,
     file_handle: Option<JoinHandle<()>>,
+    tx_stats: UnboundedSender<StatCommand>,
+    stats_handle: JoinHandle<()>,
     save_output: bool,
 ) {
     log::trace!(
-        "enter: clean_up({:?}, {:?}, {:?}, {:?}, {})",
+        "enter: clean_up({:?}, {:?}, {:?}, {:?}, {:?}, {:?}, {})",
         tx_term,
         term_handle,
         tx_file,
         file_handle,
+        tx_stats,
+        stats_handle,
         save_output
     );
-
     drop(tx_term);
     log::trace!("dropped terminal output handler's transmitter");
 
@@ -377,6 +440,9 @@ async fn clean_up(
         log::trace!("done awaiting file output handler's receiver");
     }
 
+    update_stat!(tx_stats, StatCommand::Exit); // send exit command and await the end of the future
+    stats_handle.await.unwrap_or_default();
+
     // mark all scans complete so the terminal input handler will exit cleanly
     SCAN_COMPLETE.store(true, Ordering::Relaxed);
 
@@ -384,6 +450,8 @@ async fn clean_up(
     // the final trace messages above
     PROGRESS_PRINTER.finish();
 
+    drop(tx_stats);
+
     log::trace!("exit: clean_up");
 }
 
@@ -391,17 +459,17 @@ fn main() {
     // setup logging based on the number of -v's used
     logger::initialize(CONFIGURATION.verbosity);
 
-    if CONFIGURATION.save_state {
-        // start the ctrl+c handler
-        scan_manager::initialize();
-    }
-
     // this function uses rlimit, which is not supported on windows
     #[cfg(not(target_os = "windows"))]
     set_open_file_limit(DEFAULT_OPEN_FILE_LIMIT);
 
-    if let Ok(mut runtime) = tokio::runtime::Runtime::new() {
+    if let Ok(runtime) = tokio::runtime::Builder::new_multi_thread()
+        .enable_all()
+        .build()
+    {
         let future = wrapped_main();
         runtime.block_on(future);
     }
+
+    log::trace!("exit: main");
 }

src/progress.rs

@@ -1,22 +1,42 @@
 use crate::config::{CONFIGURATION, PROGRESS_BAR};
 use indicatif::{ProgressBar, ProgressStyle};
 
+/// Types of ProgressBars that can be added to `PROGRESS_BAR`
+pub enum BarType {
+    /// no template used / not visible
+    Hidden,
+
+    /// normal directory status bar (reqs/sec shown)
+    Default,
+
+    /// similar to `Default`, except `-` is used in place of line/word/char count
+    Message,
+
+    /// bar used to show overall scan metrics
+    Total,
+}
+
 /// Add an [indicatif::ProgressBar](https://docs.rs/indicatif/latest/indicatif/struct.ProgressBar.html)
 /// to the global [PROGRESS_BAR](../config/struct.PROGRESS_BAR.html)
-pub fn add_bar(prefix: &str, length: u64, hidden: bool, hide_per_sec: bool) -> ProgressBar {
-    let style = if hidden || CONFIGURATION.quiet {
-        ProgressStyle::default_bar().template("")
-    } else if hide_per_sec {
-        ProgressStyle::default_bar()
-            .template(&format!(
+pub fn add_bar(prefix: &str, length: u64, bar_type: BarType) -> ProgressBar {
+    let mut style = ProgressStyle::default_bar().progress_chars("#>-");
+
+    style = if CONFIGURATION.quiet {
+        style.template("")
+    } else {
+        match bar_type {
+            BarType::Hidden => style.template(""),
+            BarType::Default => style.template(
+                "[{bar:.cyan/blue}] - {elapsed:<4} {pos:>7}/{len:7} {per_sec:7} {prefix}",
+            ),
+            BarType::Message => style.template(&format!(
                 "[{{bar:.cyan/blue}}] - {{elapsed:<4}} {{pos:>7}}/{{len:7}} {:7} {{prefix}}",
                 "-"
-            ))
-            .progress_chars("#>-")
-    } else {
-        ProgressStyle::default_bar()
-            .template("[{bar:.cyan/blue}] - {elapsed:<4} {pos:>7}/{len:7} {per_sec:7} {prefix}")
-            .progress_chars("#>-")
+            )),
+            BarType::Total => {
+                style.template("[{bar:.yellow/blue}] - {elapsed:<4} {pos:>7}/{len:7} {eta:7} {msg}")
+            }
+        }
     };
 
     let progress_bar = PROGRESS_BAR.add(ProgressBar::new(length));
@@ -35,16 +55,19 @@ mod tests {
     #[test]
     /// hit all code branches for add_bar
     fn add_bar_with_all_configurations() {
-        let p1 = add_bar("prefix", 2, true, false); // hidden
-        let p2 = add_bar("prefix", 2, false, true); // no per second field
-        let p3 = add_bar("prefix", 2, false, false); // normal bar
+        let p1 = add_bar("prefix", 2, BarType::Hidden); // hidden
+        let p2 = add_bar("prefix", 2, BarType::Message); // no per second field
+        let p3 = add_bar("prefix", 2, BarType::Default); // normal bar
+        let p4 = add_bar("prefix", 2, BarType::Total); // totals bar
 
         p1.finish();
         p2.finish();
         p3.finish();
+        p4.finish();
 
         assert!(p1.is_finished());
         assert!(p2.is_finished());
         assert!(p3.is_finished());
+        assert!(p4.is_finished());
     }
 }

src/reporter.rs

@@ -1,15 +1,23 @@
 use crate::{
     config::{CONFIGURATION, PROGRESS_PRINTER},
     scanner::RESPONSES,
+    statistics::{
+        StatCommand::{self, UpdateUsizeField},
+        StatField::ResourcesDiscovered,
+    },
     utils::{ferox_print, make_request, open_file},
     FeroxChannel, FeroxResponse, FeroxSerialize,
 };
 use console::strip_ansi_codes;
-use std::io::Write;
-use std::sync::{Arc, Once, RwLock};
-use std::{fs, io};
-use tokio::sync::mpsc::{self, UnboundedReceiver, UnboundedSender};
-use tokio::task::JoinHandle;
+use std::{
+    fs, io,
+    io::Write,
+    sync::{Arc, Once, RwLock},
+};
+use tokio::{
+    sync::mpsc::{self, UnboundedReceiver, UnboundedSender},
+    task::JoinHandle,
+};
 
 /// Singleton buffered file behind an Arc/RwLock; used for file writes from two locations:
 ///     - [logger::initialize](../logger/fn.initialize.html) (specifically a closure on the global logger instance)
@@ -42,27 +50,35 @@ pub fn get_cached_file_handle(filename: &str) -> Option<Arc<RwLock<io::BufWriter
 pub fn initialize(
     output_file: &str,
     save_output: bool,
+    tx_stats: UnboundedSender<StatCommand>,
 ) -> (
     UnboundedSender<FeroxResponse>,
     UnboundedSender<FeroxResponse>,
     JoinHandle<()>,
     Option<JoinHandle<()>>,
 ) {
-    log::trace!("enter: initialize({}, {})", output_file, save_output);
+    log::trace!(
+        "enter: initialize({}, {}, {:?})",
+        output_file,
+        save_output,
+        tx_stats
+    );
 
     let (tx_rpt, rx_rpt): FeroxChannel<FeroxResponse> = mpsc::unbounded_channel();
     let (tx_file, rx_file): FeroxChannel<FeroxResponse> = mpsc::unbounded_channel();
 
     let file_clone = tx_file.clone();
+    let stats_clone = tx_stats.clone();
 
-    let term_reporter =
-        tokio::spawn(async move { spawn_terminal_reporter(rx_rpt, file_clone, save_output).await });
+    let term_reporter = tokio::spawn(async move {
+        spawn_terminal_reporter(rx_rpt, file_clone, stats_clone, save_output).await
+    });
 
     let file_reporter = if save_output {
         // -o used, need to spawn the thread for writing to disk
         let file_clone = output_file.to_string();
         Some(tokio::spawn(async move {
-            spawn_file_reporter(rx_file, &file_clone).await
+            spawn_file_reporter(rx_file, tx_stats, &file_clone).await
         }))
     } else {
         None
@@ -85,12 +101,14 @@ pub fn initialize(
 async fn spawn_terminal_reporter(
     mut resp_chan: UnboundedReceiver<FeroxResponse>,
     file_chan: UnboundedSender<FeroxResponse>,
+    tx_stats: UnboundedSender<StatCommand>,
     save_output: bool,
 ) {
     log::trace!(
-        "enter: spawn_terminal_reporter({:?}, {:?}, {})",
+        "enter: spawn_terminal_reporter({:?}, {:?}, {:?}, {})",
         resp_chan,
         file_chan,
+        tx_stats,
         save_output
     );
 
@@ -105,6 +123,8 @@ async fn spawn_terminal_reporter(
             // print to stdout
             ferox_print(&resp.as_str(), &PROGRESS_PRINTER);
 
+            update_stat!(tx_stats, UpdateUsizeField(ResourcesDiscovered, 1));
+
             if save_output {
                 // -o used, need to send the report to be written out to disk
                 match file_chan.send(resp.clone()) {
@@ -122,7 +142,13 @@ async fn spawn_terminal_reporter(
         if CONFIGURATION.replay_client.is_some() && should_process_response {
             // replay proxy specified/client created and this response's status code is one that
             // should be replayed
-            match make_request(CONFIGURATION.replay_client.as_ref().unwrap(), &resp.url()).await {
+            match make_request(
+                CONFIGURATION.replay_client.as_ref().unwrap(),
+                &resp.url(),
+                tx_stats.clone(),
+            )
+            .await
+            {
                 Ok(_) => {}
                 Err(e) => {
                     log::error!("{}", e);
@@ -151,6 +177,7 @@ async fn spawn_terminal_reporter(
 /// the given reporting criteria
 async fn spawn_file_reporter(
     mut report_channel: UnboundedReceiver<FeroxResponse>,
+    tx_stats: UnboundedSender<StatCommand>,
     output_file: &str,
 ) {
     let buffered_file = match get_cached_file_handle(&CONFIGURATION.output) {
@@ -173,6 +200,8 @@ async fn spawn_file_reporter(
         safe_file_write(&response, buffered_file.clone(), CONFIGURATION.json);
     }
 
+    update_stat!(tx_stats, StatCommand::Save);
+
     log::trace!("exit: spawn_file_reporter");
 }
 

src/scan_manager.rs

@@ -1,43 +1,36 @@
-use crate::config::Configuration;
-use crate::reporter::safe_file_write;
-use crate::utils::open_file;
 use crate::{
-    config::{CONFIGURATION, PROGRESS_PRINTER},
+    config::{Configuration, CONFIGURATION, PROGRESS_BAR, PROGRESS_PRINTER},
     parser::TIMESPEC_REGEX,
-    progress,
-    scanner::{NUMBER_OF_REQUESTS, RESPONSES, SCANNED_URLS},
+    progress::{add_bar, BarType},
+    reporter::safe_file_write,
+    scanner::{RESPONSES, SCANNED_URLS},
+    statistics::Stats,
+    utils::open_file,
     FeroxResponse, FeroxSerialize, SLEEP_DURATION,
 };
-use console::style;
-use indicatif::{ProgressBar, ProgressStyle};
-use lazy_static::lazy_static;
+use console::{measure_text_width, pad_str, style, Alignment, Term};
+use indicatif::{ProgressBar, ProgressDrawTarget};
 use serde::{
     ser::{SerializeSeq, SerializeStruct},
     Deserialize, Deserializer, Serialize, Serializer,
 };
 use serde_json::Value;
-use std::collections::HashMap;
 use std::{
     cmp::PartialEq,
+    collections::HashMap,
     fmt,
     fs::File,
     io::BufReader,
+    ops::Index,
+    sync::atomic::{AtomicBool, AtomicUsize, Ordering},
     sync::{Arc, Mutex, RwLock},
+    thread::sleep,
     time::{SystemTime, UNIX_EPOCH},
 };
-use std::{
-    io::{stderr, Write},
-    sync::atomic::{AtomicBool, AtomicUsize, Ordering},
-};
+use tokio::time::Duration;
 use tokio::{task::JoinHandle, time};
 use uuid::Uuid;
 
-lazy_static! {
-    /// A clock spinner protected with a RwLock to allow for a single thread to use at a time
-    // todo remove this when issue #107 is resolved
-    static ref SINGLE_SPINNER: RwLock<ProgressBar> = RwLock::new(get_single_spinner());
-}
-
 /// Single atomic number that gets incremented once, used to track first thread to interact with
 /// when pausing a scan
 static INTERACTIVE_BARRIER: AtomicUsize = AtomicUsize::new(0);
@@ -75,11 +68,14 @@ pub struct FeroxScan {
     /// The type of scan
     pub scan_type: ScanType,
 
-    /// Whether or not this scan has completed
-    pub complete: bool,
+    /// Number of requests to populate the progress bar with
+    pub num_requests: u64,
+
+    /// Status of this scan
+    pub status: ScanStatus,
 
     /// The spawned tokio task performing this scan
-    pub task: Option<JoinHandle<()>>,
+    pub task: Option<Arc<JoinHandle<()>>>,
 
     /// The progress bar associated with this scan
     pub progress_bar: Option<ProgressBar>,
@@ -94,7 +90,8 @@ impl Default for FeroxScan {
         FeroxScan {
             id: new_id,
             task: None,
-            complete: false,
+            status: ScanStatus::default(),
+            num_requests: 0,
             url: String::new(),
             progress_bar: None,
             scan_type: ScanType::File,
@@ -105,18 +102,18 @@ impl Default for FeroxScan {
 /// Implementation of FeroxScan
 impl FeroxScan {
     /// Stop a currently running scan
-    pub fn abort(&self) {
-        self.stop_progress_bar();
-
-        if let Some(_task) = &self.task {
-            // task.abort();  todo uncomment once upgraded to tokio 0.3 (issue #107)
+    pub fn abort(&mut self) {
+        if let Some(task) = &self.task {
+            self.status = ScanStatus::Cancelled;
+            self.stop_progress_bar();
+            task.abort();
         }
     }
 
     /// Simple helper to call .finish on the scan's progress bar
     fn stop_progress_bar(&self) {
         if let Some(pb) = &self.progress_bar {
-            pb.finish();
+            pb.finish_at_current_pos();
         }
     }
 
@@ -125,8 +122,7 @@ impl FeroxScan {
         if let Some(pb) = &self.progress_bar {
             pb.clone()
         } else {
-            let num_requests = NUMBER_OF_REQUESTS.load(Ordering::Relaxed);
-            let pb = progress::add_bar(&self.url, num_requests, false, false);
+            let pb = add_bar(&self.url, self.num_requests, BarType::Default);
 
             pb.reset_elapsed();
 
@@ -137,19 +133,24 @@ impl FeroxScan {
     }
 
     /// Given a URL and ProgressBar, create a new FeroxScan, wrap it in an Arc and return it
-    pub fn new(url: &str, scan_type: ScanType, pb: Option<ProgressBar>) -> Arc<Mutex<Self>> {
-        let mut me = Self::default();
-
-        me.url = url.to_string();
-        me.scan_type = scan_type;
-        me.progress_bar = pb;
-
-        Arc::new(Mutex::new(me))
+    pub fn new(
+        url: &str,
+        scan_type: ScanType,
+        num_requests: u64,
+        pb: Option<ProgressBar>,
+    ) -> Arc<Mutex<Self>> {
+        Arc::new(Mutex::new(Self {
+            url: url.to_string(),
+            scan_type,
+            num_requests,
+            progress_bar: pb,
+            ..Default::default()
+        }))
     }
 
     /// Mark the scan as complete and stop the scan's progress bar
     pub fn finish(&mut self) {
-        self.complete = true;
+        self.status = ScanStatus::Complete;
         self.stop_progress_bar();
     }
 }
@@ -157,13 +158,14 @@ impl FeroxScan {
 /// Display implementation
 impl fmt::Display for FeroxScan {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        let complete = if self.complete {
-            style("complete").green()
-        } else {
-            style("incomplete").red()
+        let status = match self.status {
+            ScanStatus::NotStarted => style("not started").bright().blue(),
+            ScanStatus::Complete => style("complete").green(),
+            ScanStatus::Cancelled => style("cancelled").red(),
+            ScanStatus::Running => style("running").bright().yellow(),
         };
 
-        write!(f, "{:10} {}", complete, self.url)
+        write!(f, "{:12} {}", status, self.url)
     }
 }
 
@@ -186,7 +188,8 @@ impl Serialize for FeroxScan {
         state.serialize_field("id", &self.id)?;
         state.serialize_field("url", &self.url)?;
         state.serialize_field("scan_type", &self.scan_type)?;
-        state.serialize_field("complete", &self.complete)?;
+        state.serialize_field("status", &self.status)?;
+        state.serialize_field("num_requests", &self.num_requests)?;
 
         state.end()
     }
@@ -219,9 +222,15 @@ impl<'de> Deserialize<'de> for FeroxScan {
                         }
                     }
                 }
-                "complete" => {
-                    if let Some(complete) = value.as_bool() {
-                        scan.complete = complete;
+                "status" => {
+                    if let Some(status) = value.as_str() {
+                        scan.status = match status {
+                            "NotStarted" => ScanStatus::NotStarted,
+                            "Running" => ScanStatus::Running,
+                            "Complete" => ScanStatus::Complete,
+                            "Cancelled" => ScanStatus::Cancelled,
+                            _ => ScanStatus::default(),
+                        }
                     }
                 }
                 "url" => {
@@ -229,6 +238,11 @@ impl<'de> Deserialize<'de> for FeroxScan {
                         scan.url = url.to_string();
                     }
                 }
+                "num_requests" => {
+                    if let Some(num_requests) = value.as_u64() {
+                        scan.num_requests = num_requests;
+                    }
+                }
                 _ => {}
             }
         }
@@ -237,14 +251,175 @@ impl<'de> Deserialize<'de> for FeroxScan {
     }
 }
 
+#[derive(Debug, Serialize, Deserialize)]
+/// Simple enum to represent a scan's current status ([in]complete, cancelled)
+pub enum ScanStatus {
+    /// Scan hasn't started yet
+    NotStarted,
+
+    /// Scan finished normally
+    Complete,
+
+    /// Scan was cancelled by the user
+    Cancelled,
+
+    /// Scan has started, but hasn't finished, nor been cancelled
+    Running,
+}
+
+/// Default implementation for ScanStatus
+impl Default for ScanStatus {
+    /// Default variant for ScanStatus is NotStarted
+    fn default() -> Self {
+        Self::NotStarted
+    }
+}
+
+/// Interactive scan cancellation menu
+#[derive(Debug)]
+struct Menu {
+    /// character to use as visual separator of lines
+    separator: String,
+
+    /// name of menu
+    name: String,
+
+    /// header: name surrounded by separators
+    header: String,
+
+    /// instructions
+    instructions: String,
+
+    /// footer: instructions surrounded by separators
+    footer: String,
+
+    /// target for output
+    term: Term,
+}
+
+/// Implementation of Menu
+impl Menu {
+    /// Creates new Menu
+    fn new() -> Self {
+        let separator = "─".to_string();
+
+        let instructions = format!(
+            "Enter a {} list of indexes to {} (ex: 2,3)",
+            style("comma-separated").yellow(),
+            style("cancel").red(),
+        );
+
+        let name = format!(
+            "{} {} {}",
+            "💀",
+            style("Scan Cancel Menu").bright().yellow(),
+            "💀"
+        );
+
+        let longest = measure_text_width(&instructions).max(measure_text_width(&name));
+
+        let border = separator.repeat(longest);
+
+        let padded_name = pad_str(&name, longest, Alignment::Center, None);
+
+        let header = format!("{}\n{}\n{}", border, padded_name, border);
+        let footer = format!("{}\n{}\n{}", border, instructions, border);
+
+        Self {
+            separator,
+            name,
+            header,
+            instructions,
+            footer,
+            term: Term::stderr(),
+        }
+    }
+
+    /// print menu header
+    fn print_header(&self) {
+        self.println(&self.header);
+    }
+
+    /// print menu footer
+    fn print_footer(&self) {
+        self.println(&self.footer);
+    }
+
+    /// set PROGRESS_BAR bar target to hidden
+    fn hide_progress_bars(&self) {
+        PROGRESS_BAR.set_draw_target(ProgressDrawTarget::hidden());
+    }
+
+    /// set PROGRESS_BAR bar target to hidden
+    fn show_progress_bars(&self) {
+        PROGRESS_BAR.set_draw_target(ProgressDrawTarget::stdout());
+    }
+
+    /// Wrapper around console's Term::clear_screen and flush
+    fn clear_screen(&self) {
+        self.term.clear_screen().unwrap_or_default();
+        self.term.flush().unwrap_or_default();
+    }
+
+    /// Wrapper around console's Term::write_line
+    fn println(&self, msg: &str) {
+        self.term.write_line(msg).unwrap_or_default();
+    }
+
+    /// split a string into vec of usizes
+    fn split_to_nums(&self, line: &str) -> Vec<usize> {
+        line.split(',')
+            .map(|s| {
+                s.trim().to_string().parse::<usize>().unwrap_or_else(|e| {
+                    self.println(&format!("Found non-numeric input: {}", e));
+                    0
+                })
+            })
+            .filter(|m| *m != 0)
+            .collect()
+    }
+
+    /// get comma-separated list of scan indexes from the user
+    fn get_scans_from_user(&self) -> Option<Vec<usize>> {
+        if let Ok(line) = self.term.read_line() {
+            Some(self.split_to_nums(&line))
+        } else {
+            None
+        }
+    }
+
+    /// Given a url, confirm with user that we should cancel
+    fn confirm_cancellation(&self, url: &str) -> char {
+        self.println(&format!(
+            "You sure you wanna cancel this scan: {}? [Y/n]",
+            url
+        ));
+
+        self.term.read_char().unwrap_or('n')
+    }
+}
+
+/// Default implementation for Menu
+impl Default for Menu {
+    /// return Menu::new as default
+    fn default() -> Menu {
+        Menu::new()
+    }
+}
+
 /// Container around a locked hashset of `FeroxScan`s, adds wrappers for insertion and searching
 #[derive(Debug, Default)]
 pub struct FeroxScans {
     /// Internal structure: locked hashset of `FeroxScan`s
     pub scans: Mutex<Vec<Arc<Mutex<FeroxScan>>>>,
+
+    /// menu used for providing a way for users to cancel a scan
+    menu: Menu,
 }
 
 /// Serialize implementation for FeroxScans
+///
+/// purposefully skips menu attribute
 impl Serialize for FeroxScans {
     /// Function that handles serialization of FeroxScans
     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
@@ -350,20 +525,82 @@ impl FeroxScans {
         if let Ok(scans) = self.scans.lock() {
             for (i, scan) in scans.iter().enumerate() {
                 if let Ok(unlocked_scan) = scan.lock() {
-                    match unlocked_scan.scan_type {
-                        ScanType::Directory => {
-                            PROGRESS_PRINTER.println(format!("{:3}: {}", i, unlocked_scan));
-                        }
-                        ScanType::File => {
-                            // we're only interested in displaying directory scans, as those are
-                            // the only ones that make sense to be stopped
-                        }
+                    if unlocked_scan.task.is_none() {
+                        // no JoinHandle associated with this FeroxScan, meaning it was an original
+                        // target passed in via either -u or --stdin
+                        continue;
+                    }
+
+                    if matches!(unlocked_scan.scan_type, ScanType::Directory) {
+                        // we're only interested in displaying directory scans, as those are
+                        // the only ones that make sense to be stopped
+                        let scan_msg = format!("{:3}: {}", i, unlocked_scan);
+                        self.menu.println(&scan_msg);
                     }
                 }
             }
         }
     }
 
+    /// Given a list of indexes, cancel their associated FeroxScans
+    fn cancel_scans(&self, indexes: Vec<usize>) {
+        let menu_pause_duration = Duration::from_millis(SLEEP_DURATION);
+
+        for num in indexes {
+            if let Ok(u_scans) = self.scans.lock() {
+                // check if number provided is out of range
+                if num >= u_scans.len() {
+                    // usize can't be negative, just need to handle exceeding bounds
+                    self.menu
+                        .println(&format!("The number {} is not a valid choice.", num));
+                    sleep(menu_pause_duration);
+                    continue;
+                }
+
+                let selected = u_scans.index(num).clone();
+
+                if let Ok(mut ferox_scan) = selected.lock() {
+                    let input = self.menu.confirm_cancellation(&ferox_scan.url);
+
+                    if input == 'y' || input == '\n' {
+                        self.menu
+                            .println(&format!("Stopping {}...", ferox_scan.url));
+                        ferox_scan.abort();
+                    } else {
+                        self.menu.println("Ok, doing nothing...");
+                    }
+                }
+
+                sleep(menu_pause_duration);
+            }
+        }
+    }
+
+    /// CLI menu that allows for interactive cancellation of recursed-into directories
+    async fn interactive_menu(&self) {
+        self.menu.hide_progress_bars();
+        self.menu.clear_screen();
+        self.menu.print_header();
+        self.display_scans();
+        self.menu.print_footer();
+
+        if let Some(input) = self.menu.get_scans_from_user() {
+            self.cancel_scans(input);
+        }
+
+        self.menu.clear_screen();
+        self.menu.show_progress_bars();
+    }
+
+    /// prints all known responses that the scanner has already seen
+    pub fn print_known_responses(&self) {
+        if let Ok(responses) = RESPONSES.responses.read() {
+            for response in responses.iter() {
+                PROGRESS_PRINTER.println(response.as_str());
+            }
+        }
+    }
+
     /// Forced the calling thread into a busy loop
     ///
     /// Every `SLEEP_DURATION` milliseconds, the function examines the result stored in `PAUSE_SCAN`
@@ -377,38 +614,16 @@ impl FeroxScans {
         // concurrent scans rose when SLEEP_DURATION was set to 500, using that as the default for now
         let mut interval = time::interval(time::Duration::from_millis(SLEEP_DURATION));
 
-        // ignore any error returned
-        let _ = stderr().flush();
-
         if INTERACTIVE_BARRIER.load(Ordering::Relaxed) == 0 {
             INTERACTIVE_BARRIER.fetch_add(1, Ordering::Relaxed);
 
             if get_user_input {
-                self.display_scans();
-
-                let mut user_input = String::new();
-                std::io::stdin().read_line(&mut user_input).unwrap();
-                // todo (issue #107) actual logic for parsing user input in a way that allows for
-                // calling .abort on the scan retrieved based on the input
+                self.interactive_menu().await;
+                PAUSE_SCAN.store(false, Ordering::Relaxed);
+                self.print_known_responses();
             }
         }
 
-        if SINGLE_SPINNER.read().unwrap().is_finished() {
-            // todo remove this when issue #107 is resolved
-
-            // in order to not leave draw artifacts laying around in the terminal, we call
-            // finish_and_clear on the progress bar when resuming scans. For this reason, we need to
-            // check if the spinner is finished, and repopulate the RwLock with a new spinner if
-            // necessary
-            if let Ok(mut guard) = SINGLE_SPINNER.write() {
-                *guard = get_single_spinner();
-            }
-        }
-
-        if let Ok(spinner) = SINGLE_SPINNER.write() {
-            spinner.enable_steady_tick(120);
-        }
-
         loop {
             // first tick happens immediately, all others wait the specified duration
             interval.tick().await;
@@ -420,13 +635,6 @@ impl FeroxScans {
                     INTERACTIVE_BARRIER.fetch_sub(1, Ordering::Relaxed);
                 }
 
-                if let Ok(spinner) = SINGLE_SPINNER.write() {
-                    // todo remove this when issue #107 is resolved
-                    spinner.finish_and_clear();
-                }
-
-                let _ = stderr().flush();
-
                 log::trace!("exit: pause_scan");
                 return;
             }
@@ -438,15 +646,17 @@ impl FeroxScans {
     /// If `FeroxScans` did not already contain the scan, return true; otherwise return false
     ///
     /// Also return a reference to the new `FeroxScan`
-    fn add_scan(&self, url: &str, scan_type: ScanType) -> (bool, Arc<Mutex<FeroxScan>>) {
+    fn add_scan(
+        &self,
+        url: &str,
+        scan_type: ScanType,
+        stats: Arc<Stats>,
+    ) -> (bool, Arc<Mutex<FeroxScan>>) {
+        let num_requests = stats.expected_per_scan.load(Ordering::Relaxed) as u64;
+
         let bar = match scan_type {
             ScanType::Directory => {
-                let progress_bar = progress::add_bar(
-                    &url,
-                    NUMBER_OF_REQUESTS.load(Ordering::Relaxed),
-                    false,
-                    false,
-                );
+                let progress_bar = add_bar(&url, num_requests, BarType::Default);
 
                 progress_bar.reset_elapsed();
 
@@ -455,7 +665,7 @@ impl FeroxScans {
             ScanType::File => None,
         };
 
-        let ferox_scan = FeroxScan::new(&url, scan_type, bar);
+        let ferox_scan = FeroxScan::new(&url, scan_type, num_requests, bar);
 
         // If the set did not contain the scan, true is returned.
         // If the set did contain the scan, false is returned.
@@ -469,8 +679,12 @@ impl FeroxScans {
     /// If `FeroxScans` did not already contain the scan, return true; otherwise return false
     ///
     /// Also return a reference to the new `FeroxScan`
-    pub fn add_directory_scan(&self, url: &str) -> (bool, Arc<Mutex<FeroxScan>>) {
-        self.add_scan(&url, ScanType::Directory)
+    pub fn add_directory_scan(
+        &self,
+        url: &str,
+        stats: Arc<Stats>,
+    ) -> (bool, Arc<Mutex<FeroxScan>>) {
+        self.add_scan(&url, ScanType::Directory, stats)
     }
 
     /// Given a url, create a new `FeroxScan` and add it to `FeroxScans` as a File Scan
@@ -478,31 +692,11 @@ impl FeroxScans {
     /// If `FeroxScans` did not already contain the scan, return true; otherwise return false
     ///
     /// Also return a reference to the new `FeroxScan`
-    pub fn add_file_scan(&self, url: &str) -> (bool, Arc<Mutex<FeroxScan>>) {
-        self.add_scan(&url, ScanType::File)
+    pub fn add_file_scan(&self, url: &str, stats: Arc<Stats>) -> (bool, Arc<Mutex<FeroxScan>>) {
+        self.add_scan(&url, ScanType::File, stats)
     }
 }
 
-/// Return a clock spinner, used when scans are paused
-// todo remove this when issue #107 is resolved
-fn get_single_spinner() -> ProgressBar {
-    log::trace!("enter: get_single_spinner");
-
-    let spinner = ProgressBar::new_spinner().with_style(
-        ProgressStyle::default_spinner()
-            .tick_strings(&[
-                "🕛", "🕐", "🕑", "🕒", "🕓", "🕔", "🕕", "🕖", "🕗", "🕘", "🕙", "🕚",
-            ])
-            .template(&format!(
-                "\t-= All Scans {{spinner}} {} =-",
-                style("Paused").red()
-            )),
-    );
-
-    log::trace!("exit: get_single_spinner -> {:?}", spinner);
-    spinner
-}
-
 /// Container around a locked vector of `FeroxResponse`s, adds wrappers for insertion and search
 #[derive(Debug, Default)]
 pub struct FeroxResponses {
@@ -575,6 +769,9 @@ pub struct FeroxState {
 
     /// Known responses
     responses: &'static FeroxResponses,
+
+    /// Gathered statistics
+    statistics: Arc<Stats>,
 }
 
 /// FeroxSerialize implementation for FeroxState
@@ -594,7 +791,7 @@ impl FeroxSerialize for FeroxState {
 /// that representation to seconds and then wait for those seconds to elapse.  Once that period
 /// of time has elapsed, kill all currently running scans and dump a state file to disk that can
 /// be used to resume any unfinished scan.
-pub async fn start_max_time_thread(time_spec: &str) {
+pub async fn start_max_time_thread(time_spec: &str, stats: Arc<Stats>) {
     log::trace!("enter: start_max_time_thread({})", time_spec);
 
     // as this function has already made it through the parser, which calls is_match on
@@ -619,14 +816,14 @@ pub async fn start_max_time_thread(time_spec: &str) {
             length_in_secs
         );
 
-        time::delay_for(time::Duration::new(length_in_secs, 0)).await;
+        time::sleep(time::Duration::new(length_in_secs, 0)).await;
 
         log::trace!("exit: start_max_time_thread");
 
         #[cfg(test)]
-        panic!();
+        panic!(stats);
         #[cfg(not(test))]
-        sigint_handler();
+        sigint_handler(stats);
     }
 
     log::error!(
@@ -636,8 +833,8 @@ pub async fn start_max_time_thread(time_spec: &str) {
 }
 
 /// Writes the current state of the program to disk (if save_state is true) and then exits
-fn sigint_handler() {
-    log::trace!("enter: sigint_handler");
+fn sigint_handler(stats: Arc<Stats>) {
+    log::trace!("enter: sigint_handler({:?})", stats);
 
     let ts = SystemTime::now()
         .duration_since(UNIX_EPOCH)
@@ -669,6 +866,7 @@ fn sigint_handler() {
         config: &CONFIGURATION,
         scans: &SCANNED_URLS,
         responses: &RESPONSES,
+        statistics: stats,
     };
 
     let state_file = open_file(&filename);
@@ -682,10 +880,12 @@ fn sigint_handler() {
 }
 
 /// Initialize the ctrl+c handler that saves scan state to disk
-pub fn initialize() {
-    log::trace!("enter: initialize");
+pub fn initialize(stats: Arc<Stats>) {
+    log::trace!("enter: initialize({:?})", stats);
 
-    let result = ctrlc::set_handler(sigint_handler);
+    let result = ctrlc::set_handler(move || {
+        sigint_handler(stats.clone());
+    });
 
     if result.is_err() {
         log::error!("Could not set Ctrl+c handler");
@@ -761,15 +961,7 @@ mod tests {
         }
     }
 
-    #[test]
-    /// test that get_single_spinner returns the correct spinner
-    // todo remove this when issue #107 is resolved
-    fn scanner_get_single_spinner_returns_spinner() {
-        let spinner = get_single_spinner();
-        assert!(!spinner.is_finished());
-    }
-
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// tests that pause_scan pauses execution and releases execution when PAUSE_SCAN is toggled
     /// the spinner used during the test has had .finish_and_clear called on it, meaning that
     /// a new one will be created, taking the if branch within the function
@@ -782,7 +974,7 @@ mod tests {
         let expected = time::Duration::from_secs(2);
 
         tokio::spawn(async move {
-            time::delay_for(expected).await;
+            time::sleep(expected).await;
             PAUSE_SCAN.store(false, Ordering::Relaxed);
         });
 
@@ -795,8 +987,9 @@ mod tests {
     /// add an unknown url to the hashset, expect true
     fn add_url_to_list_of_scanned_urls_with_unknown_url() {
         let urls = FeroxScans::default();
+        let stats = Arc::new(Stats::new());
         let url = "http://unknown_url";
-        let (result, _scan) = urls.add_scan(url, ScanType::Directory);
+        let (result, _scan) = urls.add_scan(url, ScanType::Directory, stats);
         assert_eq!(result, true);
     }
 
@@ -806,21 +999,24 @@ mod tests {
         let urls = FeroxScans::default();
         let pb = ProgressBar::new(1);
         let url = "http://unknown_url/";
-        let scan = FeroxScan::new(url, ScanType::Directory, Some(pb));
+        let stats = Arc::new(Stats::new());
+
+        let scan = FeroxScan::new(url, ScanType::Directory, pb.length(), Some(pb));
 
         assert_eq!(urls.insert(scan), true);
 
-        let (result, _scan) = urls.add_scan(url, ScanType::Directory);
+        let (result, _scan) = urls.add_scan(url, ScanType::Directory, stats);
 
         assert_eq!(result, false);
     }
 
     #[test]
-    /// abort should call stop_progress_bar, marking it as finished
-    fn abort_stops_progress_bar() {
+    /// stop_progress_bar should stop the progress bar
+    fn stop_progress_bar_stops_bar() {
         let pb = ProgressBar::new(1);
         let url = "http://unknown_url/";
-        let scan = FeroxScan::new(url, ScanType::Directory, Some(pb));
+
+        let scan = FeroxScan::new(url, ScanType::Directory, pb.length(), Some(pb));
 
         assert_eq!(
             scan.lock()
@@ -832,7 +1028,7 @@ mod tests {
             false
         );
 
-        scan.lock().unwrap().abort();
+        scan.lock().unwrap().stop_progress_bar();
 
         assert_eq!(
             scan.lock()
@@ -850,29 +1046,37 @@ mod tests {
     fn add_url_to_list_of_scanned_urls_with_known_url_without_slash() {
         let urls = FeroxScans::default();
         let url = "http://unknown_url";
-        let scan = FeroxScan::new(url, ScanType::File, None);
+        let stats = Arc::new(Stats::new());
+
+        let scan = FeroxScan::new(url, ScanType::File, 0, None);
 
         assert_eq!(urls.insert(scan), true);
 
-        let (result, _scan) = urls.add_scan(url, ScanType::File);
+        let (result, _scan) = urls.add_scan(url, ScanType::File, stats);
 
         assert_eq!(result, false);
     }
 
-    #[test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// just increasing coverage, no real expectations
-    fn call_display_scans() {
+    async fn call_display_scans() {
         let urls = FeroxScans::default();
         let pb = ProgressBar::new(1);
         let pb_two = ProgressBar::new(2);
         let url = "http://unknown_url/";
         let url_two = "http://unknown_url/fa";
-        let scan = FeroxScan::new(url, ScanType::Directory, Some(pb));
-        let scan_two = FeroxScan::new(url_two, ScanType::Directory, Some(pb_two));
+        let scan = FeroxScan::new(url, ScanType::Directory, pb.length(), Some(pb));
+        let scan_two = FeroxScan::new(url_two, ScanType::Directory, pb_two.length(), Some(pb_two));
 
-        scan_two.lock().unwrap().finish(); // one complete, one incomplete
+        if let Ok(mut s2) = scan_two.lock() {
+            s2.finish(); // one complete, one incomplete
+            s2.task = Some(Arc::new(tokio::spawn(async move {
+                sleep(Duration::from_millis(SLEEP_DURATION));
+            })));
+        }
 
         assert_eq!(urls.insert(scan), true);
+        assert_eq!(urls.insert(scan_two), true);
 
         urls.display_scans();
     }
@@ -881,8 +1085,8 @@ mod tests {
     /// ensure that PartialEq compares FeroxScan.id fields
     fn partial_eq_compares_the_id_field() {
         let url = "http://unknown_url/";
-        let scan = FeroxScan::new(url, ScanType::Directory, None);
-        let scan_two = FeroxScan::new(url, ScanType::Directory, None);
+        let scan = FeroxScan::new(url, ScanType::Directory, 0, None);
+        let scan_two = FeroxScan::new(url, ScanType::Directory, 0, None);
 
         assert!(!scan.lock().unwrap().eq(&scan_two.lock().unwrap()));
 
@@ -908,11 +1112,13 @@ mod tests {
     /// given a JSON entry representing a FeroxScan, test that it deserializes into the proper type
     /// with the right attributes
     fn ferox_scan_deserialize() {
-        let fs_json = r#"{"id":"057016a14769414aac9a7a62707598cb","url":"https://spiritanimal.com","scan_type":"Directory","complete":true}"#;
-        let fs_json_two = r#"{"id":"057016a14769414aac9a7a62707598cb","url":"https://spiritanimal.com","scan_type":"Not Correct","complete":true}"#;
+        let fs_json = r#"{"id":"057016a14769414aac9a7a62707598cb","url":"https://spiritanimal.com","scan_type":"Directory","status":"Complete"}"#;
+        let fs_json_two = r#"{"id":"057016a14769414aac9a7a62707598cb","url":"https://spiritanimal.com","scan_type":"Not Correct","status":"Cancelled"}"#;
+        let fs_json_three = r#"{"id":"057016a14769414aac9a7a62707598cb","url":"https://spiritanimal.com","scan_type":"Not Correct","status":"","num_requests":42}"#;
 
         let fs: FeroxScan = serde_json::from_str(fs_json).unwrap();
         let fs_two: FeroxScan = serde_json::from_str(fs_json_two).unwrap();
+        let fs_three: FeroxScan = serde_json::from_str(fs_json_three).unwrap();
         assert_eq!(fs.url, "https://spiritanimal.com");
 
         match fs.scan_type {
@@ -934,16 +1140,19 @@ mod tests {
                 panic!();
             }
         }
-        assert_eq!(fs.complete, true);
+        assert!(matches!(fs.status, ScanStatus::Complete));
+        assert!(matches!(fs_two.status, ScanStatus::Cancelled));
+        assert!(matches!(fs_three.status, ScanStatus::NotStarted));
+        assert_eq!(fs_three.num_requests, 42);
         assert_eq!(fs.id, "057016a14769414aac9a7a62707598cb");
     }
 
     #[test]
     /// given a FeroxScan, test that it serializes into the proper JSON entry
     fn ferox_scan_serialize() {
-        let fs = FeroxScan::new("https://spiritanimal.com", ScanType::Directory, None);
+        let fs = FeroxScan::new("https://spiritanimal.com", ScanType::Directory, 0, None);
         let fs_json = format!(
-            r#"{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","complete":false}}"#,
+            r#"{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","status":"NotStarted","num_requests":0}}"#,
             fs.lock().unwrap().id
         );
         assert_eq!(
@@ -955,10 +1164,10 @@ mod tests {
     #[test]
     /// given a FeroxScans, test that it serializes into the proper JSON entry
     fn ferox_scans_serialize() {
-        let ferox_scan = FeroxScan::new("https://spiritanimal.com", ScanType::Directory, None);
+        let ferox_scan = FeroxScan::new("https://spiritanimal.com", ScanType::Directory, 0, None);
         let ferox_scans = FeroxScans::default();
         let ferox_scans_json = format!(
-            r#"[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","complete":false}}]"#,
+            r#"[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","status":"NotStarted","num_requests":0}}]"#,
             ferox_scan.lock().unwrap().id
         );
         ferox_scans.scans.lock().unwrap().push(ferox_scan);
@@ -1009,10 +1218,12 @@ mod tests {
     #[test]
     /// test FeroxSerialize implementation of FeroxState
     fn feroxstates_feroxserialize_implementation() {
-        let ferox_scan = FeroxScan::new("https://spiritanimal.com", ScanType::Directory, None);
+        let ferox_scan = FeroxScan::new("https://spiritanimal.com", ScanType::Directory, 0, None);
         let saved_id = ferox_scan.lock().unwrap().id.clone();
         SCANNED_URLS.insert(ferox_scan);
 
+        let stats = Arc::new(Stats::new());
+
         let json_response = r#"{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{"server":"nginx/1.16.1"}}"#;
         let response: FeroxResponse = serde_json::from_str(json_response).unwrap();
         RESPONSES.insert(response);
@@ -1021,6 +1232,7 @@ mod tests {
             scans: &SCANNED_URLS,
             responses: &RESPONSES,
             config: &CONFIGURATION,
+            statistics: stats,
         };
 
         let expected_strs = predicates::str::contains("scans: FeroxScans").and(
@@ -1035,36 +1247,120 @@ mod tests {
 
         let json_state = ferox_state.as_json();
         let expected = format!(
-            r#"{{"scans":[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","complete":false}}],"config":{{"type":"configuration","wordlist":"/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt","config":"","proxy":"","replay_proxy":"","target_url":"","status_codes":[200,204,301,302,307,308,401,403,405],"replay_codes":[200,204,301,302,307,308,401,403,405],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"quiet":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/{}","redirects":false,"insecure":false,"extensions":[],"headers":{{}},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":false,"depth":4,"scan_limit":0,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false,"resumed":false,"save_state":false,"time_limit":"","filter_similar":[]}},"responses":[{{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{{"server":"nginx/1.16.1"}}}}]}}"#,
+            r#"{{"scans":[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","status":"NotStarted","num_requests":0}}],"config":{{"type":"configuration","wordlist":"/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt","config":"","proxy":"","replay_proxy":"","target_url":"","status_codes":[200,204,301,302,307,308,401,403,405],"replay_codes":[200,204,301,302,307,308,401,403,405],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"quiet":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/{}","redirects":false,"insecure":false,"extensions":[],"headers":{{}},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":false,"depth":4,"scan_limit":0,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false,"resumed":false,"resume_from":"","save_state":false,"time_limit":"","filter_similar":[]}},"responses":[{{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{{"server":"nginx/1.16.1"}}}}]"#,
             saved_id, VERSION
         );
         println!("{}\n{}", expected, json_state);
-        assert!(predicates::str::similar(expected).eval(&json_state));
+        assert!(predicates::str::contains(expected).eval(&json_state));
     }
 
     #[should_panic]
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// call start_max_time_thread with a valid timespec, expect a panic, but only after a certain
     /// number of seconds
     async fn start_max_time_thread_panics_after_delay() {
         let now = time::Instant::now();
         let delay = time::Duration::new(3, 0);
+        let stats = Arc::new(Stats::new());
 
-        start_max_time_thread("3s").await;
+        start_max_time_thread("3s", stats).await;
 
         assert!(now.elapsed() > delay);
     }
 
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// call start_max_time_thread with a timespec that's too large to be parsed correctly, expect
     /// immediate return and no panic, as the sigint handler is never called
     async fn start_max_time_thread_returns_immediately_with_too_large_input() {
         let now = time::Instant::now();
         let delay = time::Duration::new(1, 0);
+        let stats = Arc::new(Stats::new());
 
         // pub const MAX: usize = usize::MAX; // 18_446_744_073_709_551_615usize
-        start_max_time_thread("18446744073709551616m").await; // can't fit in dest u64
+        start_max_time_thread("18446744073709551616m", stats).await; // can't fit in dest u64
 
         assert!(now.elapsed() < delay); // assuming function call will take less than 1second
     }
+
+    #[test]
+    /// coverage for FeroxScan's Display implementation
+    fn feroxscan_display() {
+        let mut scan = FeroxScan {
+            id: "".to_string(),
+            url: String::from("http://localhost"),
+            scan_type: Default::default(),
+            num_requests: 0,
+            status: Default::default(),
+            task: None,
+            progress_bar: None,
+        };
+
+        let not_started = format!("{}", scan);
+
+        assert!(predicate::str::contains("not started")
+            .and(predicate::str::contains("localhost"))
+            .eval(&not_started));
+
+        scan.status = ScanStatus::Complete;
+        let complete = format!("{}", scan);
+        assert!(predicate::str::contains("complete")
+            .and(predicate::str::contains("localhost"))
+            .eval(&complete));
+
+        scan.status = ScanStatus::Cancelled;
+        let cancelled = format!("{}", scan);
+        assert!(predicate::str::contains("cancelled")
+            .and(predicate::str::contains("localhost"))
+            .eval(&cancelled));
+
+        scan.status = ScanStatus::Running;
+        let running = format!("{}", scan);
+        assert!(predicate::str::contains("running")
+            .and(predicate::str::contains("localhost"))
+            .eval(&running));
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// call FeroxScan::abort, ensure status becomes cancelled
+    async fn ferox_scan_abort() {
+        let mut scan = FeroxScan {
+            id: "".to_string(),
+            url: String::from("http://localhost"),
+            scan_type: Default::default(),
+            num_requests: 0,
+            status: ScanStatus::Running,
+            task: Some(Arc::new(tokio::spawn(async move {
+                sleep(Duration::from_millis(SLEEP_DURATION * 2));
+            }))),
+            progress_bar: None,
+        };
+
+        scan.abort();
+
+        assert!(matches!(scan.status, ScanStatus::Cancelled));
+    }
+
+    #[test]
+    /// call a few menu functions for coverage's sake
+    ///
+    /// there's not a trivial way to test these programmatically (at least i'm too lazy rn to do it)
+    /// and their correctness can be verified easily manually; just calling for now
+    fn menu_print_header_and_footer() {
+        let menu = Menu::new();
+        menu.clear_screen();
+        menu.print_header();
+        menu.print_footer();
+        menu.hide_progress_bars();
+        menu.show_progress_bars();
+    }
+
+    #[test]
+    /// ensure spaces are trimmed and numbers are returned from split_to_nums
+    fn split_to_nums_is_correct() {
+        let menu = Menu::new();
+
+        let nums = menu.split_to_nums("1, 3,      4");
+
+        assert_eq!(nums, vec![1, 3, 4]);
+    }
 }

src/scanner.rs

@@ -1,12 +1,17 @@
 use crate::{
     config::{Configuration, CONFIGURATION},
-    extractor::{get_links, request_feroxresponse_from_new_link},
+    extractor::{extract_robots_txt, get_links, request_feroxresponse_from_new_link},
     filters::{
         FeroxFilter, LinesFilter, RegexFilter, SimilarityFilter, SizeFilter, StatusCodeFilter,
         WildcardFilter, WordsFilter,
     },
     heuristics,
-    scan_manager::{FeroxResponses, FeroxScans, PAUSE_SCAN},
+    scan_manager::{FeroxResponses, FeroxScans, ScanStatus, PAUSE_SCAN},
+    statistics::{
+        StatCommand::{self, UpdateF64Field, UpdateUsizeField},
+        StatField::{DirScanTimes, ExpectedPerScan, TotalScans, WildcardsFiltered},
+        Stats,
+    },
     utils::{format_url, get_current_depth, make_request},
     FeroxChannel, FeroxResponse, SIMILARITY_THRESHOLD,
 };
@@ -17,15 +22,16 @@ use futures::{
 use fuzzyhash::FuzzyHash;
 use lazy_static::lazy_static;
 use regex::Regex;
-use reqwest::Url;
+use reqwest::{StatusCode, Url};
 #[cfg(not(test))]
 use std::process::exit;
 use std::{
     collections::HashSet,
     convert::TryInto,
     ops::Deref,
-    sync::atomic::{AtomicU64, AtomicUsize, Ordering},
+    sync::atomic::{AtomicUsize, Ordering},
     sync::{Arc, RwLock},
+    time::Instant,
 };
 use tokio::{
     sync::{
@@ -35,12 +41,13 @@ use tokio::{
     task::JoinHandle,
 };
 
-/// Single atomic number that gets incremented once, used to track first scan vs. all others
+/// Single atomic number that gets incremented at least once, used to track first scan(s) vs. all
+/// others found during recursion
+///
+/// -u means this will be incremented once
+/// --stdin means this will be incremented by the number of targets passed via STDIN
 static CALL_COUNT: AtomicUsize = AtomicUsize::new(0);
 
-/// Single atomic number that gets holds the number of requests to be sent per directory scanned
-pub static NUMBER_OF_REQUESTS: AtomicU64 = AtomicU64::new(0);
-
 lazy_static! {
     /// Set of urls that have been sent to [scan_url](fn.scan_url.html), used for deduplication
     pub static ref SCANNED_URLS: FeroxScans = FeroxScans::default();
@@ -53,6 +60,8 @@ lazy_static! {
 
     /// Bounded semaphore used as a barrier to limit concurrent scans
     static ref SCAN_LIMITER: Semaphore = Semaphore::new(CONFIGURATION.scan_limit);
+
+
 }
 
 /// Adds the given FeroxFilter to the given list of FeroxFilter implementors
@@ -98,35 +107,41 @@ fn spawn_recursion_handler(
     mut recursion_channel: UnboundedReceiver<String>,
     wordlist: Arc<HashSet<String>>,
     base_depth: usize,
-    num_targets: usize,
+    stats: Arc<Stats>,
     tx_term: UnboundedSender<FeroxResponse>,
     tx_file: UnboundedSender<FeroxResponse>,
-) -> BoxFuture<'static, Vec<JoinHandle<()>>> {
+    tx_stats: UnboundedSender<StatCommand>,
+) -> BoxFuture<'static, Vec<Arc<JoinHandle<()>>>> {
     log::trace!(
-        "enter: spawn_recursion_handler({:?}, wordlist[{} words...], {}, {}, {:?}, {:?})",
+        "enter: spawn_recursion_handler({:?}, wordlist[{} words...], {}, {:?}, {:?}, {:?}, {:?})",
         recursion_channel,
         wordlist.len(),
         base_depth,
-        num_targets,
+        stats,
         tx_term,
-        tx_file
+        tx_file,
+        tx_stats
     );
 
     let boxed_future = async move {
         let mut scans = vec![];
 
         while let Some(resp) = recursion_channel.recv().await {
-            let (unknown, _) = SCANNED_URLS.add_directory_scan(&resp);
+            let (unknown, scan) = SCANNED_URLS.add_directory_scan(&resp, stats.clone());
 
             if !unknown {
                 // not unknown, i.e. we've seen the url before and don't need to scan again
                 continue;
             }
 
+            update_stat!(tx_stats, UpdateUsizeField(TotalScans, 1));
+
             log::info!("received {} on recursion channel", resp);
 
             let term_clone = tx_term.clone();
             let file_clone = tx_file.clone();
+            let tx_stats_clone = tx_stats.clone();
+            let stats_clone = stats.clone();
             let resp_clone = resp.clone();
             let list_clone = wordlist.clone();
 
@@ -135,14 +150,21 @@ fn spawn_recursion_handler(
                     resp_clone.to_owned().as_str(),
                     list_clone,
                     base_depth,
-                    num_targets,
+                    stats_clone,
                     term_clone,
                     file_clone,
+                    tx_stats_clone,
                 )
                 .await
             });
 
-            scans.push(future);
+            let shared_task = Arc::new(future);
+
+            if let Ok(mut u_scan) = scan.lock() {
+                u_scan.task = Some(shared_task.clone());
+            }
+
+            scans.push(shared_task);
         }
         scans
     }
@@ -158,12 +180,18 @@ fn spawn_recursion_handler(
 ///
 /// If any extensions were passed to the program, each extension will add a
 /// (base_url + word + ext) Url to the vector
-fn create_urls(target_url: &str, word: &str, extensions: &[String]) -> Vec<Url> {
+fn create_urls(
+    target_url: &str,
+    word: &str,
+    extensions: &[String],
+    tx_stats: UnboundedSender<StatCommand>,
+) -> Vec<Url> {
     log::trace!(
-        "enter: create_urls({}, {}, {:?})",
+        "enter: create_urls({}, {}, {:?}, {:?})",
         target_url,
         word,
-        extensions
+        extensions,
+        tx_stats
     );
 
     let mut urls = vec![];
@@ -174,6 +202,7 @@ fn create_urls(target_url: &str, word: &str, extensions: &[String]) -> Vec<Url>
         CONFIGURATION.add_slash,
         &CONFIGURATION.queries,
         None,
+        tx_stats.clone(),
     ) {
         urls.push(url); // default request, i.e. no extension
     }
@@ -185,6 +214,7 @@ fn create_urls(target_url: &str, word: &str, extensions: &[String]) -> Vec<Url>
             CONFIGURATION.add_slash,
             &CONFIGURATION.queries,
             Some(ext),
+            tx_stats.clone(),
         ) {
             urls.push(url); // any extensions passed in
         }
@@ -230,8 +260,9 @@ fn response_is_directory(response: &FeroxResponse) -> bool {
                 return false;
             }
         }
-    } else if response.status().is_success() {
-        // status code is 2xx, need to check if it ends in /
+    } else if response.status().is_success() || matches!(response.status(), &StatusCode::FORBIDDEN)
+    {
+        // status code is 2xx or 403, need to check if it ends in /
 
         if response.url().as_str().ends_with('/') {
             log::debug!("{} is directory suitable for recursion", response.url());
@@ -284,7 +315,7 @@ async fn try_recursion(
         "enter: try_recursion({}, {}, {:?})",
         response,
         base_depth,
-        transmitter
+        transmitter,
     );
 
     if !reached_max_depth(response.url(), base_depth, CONFIGURATION.depth)
@@ -328,12 +359,18 @@ async fn try_recursion(
 
 /// Simple helper to stay DRY; determines whether or not a given `FeroxResponse` should be reported
 /// to the user or not.
-pub fn should_filter_response(response: &FeroxResponse) -> bool {
+pub fn should_filter_response(
+    response: &FeroxResponse,
+    tx_stats: UnboundedSender<StatCommand>,
+) -> bool {
     match FILTERS.read() {
         Ok(filters) => {
             for filter in filters.iter() {
                 // wildcard.should_filter goes here
                 if filter.should_filter_response(&response) {
+                    if filter.as_any().downcast_ref::<WildcardFilter>().is_some() {
+                        update_stat!(tx_stats, UpdateUsizeField(WildcardsFiltered, 1))
+                    }
                     return true;
                 }
             }
@@ -354,22 +391,31 @@ async fn make_requests(
     target_url: &str,
     word: &str,
     base_depth: usize,
+    stats: Arc<Stats>,
     dir_chan: UnboundedSender<String>,
     report_chan: UnboundedSender<FeroxResponse>,
+    tx_stats: UnboundedSender<StatCommand>,
 ) {
     log::trace!(
-        "enter: make_requests({}, {}, {}, {:?}, {:?})",
+        "enter: make_requests({}, {}, {}, {:?}, {:?}, {:?}, {:?})",
         target_url,
         word,
         base_depth,
+        stats,
         dir_chan,
-        report_chan
+        report_chan,
+        tx_stats
     );
 
-    let urls = create_urls(&target_url, &word, &CONFIGURATION.extensions);
+    let urls = create_urls(
+        &target_url,
+        &word,
+        &CONFIGURATION.extensions,
+        tx_stats.clone(),
+    );
 
     for url in urls {
-        if let Ok(response) = make_request(&CONFIGURATION.client, &url).await {
+        if let Ok(response) = make_request(&CONFIGURATION.client, &url, tx_stats.clone()).await {
             // response came back without error, convert it to FeroxResponse
             let ferox_response = FeroxResponse::from(response, true).await;
 
@@ -381,22 +427,26 @@ async fn make_requests(
             // purposefully doing recursion before filtering. the thought process is that
             // even though this particular url is filtered, subsequent urls may not
 
-            if should_filter_response(&ferox_response) {
+            if should_filter_response(&ferox_response, tx_stats.clone()) {
                 continue;
             }
 
             if CONFIGURATION.extract_links && !ferox_response.status().is_redirection() {
-                let new_links = get_links(&ferox_response).await;
+                let new_links = get_links(&ferox_response, tx_stats.clone()).await;
 
                 for new_link in new_links {
-                    let mut new_ferox_response =
-                        match request_feroxresponse_from_new_link(&new_link).await {
-                            Some(resp) => resp,
-                            None => continue,
-                        };
+                    let mut new_ferox_response = match request_feroxresponse_from_new_link(
+                        &new_link,
+                        tx_stats.clone(),
+                    )
+                    .await
+                    {
+                        Some(resp) => resp,
+                        None => continue,
+                    };
 
                     // filter if necessary
-                    if should_filter_response(&new_ferox_response) {
+                    if should_filter_response(&new_ferox_response, tx_stats.clone()) {
                         continue;
                     }
 
@@ -404,7 +454,8 @@ async fn make_requests(
                         // very likely a file, simply request and report
                         log::debug!("Singular extraction: {}", new_ferox_response);
 
-                        SCANNED_URLS.add_file_scan(&new_ferox_response.url().to_string());
+                        SCANNED_URLS
+                            .add_file_scan(&new_ferox_response.url().to_string(), stats.clone());
 
                         send_report(report_chan.clone(), new_ferox_response);
 
@@ -414,10 +465,14 @@ async fn make_requests(
                     if !CONFIGURATION.no_recursion {
                         log::debug!("Recursive extraction: {}", new_ferox_response);
 
-                        if new_ferox_response.status().is_success()
-                            && !new_ferox_response.url().as_str().ends_with('/')
+                        if !new_ferox_response.url().as_str().ends_with('/')
+                            && (new_ferox_response.status().is_success()
+                                || matches!(new_ferox_response.status(), &StatusCode::FORBIDDEN))
                         {
-                            // since all of these are 2xx, recursion is only attempted if the
+                            // if the url doesn't end with a /
+                            // and the response code is either a 2xx or 403
+
+                            // since all of these are 2xx or 403, recursion is only attempted if the
                             // url ends in a /. I am actually ok with adding the slash and not
                             // adding it, as both have merit.  Leaving it in for now to see how
                             // things turn out (current as of: v1.1.0)
@@ -450,6 +505,61 @@ pub fn send_report(report_sender: UnboundedSender<FeroxResponse>, response: Fero
     log::trace!("exit: send_report");
 }
 
+/// Request /robots.txt from given url
+async fn scan_robots_txt(
+    target_url: &str,
+    base_depth: usize,
+    stats: Arc<Stats>,
+    tx_term: UnboundedSender<FeroxResponse>,
+    tx_dir: UnboundedSender<String>,
+    tx_stats: UnboundedSender<StatCommand>,
+) {
+    log::trace!(
+        "enter: scan_robots_txt({}, {}, {:?}, {:?}, {:?}, {:?})",
+        target_url,
+        base_depth,
+        stats,
+        tx_term,
+        tx_dir,
+        tx_stats
+    );
+
+    let robots_links = extract_robots_txt(&target_url, &CONFIGURATION, tx_stats.clone()).await;
+
+    for robot_link in robots_links {
+        // create a url based on the given command line options, continue on error
+        let mut ferox_response =
+            match request_feroxresponse_from_new_link(&robot_link, tx_stats.clone()).await {
+                Some(resp) => resp,
+                None => continue,
+            };
+
+        if should_filter_response(&ferox_response, tx_stats.clone()) {
+            continue;
+        }
+
+        if ferox_response.is_file() {
+            log::debug!("File extracted from robots.txt: {}", ferox_response);
+            SCANNED_URLS.add_file_scan(&robot_link, stats.clone());
+            send_report(tx_term.clone(), ferox_response);
+        } else if !CONFIGURATION.no_recursion {
+            log::debug!("Directory extracted from robots.txt: {}", ferox_response);
+            // todo this code is essentially the same as another piece around ~467 of this file
+            if !ferox_response.url().as_str().ends_with('/')
+                && (ferox_response.status().is_success()
+                    || matches!(ferox_response.status(), &StatusCode::FORBIDDEN))
+            {
+                // if the url doesn't end with a /
+                // and the response code is either a 2xx or 403
+                ferox_response.set_url(&format!("{}/", ferox_response.url()));
+            }
+
+            try_recursion(&ferox_response, base_depth, tx_dir.clone()).await;
+        }
+    }
+    log::trace!("exit: scan_robots_txt");
+}
+
 /// Scan a given url using a given wordlist
 ///
 /// This is the primary entrypoint for the scanner
@@ -457,34 +567,59 @@ pub async fn scan_url(
     target_url: &str,
     wordlist: Arc<HashSet<String>>,
     base_depth: usize,
-    num_targets: usize,
+    stats: Arc<Stats>,
     tx_term: UnboundedSender<FeroxResponse>,
     tx_file: UnboundedSender<FeroxResponse>,
+    tx_stats: UnboundedSender<StatCommand>,
 ) {
     log::trace!(
-        "enter: scan_url({:?}, wordlist[{} words...], {}, {}, {:?}, {:?})",
+        "enter: scan_url({:?}, wordlist[{} words...], {}, {:?}, {:?}, {:?}, {:?})",
         target_url,
         wordlist.len(),
         base_depth,
-        num_targets,
+        stats,
         tx_term,
-        tx_file
+        tx_file,
+        tx_stats
     );
 
     log::info!("Starting scan against: {}", target_url);
 
+    let scan_timer = Instant::now();
+
     let (tx_dir, rx_dir): FeroxChannel<String> = mpsc::unbounded_channel();
 
-    if CALL_COUNT.load(Ordering::Relaxed) < num_targets {
+    if CALL_COUNT.load(Ordering::Relaxed) < stats.initial_targets.load(Ordering::Relaxed) {
         CALL_COUNT.fetch_add(1, Ordering::Relaxed);
 
+        if CONFIGURATION.extract_links {
+            // only grab robots.txt on the initial scan_url calls. all fresh dirs will be passed
+            // to try_recursion
+            scan_robots_txt(
+                target_url,
+                base_depth,
+                stats.clone(),
+                tx_term.clone(),
+                tx_dir.clone(),
+                tx_stats.clone(),
+            )
+            .await;
+        }
+
+        update_stat!(tx_stats, UpdateUsizeField(TotalScans, 1));
+
         // this protection allows us to add the first scanned url to SCANNED_URLS
         // from within the scan_url function instead of the recursion handler
-        SCANNED_URLS.add_directory_scan(&target_url);
+        SCANNED_URLS.add_directory_scan(&target_url, stats.clone());
     }
 
     let ferox_scan = match SCANNED_URLS.get_scan_by_url(&target_url) {
-        Some(scan) => scan,
+        Some(scan) => {
+            if let Ok(mut u_scan) = scan.lock() {
+                u_scan.status = ScanStatus::Running;
+            }
+            scan
+        }
         None => {
             log::error!(
                 "Could not find FeroxScan associated with {}; this shouldn't happen... exiting",
@@ -511,29 +646,39 @@ pub async fn scan_url(
     // Arc clones to be passed around to the various scans
     let wildcard_bar = progress_bar.clone();
     let heuristics_term_clone = tx_term.clone();
+    let heuristics_stats_clone = tx_stats.clone();
     let recurser_term_clone = tx_term.clone();
     let recurser_file_clone = tx_file.clone();
+    let recurser_stats_clone = tx_stats.clone();
     let recurser_words = wordlist.clone();
     let looping_words = wordlist.clone();
+    let looping_stats = stats.clone();
 
     let recurser = tokio::spawn(async move {
         spawn_recursion_handler(
             rx_dir,
             recurser_words,
             base_depth,
-            num_targets,
+            stats.clone(),
             recurser_term_clone,
             recurser_file_clone,
+            recurser_stats_clone,
         )
         .await
     });
 
     // add any wildcard filters to `FILTERS`
-    let filter =
-        match heuristics::wildcard_test(&target_url, wildcard_bar, heuristics_term_clone).await {
-            Some(f) => Box::new(f),
-            None => Box::new(WildcardFilter::default()),
-        };
+    let filter = match heuristics::wildcard_test(
+        &target_url,
+        wildcard_bar,
+        heuristics_term_clone,
+        heuristics_stats_clone,
+    )
+    .await
+    {
+        Some(f) => Box::new(f),
+        None => Box::new(WildcardFilter::default()),
+    };
 
     add_filter_to_list_of_ferox_filters(filter, FILTERS.clone());
 
@@ -542,19 +687,19 @@ pub async fn scan_url(
         .map(|word| {
             let txd = tx_dir.clone();
             let txr = tx_term.clone();
+            let txs = tx_stats.clone();
             let pb = progress_bar.clone(); // progress bar is an Arc around internal state
             let tgt = target_url.to_string(); // done to satisfy 'static lifetime below
+            let lst = looping_stats.clone();
             (
                 tokio::spawn(async move {
                     if PAUSE_SCAN.load(Ordering::Acquire) {
                         // for every word in the wordlist, check to see if PAUSE_SCAN is set to true
                         // when true; enter a busy loop that only exits by setting PAUSE_SCAN back
                         // to false
-
-                        // todo change to true when issue #107 is resolved
-                        SCANNED_URLS.pause(false).await;
+                        SCANNED_URLS.pause(true).await;
                     }
-                    make_requests(&tgt, &word, base_depth, txd, txr).await
+                    make_requests(&tgt, &word, base_depth, lst, txd, txr, txs).await
                 }),
                 pb,
             )
@@ -575,6 +720,11 @@ pub async fn scan_url(
     producers.await;
     log::trace!("done awaiting scan producers");
 
+    update_stat!(
+        tx_stats,
+        UpdateF64Field(DirScanTimes, scan_timer.elapsed().as_secs_f64())
+    );
+
     // drop the current permit so the semaphore will allow another scan to proceed
     drop(permit);
 
@@ -586,18 +736,27 @@ pub async fn scan_url(
     log::trace!("dropped recursion handler's transmitter");
     drop(tx_dir);
 
-    // await rx tasks
-    log::trace!("awaiting recursive scan receiver/scans");
-    futures::future::join_all(recurser.await.unwrap()).await;
-    log::trace!("done awaiting recursive scan receiver/scans");
+    // note: in v1.11.2 i removed the join_all call that used to handle the recurser handles.
+    // nothing appears to change by having them removed, however, if ever a revert is needed
+    // this is the place and anything prior to 1.11.2 will have the code to do so
+    let _ = recurser.await.unwrap_or_default();
 
     log::trace!("exit: scan_url");
 }
 
 /// Perform steps necessary to run scans that only need to be performed once (warming up the
 /// engine, as it were)
-pub async fn initialize(num_words: usize, config: &Configuration) {
-    log::trace!("enter: initialize({}, {:?})", num_words, config,);
+pub async fn initialize(
+    num_words: usize,
+    config: &Configuration,
+    tx_stats: UnboundedSender<StatCommand>,
+) {
+    log::trace!(
+        "enter: initialize({}, {:?}, {:?})",
+        num_words,
+        config,
+        tx_stats
+    );
 
     // number of requests only needs to be calculated once, and then can be reused
     let num_reqs_expected: u64 = if config.extensions.is_empty() {
@@ -607,7 +766,11 @@ pub async fn initialize(num_words: usize, config: &Configuration) {
         total.try_into().unwrap()
     };
 
-    NUMBER_OF_REQUESTS.store(num_reqs_expected, Ordering::Relaxed);
+    // tell Stats object about the number of expected requests
+    update_stat!(
+        tx_stats,
+        UpdateUsizeField(ExpectedPerScan, num_reqs_expected as usize)
+    );
 
     // add any status code filters to `FILTERS` (-C|--filter-status)
     for code_filter in &config.filter_status {
@@ -670,9 +833,16 @@ pub async fn initialize(num_words: usize, config: &Configuration) {
     // add any similarity filters to `FILTERS` (--filter-similar-to)
     for similarity_filter in &config.filter_similar {
         // url as-is based on input, ignores user-specified url manipulation options (add-slash etc)
-        if let Ok(url) = format_url(&similarity_filter, &"", false, &Vec::new(), None) {
+        if let Ok(url) = format_url(
+            &similarity_filter,
+            &"",
+            false,
+            &Vec::new(),
+            None,
+            tx_stats.clone(),
+        ) {
             // attempt to request the given url
-            if let Ok(resp) = make_request(&CONFIGURATION.client, &url).await {
+            if let Ok(resp) = make_request(&CONFIGURATION.client, &url, tx_stats.clone()).await {
                 // if successful, create a filter based on the response's body
                 let fr = FeroxResponse::from(resp, true).await;
 
@@ -707,14 +877,16 @@ mod tests {
     #[test]
     /// sending url + word without any extensions should get back one url with the joined word
     fn create_urls_no_extension_returns_base_url_with_word() {
-        let urls = create_urls("http://localhost", "turbo", &[]);
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+        let urls = create_urls("http://localhost", "turbo", &[], tx);
         assert_eq!(urls, [Url::parse("http://localhost/turbo").unwrap()])
     }
 
     #[test]
     /// sending url + word + 1 extension should get back two urls, one base and one with extension
     fn create_urls_one_extension_returns_two_urls() {
-        let urls = create_urls("http://localhost", "turbo", &[String::from("js")]);
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+        let urls = create_urls("http://localhost", "turbo", &[String::from("js")], tx);
         assert_eq!(
             urls,
             [
@@ -752,8 +924,10 @@ mod tests {
             vec![base, js, php, pdf, tar],
         ];
 
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
         for (i, ext_set) in ext_vec.into_iter().enumerate() {
-            let urls = create_urls("http://localhost", "turbo", &ext_set);
+            let urls = create_urls("http://localhost", "turbo", &ext_set, tx.clone());
             assert_eq!(urls, expected[i]);
         }
     }
@@ -798,12 +972,15 @@ mod tests {
         assert!(result);
     }
 
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     #[should_panic]
     /// call initialize with a bad regex, triggering a panic
     async fn initialize_panics_on_bad_regex() {
-        let mut config = Configuration::default();
-        config.filter_regex = vec![r"(".to_string()];
-        initialize(1, &config).await;
+        let config = Configuration {
+            filter_regex: vec![r"(".to_string()],
+            ..Default::default()
+        };
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+        initialize(1, &config, tx).await;
     }
 }

src/statistics.rs

@@ -0,0 +1,833 @@
+use crate::{
+    config::CONFIGURATION,
+    progress::{add_bar, BarType},
+    reporter::{get_cached_file_handle, safe_file_write},
+    FeroxChannel, FeroxSerialize,
+};
+use console::style;
+use indicatif::ProgressBar;
+use reqwest::StatusCode;
+use serde::{Deserialize, Serialize};
+use std::{
+    fs::File,
+    io::BufReader,
+    sync::{
+        atomic::{AtomicUsize, Ordering},
+        Arc, Mutex,
+    },
+    time::Instant,
+};
+use tokio::{
+    sync::mpsc::{self, UnboundedReceiver, UnboundedSender},
+    task::JoinHandle,
+};
+
+/// Wrapper `Atomic*.fetch_add` to save me from writing Ordering::Relaxed a bajillion times
+///
+/// default is to increment by 1, second arg can be used to increment by a different value
+macro_rules! atomic_increment {
+    ($metric:expr) => {
+        $metric.fetch_add(1, Ordering::Relaxed);
+    };
+
+    ($metric:expr, $value:expr) => {
+        $metric.fetch_add($value, Ordering::Relaxed);
+    };
+}
+
+/// Wrapper around `Atomic*.load` to save me from writing Ordering::Relaxed a bajillion times
+macro_rules! atomic_load {
+    ($metric:expr) => {
+        $metric.load(Ordering::Relaxed);
+    };
+}
+
+/// Data collection of statistics related to a scan
+#[derive(Default, Deserialize, Debug, Serialize)]
+pub struct Stats {
+    #[serde(rename = "type")]
+    /// Name of this type of struct, used for serialization, i.e. `{"type":"statistics"}`
+    kind: String,
+
+    /// tracker for number of timeouts seen by the client
+    timeouts: AtomicUsize,
+
+    /// tracker for total number of requests sent by the client
+    requests: AtomicUsize,
+
+    /// tracker for total number of requests expected to send if the scan runs to completion
+    ///
+    /// Note: this is a per-scan expectation; `expected_requests * current # of scans` would be
+    /// indicative of the current expectation at any given time, but is a moving target.  
+    pub expected_per_scan: AtomicUsize,
+
+    /// tracker for accumulating total number of requests expected (i.e. as a new scan is started
+    /// this value should increase by `expected_requests`
+    total_expected: AtomicUsize,
+
+    /// tracker for total number of errors encountered by the client
+    errors: AtomicUsize,
+
+    /// tracker for overall number of 2xx status codes seen by the client
+    successes: AtomicUsize,
+
+    /// tracker for overall number of 3xx status codes seen by the client
+    redirects: AtomicUsize,
+
+    /// tracker for overall number of 4xx status codes seen by the client
+    client_errors: AtomicUsize,
+
+    /// tracker for overall number of 5xx status codes seen by the client
+    server_errors: AtomicUsize,
+
+    /// tracker for number of scans performed, this directly equates to number of directories
+    /// recursed into and affects the total number of expected requests
+    total_scans: AtomicUsize,
+
+    /// tracker for initial number of requested targets
+    pub initial_targets: AtomicUsize,
+
+    /// tracker for number of links extracted when `--extract-links` is used; sources are
+    /// response bodies and robots.txt as of v1.11.0
+    links_extracted: AtomicUsize,
+
+    /// tracker for overall number of 200s seen by the client
+    status_200s: AtomicUsize,
+
+    /// tracker for overall number of 301s seen by the client
+    status_301s: AtomicUsize,
+
+    /// tracker for overall number of 302s seen by the client
+    status_302s: AtomicUsize,
+
+    /// tracker for overall number of 401s seen by the client
+    status_401s: AtomicUsize,
+
+    /// tracker for overall number of 403s seen by the client
+    status_403s: AtomicUsize,
+
+    /// tracker for overall number of 429s seen by the client
+    status_429s: AtomicUsize,
+
+    /// tracker for overall number of 500s seen by the client
+    status_500s: AtomicUsize,
+
+    /// tracker for overall number of 503s seen by the client
+    status_503s: AtomicUsize,
+
+    /// tracker for overall number of 504s seen by the client
+    status_504s: AtomicUsize,
+
+    /// tracker for overall number of 508s seen by the client
+    status_508s: AtomicUsize,
+
+    /// tracker for overall number of wildcard urls filtered out by the client
+    wildcards_filtered: AtomicUsize,
+
+    /// tracker for overall number of all filtered responses
+    responses_filtered: AtomicUsize,
+
+    /// tracker for number of files found
+    resources_discovered: AtomicUsize,
+
+    /// tracker for number of errors triggered during URL formatting
+    url_format_errors: AtomicUsize,
+
+    /// tracker for number of errors triggered by the `reqwest::RedirectPolicy`
+    redirection_errors: AtomicUsize,
+
+    /// tracker for number of errors related to the connecting
+    connection_errors: AtomicUsize,
+
+    /// tracker for number of errors related to the request used
+    request_errors: AtomicUsize,
+
+    /// tracker for each directory's total scan time in seconds as a float
+    directory_scan_times: Mutex<Vec<f64>>,
+
+    /// tracker for total runtime
+    total_runtime: Mutex<Vec<f64>>,
+}
+
+/// FeroxSerialize implementation for Stats
+impl FeroxSerialize for Stats {
+    /// Simply return empty string here to disable serializing this to the output file as a string
+    /// due to it looking like garbage
+    fn as_str(&self) -> String {
+        String::new()
+    }
+
+    /// Simple call to produce a JSON string using the given Stats object
+    fn as_json(&self) -> String {
+        serde_json::to_string(&self).unwrap_or_default()
+    }
+}
+
+/// implementation of statistics data collection struct
+impl Stats {
+    /// Small wrapper for default to set `kind` to "statistics" and `total_runtime` to have at least
+    /// one value
+    pub fn new() -> Self {
+        Self {
+            kind: String::from("statistics"),
+            total_runtime: Mutex::new(vec![0.0]),
+            ..Default::default()
+        }
+    }
+
+    /// increment `requests` field by one
+    fn add_request(&self) {
+        atomic_increment!(self.requests);
+    }
+
+    /// given an `Instant` update total runtime
+    fn update_runtime(&self, seconds: f64) {
+        if let Ok(mut runtime) = self.total_runtime.lock() {
+            runtime[0] = seconds;
+        }
+    }
+
+    /// save an instance of `Stats` to disk after updating the total runtime for the scan
+    fn save(&self, seconds: f64, location: &str) {
+        let buffered_file = match get_cached_file_handle(location) {
+            Some(file) => file,
+            None => {
+                return;
+            }
+        };
+
+        self.update_runtime(seconds);
+
+        safe_file_write(self, buffered_file, CONFIGURATION.json);
+    }
+
+    /// Inspect the given `StatError` and increment the appropriate fields
+    ///
+    /// Implies incrementing:
+    ///     - requests
+    ///     - errors
+    pub fn add_error(&self, error: StatError) {
+        self.add_request();
+        atomic_increment!(self.errors);
+
+        match error {
+            StatError::Timeout => {
+                atomic_increment!(self.timeouts);
+            }
+            StatError::Status403 => {
+                atomic_increment!(self.status_403s);
+                atomic_increment!(self.client_errors);
+            }
+            StatError::UrlFormat => {
+                atomic_increment!(self.url_format_errors);
+            }
+            StatError::Redirection => {
+                atomic_increment!(self.redirection_errors);
+            }
+            StatError::Connection => {
+                atomic_increment!(self.connection_errors);
+            }
+            StatError::Request => {
+                atomic_increment!(self.request_errors);
+            }
+            StatError::Other => {
+                atomic_increment!(self.errors);
+            }
+        }
+    }
+
+    /// Inspect the given `StatusCode` and increment the appropriate fields
+    ///
+    /// Implies incrementing:
+    ///     - requests
+    ///     - status_403s (when code is 403)
+    ///     - errors (when code is [45]xx)
+    fn add_status_code(&self, status: StatusCode) {
+        self.add_request();
+
+        if status.is_success() {
+            atomic_increment!(self.successes);
+        } else if status.is_redirection() {
+            atomic_increment!(self.redirects);
+        } else if status.is_client_error() {
+            atomic_increment!(self.client_errors);
+        } else if status.is_server_error() {
+            atomic_increment!(self.server_errors);
+        }
+
+        match status {
+            StatusCode::FORBIDDEN => {
+                atomic_increment!(self.status_403s);
+            }
+            StatusCode::OK => {
+                atomic_increment!(self.status_200s);
+            }
+            StatusCode::MOVED_PERMANENTLY => {
+                atomic_increment!(self.status_301s);
+            }
+            StatusCode::FOUND => {
+                atomic_increment!(self.status_302s);
+            }
+            StatusCode::UNAUTHORIZED => {
+                atomic_increment!(self.status_401s);
+            }
+            StatusCode::TOO_MANY_REQUESTS => {
+                atomic_increment!(self.status_429s);
+            }
+            StatusCode::INTERNAL_SERVER_ERROR => {
+                atomic_increment!(self.status_500s);
+            }
+            StatusCode::SERVICE_UNAVAILABLE => {
+                atomic_increment!(self.status_503s);
+            }
+            StatusCode::GATEWAY_TIMEOUT => {
+                atomic_increment!(self.status_504s);
+            }
+            StatusCode::LOOP_DETECTED => {
+                atomic_increment!(self.status_508s);
+            }
+            _ => {} // other status codes ignored for stat gathering
+        }
+    }
+
+    /// Update a `Stats` field of type f64
+    fn update_f64_field(&self, field: StatField, value: f64) {
+        if let StatField::DirScanTimes = field {
+            if let Ok(mut locked_times) = self.directory_scan_times.lock() {
+                locked_times.push(value);
+            }
+        }
+    }
+
+    /// Update a `Stats` field of type usize
+    fn update_usize_field(&self, field: StatField, value: usize) {
+        match field {
+            StatField::ExpectedPerScan => {
+                atomic_increment!(self.expected_per_scan, value);
+            }
+            StatField::TotalScans => {
+                let multiplier = CONFIGURATION.extensions.len().max(1);
+
+                atomic_increment!(self.total_scans, value);
+                atomic_increment!(
+                    self.total_expected,
+                    value * self.expected_per_scan.load(Ordering::Relaxed) * multiplier
+                );
+            }
+            StatField::TotalExpected => {
+                atomic_increment!(self.total_expected, value);
+            }
+            StatField::LinksExtracted => {
+                atomic_increment!(self.links_extracted, value);
+            }
+            StatField::WildcardsFiltered => {
+                atomic_increment!(self.wildcards_filtered, value);
+                atomic_increment!(self.responses_filtered, value);
+            }
+            StatField::ResponsesFiltered => {
+                atomic_increment!(self.responses_filtered, value);
+            }
+            StatField::ResourcesDiscovered => {
+                atomic_increment!(self.resources_discovered, value);
+            }
+            StatField::InitialTargets => {
+                atomic_increment!(self.initial_targets, value);
+            }
+            _ => {} // f64 fields
+        }
+    }
+
+    /// Merge a given `Stats` object from a json entry written to disk when handling a Ctrl+c
+    ///
+    /// This is only ever called when resuming a scan from disk
+    pub fn merge_from(&self, filename: &str) {
+        if let Ok(file) = File::open(filename) {
+            let reader = BufReader::new(file);
+            let state: serde_json::Value = serde_json::from_reader(reader).unwrap();
+
+            if let Some(state_stats) = state.get("statistics") {
+                if let Ok(d_stats) = serde_json::from_value::<Stats>(state_stats.clone()) {
+                    atomic_increment!(self.successes, atomic_load!(d_stats.successes));
+                    atomic_increment!(self.timeouts, atomic_load!(d_stats.timeouts));
+                    atomic_increment!(self.requests, atomic_load!(d_stats.requests));
+                    atomic_increment!(self.errors, atomic_load!(d_stats.errors));
+                    atomic_increment!(self.redirects, atomic_load!(d_stats.redirects));
+                    atomic_increment!(self.client_errors, atomic_load!(d_stats.client_errors));
+                    atomic_increment!(self.server_errors, atomic_load!(d_stats.server_errors));
+                    atomic_increment!(self.links_extracted, atomic_load!(d_stats.links_extracted));
+                    atomic_increment!(self.status_200s, atomic_load!(d_stats.status_200s));
+                    atomic_increment!(self.status_301s, atomic_load!(d_stats.status_301s));
+                    atomic_increment!(self.status_302s, atomic_load!(d_stats.status_302s));
+                    atomic_increment!(self.status_401s, atomic_load!(d_stats.status_401s));
+                    atomic_increment!(self.status_403s, atomic_load!(d_stats.status_403s));
+                    atomic_increment!(self.status_429s, atomic_load!(d_stats.status_429s));
+                    atomic_increment!(self.status_500s, atomic_load!(d_stats.status_500s));
+                    atomic_increment!(self.status_503s, atomic_load!(d_stats.status_503s));
+                    atomic_increment!(self.status_504s, atomic_load!(d_stats.status_504s));
+                    atomic_increment!(self.status_508s, atomic_load!(d_stats.status_508s));
+                    atomic_increment!(
+                        self.wildcards_filtered,
+                        atomic_load!(d_stats.wildcards_filtered)
+                    );
+                    atomic_increment!(
+                        self.responses_filtered,
+                        atomic_load!(d_stats.responses_filtered)
+                    );
+                    atomic_increment!(
+                        self.resources_discovered,
+                        atomic_load!(d_stats.resources_discovered)
+                    );
+                    atomic_increment!(
+                        self.url_format_errors,
+                        atomic_load!(d_stats.url_format_errors)
+                    );
+                    atomic_increment!(
+                        self.connection_errors,
+                        atomic_load!(d_stats.connection_errors)
+                    );
+                    atomic_increment!(
+                        self.redirection_errors,
+                        atomic_load!(d_stats.redirection_errors)
+                    );
+                    atomic_increment!(self.request_errors, atomic_load!(d_stats.request_errors));
+
+                    if let Ok(scan_times) = d_stats.directory_scan_times.lock() {
+                        for scan_time in scan_times.iter() {
+                            self.update_f64_field(StatField::DirScanTimes, *scan_time);
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+#[derive(Debug)]
+/// Enum variants used to inform the `StatCommand` protocol what `Stats` fields should be updated
+pub enum StatError {
+    /// Represents a 403 response code
+    Status403,
+
+    /// Represents a timeout error
+    Timeout,
+
+    /// Represents a URL formatting error
+    UrlFormat,
+
+    /// Represents an error encountered during redirection
+    Redirection,
+
+    /// Represents an error encountered during connection
+    Connection,
+
+    /// Represents an error resulting from the client's request
+    Request,
+
+    /// Represents any other error not explicitly defined above
+    Other,
+}
+
+/// Protocol definition for updating a Stats object via mpsc
+#[derive(Debug)]
+pub enum StatCommand {
+    /// Add one to the total number of requests
+    AddRequest,
+
+    /// Add one to the proper field(s) based on the given `StatError`
+    AddError(StatError),
+
+    /// Add one to the proper field(s) based on the given `StatusCode`
+    AddStatus(StatusCode),
+
+    /// Create the progress bar (`BarType::Total`) that is updated from the stats thread
+    CreateBar,
+
+    /// Update a `Stats` field that corresponds to the given `StatField` by the given `usize` value
+    UpdateUsizeField(StatField, usize),
+
+    /// Update a `Stats` field that corresponds to the given `StatField` by the given `f64` value
+    UpdateF64Field(StatField, f64),
+
+    /// Save a `Stats` object to disk using `reporter::get_cached_file_handle`
+    Save,
+
+    /// Load a `Stats` object from disk
+    LoadStats(String),
+
+    /// Break out of the (infinite) mpsc receive loop
+    Exit,
+}
+
+/// Enum representing fields whose updates need to be performed in batches instead of one at
+/// a time
+#[derive(Debug)]
+pub enum StatField {
+    /// Due to the necessary order of events, the number of requests expected to be sent isn't
+    /// known until after `statistics::initialize` is called. This command allows for updating
+    /// the `expected_per_scan` field after initialization
+    ExpectedPerScan,
+
+    /// Translates to `total_scans`
+    TotalScans,
+
+    /// Translates to `links_extracted`
+    LinksExtracted,
+
+    /// Translates to `total_expected`
+    TotalExpected,
+
+    /// Translates to `wildcards_filtered`
+    WildcardsFiltered,
+
+    /// Translates to `responses_filtered`
+    ResponsesFiltered,
+
+    /// Translates to `resources_discovered`
+    ResourcesDiscovered,
+
+    /// Translates to `initial_targets`
+    InitialTargets,
+
+    /// Translates to `directory_scan_times`; assumes a single append to the vector
+    DirScanTimes,
+}
+
+/// Spawn a single consumer task (sc side of mpsc)
+///
+/// The consumer simply receives `StatCommands` and updates the given `Stats` object as appropriate
+pub async fn spawn_statistics_handler(
+    mut rx_stats: UnboundedReceiver<StatCommand>,
+    stats: Arc<Stats>,
+    tx_stats: UnboundedSender<StatCommand>,
+) {
+    log::trace!(
+        "enter: spawn_statistics_handler({:?}, {:?}, {:?})",
+        rx_stats,
+        stats,
+        tx_stats
+    );
+
+    // will be updated later via StatCommand; delay is for banner to print first
+    let mut bar = ProgressBar::hidden();
+
+    let start = Instant::now();
+
+    while let Some(command) = rx_stats.recv().await {
+        match command as StatCommand {
+            StatCommand::AddError(err) => {
+                stats.add_error(err);
+                increment_bar(&bar, stats.clone());
+            }
+            StatCommand::AddStatus(status) => {
+                stats.add_status_code(status);
+                increment_bar(&bar, stats.clone());
+            }
+            StatCommand::AddRequest => {
+                stats.add_request();
+                increment_bar(&bar, stats.clone());
+            }
+            StatCommand::Save => stats.save(start.elapsed().as_secs_f64(), &CONFIGURATION.output),
+            StatCommand::UpdateUsizeField(field, value) => {
+                let update_len = matches!(field, StatField::TotalScans);
+                stats.update_usize_field(field, value);
+
+                if update_len {
+                    bar.set_length(atomic_load!(stats.total_expected) as u64)
+                }
+            }
+            StatCommand::UpdateF64Field(field, value) => stats.update_f64_field(field, value),
+            StatCommand::CreateBar => {
+                bar = add_bar(
+                    "",
+                    atomic_load!(stats.total_expected) as u64,
+                    BarType::Total,
+                );
+            }
+            StatCommand::LoadStats(filename) => {
+                stats.merge_from(&filename);
+            }
+            StatCommand::Exit => break,
+        }
+    }
+
+    bar.finish();
+
+    log::debug!("{:#?}", *stats);
+    log::trace!("exit: spawn_statistics_handler")
+}
+
+/// Wrapper around incrementing the overall scan's progress bar
+fn increment_bar(bar: &ProgressBar, stats: Arc<Stats>) {
+    let msg = format!(
+        "{}:{:<7} {}:{:<7}",
+        style("found").green(),
+        atomic_load!(stats.resources_discovered),
+        style("errors").red(),
+        atomic_load!(stats.errors),
+    );
+
+    bar.set_message(&msg);
+    bar.inc(1);
+}
+
+/// Initialize new `Stats` object and the sc side of an mpsc channel that is responsible for
+/// updates to the aforementioned object.
+pub fn initialize() -> (Arc<Stats>, UnboundedSender<StatCommand>, JoinHandle<()>) {
+    log::trace!("enter: initialize");
+
+    let stats_tracker = Arc::new(Stats::new());
+    let stats_cloned = stats_tracker.clone();
+    let (tx_stats, rx_stats): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+    let tx_stats_cloned = tx_stats.clone();
+    let stats_thread = tokio::spawn(async move {
+        spawn_statistics_handler(rx_stats, stats_cloned, tx_stats_cloned).await
+    });
+
+    log::trace!(
+        "exit: initialize -> ({:?}, {:?}, {:?})",
+        stats_tracker,
+        tx_stats,
+        stats_thread
+    );
+
+    (stats_tracker, tx_stats, stats_thread)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::fs::write;
+    use tempfile::NamedTempFile;
+
+    /// simple helper to reduce code reuse
+    fn setup_stats_test() -> (Arc<Stats>, UnboundedSender<StatCommand>, JoinHandle<()>) {
+        initialize()
+    }
+
+    /// another helper to stay DRY; must be called after any sent commands and before any checks
+    /// performed against the Stats object
+    async fn teardown_stats_test(sender: UnboundedSender<StatCommand>, handle: JoinHandle<()>) {
+        // send exit and await, once the await completes, stats should be updated
+        sender.send(StatCommand::Exit).unwrap_or_default();
+        handle.await.unwrap();
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// when sent StatCommand::Exit, function should exit its while loop (runs forever otherwise)
+    async fn statistics_handler_exits() {
+        let (_, sender, handle) = setup_stats_test();
+
+        sender.send(StatCommand::Exit).unwrap_or_default();
+
+        handle.await.unwrap(); // blocks on the handler's while loop
+
+        // if we've made it here, the test has succeeded
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// when sent StatCommand::AddRequest, stats object should reflect the change
+    async fn statistics_handler_increments_requests() {
+        let (stats, tx, handle) = setup_stats_test();
+
+        tx.send(StatCommand::AddRequest).unwrap_or_default();
+        tx.send(StatCommand::AddRequest).unwrap_or_default();
+        tx.send(StatCommand::AddRequest).unwrap_or_default();
+
+        teardown_stats_test(tx, handle).await;
+
+        assert_eq!(stats.requests.load(Ordering::Relaxed), 3);
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// when sent StatCommand::AddRequest, stats object should reflect the change
+    ///
+    /// incrementing a 403 (tracked in status_403s) should also increment:
+    ///     - errors
+    ///     - requests
+    ///     - client_errors
+    async fn statistics_handler_increments_403() {
+        let (stats, tx, handle) = setup_stats_test();
+
+        let err = StatCommand::AddError(StatError::Status403);
+        let err2 = StatCommand::AddError(StatError::Status403);
+
+        tx.send(err).unwrap_or_default();
+        tx.send(err2).unwrap_or_default();
+
+        teardown_stats_test(tx, handle).await;
+
+        assert_eq!(stats.errors.load(Ordering::Relaxed), 2);
+        assert_eq!(stats.requests.load(Ordering::Relaxed), 2);
+        assert_eq!(stats.status_403s.load(Ordering::Relaxed), 2);
+        assert_eq!(stats.client_errors.load(Ordering::Relaxed), 2);
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// when sent StatCommand::AddRequest, stats object should reflect the change
+    ///
+    /// incrementing a 403 (tracked in status_403s) should also increment:
+    ///     - requests
+    ///     - client_errors
+    async fn statistics_handler_increments_403_via_status_code() {
+        let (stats, tx, handle) = setup_stats_test();
+
+        let err = StatCommand::AddStatus(reqwest::StatusCode::FORBIDDEN);
+        let err2 = StatCommand::AddStatus(reqwest::StatusCode::FORBIDDEN);
+
+        tx.send(err).unwrap_or_default();
+        tx.send(err2).unwrap_or_default();
+
+        teardown_stats_test(tx, handle).await;
+
+        assert_eq!(stats.requests.load(Ordering::Relaxed), 2);
+        assert_eq!(stats.status_403s.load(Ordering::Relaxed), 2);
+        assert_eq!(stats.client_errors.load(Ordering::Relaxed), 2);
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// when sent StatCommand::AddStatus, stats object should reflect the change
+    ///
+    /// incrementing a 500 (tracked in server_errors) should also increment:
+    ///     - requests
+    async fn statistics_handler_increments_500_via_status_code() {
+        let (stats, tx, handle) = setup_stats_test();
+
+        let err = StatCommand::AddStatus(reqwest::StatusCode::INTERNAL_SERVER_ERROR);
+        let err2 = StatCommand::AddStatus(reqwest::StatusCode::INTERNAL_SERVER_ERROR);
+
+        tx.send(err).unwrap_or_default();
+        tx.send(err2).unwrap_or_default();
+
+        teardown_stats_test(tx, handle).await;
+
+        assert_eq!(stats.requests.load(Ordering::Relaxed), 2);
+        assert_eq!(stats.server_errors.load(Ordering::Relaxed), 2);
+    }
+
+    #[test]
+    /// when Stats::add_error receives StatError::Timeout, it should increment the following:
+    ///     - timeouts
+    ///     - requests
+    ///     - errors
+    fn stats_increments_timeouts() {
+        let stats = Stats::new();
+        stats.add_error(StatError::Timeout);
+        stats.add_error(StatError::Timeout);
+        stats.add_error(StatError::Timeout);
+        stats.add_error(StatError::Timeout);
+
+        assert_eq!(stats.errors.load(Ordering::Relaxed), 4);
+        assert_eq!(stats.requests.load(Ordering::Relaxed), 4);
+        assert_eq!(stats.timeouts.load(Ordering::Relaxed), 4);
+    }
+
+    #[test]
+    /// when Stats::update_usize_field receives StatField::WildcardsFiltered, it should increment
+    /// the following:
+    ///     - responses_filtered
+    fn stats_increments_wildcards() {
+        let stats = Stats::new();
+        assert_eq!(stats.responses_filtered.load(Ordering::Relaxed), 0);
+        assert_eq!(stats.wildcards_filtered.load(Ordering::Relaxed), 0);
+
+        stats.update_usize_field(StatField::WildcardsFiltered, 1);
+        stats.update_usize_field(StatField::WildcardsFiltered, 1);
+
+        assert_eq!(stats.responses_filtered.load(Ordering::Relaxed), 2);
+        assert_eq!(stats.wildcards_filtered.load(Ordering::Relaxed), 2);
+    }
+
+    #[test]
+    /// when Stats::update_usize_field receives StatField::ResponsesFiltered, it should increment
+    fn stats_increments_responses_filtered() {
+        let stats = Stats::new();
+        assert_eq!(stats.responses_filtered.load(Ordering::Relaxed), 0);
+
+        stats.update_usize_field(StatField::ResponsesFiltered, 1);
+        stats.update_usize_field(StatField::ResponsesFiltered, 1);
+        stats.update_usize_field(StatField::ResponsesFiltered, 1);
+
+        assert_eq!(stats.responses_filtered.load(Ordering::Relaxed), 3);
+    }
+
+    #[test]
+    /// Stats::merge_from should properly incrememnt expected fields and ignore others
+    fn stats_merge_from_alters_correct_fields() {
+        let contents = r#"{"statistics":{"type":"statistics","timeouts":1,"requests":9207,"expected_per_scan":707,"total_expected":9191,"errors":3,"successes":720,"redirects":13,"client_errors":8474,"server_errors":2,"total_scans":13,"initial_targets":1,"links_extracted":51,"status_403s":3,"status_200s":720,"status_301s":12,"status_302s":1,"status_401s":4,"status_429s":2,"status_500s":5,"status_503s":9,"status_504s":6,"status_508s":7,"wildcards_filtered":707,"responses_filtered":707,"resources_discovered":27,"directory_scan_times":[2.211973078,1.989015505,1.898675839,3.9714468910000003,4.938152838,5.256073528,6.021986595,6.065740734,6.42633762,7.095142125,7.336982137,5.319785619,4.843649778],"total_runtime":[11.556575456000001],"url_format_errors":17,"redirection_errors":12,"connection_errors":21,"request_errors":4}}"#;
+        let stats = Stats::new();
+        let tfile = NamedTempFile::new().unwrap();
+        write(&tfile, contents).unwrap();
+
+        stats.merge_from(tfile.path().to_str().unwrap());
+
+        // as of 1.11.1; all Stats fields are accounted for whether they're updated in merge_from
+        // or not
+        assert_eq!(atomic_load!(stats.timeouts), 1);
+        assert_eq!(atomic_load!(stats.requests), 9207);
+        assert_eq!(atomic_load!(stats.expected_per_scan), 0); // not updated in merge_from
+        assert_eq!(atomic_load!(stats.total_expected), 0); // not updated in merge_from
+        assert_eq!(atomic_load!(stats.errors), 3);
+        assert_eq!(atomic_load!(stats.successes), 720);
+        assert_eq!(atomic_load!(stats.redirects), 13);
+        assert_eq!(atomic_load!(stats.client_errors), 8474);
+        assert_eq!(atomic_load!(stats.server_errors), 2);
+        assert_eq!(atomic_load!(stats.total_scans), 0); // not updated in merge_from
+        assert_eq!(atomic_load!(stats.initial_targets), 0); // not updated in merge_from
+        assert_eq!(atomic_load!(stats.links_extracted), 51);
+        assert_eq!(atomic_load!(stats.status_200s), 720);
+        assert_eq!(atomic_load!(stats.status_301s), 12);
+        assert_eq!(atomic_load!(stats.status_302s), 1);
+        assert_eq!(atomic_load!(stats.status_401s), 4);
+        assert_eq!(atomic_load!(stats.status_403s), 3);
+        assert_eq!(atomic_load!(stats.status_429s), 2);
+        assert_eq!(atomic_load!(stats.status_500s), 5);
+        assert_eq!(atomic_load!(stats.status_503s), 9);
+        assert_eq!(atomic_load!(stats.status_504s), 6);
+        assert_eq!(atomic_load!(stats.status_508s), 7);
+        assert_eq!(atomic_load!(stats.wildcards_filtered), 707);
+        assert_eq!(atomic_load!(stats.responses_filtered), 707);
+        assert_eq!(atomic_load!(stats.resources_discovered), 27);
+        assert_eq!(atomic_load!(stats.url_format_errors), 17);
+        assert_eq!(atomic_load!(stats.redirection_errors), 12);
+        assert_eq!(atomic_load!(stats.connection_errors), 21);
+        assert_eq!(atomic_load!(stats.request_errors), 4);
+        assert_eq!(stats.directory_scan_times.lock().unwrap().len(), 13);
+        for scan in stats.directory_scan_times.lock().unwrap().iter() {
+            assert!(scan.max(0.0) > 0.0); // all scans are non-zero
+        }
+        // total_runtime not updated in merge_from
+        assert_eq!(stats.total_runtime.lock().unwrap().len(), 1);
+        assert!((stats.total_runtime.lock().unwrap()[0] - 0.0).abs() < f64::EPSILON);
+    }
+
+    #[test]
+    /// ensure update runtime overwrites the default 0th entry
+    fn update_runtime_works() {
+        let stats = Stats::new();
+        assert!((stats.total_runtime.lock().unwrap()[0] - 0.0).abs() < f64::EPSILON);
+        stats.update_runtime(20.2);
+        assert!((stats.total_runtime.lock().unwrap()[0] - 20.2).abs() < f64::EPSILON);
+    }
+
+    #[test]
+    /// Stats::save should write contents of Stats to disk
+    fn save_writes_stats_object_to_disk() {
+        let stats = Stats::new();
+        stats.add_request();
+        stats.add_request();
+        stats.add_request();
+        stats.add_request();
+        stats.add_error(StatError::Timeout);
+        stats.add_error(StatError::Timeout);
+        stats.add_error(StatError::Timeout);
+        stats.add_error(StatError::Timeout);
+        stats.add_status_code(StatusCode::OK);
+        stats.add_status_code(StatusCode::OK);
+        stats.add_status_code(StatusCode::OK);
+        let outfile = "/tmp/stuff";
+        stats.save(174.33, outfile);
+        assert!(stats.as_json().contains("statistics"));
+        assert!(stats.as_json().contains("11")); // requests made
+        assert!(stats.as_str().is_empty());
+    }
+}

src/utils.rs

@@ -1,5 +1,10 @@
+#![macro_use]
 use crate::{
     config::{CONFIGURATION, PROGRESS_PRINTER},
+    statistics::{
+        StatCommand::{self, AddError, AddStatus},
+        StatError::{Connection, Other, Redirection, Request, Timeout, UrlFormat},
+    },
     FeroxError, FeroxResult,
 };
 use console::{strip_ansi_codes, style, user_attended};
@@ -10,6 +15,7 @@ use rlimit::{getrlimit, setrlimit, Resource, Rlim};
 use std::convert::TryInto;
 use std::sync::{Arc, RwLock};
 use std::{fs, io};
+use tokio::sync::mpsc::UnboundedSender;
 
 /// Given the path to a file, open the file in append mode (create it if it doesn't exist) and
 /// return a reference to the file that is buffered and locked
@@ -160,6 +166,14 @@ pub fn ferox_print(msg: &str, bar: &ProgressBar) {
     }
 }
 
+#[macro_export]
+/// wrapper to improve code readability
+macro_rules! update_stat {
+    ($tx:expr, $value:expr) => {
+        $tx.send($value).unwrap_or_default();
+    };
+}
+
 /// Simple helper to generate a `Url`
 ///
 /// Errors during parsing `url` or joining `word` are propagated up the call stack
@@ -169,14 +183,16 @@ pub fn format_url(
     add_slash: bool,
     queries: &[(String, String)],
     extension: Option<&str>,
+    tx_stats: UnboundedSender<StatCommand>,
 ) -> FeroxResult<Url> {
     log::trace!(
-        "enter: format_url({}, {}, {}, {:?} {:?})",
+        "enter: format_url({}, {}, {}, {:?} {:?}, {:?})",
         url,
         word,
         add_slash,
         queries,
-        extension
+        extension,
+        tx_stats
     );
 
     if Url::parse(&word).is_ok() {
@@ -193,8 +209,9 @@ pub fn format_url(
         );
         log::warn!("{}", message);
 
-        let mut err = FeroxError::default();
-        err.message = message;
+        let err = FeroxError { message };
+
+        update_stat!(tx_stats, AddError(UrlFormat));
 
         log::trace!("exit: format_url -> {}", err);
         return Err(Box::new(err));
@@ -208,7 +225,7 @@ pub fn format_url(
     // the transforms that occur here will need to keep this in mind, i.e. add a slash to preserve
     // the current directory sent as part of the url
     let url = if word.is_empty() {
-        // v1.0.6: added during --extract-links feature inplementation to support creating urls
+        // v1.0.6: added during --extract-links feature implementation to support creating urls
         // that were extracted from response bodies, i.e. http://localhost/some/path/js/main.js
         url.to_string()
     } else if !url.ends_with('/') {
@@ -255,6 +272,7 @@ pub fn format_url(
             }
         }
         Err(e) => {
+            update_stat!(tx_stats, AddError(UrlFormat));
             log::trace!("exit: format_url -> {}", e);
             log::error!("Could not join {} with {}", word, base_url);
             Err(Box::new(e))
@@ -263,37 +281,63 @@ pub fn format_url(
 }
 
 /// Initiate request to the given `Url` using `Client`
-pub async fn make_request(client: &Client, url: &Url) -> FeroxResult<Response> {
-    log::trace!("enter: make_request(CONFIGURATION.Client, {})", url);
+pub async fn make_request(
+    client: &Client,
+    url: &Url,
+    tx_stats: UnboundedSender<StatCommand>,
+) -> FeroxResult<Response> {
+    log::trace!(
+        "enter: make_request(CONFIGURATION.Client, {}, {:?})",
+        url,
+        tx_stats
+    );
 
     match client.get(url.to_owned()).send().await {
-        Ok(resp) => {
-            log::trace!("exit: make_request -> {:?}", resp);
-            Ok(resp)
-        }
         Err(e) => {
+            let mut log_level = log::Level::Error;
+
             log::trace!("exit: make_request -> {}", e);
-            if e.to_string().contains("operation timed out") {
+            if e.is_timeout() {
                 // only warn for timeouts, while actual errors are still left as errors
-                log::warn!("Error while making request: {}", e);
+                log_level = log::Level::Warn;
+                update_stat!(tx_stats, AddError(Timeout));
             } else if e.is_redirect() {
                 if let Some(last_redirect) = e.url() {
                     // get where we were headed (last_redirect) and where we came from (url)
                     let fancy_message = format!("{} !=> {}", url, last_redirect);
 
                     let report = if let Some(msg_status) = e.status() {
+                        update_stat!(tx_stats, AddStatus(msg_status));
                         create_report_string(msg_status.as_str(), "-1", "-1", "-1", &fancy_message)
                     } else {
                         create_report_string("UNK", "-1", "-1", "-1", &fancy_message)
                     };
 
+                    update_stat!(tx_stats, AddError(Redirection));
+
                     ferox_print(&report, &PROGRESS_PRINTER)
                 };
+            } else if e.is_connect() {
+                update_stat!(tx_stats, AddError(Connection));
+            } else if e.is_request() {
+                update_stat!(tx_stats, AddError(Request));
             } else {
-                log::error!("Error while making request: {}", e);
+                update_stat!(tx_stats, AddError(Other));
             }
+
+            if matches!(log_level, log::Level::Error) {
+                log::error!("Error while making request: {}", e);
+            } else {
+                log::warn!("Error while making request: {}", e);
+            }
+
             Err(Box::new(e))
         }
+        Ok(resp) => {
+            log::trace!("exit: make_request -> {:?}", resp);
+            update_stat!(tx_stats, AddStatus(resp.status()));
+            Ok(resp)
+        }
     }
 }
 
@@ -391,6 +435,8 @@ pub fn normalize_url(url: &str) -> String {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::FeroxChannel;
+    use tokio::sync::mpsc;
 
     #[test]
     /// set_open_file_limit with a low requested limit succeeds
@@ -458,8 +504,9 @@ mod tests {
     #[test]
     /// base url + 1 word + no slash + no extension
     fn format_url_normal() {
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
         assert_eq!(
-            format_url("http://localhost", "stuff", false, &Vec::new(), None).unwrap(),
+            format_url("http://localhost", "stuff", false, &Vec::new(), None, tx).unwrap(),
             reqwest::Url::parse("http://localhost/stuff").unwrap()
         );
     }
@@ -467,8 +514,9 @@ mod tests {
     #[test]
     /// base url + no word + no slash + no extension
     fn format_url_no_word() {
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
         assert_eq!(
-            format_url("http://localhost", "", false, &Vec::new(), None).unwrap(),
+            format_url("http://localhost", "", false, &Vec::new(), None, tx).unwrap(),
             reqwest::Url::parse("http://localhost").unwrap()
         );
     }
@@ -476,13 +524,15 @@ mod tests {
     #[test]
     /// base url + word + no slash + no extension + queries
     fn format_url_joins_queries() {
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
         assert_eq!(
             format_url(
                 "http://localhost",
                 "lazer",
                 false,
                 &[(String::from("stuff"), String::from("things"))],
-                None
+                None,
+                tx
             )
             .unwrap(),
             reqwest::Url::parse("http://localhost/lazer?stuff=things").unwrap()
@@ -492,13 +542,15 @@ mod tests {
     #[test]
     /// base url + no word + no slash + no extension + queries
     fn format_url_without_word_joins_queries() {
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
         assert_eq!(
             format_url(
                 "http://localhost",
                 "",
                 false,
                 &[(String::from("stuff"), String::from("things"))],
-                None
+                None,
+                tx
             )
             .unwrap(),
             reqwest::Url::parse("http://localhost/?stuff=things").unwrap()
@@ -509,14 +561,16 @@ mod tests {
     #[should_panic]
     /// no base url is an error
     fn format_url_no_url() {
-        format_url("", "stuff", false, &Vec::new(), None).unwrap();
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+        format_url("", "stuff", false, &Vec::new(), None, tx).unwrap();
     }
 
     #[test]
     /// word prepended with slash is adjusted for correctness
     fn format_url_word_with_preslash() {
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
         assert_eq!(
-            format_url("http://localhost", "/stuff", false, &Vec::new(), None).unwrap(),
+            format_url("http://localhost", "/stuff", false, &Vec::new(), None, tx).unwrap(),
             reqwest::Url::parse("http://localhost/stuff").unwrap()
         );
     }
@@ -524,8 +578,9 @@ mod tests {
     #[test]
     /// word with appended slash allows the slash to persist
     fn format_url_word_with_postslash() {
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
         assert_eq!(
-            format_url("http://localhost", "stuff/", false, &Vec::new(), None).unwrap(),
+            format_url("http://localhost", "stuff/", false, &Vec::new(), None, tx).unwrap(),
             reqwest::Url::parse("http://localhost/stuff/").unwrap()
         );
     }
@@ -533,12 +588,14 @@ mod tests {
     #[test]
     /// word that is a fully formed url, should return an error
     fn format_url_word_that_is_a_url() {
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
         let url = format_url(
             "http://localhost",
             "http://schmocalhost",
             false,
             &Vec::new(),
             None,
+            tx,
         );
         assert!(url.is_err());
     }

tests/test_extractor.rs

@@ -252,11 +252,20 @@ fn extractor_finds_robots_txt_links_and_displays_files_or_scans_directories() {
         then.status(200).body("im a little teapot too"); // 22
     });
 
-    let mock_dir = srv.mock(|when, then| {
-        when.method(GET).path("/disallowed-subdir/LICENSE");
+    let mock_scanned_file = srv.mock(|when, then| {
+        when.method(GET).path("/misc/LICENSE");
         then.status(200).body("i too, am a container for tea"); // 29
     });
 
+    let mock_dir = srv.mock(|when, _| {
+        when.method(GET).path("/misc/");
+    });
+
+    let mock_disallowed = srv.mock(|when, then| {
+        when.method(GET).path("/disallowed-subdir");
+        then.status(404);
+    });
+
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
         .arg("--url")
@@ -264,6 +273,7 @@ fn extractor_finds_robots_txt_links_and_displays_files_or_scans_directories() {
         .arg("--wordlist")
         .arg(file.as_os_str())
         .arg("--extract-links")
+        .arg("-vvvv")
         .unwrap();
 
     cmd.assert().success().stdout(
@@ -272,7 +282,7 @@ fn extractor_finds_robots_txt_links_and_displays_files_or_scans_directories() {
             .and(predicate::str::contains("18c"))
             .and(predicate::str::contains("/misc/stupidfile.php"))
             .and(predicate::str::contains("22c"))
-            .and(predicate::str::contains("/disallowed-subdir/LICENSE"))
+            .and(predicate::str::contains("/misc/LICENSE"))
             .and(predicate::str::contains("29c"))
             .and(predicate::str::contains("200").count(3)),
     );
@@ -281,5 +291,58 @@ fn extractor_finds_robots_txt_links_and_displays_files_or_scans_directories() {
     assert_eq!(mock_dir.hits(), 1);
     assert_eq!(mock_two.hits(), 1);
     assert_eq!(mock_file.hits(), 1);
+    assert_eq!(mock_disallowed.hits(), 1);
+    assert_eq!(mock_scanned_file.hits(), 1);
     teardown_tmp_directory(tmp_dir);
 }
+
+#[test]
+/// send a request to a page that contains a link that contains a directory that returns a 403
+/// --extract-links should find the link and make recurse into the 403 directory, finding LICENSE
+fn extractor_recurses_into_403_directories() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200)
+            .body(&srv.url("'/homepage/assets/img/icons/handshake.svg'"));
+    });
+
+    let mock_two = srv.mock(|when, then| {
+        when.method(GET).path("/homepage/assets/img/icons/LICENSE");
+        then.status(200).body("that's just like, your opinion man");
+    });
+
+    let forbidden_dir = srv.mock(|when, then| {
+        when.method(GET).path("/homepage/assets/img/icons/");
+        then.status(403);
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--extract-links")
+        .arg("--depth") // need to go past default 4 directories
+        .arg("0")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .count(2)
+            .and(predicate::str::contains("1w")) // link in /LICENSE
+            .and(predicate::str::contains("34c")) // recursed LICENSE
+            .and(predicate::str::contains(
+                "/homepage/assets/img/icons/LICENSE",
+            )),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_two.hits(), 1);
+    assert_eq!(forbidden_dir.hits(), 1);
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}

tests/test_filters.rs

@@ -193,7 +193,7 @@ fn filters_size_should_filter_response() {
 #[test]
 /// create a FeroxResponse that should elicit a true from
 /// SimilarityFilter::should_filter_response
-fn filter_similar_should_filter_response() {
+fn filters_similar_should_filter_response() {
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(
         &["not-similar".to_string(), "similar".to_string()],

tests/test_scan_manager.rs

@@ -20,11 +20,11 @@ fn resume_scan_works() {
     // localhost:PORT/ <- complete
     // localhost:PORT/js <- will get scanned with /css and /stuff
     let complete_scan = format!(
-        r#"{{"id":"057016a14769414aac9a7a62707598cb","url":"{}","scan_type":"Directory","complete":true}}"#,
+        r#"{{"id":"057016a14769414aac9a7a62707598cb","url":"{}","scan_type":"Directory","status":"Complete"}}"#,
         srv.url("/")
     );
     let incomplete_scan = format!(
-        r#"{{"id":"400b2323a16f43468a04ffcbbeba34c6","url":"{}","scan_type":"Directory","complete":false}}"#,
+        r#"{{"id":"400b2323a16f43468a04ffcbbeba34c6","url":"{}","scan_type":"Directory","status":"NotStarted"}}"#,
         srv.url("/js")
     );
     let scans = format!(r#""scans":[{},{}]"#, complete_scan, incomplete_scan);

tests/test_scanner.rs

@@ -546,3 +546,53 @@ fn scanner_single_request_scan_with_regex_filtered_result() {
     assert_eq!(filtered_mock.hits(), 1);
     teardown_tmp_directory(tmp_dir);
 }
+
+#[test]
+/// send a request to a 403 directory, expect recursion to work into the 403
+fn scanner_recursion_works_with_403_directories() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) =
+        setup_tmp_directory(&["LICENSE".to_string(), "ignored/".to_string()], "wordlist").unwrap();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
+
+    let forbidden_dir = srv.mock(|when, then| {
+        when.method(GET).path("/ignored/");
+        then.status(403);
+    });
+
+    let found_anyway = srv.mock(|when, then| {
+        when.method(GET).path("/ignored/LICENSE");
+        then.status(200)
+            .body("this is a test\nThat rug really tied the room together");
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .count(2)
+            .and(predicate::str::contains("200").count(2))
+            .and(predicate::str::contains("403"))
+            .and(predicate::str::contains("53c"))
+            .and(predicate::str::contains("14c"))
+            .and(predicate::str::contains("0c"))
+            .and(predicate::str::contains("ignored").count(2))
+            .and(predicate::str::contains("/ignored/LICENSE")),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(found_anyway.hits(), 1);
+    assert_eq!(forbidden_dir.hits(), 1);
+
+    teardown_tmp_directory(tmp_dir);
+}