bumped version to 1.12.3

fixed url parsing issue when word starts with 2 or more /

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "feroxbuster"
-version = "1.11.1"
+version = "1.12.3"
 authors = ["Ben 'epi' Risher <epibar052@gmail.com>"]
 license = "MIT"
 edition = "2018"
@@ -22,11 +22,11 @@ lazy_static = "1.4"
 
 [dependencies]
 futures = { version = "0.3"}
-tokio = { version = "0.2", features = ["full"] }
-tokio-util = {version = "0.3", features = ["codec"]}
+tokio = { version = "1.0", features = ["full"] }
+tokio-util = {version = "0.6", features = ["codec"]}
 log = "0.4"
 env_logger = "0.8"
-reqwest = { version = "0.10", features = ["socks"] }
+reqwest = { version = "0.11", features = ["socks"] }
 clap = "2.33"
 lazy_static = "1.4"
 toml = "0.5"
@@ -41,7 +41,7 @@ regex = "1"
 crossterm = "0.19"
 rlimit = "0.5"
 ctrlc = "3.1"
-fuzzyhash = "0.2"
+fuzzyhash = "0.2.1"
 
 [dev-dependencies]
 tempfile = "3.1"

README.md

@@ -101,6 +101,7 @@ Enumeration.
     - [Enforce a Time Limit on Your Scan (new in `v1.10.0`)](#enforce-a-time-limit-on-your-scan-new-in-v1100)
     - [Extract Links from robots.txt (New in `v1.10.2`)](#extract-links-from-robotstxt-new-in-v1102)
     - [Filter Response by Similarity to A Given Page (fuzzy filter) (new in `v1.11.0`)](#filter-response-by-similarity-to-a-given-page-fuzzy-filter-new-in-v1110)
+    - [Cancel a Recursive Scan Interactively (new in `v1.12.0`)](#cancel-a-recursive-scan-interactively-new-in-v1120)
 - [Comparison w/ Similar Tools](#-comparison-w-similar-tools)
 - [Common Problems/Issues (FAQ)](#-common-problemsissues-faq)
     - [No file descriptors available](#no-file-descriptors-available)
@@ -596,9 +597,11 @@ is checked against a list of known filters and either displayed or not according
 
 ### Pause an Active Scan (new in `v1.4.0`)
 
-Scans can be paused and resumed by pressing the ENTER key (shown below)
+**NOTE**: [v1.12.0](#cancel-a-recursive-scan-interactively-new-in-v1120) added an interactive menu to the pause/resume
+functionality.  Active scans can still be paused, however, now you're presented with the option to cancel a scan instead
+of simply seeing a spinner.
 
-![pause-resume-demo](img/pause-resume-demo.gif)
+Scans can be paused and resumed by pressing the ENTER key (~~shown below~~, please see [v1.12.0](#cancel-a-recursive-scan-interactively-new-in-v1120)'s entry for the latest visual representation)
 
 ### Replay Responses to a Proxy based on Status Code (new in `v1.5.0`)
 
@@ -768,6 +771,27 @@ magnitude slower on requests/second).
   - The lack of accuracy with very small responses is considered a fair trade-off for not negatively impacting performance
 - Using a bunch of `--filter-similar-to` values **may** negatively impact performance
 
+### Cancel a Recursive Scan Interactively (new in `v1.12.0`)
+
+Version 1.12.0 expanded the pause/resume functionality introduced in [v1.4.0](#pause-an-active-scan-new-in-v140) by 
+adding an interactive menu from which currently running recursive scans can be cancelled, without affecting the overall scan.  Scans can still be paused indefinitely by pressing `ENTER`, however, the   
+
+Scans that are started via `-u` or passed in through `--stdin` cannot be cancelled, only scans found via `--extract-links` or recursion are eligible.
+
+Below is an example of the Scan Cancel Menu™.
+
+![cancel-menu](img/cancel-menu.png)
+
+Using the menu is pretty simple:
+- Press `ENTER` to view the menu
+- Choose a scan to cancel by entering its scan index (`1`)
+  - more than one scan can be selected by using a comma-separated list (`1,2,3` ... etc)
+- Confirm selections, after which all non-cancelled scans will resume
+
+Here is a short demonstration of cancelling two in-progress scans found via recursion.
+
+![cancel-scan](img/cancel-scan.gif)
+
 ## 🧐 Comparison w/ Similar Tools
 
 There are quite a few similar tools for forced browsing/content discovery. Burp Suite Pro, Dirb, Dirbuster, etc...
@@ -813,6 +837,7 @@ few of the use-cases in which feroxbuster may be a better fit:
 | maximum run time limit (`v1.10.0`)                                           | ✔ |   | ✔ |
 | use robots.txt to increase scan coverage (`v1.10.2`)                         | ✔ |   |   |
 | use example page's response to fuzzily filter similar pages  (`v1.11.0`)     | ✔ |   |   |
+| cancel a recursive scan interactively (`v1.12.0`)                            | ✔ |   |   |
 | **huge** number of other options                                             |   |   | ✔ |
 
 Of note, there's another written-in-rust content discovery tool, [rustbuster](https://github.com/phra/rustbuster). I

install-nix.sh

@@ -36,18 +36,23 @@ elif [[ "$(expr substr $(uname -s) 1 5)" == "Linux" ]]; then
         rm "${LIN64_ZIP}"
     fi
 
-    echo "[=] Installing Noto Emoji Font"
-    mkdir -p ~/.fonts
-    pushd ~/.fonts 2>&1 >/dev/null
+    if [[ -e ~/.fonts/NotoColorEmoji.ttf ]]; then
+       echo "[=] Found Noto Emoji Font, skipping install" 
+    else 
+        echo "[=] Installing Noto Emoji Font"
+        mkdir -p ~/.fonts
+        pushd ~/.fonts 2>&1 >/dev/null
 
-    curl -sLO "${EMOJI_URL}"
+        curl -sLO "${EMOJI_URL}"
 
-    fc-cache -f -v >/dev/null
+        fc-cache -f -v >/dev/null
 
-    popd 2>&1 >/dev/null
-    echo "[+] Noto Emoji Font installed"
+        popd 2>&1 >/dev/null
+        echo "[+] Noto Emoji Font installed"
+    fi
 fi
 
+
 chmod +x ./feroxbuster
 
 echo "[+] Installed feroxbuster version $(./feroxbuster -V)"

src/banner.rs

@@ -227,12 +227,12 @@ by Ben "epi" Risher {}                 ver: {}"#,
             &mut writer,
             "{}",
             format_banner_entry!(
-                format_emoji("🗑"),
+                format_emoji("💢"),
                 "Status Code Filters",
                 format!("[{}]", code_filters.join(", "))
             )
         )
-        .unwrap_or_default(); // 🗑
+        .unwrap_or_default(); // 💢
     }
 
     writeln!(
@@ -536,11 +536,10 @@ by Ben "epi" Risher {}                 ver: {}"#,
     // ⏯
     writeln!(
         &mut writer,
-        " {}   Press [{}] to {}|{} your scan",
-        format_emoji("⏯"),
+        " {}  Press [{}] to use the {}™",
+        format_emoji("🏁"),
         style("ENTER").yellow(),
-        style("pause").red(),
-        style("resume").green()
+        style("Scan Cancel Menu").bright().yellow(),
     )
     .unwrap_or_default();
 
@@ -559,7 +558,7 @@ mod tests {
     use tempfile::NamedTempFile;
     use tokio::sync::mpsc;
 
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// test to hit no execution of targets for loop in banner
     async fn banner_intialize_without_targets() {
         let config = Configuration::default();
@@ -568,7 +567,7 @@ mod tests {
         initialize(&[], &config, VERSION, stderr(), tx).await;
     }
 
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// test to hit no execution of statuscode for loop in banner
     async fn banner_intialize_without_status_codes() {
         let config = Configuration {
@@ -588,7 +587,7 @@ mod tests {
         .await;
     }
 
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// test to hit an empty config file
     async fn banner_intialize_without_config_file() {
         let config = Configuration {
@@ -608,7 +607,7 @@ mod tests {
         .await;
     }
 
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// test to hit an empty config file
     async fn banner_intialize_without_queries() {
         let config = Configuration {
@@ -628,7 +627,8 @@ mod tests {
         .await;
     }
 
-    #[tokio::test(core_threads = 1)]
+    #[ignore]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// test to show that a new version is available for download
     async fn banner_intialize_with_mismatched_version() {
         let config = Configuration::default();
@@ -649,7 +649,7 @@ mod tests {
         assert!(contents.contains("https://github.com/epi052/feroxbuster/releases/latest"));
     }
 
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// test that
     async fn banner_needs_update_returns_unknown_with_bad_url() {
         let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
@@ -658,7 +658,7 @@ mod tests {
         assert!(matches!(result, UpdateStatus::Unknown));
     }
 
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// test return value of good url to needs_update
     async fn banner_needs_update_returns_up_to_date() {
         let srv = MockServer::start();
@@ -676,7 +676,7 @@ mod tests {
         assert!(matches!(result, UpdateStatus::UpToDate));
     }
 
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// test return value of good url to needs_update that returns a newer version than current
     async fn banner_needs_update_returns_out_of_date() {
         let srv = MockServer::start();
@@ -694,7 +694,7 @@ mod tests {
         assert!(matches!(result, UpdateStatus::OutOfDate));
     }
 
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// test return value of good url that times out
     async fn banner_needs_update_returns_unknown_on_timeout() {
         let srv = MockServer::start();
@@ -714,7 +714,7 @@ mod tests {
         assert!(matches!(result, UpdateStatus::Unknown));
     }
 
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// test return value of good url with bad json response
     async fn banner_needs_update_returns_unknown_on_bad_json_response() {
         let srv = MockServer::start();
@@ -732,7 +732,7 @@ mod tests {
         assert!(matches!(result, UpdateStatus::Unknown));
     }
 
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// test return value of good url with json response that lacks the tag_name field
     async fn banner_needs_update_returns_unknown_on_json_without_correct_tag() {
         let srv = MockServer::start();

src/extractor.rs

@@ -52,20 +52,27 @@ fn get_sub_paths_from_path(path: &str) -> Vec<String> {
 
     let length = parts.len();
 
-    for _ in 0..length {
+    for i in 0..length {
         // iterate over all parts of the path
         if parts.is_empty() {
             // pop left us with an empty vector, we're done
             break;
         }
 
-        let possible_path = parts.join("/");
+        let mut possible_path = parts.join("/");
 
         if possible_path.is_empty() {
             // .join can result in an empty string, which we don't need, ignore
             continue;
         }
 
+        if i > 0 {
+            // this isn't the last index of the parts array
+            // ex: /buried/misc/stupidfile.php
+            // this block skips the file but sees all parent folders
+            possible_path = format!("{}/", possible_path);
+        }
+
         paths.push(possible_path); // good sub-path found
         parts.pop(); // use .pop() to remove the last part of the path and continue iteration
     }
@@ -355,10 +362,10 @@ mod tests {
         let path = "homepage/assets/img/icons/handshake.svg";
         let paths = get_sub_paths_from_path(&path);
         let expected = vec![
-            "homepage",
-            "homepage/assets",
-            "homepage/assets/img",
-            "homepage/assets/img/icons",
+            "homepage/",
+            "homepage/assets/",
+            "homepage/assets/img/",
+            "homepage/assets/img/icons/",
             "homepage/assets/img/icons/handshake.svg",
         ];
 
@@ -375,7 +382,7 @@ mod tests {
     fn extractor_get_sub_paths_from_path_with_enclosing_slashes() {
         let path = "/homepage/assets/";
         let paths = get_sub_paths_from_path(&path);
-        let expected = vec!["homepage", "homepage/assets"];
+        let expected = vec!["homepage/", "homepage/assets"];
 
         assert_eq!(paths.len(), expected.len());
         for expected_path in expected {
@@ -439,7 +446,7 @@ mod tests {
         assert!(links.is_empty());
     }
 
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// use make_request to generate a Response, and use the Response to test get_links;
     /// the response will contain an absolute path to a domain that is not part of the scanned
     /// domain; expect an empty set returned
@@ -470,7 +477,7 @@ mod tests {
         Ok(())
     }
 
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// test that /robots.txt is correctly requested given a base url (happy path)
     async fn request_robots_txt_with_and_without_proxy() {
         let srv = MockServer::start();

src/filters.rs

@@ -1,513 +0,0 @@
-use crate::config::CONFIGURATION;
-use crate::utils::get_url_path_length;
-use crate::{FeroxResponse, FeroxSerialize};
-use fuzzyhash::FuzzyHash;
-use regex::Regex;
-use std::any::Any;
-use std::fmt::Debug;
-
-// references:
-//   https://dev.to/magnusstrale/rust-trait-objects-in-a-vector-non-trivial-4co5
-//   https://stackoverflow.com/questions/25339603/how-to-test-for-equality-between-trait-objects
-
-/// FeroxFilter trait; represents different types of possible filters that can be applied to
-/// responses
-pub trait FeroxFilter: Debug + Send + Sync {
-    /// Determine whether or not this particular filter should be applied or not
-    fn should_filter_response(&self, response: &FeroxResponse) -> bool;
-
-    /// delegates to the FeroxFilter-implementing type which gives us the actual type of self
-    fn box_eq(&self, other: &dyn Any) -> bool;
-
-    /// gives us `other` as Any in box_eq
-    fn as_any(&self) -> &dyn Any;
-}
-
-/// implementation of PartialEq, necessary long-form due to "trait cannot be made into an object"
-/// error when attempting to derive PartialEq on the trait itself
-impl PartialEq for Box<dyn FeroxFilter> {
-    /// Perform a comparison of two implementors of the FeroxFilter trait
-    fn eq(&self, other: &Box<dyn FeroxFilter>) -> bool {
-        self.box_eq(other.as_any())
-    }
-}
-
-/// Data holder for two pieces of data needed when auto-filtering out wildcard responses
-///
-/// `dynamic` is the size of the response that will later be combined with the length
-/// of the path of the url requested and used to determine interesting pages from custom
-/// 404s where the requested url is reflected back in the response
-///
-/// `size` is size of the response that should be included with filters passed via runtime
-/// configuration and any static wildcard lengths.
-#[derive(Debug, Default, Clone, PartialEq)]
-pub struct WildcardFilter {
-    /// size of the response that will later be combined with the length of the path of the url
-    /// requested
-    pub dynamic: u64,
-
-    /// size of the response that should be included with filters passed via runtime configuration
-    pub size: u64,
-}
-
-/// implementation of FeroxFilter for WildcardFilter
-impl FeroxFilter for WildcardFilter {
-    /// Examine size, dynamic, and content_len to determine whether or not the response received
-    /// is a wildcard response and therefore should be filtered out
-    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
-        log::trace!("enter: should_filter_response({:?} {})", self, response);
-
-        // quick return if dont_filter is set
-        if CONFIGURATION.dont_filter {
-            // --dont-filter applies specifically to wildcard filters, it is not a 100% catch all
-            // for not filtering anything.  As such, it should live in the implementation of
-            // a wildcard filter
-            return false;
-        }
-
-        if self.size > 0 && self.size == response.content_length() {
-            // static wildcard size found during testing
-            // size isn't default, size equals response length, and auto-filter is on
-            log::debug!("static wildcard: filtered out {}", response.url());
-            log::trace!("exit: should_filter_response -> true");
-            return true;
-        }
-
-        if self.dynamic > 0 {
-            // dynamic wildcard offset found during testing
-
-            // I'm about to manually split this url path instead of using reqwest::Url's
-            // builtin parsing. The reason is that they call .split() on the url path
-            // except that I don't want an empty string taking up the last index in the
-            // event that the url ends with a forward slash.  It's ugly enough to be split
-            // into its own function for readability.
-            let url_len = get_url_path_length(&response.url());
-
-            if url_len + self.dynamic == response.content_length() {
-                log::debug!("dynamic wildcard: filtered out {}", response.url());
-                log::trace!("exit: should_filter_response -> true");
-                return true;
-            }
-        }
-        log::trace!("exit: should_filter_response -> false");
-        false
-    }
-
-    /// Compare one WildcardFilter to another
-    fn box_eq(&self, other: &dyn Any) -> bool {
-        other.downcast_ref::<Self>().map_or(false, |a| self == a)
-    }
-
-    /// Return self as Any for dynamic dispatch purposes
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-}
-
-/// Simple implementor of FeroxFilter; used to filter out status codes specified using
-/// -C|--filter-status
-#[derive(Default, Debug, PartialEq)]
-pub struct StatusCodeFilter {
-    /// Status code that should not be displayed to the user
-    pub filter_code: u16,
-}
-
-/// implementation of FeroxFilter for StatusCodeFilter
-impl FeroxFilter for StatusCodeFilter {
-    /// Check `filter_code` against what was passed in via -C|--filter-status
-    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
-        log::trace!("enter: should_filter_response({:?} {})", self, response);
-
-        if response.status().as_u16() == self.filter_code {
-            log::debug!(
-                "filtered out {} based on --filter-status of {}",
-                response.url(),
-                self.filter_code
-            );
-            log::trace!("exit: should_filter_response -> true");
-            return true;
-        }
-
-        log::trace!("exit: should_filter_response -> false");
-        false
-    }
-
-    /// Compare one StatusCodeFilter to another
-    fn box_eq(&self, other: &dyn Any) -> bool {
-        other.downcast_ref::<Self>().map_or(false, |a| self == a)
-    }
-
-    /// Return self as Any for dynamic dispatch purposes
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-}
-
-/// Simple implementor of FeroxFilter; used to filter out responses based on the number of lines
-/// in a Response body; specified using -N|--filter-lines
-#[derive(Default, Debug, PartialEq)]
-pub struct LinesFilter {
-    /// Number of lines in a Response's body that should be filtered
-    pub line_count: usize,
-}
-
-/// implementation of FeroxFilter for LinesFilter
-impl FeroxFilter for LinesFilter {
-    /// Check `line_count` against what was passed in via -N|--filter-lines
-    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
-        log::trace!("enter: should_filter_response({:?} {})", self, response);
-
-        let result = response.line_count() == self.line_count;
-
-        log::trace!("exit: should_filter_response -> {}", result);
-
-        result
-    }
-
-    /// Compare one LinesFilter to another
-    fn box_eq(&self, other: &dyn Any) -> bool {
-        other.downcast_ref::<Self>().map_or(false, |a| self == a)
-    }
-
-    /// Return self as Any for dynamic dispatch purposes
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-}
-
-/// Simple implementor of FeroxFilter; used to filter out responses based on the number of words
-/// in a Response body; specified using -W|--filter-words
-#[derive(Default, Debug, PartialEq)]
-pub struct WordsFilter {
-    /// Number of words in a Response's body that should be filtered
-    pub word_count: usize,
-}
-
-/// implementation of FeroxFilter for WordsFilter
-impl FeroxFilter for WordsFilter {
-    /// Check `word_count` against what was passed in via -W|--filter-words
-    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
-        log::trace!("enter: should_filter_response({:?} {})", self, response);
-
-        let result = response.word_count() == self.word_count;
-
-        log::trace!("exit: should_filter_response -> {}", result);
-
-        result
-    }
-
-    /// Compare one WordsFilter to another
-    fn box_eq(&self, other: &dyn Any) -> bool {
-        other.downcast_ref::<Self>().map_or(false, |a| self == a)
-    }
-
-    /// Return self as Any for dynamic dispatch purposes
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-}
-
-/// Simple implementor of FeroxFilter; used to filter out responses based on the length of a
-/// Response body; specified using -S|--filter-size
-#[derive(Default, Debug, PartialEq)]
-pub struct SizeFilter {
-    /// Overall length of a Response's body that should be filtered
-    pub content_length: u64,
-}
-
-/// implementation of FeroxFilter for SizeFilter
-impl FeroxFilter for SizeFilter {
-    /// Check `content_length` against what was passed in via -S|--filter-size
-    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
-        log::trace!("enter: should_filter_response({:?} {})", self, response);
-
-        let result = response.content_length() == self.content_length;
-
-        log::trace!("exit: should_filter_response -> {}", result);
-
-        result
-    }
-
-    /// Compare one SizeFilter to another
-    fn box_eq(&self, other: &dyn Any) -> bool {
-        other.downcast_ref::<Self>().map_or(false, |a| self == a)
-    }
-
-    /// Return self as Any for dynamic dispatch purposes
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-}
-
-/// Simple implementor of FeroxFilter; used to filter out responses based on a given regular
-/// expression; specified using -X|--filter-regex
-#[derive(Debug)]
-pub struct RegexFilter {
-    /// Regular expression to be applied to the response body for filtering, compiled
-    pub compiled: Regex,
-
-    /// Regular expression as passed in on the command line, not compiled
-    pub raw_string: String,
-}
-
-/// implementation of FeroxFilter for RegexFilter
-impl FeroxFilter for RegexFilter {
-    /// Check `expression` against the response body, if the expression matches, the response
-    /// should be filtered out
-    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
-        log::trace!("enter: should_filter_response({:?} {})", self, response);
-
-        let result = self.compiled.is_match(response.text());
-
-        log::trace!("exit: should_filter_response -> {}", result);
-
-        result
-    }
-
-    /// Compare one SizeFilter to another
-    fn box_eq(&self, other: &dyn Any) -> bool {
-        other.downcast_ref::<Self>().map_or(false, |a| self == a)
-    }
-
-    /// Return self as Any for dynamic dispatch purposes
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-}
-
-/// PartialEq implementation for RegexFilter
-impl PartialEq for RegexFilter {
-    /// Simple comparison of the raw string passed in via the command line
-    fn eq(&self, other: &RegexFilter) -> bool {
-        self.raw_string == other.raw_string
-    }
-}
-
-/// Simple implementor of FeroxFilter; used to filter out responses based on the similarity of a
-/// Response body with a known response; specified using --filter-similar-to
-#[derive(Default, Debug, PartialEq)]
-pub struct SimilarityFilter {
-    /// Response's body to be used for comparison for similarity
-    pub text: String,
-
-    /// Percentage of similarity at which a page is determined to be a near-duplicate of another
-    pub threshold: u32,
-}
-
-/// implementation of FeroxFilter for SimilarityFilter
-impl FeroxFilter for SimilarityFilter {
-    /// Check `FeroxResponse::text` against what was requested from the site passed in via
-    /// --filter-similar-to
-    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
-        let other = FuzzyHash::new(&response.text);
-
-        if let Ok(result) = FuzzyHash::compare(&self.text, &other.to_string()) {
-            return result >= self.threshold;
-        }
-
-        // couldn't hash the response, don't filter
-        log::warn!("Could not hash body from {}", response.as_str());
-        false
-    }
-
-    /// Compare one SimilarityFilter to another
-    fn box_eq(&self, other: &dyn Any) -> bool {
-        other.downcast_ref::<Self>().map_or(false, |a| self == a)
-    }
-
-    /// Return self as Any for dynamic dispatch purposes
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use reqwest::Url;
-
-    #[test]
-    /// just a simple test to increase code coverage by hitting as_any and the inner value
-    fn lines_filter_as_any() {
-        let filter = LinesFilter { line_count: 1 };
-
-        assert_eq!(filter.line_count, 1);
-        assert_eq!(
-            *filter.as_any().downcast_ref::<LinesFilter>().unwrap(),
-            filter
-        );
-    }
-
-    #[test]
-    /// just a simple test to increase code coverage by hitting as_any and the inner value
-    fn words_filter_as_any() {
-        let filter = WordsFilter { word_count: 1 };
-
-        assert_eq!(filter.word_count, 1);
-        assert_eq!(
-            *filter.as_any().downcast_ref::<WordsFilter>().unwrap(),
-            filter
-        );
-    }
-
-    #[test]
-    /// just a simple test to increase code coverage by hitting as_any and the inner value
-    fn size_filter_as_any() {
-        let filter = SizeFilter { content_length: 1 };
-
-        assert_eq!(filter.content_length, 1);
-        assert_eq!(
-            *filter.as_any().downcast_ref::<SizeFilter>().unwrap(),
-            filter
-        );
-    }
-
-    #[test]
-    /// just a simple test to increase code coverage by hitting as_any and the inner value
-    fn status_code_filter_as_any() {
-        let filter = StatusCodeFilter { filter_code: 200 };
-
-        assert_eq!(filter.filter_code, 200);
-        assert_eq!(
-            *filter.as_any().downcast_ref::<StatusCodeFilter>().unwrap(),
-            filter
-        );
-    }
-
-    #[test]
-    /// just a simple test to increase code coverage by hitting as_any and the inner value
-    fn regex_filter_as_any() {
-        let raw = r".*\.txt$";
-        let compiled = Regex::new(raw).unwrap();
-        let filter = RegexFilter {
-            compiled,
-            raw_string: raw.to_string(),
-        };
-
-        assert_eq!(filter.raw_string, r".*\.txt$");
-        assert_eq!(
-            *filter.as_any().downcast_ref::<RegexFilter>().unwrap(),
-            filter
-        );
-    }
-
-    #[test]
-    /// test should_filter on WilcardFilter where static logic matches
-    fn wildcard_should_filter_when_static_wildcard_found() {
-        let resp = FeroxResponse {
-            text: String::new(),
-            wildcard: true,
-            url: Url::parse("http://localhost").unwrap(),
-            content_length: 100,
-            word_count: 50,
-            line_count: 25,
-            headers: reqwest::header::HeaderMap::new(),
-            status: reqwest::StatusCode::OK,
-        };
-
-        let filter = WildcardFilter {
-            size: 100,
-            dynamic: 0,
-        };
-
-        assert!(filter.should_filter_response(&resp));
-    }
-
-    #[test]
-    /// test should_filter on WilcardFilter where dynamic logic matches
-    fn wildcard_should_filter_when_dynamic_wildcard_found() {
-        let resp = FeroxResponse {
-            text: String::new(),
-            wildcard: true,
-            url: Url::parse("http://localhost/stuff").unwrap(),
-            content_length: 100,
-            word_count: 50,
-            line_count: 25,
-            headers: reqwest::header::HeaderMap::new(),
-            status: reqwest::StatusCode::OK,
-        };
-
-        let filter = WildcardFilter {
-            size: 0,
-            dynamic: 95,
-        };
-
-        assert!(filter.should_filter_response(&resp));
-    }
-
-    #[test]
-    /// test should_filter on RegexFilter where regex matches body
-    fn regexfilter_should_filter_when_regex_matches_on_response_body() {
-        let resp = FeroxResponse {
-            text: String::from("im a body response hurr durr!"),
-            wildcard: false,
-            url: Url::parse("http://localhost/stuff").unwrap(),
-            content_length: 100,
-            word_count: 50,
-            line_count: 25,
-            headers: reqwest::header::HeaderMap::new(),
-            status: reqwest::StatusCode::OK,
-        };
-
-        let raw = r"response...rr";
-
-        let filter = RegexFilter {
-            raw_string: raw.to_string(),
-            compiled: Regex::new(raw).unwrap(),
-        };
-
-        assert!(filter.should_filter_response(&resp));
-    }
-
-    #[test]
-    /// a few simple tests for similarity filter
-    fn similarity_filter_is_accurate() {
-        let mut resp = FeroxResponse {
-            text: String::from("sitting"),
-            wildcard: false,
-            url: Url::parse("http://localhost/stuff").unwrap(),
-            content_length: 100,
-            word_count: 50,
-            line_count: 25,
-            headers: reqwest::header::HeaderMap::new(),
-            status: reqwest::StatusCode::OK,
-        };
-
-        let mut filter = SimilarityFilter {
-            text: FuzzyHash::new("kitten").to_string(),
-            threshold: 95,
-        };
-
-        // kitten/sitting is 57% similar, so a threshold of 95 should not be filtered
-        assert!(!filter.should_filter_response(&resp));
-
-        resp.text = String::new();
-        filter.text = String::new();
-        filter.threshold = 100;
-
-        // two empty strings are the same, however ssdeep doesn't accept empty strings, expect false
-        assert!(!filter.should_filter_response(&resp));
-
-        resp.text = String::from("some data to hash for the purposes of running a test");
-        filter.text =
-            FuzzyHash::new("some data to hash for the purposes of running a te").to_string();
-        filter.threshold = 17;
-
-        assert!(filter.should_filter_response(&resp));
-    }
-
-    #[test]
-    /// just a simple test to increase code coverage by hitting as_any and the inner value
-    fn similarity_filter_as_any() {
-        let filter = SimilarityFilter {
-            text: String::from("stuff"),
-            threshold: 95,
-        };
-
-        assert_eq!(filter.text, "stuff");
-        assert_eq!(
-            *filter.as_any().downcast_ref::<SimilarityFilter>().unwrap(),
-            filter
-        );
-    }
-}

src/filters/lines.rs

@@ -0,0 +1,33 @@
+use super::*;
+
+/// Simple implementor of FeroxFilter; used to filter out responses based on the number of lines
+/// in a Response body; specified using -N|--filter-lines
+#[derive(Default, Debug, PartialEq)]
+pub struct LinesFilter {
+    /// Number of lines in a Response's body that should be filtered
+    pub line_count: usize,
+}
+
+/// implementation of FeroxFilter for LinesFilter
+impl FeroxFilter for LinesFilter {
+    /// Check `line_count` against what was passed in via -N|--filter-lines
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        let result = response.line_count() == self.line_count;
+
+        log::trace!("exit: should_filter_response -> {}", result);
+
+        result
+    }
+
+    /// Compare one LinesFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/filters/mod.rs

@@ -0,0 +1,24 @@
+//! module containing all of feroxbuster's filters
+mod traits;
+mod wildcard;
+mod status_code;
+mod words;
+mod lines;
+mod size;
+mod regex;
+mod similarity;
+#[cfg(test)]
+mod tests;
+
+pub use self::lines::LinesFilter;
+pub use self::regex::RegexFilter;
+pub use self::similarity::SimilarityFilter;
+pub use self::size::SizeFilter;
+pub use self::status_code::StatusCodeFilter;
+pub use self::traits::FeroxFilter;
+pub use self::wildcard::WildcardFilter;
+pub use self::words::WordsFilter;
+
+use crate::{config::CONFIGURATION, utils::get_url_path_length, FeroxResponse, FeroxSerialize};
+use std::any::Any;
+use std::fmt::Debug;

src/filters/regex.rs

@@ -0,0 +1,46 @@
+use super::*;
+use ::regex::Regex;
+
+/// Simple implementor of FeroxFilter; used to filter out responses based on a given regular
+/// expression; specified using -X|--filter-regex
+#[derive(Debug)]
+pub struct RegexFilter {
+    /// Regular expression to be applied to the response body for filtering, compiled
+    pub compiled: Regex,
+
+    /// Regular expression as passed in on the command line, not compiled
+    pub raw_string: String,
+}
+
+/// implementation of FeroxFilter for RegexFilter
+impl FeroxFilter for RegexFilter {
+    /// Check `expression` against the response body, if the expression matches, the response
+    /// should be filtered out
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        let result = self.compiled.is_match(response.text());
+
+        log::trace!("exit: should_filter_response -> {}", result);
+
+        result
+    }
+
+    /// Compare one SizeFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
+
+/// PartialEq implementation for RegexFilter
+impl PartialEq for RegexFilter {
+    /// Simple comparison of the raw string passed in via the command line
+    fn eq(&self, other: &RegexFilter) -> bool {
+        self.raw_string == other.raw_string
+    }
+}

src/filters/similarity.rs

@@ -0,0 +1,40 @@
+use super::*;
+use fuzzyhash::FuzzyHash;
+
+/// Simple implementor of FeroxFilter; used to filter out responses based on the similarity of a
+/// Response body with a known response; specified using --filter-similar-to
+#[derive(Default, Debug, PartialEq)]
+pub struct SimilarityFilter {
+    /// Response's body to be used for comparison for similarity
+    pub text: String,
+
+    /// Percentage of similarity at which a page is determined to be a near-duplicate of another
+    pub threshold: u32,
+}
+
+/// implementation of FeroxFilter for SimilarityFilter
+impl FeroxFilter for SimilarityFilter {
+    /// Check `FeroxResponse::text` against what was requested from the site passed in via
+    /// --filter-similar-to
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        let other = FuzzyHash::new(&response.text);
+
+        if let Ok(result) = FuzzyHash::compare(&self.text, &other.to_string()) {
+            return result >= self.threshold;
+        }
+
+        // couldn't hash the response, don't filter
+        log::warn!("Could not hash body from {}", response.as_str());
+        false
+    }
+
+    /// Compare one SimilarityFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/filters/size.rs

@@ -0,0 +1,33 @@
+use super::*;
+
+/// Simple implementor of FeroxFilter; used to filter out responses based on the length of a
+/// Response body; specified using -S|--filter-size
+#[derive(Default, Debug, PartialEq)]
+pub struct SizeFilter {
+    /// Overall length of a Response's body that should be filtered
+    pub content_length: u64,
+}
+
+/// implementation of FeroxFilter for SizeFilter
+impl FeroxFilter for SizeFilter {
+    /// Check `content_length` against what was passed in via -S|--filter-size
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        let result = response.content_length() == self.content_length;
+
+        log::trace!("exit: should_filter_response -> {}", result);
+
+        result
+    }
+
+    /// Compare one SizeFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/filters/status_code.rs

@@ -0,0 +1,40 @@
+use super::*;
+
+/// Simple implementor of FeroxFilter; used to filter out status codes specified using
+/// -C|--filter-status
+#[derive(Default, Debug, PartialEq)]
+pub struct StatusCodeFilter {
+    /// Status code that should not be displayed to the user
+    pub filter_code: u16,
+}
+
+/// implementation of FeroxFilter for StatusCodeFilter
+impl FeroxFilter for StatusCodeFilter {
+    /// Check `filter_code` against what was passed in via -C|--filter-status
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        if response.status().as_u16() == self.filter_code {
+            log::debug!(
+                "filtered out {} based on --filter-status of {}",
+                response.url(),
+                self.filter_code
+            );
+            log::trace!("exit: should_filter_response -> true");
+            return true;
+        }
+
+        log::trace!("exit: should_filter_response -> false");
+        false
+    }
+
+    /// Compare one StatusCodeFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/filters/tests.rs

@@ -0,0 +1,188 @@
+use super::*;
+use ::fuzzyhash::FuzzyHash;
+use ::regex::Regex;
+use reqwest::Url;
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn lines_filter_as_any() {
+    let filter = LinesFilter { line_count: 1 };
+
+    assert_eq!(filter.line_count, 1);
+    assert_eq!(
+        *filter.as_any().downcast_ref::<LinesFilter>().unwrap(),
+        filter
+    );
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn words_filter_as_any() {
+    let filter = WordsFilter { word_count: 1 };
+
+    assert_eq!(filter.word_count, 1);
+    assert_eq!(
+        *filter.as_any().downcast_ref::<WordsFilter>().unwrap(),
+        filter
+    );
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn size_filter_as_any() {
+    let filter = SizeFilter { content_length: 1 };
+
+    assert_eq!(filter.content_length, 1);
+    assert_eq!(
+        *filter.as_any().downcast_ref::<SizeFilter>().unwrap(),
+        filter
+    );
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn status_code_filter_as_any() {
+    let filter = StatusCodeFilter { filter_code: 200 };
+
+    assert_eq!(filter.filter_code, 200);
+    assert_eq!(
+        *filter.as_any().downcast_ref::<StatusCodeFilter>().unwrap(),
+        filter
+    );
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn regex_filter_as_any() {
+    let raw = r".*\.txt$";
+    let compiled = Regex::new(raw).unwrap();
+    let filter = RegexFilter {
+        compiled,
+        raw_string: raw.to_string(),
+    };
+
+    assert_eq!(filter.raw_string, r".*\.txt$");
+    assert_eq!(
+        *filter.as_any().downcast_ref::<RegexFilter>().unwrap(),
+        filter
+    );
+}
+
+#[test]
+/// test should_filter on WilcardFilter where static logic matches
+fn wildcard_should_filter_when_static_wildcard_found() {
+    let resp = FeroxResponse {
+        text: String::new(),
+        wildcard: true,
+        url: Url::parse("http://localhost").unwrap(),
+        content_length: 100,
+        word_count: 50,
+        line_count: 25,
+        headers: reqwest::header::HeaderMap::new(),
+        status: reqwest::StatusCode::OK,
+    };
+
+    let filter = WildcardFilter {
+        size: 100,
+        dynamic: 0,
+    };
+
+    assert!(filter.should_filter_response(&resp));
+}
+
+#[test]
+/// test should_filter on WilcardFilter where dynamic logic matches
+fn wildcard_should_filter_when_dynamic_wildcard_found() {
+    let resp = FeroxResponse {
+        text: String::new(),
+        wildcard: true,
+        url: Url::parse("http://localhost/stuff").unwrap(),
+        content_length: 100,
+        word_count: 50,
+        line_count: 25,
+        headers: reqwest::header::HeaderMap::new(),
+        status: reqwest::StatusCode::OK,
+    };
+
+    let filter = WildcardFilter {
+        size: 0,
+        dynamic: 95,
+    };
+
+    assert!(filter.should_filter_response(&resp));
+}
+
+#[test]
+/// test should_filter on RegexFilter where regex matches body
+fn regexfilter_should_filter_when_regex_matches_on_response_body() {
+    let resp = FeroxResponse {
+        text: String::from("im a body response hurr durr!"),
+        wildcard: false,
+        url: Url::parse("http://localhost/stuff").unwrap(),
+        content_length: 100,
+        word_count: 50,
+        line_count: 25,
+        headers: reqwest::header::HeaderMap::new(),
+        status: reqwest::StatusCode::OK,
+    };
+
+    let raw = r"response...rr";
+
+    let filter = RegexFilter {
+        raw_string: raw.to_string(),
+        compiled: Regex::new(raw).unwrap(),
+    };
+
+    assert!(filter.should_filter_response(&resp));
+}
+
+#[test]
+/// a few simple tests for similarity filter
+fn similarity_filter_is_accurate() {
+    let mut resp = FeroxResponse {
+        text: String::from("sitting"),
+        wildcard: false,
+        url: Url::parse("http://localhost/stuff").unwrap(),
+        content_length: 100,
+        word_count: 50,
+        line_count: 25,
+        headers: reqwest::header::HeaderMap::new(),
+        status: reqwest::StatusCode::OK,
+    };
+
+    let mut filter = SimilarityFilter {
+        text: FuzzyHash::new("kitten").to_string(),
+        threshold: 95,
+    };
+
+    // kitten/sitting is 57% similar, so a threshold of 95 should not be filtered
+    assert!(!filter.should_filter_response(&resp));
+
+    resp.text = String::new();
+    filter.text = String::new();
+    filter.threshold = 100;
+
+    // two empty strings are the same, however ssdeep doesn't accept empty strings, expect false
+    assert!(!filter.should_filter_response(&resp));
+
+    resp.text = String::from("some data to hash for the purposes of running a test");
+    filter.text = FuzzyHash::new("some data to hash for the purposes of running a te").to_string();
+    filter.threshold = 17;
+
+    assert!(filter.should_filter_response(&resp));
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn similarity_filter_as_any() {
+    let filter = SimilarityFilter {
+        text: String::from("stuff"),
+        threshold: 95,
+    };
+
+    assert_eq!(filter.text, "stuff");
+    assert_eq!(
+        *filter.as_any().downcast_ref::<SimilarityFilter>().unwrap(),
+        filter
+    );
+}

src/filters/traits.rs

@@ -0,0 +1,27 @@
+use super::*;
+
+// references:
+//   https://dev.to/magnusstrale/rust-trait-objects-in-a-vector-non-trivial-4co5
+//   https://stackoverflow.com/questions/25339603/how-to-test-for-equality-between-trait-objects
+
+/// FeroxFilter trait; represents different types of possible filters that can be applied to
+/// responses
+pub trait FeroxFilter: Debug + Send + Sync {
+    /// Determine whether or not this particular filter should be applied or not
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool;
+
+    /// delegates to the FeroxFilter-implementing type which gives us the actual type of self
+    fn box_eq(&self, other: &dyn Any) -> bool;
+
+    /// gives us `other` as Any in box_eq
+    fn as_any(&self) -> &dyn Any;
+}
+
+/// implementation of PartialEq, necessary long-form due to "trait cannot be made into an object"
+/// error when attempting to derive PartialEq on the trait itself
+impl PartialEq for Box<dyn FeroxFilter> {
+    /// Perform a comparison of two implementors of the FeroxFilter trait
+    fn eq(&self, other: &Box<dyn FeroxFilter>) -> bool {
+        self.box_eq(other.as_any())
+    }
+}

src/filters/wildcard.rs

@@ -0,0 +1,73 @@
+use super::*;
+
+/// Data holder for two pieces of data needed when auto-filtering out wildcard responses
+///
+/// `dynamic` is the size of the response that will later be combined with the length
+/// of the path of the url requested and used to determine interesting pages from custom
+/// 404s where the requested url is reflected back in the response
+///
+/// `size` is size of the response that should be included with filters passed via runtime
+/// configuration and any static wildcard lengths.
+#[derive(Debug, Default, Clone, PartialEq)]
+pub struct WildcardFilter {
+    /// size of the response that will later be combined with the length of the path of the url
+    /// requested
+    pub dynamic: u64,
+
+    /// size of the response that should be included with filters passed via runtime configuration
+    pub size: u64,
+}
+
+/// implementation of FeroxFilter for WildcardFilter
+impl FeroxFilter for WildcardFilter {
+    /// Examine size, dynamic, and content_len to determine whether or not the response received
+    /// is a wildcard response and therefore should be filtered out
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        // quick return if dont_filter is set
+        if CONFIGURATION.dont_filter {
+            // --dont-filter applies specifically to wildcard filters, it is not a 100% catch all
+            // for not filtering anything.  As such, it should live in the implementation of
+            // a wildcard filter
+            return false;
+        }
+
+        if self.size > 0 && self.size == response.content_length() {
+            // static wildcard size found during testing
+            // size isn't default, size equals response length, and auto-filter is on
+            log::debug!("static wildcard: filtered out {}", response.url());
+            log::trace!("exit: should_filter_response -> true");
+            return true;
+        }
+
+        if self.dynamic > 0 {
+            // dynamic wildcard offset found during testing
+
+            // I'm about to manually split this url path instead of using reqwest::Url's
+            // builtin parsing. The reason is that they call .split() on the url path
+            // except that I don't want an empty string taking up the last index in the
+            // event that the url ends with a forward slash.  It's ugly enough to be split
+            // into its own function for readability.
+            let url_len = get_url_path_length(&response.url());
+
+            if url_len + self.dynamic == response.content_length() {
+                log::debug!("dynamic wildcard: filtered out {}", response.url());
+                log::trace!("exit: should_filter_response -> true");
+                return true;
+            }
+        }
+        log::trace!("exit: should_filter_response -> false");
+        false
+    }
+
+    /// Compare one WildcardFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/filters/words.rs

@@ -0,0 +1,33 @@
+use super::*;
+
+/// Simple implementor of FeroxFilter; used to filter out responses based on the number of words
+/// in a Response body; specified using -W|--filter-words
+#[derive(Default, Debug, PartialEq)]
+pub struct WordsFilter {
+    /// Number of words in a Response's body that should be filtered
+    pub word_count: usize,
+}
+
+/// implementation of FeroxFilter for WordsFilter
+impl FeroxFilter for WordsFilter {
+    /// Check `word_count` against what was passed in via -W|--filter-words
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        let result = response.word_count() == self.word_count;
+
+        log::trace!("exit: should_filter_response -> {}", result);
+
+        result
+    }
+
+    /// Compare one WordsFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/main.rs

@@ -2,12 +2,11 @@ use crossterm::event::{self, Event, KeyCode};
 use feroxbuster::{
     banner,
     config::{CONFIGURATION, PROGRESS_BAR, PROGRESS_PRINTER},
-    extractor::{extract_robots_txt, request_feroxresponse_from_new_link},
     heuristics, logger,
     progress::{add_bar, BarType},
     reporter,
-    scan_manager::{self, PAUSE_SCAN},
-    scanner::{self, scan_url, send_report, RESPONSES, SCANNED_URLS},
+    scan_manager::{self, ScanStatus, PAUSE_SCAN},
+    scanner::{self, scan_url, SCANNED_URLS},
     statistics::{
         self,
         StatCommand::{self, CreateBar, LoadStats, UpdateUsizeField},
@@ -16,7 +15,7 @@ use feroxbuster::{
     },
     update_stat,
     utils::{ferox_print, get_current_depth, module_colorizer, status_colorizer},
-    FeroxError, FeroxResponse, FeroxResult, FeroxSerialize, SLEEP_DURATION, VERSION,
+    FeroxError, FeroxResponse, FeroxResult, SLEEP_DURATION, VERSION,
 };
 #[cfg(not(target_os = "windows"))]
 use feroxbuster::{utils::set_open_file_limit, DEFAULT_OPEN_FILE_LIMIT};
@@ -31,6 +30,7 @@ use std::{
         atomic::{AtomicBool, Ordering},
         Arc,
     },
+    thread::sleep,
     time::Duration,
 };
 use tokio::{io, sync::mpsc::UnboundedSender, task::JoinHandle};
@@ -44,17 +44,20 @@ fn terminal_input_handler() {
     log::trace!("enter: terminal_input_handler");
 
     loop {
-        if event::poll(Duration::from_millis(SLEEP_DURATION)).unwrap_or(false) {
+        if PAUSE_SCAN.load(Ordering::Relaxed) {
+            // if the scan is already paused, we don't want this event poller fighting the user
+            // over stdin
+            sleep(Duration::from_millis(SLEEP_DURATION));
+        } else if event::poll(Duration::from_millis(SLEEP_DURATION)).unwrap_or(false) {
             // It's guaranteed that the `read()` won't block when the `poll()`
             // function returns `true`
 
             if let Ok(key_pressed) = event::read() {
+                // ignore any other keys
                 if key_pressed == Event::Key(KeyCode::Enter.into()) {
-                    // if the user presses Enter, toggle the value stored in PAUSE_SCAN
-                    // ignore any other keys
-                    let current = PAUSE_SCAN.load(Ordering::Acquire);
-
-                    PAUSE_SCAN.store(!current, Ordering::Release);
+                    // if the user presses Enter, set PAUSE_SCAN to true. The interactive menu
+                    // will be triggered and will handle setting PAUSE_SCAN to false
+                    PAUSE_SCAN.store(true, Ordering::Release);
                 }
             }
         } else {
@@ -106,7 +109,7 @@ fn get_unique_words_from_wordlist(path: &str) -> FeroxResult<Arc<HashSet<String>
 
 /// Determine whether it's a single url scan or urls are coming from stdin, then scan as needed
 async fn scan(
-    mut targets: Vec<String>,
+    targets: Vec<String>,
     stats: Arc<Stats>,
     tx_term: UnboundedSender<FeroxResponse>,
     tx_file: UnboundedSender<FeroxResponse>,
@@ -148,16 +151,12 @@ async fn scan(
     if CONFIGURATION.resumed {
         update_stat!(tx_stats, LoadStats(CONFIGURATION.resume_from.clone()));
 
-        if let Ok(responses) = RESPONSES.responses.read() {
-            for response in responses.iter() {
-                PROGRESS_PRINTER.println(response.as_str());
-            }
-        }
+        SCANNED_URLS.print_known_responses();
 
         if let Ok(scans) = SCANNED_URLS.scans.lock() {
             for scan in scans.iter() {
                 if let Ok(locked_scan) = scan.lock() {
-                    if locked_scan.complete {
+                    if matches!(locked_scan.status, ScanStatus::Complete) {
                         // these scans are complete, and just need to be shown to the user
                         let pb = add_bar(
                             &locked_scan.url,
@@ -171,42 +170,6 @@ async fn scan(
         }
     }
 
-    if CONFIGURATION.extract_links {
-        for target in targets.clone() {
-            // modifying the targets vector, so we can't have a reference to it while we borrow
-            // it as mutable; thus the clone
-            let robots_links = extract_robots_txt(&target, &CONFIGURATION, tx_stats.clone()).await;
-
-            for robot_link in robots_links {
-                // create a url based on the given command line options, continue on error
-                let ferox_response = match request_feroxresponse_from_new_link(
-                    &robot_link,
-                    tx_stats.clone(),
-                )
-                .await
-                {
-                    Some(resp) => resp,
-                    None => continue,
-                };
-
-                if ferox_response.is_file() {
-                    SCANNED_URLS.add_file_scan(&robot_link, stats.clone());
-                    send_report(tx_term.clone(), ferox_response);
-                } else {
-                    let (unknown, _) = SCANNED_URLS.add_directory_scan(&robot_link, stats.clone());
-
-                    if !unknown {
-                        // known directory; can skip (unlikely)
-                        continue;
-                    }
-
-                    // unknown directory; add to targets for scanning
-                    targets.push(robot_link);
-                }
-            }
-        }
-    }
-
     let mut tasks = vec![];
 
     for target in targets {
@@ -263,7 +226,7 @@ async fn get_targets() -> FeroxResult<Vec<String>> {
                 // SCANNED_URLS gets deserialized scans added to it at program start if --resume-from
                 // is used, so scans that aren't marked complete still need to be scanned
                 if let Ok(locked_scan) = scan.lock() {
-                    if locked_scan.complete {
+                    if matches!(locked_scan.status, ScanStatus::Complete) {
                         // this one's already done, ignore it
                         continue;
                     }
@@ -422,7 +385,7 @@ async fn wrapped_main() {
     )
     .await;
 
-    log::trace!("exit: main");
+    log::trace!("exit: wrapped_main");
 }
 
 /// Single cleanup function that handles all the necessary drops/finishes etc required to gracefully
@@ -446,7 +409,6 @@ async fn clean_up(
         stats_handle,
         save_output
     );
-
     drop(tx_term);
     log::trace!("dropped terminal output handler's transmitter");
 
@@ -478,7 +440,6 @@ async fn clean_up(
         log::trace!("done awaiting file output handler's receiver");
     }
 
-    log::trace!("tx_stats: {:?}", tx_stats);
     update_stat!(tx_stats, StatCommand::Exit); // send exit command and await the end of the future
     stats_handle.await.unwrap_or_default();
 
@@ -489,6 +450,8 @@ async fn clean_up(
     // the final trace messages above
     PROGRESS_PRINTER.finish();
 
+    drop(tx_stats);
+
     log::trace!("exit: clean_up");
 }
 
@@ -500,8 +463,13 @@ fn main() {
     #[cfg(not(target_os = "windows"))]
     set_open_file_limit(DEFAULT_OPEN_FILE_LIMIT);
 
-    if let Ok(mut runtime) = tokio::runtime::Runtime::new() {
+    if let Ok(runtime) = tokio::runtime::Builder::new_multi_thread()
+        .enable_all()
+        .build()
+    {
         let future = wrapped_main();
         runtime.block_on(future);
     }
+
+    log::trace!("exit: main");
 }

src/reporter.rs

@@ -14,8 +14,10 @@ use std::{
     io::Write,
     sync::{Arc, Once, RwLock},
 };
-use tokio::sync::mpsc::{self, UnboundedReceiver, UnboundedSender};
-use tokio::task::JoinHandle;
+use tokio::{
+    sync::mpsc::{self, UnboundedReceiver, UnboundedSender},
+    task::JoinHandle,
+};
 
 /// Singleton buffered file behind an Arc/RwLock; used for file writes from two locations:
 ///     - [logger::initialize](../logger/fn.initialize.html) (specifically a closure on the global logger instance)

src/scan_manager.rs

@@ -1,5 +1,5 @@
 use crate::{
-    config::{Configuration, CONFIGURATION, PROGRESS_PRINTER},
+    config::{Configuration, CONFIGURATION, PROGRESS_BAR, PROGRESS_PRINTER},
     parser::TIMESPEC_REGEX,
     progress::{add_bar, BarType},
     reporter::safe_file_write,
@@ -8,9 +8,8 @@ use crate::{
     utils::open_file,
     FeroxResponse, FeroxSerialize, SLEEP_DURATION,
 };
-use console::style;
-use indicatif::{ProgressBar, ProgressStyle};
-use lazy_static::lazy_static;
+use console::{measure_text_width, pad_str, style, Alignment, Term};
+use indicatif::{ProgressBar, ProgressDrawTarget};
 use serde::{
     ser::{SerializeSeq, SerializeStruct},
     Deserialize, Deserializer, Serialize, Serializer,
@@ -22,20 +21,16 @@ use std::{
     fmt,
     fs::File,
     io::BufReader,
-    io::{stderr, Write},
+    ops::Index,
     sync::atomic::{AtomicBool, AtomicUsize, Ordering},
     sync::{Arc, Mutex, RwLock},
+    thread::sleep,
     time::{SystemTime, UNIX_EPOCH},
 };
+use tokio::time::Duration;
 use tokio::{task::JoinHandle, time};
 use uuid::Uuid;
 
-lazy_static! {
-    /// A clock spinner protected with a RwLock to allow for a single thread to use at a time
-    // todo remove this when issue #107 is resolved
-    static ref SINGLE_SPINNER: RwLock<ProgressBar> = RwLock::new(get_single_spinner());
-}
-
 /// Single atomic number that gets incremented once, used to track first thread to interact with
 /// when pausing a scan
 static INTERACTIVE_BARRIER: AtomicUsize = AtomicUsize::new(0);
@@ -76,11 +71,11 @@ pub struct FeroxScan {
     /// Number of requests to populate the progress bar with
     pub num_requests: u64,
 
-    /// Whether or not this scan has completed
-    pub complete: bool,
+    /// Status of this scan
+    pub status: ScanStatus,
 
     /// The spawned tokio task performing this scan
-    pub task: Option<JoinHandle<()>>,
+    pub task: Option<Arc<JoinHandle<()>>>,
 
     /// The progress bar associated with this scan
     pub progress_bar: Option<ProgressBar>,
@@ -95,7 +90,7 @@ impl Default for FeroxScan {
         FeroxScan {
             id: new_id,
             task: None,
-            complete: false,
+            status: ScanStatus::default(),
             num_requests: 0,
             url: String::new(),
             progress_bar: None,
@@ -107,18 +102,18 @@ impl Default for FeroxScan {
 /// Implementation of FeroxScan
 impl FeroxScan {
     /// Stop a currently running scan
-    pub fn abort(&self) {
-        self.stop_progress_bar();
-
-        if let Some(_task) = &self.task {
-            // task.abort();  todo uncomment once upgraded to tokio 0.3 (issue #107)
+    pub fn abort(&mut self) {
+        if let Some(task) = &self.task {
+            self.status = ScanStatus::Cancelled;
+            self.stop_progress_bar();
+            task.abort();
         }
     }
 
     /// Simple helper to call .finish on the scan's progress bar
     fn stop_progress_bar(&self) {
         if let Some(pb) = &self.progress_bar {
-            pb.finish();
+            pb.finish_at_current_pos();
         }
     }
 
@@ -155,7 +150,7 @@ impl FeroxScan {
 
     /// Mark the scan as complete and stop the scan's progress bar
     pub fn finish(&mut self) {
-        self.complete = true;
+        self.status = ScanStatus::Complete;
         self.stop_progress_bar();
     }
 }
@@ -163,13 +158,14 @@ impl FeroxScan {
 /// Display implementation
 impl fmt::Display for FeroxScan {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        let complete = if self.complete {
-            style("complete").green()
-        } else {
-            style("incomplete").red()
+        let status = match self.status {
+            ScanStatus::NotStarted => style("not started").bright().blue(),
+            ScanStatus::Complete => style("complete").green(),
+            ScanStatus::Cancelled => style("cancelled").red(),
+            ScanStatus::Running => style("running").bright().yellow(),
         };
 
-        write!(f, "{:10} {}", complete, self.url)
+        write!(f, "{:12} {}", status, self.url)
     }
 }
 
@@ -192,7 +188,7 @@ impl Serialize for FeroxScan {
         state.serialize_field("id", &self.id)?;
         state.serialize_field("url", &self.url)?;
         state.serialize_field("scan_type", &self.scan_type)?;
-        state.serialize_field("complete", &self.complete)?;
+        state.serialize_field("status", &self.status)?;
         state.serialize_field("num_requests", &self.num_requests)?;
 
         state.end()
@@ -226,9 +222,15 @@ impl<'de> Deserialize<'de> for FeroxScan {
                         }
                     }
                 }
-                "complete" => {
-                    if let Some(complete) = value.as_bool() {
-                        scan.complete = complete;
+                "status" => {
+                    if let Some(status) = value.as_str() {
+                        scan.status = match status {
+                            "NotStarted" => ScanStatus::NotStarted,
+                            "Running" => ScanStatus::Running,
+                            "Complete" => ScanStatus::Complete,
+                            "Cancelled" => ScanStatus::Cancelled,
+                            _ => ScanStatus::default(),
+                        }
                     }
                 }
                 "url" => {
@@ -249,14 +251,175 @@ impl<'de> Deserialize<'de> for FeroxScan {
     }
 }
 
+#[derive(Debug, Serialize, Deserialize)]
+/// Simple enum to represent a scan's current status ([in]complete, cancelled)
+pub enum ScanStatus {
+    /// Scan hasn't started yet
+    NotStarted,
+
+    /// Scan finished normally
+    Complete,
+
+    /// Scan was cancelled by the user
+    Cancelled,
+
+    /// Scan has started, but hasn't finished, nor been cancelled
+    Running,
+}
+
+/// Default implementation for ScanStatus
+impl Default for ScanStatus {
+    /// Default variant for ScanStatus is NotStarted
+    fn default() -> Self {
+        Self::NotStarted
+    }
+}
+
+/// Interactive scan cancellation menu
+#[derive(Debug)]
+struct Menu {
+    /// character to use as visual separator of lines
+    separator: String,
+
+    /// name of menu
+    name: String,
+
+    /// header: name surrounded by separators
+    header: String,
+
+    /// instructions
+    instructions: String,
+
+    /// footer: instructions surrounded by separators
+    footer: String,
+
+    /// target for output
+    term: Term,
+}
+
+/// Implementation of Menu
+impl Menu {
+    /// Creates new Menu
+    fn new() -> Self {
+        let separator = "─".to_string();
+
+        let instructions = format!(
+            "Enter a {} list of indexes to {} (ex: 2,3)",
+            style("comma-separated").yellow(),
+            style("cancel").red(),
+        );
+
+        let name = format!(
+            "{} {} {}",
+            "💀",
+            style("Scan Cancel Menu").bright().yellow(),
+            "💀"
+        );
+
+        let longest = measure_text_width(&instructions).max(measure_text_width(&name));
+
+        let border = separator.repeat(longest);
+
+        let padded_name = pad_str(&name, longest, Alignment::Center, None);
+
+        let header = format!("{}\n{}\n{}", border, padded_name, border);
+        let footer = format!("{}\n{}\n{}", border, instructions, border);
+
+        Self {
+            separator,
+            name,
+            header,
+            instructions,
+            footer,
+            term: Term::stderr(),
+        }
+    }
+
+    /// print menu header
+    fn print_header(&self) {
+        self.println(&self.header);
+    }
+
+    /// print menu footer
+    fn print_footer(&self) {
+        self.println(&self.footer);
+    }
+
+    /// set PROGRESS_BAR bar target to hidden
+    fn hide_progress_bars(&self) {
+        PROGRESS_BAR.set_draw_target(ProgressDrawTarget::hidden());
+    }
+
+    /// set PROGRESS_BAR bar target to hidden
+    fn show_progress_bars(&self) {
+        PROGRESS_BAR.set_draw_target(ProgressDrawTarget::stdout());
+    }
+
+    /// Wrapper around console's Term::clear_screen and flush
+    fn clear_screen(&self) {
+        self.term.clear_screen().unwrap_or_default();
+        self.term.flush().unwrap_or_default();
+    }
+
+    /// Wrapper around console's Term::write_line
+    fn println(&self, msg: &str) {
+        self.term.write_line(msg).unwrap_or_default();
+    }
+
+    /// split a string into vec of usizes
+    fn split_to_nums(&self, line: &str) -> Vec<usize> {
+        line.split(',')
+            .map(|s| {
+                s.trim().to_string().parse::<usize>().unwrap_or_else(|e| {
+                    self.println(&format!("Found non-numeric input: {}", e));
+                    0
+                })
+            })
+            .filter(|m| *m != 0)
+            .collect()
+    }
+
+    /// get comma-separated list of scan indexes from the user
+    fn get_scans_from_user(&self) -> Option<Vec<usize>> {
+        if let Ok(line) = self.term.read_line() {
+            Some(self.split_to_nums(&line))
+        } else {
+            None
+        }
+    }
+
+    /// Given a url, confirm with user that we should cancel
+    fn confirm_cancellation(&self, url: &str) -> char {
+        self.println(&format!(
+            "You sure you wanna cancel this scan: {}? [Y/n]",
+            url
+        ));
+
+        self.term.read_char().unwrap_or('n')
+    }
+}
+
+/// Default implementation for Menu
+impl Default for Menu {
+    /// return Menu::new as default
+    fn default() -> Menu {
+        Menu::new()
+    }
+}
+
 /// Container around a locked hashset of `FeroxScan`s, adds wrappers for insertion and searching
 #[derive(Debug, Default)]
 pub struct FeroxScans {
     /// Internal structure: locked hashset of `FeroxScan`s
     pub scans: Mutex<Vec<Arc<Mutex<FeroxScan>>>>,
+
+    /// menu used for providing a way for users to cancel a scan
+    menu: Menu,
 }
 
 /// Serialize implementation for FeroxScans
+///
+/// purposefully skips menu attribute
 impl Serialize for FeroxScans {
     /// Function that handles serialization of FeroxScans
     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
@@ -362,20 +525,82 @@ impl FeroxScans {
         if let Ok(scans) = self.scans.lock() {
             for (i, scan) in scans.iter().enumerate() {
                 if let Ok(unlocked_scan) = scan.lock() {
-                    match unlocked_scan.scan_type {
-                        ScanType::Directory => {
-                            PROGRESS_PRINTER.println(format!("{:3}: {}", i, unlocked_scan));
-                        }
-                        ScanType::File => {
-                            // we're only interested in displaying directory scans, as those are
-                            // the only ones that make sense to be stopped
-                        }
+                    if unlocked_scan.task.is_none() {
+                        // no JoinHandle associated with this FeroxScan, meaning it was an original
+                        // target passed in via either -u or --stdin
+                        continue;
+                    }
+
+                    if matches!(unlocked_scan.scan_type, ScanType::Directory) {
+                        // we're only interested in displaying directory scans, as those are
+                        // the only ones that make sense to be stopped
+                        let scan_msg = format!("{:3}: {}", i, unlocked_scan);
+                        self.menu.println(&scan_msg);
                     }
                 }
             }
         }
     }
 
+    /// Given a list of indexes, cancel their associated FeroxScans
+    fn cancel_scans(&self, indexes: Vec<usize>) {
+        let menu_pause_duration = Duration::from_millis(SLEEP_DURATION);
+
+        for num in indexes {
+            if let Ok(u_scans) = self.scans.lock() {
+                // check if number provided is out of range
+                if num >= u_scans.len() {
+                    // usize can't be negative, just need to handle exceeding bounds
+                    self.menu
+                        .println(&format!("The number {} is not a valid choice.", num));
+                    sleep(menu_pause_duration);
+                    continue;
+                }
+
+                let selected = u_scans.index(num).clone();
+
+                if let Ok(mut ferox_scan) = selected.lock() {
+                    let input = self.menu.confirm_cancellation(&ferox_scan.url);
+
+                    if input == 'y' || input == '\n' {
+                        self.menu
+                            .println(&format!("Stopping {}...", ferox_scan.url));
+                        ferox_scan.abort();
+                    } else {
+                        self.menu.println("Ok, doing nothing...");
+                    }
+                }
+
+                sleep(menu_pause_duration);
+            }
+        }
+    }
+
+    /// CLI menu that allows for interactive cancellation of recursed-into directories
+    async fn interactive_menu(&self) {
+        self.menu.hide_progress_bars();
+        self.menu.clear_screen();
+        self.menu.print_header();
+        self.display_scans();
+        self.menu.print_footer();
+
+        if let Some(input) = self.menu.get_scans_from_user() {
+            self.cancel_scans(input);
+        }
+
+        self.menu.clear_screen();
+        self.menu.show_progress_bars();
+    }
+
+    /// prints all known responses that the scanner has already seen
+    pub fn print_known_responses(&self) {
+        if let Ok(responses) = RESPONSES.responses.read() {
+            for response in responses.iter() {
+                PROGRESS_PRINTER.println(response.as_str());
+            }
+        }
+    }
+
     /// Forced the calling thread into a busy loop
     ///
     /// Every `SLEEP_DURATION` milliseconds, the function examines the result stored in `PAUSE_SCAN`
@@ -389,38 +614,16 @@ impl FeroxScans {
         // concurrent scans rose when SLEEP_DURATION was set to 500, using that as the default for now
         let mut interval = time::interval(time::Duration::from_millis(SLEEP_DURATION));
 
-        // ignore any error returned
-        let _ = stderr().flush();
-
         if INTERACTIVE_BARRIER.load(Ordering::Relaxed) == 0 {
             INTERACTIVE_BARRIER.fetch_add(1, Ordering::Relaxed);
 
             if get_user_input {
-                self.display_scans();
-
-                let mut user_input = String::new();
-                std::io::stdin().read_line(&mut user_input).unwrap();
-                // todo (issue #107) actual logic for parsing user input in a way that allows for
-                // calling .abort on the scan retrieved based on the input
+                self.interactive_menu().await;
+                PAUSE_SCAN.store(false, Ordering::Relaxed);
+                self.print_known_responses();
             }
         }
 
-        if SINGLE_SPINNER.read().unwrap().is_finished() {
-            // todo remove this when issue #107 is resolved
-
-            // in order to not leave draw artifacts laying around in the terminal, we call
-            // finish_and_clear on the progress bar when resuming scans. For this reason, we need to
-            // check if the spinner is finished, and repopulate the RwLock with a new spinner if
-            // necessary
-            if let Ok(mut guard) = SINGLE_SPINNER.write() {
-                *guard = get_single_spinner();
-            }
-        }
-
-        if let Ok(spinner) = SINGLE_SPINNER.write() {
-            spinner.enable_steady_tick(120);
-        }
-
         loop {
             // first tick happens immediately, all others wait the specified duration
             interval.tick().await;
@@ -432,13 +635,6 @@ impl FeroxScans {
                     INTERACTIVE_BARRIER.fetch_sub(1, Ordering::Relaxed);
                 }
 
-                if let Ok(spinner) = SINGLE_SPINNER.write() {
-                    // todo remove this when issue #107 is resolved
-                    spinner.finish_and_clear();
-                }
-
-                let _ = stderr().flush();
-
                 log::trace!("exit: pause_scan");
                 return;
             }
@@ -501,26 +697,6 @@ impl FeroxScans {
     }
 }
 
-/// Return a clock spinner, used when scans are paused
-// todo remove this when issue #107 is resolved
-fn get_single_spinner() -> ProgressBar {
-    log::trace!("enter: get_single_spinner");
-
-    let spinner = ProgressBar::new_spinner().with_style(
-        ProgressStyle::default_spinner()
-            .tick_strings(&[
-                "🕛", "🕐", "🕑", "🕒", "🕓", "🕔", "🕕", "🕖", "🕗", "🕘", "🕙", "🕚",
-            ])
-            .template(&format!(
-                "\t-= All Scans {{spinner}} {} =-",
-                style("Paused").red()
-            )),
-    );
-
-    log::trace!("exit: get_single_spinner -> {:?}", spinner);
-    spinner
-}
-
 /// Container around a locked vector of `FeroxResponse`s, adds wrappers for insertion and search
 #[derive(Debug, Default)]
 pub struct FeroxResponses {
@@ -640,7 +816,7 @@ pub async fn start_max_time_thread(time_spec: &str, stats: Arc<Stats>) {
             length_in_secs
         );
 
-        time::delay_for(time::Duration::new(length_in_secs, 0)).await;
+        time::sleep(time::Duration::new(length_in_secs, 0)).await;
 
         log::trace!("exit: start_max_time_thread");
 
@@ -785,15 +961,7 @@ mod tests {
         }
     }
 
-    #[test]
-    /// test that get_single_spinner returns the correct spinner
-    // todo remove this when issue #107 is resolved
-    fn scanner_get_single_spinner_returns_spinner() {
-        let spinner = get_single_spinner();
-        assert!(!spinner.is_finished());
-    }
-
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// tests that pause_scan pauses execution and releases execution when PAUSE_SCAN is toggled
     /// the spinner used during the test has had .finish_and_clear called on it, meaning that
     /// a new one will be created, taking the if branch within the function
@@ -806,7 +974,7 @@ mod tests {
         let expected = time::Duration::from_secs(2);
 
         tokio::spawn(async move {
-            time::delay_for(expected).await;
+            time::sleep(expected).await;
             PAUSE_SCAN.store(false, Ordering::Relaxed);
         });
 
@@ -843,8 +1011,8 @@ mod tests {
     }
 
     #[test]
-    /// abort should call stop_progress_bar, marking it as finished
-    fn abort_stops_progress_bar() {
+    /// stop_progress_bar should stop the progress bar
+    fn stop_progress_bar_stops_bar() {
         let pb = ProgressBar::new(1);
         let url = "http://unknown_url/";
 
@@ -860,7 +1028,7 @@ mod tests {
             false
         );
 
-        scan.lock().unwrap().abort();
+        scan.lock().unwrap().stop_progress_bar();
 
         assert_eq!(
             scan.lock()
@@ -889,9 +1057,9 @@ mod tests {
         assert_eq!(result, false);
     }
 
-    #[test]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// just increasing coverage, no real expectations
-    fn call_display_scans() {
+    async fn call_display_scans() {
         let urls = FeroxScans::default();
         let pb = ProgressBar::new(1);
         let pb_two = ProgressBar::new(2);
@@ -900,9 +1068,15 @@ mod tests {
         let scan = FeroxScan::new(url, ScanType::Directory, pb.length(), Some(pb));
         let scan_two = FeroxScan::new(url_two, ScanType::Directory, pb_two.length(), Some(pb_two));
 
-        scan_two.lock().unwrap().finish(); // one complete, one incomplete
+        if let Ok(mut s2) = scan_two.lock() {
+            s2.finish(); // one complete, one incomplete
+            s2.task = Some(Arc::new(tokio::spawn(async move {
+                sleep(Duration::from_millis(SLEEP_DURATION));
+            })));
+        }
 
         assert_eq!(urls.insert(scan), true);
+        assert_eq!(urls.insert(scan_two), true);
 
         urls.display_scans();
     }
@@ -938,11 +1112,13 @@ mod tests {
     /// given a JSON entry representing a FeroxScan, test that it deserializes into the proper type
     /// with the right attributes
     fn ferox_scan_deserialize() {
-        let fs_json = r#"{"id":"057016a14769414aac9a7a62707598cb","url":"https://spiritanimal.com","scan_type":"Directory","complete":true}"#;
-        let fs_json_two = r#"{"id":"057016a14769414aac9a7a62707598cb","url":"https://spiritanimal.com","scan_type":"Not Correct","complete":true}"#;
+        let fs_json = r#"{"id":"057016a14769414aac9a7a62707598cb","url":"https://spiritanimal.com","scan_type":"Directory","status":"Complete"}"#;
+        let fs_json_two = r#"{"id":"057016a14769414aac9a7a62707598cb","url":"https://spiritanimal.com","scan_type":"Not Correct","status":"Cancelled"}"#;
+        let fs_json_three = r#"{"id":"057016a14769414aac9a7a62707598cb","url":"https://spiritanimal.com","scan_type":"Not Correct","status":"","num_requests":42}"#;
 
         let fs: FeroxScan = serde_json::from_str(fs_json).unwrap();
         let fs_two: FeroxScan = serde_json::from_str(fs_json_two).unwrap();
+        let fs_three: FeroxScan = serde_json::from_str(fs_json_three).unwrap();
         assert_eq!(fs.url, "https://spiritanimal.com");
 
         match fs.scan_type {
@@ -964,7 +1140,10 @@ mod tests {
                 panic!();
             }
         }
-        assert_eq!(fs.complete, true);
+        assert!(matches!(fs.status, ScanStatus::Complete));
+        assert!(matches!(fs_two.status, ScanStatus::Cancelled));
+        assert!(matches!(fs_three.status, ScanStatus::NotStarted));
+        assert_eq!(fs_three.num_requests, 42);
         assert_eq!(fs.id, "057016a14769414aac9a7a62707598cb");
     }
 
@@ -973,7 +1152,7 @@ mod tests {
     fn ferox_scan_serialize() {
         let fs = FeroxScan::new("https://spiritanimal.com", ScanType::Directory, 0, None);
         let fs_json = format!(
-            r#"{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","complete":false,"num_requests":0}}"#,
+            r#"{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","status":"NotStarted","num_requests":0}}"#,
             fs.lock().unwrap().id
         );
         assert_eq!(
@@ -988,7 +1167,7 @@ mod tests {
         let ferox_scan = FeroxScan::new("https://spiritanimal.com", ScanType::Directory, 0, None);
         let ferox_scans = FeroxScans::default();
         let ferox_scans_json = format!(
-            r#"[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","complete":false,"num_requests":0}}]"#,
+            r#"[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","status":"NotStarted","num_requests":0}}]"#,
             ferox_scan.lock().unwrap().id
         );
         ferox_scans.scans.lock().unwrap().push(ferox_scan);
@@ -1068,7 +1247,7 @@ mod tests {
 
         let json_state = ferox_state.as_json();
         let expected = format!(
-            r#"{{"scans":[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","complete":false,"num_requests":0}}],"config":{{"type":"configuration","wordlist":"/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt","config":"","proxy":"","replay_proxy":"","target_url":"","status_codes":[200,204,301,302,307,308,401,403,405],"replay_codes":[200,204,301,302,307,308,401,403,405],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"quiet":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/{}","redirects":false,"insecure":false,"extensions":[],"headers":{{}},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":false,"depth":4,"scan_limit":0,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false,"resumed":false,"resume_from":"","save_state":false,"time_limit":"","filter_similar":[]}},"responses":[{{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{{"server":"nginx/1.16.1"}}}}]"#,
+            r#"{{"scans":[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","status":"NotStarted","num_requests":0}}],"config":{{"type":"configuration","wordlist":"/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt","config":"","proxy":"","replay_proxy":"","target_url":"","status_codes":[200,204,301,302,307,308,401,403,405],"replay_codes":[200,204,301,302,307,308,401,403,405],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"quiet":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/{}","redirects":false,"insecure":false,"extensions":[],"headers":{{}},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":false,"depth":4,"scan_limit":0,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false,"resumed":false,"resume_from":"","save_state":false,"time_limit":"","filter_similar":[]}},"responses":[{{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{{"server":"nginx/1.16.1"}}}}]"#,
             saved_id, VERSION
         );
         println!("{}\n{}", expected, json_state);
@@ -1076,7 +1255,7 @@ mod tests {
     }
 
     #[should_panic]
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// call start_max_time_thread with a valid timespec, expect a panic, but only after a certain
     /// number of seconds
     async fn start_max_time_thread_panics_after_delay() {
@@ -1089,7 +1268,7 @@ mod tests {
         assert!(now.elapsed() > delay);
     }
 
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// call start_max_time_thread with a timespec that's too large to be parsed correctly, expect
     /// immediate return and no panic, as the sigint handler is never called
     async fn start_max_time_thread_returns_immediately_with_too_large_input() {
@@ -1102,4 +1281,86 @@ mod tests {
 
         assert!(now.elapsed() < delay); // assuming function call will take less than 1second
     }
+
+    #[test]
+    /// coverage for FeroxScan's Display implementation
+    fn feroxscan_display() {
+        let mut scan = FeroxScan {
+            id: "".to_string(),
+            url: String::from("http://localhost"),
+            scan_type: Default::default(),
+            num_requests: 0,
+            status: Default::default(),
+            task: None,
+            progress_bar: None,
+        };
+
+        let not_started = format!("{}", scan);
+
+        assert!(predicate::str::contains("not started")
+            .and(predicate::str::contains("localhost"))
+            .eval(&not_started));
+
+        scan.status = ScanStatus::Complete;
+        let complete = format!("{}", scan);
+        assert!(predicate::str::contains("complete")
+            .and(predicate::str::contains("localhost"))
+            .eval(&complete));
+
+        scan.status = ScanStatus::Cancelled;
+        let cancelled = format!("{}", scan);
+        assert!(predicate::str::contains("cancelled")
+            .and(predicate::str::contains("localhost"))
+            .eval(&cancelled));
+
+        scan.status = ScanStatus::Running;
+        let running = format!("{}", scan);
+        assert!(predicate::str::contains("running")
+            .and(predicate::str::contains("localhost"))
+            .eval(&running));
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// call FeroxScan::abort, ensure status becomes cancelled
+    async fn ferox_scan_abort() {
+        let mut scan = FeroxScan {
+            id: "".to_string(),
+            url: String::from("http://localhost"),
+            scan_type: Default::default(),
+            num_requests: 0,
+            status: ScanStatus::Running,
+            task: Some(Arc::new(tokio::spawn(async move {
+                sleep(Duration::from_millis(SLEEP_DURATION * 2));
+            }))),
+            progress_bar: None,
+        };
+
+        scan.abort();
+
+        assert!(matches!(scan.status, ScanStatus::Cancelled));
+    }
+
+    #[test]
+    /// call a few menu functions for coverage's sake
+    ///
+    /// there's not a trivial way to test these programmatically (at least i'm too lazy rn to do it)
+    /// and their correctness can be verified easily manually; just calling for now
+    fn menu_print_header_and_footer() {
+        let menu = Menu::new();
+        menu.clear_screen();
+        menu.print_header();
+        menu.print_footer();
+        menu.hide_progress_bars();
+        menu.show_progress_bars();
+    }
+
+    #[test]
+    /// ensure spaces are trimmed and numbers are returned from split_to_nums
+    fn split_to_nums_is_correct() {
+        let menu = Menu::new();
+
+        let nums = menu.split_to_nums("1, 3,      4");
+
+        assert_eq!(nums, vec![1, 3, 4]);
+    }
 }

src/scanner.rs

@@ -1,16 +1,16 @@
-use crate::statistics::Stats;
 use crate::{
     config::{Configuration, CONFIGURATION},
-    extractor::{get_links, request_feroxresponse_from_new_link},
+    extractor::{extract_robots_txt, get_links, request_feroxresponse_from_new_link},
     filters::{
         FeroxFilter, LinesFilter, RegexFilter, SimilarityFilter, SizeFilter, StatusCodeFilter,
         WildcardFilter, WordsFilter,
     },
     heuristics,
-    scan_manager::{FeroxResponses, FeroxScans, PAUSE_SCAN},
+    scan_manager::{FeroxResponses, FeroxScans, ScanStatus, PAUSE_SCAN},
     statistics::{
         StatCommand::{self, UpdateF64Field, UpdateUsizeField},
         StatField::{DirScanTimes, ExpectedPerScan, TotalScans, WildcardsFiltered},
+        Stats,
     },
     utils::{format_url, get_current_depth, make_request},
     FeroxChannel, FeroxResponse, SIMILARITY_THRESHOLD,
@@ -22,7 +22,7 @@ use futures::{
 use fuzzyhash::FuzzyHash;
 use lazy_static::lazy_static;
 use regex::Regex;
-use reqwest::Url;
+use reqwest::{StatusCode, Url};
 #[cfg(not(test))]
 use std::process::exit;
 use std::{
@@ -60,6 +60,8 @@ lazy_static! {
 
     /// Bounded semaphore used as a barrier to limit concurrent scans
     static ref SCAN_LIMITER: Semaphore = Semaphore::new(CONFIGURATION.scan_limit);
+
+
 }
 
 /// Adds the given FeroxFilter to the given list of FeroxFilter implementors
@@ -109,7 +111,7 @@ fn spawn_recursion_handler(
     tx_term: UnboundedSender<FeroxResponse>,
     tx_file: UnboundedSender<FeroxResponse>,
     tx_stats: UnboundedSender<StatCommand>,
-) -> BoxFuture<'static, Vec<JoinHandle<()>>> {
+) -> BoxFuture<'static, Vec<Arc<JoinHandle<()>>>> {
     log::trace!(
         "enter: spawn_recursion_handler({:?}, wordlist[{} words...], {}, {:?}, {:?}, {:?}, {:?})",
         recursion_channel,
@@ -125,7 +127,7 @@ fn spawn_recursion_handler(
         let mut scans = vec![];
 
         while let Some(resp) = recursion_channel.recv().await {
-            let (unknown, _) = SCANNED_URLS.add_directory_scan(&resp, stats.clone());
+            let (unknown, scan) = SCANNED_URLS.add_directory_scan(&resp, stats.clone());
 
             if !unknown {
                 // not unknown, i.e. we've seen the url before and don't need to scan again
@@ -156,7 +158,13 @@ fn spawn_recursion_handler(
                 .await
             });
 
-            scans.push(future);
+            let shared_task = Arc::new(future);
+
+            if let Ok(mut u_scan) = scan.lock() {
+                u_scan.task = Some(shared_task.clone());
+            }
+
+            scans.push(shared_task);
         }
         scans
     }
@@ -252,8 +260,9 @@ fn response_is_directory(response: &FeroxResponse) -> bool {
                 return false;
             }
         }
-    } else if response.status().is_success() {
-        // status code is 2xx, need to check if it ends in /
+    } else if response.status().is_success() || matches!(response.status(), &StatusCode::FORBIDDEN)
+    {
+        // status code is 2xx or 403, need to check if it ends in /
 
         if response.url().as_str().ends_with('/') {
             log::debug!("{} is directory suitable for recursion", response.url());
@@ -456,10 +465,14 @@ async fn make_requests(
                     if !CONFIGURATION.no_recursion {
                         log::debug!("Recursive extraction: {}", new_ferox_response);
 
-                        if new_ferox_response.status().is_success()
-                            && !new_ferox_response.url().as_str().ends_with('/')
+                        if !new_ferox_response.url().as_str().ends_with('/')
+                            && (new_ferox_response.status().is_success()
+                                || matches!(new_ferox_response.status(), &StatusCode::FORBIDDEN))
                         {
-                            // since all of these are 2xx, recursion is only attempted if the
+                            // if the url doesn't end with a /
+                            // and the response code is either a 2xx or 403
+
+                            // since all of these are 2xx or 403, recursion is only attempted if the
                             // url ends in a /. I am actually ok with adding the slash and not
                             // adding it, as both have merit.  Leaving it in for now to see how
                             // things turn out (current as of: v1.1.0)
@@ -492,6 +505,61 @@ pub fn send_report(report_sender: UnboundedSender<FeroxResponse>, response: Fero
     log::trace!("exit: send_report");
 }
 
+/// Request /robots.txt from given url
+async fn scan_robots_txt(
+    target_url: &str,
+    base_depth: usize,
+    stats: Arc<Stats>,
+    tx_term: UnboundedSender<FeroxResponse>,
+    tx_dir: UnboundedSender<String>,
+    tx_stats: UnboundedSender<StatCommand>,
+) {
+    log::trace!(
+        "enter: scan_robots_txt({}, {}, {:?}, {:?}, {:?}, {:?})",
+        target_url,
+        base_depth,
+        stats,
+        tx_term,
+        tx_dir,
+        tx_stats
+    );
+
+    let robots_links = extract_robots_txt(&target_url, &CONFIGURATION, tx_stats.clone()).await;
+
+    for robot_link in robots_links {
+        // create a url based on the given command line options, continue on error
+        let mut ferox_response =
+            match request_feroxresponse_from_new_link(&robot_link, tx_stats.clone()).await {
+                Some(resp) => resp,
+                None => continue,
+            };
+
+        if should_filter_response(&ferox_response, tx_stats.clone()) {
+            continue;
+        }
+
+        if ferox_response.is_file() {
+            log::debug!("File extracted from robots.txt: {}", ferox_response);
+            SCANNED_URLS.add_file_scan(&robot_link, stats.clone());
+            send_report(tx_term.clone(), ferox_response);
+        } else if !CONFIGURATION.no_recursion {
+            log::debug!("Directory extracted from robots.txt: {}", ferox_response);
+            // todo this code is essentially the same as another piece around ~467 of this file
+            if !ferox_response.url().as_str().ends_with('/')
+                && (ferox_response.status().is_success()
+                    || matches!(ferox_response.status(), &StatusCode::FORBIDDEN))
+            {
+                // if the url doesn't end with a /
+                // and the response code is either a 2xx or 403
+                ferox_response.set_url(&format!("{}/", ferox_response.url()));
+            }
+
+            try_recursion(&ferox_response, base_depth, tx_dir.clone()).await;
+        }
+    }
+    log::trace!("exit: scan_robots_txt");
+}
+
 /// Scan a given url using a given wordlist
 ///
 /// This is the primary entrypoint for the scanner
@@ -524,6 +592,20 @@ pub async fn scan_url(
     if CALL_COUNT.load(Ordering::Relaxed) < stats.initial_targets.load(Ordering::Relaxed) {
         CALL_COUNT.fetch_add(1, Ordering::Relaxed);
 
+        if CONFIGURATION.extract_links {
+            // only grab robots.txt on the initial scan_url calls. all fresh dirs will be passed
+            // to try_recursion
+            scan_robots_txt(
+                target_url,
+                base_depth,
+                stats.clone(),
+                tx_term.clone(),
+                tx_dir.clone(),
+                tx_stats.clone(),
+            )
+            .await;
+        }
+
         update_stat!(tx_stats, UpdateUsizeField(TotalScans, 1));
 
         // this protection allows us to add the first scanned url to SCANNED_URLS
@@ -532,7 +614,12 @@ pub async fn scan_url(
     }
 
     let ferox_scan = match SCANNED_URLS.get_scan_by_url(&target_url) {
-        Some(scan) => scan,
+        Some(scan) => {
+            if let Ok(mut u_scan) = scan.lock() {
+                u_scan.status = ScanStatus::Running;
+            }
+            scan
+        }
         None => {
             log::error!(
                 "Could not find FeroxScan associated with {}; this shouldn't happen... exiting",
@@ -610,9 +697,7 @@ pub async fn scan_url(
                         // for every word in the wordlist, check to see if PAUSE_SCAN is set to true
                         // when true; enter a busy loop that only exits by setting PAUSE_SCAN back
                         // to false
-
-                        // todo change to true when issue #107 is resolved
-                        SCANNED_URLS.pause(false).await;
+                        SCANNED_URLS.pause(true).await;
                     }
                     make_requests(&tgt, &word, base_depth, lst, txd, txr, txs).await
                 }),
@@ -651,10 +736,10 @@ pub async fn scan_url(
     log::trace!("dropped recursion handler's transmitter");
     drop(tx_dir);
 
-    // await rx tasks
-    log::trace!("awaiting recursive scan receiver/scans");
-    futures::future::join_all(recurser.await.unwrap()).await;
-    log::trace!("done awaiting recursive scan receiver/scans");
+    // note: in v1.11.2 i removed the join_all call that used to handle the recurser handles.
+    // nothing appears to change by having them removed, however, if ever a revert is needed
+    // this is the place and anything prior to 1.11.2 will have the code to do so
+    let _ = recurser.await.unwrap_or_default();
 
     log::trace!("exit: scan_url");
 }
@@ -887,7 +972,7 @@ mod tests {
         assert!(result);
     }
 
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     #[should_panic]
     /// call initialize with a bad regex, triggering a panic
     async fn initialize_panics_on_bad_regex() {

src/statistics.rs

@@ -151,7 +151,8 @@ pub struct Stats {
 
 /// FeroxSerialize implementation for Stats
 impl FeroxSerialize for Stats {
-    /// Simply return debug format of Stats to satisfy as_str
+    /// Simply return empty string here to disable serializing this to the output file as a string
+    /// due to it looking like garbage
     fn as_str(&self) -> String {
         String::new()
     }
@@ -187,8 +188,8 @@ impl Stats {
     }
 
     /// save an instance of `Stats` to disk after updating the total runtime for the scan
-    fn save(&self, seconds: f64) {
-        let buffered_file = match get_cached_file_handle(&CONFIGURATION.output) {
+    fn save(&self, seconds: f64, location: &str) {
+        let buffered_file = match get_cached_file_handle(location) {
             Some(file) => file,
             None => {
                 return;
@@ -515,12 +516,17 @@ pub async fn spawn_statistics_handler(
         match command as StatCommand {
             StatCommand::AddError(err) => {
                 stats.add_error(err);
+                increment_bar(&bar, stats.clone());
             }
             StatCommand::AddStatus(status) => {
                 stats.add_status_code(status);
+                increment_bar(&bar, stats.clone());
             }
-            StatCommand::AddRequest => stats.add_request(),
-            StatCommand::Save => stats.save(start.elapsed().as_secs_f64()),
+            StatCommand::AddRequest => {
+                stats.add_request();
+                increment_bar(&bar, stats.clone());
+            }
+            StatCommand::Save => stats.save(start.elapsed().as_secs_f64(), &CONFIGURATION.output),
             StatCommand::UpdateUsizeField(field, value) => {
                 let update_len = matches!(field, StatField::TotalScans);
                 stats.update_usize_field(field, value);
@@ -542,17 +548,6 @@ pub async fn spawn_statistics_handler(
             }
             StatCommand::Exit => break,
         }
-
-        let msg = format!(
-            "{}:{:<7} {}:{:<7}",
-            style("found").green(),
-            atomic_load!(stats.resources_discovered),
-            style("errors").red(),
-            atomic_load!(stats.errors),
-        );
-
-        bar.set_message(&msg);
-        bar.inc(1);
     }
 
     bar.finish();
@@ -561,6 +556,20 @@ pub async fn spawn_statistics_handler(
     log::trace!("exit: spawn_statistics_handler")
 }
 
+/// Wrapper around incrementing the overall scan's progress bar
+fn increment_bar(bar: &ProgressBar, stats: Arc<Stats>) {
+    let msg = format!(
+        "{}:{:<7} {}:{:<7}",
+        style("found").green(),
+        atomic_load!(stats.resources_discovered),
+        style("errors").red(),
+        atomic_load!(stats.errors),
+    );
+
+    bar.set_message(&msg);
+    bar.inc(1);
+}
+
 /// Initialize new `Stats` object and the sc side of an mpsc channel that is responsible for
 /// updates to the aforementioned object.
 pub fn initialize() -> (Arc<Stats>, UnboundedSender<StatCommand>, JoinHandle<()>) {
@@ -603,7 +612,7 @@ mod tests {
         handle.await.unwrap();
     }
 
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// when sent StatCommand::Exit, function should exit its while loop (runs forever otherwise)
     async fn statistics_handler_exits() {
         let (_, sender, handle) = setup_stats_test();
@@ -615,7 +624,7 @@ mod tests {
         // if we've made it here, the test has succeeded
     }
 
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// when sent StatCommand::AddRequest, stats object should reflect the change
     async fn statistics_handler_increments_requests() {
         let (stats, tx, handle) = setup_stats_test();
@@ -629,7 +638,7 @@ mod tests {
         assert_eq!(stats.requests.load(Ordering::Relaxed), 3);
     }
 
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// when sent StatCommand::AddRequest, stats object should reflect the change
     ///
     /// incrementing a 403 (tracked in status_403s) should also increment:
@@ -653,7 +662,7 @@ mod tests {
         assert_eq!(stats.client_errors.load(Ordering::Relaxed), 2);
     }
 
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// when sent StatCommand::AddRequest, stats object should reflect the change
     ///
     /// incrementing a 403 (tracked in status_403s) should also increment:
@@ -675,7 +684,7 @@ mod tests {
         assert_eq!(stats.client_errors.load(Ordering::Relaxed), 2);
     }
 
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// when sent StatCommand::AddStatus, stats object should reflect the change
     ///
     /// incrementing a 500 (tracked in server_errors) should also increment:
@@ -790,4 +799,35 @@ mod tests {
         assert_eq!(stats.total_runtime.lock().unwrap().len(), 1);
         assert!((stats.total_runtime.lock().unwrap()[0] - 0.0).abs() < f64::EPSILON);
     }
+
+    #[test]
+    /// ensure update runtime overwrites the default 0th entry
+    fn update_runtime_works() {
+        let stats = Stats::new();
+        assert!((stats.total_runtime.lock().unwrap()[0] - 0.0).abs() < f64::EPSILON);
+        stats.update_runtime(20.2);
+        assert!((stats.total_runtime.lock().unwrap()[0] - 20.2).abs() < f64::EPSILON);
+    }
+
+    #[test]
+    /// Stats::save should write contents of Stats to disk
+    fn save_writes_stats_object_to_disk() {
+        let stats = Stats::new();
+        stats.add_request();
+        stats.add_request();
+        stats.add_request();
+        stats.add_request();
+        stats.add_error(StatError::Timeout);
+        stats.add_error(StatError::Timeout);
+        stats.add_error(StatError::Timeout);
+        stats.add_error(StatError::Timeout);
+        stats.add_status_code(StatusCode::OK);
+        stats.add_status_code(StatusCode::OK);
+        stats.add_status_code(StatusCode::OK);
+        let outfile = "/tmp/stuff";
+        stats.save(174.33, outfile);
+        assert!(stats.as_json().contains("statistics"));
+        assert!(stats.as_json().contains("11")); // requests made
+        assert!(stats.as_str().is_empty());
+    }
 }

src/utils.rs

@@ -242,6 +242,15 @@ pub fn format_url(
     } else if add_slash && !word.ends_with('/') {
         // -f used, and word doesn't already end with a /
         format!("{}/", word)
+    } else if word.starts_with("//") {
+        // bug ID'd by @Sicks3c, when a wordlist contains words that begin with 2 forward slashes
+        // i.e. //1_40_0/static/js, it gets joined onto the base url in a surprising way
+        // ex: https://localhost/ + //1_40_0/static/js -> https://1_40_0/static/js
+        // this is due to the fact that //... is a valid url. The fix is introduced here in 1.12.2
+        // and simply removes prefixed forward slashes if there are two of them. Additionally,
+        // trim_start_matches will trim the pattern until it's gone, so even if there are more than
+        // 2 /'s, they'll still be trimmed
+        word.trim_start_matches('/').to_string()
     } else {
         String::from(word)
     };
@@ -585,6 +594,27 @@ mod tests {
         );
     }
 
+    #[test]
+    /// word with two prepended slashes doesn't discard the entire domain
+    fn format_url_word_with_two_prepended_slashes() {
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
+        let result = format_url(
+            "http://localhost",
+            "//upload/img",
+            false,
+            &Vec::new(),
+            None,
+            tx,
+        )
+        .unwrap();
+
+        assert_eq!(
+            result,
+            reqwest::Url::parse("http://localhost/upload/img").unwrap()
+        );
+    }
+
     #[test]
     /// word that is a fully formed url, should return an error
     fn format_url_word_that_is_a_url() {

tests/test_extractor.rs

@@ -252,11 +252,20 @@ fn extractor_finds_robots_txt_links_and_displays_files_or_scans_directories() {
         then.status(200).body("im a little teapot too"); // 22
     });
 
-    let mock_dir = srv.mock(|when, then| {
-        when.method(GET).path("/disallowed-subdir/LICENSE");
+    let mock_scanned_file = srv.mock(|when, then| {
+        when.method(GET).path("/misc/LICENSE");
         then.status(200).body("i too, am a container for tea"); // 29
     });
 
+    let mock_dir = srv.mock(|when, _| {
+        when.method(GET).path("/misc/");
+    });
+
+    let mock_disallowed = srv.mock(|when, then| {
+        when.method(GET).path("/disallowed-subdir");
+        then.status(404);
+    });
+
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
         .arg("--url")
@@ -273,7 +282,7 @@ fn extractor_finds_robots_txt_links_and_displays_files_or_scans_directories() {
             .and(predicate::str::contains("18c"))
             .and(predicate::str::contains("/misc/stupidfile.php"))
             .and(predicate::str::contains("22c"))
-            .and(predicate::str::contains("/disallowed-subdir/LICENSE"))
+            .and(predicate::str::contains("/misc/LICENSE"))
             .and(predicate::str::contains("29c"))
             .and(predicate::str::contains("200").count(3)),
     );
@@ -282,5 +291,58 @@ fn extractor_finds_robots_txt_links_and_displays_files_or_scans_directories() {
     assert_eq!(mock_dir.hits(), 1);
     assert_eq!(mock_two.hits(), 1);
     assert_eq!(mock_file.hits(), 1);
+    assert_eq!(mock_disallowed.hits(), 1);
+    assert_eq!(mock_scanned_file.hits(), 1);
     teardown_tmp_directory(tmp_dir);
 }
+
+#[test]
+/// send a request to a page that contains a link that contains a directory that returns a 403
+/// --extract-links should find the link and make recurse into the 403 directory, finding LICENSE
+fn extractor_recurses_into_403_directories() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200)
+            .body(&srv.url("'/homepage/assets/img/icons/handshake.svg'"));
+    });
+
+    let mock_two = srv.mock(|when, then| {
+        when.method(GET).path("/homepage/assets/img/icons/LICENSE");
+        then.status(200).body("that's just like, your opinion man");
+    });
+
+    let forbidden_dir = srv.mock(|when, then| {
+        when.method(GET).path("/homepage/assets/img/icons/");
+        then.status(403);
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--extract-links")
+        .arg("--depth") // need to go past default 4 directories
+        .arg("0")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .count(2)
+            .and(predicate::str::contains("1w")) // link in /LICENSE
+            .and(predicate::str::contains("34c")) // recursed LICENSE
+            .and(predicate::str::contains(
+                "/homepage/assets/img/icons/LICENSE",
+            )),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_two.hits(), 1);
+    assert_eq!(forbidden_dir.hits(), 1);
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}

tests/test_filters.rs

@@ -193,7 +193,7 @@ fn filters_size_should_filter_response() {
 #[test]
 /// create a FeroxResponse that should elicit a true from
 /// SimilarityFilter::should_filter_response
-fn filter_similar_should_filter_response() {
+fn filters_similar_should_filter_response() {
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(
         &["not-similar".to_string(), "similar".to_string()],

tests/test_scan_manager.rs

@@ -20,11 +20,11 @@ fn resume_scan_works() {
     // localhost:PORT/ <- complete
     // localhost:PORT/js <- will get scanned with /css and /stuff
     let complete_scan = format!(
-        r#"{{"id":"057016a14769414aac9a7a62707598cb","url":"{}","scan_type":"Directory","complete":true}}"#,
+        r#"{{"id":"057016a14769414aac9a7a62707598cb","url":"{}","scan_type":"Directory","status":"Complete"}}"#,
         srv.url("/")
     );
     let incomplete_scan = format!(
-        r#"{{"id":"400b2323a16f43468a04ffcbbeba34c6","url":"{}","scan_type":"Directory","complete":false}}"#,
+        r#"{{"id":"400b2323a16f43468a04ffcbbeba34c6","url":"{}","scan_type":"Directory","status":"NotStarted"}}"#,
         srv.url("/js")
     );
     let scans = format!(r#""scans":[{},{}]"#, complete_scan, incomplete_scan);

tests/test_scanner.rs

@@ -546,3 +546,53 @@ fn scanner_single_request_scan_with_regex_filtered_result() {
     assert_eq!(filtered_mock.hits(), 1);
     teardown_tmp_directory(tmp_dir);
 }
+
+#[test]
+/// send a request to a 403 directory, expect recursion to work into the 403
+fn scanner_recursion_works_with_403_directories() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) =
+        setup_tmp_directory(&["LICENSE".to_string(), "ignored/".to_string()], "wordlist").unwrap();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
+
+    let forbidden_dir = srv.mock(|when, then| {
+        when.method(GET).path("/ignored/");
+        then.status(403);
+    });
+
+    let found_anyway = srv.mock(|when, then| {
+        when.method(GET).path("/ignored/LICENSE");
+        then.status(200)
+            .body("this is a test\nThat rug really tied the room together");
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .count(2)
+            .and(predicate::str::contains("200").count(2))
+            .and(predicate::str::contains("403"))
+            .and(predicate::str::contains("53c"))
+            .and(predicate::str::contains("14c"))
+            .and(predicate::str::contains("0c"))
+            .and(predicate::str::contains("ignored").count(2))
+            .and(predicate::str::contains("/ignored/LICENSE")),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(found_anyway.hits(), 1);
+    assert_eq!(forbidden_dir.hits(), 1);
+
+    teardown_tmp_directory(tmp_dir);
+}