bumped version to 1.12.3

fixed url parsing issue when word starts with 2 or more /

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "feroxbuster"
-version = "1.12.0"
+version = "1.12.3"
 authors = ["Ben 'epi' Risher <epibar052@gmail.com>"]
 license = "MIT"
 edition = "2018"
@@ -41,7 +41,7 @@ regex = "1"
 crossterm = "0.19"
 rlimit = "0.5"
 ctrlc = "3.1"
-fuzzyhash = "0.2"
+fuzzyhash = "0.2.1"
 
 [dev-dependencies]
 tempfile = "3.1"

src/banner.rs

@@ -227,12 +227,12 @@ by Ben "epi" Risher {}                 ver: {}"#,
             &mut writer,
             "{}",
             format_banner_entry!(
-                format_emoji("🗑"),
+                format_emoji("💢"),
                 "Status Code Filters",
                 format!("[{}]", code_filters.join(", "))
             )
         )
-        .unwrap_or_default(); // 🗑
+        .unwrap_or_default(); // 💢
     }
 
     writeln!(

src/filters.rs

@@ -1,513 +0,0 @@
-use crate::config::CONFIGURATION;
-use crate::utils::get_url_path_length;
-use crate::{FeroxResponse, FeroxSerialize};
-use fuzzyhash::FuzzyHash;
-use regex::Regex;
-use std::any::Any;
-use std::fmt::Debug;
-
-// references:
-//   https://dev.to/magnusstrale/rust-trait-objects-in-a-vector-non-trivial-4co5
-//   https://stackoverflow.com/questions/25339603/how-to-test-for-equality-between-trait-objects
-
-/// FeroxFilter trait; represents different types of possible filters that can be applied to
-/// responses
-pub trait FeroxFilter: Debug + Send + Sync {
-    /// Determine whether or not this particular filter should be applied or not
-    fn should_filter_response(&self, response: &FeroxResponse) -> bool;
-
-    /// delegates to the FeroxFilter-implementing type which gives us the actual type of self
-    fn box_eq(&self, other: &dyn Any) -> bool;
-
-    /// gives us `other` as Any in box_eq
-    fn as_any(&self) -> &dyn Any;
-}
-
-/// implementation of PartialEq, necessary long-form due to "trait cannot be made into an object"
-/// error when attempting to derive PartialEq on the trait itself
-impl PartialEq for Box<dyn FeroxFilter> {
-    /// Perform a comparison of two implementors of the FeroxFilter trait
-    fn eq(&self, other: &Box<dyn FeroxFilter>) -> bool {
-        self.box_eq(other.as_any())
-    }
-}
-
-/// Data holder for two pieces of data needed when auto-filtering out wildcard responses
-///
-/// `dynamic` is the size of the response that will later be combined with the length
-/// of the path of the url requested and used to determine interesting pages from custom
-/// 404s where the requested url is reflected back in the response
-///
-/// `size` is size of the response that should be included with filters passed via runtime
-/// configuration and any static wildcard lengths.
-#[derive(Debug, Default, Clone, PartialEq)]
-pub struct WildcardFilter {
-    /// size of the response that will later be combined with the length of the path of the url
-    /// requested
-    pub dynamic: u64,
-
-    /// size of the response that should be included with filters passed via runtime configuration
-    pub size: u64,
-}
-
-/// implementation of FeroxFilter for WildcardFilter
-impl FeroxFilter for WildcardFilter {
-    /// Examine size, dynamic, and content_len to determine whether or not the response received
-    /// is a wildcard response and therefore should be filtered out
-    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
-        log::trace!("enter: should_filter_response({:?} {})", self, response);
-
-        // quick return if dont_filter is set
-        if CONFIGURATION.dont_filter {
-            // --dont-filter applies specifically to wildcard filters, it is not a 100% catch all
-            // for not filtering anything.  As such, it should live in the implementation of
-            // a wildcard filter
-            return false;
-        }
-
-        if self.size > 0 && self.size == response.content_length() {
-            // static wildcard size found during testing
-            // size isn't default, size equals response length, and auto-filter is on
-            log::debug!("static wildcard: filtered out {}", response.url());
-            log::trace!("exit: should_filter_response -> true");
-            return true;
-        }
-
-        if self.dynamic > 0 {
-            // dynamic wildcard offset found during testing
-
-            // I'm about to manually split this url path instead of using reqwest::Url's
-            // builtin parsing. The reason is that they call .split() on the url path
-            // except that I don't want an empty string taking up the last index in the
-            // event that the url ends with a forward slash.  It's ugly enough to be split
-            // into its own function for readability.
-            let url_len = get_url_path_length(&response.url());
-
-            if url_len + self.dynamic == response.content_length() {
-                log::debug!("dynamic wildcard: filtered out {}", response.url());
-                log::trace!("exit: should_filter_response -> true");
-                return true;
-            }
-        }
-        log::trace!("exit: should_filter_response -> false");
-        false
-    }
-
-    /// Compare one WildcardFilter to another
-    fn box_eq(&self, other: &dyn Any) -> bool {
-        other.downcast_ref::<Self>().map_or(false, |a| self == a)
-    }
-
-    /// Return self as Any for dynamic dispatch purposes
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-}
-
-/// Simple implementor of FeroxFilter; used to filter out status codes specified using
-/// -C|--filter-status
-#[derive(Default, Debug, PartialEq)]
-pub struct StatusCodeFilter {
-    /// Status code that should not be displayed to the user
-    pub filter_code: u16,
-}
-
-/// implementation of FeroxFilter for StatusCodeFilter
-impl FeroxFilter for StatusCodeFilter {
-    /// Check `filter_code` against what was passed in via -C|--filter-status
-    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
-        log::trace!("enter: should_filter_response({:?} {})", self, response);
-
-        if response.status().as_u16() == self.filter_code {
-            log::debug!(
-                "filtered out {} based on --filter-status of {}",
-                response.url(),
-                self.filter_code
-            );
-            log::trace!("exit: should_filter_response -> true");
-            return true;
-        }
-
-        log::trace!("exit: should_filter_response -> false");
-        false
-    }
-
-    /// Compare one StatusCodeFilter to another
-    fn box_eq(&self, other: &dyn Any) -> bool {
-        other.downcast_ref::<Self>().map_or(false, |a| self == a)
-    }
-
-    /// Return self as Any for dynamic dispatch purposes
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-}
-
-/// Simple implementor of FeroxFilter; used to filter out responses based on the number of lines
-/// in a Response body; specified using -N|--filter-lines
-#[derive(Default, Debug, PartialEq)]
-pub struct LinesFilter {
-    /// Number of lines in a Response's body that should be filtered
-    pub line_count: usize,
-}
-
-/// implementation of FeroxFilter for LinesFilter
-impl FeroxFilter for LinesFilter {
-    /// Check `line_count` against what was passed in via -N|--filter-lines
-    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
-        log::trace!("enter: should_filter_response({:?} {})", self, response);
-
-        let result = response.line_count() == self.line_count;
-
-        log::trace!("exit: should_filter_response -> {}", result);
-
-        result
-    }
-
-    /// Compare one LinesFilter to another
-    fn box_eq(&self, other: &dyn Any) -> bool {
-        other.downcast_ref::<Self>().map_or(false, |a| self == a)
-    }
-
-    /// Return self as Any for dynamic dispatch purposes
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-}
-
-/// Simple implementor of FeroxFilter; used to filter out responses based on the number of words
-/// in a Response body; specified using -W|--filter-words
-#[derive(Default, Debug, PartialEq)]
-pub struct WordsFilter {
-    /// Number of words in a Response's body that should be filtered
-    pub word_count: usize,
-}
-
-/// implementation of FeroxFilter for WordsFilter
-impl FeroxFilter for WordsFilter {
-    /// Check `word_count` against what was passed in via -W|--filter-words
-    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
-        log::trace!("enter: should_filter_response({:?} {})", self, response);
-
-        let result = response.word_count() == self.word_count;
-
-        log::trace!("exit: should_filter_response -> {}", result);
-
-        result
-    }
-
-    /// Compare one WordsFilter to another
-    fn box_eq(&self, other: &dyn Any) -> bool {
-        other.downcast_ref::<Self>().map_or(false, |a| self == a)
-    }
-
-    /// Return self as Any for dynamic dispatch purposes
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-}
-
-/// Simple implementor of FeroxFilter; used to filter out responses based on the length of a
-/// Response body; specified using -S|--filter-size
-#[derive(Default, Debug, PartialEq)]
-pub struct SizeFilter {
-    /// Overall length of a Response's body that should be filtered
-    pub content_length: u64,
-}
-
-/// implementation of FeroxFilter for SizeFilter
-impl FeroxFilter for SizeFilter {
-    /// Check `content_length` against what was passed in via -S|--filter-size
-    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
-        log::trace!("enter: should_filter_response({:?} {})", self, response);
-
-        let result = response.content_length() == self.content_length;
-
-        log::trace!("exit: should_filter_response -> {}", result);
-
-        result
-    }
-
-    /// Compare one SizeFilter to another
-    fn box_eq(&self, other: &dyn Any) -> bool {
-        other.downcast_ref::<Self>().map_or(false, |a| self == a)
-    }
-
-    /// Return self as Any for dynamic dispatch purposes
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-}
-
-/// Simple implementor of FeroxFilter; used to filter out responses based on a given regular
-/// expression; specified using -X|--filter-regex
-#[derive(Debug)]
-pub struct RegexFilter {
-    /// Regular expression to be applied to the response body for filtering, compiled
-    pub compiled: Regex,
-
-    /// Regular expression as passed in on the command line, not compiled
-    pub raw_string: String,
-}
-
-/// implementation of FeroxFilter for RegexFilter
-impl FeroxFilter for RegexFilter {
-    /// Check `expression` against the response body, if the expression matches, the response
-    /// should be filtered out
-    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
-        log::trace!("enter: should_filter_response({:?} {})", self, response);
-
-        let result = self.compiled.is_match(response.text());
-
-        log::trace!("exit: should_filter_response -> {}", result);
-
-        result
-    }
-
-    /// Compare one SizeFilter to another
-    fn box_eq(&self, other: &dyn Any) -> bool {
-        other.downcast_ref::<Self>().map_or(false, |a| self == a)
-    }
-
-    /// Return self as Any for dynamic dispatch purposes
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-}
-
-/// PartialEq implementation for RegexFilter
-impl PartialEq for RegexFilter {
-    /// Simple comparison of the raw string passed in via the command line
-    fn eq(&self, other: &RegexFilter) -> bool {
-        self.raw_string == other.raw_string
-    }
-}
-
-/// Simple implementor of FeroxFilter; used to filter out responses based on the similarity of a
-/// Response body with a known response; specified using --filter-similar-to
-#[derive(Default, Debug, PartialEq)]
-pub struct SimilarityFilter {
-    /// Response's body to be used for comparison for similarity
-    pub text: String,
-
-    /// Percentage of similarity at which a page is determined to be a near-duplicate of another
-    pub threshold: u32,
-}
-
-/// implementation of FeroxFilter for SimilarityFilter
-impl FeroxFilter for SimilarityFilter {
-    /// Check `FeroxResponse::text` against what was requested from the site passed in via
-    /// --filter-similar-to
-    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
-        let other = FuzzyHash::new(&response.text);
-
-        if let Ok(result) = FuzzyHash::compare(&self.text, &other.to_string()) {
-            return result >= self.threshold;
-        }
-
-        // couldn't hash the response, don't filter
-        log::warn!("Could not hash body from {}", response.as_str());
-        false
-    }
-
-    /// Compare one SimilarityFilter to another
-    fn box_eq(&self, other: &dyn Any) -> bool {
-        other.downcast_ref::<Self>().map_or(false, |a| self == a)
-    }
-
-    /// Return self as Any for dynamic dispatch purposes
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use reqwest::Url;
-
-    #[test]
-    /// just a simple test to increase code coverage by hitting as_any and the inner value
-    fn lines_filter_as_any() {
-        let filter = LinesFilter { line_count: 1 };
-
-        assert_eq!(filter.line_count, 1);
-        assert_eq!(
-            *filter.as_any().downcast_ref::<LinesFilter>().unwrap(),
-            filter
-        );
-    }
-
-    #[test]
-    /// just a simple test to increase code coverage by hitting as_any and the inner value
-    fn words_filter_as_any() {
-        let filter = WordsFilter { word_count: 1 };
-
-        assert_eq!(filter.word_count, 1);
-        assert_eq!(
-            *filter.as_any().downcast_ref::<WordsFilter>().unwrap(),
-            filter
-        );
-    }
-
-    #[test]
-    /// just a simple test to increase code coverage by hitting as_any and the inner value
-    fn size_filter_as_any() {
-        let filter = SizeFilter { content_length: 1 };
-
-        assert_eq!(filter.content_length, 1);
-        assert_eq!(
-            *filter.as_any().downcast_ref::<SizeFilter>().unwrap(),
-            filter
-        );
-    }
-
-    #[test]
-    /// just a simple test to increase code coverage by hitting as_any and the inner value
-    fn status_code_filter_as_any() {
-        let filter = StatusCodeFilter { filter_code: 200 };
-
-        assert_eq!(filter.filter_code, 200);
-        assert_eq!(
-            *filter.as_any().downcast_ref::<StatusCodeFilter>().unwrap(),
-            filter
-        );
-    }
-
-    #[test]
-    /// just a simple test to increase code coverage by hitting as_any and the inner value
-    fn regex_filter_as_any() {
-        let raw = r".*\.txt$";
-        let compiled = Regex::new(raw).unwrap();
-        let filter = RegexFilter {
-            compiled,
-            raw_string: raw.to_string(),
-        };
-
-        assert_eq!(filter.raw_string, r".*\.txt$");
-        assert_eq!(
-            *filter.as_any().downcast_ref::<RegexFilter>().unwrap(),
-            filter
-        );
-    }
-
-    #[test]
-    /// test should_filter on WilcardFilter where static logic matches
-    fn wildcard_should_filter_when_static_wildcard_found() {
-        let resp = FeroxResponse {
-            text: String::new(),
-            wildcard: true,
-            url: Url::parse("http://localhost").unwrap(),
-            content_length: 100,
-            word_count: 50,
-            line_count: 25,
-            headers: reqwest::header::HeaderMap::new(),
-            status: reqwest::StatusCode::OK,
-        };
-
-        let filter = WildcardFilter {
-            size: 100,
-            dynamic: 0,
-        };
-
-        assert!(filter.should_filter_response(&resp));
-    }
-
-    #[test]
-    /// test should_filter on WilcardFilter where dynamic logic matches
-    fn wildcard_should_filter_when_dynamic_wildcard_found() {
-        let resp = FeroxResponse {
-            text: String::new(),
-            wildcard: true,
-            url: Url::parse("http://localhost/stuff").unwrap(),
-            content_length: 100,
-            word_count: 50,
-            line_count: 25,
-            headers: reqwest::header::HeaderMap::new(),
-            status: reqwest::StatusCode::OK,
-        };
-
-        let filter = WildcardFilter {
-            size: 0,
-            dynamic: 95,
-        };
-
-        assert!(filter.should_filter_response(&resp));
-    }
-
-    #[test]
-    /// test should_filter on RegexFilter where regex matches body
-    fn regexfilter_should_filter_when_regex_matches_on_response_body() {
-        let resp = FeroxResponse {
-            text: String::from("im a body response hurr durr!"),
-            wildcard: false,
-            url: Url::parse("http://localhost/stuff").unwrap(),
-            content_length: 100,
-            word_count: 50,
-            line_count: 25,
-            headers: reqwest::header::HeaderMap::new(),
-            status: reqwest::StatusCode::OK,
-        };
-
-        let raw = r"response...rr";
-
-        let filter = RegexFilter {
-            raw_string: raw.to_string(),
-            compiled: Regex::new(raw).unwrap(),
-        };
-
-        assert!(filter.should_filter_response(&resp));
-    }
-
-    #[test]
-    /// a few simple tests for similarity filter
-    fn similarity_filter_is_accurate() {
-        let mut resp = FeroxResponse {
-            text: String::from("sitting"),
-            wildcard: false,
-            url: Url::parse("http://localhost/stuff").unwrap(),
-            content_length: 100,
-            word_count: 50,
-            line_count: 25,
-            headers: reqwest::header::HeaderMap::new(),
-            status: reqwest::StatusCode::OK,
-        };
-
-        let mut filter = SimilarityFilter {
-            text: FuzzyHash::new("kitten").to_string(),
-            threshold: 95,
-        };
-
-        // kitten/sitting is 57% similar, so a threshold of 95 should not be filtered
-        assert!(!filter.should_filter_response(&resp));
-
-        resp.text = String::new();
-        filter.text = String::new();
-        filter.threshold = 100;
-
-        // two empty strings are the same, however ssdeep doesn't accept empty strings, expect false
-        assert!(!filter.should_filter_response(&resp));
-
-        resp.text = String::from("some data to hash for the purposes of running a test");
-        filter.text =
-            FuzzyHash::new("some data to hash for the purposes of running a te").to_string();
-        filter.threshold = 17;
-
-        assert!(filter.should_filter_response(&resp));
-    }
-
-    #[test]
-    /// just a simple test to increase code coverage by hitting as_any and the inner value
-    fn similarity_filter_as_any() {
-        let filter = SimilarityFilter {
-            text: String::from("stuff"),
-            threshold: 95,
-        };
-
-        assert_eq!(filter.text, "stuff");
-        assert_eq!(
-            *filter.as_any().downcast_ref::<SimilarityFilter>().unwrap(),
-            filter
-        );
-    }
-}

src/filters/lines.rs

@@ -0,0 +1,33 @@
+use super::*;
+
+/// Simple implementor of FeroxFilter; used to filter out responses based on the number of lines
+/// in a Response body; specified using -N|--filter-lines
+#[derive(Default, Debug, PartialEq)]
+pub struct LinesFilter {
+    /// Number of lines in a Response's body that should be filtered
+    pub line_count: usize,
+}
+
+/// implementation of FeroxFilter for LinesFilter
+impl FeroxFilter for LinesFilter {
+    /// Check `line_count` against what was passed in via -N|--filter-lines
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        let result = response.line_count() == self.line_count;
+
+        log::trace!("exit: should_filter_response -> {}", result);
+
+        result
+    }
+
+    /// Compare one LinesFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/filters/mod.rs

@@ -0,0 +1,24 @@
+//! module containing all of feroxbuster's filters
+mod traits;
+mod wildcard;
+mod status_code;
+mod words;
+mod lines;
+mod size;
+mod regex;
+mod similarity;
+#[cfg(test)]
+mod tests;
+
+pub use self::lines::LinesFilter;
+pub use self::regex::RegexFilter;
+pub use self::similarity::SimilarityFilter;
+pub use self::size::SizeFilter;
+pub use self::status_code::StatusCodeFilter;
+pub use self::traits::FeroxFilter;
+pub use self::wildcard::WildcardFilter;
+pub use self::words::WordsFilter;
+
+use crate::{config::CONFIGURATION, utils::get_url_path_length, FeroxResponse, FeroxSerialize};
+use std::any::Any;
+use std::fmt::Debug;

src/filters/regex.rs

@@ -0,0 +1,46 @@
+use super::*;
+use ::regex::Regex;
+
+/// Simple implementor of FeroxFilter; used to filter out responses based on a given regular
+/// expression; specified using -X|--filter-regex
+#[derive(Debug)]
+pub struct RegexFilter {
+    /// Regular expression to be applied to the response body for filtering, compiled
+    pub compiled: Regex,
+
+    /// Regular expression as passed in on the command line, not compiled
+    pub raw_string: String,
+}
+
+/// implementation of FeroxFilter for RegexFilter
+impl FeroxFilter for RegexFilter {
+    /// Check `expression` against the response body, if the expression matches, the response
+    /// should be filtered out
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        let result = self.compiled.is_match(response.text());
+
+        log::trace!("exit: should_filter_response -> {}", result);
+
+        result
+    }
+
+    /// Compare one SizeFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
+
+/// PartialEq implementation for RegexFilter
+impl PartialEq for RegexFilter {
+    /// Simple comparison of the raw string passed in via the command line
+    fn eq(&self, other: &RegexFilter) -> bool {
+        self.raw_string == other.raw_string
+    }
+}

src/filters/similarity.rs

@@ -0,0 +1,40 @@
+use super::*;
+use fuzzyhash::FuzzyHash;
+
+/// Simple implementor of FeroxFilter; used to filter out responses based on the similarity of a
+/// Response body with a known response; specified using --filter-similar-to
+#[derive(Default, Debug, PartialEq)]
+pub struct SimilarityFilter {
+    /// Response's body to be used for comparison for similarity
+    pub text: String,
+
+    /// Percentage of similarity at which a page is determined to be a near-duplicate of another
+    pub threshold: u32,
+}
+
+/// implementation of FeroxFilter for SimilarityFilter
+impl FeroxFilter for SimilarityFilter {
+    /// Check `FeroxResponse::text` against what was requested from the site passed in via
+    /// --filter-similar-to
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        let other = FuzzyHash::new(&response.text);
+
+        if let Ok(result) = FuzzyHash::compare(&self.text, &other.to_string()) {
+            return result >= self.threshold;
+        }
+
+        // couldn't hash the response, don't filter
+        log::warn!("Could not hash body from {}", response.as_str());
+        false
+    }
+
+    /// Compare one SimilarityFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/filters/size.rs

@@ -0,0 +1,33 @@
+use super::*;
+
+/// Simple implementor of FeroxFilter; used to filter out responses based on the length of a
+/// Response body; specified using -S|--filter-size
+#[derive(Default, Debug, PartialEq)]
+pub struct SizeFilter {
+    /// Overall length of a Response's body that should be filtered
+    pub content_length: u64,
+}
+
+/// implementation of FeroxFilter for SizeFilter
+impl FeroxFilter for SizeFilter {
+    /// Check `content_length` against what was passed in via -S|--filter-size
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        let result = response.content_length() == self.content_length;
+
+        log::trace!("exit: should_filter_response -> {}", result);
+
+        result
+    }
+
+    /// Compare one SizeFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/filters/status_code.rs

@@ -0,0 +1,40 @@
+use super::*;
+
+/// Simple implementor of FeroxFilter; used to filter out status codes specified using
+/// -C|--filter-status
+#[derive(Default, Debug, PartialEq)]
+pub struct StatusCodeFilter {
+    /// Status code that should not be displayed to the user
+    pub filter_code: u16,
+}
+
+/// implementation of FeroxFilter for StatusCodeFilter
+impl FeroxFilter for StatusCodeFilter {
+    /// Check `filter_code` against what was passed in via -C|--filter-status
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        if response.status().as_u16() == self.filter_code {
+            log::debug!(
+                "filtered out {} based on --filter-status of {}",
+                response.url(),
+                self.filter_code
+            );
+            log::trace!("exit: should_filter_response -> true");
+            return true;
+        }
+
+        log::trace!("exit: should_filter_response -> false");
+        false
+    }
+
+    /// Compare one StatusCodeFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/filters/tests.rs

@@ -0,0 +1,188 @@
+use super::*;
+use ::fuzzyhash::FuzzyHash;
+use ::regex::Regex;
+use reqwest::Url;
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn lines_filter_as_any() {
+    let filter = LinesFilter { line_count: 1 };
+
+    assert_eq!(filter.line_count, 1);
+    assert_eq!(
+        *filter.as_any().downcast_ref::<LinesFilter>().unwrap(),
+        filter
+    );
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn words_filter_as_any() {
+    let filter = WordsFilter { word_count: 1 };
+
+    assert_eq!(filter.word_count, 1);
+    assert_eq!(
+        *filter.as_any().downcast_ref::<WordsFilter>().unwrap(),
+        filter
+    );
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn size_filter_as_any() {
+    let filter = SizeFilter { content_length: 1 };
+
+    assert_eq!(filter.content_length, 1);
+    assert_eq!(
+        *filter.as_any().downcast_ref::<SizeFilter>().unwrap(),
+        filter
+    );
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn status_code_filter_as_any() {
+    let filter = StatusCodeFilter { filter_code: 200 };
+
+    assert_eq!(filter.filter_code, 200);
+    assert_eq!(
+        *filter.as_any().downcast_ref::<StatusCodeFilter>().unwrap(),
+        filter
+    );
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn regex_filter_as_any() {
+    let raw = r".*\.txt$";
+    let compiled = Regex::new(raw).unwrap();
+    let filter = RegexFilter {
+        compiled,
+        raw_string: raw.to_string(),
+    };
+
+    assert_eq!(filter.raw_string, r".*\.txt$");
+    assert_eq!(
+        *filter.as_any().downcast_ref::<RegexFilter>().unwrap(),
+        filter
+    );
+}
+
+#[test]
+/// test should_filter on WilcardFilter where static logic matches
+fn wildcard_should_filter_when_static_wildcard_found() {
+    let resp = FeroxResponse {
+        text: String::new(),
+        wildcard: true,
+        url: Url::parse("http://localhost").unwrap(),
+        content_length: 100,
+        word_count: 50,
+        line_count: 25,
+        headers: reqwest::header::HeaderMap::new(),
+        status: reqwest::StatusCode::OK,
+    };
+
+    let filter = WildcardFilter {
+        size: 100,
+        dynamic: 0,
+    };
+
+    assert!(filter.should_filter_response(&resp));
+}
+
+#[test]
+/// test should_filter on WilcardFilter where dynamic logic matches
+fn wildcard_should_filter_when_dynamic_wildcard_found() {
+    let resp = FeroxResponse {
+        text: String::new(),
+        wildcard: true,
+        url: Url::parse("http://localhost/stuff").unwrap(),
+        content_length: 100,
+        word_count: 50,
+        line_count: 25,
+        headers: reqwest::header::HeaderMap::new(),
+        status: reqwest::StatusCode::OK,
+    };
+
+    let filter = WildcardFilter {
+        size: 0,
+        dynamic: 95,
+    };
+
+    assert!(filter.should_filter_response(&resp));
+}
+
+#[test]
+/// test should_filter on RegexFilter where regex matches body
+fn regexfilter_should_filter_when_regex_matches_on_response_body() {
+    let resp = FeroxResponse {
+        text: String::from("im a body response hurr durr!"),
+        wildcard: false,
+        url: Url::parse("http://localhost/stuff").unwrap(),
+        content_length: 100,
+        word_count: 50,
+        line_count: 25,
+        headers: reqwest::header::HeaderMap::new(),
+        status: reqwest::StatusCode::OK,
+    };
+
+    let raw = r"response...rr";
+
+    let filter = RegexFilter {
+        raw_string: raw.to_string(),
+        compiled: Regex::new(raw).unwrap(),
+    };
+
+    assert!(filter.should_filter_response(&resp));
+}
+
+#[test]
+/// a few simple tests for similarity filter
+fn similarity_filter_is_accurate() {
+    let mut resp = FeroxResponse {
+        text: String::from("sitting"),
+        wildcard: false,
+        url: Url::parse("http://localhost/stuff").unwrap(),
+        content_length: 100,
+        word_count: 50,
+        line_count: 25,
+        headers: reqwest::header::HeaderMap::new(),
+        status: reqwest::StatusCode::OK,
+    };
+
+    let mut filter = SimilarityFilter {
+        text: FuzzyHash::new("kitten").to_string(),
+        threshold: 95,
+    };
+
+    // kitten/sitting is 57% similar, so a threshold of 95 should not be filtered
+    assert!(!filter.should_filter_response(&resp));
+
+    resp.text = String::new();
+    filter.text = String::new();
+    filter.threshold = 100;
+
+    // two empty strings are the same, however ssdeep doesn't accept empty strings, expect false
+    assert!(!filter.should_filter_response(&resp));
+
+    resp.text = String::from("some data to hash for the purposes of running a test");
+    filter.text = FuzzyHash::new("some data to hash for the purposes of running a te").to_string();
+    filter.threshold = 17;
+
+    assert!(filter.should_filter_response(&resp));
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn similarity_filter_as_any() {
+    let filter = SimilarityFilter {
+        text: String::from("stuff"),
+        threshold: 95,
+    };
+
+    assert_eq!(filter.text, "stuff");
+    assert_eq!(
+        *filter.as_any().downcast_ref::<SimilarityFilter>().unwrap(),
+        filter
+    );
+}

src/filters/traits.rs

@@ -0,0 +1,27 @@
+use super::*;
+
+// references:
+//   https://dev.to/magnusstrale/rust-trait-objects-in-a-vector-non-trivial-4co5
+//   https://stackoverflow.com/questions/25339603/how-to-test-for-equality-between-trait-objects
+
+/// FeroxFilter trait; represents different types of possible filters that can be applied to
+/// responses
+pub trait FeroxFilter: Debug + Send + Sync {
+    /// Determine whether or not this particular filter should be applied or not
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool;
+
+    /// delegates to the FeroxFilter-implementing type which gives us the actual type of self
+    fn box_eq(&self, other: &dyn Any) -> bool;
+
+    /// gives us `other` as Any in box_eq
+    fn as_any(&self) -> &dyn Any;
+}
+
+/// implementation of PartialEq, necessary long-form due to "trait cannot be made into an object"
+/// error when attempting to derive PartialEq on the trait itself
+impl PartialEq for Box<dyn FeroxFilter> {
+    /// Perform a comparison of two implementors of the FeroxFilter trait
+    fn eq(&self, other: &Box<dyn FeroxFilter>) -> bool {
+        self.box_eq(other.as_any())
+    }
+}

src/filters/wildcard.rs

@@ -0,0 +1,73 @@
+use super::*;
+
+/// Data holder for two pieces of data needed when auto-filtering out wildcard responses
+///
+/// `dynamic` is the size of the response that will later be combined with the length
+/// of the path of the url requested and used to determine interesting pages from custom
+/// 404s where the requested url is reflected back in the response
+///
+/// `size` is size of the response that should be included with filters passed via runtime
+/// configuration and any static wildcard lengths.
+#[derive(Debug, Default, Clone, PartialEq)]
+pub struct WildcardFilter {
+    /// size of the response that will later be combined with the length of the path of the url
+    /// requested
+    pub dynamic: u64,
+
+    /// size of the response that should be included with filters passed via runtime configuration
+    pub size: u64,
+}
+
+/// implementation of FeroxFilter for WildcardFilter
+impl FeroxFilter for WildcardFilter {
+    /// Examine size, dynamic, and content_len to determine whether or not the response received
+    /// is a wildcard response and therefore should be filtered out
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        // quick return if dont_filter is set
+        if CONFIGURATION.dont_filter {
+            // --dont-filter applies specifically to wildcard filters, it is not a 100% catch all
+            // for not filtering anything.  As such, it should live in the implementation of
+            // a wildcard filter
+            return false;
+        }
+
+        if self.size > 0 && self.size == response.content_length() {
+            // static wildcard size found during testing
+            // size isn't default, size equals response length, and auto-filter is on
+            log::debug!("static wildcard: filtered out {}", response.url());
+            log::trace!("exit: should_filter_response -> true");
+            return true;
+        }
+
+        if self.dynamic > 0 {
+            // dynamic wildcard offset found during testing
+
+            // I'm about to manually split this url path instead of using reqwest::Url's
+            // builtin parsing. The reason is that they call .split() on the url path
+            // except that I don't want an empty string taking up the last index in the
+            // event that the url ends with a forward slash.  It's ugly enough to be split
+            // into its own function for readability.
+            let url_len = get_url_path_length(&response.url());
+
+            if url_len + self.dynamic == response.content_length() {
+                log::debug!("dynamic wildcard: filtered out {}", response.url());
+                log::trace!("exit: should_filter_response -> true");
+                return true;
+            }
+        }
+        log::trace!("exit: should_filter_response -> false");
+        false
+    }
+
+    /// Compare one WildcardFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/filters/words.rs

@@ -0,0 +1,33 @@
+use super::*;
+
+/// Simple implementor of FeroxFilter; used to filter out responses based on the number of words
+/// in a Response body; specified using -W|--filter-words
+#[derive(Default, Debug, PartialEq)]
+pub struct WordsFilter {
+    /// Number of words in a Response's body that should be filtered
+    pub word_count: usize,
+}
+
+/// implementation of FeroxFilter for WordsFilter
+impl FeroxFilter for WordsFilter {
+    /// Check `word_count` against what was passed in via -W|--filter-words
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        let result = response.word_count() == self.word_count;
+
+        log::trace!("exit: should_filter_response -> {}", result);
+
+        result
+    }
+
+    /// Compare one WordsFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/main.rs

@@ -409,8 +409,6 @@ async fn clean_up(
         stats_handle,
         save_output
     );
-    update_stat!(tx_stats, StatCommand::Exit); // send exit command and await the end of the future
-
     drop(tx_term);
     log::trace!("dropped terminal output handler's transmitter");
 
@@ -442,6 +440,7 @@ async fn clean_up(
         log::trace!("done awaiting file output handler's receiver");
     }
 
+    update_stat!(tx_stats, StatCommand::Exit); // send exit command and await the end of the future
     stats_handle.await.unwrap_or_default();
 
     // mark all scans complete so the terminal input handler will exit cleanly

src/scanner.rs

@@ -22,7 +22,7 @@ use futures::{
 use fuzzyhash::FuzzyHash;
 use lazy_static::lazy_static;
 use regex::Regex;
-use reqwest::Url;
+use reqwest::{StatusCode, Url};
 #[cfg(not(test))]
 use std::process::exit;
 use std::{
@@ -260,8 +260,9 @@ fn response_is_directory(response: &FeroxResponse) -> bool {
                 return false;
             }
         }
-    } else if response.status().is_success() {
-        // status code is 2xx, need to check if it ends in /
+    } else if response.status().is_success() || matches!(response.status(), &StatusCode::FORBIDDEN)
+    {
+        // status code is 2xx or 403, need to check if it ends in /
 
         if response.url().as_str().ends_with('/') {
             log::debug!("{} is directory suitable for recursion", response.url());
@@ -464,10 +465,14 @@ async fn make_requests(
                     if !CONFIGURATION.no_recursion {
                         log::debug!("Recursive extraction: {}", new_ferox_response);
 
-                        if new_ferox_response.status().is_success()
-                            && !new_ferox_response.url().as_str().ends_with('/')
+                        if !new_ferox_response.url().as_str().ends_with('/')
+                            && (new_ferox_response.status().is_success()
+                                || matches!(new_ferox_response.status(), &StatusCode::FORBIDDEN))
                         {
-                            // since all of these are 2xx, recursion is only attempted if the
+                            // if the url doesn't end with a /
+                            // and the response code is either a 2xx or 403
+
+                            // since all of these are 2xx or 403, recursion is only attempted if the
                             // url ends in a /. I am actually ok with adding the slash and not
                             // adding it, as both have merit.  Leaving it in for now to see how
                             // things turn out (current as of: v1.1.0)
@@ -540,8 +545,12 @@ async fn scan_robots_txt(
         } else if !CONFIGURATION.no_recursion {
             log::debug!("Directory extracted from robots.txt: {}", ferox_response);
             // todo this code is essentially the same as another piece around ~467 of this file
-            if ferox_response.status().is_success() && !ferox_response.url().as_str().ends_with('/')
+            if !ferox_response.url().as_str().ends_with('/')
+                && (ferox_response.status().is_success()
+                    || matches!(ferox_response.status(), &StatusCode::FORBIDDEN))
             {
+                // if the url doesn't end with a /
+                // and the response code is either a 2xx or 403
                 ferox_response.set_url(&format!("{}/", ferox_response.url()));
             }
 

src/utils.rs

@@ -242,6 +242,15 @@ pub fn format_url(
     } else if add_slash && !word.ends_with('/') {
         // -f used, and word doesn't already end with a /
         format!("{}/", word)
+    } else if word.starts_with("//") {
+        // bug ID'd by @Sicks3c, when a wordlist contains words that begin with 2 forward slashes
+        // i.e. //1_40_0/static/js, it gets joined onto the base url in a surprising way
+        // ex: https://localhost/ + //1_40_0/static/js -> https://1_40_0/static/js
+        // this is due to the fact that //... is a valid url. The fix is introduced here in 1.12.2
+        // and simply removes prefixed forward slashes if there are two of them. Additionally,
+        // trim_start_matches will trim the pattern until it's gone, so even if there are more than
+        // 2 /'s, they'll still be trimmed
+        word.trim_start_matches('/').to_string()
     } else {
         String::from(word)
     };
@@ -585,6 +594,27 @@ mod tests {
         );
     }
 
+    #[test]
+    /// word with two prepended slashes doesn't discard the entire domain
+    fn format_url_word_with_two_prepended_slashes() {
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
+        let result = format_url(
+            "http://localhost",
+            "//upload/img",
+            false,
+            &Vec::new(),
+            None,
+            tx,
+        )
+        .unwrap();
+
+        assert_eq!(
+            result,
+            reqwest::Url::parse("http://localhost/upload/img").unwrap()
+        );
+    }
+
     #[test]
     /// word that is a fully formed url, should return an error
     fn format_url_word_that_is_a_url() {

tests/test_extractor.rs

@@ -295,3 +295,54 @@ fn extractor_finds_robots_txt_links_and_displays_files_or_scans_directories() {
     assert_eq!(mock_scanned_file.hits(), 1);
     teardown_tmp_directory(tmp_dir);
 }
+
+#[test]
+/// send a request to a page that contains a link that contains a directory that returns a 403
+/// --extract-links should find the link and make recurse into the 403 directory, finding LICENSE
+fn extractor_recurses_into_403_directories() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200)
+            .body(&srv.url("'/homepage/assets/img/icons/handshake.svg'"));
+    });
+
+    let mock_two = srv.mock(|when, then| {
+        when.method(GET).path("/homepage/assets/img/icons/LICENSE");
+        then.status(200).body("that's just like, your opinion man");
+    });
+
+    let forbidden_dir = srv.mock(|when, then| {
+        when.method(GET).path("/homepage/assets/img/icons/");
+        then.status(403);
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--extract-links")
+        .arg("--depth") // need to go past default 4 directories
+        .arg("0")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .count(2)
+            .and(predicate::str::contains("1w")) // link in /LICENSE
+            .and(predicate::str::contains("34c")) // recursed LICENSE
+            .and(predicate::str::contains(
+                "/homepage/assets/img/icons/LICENSE",
+            )),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_two.hits(), 1);
+    assert_eq!(forbidden_dir.hits(), 1);
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}

tests/test_filters.rs

@@ -193,7 +193,7 @@ fn filters_size_should_filter_response() {
 #[test]
 /// create a FeroxResponse that should elicit a true from
 /// SimilarityFilter::should_filter_response
-fn filter_similar_should_filter_response() {
+fn filters_similar_should_filter_response() {
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(
         &["not-similar".to_string(), "similar".to_string()],

tests/test_scanner.rs

@@ -546,3 +546,53 @@ fn scanner_single_request_scan_with_regex_filtered_result() {
     assert_eq!(filtered_mock.hits(), 1);
     teardown_tmp_directory(tmp_dir);
 }
+
+#[test]
+/// send a request to a 403 directory, expect recursion to work into the 403
+fn scanner_recursion_works_with_403_directories() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) =
+        setup_tmp_directory(&["LICENSE".to_string(), "ignored/".to_string()], "wordlist").unwrap();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
+
+    let forbidden_dir = srv.mock(|when, then| {
+        when.method(GET).path("/ignored/");
+        then.status(403);
+    });
+
+    let found_anyway = srv.mock(|when, then| {
+        when.method(GET).path("/ignored/LICENSE");
+        then.status(200)
+            .body("this is a test\nThat rug really tied the room together");
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .count(2)
+            .and(predicate::str::contains("200").count(2))
+            .and(predicate::str::contains("403"))
+            .and(predicate::str::contains("53c"))
+            .and(predicate::str::contains("14c"))
+            .and(predicate::str::contains("0c"))
+            .and(predicate::str::contains("ignored").count(2))
+            .and(predicate::str::contains("/ignored/LICENSE")),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(found_anyway.hits(), 1);
+    assert_eq!(forbidden_dir.hits(), 1);
+
+    teardown_tmp_directory(tmp_dir);
+}