Merge pull request #100 from epi052/FEATURE-response-code-blacklist

Feature response code blacklist

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "feroxbuster"
-version = "1.2.0"
+version = "1.3.0"
 authors = ["Ben 'epi' Risher <epibar052@gmail.com>"]
 license = "MIT"
 edition = "2018"

README.md

@@ -80,9 +80,11 @@ This attack is also known as Predictable Resource Location, File Enumeration, Di
     - [Proxy traffic through a SOCKS proxy](#proxy-traffic-through-a-socks-proxy)
     - [Pass auth token via query parameter](#pass-auth-token-via-query-parameter)
     - [Limit Total Number of Concurrent Scans (new in `v1.2.0`)](#limit-total-number-of-concurrent-scans-new-in-v120)
+    - [Filter Response by Status Code  (new in `v1.3.0`)](#filter-response-by-status-code--new-in-v130)
 - [Comparison w/ Similar Tools](#-comparison-w-similar-tools)
 - [Common Problems/Issues (FAQ)](#-common-problemsissues-faq)
     - [No file descriptors available](#no-file-descriptors-available)
+    - [Progress bars print one line at a time](#progress-bars-print-one-line-at-a-time)
 
 ## 💿 Installation
 
@@ -90,28 +92,9 @@ This attack is also known as Predictable Resource Location, File Enumeration, Di
 
 Releases for multiple architectures can be found in the [Releases](https://github.com/epi052/feroxbuster/releases) section.  The latest release for each of the following systems can be downloaded and executed as shown below.
 
-#### Linux x86
+#### Linux (32 and 64-bit) & MacOS
 ```
-curl -sLO https://github.com/epi052/feroxbuster/releases/latest/download/x86-linux-feroxbuster.zip
-unzip x86-linux-feroxbuster.zip
-chmod +x ./feroxbuster
-./feroxbuster -V
-```
-#### Linux x86_64
-
-```
-curl -sLO https://github.com/epi052/feroxbuster/releases/latest/download/x86_64-linux-feroxbuster.zip
-unzip x86_64-linux-feroxbuster.zip
-chmod +x ./feroxbuster
-./feroxbuster -V
-```
-
-#### MacOS x86_64
-```
-curl -sLO https://github.com/epi052/feroxbuster/releases/latest/download/x86_64-macos-feroxbuster.zip
-unzip x86_64-macos-feroxbuster.zip
-chmod +x ./feroxbuster
-./feroxbuster -V
+curl -sL https://raw.githubusercontent.com/epi052/feroxbuster/master/install-nix.sh | bash
 ```
 
 #### Windows x86
@@ -239,8 +222,8 @@ Configuration begins with with the following built-in default values baked into
 - threads: `50`
 - verbosity: `0` (no logging enabled)
 - scan_limit: `0` (no limit imposed on concurrent scans)
-- statuscodes: `200 204 301 302 307 308 401 403 405`
-- useragent: `feroxbuster/VERSION`
+- status_codes: `200 204 301 302 307 308 401 403 405`
+- user_agent: `feroxbuster/VERSION`
 - recursion depth: `4`
 - auto-filter wildcards - `true`
 - output: `stdout`
@@ -290,7 +273,8 @@ A pre-made configuration file with examples of all available settings can be fou
 # Any setting used here can be overridden by the corresponding command line option/argument
 #
 # wordlist = "/wordlists/jhaddix/all.txt"
-# statuscodes = [200, 500]
+# status_codes = [200, 500]
+# filter_status = [301]
 # threads = 1
 # timeout = 5
 # proxy = "http://127.0.0.1:8080"
@@ -298,17 +282,17 @@ A pre-made configuration file with examples of all available settings can be fou
 # scan_limit = 6
 # quiet = true
 # output = "/targets/ellingson_mineral_company/gibson.txt"
-# useragent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0"
+# user_agent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0"
 # redirects = true
 # insecure = true
 # extensions = ["php", "html"]
-# norecursion = true
-# addslash = true
+# no_recursion = true
+# add_slash = true
 # stdin = true
-# dontfilter = true
+# dont_filter = true
 # extract_links = true
 # depth = 1
-# sizefilters = [5174]
+# filter_size = [5174]
 # queries = [["name","value"], ["rick", "astley"]]
 
 # headers can be specified on multiple lines or as an inline table
@@ -333,13 +317,13 @@ USAGE:
     feroxbuster [FLAGS] [OPTIONS] --url <URL>...
 
 FLAGS:
-    -f, --addslash         Append / to each request
-    -D, --dontfilter       Don't auto-filter wildcard responses
+    -f, --add-slash        Append / to each request
+    -D, --dont-filter      Don't auto-filter wildcard responses
     -e, --extract-links    Extract links from response body (html, javascript, etc...); make new requests based on
                            findings (default: false)
     -h, --help             Prints help information
     -k, --insecure         Disables TLS certificate validation
-    -n, --norecursion      Do not scan recursively
+    -n, --no-recursion     Do not scan recursively
     -q, --quiet            Only print URLs; Don't print status codes, response size, running config, etc...
     -r, --redirects        Follow redirects
         --stdin            Read url(s) from STDIN
@@ -349,17 +333,19 @@ FLAGS:
 OPTIONS:
     -d, --depth <RECURSION_DEPTH>           Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)
     -x, --extensions <FILE_EXTENSION>...    File extension(s) to search for (ex: -x php -x pdf js)
+    -S, --filter-size <SIZE>...             Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)
+    -C, --filter-status <STATUS_CODE>...    Filter out status codes (deny list) (ex: -C 200 -S 401)
     -H, --headers <HEADER>...               Specify HTTP headers (ex: -H Header:val 'stuff: things')
     -o, --output <FILE>                     Output file to write results to (default: stdout)
     -p, --proxy <PROXY>                     Proxy to use for requests (ex: http(s)://host:port, socks5://host:port)
     -Q, --query <QUERY>...                  Specify URL query parameters (ex: -Q token=stuff -Q secret=key)
-    -L, --scan-limit <SCAN_LIMIT>           Limit total number of concurrent scans (default: 7)
-    -S, --sizefilter <SIZE>...              Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)
-    -s, --statuscodes <STATUS_CODE>...      Status Codes of interest (default: 200 204 301 302 307 308 401 403 405)
+    -L, --scan-limit <SCAN_LIMIT>           Limit total number of concurrent scans (default: 0, i.e. no limit)
+    -s, --status-codes <STATUS_CODE>...     Status Codes to include (allow list) (default: 200 204 301 302 307 308 401
+                                            403 405)
     -t, --threads <THREADS>                 Number of concurrent threads (default: 50)
     -T, --timeout <SECONDS>                 Number of seconds before a request times out (default: 7)
     -u, --url <URL>...                      The target URL(s) (required, unless --stdin used)
-    -a, --useragent <USER_AGENT>            Sets the User-Agent (default: feroxbuster/VERSION)
+    -a, --user-agent <USER_AGENT>           Sets the User-Agent (default: feroxbuster/VERSION)
     -w, --wordlist <FILE>                   Path to the wordlist
 ```
 
@@ -403,10 +389,21 @@ Example request/response with `--extract-links` enabled:
 ./feroxbuster -u http://127.1 --extract-links
 ```
 
+Here's a comparison of a wordlist-only scan vs `--extract-links` using [Feline](https://www.hackthebox.eu/home/machines/profile/274) from Hack the Box:
+
+Wordlist only
+
+![normal-scan-cmp-extract](img/normal-scan-cmp-extract.gif)
+
+With `--extract-links`
+
+![extract-scan-cmp-normal](img/extract-scan-cmp-normal.gif)
+
+
 ### IPv6, non-recursive scan with INFO-level logging enabled
 
 ```
-./feroxbuster -u http://[::1] --norecursion -vv
+./feroxbuster -u http://[::1] --no-recursion -vv
 ```
 
 ### Read urls from STDIN; pipe only resulting urls out to another tool
@@ -445,6 +442,16 @@ discovered directories can only begin scanning when the total number of active s
 
 ![limit-demo](img/limit-demo.gif)
 
+### Filter Response by Status Code  (new in `v1.3.0`)
+
+Version 1.3.0 included an overhaul to the filtering system which will allow for a wide array of filters to be added 
+with minimal effort. The first such filter is a Status Code Filter. As responses come back from the scanned server,
+each one is checked against a list of known filters and either displayed or not according to which filters are set.
+
+```
+./feroxbuster -u http://127.1 --filter-status 301
+```
+
 ## 🧐 Comparison w/ Similar Tools
 
 There are quite a few similar tools for forced browsing/content discovery.  Burp Suite Pro, Dirb, Dirbuster, etc... 
@@ -467,7 +474,7 @@ a few of the use-cases in which feroxbuster may be a better fit:
 |------------------------------------------------------------------|---|---|---|
 | fast                                                             | ✔ | ✔ | ✔ |
 | easy to use                                                      | ✔ | ✔ |   |
-| blacklist status codes (in addition to whitelist)                |   | ✔ | ✔ |
+| filter out responses by status code (new in `v1.3.0`)            | ✔ | ✔ | ✔ |
 | allows recursion                                                 | ✔ |   | ✔ |
 | can specify query parameters                                     | ✔ |   | ✔ |
 | SOCKS proxy support                                              | ✔ |   |   |
@@ -558,3 +565,12 @@ This allows fast cycling of sockets in time_wait state and re-using them. Make s
 ```
 sudo sysctl net.ipv4.tcp_tw_reuse=1 
 ```
+
+### Progress bars print one line at a time
+
+`feroxbuster` needs a terminal width of at least the size of what's being printed in order to do progress bar printing correctly.  If your width is too small, you may see output like what's shown below.
+
+![small-term](img/small-term.png)
+
+If you can, simply make the terminal wider and rerun.  If you're unable to make your terminal wider
+consider using `-q` to suppress the progress bars.

ferox-config.toml.example

@@ -8,7 +8,8 @@
 # Any setting used here can be overridden by the corresponding command line option/argument
 #
 # wordlist = "/wordlists/seclists/Discovery/Web-Content/raft-medium-directories.txt"
-# statuscodes = [200, 500]
+# status_codes = [200, 500]
+# filter_status = [301]
 # threads = 1
 # timeout = 5
 # proxy = "http://127.0.0.1:8080"
@@ -16,17 +17,17 @@
 # scan_limit = 6
 # quiet = true
 # output = "/targets/ellingson_mineral_company/gibson.txt"
-# useragent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0"
+# user_agent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0"
 # redirects = true
 # insecure = true
 # extensions = ["php", "html"]
-# norecursion = true
-# addslash = true
+# no_recursion = true
+# add_slash = true
 # stdin = true
-# dontfilter = true
+# dont_filter = true
 # extract_links = true
 # depth = 1
-# sizefilters = [5174]
+# filter_size = [5174]
 # queries = [["name","value"], ["rick", "astley"]]
 
 # headers can be specified on multiple lines or as an inline table

install-nix.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+
+BASE_URL=https://github.com/epi052/feroxbuster/releases/latest/download
+
+MAC_ZIP=x86_64-macos-feroxbuster.zip
+MAC_URL="${BASE_URL}/${MAC_ZIP}"
+
+LIN32_ZIP=x86-linux-feroxbuster.zip
+LIN32_URL="${BASE_URL}/${LIN32_ZIP}"
+
+LIN64_ZIP=x86_64-linux-feroxbuster.zip
+LIN64_URL="${BASE_URL}/${LIN64_ZIP}"
+
+EMOJI_URL=https://gist.github.com/epi052/8196b550ea51d0907ad4b93751b1b57d/raw/6112c9f32ae07922983fdc549c54fd3fb9a38e4c/NotoColorEmoji.ttf
+
+echo "[+] Installing feroxbuster!"
+
+if [[ "$(uname)" == "Darwin" ]]; then
+    echo "[=] Found MacOS, downloading from ${MAC_URL}"
+
+    curl -sLO "${MAC_URL}"
+    unzip -o "${MAC_ZIP}" > /dev/null
+    rm "${MAC_ZIP}"
+elif [[ "$(expr substr $(uname -s) 1 5)" == "Linux" ]]; then
+    if [[ $(getconf LONG_BIT) == 32 ]]; then
+        echo "[=] Found 32-bit Linux, downloading from ${LIN32_URL}"
+
+        curl -sLO "${LIN32_URL}"
+        unzip -o "${LIN32_ZIP}" > /dev/null
+        rm "${LIN32_ZIP}"
+    else
+        echo "[=] Found 64-bit Linux, downloading from ${LIN64_URL}"
+
+        curl -sLO "${LIN64_URL}"
+        unzip -o "${LIN64_ZIP}" > /dev/null
+        rm "${LIN64_ZIP}"
+    fi
+
+    echo "[=] Installing Noto Emoji Font"
+    mkdir -p ~/.fonts
+    pushd ~/.fonts 2>&1 >/dev/null
+
+    curl -sLO "${EMOJI_URL}"
+
+    fc-cache -f -v >/dev/null
+
+    popd 2>&1 >/dev/null
+    echo "[+] Noto Emoji Font installed"
+fi
+
+chmod +x ./feroxbuster
+
+echo "[+] Installed feroxbuster version $(./feroxbuster -V)"
+
+
+

src/banner.rs

@@ -161,7 +161,7 @@ by Ben "epi" Risher {}                  ver: {}"#,
 
     let mut codes = vec![];
 
-    for code in &config.statuscodes {
+    for code in &config.status_codes {
         codes.push(status_colorizer(&code.to_string()))
     }
 
@@ -190,6 +190,27 @@ by Ben "epi" Risher {}                  ver: {}"#,
     )
     .unwrap_or_default(); // 🆗
 
+    if !config.filter_status.is_empty() {
+        // exception here for optional print due to me wanting the allows and denys to be printed
+        // one after the other
+        let mut code_filters = vec![];
+
+        for code in &config.filter_status {
+            code_filters.push(status_colorizer(&code.to_string()))
+        }
+
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!(
+                "\u{1f5d1}",
+                "Status Code Filters",
+                format!("[{}]", code_filters.join(", "))
+            )
+        )
+        .unwrap_or_default(); // 🗑
+    }
+
     writeln!(
         &mut writer,
         "{}",
@@ -200,7 +221,7 @@ by Ben "epi" Risher {}                  ver: {}"#,
     writeln!(
         &mut writer,
         "{}",
-        format_banner_entry!("\u{1F9a1}", "User-Agent", config.useragent)
+        format_banner_entry!("\u{1F9a1}", "User-Agent", config.user_agent)
     )
     .unwrap_or_default(); // 🦡
 
@@ -234,8 +255,8 @@ by Ben "epi" Risher {}                  ver: {}"#,
         }
     }
 
-    if !config.sizefilters.is_empty() {
-        for filter in &config.sizefilters {
+    if !config.filter_size.is_empty() {
+        for filter in &config.filter_size {
             writeln!(
                 &mut writer,
                 "{}",
@@ -309,11 +330,11 @@ by Ben "epi" Risher {}                  ver: {}"#,
         .unwrap_or_default(); // 📍
     }
 
-    if config.dontfilter {
+    if config.dont_filter {
         writeln!(
             &mut writer,
             "{}",
-            format_banner_entry!("\u{1f92a}", "Filter Wildcards", !config.dontfilter)
+            format_banner_entry!("\u{1f92a}", "Filter Wildcards", !config.dont_filter)
         )
         .unwrap_or_default(); // 🤪
     }
@@ -355,16 +376,16 @@ by Ben "epi" Risher {}                  ver: {}"#,
         _ => {}
     }
 
-    if config.addslash {
+    if config.add_slash {
         writeln!(
             &mut writer,
             "{}",
-            format_banner_entry!("\u{1fa93}", "Add Slash", config.addslash)
+            format_banner_entry!("\u{1fa93}", "Add Slash", config.add_slash)
         )
         .unwrap_or_default(); // 🪓
     }
 
-    if !config.norecursion {
+    if !config.no_recursion {
         if config.depth == 0 {
             writeln!(
                 &mut writer,
@@ -384,7 +405,7 @@ by Ben "epi" Risher {}                  ver: {}"#,
         writeln!(
             &mut writer,
             "{}",
-            format_banner_entry!("\u{1f6ab}", "Do Not Recurse", config.norecursion)
+            format_banner_entry!("\u{1f6ab}", "Do Not Recurse", config.no_recursion)
         )
         .unwrap_or_default(); // 🚫
     }
@@ -436,7 +457,7 @@ mod tests {
     /// test to hit no execution of statuscode for loop in banner
     async fn banner_intialize_without_status_codes() {
         let mut config = Configuration::default();
-        config.statuscodes = vec![];
+        config.status_codes = vec![];
         initialize(
             &[String::from("http://localhost")],
             &config,

src/client.rs

@@ -10,7 +10,7 @@ use std::time::Duration;
 /// Create and return an instance of [reqwest::Client](https://docs.rs/reqwest/latest/reqwest/struct.Client.html)
 pub fn initialize(
     timeout: u64,
-    useragent: &str,
+    user_agent: &str,
     redirects: bool,
     insecure: bool,
     headers: &HashMap<String, String>,
@@ -27,7 +27,7 @@ pub fn initialize(
 
     let client = Client::builder()
         .timeout(Duration::new(timeout, 0))
-        .user_agent(useragent)
+        .user_agent(user_agent)
         .danger_accept_invalid_certs(insecure)
         .default_headers(header_map)
         .redirect(policy);

src/config.rs

@@ -51,9 +51,13 @@ pub struct Configuration {
     #[serde(default)]
     pub target_url: String,
 
-    /// Status Codes of interest (default: 200 204 301 302 307 308 401 403 405)
-    #[serde(default = "statuscodes")]
-    pub statuscodes: Vec<u16>,
+    /// Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)
+    #[serde(default = "status_codes")]
+    pub status_codes: Vec<u16>,
+
+    /// Status Codes to filter out (deny list)
+    #[serde(default)]
+    pub filter_status: Vec<u16>,
 
     /// Instance of [reqwest::Client](https://docs.rs/reqwest/latest/reqwest/struct.Client.html)
     #[serde(skip)]
@@ -80,8 +84,8 @@ pub struct Configuration {
     pub output: String,
 
     /// Sets the User-Agent (default: feroxbuster/VERSION)
-    #[serde(default = "useragent")]
-    pub useragent: String,
+    #[serde(default = "user_agent")]
+    pub user_agent: String,
 
     /// Follow redirects
     #[serde(default)]
@@ -105,7 +109,7 @@ pub struct Configuration {
 
     /// Do not scan recursively
     #[serde(default)]
-    pub norecursion: bool,
+    pub no_recursion: bool,
 
     /// Extract links from html/javscript
     #[serde(default)]
@@ -113,7 +117,7 @@ pub struct Configuration {
 
     /// Append / to each request
     #[serde(default)]
-    pub addslash: bool,
+    pub add_slash: bool,
 
     /// Read url(s) from STDIN
     #[serde(default)]
@@ -129,14 +133,14 @@ pub struct Configuration {
 
     /// Filter out messages of a particular size
     #[serde(default)]
-    pub sizefilters: Vec<u64>,
+    pub filter_size: Vec<u64>,
 
     /// Don't auto-filter wildcard responses
     #[serde(default)]
-    pub dontfilter: bool,
+    pub dont_filter: bool,
 }
 
-// functions timeout, threads, statuscodes, useragent, wordlist, and depth are used to provide
+// functions timeout, threads, status_codes, user_agent, wordlist, and depth are used to provide
 // defaults in the event that a ferox-config.toml is found but one or more of the values below
 // aren't listed in the config.  This way, we get the correct defaults upon Deserialization
 
@@ -151,7 +155,7 @@ fn threads() -> usize {
 }
 
 /// default status codes
-fn statuscodes() -> Vec<u16> {
+fn status_codes() -> Vec<u16> {
     DEFAULT_STATUS_CODES
         .iter()
         .map(|code| code.as_u16())
@@ -163,8 +167,8 @@ fn wordlist() -> String {
     String::from(DEFAULT_WORDLIST)
 }
 
-/// default useragent
-fn useragent() -> String {
+/// default user-agent
+fn user_agent() -> String {
     format!("feroxbuster/{}", VERSION)
 }
 
@@ -177,22 +181,22 @@ impl Default for Configuration {
     /// Builds the default Configuration for feroxbuster
     fn default() -> Self {
         let timeout = timeout();
-        let useragent = useragent();
-        let client = client::initialize(timeout, &useragent, false, false, &HashMap::new(), None);
+        let user_agent = user_agent();
+        let client = client::initialize(timeout, &user_agent, false, false, &HashMap::new(), None);
 
         Configuration {
             client,
             timeout,
-            useragent,
-            dontfilter: false,
+            user_agent,
+            dont_filter: false,
             quiet: false,
             stdin: false,
             verbosity: 0,
             scan_limit: 0,
-            addslash: false,
+            add_slash: false,
             insecure: false,
             redirects: false,
-            norecursion: false,
+            no_recursion: false,
             extract_links: false,
             proxy: String::new(),
             config: String::new(),
@@ -200,12 +204,13 @@ impl Default for Configuration {
             target_url: String::new(),
             queries: Vec::new(),
             extensions: Vec::new(),
-            sizefilters: Vec::new(),
+            filter_size: Vec::new(),
+            filter_status: Vec::new(),
             headers: HashMap::new(),
             threads: threads(),
             depth: depth(),
             wordlist: wordlist(),
-            statuscodes: statuscodes(),
+            status_codes: status_codes(),
         }
     }
 }
@@ -223,19 +228,20 @@ impl Configuration {
     /// - **timeout**: `7` seconds
     /// - **verbosity**: `0` (no logging enabled)
     /// - **proxy**: `None`
-    /// - **statuscodes**: [`DEFAULT_RESPONSE_CODES`](constant.DEFAULT_RESPONSE_CODES.html)
+    /// - **status_codes**: [`DEFAULT_RESPONSE_CODES`](constant.DEFAULT_RESPONSE_CODES.html)
+    /// - **filter_status**: `None`
     /// - **output**: `None` (print to stdout)
     /// - **quiet**: `false`
-    /// - **useragent**: `feroxer/VERSION`
+    /// - **user_agent**: `feroxer/VERSION`
     /// - **insecure**: `false` (don't be insecure, i.e. don't allow invalid certs)
     /// - **extensions**: `None`
-    /// - **sizefilters**: `None`
+    /// - **filter_size**: `None`
     /// - **headers**: `None`
     /// - **queries**: `None`
-    /// - **norecursion**: `false` (recursively scan enumerated sub-directories)
-    /// - **addslash**: `false`
+    /// - **no_recursion**: `false` (recursively scan enumerated sub-directories)
+    /// - **add_slash**: `false`
     /// - **stdin**: `false`
-    /// - **dontfilter**: `false` (auto filter wildcard responses)
+    /// - **dont_filter**: `false` (auto filter wildcard responses)
     /// - **depth**: `4` (maximum recursion depth)
     /// - **scan_limit**: `0` (no limit on concurrent scans imposed)
     ///
@@ -336,9 +342,29 @@ impl Configuration {
             config.output = String::from(args.value_of("output").unwrap());
         }
 
-        if args.values_of("statuscodes").is_some() {
-            config.statuscodes = args
-                .values_of("statuscodes")
+        if args.values_of("status_codes").is_some() {
+            config.status_codes = args
+                .values_of("status_codes")
+                .unwrap() // already known good
+                .map(|code| {
+                    StatusCode::from_bytes(code.as_bytes())
+                        .unwrap_or_else(|e| {
+                            eprintln!(
+                                "{} {}: {}",
+                                status_colorizer("ERROR"),
+                                module_colorizer("Configuration::new"),
+                                e
+                            );
+                            exit(1)
+                        })
+                        .as_u16()
+                })
+                .collect();
+        }
+
+        if args.values_of("filter_status").is_some() {
+            config.filter_status = args
+                .values_of("filter_status")
                 .unwrap() // already known good
                 .map(|code| {
                     StatusCode::from_bytes(code.as_bytes())
@@ -364,9 +390,9 @@ impl Configuration {
                 .collect();
         }
 
-        if args.values_of("sizefilters").is_some() {
-            config.sizefilters = args
-                .values_of("sizefilters")
+        if args.values_of("filter_size").is_some() {
+            config.filter_size = args
+                .values_of("filter_size")
                 .unwrap() // already known good
                 .map(|size| {
                     size.parse::<u64>().unwrap_or_else(|e| {
@@ -390,8 +416,8 @@ impl Configuration {
             config.quiet = args.is_present("quiet");
         }
 
-        if args.is_present("dontfilter") {
-            config.dontfilter = args.is_present("dontfilter");
+        if args.is_present("dont_filter") {
+            config.dont_filter = args.is_present("dont_filter");
         }
 
         if args.occurrences_of("verbosity") > 0 {
@@ -400,12 +426,12 @@ impl Configuration {
             config.verbosity = args.occurrences_of("verbosity") as u8;
         }
 
-        if args.is_present("norecursion") {
-            config.norecursion = args.is_present("norecursion");
+        if args.is_present("no_recursion") {
+            config.no_recursion = args.is_present("no_recursion");
         }
 
-        if args.is_present("addslash") {
-            config.addslash = args.is_present("addslash");
+        if args.is_present("add_slash") {
+            config.add_slash = args.is_present("add_slash");
         }
 
         if args.is_present("extract_links") {
@@ -425,8 +451,8 @@ impl Configuration {
             config.proxy = String::from(args.value_of("proxy").unwrap());
         }
 
-        if args.value_of("useragent").is_some() {
-            config.useragent = String::from(args.value_of("useragent").unwrap());
+        if args.value_of("user_agent").is_some() {
+            config.user_agent = String::from(args.value_of("user_agent").unwrap());
         }
 
         if args.value_of("timeout").is_some() {
@@ -474,7 +500,7 @@ impl Configuration {
         // the client and store it in the config struct
         if !config.proxy.is_empty()
             || config.timeout != timeout()
-            || config.useragent != useragent()
+            || config.user_agent != user_agent()
             || config.redirects
             || config.insecure
             || !config.headers.is_empty()
@@ -482,7 +508,7 @@ impl Configuration {
             if config.proxy.is_empty() {
                 config.client = client::initialize(
                     config.timeout,
-                    &config.useragent,
+                    &config.user_agent,
                     config.redirects,
                     config.insecure,
                     &config.headers,
@@ -491,7 +517,7 @@ impl Configuration {
             } else {
                 config.client = client::initialize(
                     config.timeout,
-                    &config.useragent,
+                    &config.user_agent,
                     config.redirects,
                     config.insecure,
                     &config.headers,
@@ -527,25 +553,26 @@ impl Configuration {
     fn merge_config(settings: &mut Self, settings_to_merge: Self) {
         settings.threads = settings_to_merge.threads;
         settings.wordlist = settings_to_merge.wordlist;
-        settings.statuscodes = settings_to_merge.statuscodes;
+        settings.status_codes = settings_to_merge.status_codes;
         settings.proxy = settings_to_merge.proxy;
         settings.timeout = settings_to_merge.timeout;
         settings.verbosity = settings_to_merge.verbosity;
         settings.quiet = settings_to_merge.quiet;
         settings.output = settings_to_merge.output;
-        settings.useragent = settings_to_merge.useragent;
+        settings.user_agent = settings_to_merge.user_agent;
         settings.redirects = settings_to_merge.redirects;
         settings.insecure = settings_to_merge.insecure;
         settings.extract_links = settings_to_merge.extract_links;
         settings.extensions = settings_to_merge.extensions;
         settings.headers = settings_to_merge.headers;
         settings.queries = settings_to_merge.queries;
-        settings.norecursion = settings_to_merge.norecursion;
-        settings.addslash = settings_to_merge.addslash;
+        settings.no_recursion = settings_to_merge.no_recursion;
+        settings.add_slash = settings_to_merge.add_slash;
         settings.stdin = settings_to_merge.stdin;
         settings.depth = settings_to_merge.depth;
-        settings.sizefilters = settings_to_merge.sizefilters;
-        settings.dontfilter = settings_to_merge.dontfilter;
+        settings.filter_size = settings_to_merge.filter_size;
+        settings.filter_status = settings_to_merge.filter_status;
+        settings.dont_filter = settings_to_merge.dont_filter;
         settings.scan_limit = settings_to_merge.scan_limit;
     }
 
@@ -582,7 +609,7 @@ mod tests {
     fn setup_config_test() -> Configuration {
         let data = r#"
             wordlist = "/some/path"
-            statuscodes = [201, 301, 401]
+            status_codes = [201, 301, 401]
             threads = 40
             timeout = 5
             proxy = "http://127.0.0.1:8080"
@@ -595,13 +622,14 @@ mod tests {
             extensions = ["html", "php", "js"]
             headers = {stuff = "things", mostuff = "mothings"}
             queries = [["name","value"], ["rick", "astley"]]
-            norecursion = true
-            addslash = true
+            no_recursion = true
+            add_slash = true
             stdin = true
-            dontfilter = true
+            dont_filter = true
             extract_links = true
             depth = 1
-            sizefilters = [4120]
+            filter_size = [4120]
+            filter_status = [201]
         "#;
         let tmp_dir = TempDir::new().unwrap();
         let file = tmp_dir.path().join(DEFAULT_CONFIG_NAME);
@@ -617,23 +645,24 @@ mod tests {
         assert_eq!(config.proxy, String::new());
         assert_eq!(config.target_url, String::new());
         assert_eq!(config.config, String::new());
-        assert_eq!(config.statuscodes, statuscodes());
+        assert_eq!(config.status_codes, status_codes());
         assert_eq!(config.threads, threads());
         assert_eq!(config.depth, depth());
         assert_eq!(config.timeout, timeout());
         assert_eq!(config.verbosity, 0);
         assert_eq!(config.scan_limit, 0);
         assert_eq!(config.quiet, false);
-        assert_eq!(config.dontfilter, false);
-        assert_eq!(config.norecursion, false);
+        assert_eq!(config.dont_filter, false);
+        assert_eq!(config.no_recursion, false);
         assert_eq!(config.stdin, false);
-        assert_eq!(config.addslash, false);
+        assert_eq!(config.add_slash, false);
         assert_eq!(config.redirects, false);
         assert_eq!(config.extract_links, false);
         assert_eq!(config.insecure, false);
         assert_eq!(config.queries, Vec::new());
         assert_eq!(config.extensions, Vec::<String>::new());
-        assert_eq!(config.sizefilters, Vec::<u64>::new());
+        assert_eq!(config.filter_size, Vec::<u64>::new());
+        assert_eq!(config.filter_status, Vec::<u16>::new());
         assert_eq!(config.headers, HashMap::new());
     }
 
@@ -646,9 +675,9 @@ mod tests {
 
     #[test]
     /// parse the test config and see that the value parsed is correct
-    fn config_reads_statuscodes() {
+    fn config_reads_status_codes() {
         let config = setup_config_test();
-        assert_eq!(config.statuscodes, vec![201, 301, 401]);
+        assert_eq!(config.status_codes, vec![201, 301, 401]);
     }
 
     #[test]
@@ -723,9 +752,9 @@ mod tests {
 
     #[test]
     /// parse the test config and see that the value parsed is correct
-    fn config_reads_norecursion() {
+    fn config_reads_no_recursion() {
         let config = setup_config_test();
-        assert_eq!(config.norecursion, true);
+        assert_eq!(config.no_recursion, true);
     }
 
     #[test]
@@ -737,16 +766,16 @@ mod tests {
 
     #[test]
     /// parse the test config and see that the value parsed is correct
-    fn config_reads_dontfilter() {
+    fn config_reads_dont_filter() {
         let config = setup_config_test();
-        assert_eq!(config.dontfilter, true);
+        assert_eq!(config.dont_filter, true);
     }
 
     #[test]
     /// parse the test config and see that the value parsed is correct
-    fn config_reads_addslash() {
+    fn config_reads_add_slash() {
         let config = setup_config_test();
-        assert_eq!(config.addslash, true);
+        assert_eq!(config.add_slash, true);
     }
 
     #[test]
@@ -765,9 +794,16 @@ mod tests {
 
     #[test]
     /// parse the test config and see that the value parsed is correct
-    fn config_reads_sizefilters() {
+    fn config_reads_filter_size() {
         let config = setup_config_test();
-        assert_eq!(config.sizefilters, vec![4120]);
+        assert_eq!(config.filter_size, vec![4120]);
+    }
+
+    #[test]
+    /// parse the test config and see that the value parsed is correct
+    fn config_reads_filter_status() {
+        let config = setup_config_test();
+        assert_eq!(config.filter_status, vec![201]);
     }
 
     #[test]

src/filters.rs

@@ -0,0 +1,142 @@
+use crate::config::CONFIGURATION;
+use crate::utils::get_url_path_length;
+use crate::FeroxResponse;
+use std::any::Any;
+use std::fmt::Debug;
+
+// references:
+//   https://dev.to/magnusstrale/rust-trait-objects-in-a-vector-non-trivial-4co5
+//   https://stackoverflow.com/questions/25339603/how-to-test-for-equality-between-trait-objects
+
+/// FeroxFilter trait; represents different types of possible filters that can be applied to
+/// responses
+pub trait FeroxFilter: Debug + Send + Sync {
+    /// Determine whether or not this particular filter should be applied or not
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool;
+
+    /// delegates to the FeroxFilter-implementing type which gives us the actual type of self
+    fn box_eq(&self, other: &dyn Any) -> bool;
+
+    /// gives us `other` as Any in box_eq
+    fn as_any(&self) -> &dyn Any;
+}
+
+/// implementation of PartialEq, necessary long-form due to "trait cannot be made into an object"
+/// error when attempting to derive PartialEq on the trait itself
+impl PartialEq for Box<dyn FeroxFilter> {
+    /// Perform a comparison of two implementors of the FeroxFilter trait
+    fn eq(&self, other: &Box<dyn FeroxFilter>) -> bool {
+        self.box_eq(other.as_any())
+    }
+}
+
+/// Data holder for two pieces of data needed when auto-filtering out wildcard responses
+///
+/// `dynamic` is the size of the response that will later be combined with the length
+/// of the path of the url requested and used to determine interesting pages from custom
+/// 404s where the requested url is reflected back in the response
+///
+/// `size` is size of the response that should be included with filters passed via runtime
+/// configuration and any static wildcard lengths.
+#[derive(Debug, Default, Clone, PartialEq)]
+pub struct WildcardFilter {
+    /// size of the response that will later be combined with the length of the path of the url
+    /// requested
+    pub dynamic: u64,
+
+    /// size of the response that should be included with filters passed via runtime configuration
+    pub size: u64,
+}
+
+/// implementation of FeroxFilter for WildcardFilter
+impl FeroxFilter for WildcardFilter {
+    /// Examine size, dynamic, and content_len to determine whether or not the response received
+    /// is a wildcard response and therefore should be filtered out
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {:?})", self, response);
+
+        // quick return if dont_filter is set
+        if CONFIGURATION.dont_filter {
+            // --dont-filter applies specifically to wildcard filters, it is not a 100% catch all
+            // for not filtering anything.  As such, it should live in the implementation of
+            // a wildcard filter
+            return false;
+        }
+
+        if self.size > 0 && self.size == response.content_length() {
+            // static wildcard size found during testing
+            // size isn't default, size equals response length, and auto-filter is on
+            log::debug!("static wildcard: filtered out {}", response.url());
+            log::trace!("exit: should_filter_response -> true");
+            return true;
+        }
+
+        if self.dynamic > 0 {
+            // dynamic wildcard offset found during testing
+
+            // I'm about to manually split this url path instead of using reqwest::Url's
+            // builtin parsing. The reason is that they call .split() on the url path
+            // except that I don't want an empty string taking up the last index in the
+            // event that the url ends with a forward slash.  It's ugly enough to be split
+            // into its own function for readability.
+            let url_len = get_url_path_length(&response.url());
+
+            if url_len + self.dynamic == response.content_length() {
+                log::debug!("dynamic wildcard: filtered out {}", response.url());
+                log::trace!("exit: should_filter_response -> true");
+                return true;
+            }
+        }
+        log::trace!("exit: should_filter_response -> false");
+        false
+    }
+
+    /// Compare one WildcardFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
+
+/// Simple implementor of FeroxFilter; used to filter out status codes specified using
+/// -C|--filter-status
+#[derive(Default, Debug, PartialEq)]
+pub struct StatusCodeFilter {
+    /// Status code that should not be displayed to the user
+    pub filter_code: u16,
+}
+
+/// implementation of FeroxFilter for StatusCodeFilter
+impl FeroxFilter for StatusCodeFilter {
+    /// Check `filter_code` against what was passed in via -C|--filter-status
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {:?})", self, response);
+
+        if response.status().as_u16() == self.filter_code {
+            log::debug!(
+                "filtered out {} based on --filter-status of {}",
+                response.url(),
+                self.filter_code
+            );
+            log::trace!("exit: should_filter_response -> true");
+            return true;
+        }
+
+        log::trace!("exit: should_filter_response -> false");
+        false
+    }
+
+    /// Compare one StatusCodeFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/heuristics.rs

@@ -1,11 +1,12 @@
 use crate::config::{CONFIGURATION, PROGRESS_PRINTER};
+use crate::filters::WildcardFilter;
 use crate::scanner::should_filter_response;
 use crate::utils::{
     ferox_print, format_url, get_url_path_length, make_request, module_colorizer, status_colorizer,
 };
+use crate::FeroxResponse;
 use console::style;
 use indicatif::ProgressBar;
-use reqwest::Response;
 use std::process;
 use tokio::sync::mpsc::UnboundedSender;
 use uuid::Uuid;
@@ -13,24 +14,6 @@ use uuid::Uuid;
 /// length of a standard UUID, used when determining wildcard responses
 const UUID_LENGTH: u64 = 32;
 
-/// Data holder for two pieces of data needed when auto-filtering out wildcard responses
-///
-/// `dynamic` is the size of the response that will later be combined with the length
-/// of the path of the url requested and used to determine interesting pages from custom
-/// 404s where the requested url is reflected back in the response
-///
-/// `size` is size of the response that should be included with filters passed via runtime
-/// configuration and any static wildcard lengths.
-#[derive(Default, Debug, PartialEq, Copy, Clone)]
-pub struct WildcardFilter {
-    /// size of the response that will later be combined with the length of the path of the url
-    /// requested
-    pub dynamic: u64,
-
-    /// size of the response that should be included with filters passed via runtime configuration
-    pub size: u64,
-}
-
 /// Simple helper to return a uuid, formatted as lowercase without hyphens
 ///
 /// `length` determines the number of uuids to string together. Each uuid
@@ -66,8 +49,8 @@ pub async fn wildcard_test(
         tx_file
     );
 
-    if CONFIGURATION.dontfilter {
-        // early return, dontfilter scans don't need tested
+    if CONFIGURATION.dont_filter {
+        // early return, dont_filter scans don't need tested
         log::trace!("exit: wildcard_test -> None");
         return None;
     }
@@ -75,13 +58,13 @@ pub async fn wildcard_test(
     let clone_req_one = tx_file.clone();
     let clone_req_two = tx_file.clone();
 
-    if let Some(resp_one) = make_wildcard_request(&target_url, 1, clone_req_one).await {
+    if let Some(ferox_response) = make_wildcard_request(&target_url, 1, clone_req_one).await {
         bar.inc(1);
 
         // found a wildcard response
         let mut wildcard = WildcardFilter::default();
 
-        let wc_length = resp_one.content_length().unwrap_or(0);
+        let wc_length = ferox_response.content_length();
 
         if wc_length == 0 {
             log::trace!("exit: wildcard_test -> Some({:?})", wildcard);
@@ -93,25 +76,23 @@ pub async fn wildcard_test(
         if let Some(resp_two) = make_wildcard_request(&target_url, 3, clone_req_two).await {
             bar.inc(1);
 
-            let wc2_length = resp_two.content_length().unwrap_or(0);
+            let wc2_length = resp_two.content_length();
 
             if wc2_length == wc_length + (UUID_LENGTH * 2) {
                 // second length is what we'd expect to see if the requested url is
                 // reflected in the response along with some static content; aka custom 404
-                let url_len = get_url_path_length(&resp_one.url());
+                let url_len = get_url_path_length(&ferox_response.url());
 
                 wildcard.dynamic = wc_length - url_len;
 
-                if !CONFIGURATION.quiet
-                    && !should_filter_response(&wildcard.dynamic, &resp_one.url())
-                {
+                if !CONFIGURATION.quiet {
                     let msg = format!(
                             "{} {:>10} Wildcard response is dynamic; {} ({} + url length) responses; toggle this behavior by using {}\n",
                             status_colorizer("WLD"),
                             wildcard.dynamic,
                             style("auto-filtering").yellow(),
                             style(wc_length - url_len).cyan(),
-                            style("--dontfilter").yellow()
+                            style("--dont-filter").yellow()
                         );
 
                     ferox_print(&msg, &PROGRESS_PRINTER);
@@ -125,15 +106,14 @@ pub async fn wildcard_test(
             } else if wc_length == wc2_length {
                 wildcard.size = wc_length;
 
-                if !CONFIGURATION.quiet && !should_filter_response(&wildcard.size, &resp_one.url())
-                {
+                if !CONFIGURATION.quiet {
                     let msg = format!(
                         "{} {:>10} Wildcard response is static; {} {} responses; toggle this behavior by using {}\n",
                         status_colorizer("WLD"),
                         wc_length,
                         style("auto-filtering").yellow(),
                         style(wc_length).cyan(),
-                        style("--dontfilter").yellow()
+                        style("--dont-filter").yellow()
                     );
 
                     ferox_print(&msg, &PROGRESS_PRINTER);
@@ -167,7 +147,7 @@ async fn make_wildcard_request(
     target_url: &str,
     length: usize,
     tx_file: UnboundedSender<String>,
-) -> Option<Response> {
+) -> Option<FeroxResponse> {
     log::trace!(
         "enter: make_wildcard_request({}, {}, {:?})",
         target_url,
@@ -180,7 +160,7 @@ async fn make_wildcard_request(
     let nonexistent = match format_url(
         target_url,
         &unique_str,
-        CONFIGURATION.addslash,
+        CONFIGURATION.add_slash,
         &CONFIGURATION.queries,
         None,
     ) {
@@ -197,20 +177,21 @@ async fn make_wildcard_request(
     match make_request(&CONFIGURATION.client, &nonexistent.to_owned()).await {
         Ok(response) => {
             if CONFIGURATION
-                .statuscodes
+                .status_codes
                 .contains(&response.status().as_u16())
             {
                 // found a wildcard response
-                let url_len = get_url_path_length(&response.url());
-                let content_len = response.content_length().unwrap_or(0);
+                let ferox_response = FeroxResponse::from(response, false).await;
+                let url_len = get_url_path_length(&ferox_response.url());
+                let content_len = ferox_response.content_length();
 
-                if !CONFIGURATION.quiet && !should_filter_response(&content_len, &response.url()) {
+                if !CONFIGURATION.quiet && !should_filter_response(&ferox_response) {
                     let msg = format!(
                         "{} {:>10} Got {} for {} (url length: {})\n",
                         wildcard,
                         content_len,
-                        status_colorizer(&response.status().as_str()),
-                        response.url(),
+                        status_colorizer(&ferox_response.status().as_str()),
+                        ferox_response.url(),
                         url_len
                     );
 
@@ -223,18 +204,16 @@ async fn make_wildcard_request(
                     );
                 }
 
-                if response.status().is_redirection() {
+                if ferox_response.status().is_redirection() {
                     // show where it goes, if possible
-                    if let Some(next_loc) = response.headers().get("Location") {
+                    if let Some(next_loc) = ferox_response.headers().get("Location") {
                         let next_loc_str = next_loc.to_str().unwrap_or("Unknown");
-                        if !CONFIGURATION.quiet
-                            && !should_filter_response(&content_len, &response.url())
-                        {
+                        if !CONFIGURATION.quiet && !should_filter_response(&ferox_response) {
                             let msg = format!(
                                 "{} {:>10} {} redirects to => {}\n",
                                 wildcard,
                                 content_len,
-                                response.url(),
+                                ferox_response.url(),
                                 next_loc_str
                             );
 
@@ -248,8 +227,8 @@ async fn make_wildcard_request(
                         }
                     }
                 }
-                log::trace!("exit: make_wildcard_request -> {:?}", response);
-                return Some(response);
+                log::trace!("exit: make_wildcard_request -> {:?}", ferox_response);
+                return Some(ferox_response);
             }
         }
         Err(e) => {
@@ -276,7 +255,7 @@ pub async fn connectivity_test(target_urls: &[String]) -> Vec<String> {
         let request = match format_url(
             target_url,
             "",
-            CONFIGURATION.addslash,
+            CONFIGURATION.add_slash,
             &CONFIGURATION.queries,
             None,
         ) {

src/lib.rs

@@ -2,6 +2,7 @@ pub mod banner;
 pub mod client;
 pub mod config;
 pub mod extractor;
+pub mod filters;
 pub mod heuristics;
 pub mod logger;
 pub mod parser;
@@ -61,7 +62,7 @@ pub const DEFAULT_STATUS_CODES: [StatusCode; 9] = [
 pub const DEFAULT_CONFIG_NAME: &str = "ferox-config.toml";
 
 /// A `FeroxResponse`, derived from a `Response` to a submitted `Request`
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct FeroxResponse {
     /// The final `Url` of this `FeroxResponse`
     url: Url,

src/parser.rs

@@ -68,15 +68,15 @@ pub fn initialize() -> App<'static, 'static> {
                 ),
         )
         .arg(
-            Arg::with_name("statuscodes")
+            Arg::with_name("status_codes")
                 .short("s")
-                .long("statuscodes")
+                .long("status-codes")
                 .value_name("STATUS_CODE")
                 .takes_value(true)
                 .multiple(true)
                 .use_delimiter(true)
                 .help(
-                    "Status Codes of interest (default: 200 204 301 302 307 308 401 403 405)",
+                    "Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)",
                 ),
         )
         .arg(
@@ -87,9 +87,9 @@ pub fn initialize() -> App<'static, 'static> {
                 .help("Only print URLs; Don't print status codes, response size, running config, etc...")
         )
         .arg(
-            Arg::with_name("dontfilter")
+            Arg::with_name("dont_filter")
                 .short("D")
-                .long("dontfilter")
+                .long("dont-filter")
                 .takes_value(false)
                 .help("Don't auto-filter wildcard responses")
         )
@@ -102,9 +102,9 @@ pub fn initialize() -> App<'static, 'static> {
                 .takes_value(true),
         )
         .arg(
-            Arg::with_name("useragent")
+            Arg::with_name("user_agent")
                 .short("a")
-                .long("useragent")
+                .long("user-agent")
                 .value_name("USER_AGENT")
                 .takes_value(true)
                 .help(
@@ -162,16 +162,16 @@ pub fn initialize() -> App<'static, 'static> {
                 ),
         )
         .arg(
-            Arg::with_name("norecursion")
+            Arg::with_name("no_recursion")
                 .short("n")
-                .long("norecursion")
+                .long("no-recursion")
                 .takes_value(false)
                 .help("Do not scan recursively")
         )
         .arg(
-            Arg::with_name("addslash")
+            Arg::with_name("add_slash")
                 .short("f")
-                .long("addslash")
+                .long("add-slash")
                 .takes_value(false)
                 .conflicts_with("extensions")
                 .help("Append / to each request")
@@ -184,9 +184,9 @@ pub fn initialize() -> App<'static, 'static> {
                 .conflicts_with("url")
         )
         .arg(
-            Arg::with_name("sizefilters")
+            Arg::with_name("filter_size")
                 .short("S")
-                .long("sizefilter")
+                .long("filter-size")
                 .value_name("SIZE")
                 .takes_value(true)
                 .multiple(true)
@@ -195,6 +195,18 @@ pub fn initialize() -> App<'static, 'static> {
                     "Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)",
                 ),
         )
+        .arg(
+            Arg::with_name("filter_status")
+                .short("C")
+                .long("filter-status")
+                .value_name("STATUS_CODE")
+                .takes_value(true)
+                .multiple(true)
+                .use_delimiter(true)
+                .help(
+                    "Filter out status codes (deny list) (ex: -C 200 -S 401)",
+                ),
+        )
         .arg(
             Arg::with_name("extract_links")
                 .short("e")
@@ -225,7 +237,7 @@ EXAMPLES:
         ./feroxbuster -u http://127.1 -H Accept:application/json "Authorization: Bearer {token}"
 
     IPv6, non-recursive scan with INFO-level logging enabled:
-        ./feroxbuster -u http://[::1] --norecursion -vv
+        ./feroxbuster -u http://[::1] --no-recursion -vv
 
     Read urls from STDIN; pipe only resulting urls out to another tool
         cat targets | ./feroxbuster --stdin --quiet -s 200 301 302 --redirects -x js | fff -s 200 -o js-files

src/reporter.rs

@@ -94,7 +94,7 @@ async fn spawn_terminal_reporter(
     while let Some(resp) = resp_chan.recv().await {
         log::debug!("received {} on reporting channel", resp.url());
 
-        if CONFIGURATION.statuscodes.contains(&resp.status().as_u16()) {
+        if CONFIGURATION.status_codes.contains(&resp.status().as_u16()) {
             let report = if CONFIGURATION.quiet {
                 // -q used, just need the url
                 format!("{}\n", resp.url())

src/scanner.rs

@@ -1,7 +1,7 @@
 use crate::config::{CONFIGURATION, PROGRESS_BAR};
 use crate::extractor::get_links;
-use crate::heuristics::WildcardFilter;
-use crate::utils::{format_url, get_current_depth, get_url_path_length, make_request};
+use crate::filters::{FeroxFilter, StatusCodeFilter, WildcardFilter};
+use crate::utils::{format_url, get_current_depth, make_request};
 use crate::{heuristics, progress, FeroxChannel, FeroxResponse};
 use futures::future::{BoxFuture, FutureExt};
 use futures::{stream, StreamExt};
@@ -23,8 +23,8 @@ lazy_static! {
     /// Set of urls that have been sent to [scan_url](fn.scan_url.html), used for deduplication
     static ref SCANNED_URLS: RwLock<HashSet<String>> = RwLock::new(HashSet::new());
 
-    /// Vector of WildcardFilters that have been ID'd through heuristics
-    static ref WILDCARD_FILTERS: Arc<RwLock<Vec<Arc<WildcardFilter>>>> = Arc::new(RwLock::new(Vec::<Arc<WildcardFilter>>::new()));
+    /// Vector of implementors of the FeroxFilter trait
+    static ref FILTERS: Arc<RwLock<Vec<Box<dyn FeroxFilter>>>> = Arc::new(RwLock::new(Vec::<Box<dyn FeroxFilter>>::new()));
 
     /// Bounded semaphore used as a barrier to limit concurrent scans
     static ref SCAN_LIMITER: Semaphore = Semaphore::new(CONFIGURATION.scan_limit);
@@ -67,37 +67,37 @@ fn add_url_to_list_of_scanned_urls(resp: &str, scanned_urls: &RwLock<HashSet<Str
     }
 }
 
-/// Adds the given WildcardFilter to `WILDCARD_FILTERS`
+/// Adds the given FeroxFilter to the given list of FeroxFilter implementors
 ///
-/// If `WILDCARD_FILTERS` did not already contain the filter, return true; otherwise return false
-fn add_filter_to_list_of_wildcard_filters(
-    filter: Arc<WildcardFilter>,
-    wildcard_filters: Arc<RwLock<Vec<Arc<WildcardFilter>>>>,
+/// If the given list did not already contain the filter, return true; otherwise return false
+fn add_filter_to_list_of_ferox_filters(
+    filter: Box<dyn FeroxFilter>,
+    ferox_filters: Arc<RwLock<Vec<Box<dyn FeroxFilter>>>>,
 ) -> bool {
     log::trace!(
-        "enter: add_filter_to_list_of_wildcard_filters({:?}, {:?})",
+        "enter: add_filter_to_list_of_ferox_filters({:?}, {:?})",
         filter,
-        wildcard_filters
+        ferox_filters
     );
 
-    match wildcard_filters.write() {
+    match ferox_filters.write() {
         Ok(mut filters) => {
             // If the set did not contain the assigned filter, true is returned.
             // If the set did contain the assigned filter, false is returned.
             if filters.contains(&filter) {
-                log::trace!("exit: add_filter_to_list_of_wildcard_filters -> false");
+                log::trace!("exit: add_filter_to_list_of_ferox_filters -> false");
                 return false;
             }
 
             filters.push(filter);
 
-            log::trace!("exit: add_filter_to_list_of_wildcard_filters -> true");
+            log::trace!("exit: add_filter_to_list_of_ferox_filters -> true");
             true
         }
         Err(e) => {
             // poisoned lock
             log::error!("Set of wildcard filters poisoned: {}", e);
-            log::trace!("exit: add_filter_to_list_of_wildcard_filters -> false");
+            log::trace!("exit: add_filter_to_list_of_ferox_filters -> false");
             false
         }
     }
@@ -178,7 +178,7 @@ fn create_urls(target_url: &str, word: &str, extensions: &[String]) -> Vec<Url>
     if let Ok(url) = format_url(
         &target_url,
         &word,
-        CONFIGURATION.addslash,
+        CONFIGURATION.add_slash,
         &CONFIGURATION.queries,
         None,
     ) {
@@ -189,7 +189,7 @@ fn create_urls(target_url: &str, word: &str, extensions: &[String]) -> Vec<Url>
         if let Ok(url) = format_url(
             &target_url,
             &word,
-            CONFIGURATION.addslash,
+            CONFIGURATION.add_slash,
             &CONFIGURATION.queries,
             Some(ext),
         ) {
@@ -242,6 +242,7 @@ fn response_is_directory(response: &FeroxResponse) -> bool {
         }
     } else if response.status().is_success() {
         // status code is 2xx, need to check if it ends in /
+
         if response.url().as_str().ends_with('/') {
             log::debug!("{} is directory suitable for recursion", response.url());
             log::trace!("exit: is_directory -> true");
@@ -337,43 +338,24 @@ async fn try_recursion(
 
 /// Simple helper to stay DRY; determines whether or not a given `FeroxResponse` should be reported
 /// to the user or not.
-pub fn should_filter_response(content_len: &u64, url: &Url) -> bool {
-    if CONFIGURATION.sizefilters.contains(content_len) {
-        // filtered value from --sizefilters, move on to the next url
-        log::debug!("size filter: filtered out {}", url);
+pub fn should_filter_response(response: &FeroxResponse) -> bool {
+    if CONFIGURATION
+        .filter_size
+        .contains(&response.content_length())
+    {
+        // filtered value from --filter-size, size filters and wildcards are two separate filters
+        // and are applied independently
+        log::debug!("size filter: filtered out {}", response.url());
         return true;
     }
 
-    match WILDCARD_FILTERS.read() {
+    match FILTERS.read() {
         Ok(filters) => {
             for filter in filters.iter() {
-                if CONFIGURATION.dontfilter {
-                    // quick return if dontfilter is set
-                    return false;
-                }
-
-                if filter.size > 0 && filter.size == *content_len {
-                    // static wildcard size found during testing
-                    // size isn't default, size equals response length, and auto-filter is on
-                    log::debug!("static wildcard: filtered out {}", url);
+                // wildcard.should_filter goes here
+                if filter.should_filter_response(&response) {
                     return true;
                 }
-
-                if filter.dynamic > 0 {
-                    // dynamic wildcard offset found during testing
-
-                    // I'm about to manually split this url path instead of using reqwest::Url's
-                    // builtin parsing. The reason is that they call .split() on the url path
-                    // except that I don't want an empty string taking up the last index in the
-                    // event that the url ends with a forward slash.  It's ugly enough to be split
-                    // into its own function for readability.
-                    let url_len = get_url_path_length(&url);
-
-                    if url_len + filter.dynamic == *content_len {
-                        log::debug!("dynamic wildcard: filtered out {}", url);
-                        return true;
-                    }
-                }
             }
         }
         Err(e) => {
@@ -412,16 +394,14 @@ async fn make_requests(
             let ferox_response = FeroxResponse::from(response, CONFIGURATION.extract_links).await;
 
             // do recursion if appropriate
-            if !CONFIGURATION.norecursion {
+            if !CONFIGURATION.no_recursion {
                 try_recursion(&ferox_response, base_depth, dir_chan.clone()).await;
             }
 
             // purposefully doing recursion before filtering. the thought process is that
             // even though this particular url is filtered, subsequent urls may not
 
-            let content_len = &ferox_response.content_length();
-
-            if should_filter_response(content_len, &ferox_response.url()) {
+            if should_filter_response(&ferox_response) {
                 continue;
             }
 
@@ -440,7 +420,7 @@ async fn make_requests(
                     let new_url = match format_url(
                         &new_link,
                         &"",
-                        CONFIGURATION.addslash,
+                        CONFIGURATION.add_slash,
                         &CONFIGURATION.queries,
                         None,
                     ) {
@@ -458,8 +438,7 @@ async fn make_requests(
                         FeroxResponse::from(new_response, CONFIGURATION.extract_links).await;
 
                     // filter if necessary
-                    let new_content_len = &new_ferox_response.content_length();
-                    if should_filter_response(new_content_len, &new_ferox_response.url()) {
+                    if should_filter_response(&new_ferox_response) {
                         continue;
                     }
 
@@ -476,7 +455,7 @@ async fn make_requests(
                         continue;
                     }
 
-                    if !CONFIGURATION.norecursion {
+                    if !CONFIGURATION.no_recursion {
                         log::debug!(
                             "Recursive extraction: {} ({})",
                             new_ferox_response.url(),
@@ -594,13 +573,23 @@ pub async fn scan_url(
         .await
     });
 
+    // add any wildcard filters to `FILTERS`
     let filter =
         match heuristics::wildcard_test(&target_url, wildcard_bar, heuristics_file_clone).await {
-            Some(f) => Arc::new(f),
-            None => Arc::new(WildcardFilter::default()),
+            Some(f) => Box::new(f),
+            None => Box::new(WildcardFilter::default()),
         };
 
-    add_filter_to_list_of_wildcard_filters(filter.clone(), WILDCARD_FILTERS.clone());
+    add_filter_to_list_of_ferox_filters(filter, FILTERS.clone());
+
+    // add any status code filters to `FILTERS`
+    for code_filter in &CONFIGURATION.filter_status {
+        let filter = StatusCodeFilter {
+            filter_code: *code_filter,
+        };
+        let boxed_filter = Box::new(filter);
+        add_filter_to_list_of_ferox_filters(boxed_filter, FILTERS.clone());
+    }
 
     // producer tasks (mp of mpsc); responsible for making requests
     let producers = stream::iter(looping_words.deref().to_owned())
@@ -779,30 +768,4 @@ mod tests {
 
         assert_eq!(add_url_to_list_of_scanned_urls(url, &urls), false);
     }
-
-    #[test]
-    /// add a wildcard filter with the `size` attribute set to WILDCARD_FILTERS and ensure that
-    /// should_filter_response correctly returns true
-    fn should_filter_response_filters_wildcard_size() {
-        let mut filter = WildcardFilter::default();
-        let url = Url::parse("http://localhost").unwrap();
-        filter.size = 18;
-        let filter = Arc::new(filter);
-        add_filter_to_list_of_wildcard_filters(filter, WILDCARD_FILTERS.clone());
-        let result = should_filter_response(&18, &url);
-        assert!(result);
-    }
-
-    #[test]
-    /// add a wildcard filter with the `dynamic` attribute set to WILDCARD_FILTERS and ensure that
-    /// should_filter_response correctly returns true
-    fn should_filter_response_filters_wildcard_dynamic() {
-        let mut filter = WildcardFilter::default();
-        let url = Url::parse("http://localhost/some-path").unwrap();
-        filter.dynamic = 9;
-        let filter = Arc::new(filter);
-        add_filter_to_list_of_wildcard_filters(filter, WILDCARD_FILTERS.clone());
-        let result = should_filter_response(&18, &url);
-        assert!(result);
-    }
 }

src/utils.rs

@@ -140,7 +140,7 @@ pub fn ferox_print(msg: &str, bar: &ProgressBar) {
 pub fn format_url(
     url: &str,
     word: &str,
-    addslash: bool,
+    add_slash: bool,
     queries: &[(String, String)],
     extension: Option<&str>,
 ) -> FeroxResult<Url> {
@@ -148,7 +148,7 @@ pub fn format_url(
         "enter: format_url({}, {}, {}, {:?} {:?})",
         url,
         word,
-        addslash,
+        add_slash,
         queries,
         extension
     );
@@ -175,7 +175,7 @@ pub fn format_url(
     // extensions and slashes are mutually exclusive cases
     let word = if extension.is_some() {
         format!("{}.{}", word, extension.unwrap())
-    } else if addslash && !word.ends_with('/') {
+    } else if add_slash && !word.ends_with('/') {
         // -f used, and word doesn't already end with a /
         format!("{}/", word)
     } else {

tests/test_banner.rs

@@ -77,14 +77,14 @@ fn banner_prints_headers() -> Result<(), Box<dyn std::error::Error>> {
 #[test]
 /// test allows non-existent wordlist to trigger the banner printing to stderr
 /// expect to see all mandatory prints + multiple size filters
-fn banner_prints_size_filters() -> Result<(), Box<dyn std::error::Error>> {
+fn banner_prints_filter_sizes() -> Result<(), Box<dyn std::error::Error>> {
     Command::cargo_bin("feroxbuster")
         .unwrap()
         .arg("--url")
         .arg("http://localhost")
         .arg("-S")
         .arg("789456123")
-        .arg("--sizefilter")
+        .arg("--filter-size")
         .arg("44444444")
         .assert()
         .failure()
@@ -277,13 +277,13 @@ fn banner_prints_extensions() -> Result<(), Box<dyn std::error::Error>> {
 
 #[test]
 /// test allows non-existent wordlist to trigger the banner printing to stderr
-/// expect to see all mandatory prints + dontfilter
-fn banner_prints_dontfilter() -> Result<(), Box<dyn std::error::Error>> {
+/// expect to see all mandatory prints + dont_filter
+fn banner_prints_dont_filter() -> Result<(), Box<dyn std::error::Error>> {
     Command::cargo_bin("feroxbuster")
         .unwrap()
         .arg("--url")
         .arg("http://localhost")
-        .arg("--dontfilter")
+        .arg("--dont-filter")
         .assert()
         .failure()
         .stderr(
@@ -591,3 +591,31 @@ fn banner_prints_scan_limit() -> Result<(), Box<dyn std::error::Error>> {
         );
     Ok(())
 }
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + filter-status
+fn banner_prints_filter_status() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-C")
+        .arg("200")
+        .assert()
+        .failure()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Status Code Filters"))
+                .and(predicate::str::contains("│ [200]"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}

tests/test_extractor.rs

@@ -181,7 +181,7 @@ fn extractor_finds_same_relative_url_twice() -> Result<(), Box<dyn std::error::E
 }
 
 #[test]
-/// send a request to a page that contains an absolute link that leads to a page with a sizefilter
+/// send a request to a page that contains an absolute link that leads to a page with a filter_size
 /// that should filter it out, expect not to see the second response reported
 fn extractor_finds_filtered_content() -> Result<(), Box<dyn std::error::Error>> {
     let srv = MockServer::start();
@@ -209,7 +209,7 @@ fn extractor_finds_filtered_content() -> Result<(), Box<dyn std::error::Error>>
         .arg("--wordlist")
         .arg(file.as_os_str())
         .arg("--extract-links")
-        .arg("--sizefilter")
+        .arg("--filter-size")
         .arg("18")
         .unwrap();
 

tests/test_filters.rs

@@ -0,0 +1,57 @@
+mod utils;
+use assert_cmd::prelude::*;
+use httpmock::Method::GET;
+use httpmock::{Mock, MockServer};
+use predicates::prelude::*;
+use std::process::Command;
+use utils::{setup_tmp_directory, teardown_tmp_directory};
+
+#[test]
+/// create a FeroxResponse that should elicit a true from
+/// StatusCodeFilter::should_filter_response
+fn filters_status_code_should_filter_response() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) =
+        setup_tmp_directory(&["LICENSE".to_string(), "file.js".to_string()], "wordlist").unwrap();
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/LICENSE")
+        .return_status(302)
+        .return_body("this is a test")
+        .create_on(&srv);
+
+    let mock_two = Mock::new()
+        .expect_method(GET)
+        .expect_path("/file.js")
+        .return_status(200)
+        .return_body("this is also a test of some import")
+        .create_on(&srv);
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-vvvv")
+        .arg("--filter-status")
+        .arg("302")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .not()
+            .and(predicate::str::contains("302"))
+            .not()
+            .and(predicate::str::contains("14"))
+            .not()
+            .and(predicate::str::contains("/file.js"))
+            .and(predicate::str::contains("200"))
+            .and(predicate::str::contains("34")),
+    );
+
+    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock_two.times_called(), 1);
+    teardown_tmp_directory(tmp_dir);
+}

tests/test_heuristics.rs

@@ -115,7 +115,7 @@ fn test_static_wildcard_request_found() -> Result<(), Box<dyn std::error::Error>
         .arg(srv.url("/"))
         .arg("--wordlist")
         .arg(file.as_os_str())
-        .arg("--addslash")
+        .arg("--add-slash")
         .unwrap();
 
     teardown_tmp_directory(tmp_dir);
@@ -158,7 +158,7 @@ fn test_dynamic_wildcard_request_found() -> Result<(), Box<dyn std::error::Error
         .arg(srv.url("/"))
         .arg("--wordlist")
         .arg(file.as_os_str())
-        .arg("--addslash")
+        .arg("--add-slash")
         .arg("--output")
         .arg(outfile.as_os_str())
         .unwrap();
@@ -198,8 +198,8 @@ fn test_dynamic_wildcard_request_found() -> Result<(), Box<dyn std::error::Error
 }
 
 #[test]
-/// uses dontfilter, so the normal wildcard test should never happen
-fn heuristics_static_wildcard_request_with_dontfilter() -> Result<(), Box<dyn std::error::Error>> {
+/// uses dont_filter, so the normal wildcard test should never happen
+fn heuristics_static_wildcard_request_with_dont_filter() -> Result<(), Box<dyn std::error::Error>> {
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
@@ -216,7 +216,7 @@ fn heuristics_static_wildcard_request_with_dontfilter() -> Result<(), Box<dyn st
         .arg(srv.url("/"))
         .arg("--wordlist")
         .arg(file.as_os_str())
-        .arg("--dontfilter")
+        .arg("--dont-filter")
         .unwrap();
 
     teardown_tmp_directory(tmp_dir);
@@ -251,7 +251,7 @@ fn heuristics_wildcard_test_with_two_static_wildcards() -> Result<(), Box<dyn st
         .arg(srv.url("/"))
         .arg("--wordlist")
         .arg(file.as_os_str())
-        .arg("--addslash")
+        .arg("--add-slash")
         .unwrap();
 
     teardown_tmp_directory(tmp_dir);
@@ -299,7 +299,7 @@ fn heuristics_wildcard_test_with_two_static_wildcards_with_quiet_enabled(
         .arg(srv.url("/"))
         .arg("--wordlist")
         .arg(file.as_os_str())
-        .arg("--addslash")
+        .arg("--add-slash")
         .arg("-q")
         .unwrap();
 
@@ -340,7 +340,7 @@ fn heuristics_wildcard_test_with_two_static_wildcards_and_output_to_file(
         .arg(srv.url("/"))
         .arg("--wordlist")
         .arg(file.as_os_str())
-        .arg("--addslash")
+        .arg("--add-slash")
         .arg("--output")
         .arg(outfile.as_os_str())
         .unwrap();
@@ -406,7 +406,7 @@ fn heuristics_wildcard_test_with_redirect_as_response_code(
         .arg(srv.url("/"))
         .arg("--wordlist")
         .arg(file.as_os_str())
-        .arg("--addslash")
+        .arg("--add-slash")
         .arg("--output")
         .arg(outfile.as_os_str())
         .unwrap();

tests/test_scanner.rs

@@ -327,7 +327,8 @@ fn scanner_single_request_quiet_scan() -> Result<(), Box<dyn std::error::Error>>
 }
 
 #[test]
-/// send single valid request, get back a 301 without a Location header, expect false
+/// send single valid request, get back a 301 without a Location header
+/// expect response_is_directory to return false when called
 fn scanner_single_request_returns_301_without_location_header(
 ) -> Result<(), Box<dyn std::error::Error>> {
     let srv = MockServer::start();
@@ -336,6 +337,7 @@ fn scanner_single_request_returns_301_without_location_header(
     let mock = Mock::new()
         .expect_method(GET)
         .expect_path("/LICENSE")
+        .return_body("this is a test")
         .return_status(301)
         .create_on(&srv);
 
@@ -345,17 +347,16 @@ fn scanner_single_request_returns_301_without_location_header(
         .arg(srv.url("/"))
         .arg("--wordlist")
         .arg(file.as_os_str())
-        .arg("-T")
+        .arg("--timeout")
         .arg("5")
-        .arg("-a")
+        .arg("--user-agent")
         .arg("some-user-agent-string")
         .unwrap();
 
     cmd.assert().success().stdout(
         predicate::str::contains(srv.url("/LICENSE"))
             .and(predicate::str::contains("301"))
-            .and(predicate::str::contains("14"))
-            .not(),
+            .and(predicate::str::contains("14")),
     );
 
     assert_eq!(mock.times_called(), 1);
@@ -401,7 +402,7 @@ fn scanner_single_request_scan_with_filtered_result() -> Result<(), Box<dyn std:
             .and(predicate::str::contains("20"))
             .and(predicate::str::contains("ignored"))
             .not()
-            .and(predicate::str::contains("14"))
+            .and(predicate::str::contains(" 14 "))
             .not(),
     );