Merge pull request #127 from epi052/125-add-url-from-whence-we-came

reduced log output by a lot; added redirection location on error

.github/workflows/check.yml

@@ -1,6 +1,6 @@
 name: CI Pipeline
 
-on: [push]
+on: [push, pull_request]
 
 jobs:
   check:
@@ -61,4 +61,4 @@ jobs:
       - uses: actions-rs/cargo@v1
         with:
           command: clippy
-          args: --all-targets --all-features -- -D warnings -A clippy::unnecessary_unwrap
+          args: --all-targets --all-features -- -D warnings -A clippy::unnecessary_unwrap -A clippy::deref_addrof

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "feroxbuster"
-version = "1.3.0"
+version = "1.6.1"
 authors = ["Ben 'epi' Risher <epibar052@gmail.com>"]
 license = "MIT"
 edition = "2018"
@@ -19,7 +19,7 @@ futures = { version = "0.3"}
 tokio = { version = "0.2", features = ["full"] }
 tokio-util = {version = "0.3", features = ["codec"]}
 log = "0.4"
-env_logger = "0.7"
+env_logger = "0.8"
 reqwest = { version = "0.10", features = ["socks"] }
 clap = "2"
 lazy_static = "1.4"
@@ -32,6 +32,8 @@ console = "0.12"
 openssl = { version = "0.10", features = ["vendored"] }
 dirs = "3.0"
 regex = "1"
+crossterm = "0.18"
+rlimit = "0.5"
 
 [dev-dependencies]
 tempfile = "3.1"

README.md

@@ -61,6 +61,7 @@ This attack is also known as Predictable Resource Location, File Enumeration, Di
 -----------------
 - [Installation](#-installation)
     - [Download a Release](#download-a-release)
+    - [Snap Install](#snap-install)
     - [Homebrew on MacOS and Linux](#homebrew-on-macos-and-linux)
     - [Cargo Install](#cargo-install)
     - [apt Install](#apt-install)
@@ -68,9 +69,11 @@ This attack is also known as Predictable Resource Location, File Enumeration, Di
     - [Docker Install](#docker-install)
 - [Configuration](#%EF%B8%8F-configuration)
     - [Default Values](#default-values)
+    - [Threads and Connection Limits At A High-Level](#threads-and-connection-limits-at-a-high-level)
     - [ferox-config.toml](#ferox-configtoml)
     - [Command Line Parsing](#command-line-parsing)
 - [Example Usage](#-example-usage)
+    - [Pause and Resume Scans (new in `v1.4.0`)](#pause-and-resume-scans-new-in-v140)
     - [Multiple Values](#multiple-values)
     - [Extract Links from Response Body (new in `v1.1.0`)](#extract-links-from-response-body-new-in-v110)
     - [Include Headers](#include-headers)
@@ -81,6 +84,7 @@ This attack is also known as Predictable Resource Location, File Enumeration, Di
     - [Pass auth token via query parameter](#pass-auth-token-via-query-parameter)
     - [Limit Total Number of Concurrent Scans (new in `v1.2.0`)](#limit-total-number-of-concurrent-scans-new-in-v120)
     - [Filter Response by Status Code  (new in `v1.3.0`)](#filter-response-by-status-code--new-in-v130)
+    - [Replay Responses to a Proxy based on Status Code (new in `v1.5.0`)](#replay-responses-to-a-proxy-based-on-status-code-new-in-v150)
 - [Comparison w/ Similar Tools](#-comparison-w-similar-tools)
 - [Common Problems/Issues (FAQ)](#-common-problemsissues-faq)
     - [No file descriptors available](#no-file-descriptors-available)
@@ -113,9 +117,34 @@ Expand-Archive .\feroxbuster.zip
 .\feroxbuster\feroxbuster.exe -V
 ```
 
+### Snap Install
+
+Install using `snap`
+
+```
+sudo snap install feroxbuster
+```
+
+The only gotcha here is that the snap package can only read wordlists from a few specific locations. There are a few 
+possible solutions, of which two are shown below.
+
+If the wordlist is on the same partition as your home directory, it can be hard-linked into `~/snap/feroxbuster/common`
+
+```
+ln /path/to/the/wordlist ~/snap/feroxbuster/common
+./feroxbuster -u http://localhost -w ~/snap/feroxbuster/common/wordlist
+``` 
+
+If the wordlist is on a separate partition, hard-linking won't work.  You'll need to copy it into the snap directory.
+
+```
+cp /path/to/the/wordlist ~/snap/feroxbuster/common
+./feroxbuster -u http://localhost -w ~/snap/feroxbuster/common/wordlist
+``` 
+
 ### Homebrew on MacOS and Linux
 
-Installable by Homebrew throughout own formulas:
+Install using Homebrew via tap
 
 🍏 [MacOS](https://github.com/TGotwig/homebrew-feroxbuster/blob/main/feroxbuster.rb)
 
@@ -228,6 +257,23 @@ Configuration begins with with the following built-in default values baked into
 - auto-filter wildcards - `true`
 - output: `stdout`
 
+### Threads and Connection Limits At A High-Level
+
+This section explains how the `-t` and `-L` options work together to determine the overall aggressiveness of a scan. The combination of the two values set by these options determines how hard your target will get hit and to some extent also determines how many resources will be consumed on your local machine.
+
+#### A Note on Green Threads
+
+`feroxbuster` uses so-called [green threads](https://en.wikipedia.org/wiki/Green_threads) as opposed to traditional kernel/OS threads. This means (at a high-level) that the threads are implemented entirely in userspace, within a single running process. As a result, a scan with 30 green threads will appear to the OS to be a single process with no additional light-weight processes associated with it as far as the kernel is concerned. As such, there will not be any impact to process (`nproc`) limits when specifying larger values for `-t`. However, these threads will still consume file descriptors, so you will need to ensure that you have a suitable `nlimit` set when scaling up the amount of threads. More detailed documentation on setting appropriate `nlimit` values can be found in the [No File Descriptors Available](#no-file-descriptors-available) section of the FAQ
+
+#### Threads and Connection Limits: The Implementation
+
+* Threads: The `-t` option specifies the maximum amount of active threads *per-directory* during a scan
+* Connection Limits: The `-L` option specifies the maximum amount of active connections per thread
+
+#### Threads and Connection Limits: Examples
+
+To truly have only 30 active requests to a site at any given time, `-t 30 -L 1` is necessary. Using `-t 30 -L 2` will result in a maximum of 60 total requests being processed at any given time for that site. And so on. For a conversation on this, please see [Issue #126](https://github.com/epi052/feroxbuster/issues/126) which may provide more (or less) clarity :wink:
+
 ### ferox-config.toml
 After setting built-in default values, any values defined in a `ferox-config.toml` config file will override the
 built-in defaults.  
@@ -275,9 +321,11 @@ A pre-made configuration file with examples of all available settings can be fou
 # wordlist = "/wordlists/jhaddix/all.txt"
 # status_codes = [200, 500]
 # filter_status = [301]
+# replay_codes = [301]
 # threads = 1
 # timeout = 5
 # proxy = "http://127.0.0.1:8080"
+# replay_proxy = "http://127.0.0.1:8081"
 # verbosity = 1
 # scan_limit = 6
 # quiet = true
@@ -293,6 +341,8 @@ A pre-made configuration file with examples of all available settings can be fou
 # extract_links = true
 # depth = 1
 # filter_size = [5174]
+# filter_word_count = [993]
+# filter_line_count = [35, 36]
 # queries = [["name","value"], ["rick", "astley"]]
 
 # headers can be specified on multiple lines or as an inline table
@@ -333,12 +383,18 @@ FLAGS:
 OPTIONS:
     -d, --depth <RECURSION_DEPTH>           Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)
     -x, --extensions <FILE_EXTENSION>...    File extension(s) to search for (ex: -x php -x pdf js)
+    -N, --filter-lines <LINES>...           Filter out messages of a particular line count (ex: -N 20 -N 31,30)
     -S, --filter-size <SIZE>...             Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)
-    -C, --filter-status <STATUS_CODE>...    Filter out status codes (deny list) (ex: -C 200 -S 401)
+    -C, --filter-status <STATUS_CODE>...    Filter out status codes (deny list) (ex: -C 200 -C 401)
+    -W, --filter-words <WORDS>...           Filter out messages of a particular word count (ex: -W 312 -W 91,82)
     -H, --headers <HEADER>...               Specify HTTP headers (ex: -H Header:val 'stuff: things')
     -o, --output <FILE>                     Output file to write results to (default: stdout)
     -p, --proxy <PROXY>                     Proxy to use for requests (ex: http(s)://host:port, socks5://host:port)
     -Q, --query <QUERY>...                  Specify URL query parameters (ex: -Q token=stuff -Q secret=key)
+    -R, --replay-codes <REPLAY_CODE>...     Status Codes to send through a Replay Proxy when found (default: --status
+                                            -codes value)
+    -P, --replay-proxy <REPLAY_PROXY>       Send only unfiltered requests through a Replay Proxy, instead of all
+                                            requests
     -L, --scan-limit <SCAN_LIMIT>           Limit total number of concurrent scans (default: 0, i.e. no limit)
     -s, --status-codes <STATUS_CODE>...     Status Codes to include (allow list) (default: 200 204 301 302 307 308 401
                                             403 405)
@@ -351,6 +407,12 @@ OPTIONS:
 
 ## 🧰 Example Usage
 
+### Pause and Resume Scans (new in `v1.4.0`)
+
+Scans can be paused and resumed by pressing the ENTER key (shown below)
+
+![pause-resume-demo](img/pause-resume-demo.gif)
+
 ### Multiple Values
 
 Options that take multiple values are very flexible.  Consider the following ways of specifying extensions:
@@ -452,6 +514,20 @@ each one is checked against a list of known filters and either displayed or not
 ./feroxbuster -u http://127.1 --filter-status 301
 ```
 
+### Replay Responses to a Proxy based on Status Code (new in `v1.5.0`)
+
+The `--replay-proxy` and `--replay-codes` options were added as a way to only send a select few responses to a proxy.  This is in stark contrast to `--proxy` which proxies EVERY request.  
+
+Imagine you only care about proxying responses that have either the status code `200` or `302` (or you just don't want to clutter up your Burp history).  These two options will allow you to fine-tune what gets proxied and what doesn't.  
+
+```
+./feroxbuster -u http://127.1 --replay-proxy http://localhost:8080 --replay-codes 200 302 --insecure
+```
+
+Of note: this means that for every response that matches your replay criteria, you'll end up sending the request that generated that response a second time.  Depending on the target and your engagement terms (if any), it may not make sense from a traffic generated perspective.
+
+![replay-proxy-demo](img/replay-proxy-demo.gif)
+
 ## 🧐 Comparison w/ Similar Tools
 
 There are quite a few similar tools for forced browsing/content discovery.  Burp Suite Pro, Dirb, Dirbuster, etc... 
@@ -483,7 +559,7 @@ a few of the use-cases in which feroxbuster may be a better fit:
 | configuration file for default value override                    | ✔ |   | ✔ |
 | can accept urls via STDIN as part of a pipeline                  | ✔ |   | ✔ |
 | can accept wordlists via STDIN                                   |   | ✔ | ✔ |
-| filter by response size                                          | ✔ |   | ✔ |
+| filter based on response size, wordcount, and linecount          | ✔ |   | ✔ |
 | auto-filter wildcard responses                                   | ✔ |   | ✔ |
 | performs other scans (vhost, dns, etc)                           |   | ✔ | ✔ |
 | time delay / rate limiting                                       |   | ✔ | ✔ |

ferox-config.toml.example

@@ -13,6 +13,8 @@
 # threads = 1
 # timeout = 5
 # proxy = "http://127.0.0.1:8080"
+# replay_proxy = "http://127.0.0.1:8081"
+# replay_codes = [200, 302]
 # verbosity = 1
 # scan_limit = 6
 # quiet = true
@@ -28,6 +30,8 @@
 # extract_links = true
 # depth = 1
 # filter_size = [5174]
+# filter_word_count = [993]
+# filter_line_count = [35, 36]
 # queries = [["name","value"], ["rick", "astley"]]
 
 # headers can be specified on multiple lines or as an inline table

snapcraft.yaml

@@ -0,0 +1,41 @@
+name: feroxbuster
+version: git
+summary: A simple, fast, recursive content discovery tool written in Rust
+description: |
+  feroxbuster is a tool designed to perform Forced Browsing.
+
+  Forced browsing is an attack where the aim is to enumerate and access resources that are not referenced by the web application, but are still accessible by an attacker.
+
+  feroxbuster uses brute force combined with a wordlist to search for unlinked content in target directories. These resources may store sensitive information about web applications and operational systems, such as source code, credentials, internal network addressing, etc...
+
+  This attack is also known as Predictable Resource Location, File Enumeration, Directory Enumeration, and Resource Enumeration.
+
+
+base: core18
+
+plugs:
+  etc-feroxbuster:
+    interface: system-files
+    read:
+    - /etc/feroxbuster
+  dot-config-feroxbuster:
+    interface: personal-files
+    read:
+    - $HOME/.config/feroxbuster
+
+architectures:
+  - build-on: amd64
+  - build-on: i386
+
+parts:
+  feroxbuster:
+    plugin: rust
+    source: .
+
+apps:
+  feroxbuster:
+    command: bin/feroxbuster
+    plugs:
+      - etc-feroxbuster
+      - dot-config-feroxbuster
+      - network

src/banner.rs

@@ -1,5 +1,6 @@
 use crate::config::{Configuration, CONFIGURATION};
 use crate::utils::{make_request, status_colorizer};
+use console::style;
 use reqwest::{Client, Url};
 use serde_json::Value;
 use std::io::Write;
@@ -144,6 +145,7 @@ by Ben "epi" Risher {}                  ver: {}"#,
     let status = needs_update(&CONFIGURATION.client, UPDATE_URL, version).await;
 
     let top = "───────────────────────────┬──────────────────────";
+    let addl_section = "──────────────────────────────────────────────────";
     let bottom = "───────────────────────────┴──────────────────────";
 
     writeln!(&mut writer, "{}", artwork).unwrap_or_default();
@@ -244,6 +246,35 @@ by Ben "epi" Risher {}                  ver: {}"#,
         .unwrap_or_default(); // 💎
     }
 
+    if !config.replay_proxy.is_empty() {
+        // i include replay codes logic here because in config.rs, replay codes are set to the
+        // value in status codes, meaning it's never empty
+
+        let mut replay_codes = vec![];
+
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!("\u{1f3a5}", "Replay Proxy", config.replay_proxy)
+        )
+        .unwrap_or_default(); // 🎥
+
+        for code in &config.replay_codes {
+            replay_codes.push(status_colorizer(&code.to_string()))
+        }
+
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!(
+                "\u{1f4fc}",
+                "Replay Proxy Codes",
+                format!("[{}]", replay_codes.join(", "))
+            )
+        )
+        .unwrap_or_default(); // 📼
+    }
+
     if !config.headers.is_empty() {
         for (name, value) in &config.headers {
             writeln!(
@@ -266,6 +297,24 @@ by Ben "epi" Risher {}                  ver: {}"#,
         }
     }
 
+    for filter in &config.filter_word_count {
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!("\u{1f4a2}", "Word Count Filter", filter)
+        )
+        .unwrap_or_default(); // 💢
+    }
+
+    for filter in &config.filter_line_count {
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!("\u{1f4a2}", "Line Count Filter", filter)
+        )
+        .unwrap_or_default(); // 💢
+    }
+
     if config.extract_links {
         writeln!(
             &mut writer,
@@ -433,6 +482,16 @@ by Ben "epi" Risher {}                  ver: {}"#,
     }
 
     writeln!(&mut writer, "{}", bottom).unwrap_or_default();
+    // ⏯
+    writeln!(
+        &mut writer,
+        " \u{23ef}   Press [{}] to {}|{} your scan",
+        style("ENTER").yellow(),
+        style("pause").red(),
+        style("resume").green()
+    )
+    .unwrap_or_default();
+    writeln!(&mut writer, "{}", addl_section).unwrap_or_default();
 }
 
 #[cfg(test)]

src/client.rs

@@ -32,41 +32,38 @@ pub fn initialize(
         .default_headers(header_map)
         .redirect(policy);
 
-    let client = if proxy.is_some() && !proxy.unwrap().is_empty() {
-        match Proxy::all(proxy.unwrap()) {
-            Ok(proxy_obj) => client.proxy(proxy_obj),
-            Err(e) => {
-                eprintln!(
-                    "{} {} Could not add proxy ({:?}) to Client configuration",
-                    status_colorizer("ERROR"),
-                    module_colorizer("Client::initialize"),
-                    proxy
-                );
-                eprintln!(
-                    "{} {} {}",
-                    status_colorizer("ERROR"),
-                    module_colorizer("Client::initialize"),
-                    e
-                );
+    let client = match proxy {
+        // a proxy is specified, need to add it to the client
+        Some(some_proxy) => {
+            if !some_proxy.is_empty() {
+                // it's not an empty string
+                match Proxy::all(some_proxy) {
+                    Ok(proxy_obj) => client.proxy(proxy_obj),
+                    Err(e) => {
+                        eprintln!(
+                            "{} {} {}",
+                            status_colorizer("ERROR"),
+                            module_colorizer("Client::initialize"),
+                            e
+                        );
 
-                #[cfg(test)]
-                panic!();
-                #[cfg(not(test))]
-                exit(1);
+                        #[cfg(test)]
+                        panic!();
+                        #[cfg(not(test))]
+                        exit(1);
+                    }
+                }
+            } else {
+                client // Some("") was used?
             }
         }
-    } else {
-        client
+        // no proxy specified
+        None => client,
     };
 
     match client.build() {
         Ok(client) => client,
         Err(e) => {
-            eprintln!(
-                "{} {} Could not create a Client with the given configuration, exiting.",
-                status_colorizer("ERROR"),
-                module_colorizer("Client::build")
-            );
             eprintln!(
                 "{} {} {}",
                 status_colorizer("ERROR"),

src/config.rs

@@ -10,6 +10,7 @@ use std::collections::HashMap;
 use std::env::{current_dir, current_exe};
 use std::fs::read_to_string;
 use std::path::PathBuf;
+#[cfg(not(test))]
 use std::process::exit;
 
 lazy_static! {
@@ -23,6 +24,21 @@ lazy_static! {
     pub static ref PROGRESS_PRINTER: ProgressBar = progress::add_bar("", 0, true);
 }
 
+/// simple helper to clean up some code reuse below; panics under test / exits in prod
+fn report_and_exit(err: &str) -> ! {
+    eprintln!(
+        "{} {}: {}",
+        status_colorizer("ERROR"),
+        module_colorizer("Configuration::new"),
+        err
+    );
+
+    #[cfg(test)]
+    panic!();
+    #[cfg(not(test))]
+    exit(1);
+}
+
 /// Represents the final, global configuration of the program.
 ///
 /// This struct is the combination of the following:
@@ -47,6 +63,10 @@ pub struct Configuration {
     #[serde(default)]
     pub proxy: String,
 
+    /// Replay Proxy to use for requests (ex: http(s)://host:port, socks5://host:port)
+    #[serde(default)]
+    pub replay_proxy: String,
+
     /// The target URL
     #[serde(default)]
     pub target_url: String,
@@ -55,6 +75,10 @@ pub struct Configuration {
     #[serde(default = "status_codes")]
     pub status_codes: Vec<u16>,
 
+    /// Status Codes to replay to the Replay Proxy (default: whatever is passed to --status-code)
+    #[serde(default)]
+    pub replay_codes: Vec<u16>,
+
     /// Status Codes to filter out (deny list)
     #[serde(default)]
     pub filter_status: Vec<u16>,
@@ -63,6 +87,10 @@ pub struct Configuration {
     #[serde(skip)]
     pub client: Client,
 
+    /// Instance of [reqwest::Client](https://docs.rs/reqwest/latest/reqwest/struct.Client.html)
+    #[serde(skip)]
+    pub replay_client: Option<Client>,
+
     /// Number of concurrent threads (default: 50)
     #[serde(default = "threads")]
     pub threads: usize,
@@ -135,6 +163,14 @@ pub struct Configuration {
     #[serde(default)]
     pub filter_size: Vec<u64>,
 
+    /// Filter out messages of a particular line count
+    #[serde(default)]
+    pub filter_line_count: Vec<usize>,
+
+    /// Filter out messages of a particular word count
+    #[serde(default)]
+    pub filter_word_count: Vec<usize>,
+
     /// Don't auto-filter wildcard responses
     #[serde(default)]
     pub dont_filter: bool,
@@ -183,11 +219,17 @@ impl Default for Configuration {
         let timeout = timeout();
         let user_agent = user_agent();
         let client = client::initialize(timeout, &user_agent, false, false, &HashMap::new(), None);
+        let replay_client = None;
+        let status_codes = status_codes();
+        let replay_codes = status_codes.clone();
 
         Configuration {
             client,
             timeout,
             user_agent,
+            replay_codes,
+            status_codes,
+            replay_client,
             dont_filter: false,
             quiet: false,
             stdin: false,
@@ -202,15 +244,17 @@ impl Default for Configuration {
             config: String::new(),
             output: String::new(),
             target_url: String::new(),
+            replay_proxy: String::new(),
             queries: Vec::new(),
             extensions: Vec::new(),
             filter_size: Vec::new(),
+            filter_line_count: Vec::new(),
+            filter_word_count: Vec::new(),
             filter_status: Vec::new(),
             headers: HashMap::new(),
-            threads: threads(),
             depth: depth(),
+            threads: threads(),
             wordlist: wordlist(),
-            status_codes: status_codes(),
         }
     }
 }
@@ -236,6 +280,8 @@ impl Configuration {
     /// - **insecure**: `false` (don't be insecure, i.e. don't allow invalid certs)
     /// - **extensions**: `None`
     /// - **filter_size**: `None`
+    /// - **filter_word_count**: `None`
+    /// - **filter_line_count**: `None`
     /// - **headers**: `None`
     /// - **queries**: `None`
     /// - **no_recursion**: `false` (recursively scan enumerated sub-directories)
@@ -244,6 +290,8 @@ impl Configuration {
     /// - **dont_filter**: `false` (auto filter wildcard responses)
     /// - **depth**: `4` (maximum recursion depth)
     /// - **scan_limit**: `0` (no limit on concurrent scans imposed)
+    /// - **replay_proxy**: `None` (no limit on concurrent scans imposed)
+    /// - **replay_codes**: [`DEFAULT_RESPONSE_CODES`](constant.DEFAULT_RESPONSE_CODES.html)
     ///
     /// After which, any values defined in a
     /// [ferox-config.toml](constant.DEFAULT_CONFIG_NAME.html) config file will override the
@@ -316,94 +364,89 @@ impl Configuration {
 
         let args = parser::initialize().get_matches();
 
-        // the .is_some appears clunky, but it allows default values to be incrementally
-        // overwritten from Struct defaults, to file config, to command line args, soooo ¯\_(ツ)_/¯
-        if args.value_of("threads").is_some() {
-            let threads = value_t!(args.value_of("threads"), usize).unwrap_or_else(|e| e.exit());
-            config.threads = threads;
+        macro_rules! update_config_if_present {
+            ($c:expr, $m:ident, $v:expr, $t:ty) => {
+                match value_t!($m, $v, $t) {
+                    Ok(value) => *$c = value, // Update value
+                    Err(clap::Error {
+                        kind: clap::ErrorKind::ArgumentNotFound,
+                        message: _,
+                        info: _,
+                    }) => {
+                        // Do nothing if argument not found
+                    }
+                    Err(e) => e.exit(), // Exit with error on parse error
+                }
+            };
         }
 
-        if args.value_of("depth").is_some() {
-            let depth = value_t!(args.value_of("depth"), usize).unwrap_or_else(|e| e.exit());
-            config.depth = depth;
-        }
+        update_config_if_present!(&mut config.threads, args, "threads", usize);
+        update_config_if_present!(&mut config.depth, args, "depth", usize);
+        update_config_if_present!(&mut config.scan_limit, args, "scan_limit", usize);
+        update_config_if_present!(&mut config.wordlist, args, "wordlist", String);
+        update_config_if_present!(&mut config.output, args, "output", String);
 
-        if args.value_of("scan_limit").is_some() {
-            let scan_limit =
-                value_t!(args.value_of("scan_limit"), usize).unwrap_or_else(|e| e.exit());
-            config.scan_limit = scan_limit;
-        }
-
-        if args.value_of("wordlist").is_some() {
-            config.wordlist = String::from(args.value_of("wordlist").unwrap());
-        }
-
-        if args.value_of("output").is_some() {
-            config.output = String::from(args.value_of("output").unwrap());
-        }
-
-        if args.values_of("status_codes").is_some() {
-            config.status_codes = args
-                .values_of("status_codes")
-                .unwrap() // already known good
+        if let Some(arg) = args.values_of("status_codes") {
+            config.status_codes = arg
                 .map(|code| {
                     StatusCode::from_bytes(code.as_bytes())
-                        .unwrap_or_else(|e| {
-                            eprintln!(
-                                "{} {}: {}",
-                                status_colorizer("ERROR"),
-                                module_colorizer("Configuration::new"),
-                                e
-                            );
-                            exit(1)
-                        })
+                        .unwrap_or_else(|e| report_and_exit(&e.to_string()))
                         .as_u16()
                 })
                 .collect();
         }
 
-        if args.values_of("filter_status").is_some() {
-            config.filter_status = args
-                .values_of("filter_status")
-                .unwrap() // already known good
+        if let Some(arg) = args.values_of("replay_codes") {
+            // replay codes passed in by the user
+            config.replay_codes = arg
                 .map(|code| {
                     StatusCode::from_bytes(code.as_bytes())
-                        .unwrap_or_else(|e| {
-                            eprintln!(
-                                "{} {}: {}",
-                                status_colorizer("ERROR"),
-                                module_colorizer("Configuration::new"),
-                                e
-                            );
-                            exit(1)
-                        })
+                        .unwrap_or_else(|e| report_and_exit(&e.to_string()))
+                        .as_u16()
+                })
+                .collect();
+        } else {
+            // not passed in by the user, use whatever value is held in status_codes
+            config.replay_codes = config.status_codes.clone();
+        }
+
+        if let Some(arg) = args.values_of("filter_status") {
+            config.filter_status = arg
+                .map(|code| {
+                    StatusCode::from_bytes(code.as_bytes())
+                        .unwrap_or_else(|e| report_and_exit(&e.to_string()))
                         .as_u16()
                 })
                 .collect();
         }
 
-        if args.values_of("extensions").is_some() {
-            config.extensions = args
-                .values_of("extensions")
-                .unwrap()
-                .map(|val| val.to_string())
-                .collect();
+        if let Some(arg) = args.values_of("extensions") {
+            config.extensions = arg.map(|val| val.to_string()).collect();
         }
 
-        if args.values_of("filter_size").is_some() {
-            config.filter_size = args
-                .values_of("filter_size")
-                .unwrap() // already known good
+        if let Some(arg) = args.values_of("filter_size") {
+            config.filter_size = arg
                 .map(|size| {
-                    size.parse::<u64>().unwrap_or_else(|e| {
-                        eprintln!(
-                            "{} {}: {}",
-                            status_colorizer("ERROR"),
-                            module_colorizer("Configuration::new"),
-                            e
-                        );
-                        exit(1)
-                    })
+                    size.parse::<u64>()
+                        .unwrap_or_else(|e| report_and_exit(&e.to_string()))
+                })
+                .collect();
+        }
+
+        if let Some(arg) = args.values_of("filter_words") {
+            config.filter_word_count = arg
+                .map(|size| {
+                    size.parse::<usize>()
+                        .unwrap_or_else(|e| report_and_exit(&e.to_string()))
+                })
+                .collect();
+        }
+
+        if let Some(arg) = args.values_of("filter_lines") {
+            config.filter_line_count = arg
+                .map(|size| {
+                    size.parse::<usize>()
+                        .unwrap_or_else(|e| report_and_exit(&e.to_string()))
                 })
                 .collect();
         }
@@ -413,11 +456,11 @@ impl Configuration {
             // consider a user specifying quiet = true in ferox-config.toml
             // if the line below is outside of the if, we'd overwrite true with
             // false if no -q is used on the command line
-            config.quiet = args.is_present("quiet");
+            config.quiet = true;
         }
 
         if args.is_present("dont_filter") {
-            config.dont_filter = args.is_present("dont_filter");
+            config.dont_filter = true;
         }
 
         if args.occurrences_of("verbosity") > 0 {
@@ -427,19 +470,19 @@ impl Configuration {
         }
 
         if args.is_present("no_recursion") {
-            config.no_recursion = args.is_present("no_recursion");
+            config.no_recursion = true;
         }
 
         if args.is_present("add_slash") {
-            config.add_slash = args.is_present("add_slash");
+            config.add_slash = true;
         }
 
         if args.is_present("extract_links") {
-            config.extract_links = args.is_present("extract_links");
+            config.extract_links = true;
         }
 
         if args.is_present("stdin") {
-            config.stdin = args.is_present("stdin");
+            config.stdin = true;
         } else {
             config.target_url = String::from(args.value_of("url").unwrap());
         }
@@ -447,29 +490,21 @@ impl Configuration {
         ////
         // organizational breakpoint; all options below alter the Client configuration
         ////
-        if args.value_of("proxy").is_some() {
-            config.proxy = String::from(args.value_of("proxy").unwrap());
-        }
-
-        if args.value_of("user_agent").is_some() {
-            config.user_agent = String::from(args.value_of("user_agent").unwrap());
-        }
-
-        if args.value_of("timeout").is_some() {
-            let timeout = value_t!(args.value_of("timeout"), u64).unwrap_or_else(|e| e.exit());
-            config.timeout = timeout;
-        }
+        update_config_if_present!(&mut config.proxy, args, "proxy", String);
+        update_config_if_present!(&mut config.replay_proxy, args, "replay_proxy", String);
+        update_config_if_present!(&mut config.user_agent, args, "user_agent", String);
+        update_config_if_present!(&mut config.timeout, args, "timeout", u64);
 
         if args.is_present("redirects") {
-            config.redirects = args.is_present("redirects");
+            config.redirects = true;
         }
 
         if args.is_present("insecure") {
-            config.insecure = args.is_present("insecure");
+            config.insecure = true;
         }
 
-        if args.values_of("headers").is_some() {
-            for val in args.values_of("headers").unwrap() {
+        if let Some(headers) = args.values_of("headers") {
+            for val in headers {
                 let mut split_val = val.split(':');
 
                 // explicitly take first split value as header's name
@@ -482,8 +517,8 @@ impl Configuration {
             }
         }
 
-        if args.values_of("queries").is_some() {
-            for val in args.values_of("queries").unwrap() {
+        if let Some(queries) = args.values_of("queries") {
+            for val in queries {
                 // same basic logic used as reading in the headers HashMap above
                 let mut split_val = val.split('=');
 
@@ -526,6 +561,18 @@ impl Configuration {
             }
         }
 
+        if !config.replay_proxy.is_empty() {
+            // only set replay_client when replay_proxy is set
+            config.replay_client = Some(client::initialize(
+                config.timeout,
+                &config.user_agent,
+                config.redirects,
+                config.insecure,
+                &config.headers,
+                Some(&config.replay_proxy),
+            ));
+        }
+
         config
     }
 
@@ -571,9 +618,13 @@ impl Configuration {
         settings.stdin = settings_to_merge.stdin;
         settings.depth = settings_to_merge.depth;
         settings.filter_size = settings_to_merge.filter_size;
+        settings.filter_word_count = settings_to_merge.filter_word_count;
+        settings.filter_line_count = settings_to_merge.filter_line_count;
         settings.filter_status = settings_to_merge.filter_status;
         settings.dont_filter = settings_to_merge.dont_filter;
         settings.scan_limit = settings_to_merge.scan_limit;
+        settings.replay_proxy = settings_to_merge.replay_proxy;
+        settings.replay_codes = settings_to_merge.replay_codes;
     }
 
     /// If present, read in `DEFAULT_CONFIG_NAME` and deserialize the specified values
@@ -610,9 +661,11 @@ mod tests {
         let data = r#"
             wordlist = "/some/path"
             status_codes = [201, 301, 401]
+            replay_codes = [201, 301]
             threads = 40
             timeout = 5
             proxy = "http://127.0.0.1:8080"
+            replay_proxy = "http://127.0.0.1:8081"
             quiet = true
             verbosity = 1
             scan_limit = 6
@@ -629,6 +682,8 @@ mod tests {
             extract_links = true
             depth = 1
             filter_size = [4120]
+            filter_word_count = [994, 992]
+            filter_line_count = [34]
             filter_status = [201]
         "#;
         let tmp_dir = TempDir::new().unwrap();
@@ -645,7 +700,10 @@ mod tests {
         assert_eq!(config.proxy, String::new());
         assert_eq!(config.target_url, String::new());
         assert_eq!(config.config, String::new());
+        assert_eq!(config.replay_proxy, String::new());
         assert_eq!(config.status_codes, status_codes());
+        assert_eq!(config.replay_codes, config.status_codes);
+        assert!(config.replay_client.is_none());
         assert_eq!(config.threads, threads());
         assert_eq!(config.depth, depth());
         assert_eq!(config.timeout, timeout());
@@ -662,6 +720,8 @@ mod tests {
         assert_eq!(config.queries, Vec::new());
         assert_eq!(config.extensions, Vec::<String>::new());
         assert_eq!(config.filter_size, Vec::<u64>::new());
+        assert_eq!(config.filter_word_count, Vec::<usize>::new());
+        assert_eq!(config.filter_line_count, Vec::<usize>::new());
         assert_eq!(config.filter_status, Vec::<u16>::new());
         assert_eq!(config.headers, HashMap::new());
     }
@@ -680,6 +740,13 @@ mod tests {
         assert_eq!(config.status_codes, vec![201, 301, 401]);
     }
 
+    #[test]
+    /// parse the test config and see that the value parsed is correct
+    fn config_reads_replay_codes() {
+        let config = setup_config_test();
+        assert_eq!(config.replay_codes, vec![201, 301]);
+    }
+
     #[test]
     /// parse the test config and see that the value parsed is correct
     fn config_reads_threads() {
@@ -715,6 +782,13 @@ mod tests {
         assert_eq!(config.proxy, "http://127.0.0.1:8080");
     }
 
+    #[test]
+    /// parse the test config and see that the value parsed is correct
+    fn config_reads_replay_proxy() {
+        let config = setup_config_test();
+        assert_eq!(config.replay_proxy, "http://127.0.0.1:8081");
+    }
+
     #[test]
     /// parse the test config and see that the value parsed is correct
     fn config_reads_quiet() {
@@ -799,6 +873,20 @@ mod tests {
         assert_eq!(config.filter_size, vec![4120]);
     }
 
+    #[test]
+    /// parse the test config and see that the value parsed is correct
+    fn config_reads_filter_word_count() {
+        let config = setup_config_test();
+        assert_eq!(config.filter_word_count, vec![994, 992]);
+    }
+
+    #[test]
+    /// parse the test config and see that the value parsed is correct
+    fn config_reads_filter_line_count() {
+        let config = setup_config_test();
+        assert_eq!(config.filter_line_count, vec![34]);
+    }
+
     #[test]
     /// parse the test config and see that the value parsed is correct
     fn config_reads_filter_status() {
@@ -825,4 +913,11 @@ mod tests {
         queries.push(("rick".to_string(), "astley".to_string()));
         assert_eq!(config.queries, queries);
     }
+
+    #[test]
+    #[should_panic]
+    /// test that an error message is printed and panic is called when report_and_exit is called
+    fn config_report_and_exit_works() {
+        report_and_exit("some message");
+    }
 }

src/filters.rs

@@ -53,7 +53,7 @@ impl FeroxFilter for WildcardFilter {
     /// Examine size, dynamic, and content_len to determine whether or not the response received
     /// is a wildcard response and therefore should be filtered out
     fn should_filter_response(&self, response: &FeroxResponse) -> bool {
-        log::trace!("enter: should_filter_response({:?} {:?})", self, response);
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
 
         // quick return if dont_filter is set
         if CONFIGURATION.dont_filter {
@@ -114,7 +114,7 @@ pub struct StatusCodeFilter {
 impl FeroxFilter for StatusCodeFilter {
     /// Check `filter_code` against what was passed in via -C|--filter-status
     fn should_filter_response(&self, response: &FeroxResponse) -> bool {
-        log::trace!("enter: should_filter_response({:?} {:?})", self, response);
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
 
         if response.status().as_u16() == self.filter_code {
             log::debug!(

src/heuristics.rs

@@ -1,13 +1,15 @@
-use crate::config::{CONFIGURATION, PROGRESS_PRINTER};
-use crate::filters::WildcardFilter;
-use crate::scanner::should_filter_response;
-use crate::utils::{
-    ferox_print, format_url, get_url_path_length, make_request, module_colorizer, status_colorizer,
+use crate::{
+    config::{CONFIGURATION, PROGRESS_PRINTER},
+    filters::WildcardFilter,
+    scanner::should_filter_response,
+    utils::{
+        ferox_print, format_url, get_url_path_length, make_request, module_colorizer,
+        status_colorizer,
+    },
+    FeroxResponse,
 };
-use crate::FeroxResponse;
 use console::style;
 use indicatif::ProgressBar;
-use std::process;
 use tokio::sync::mpsc::UnboundedSender;
 use uuid::Uuid;
 
@@ -87,8 +89,10 @@ pub async fn wildcard_test(
 
                 if !CONFIGURATION.quiet {
                     let msg = format!(
-                            "{} {:>10} Wildcard response is dynamic; {} ({} + url length) responses; toggle this behavior by using {}\n",
+                            "{} {:>8}l {:>8}w {:>8}c Wildcard response is dynamic; {} ({} + url length) responses; toggle this behavior by using {}\n",
                             status_colorizer("WLD"),
+                            ferox_response.line_count(),
+                            ferox_response.word_count(),
                             wildcard.dynamic,
                             style("auto-filtering").yellow(),
                             style(wc_length - url_len).cyan(),
@@ -108,8 +112,10 @@ pub async fn wildcard_test(
 
                 if !CONFIGURATION.quiet {
                     let msg = format!(
-                        "{} {:>10} Wildcard response is static; {} {} responses; toggle this behavior by using {}\n",
+                        "{} {:>8}l {:>8}w {:>8}c Wildcard response is static; {} {} responses; toggle this behavior by using {}\n",
                         status_colorizer("WLD"),
+                        ferox_response.line_count(),
+                        ferox_response.word_count(),
                         wc_length,
                         style("auto-filtering").yellow(),
                         style(wc_length).cyan(),
@@ -181,14 +187,18 @@ async fn make_wildcard_request(
                 .contains(&response.status().as_u16())
             {
                 // found a wildcard response
-                let ferox_response = FeroxResponse::from(response, false).await;
+                let ferox_response = FeroxResponse::from(response, true).await;
                 let url_len = get_url_path_length(&ferox_response.url());
                 let content_len = ferox_response.content_length();
+                let content_words = ferox_response.word_count();
+                let content_lines = ferox_response.line_count();
 
                 if !CONFIGURATION.quiet && !should_filter_response(&ferox_response) {
                     let msg = format!(
-                        "{} {:>10} Got {} for {} (url length: {})\n",
+                        "{} {:>8}l {:>8}w {:>8}c Got {} for {} (url length: {})\n",
                         wildcard,
+                        content_lines,
+                        content_words,
                         content_len,
                         status_colorizer(&ferox_response.status().as_str()),
                         ferox_response.url(),
@@ -210,8 +220,10 @@ async fn make_wildcard_request(
                         let next_loc_str = next_loc.to_str().unwrap_or("Unknown");
                         if !CONFIGURATION.quiet && !should_filter_response(&ferox_response) {
                             let msg = format!(
-                                "{} {:>10} {} redirects to => {}\n",
+                                "{} {:>8}l {:>8}w {:>8}c {} redirects to => {}\n",
                                 wildcard,
+                                content_lines,
+                                content_words,
                                 content_len,
                                 ferox_response.url(),
                                 next_loc_str
@@ -227,7 +239,7 @@ async fn make_wildcard_request(
                         }
                     }
                 }
-                log::trace!("exit: make_wildcard_request -> {:?}", ferox_response);
+                log::trace!("exit: make_wildcard_request -> {}", ferox_response);
                 return Some(ferox_response);
             }
         }
@@ -284,14 +296,6 @@ pub async fn connectivity_test(target_urls: &[String]) -> Vec<String> {
 
     if good_urls.is_empty() {
         log::error!("Could not connect to any target provided, exiting.");
-        log::trace!("exit: connectivity_test");
-        eprintln!(
-            "{} {} Could not connect to any target provided",
-            status_colorizer("ERROR"),
-            module_colorizer("heuristics::connectivity_test"),
-        );
-
-        process::exit(1);
     }
 
     log::trace!("exit: connectivity_test -> {:?}", good_urls);
@@ -313,8 +317,7 @@ fn try_send_message_to_file(msg: &str, tx_file: UnboundedSender<String>, save_ou
             }
             Err(e) => {
                 log::error!(
-                    "{} {} {}",
-                    status_colorizer("ERROR"),
+                    "{} {}",
                     module_colorizer("heuristics::try_send_message_to_file"),
                     e
                 );

src/lib.rs

@@ -11,13 +11,30 @@ pub mod reporter;
 pub mod scanner;
 pub mod utils;
 
-use reqwest::header::HeaderMap;
-use reqwest::{Response, StatusCode, Url};
+use reqwest::{
+    header::HeaderMap,
+    {Response, StatusCode, Url},
+};
+use std::{error, fmt};
 use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender};
 
 /// Generic Result type to ease error handling in async contexts
-pub type FeroxResult<T> =
-    std::result::Result<T, Box<dyn std::error::Error + Send + Sync + 'static>>;
+pub type FeroxResult<T> = std::result::Result<T, Box<dyn error::Error + Send + Sync + 'static>>;
+
+/// Simple Error implementation to allow for custom error returns
+#[derive(Debug, Default)]
+pub struct FeroxError {
+    /// fancy string that can be printed via Display
+    pub message: String,
+}
+
+impl error::Error for FeroxError {}
+
+impl fmt::Display for FeroxError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}", &self.message)
+    }
+}
 
 /// Generic mpsc::unbounded_channel type to tidy up some code
 pub type FeroxChannel<T> = (UnboundedSender<T>, UnboundedReceiver<T>);
@@ -25,6 +42,9 @@ pub type FeroxChannel<T> = (UnboundedSender<T>, UnboundedReceiver<T>);
 /// Version pulled from Cargo.toml at compile time
 pub const VERSION: &str = env!("CARGO_PKG_VERSION");
 
+/// Maximum number of file descriptors that can be opened during a scan
+pub const DEFAULT_OPEN_FILE_LIMIT: usize = 8192;
+
 /// Default wordlist to use when `-w|--wordlist` isn't specified and not `wordlist` isn't set
 /// in a [ferox-config.toml](constant.DEFAULT_CONFIG_NAME.html) config file.
 ///
@@ -33,6 +53,9 @@ pub const VERSION: &str = env!("CARGO_PKG_VERSION");
 pub const DEFAULT_WORDLIST: &str =
     "/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt";
 
+/// Number of milliseconds to wait between polls of `PAUSE_SCAN` when user pauses a scan
+pub static SLEEP_DURATION: u64 = 500;
+
 /// Default list of status codes to report
 ///
 /// * 200 Ok
@@ -80,6 +103,19 @@ pub struct FeroxResponse {
     headers: HeaderMap,
 }
 
+/// Implement Display for FeroxResponse
+impl fmt::Display for FeroxResponse {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(
+            f,
+            "FeroxResponse {{ url: {}, status: {}, content-length: {} }}",
+            self.url(),
+            self.status(),
+            self.content_length()
+        )
+    }
+}
+
 /// `FeroxResponse` implementation
 impl FeroxResponse {
     /// Get the `StatusCode` of this `FeroxResponse`
@@ -140,6 +176,19 @@ impl FeroxResponse {
         self.url.query_pairs().count() > 0 || has_extension
     }
 
+    /// Returns line count of the response text.
+    pub fn line_count(&self) -> usize {
+        self.text().lines().count()
+    }
+
+    /// Returns word count of the response text.
+    pub fn word_count(&self) -> usize {
+        self.text()
+            .lines()
+            .map(|s| s.split_whitespace().count())
+            .sum()
+    }
+
     /// Create a new `FeroxResponse` from the given `Response`
     pub async fn from(response: Response, read_body: bool) -> Self {
         let url = response.url().clone();

src/logger.rs

@@ -19,8 +19,8 @@ pub fn initialize(verbosity: u8) {
                 0 => (),
                 1 => env::set_var("RUST_LOG", "warn"),
                 2 => env::set_var("RUST_LOG", "info"),
-                3 => env::set_var("RUST_LOG", "debug,hyper=info,reqwest=info"),
-                _ => env::set_var("RUST_LOG", "trace,hyper=info,reqwest=info"),
+                3 => env::set_var("RUST_LOG", "feroxbuster=debug,info"),
+                _ => env::set_var("RUST_LOG", "feroxbuster=trace,info"),
             }
         }
     }
@@ -55,9 +55,10 @@ pub fn initialize(verbosity: u8) {
             };
 
             let msg = format!(
-                "{} {:10.03} {}\n",
+                "{} {:10.03} {} {}\n",
                 style(level_name).bg(level_color).black(),
                 style(t).dim(),
+                record.target(),
                 style(record.args()).dim(),
             );
 

src/main.rs

@@ -1,17 +1,61 @@
-use feroxbuster::config::{CONFIGURATION, PROGRESS_PRINTER};
-use feroxbuster::scanner::scan_url;
-use feroxbuster::utils::{ferox_print, get_current_depth, module_colorizer, status_colorizer};
-use feroxbuster::{banner, heuristics, logger, reporter, FeroxResponse, FeroxResult, VERSION};
+use crossterm::event::{self, Event, KeyCode};
+use feroxbuster::{
+    banner,
+    config::{CONFIGURATION, PROGRESS_BAR, PROGRESS_PRINTER},
+    heuristics, logger, reporter,
+    scanner::{scan_url, PAUSE_SCAN},
+    utils::{ferox_print, get_current_depth, module_colorizer, status_colorizer},
+    FeroxError, FeroxResponse, FeroxResult, SLEEP_DURATION, VERSION,
+};
+#[cfg(not(target_os = "windows"))]
+use feroxbuster::{utils::set_open_file_limit, DEFAULT_OPEN_FILE_LIMIT};
 use futures::StreamExt;
-use std::collections::HashSet;
-use std::fs::File;
-use std::io::{stderr, BufRead, BufReader};
-use std::process;
-use std::sync::Arc;
-use tokio::io;
-use tokio::sync::mpsc::UnboundedSender;
+use std::{
+    collections::HashSet,
+    fs::File,
+    io::{stderr, BufRead, BufReader},
+    process,
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        Arc,
+    },
+    time::Duration,
+};
+use tokio::{io, sync::mpsc::UnboundedSender, task::JoinHandle};
 use tokio_util::codec::{FramedRead, LinesCodec};
 
+/// Atomic boolean flag, used to determine whether or not the terminal input handler should exit
+pub static SCAN_COMPLETE: AtomicBool = AtomicBool::new(false);
+
+/// Handles specific key events triggered by the user over stdin
+fn terminal_input_handler() {
+    log::trace!("enter: terminal_input_handler");
+
+    loop {
+        if event::poll(Duration::from_millis(SLEEP_DURATION)).unwrap_or(false) {
+            // It's guaranteed that the `read()` won't block when the `poll()`
+            // function returns `true`
+
+            if let Ok(key_pressed) = event::read() {
+                if key_pressed == Event::Key(KeyCode::Enter.into()) {
+                    // if the user presses Enter, toggle the value stored in PAUSE_SCAN
+                    // ignore any other keys
+                    let current = PAUSE_SCAN.load(Ordering::Acquire);
+
+                    PAUSE_SCAN.store(!current, Ordering::Release);
+                }
+            }
+        } else {
+            // Timeout expired and no `Event` is available; use the timeout to check SCAN_COMPLETE
+            if SCAN_COMPLETE.load(Ordering::Relaxed) {
+                // scan has been marked complete by main, time to exit the loop
+                break;
+            }
+        }
+    }
+    log::trace!("exit: terminal_input_handler");
+}
+
 /// Create a HashSet of Strings from the given wordlist then stores it inside an Arc
 fn get_unique_words_from_wordlist(path: &str) -> FeroxResult<Arc<HashSet<String>>> {
     log::trace!("enter: get_unique_words_from_wordlist({})", path);
@@ -19,12 +63,6 @@ fn get_unique_words_from_wordlist(path: &str) -> FeroxResult<Arc<HashSet<String>
     let file = match File::open(&path) {
         Ok(f) => f,
         Err(e) => {
-            eprintln!(
-                "{} {} {}",
-                status_colorizer("ERROR"),
-                module_colorizer("main::get_unique_words_from_wordlist"),
-                e
-            );
             log::error!("Could not open wordlist: {}", e);
             log::trace!("exit: get_unique_words_from_wordlist -> {}", e);
 
@@ -69,13 +107,9 @@ async fn scan(
             .await??;
 
     if words.len() == 0 {
-        eprintln!(
-            "{} {} Did not find any words in {}",
-            status_colorizer("ERROR"),
-            module_colorizer("main::scan"),
-            CONFIGURATION.wordlist
-        );
-        process::exit(1);
+        let mut err = FeroxError::default();
+        err.message = format!("Did not find any words in {}", CONFIGURATION.wordlist);
+        return Err(Box::new(err));
     }
 
     let mut tasks = vec![];
@@ -100,6 +134,7 @@ async fn scan(
     Ok(())
 }
 
+/// Get targets from either commandline or stdin, pass them back to the caller as a Result<Vec>
 async fn get_targets() -> FeroxResult<Vec<String>> {
     log::trace!("enter: get_targets");
 
@@ -123,15 +158,31 @@ async fn get_targets() -> FeroxResult<Vec<String>> {
     Ok(targets)
 }
 
-#[tokio::main]
-async fn main() {
-    // setup logging based on the number of -v's used
-    logger::initialize(CONFIGURATION.verbosity);
+/// async main called from real main, broken out in this way to allow for some synchronous code
+/// to be executed before bringing the tokio runtime online
+async fn wrapped_main() {
+    // join can only be called once, otherwise it causes the thread to panic
+    tokio::task::spawn_blocking(move || {
+        // ok, lazy_static! uses (unsurprisingly in retrospect) a lazy loading model where the
+        // thing obtained through deref isn't actually created until it's used. This created a
+        // problem when initializing the logger as it relied on PROGRESS_PRINTER which may or may
+        // not have been created by the time it was needed for logging (really only occurred in
+        // heuristics / banner / main). In order to initialize logging properly, we need to ensure
+        // PROGRESS_PRINTER and PROGRESS_BAR have been used at least once.  This call satisfies
+        // that constraint
+        PROGRESS_PRINTER.println("");
+        PROGRESS_BAR.join().unwrap();
+    });
 
-    // can't trace main until after logger is initialized
+    // can't trace main until after logger is initialized and the above task is started
     log::trace!("enter: main");
     log::debug!("{:#?}", *CONFIGURATION);
 
+    // spawn a thread that listens for keyboard input on stdin, when a user presses enter
+    // the input handler will toggle PAUSE_SCAN, which in turn is used to pause and resume
+    // scans that are already running
+    tokio::task::spawn_blocking(terminal_input_handler);
+
     let save_output = !CONFIGURATION.output.is_empty(); // was -o used?
 
     let (tx_term, tx_file, term_handle, file_handle) =
@@ -142,17 +193,9 @@ async fn main() {
         Ok(t) => t,
         Err(e) => {
             // should only happen in the event that there was an error reading from stdin
-            log::error!("{}", e);
-            ferox_print(
-                &format!(
-                    "{} {} {}",
-                    status_colorizer("ERROR"),
-                    module_colorizer("main::get_targets"),
-                    e
-                ),
-                &PROGRESS_PRINTER,
-            );
-            process::exit(1);
+            log::error!("{} {}", module_colorizer("main::get_targets"), e);
+            clean_up(tx_term, term_handle, tx_file, file_handle, save_output).await;
+            return;
         }
     };
 
@@ -165,15 +208,49 @@ async fn main() {
     // discard non-responsive targets
     let live_targets = heuristics::connectivity_test(&targets).await;
 
+    if live_targets.is_empty() {
+        clean_up(tx_term, term_handle, tx_file, file_handle, save_output).await;
+        return;
+    }
+
     // kick off a scan against any targets determined to be responsive
     match scan(live_targets, tx_term.clone(), tx_file.clone()).await {
         Ok(_) => {
             log::info!("All scans complete!");
         }
-        Err(e) => log::error!("An error occurred: {}", e),
+        Err(e) => {
+            ferox_print(
+                &format!("{} while scanning: {}", status_colorizer("Error"), e),
+                &PROGRESS_PRINTER,
+            );
+            clean_up(tx_term, term_handle, tx_file, file_handle, save_output).await;
+            process::exit(1);
+        }
     };
 
-    // manually drop tx in order for the rx task's while loops to eval to false
+    clean_up(tx_term, term_handle, tx_file, file_handle, save_output).await;
+
+    log::trace!("exit: main");
+}
+
+/// Single cleanup function that handles all the necessary drops/finishes etc required to gracefully
+/// shutdown the program
+async fn clean_up(
+    tx_term: UnboundedSender<FeroxResponse>,
+    term_handle: JoinHandle<()>,
+    tx_file: UnboundedSender<String>,
+    file_handle: Option<JoinHandle<()>>,
+    save_output: bool,
+) {
+    log::trace!(
+        "enter: clean_up({:?}, {:?}, {:?}, {:?}, {}",
+        tx_term,
+        term_handle,
+        tx_file,
+        file_handle,
+        save_output
+    );
+
     drop(tx_term);
     log::trace!("dropped terminal output handler's transmitter");
 
@@ -205,9 +282,26 @@ async fn main() {
         log::trace!("done awaiting file output handler's receiver");
     }
 
-    log::trace!("exit: main");
+    // mark all scans complete so the terminal input handler will exit cleanly
+    SCAN_COMPLETE.store(true, Ordering::Relaxed);
 
     // clean-up function for the MultiProgress bar; must be called last in order to still see
-    // the final trace message above
+    // the final trace messages above
     PROGRESS_PRINTER.finish();
+
+    log::trace!("exit: clean_up");
+}
+
+fn main() {
+    // setup logging based on the number of -v's used
+    logger::initialize(CONFIGURATION.verbosity);
+
+    // this function uses rlimit, which is not supported on windows
+    #[cfg(not(target_os = "windows"))]
+    set_open_file_limit(DEFAULT_OPEN_FILE_LIMIT);
+
+    if let Ok(mut runtime) = tokio::runtime::Runtime::new() {
+        let future = wrapped_main();
+        runtime.block_on(future);
+    }
 }

src/parser.rs

@@ -55,7 +55,7 @@ pub fn initialize() -> App<'static, 'static> {
                 .long("verbosity")
                 .takes_value(false)
                 .multiple(true)
-                .help("Increase verbosity level (use -vv or more for greater effect)"),
+                .help("Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v's is probably too much)"),
         )
         .arg(
             Arg::with_name("proxy")
@@ -67,6 +67,29 @@ pub fn initialize() -> App<'static, 'static> {
                     "Proxy to use for requests (ex: http(s)://host:port, socks5://host:port)",
                 ),
         )
+        .arg(
+            Arg::with_name("replay_proxy")
+                .short("P")
+                .long("replay-proxy")
+                .takes_value(true)
+                .value_name("REPLAY_PROXY")
+                .help(
+                    "Send only unfiltered requests through a Replay Proxy, instead of all requests",
+                ),
+        )
+        .arg(
+            Arg::with_name("replay_codes")
+                .short("R")
+                .long("replay-codes")
+                .value_name("REPLAY_CODE")
+                .takes_value(true)
+                .multiple(true)
+                .use_delimiter(true)
+                .requires("replay_proxy")
+                .help(
+                    "Status Codes to send through a Replay Proxy when found (default: --status-codes value)",
+                ),
+        )
         .arg(
             Arg::with_name("status_codes")
                 .short("s")
@@ -195,6 +218,30 @@ pub fn initialize() -> App<'static, 'static> {
                     "Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)",
                 ),
         )
+        .arg(
+            Arg::with_name("filter_words")
+                .short("W")
+                .long("filter-words")
+                .value_name("WORDS")
+                .takes_value(true)
+                .multiple(true)
+                .use_delimiter(true)
+                .help(
+                    "Filter out messages of a particular word count (ex: -W 312 -W 91,82)",
+                ),
+        )
+        .arg(
+            Arg::with_name("filter_lines")
+                .short("N")
+                .long("filter-lines")
+                .value_name("LINES")
+                .takes_value(true)
+                .multiple(true)
+                .use_delimiter(true)
+                .help(
+                    "Filter out messages of a particular line count (ex: -N 20 -N 31,30)",
+                ),
+        )
         .arg(
             Arg::with_name("filter_status")
                 .short("C")
@@ -204,7 +251,7 @@ pub fn initialize() -> App<'static, 'static> {
                 .multiple(true)
                 .use_delimiter(true)
                 .help(
-                    "Filter out status codes (deny list) (ex: -C 200 -S 401)",
+                    "Filter out status codes (deny list) (ex: -C 200 -C 401)",
                 ),
         )
         .arg(

src/reporter.rs

@@ -1,5 +1,5 @@
 use crate::config::{CONFIGURATION, PROGRESS_PRINTER};
-use crate::utils::{ferox_print, status_colorizer};
+use crate::utils::{create_report_string, ferox_print, make_request};
 use crate::{FeroxChannel, FeroxResponse};
 use console::strip_ansi_codes;
 use std::io::Write;
@@ -92,24 +92,16 @@ async fn spawn_terminal_reporter(
     );
 
     while let Some(resp) = resp_chan.recv().await {
-        log::debug!("received {} on reporting channel", resp.url());
+        log::trace!("received {} on reporting channel", resp.url());
 
         if CONFIGURATION.status_codes.contains(&resp.status().as_u16()) {
-            let report = if CONFIGURATION.quiet {
-                // -q used, just need the url
-                format!("{}\n", resp.url())
-            } else {
-                // normal printing with status and size
-                let status = status_colorizer(&resp.status().as_str());
-                format!(
-                    // example output
-                    // 200       3280 https://localhost.com/FAQ
-                    "{} {:>10} {}\n",
-                    status,
-                    resp.content_length(),
-                    resp.url()
-                )
-            };
+            let report = create_report_string(
+                resp.status().as_str(),
+                &resp.line_count().to_string(),
+                &resp.word_count().to_string(),
+                &resp.content_length().to_string(),
+                &resp.url().to_string(),
+            );
 
             // print to stdout
             ferox_print(&report, &PROGRESS_PRINTER);
@@ -126,7 +118,20 @@ async fn spawn_terminal_reporter(
                 }
             }
         }
-        log::debug!("report complete: {}", resp.url());
+        log::trace!("report complete: {}", resp.url());
+
+        if CONFIGURATION.replay_client.is_some()
+            && CONFIGURATION.replay_codes.contains(&resp.status().as_u16())
+        {
+            // replay proxy specified/client created and this response's status code is one that
+            // should be replayed
+            match make_request(CONFIGURATION.replay_client.as_ref().unwrap(), &resp.url()).await {
+                Ok(_) => {}
+                Err(e) => {
+                    log::error!("{}", e);
+                }
+            }
+        }
     }
     log::trace!("exit: spawn_terminal_reporter");
 }

src/scanner.rs

@@ -1,28 +1,49 @@
-use crate::config::{CONFIGURATION, PROGRESS_BAR};
-use crate::extractor::get_links;
-use crate::filters::{FeroxFilter, StatusCodeFilter, WildcardFilter};
-use crate::utils::{format_url, get_current_depth, make_request};
-use crate::{heuristics, progress, FeroxChannel, FeroxResponse};
-use futures::future::{BoxFuture, FutureExt};
-use futures::{stream, StreamExt};
+use crate::{
+    config::CONFIGURATION,
+    extractor::get_links,
+    filters::{FeroxFilter, StatusCodeFilter, WildcardFilter},
+    heuristics, progress,
+    utils::{format_url, get_current_depth, make_request},
+    FeroxChannel, FeroxResponse, SLEEP_DURATION,
+};
+use console::style;
+use futures::{
+    future::{BoxFuture, FutureExt},
+    stream, StreamExt,
+};
+use indicatif::{ProgressBar, ProgressStyle};
 use lazy_static::lazy_static;
 use reqwest::Url;
-use std::collections::HashSet;
-use std::convert::TryInto;
-use std::ops::Deref;
-use std::sync::atomic::{AtomicUsize, Ordering};
-use std::sync::{Arc, RwLock};
-use tokio::sync::mpsc::{self, UnboundedReceiver, UnboundedSender};
-use tokio::sync::Semaphore;
-use tokio::task::JoinHandle;
+use std::{
+    collections::HashSet,
+    convert::TryInto,
+    io::{stderr, Write},
+    ops::Deref,
+    sync::atomic::{AtomicBool, AtomicUsize, Ordering},
+    sync::{Arc, RwLock},
+};
+use tokio::{
+    sync::{
+        mpsc::{self, UnboundedReceiver, UnboundedSender},
+        Semaphore,
+    },
+    task::JoinHandle,
+    time,
+};
 
 /// Single atomic number that gets incremented once, used to track first scan vs. all others
 static CALL_COUNT: AtomicUsize = AtomicUsize::new(0);
 
+/// Atomic boolean flag, used to determine whether or not a scan should pause or resume
+pub static PAUSE_SCAN: AtomicBool = AtomicBool::new(false);
+
 lazy_static! {
     /// Set of urls that have been sent to [scan_url](fn.scan_url.html), used for deduplication
     static ref SCANNED_URLS: RwLock<HashSet<String>> = RwLock::new(HashSet::new());
 
+    /// A clock spinner protected with a RwLock to allow for a single thread to use at a time
+    static ref SINGLE_SPINNER: RwLock<ProgressBar> = RwLock::new(get_single_spinner());
+
     /// Vector of implementors of the FeroxFilter trait
     static ref FILTERS: Arc<RwLock<Vec<Box<dyn FeroxFilter>>>> = Arc::new(RwLock::new(Vec::<Box<dyn FeroxFilter>>::new()));
 
@@ -30,6 +51,72 @@ lazy_static! {
     static ref SCAN_LIMITER: Semaphore = Semaphore::new(CONFIGURATION.scan_limit);
 }
 
+/// Return a clock spinner, used when scans are paused
+fn get_single_spinner() -> ProgressBar {
+    log::trace!("enter: get_single_spinner");
+
+    let spinner = ProgressBar::new_spinner().with_style(
+        ProgressStyle::default_spinner()
+            .tick_strings(&[
+                "🕛", "🕐", "🕑", "🕒", "🕓", "🕔", "🕕", "🕖", "🕗", "🕘", "🕙", "🕚",
+            ])
+            .template(&format!(
+                "\t-= All Scans {{spinner}} {} =-",
+                style("Paused").red()
+            )),
+    );
+
+    log::trace!("exit: get_single_spinner -> {:?}", spinner);
+    spinner
+}
+
+/// Forced the calling thread into a busy loop
+///
+/// Every `SLEEP_DURATION` milliseconds, the function examines the result stored in `PAUSE_SCAN`
+///
+/// When the value stored in `PAUSE_SCAN` becomes `false`, the function returns, exiting the busy
+/// loop
+async fn pause_scan() {
+    log::trace!("enter: pause_scan");
+    // function uses tokio::time, not std
+
+    // local testing showed a pretty slow increase (less than linear) in CPU usage as # of
+    // concurrent scans rose when SLEEP_DURATION was set to 500, using that as the default for now
+    let mut interval = time::interval(time::Duration::from_millis(SLEEP_DURATION));
+
+    // ignore any error returned
+    let _ = stderr().flush();
+
+    if SINGLE_SPINNER.read().unwrap().is_finished() {
+        // in order to not leave draw artifacts laying around in the terminal, we call
+        // finish_and_clear on the progress bar when resuming scans. For this reason, we need to
+        // check if the spinner is finished, and repopulate the RwLock with a new spinner if
+        // necessary
+        if let Ok(mut guard) = SINGLE_SPINNER.write() {
+            *guard = get_single_spinner();
+        }
+    }
+
+    if let Ok(spinner) = SINGLE_SPINNER.write() {
+        spinner.enable_steady_tick(120);
+    }
+
+    loop {
+        // first tick happens immediately, all others wait the specified duration
+        interval.tick().await;
+
+        if !PAUSE_SCAN.load(Ordering::Acquire) {
+            // PAUSE_SCAN is false, so we can exit the busy loop
+            if let Ok(spinner) = SINGLE_SPINNER.write() {
+                spinner.finish_and_clear();
+            }
+            let _ = stderr().flush();
+            log::trace!("exit: pause_scan");
+            return;
+        }
+    }
+}
+
 /// Adds the given url to `SCANNED_URLS`
 ///
 /// If `SCANNED_URLS` did not already contain the url, return true; otherwise return false
@@ -43,17 +130,9 @@ fn add_url_to_list_of_scanned_urls(resp: &str, scanned_urls: &RwLock<HashSet<Str
     match scanned_urls.write() {
         // check new url against what's already been scanned
         Ok(mut urls) => {
-            let normalized_url = if resp.ends_with('/') {
-                // append a / to the list of 'seen' urls, this is to prevent the case where
-                // 3xx and 2xx duplicate eachother
-                resp.to_string()
-            } else {
-                format!("{}/", resp)
-            };
-
             // If the set did not contain resp, true is returned.
             // If the set did contain resp, false is returned.
-            let response = urls.insert(normalized_url);
+            let response = urls.insert(resp.to_string());
 
             log::trace!("exit: add_url_to_list_of_scanned_urls -> {}", response);
             response
@@ -206,7 +285,7 @@ fn create_urls(target_url: &str, word: &str, extensions: &[String]) -> Vec<Url>
 /// handles 2xx and 3xx responses by either checking if the url ends with a / (2xx)
 /// or if the Location header is present and matches the base url + / (3xx)
 fn response_is_directory(response: &FeroxResponse) -> bool {
-    log::trace!("enter: is_directory({:?})", response);
+    log::trace!("enter: is_directory({})", response);
 
     if response.status().is_redirection() {
         // status code is 3xx
@@ -232,10 +311,7 @@ fn response_is_directory(response: &FeroxResponse) -> bool {
                 }
             }
             None => {
-                log::debug!(
-                    "expected Location header, but none was found: {:?}",
-                    response
-                );
+                log::debug!("expected Location header, but none was found: {}", response);
                 log::trace!("exit: is_directory -> false");
                 return false;
             }
@@ -291,7 +367,7 @@ async fn try_recursion(
     transmitter: UnboundedSender<String>,
 ) {
     log::trace!(
-        "enter: try_recursion({:?}, {}, {:?})",
+        "enter: try_recursion({}, {}, {:?})",
         response,
         base_depth,
         transmitter
@@ -342,6 +418,12 @@ pub fn should_filter_response(response: &FeroxResponse) -> bool {
     if CONFIGURATION
         .filter_size
         .contains(&response.content_length())
+        || CONFIGURATION
+            .filter_line_count
+            .contains(&response.line_count())
+        || CONFIGURATION
+            .filter_word_count
+            .contains(&response.word_count())
     {
         // filtered value from --filter-size, size filters and wildcards are two separate filters
         // and are applied independently
@@ -391,7 +473,7 @@ async fn make_requests(
     for url in urls {
         if let Ok(response) = make_request(&CONFIGURATION.client, &url).await {
             // response came back without error, convert it to FeroxResponse
-            let ferox_response = FeroxResponse::from(response, CONFIGURATION.extract_links).await;
+            let ferox_response = FeroxResponse::from(response, true).await;
 
             // do recursion if appropriate
             if !CONFIGURATION.no_recursion {
@@ -434,8 +516,7 @@ async fn make_requests(
                         Err(_) => continue,
                     };
 
-                    let mut new_ferox_response =
-                        FeroxResponse::from(new_response, CONFIGURATION.extract_links).await;
+                    let mut new_ferox_response = FeroxResponse::from(new_response, true).await;
 
                     // filter if necessary
                     if should_filter_response(&new_ferox_response) {
@@ -444,11 +525,7 @@ async fn make_requests(
 
                     if new_ferox_response.is_file() {
                         // very likely a file, simply request and report
-                        log::debug!(
-                            "Singular extraction: {} ({})",
-                            new_ferox_response.url(),
-                            new_ferox_response.status().as_str(),
-                        );
+                        log::debug!("Singular extraction: {}", new_ferox_response);
 
                         send_report(report_chan.clone(), new_ferox_response);
 
@@ -456,11 +533,7 @@ async fn make_requests(
                     }
 
                     if !CONFIGURATION.no_recursion {
-                        log::debug!(
-                            "Recursive extraction: {} ({})",
-                            new_ferox_response.url(),
-                            new_ferox_response.status().as_str()
-                        );
+                        log::debug!("Recursive extraction: {}", new_ferox_response);
 
                         if new_ferox_response.status().is_success()
                             && !new_ferox_response.url().as_str().ends_with('/')
@@ -486,7 +559,7 @@ async fn make_requests(
 
 /// Simple helper to send a `FeroxResponse` over the tx side of an `mpsc::unbounded_channel`
 fn send_report(report_sender: UnboundedSender<FeroxResponse>, response: FeroxResponse) {
-    log::trace!("enter: send_report({:?}, {:?}", report_sender, response);
+    log::trace!("enter: send_report({:?}, {}", report_sender, response);
 
     match report_sender.send(response) {
         Ok(_) => {}
@@ -532,11 +605,9 @@ pub async fn scan_url(
     progress_bar.reset_elapsed();
 
     if CALL_COUNT.load(Ordering::Relaxed) == 0 {
-        // join can only be called once, otherwise it causes the thread to panic
-        tokio::task::spawn_blocking(move || PROGRESS_BAR.join().unwrap());
         CALL_COUNT.fetch_add(1, Ordering::Relaxed);
 
-        // this protection around join also allows us to add the first scanned url to SCANNED_URLS
+        // this protection allows us to add the first scanned url to SCANNED_URLS
         // from within the scan_url function instead of the recursion handler
         add_url_to_list_of_scanned_urls(&target_url, &SCANNED_URLS);
 
@@ -599,7 +670,15 @@ pub async fn scan_url(
             let pb = progress_bar.clone(); // progress bar is an Arc around internal state
             let tgt = target_url.to_string(); // done to satisfy 'static lifetime below
             (
-                tokio::spawn(async move { make_requests(&tgt, &word, base_depth, txd, txr).await }),
+                tokio::spawn(async move {
+                    if PAUSE_SCAN.load(Ordering::Acquire) {
+                        // for every word in the wordlist, check to see if PAUSE_SCAN is set to true
+                        // when true; enter a busy loop that only exits by setting PAUSE_SCAN back
+                        // to false
+                        pause_scan().await;
+                    }
+                    make_requests(&tgt, &word, base_depth, txd, txr).await
+                }),
                 pb,
             )
         })
@@ -762,10 +841,39 @@ mod tests {
         assert_eq!(
             urls.write()
                 .unwrap()
-                .insert("http://unknown_url/".to_string()),
+                .insert("http://unknown_url".to_string()),
             true
         );
 
         assert_eq!(add_url_to_list_of_scanned_urls(url, &urls), false);
     }
+
+    #[test]
+    /// test that get_single_spinner returns the correct spinner
+    fn scanner_get_single_spinner_returns_spinner() {
+        let spinner = get_single_spinner();
+        assert!(!spinner.is_finished());
+    }
+
+    #[tokio::test(core_threads = 1)]
+    /// tests that pause_scan pauses execution and releases execution when PAUSE_SCAN is toggled
+    /// the spinner used during the test has had .finish_and_clear called on it, meaning that
+    /// a new one will be created, taking the if branch within the function
+    async fn scanner_pause_scan_with_finished_spinner() {
+        let now = time::Instant::now();
+
+        PAUSE_SCAN.store(true, Ordering::Relaxed);
+        SINGLE_SPINNER.write().unwrap().finish_and_clear();
+
+        let expected = time::Duration::from_secs(2);
+
+        tokio::spawn(async move {
+            time::delay_for(expected).await;
+            PAUSE_SCAN.store(false, Ordering::Relaxed);
+        });
+
+        pause_scan().await;
+
+        assert!(now.elapsed() > expected);
+    }
 }

src/utils.rs

@@ -1,8 +1,12 @@
-use crate::FeroxResult;
+use crate::{
+    config::{CONFIGURATION, PROGRESS_PRINTER},
+    FeroxError, FeroxResult,
+};
 use console::{strip_ansi_codes, style, user_attended};
 use indicatif::ProgressBar;
-use reqwest::Url;
-use reqwest::{Client, Response};
+use reqwest::{Client, Response, Url};
+#[cfg(not(target_os = "windows"))]
+use rlimit::{getrlimit, setrlimit, Resource, Rlim};
 use std::convert::TryInto;
 
 /// Helper function that determines the current depth of a given url
@@ -153,6 +157,27 @@ pub fn format_url(
         extension
     );
 
+    if Url::parse(&word).is_ok() {
+        // when a full url is passed in as a word to be joined to a base url using
+        // reqwest::Url::join, the result is that the word (url) completely overwrites the base
+        // url, potentially resulting in requests to places that aren't actually the target
+        // specified.
+        //
+        // in order to resolve the issue, we check if the word from the wordlist is a parsable URL
+        // and if so, don't do any further processing
+        let message = format!(
+            "word ({}) from the wordlist is actually a URL, skipping...",
+            word
+        );
+        log::warn!("{}", message);
+
+        let mut err = FeroxError::default();
+        err.message = message;
+
+        log::trace!("exit: format_url -> {}", err);
+        return Err(Box::new(err));
+    }
+
     // from reqwest::Url::join
     //   Note: a trailing slash is significant. Without it, the last path component
     //   is considered to be a “file” name to be removed to get at the “directory”
@@ -221,7 +246,6 @@ pub async fn make_request(client: &Client, url: &Url) -> FeroxResult<Response> {
 
     match client.get(url.to_owned()).send().await {
         Ok(resp) => {
-            log::debug!("requested Url: {}", resp.url());
             log::trace!("exit: make_request -> {:?}", resp);
             Ok(resp)
         }
@@ -230,6 +254,19 @@ pub async fn make_request(client: &Client, url: &Url) -> FeroxResult<Response> {
             if e.to_string().contains("operation timed out") {
                 // only warn for timeouts, while actual errors are still left as errors
                 log::warn!("Error while making request: {}", e);
+            } else if e.is_redirect() {
+                if let Some(last_redirect) = e.url() {
+                    // get where we were headed (last_redirect) and where we came from (url)
+                    let fancy_message = format!("{} !=> {}", url, last_redirect);
+
+                    let report = if let Some(msg_status) = e.status() {
+                        create_report_string(msg_status.as_str(), "-1", "-1", "-1", &fancy_message)
+                    } else {
+                        create_report_string("UNK", "-1", "-1", "-1", &fancy_message)
+                    };
+
+                    ferox_print(&report, &PROGRESS_PRINTER)
+                };
             } else {
                 log::error!("Error while making request: {}", e);
             }
@@ -238,10 +275,113 @@ pub async fn make_request(client: &Client, url: &Url) -> FeroxResult<Response> {
     }
 }
 
+/// Helper to create the standard line for output to file/terminal
+///
+/// example output:
+/// 200      127l      283w     4134c http://localhost/faq
+pub fn create_report_string(
+    status: &str,
+    line_count: &str,
+    word_count: &str,
+    content_length: &str,
+    url: &str,
+) -> String {
+    if CONFIGURATION.quiet {
+        // -q used, just need the url
+        format!("{}\n", url)
+    } else {
+        // normal printing with status and sizes
+        let color_status = status_colorizer(status);
+        format!(
+            "{} {:>8}l {:>8}w {:>8}c {}\n",
+            color_status, line_count, word_count, content_length, url
+        )
+    }
+}
+
+/// Attempts to set the soft limit for the RLIMIT_NOFILE resource
+///
+/// RLIMIT_NOFILE is the maximum number of file descriptors that can be opened by this process
+///
+/// The soft limit is the value that the kernel enforces for the corresponding resource.
+/// The hard limit acts as a ceiling for the soft limit: an unprivileged process may set only its
+/// soft limit to a value in the range from 0 up to the hard limit, and (irreversibly) lower its
+/// hard limit.
+///
+/// A child process created via fork(2) inherits its parent's resource limits. Resource limits are
+/// per-process attributes that are shared by all of the threads in a process.
+///
+/// Based on the above information, no attempt is made to restore the limit to its pre-scan value
+/// as the adjustment made here is only valid for the scan itself (and any child processes, of which
+/// there are none).
+#[cfg(not(target_os = "windows"))]
+pub fn set_open_file_limit(limit: usize) -> bool {
+    log::trace!("enter: set_open_file_limit");
+
+    if let Ok((soft, hard)) = getrlimit(Resource::NOFILE) {
+        if hard.as_usize() > limit {
+            // our default open file limit is less than the current hard limit, this means we can
+            // set the soft limit to our default
+            let new_soft_limit = Rlim::from_usize(limit);
+
+            if setrlimit(Resource::NOFILE, new_soft_limit, hard).is_ok() {
+                log::debug!("set open file descriptor limit to {}", limit);
+
+                log::trace!("exit: set_open_file_limit -> {}", true);
+                return true;
+            }
+        } else if soft != hard {
+            // hard limit is lower than our default, the next best option is to set the soft limit as
+            // high as the hard limit will allow
+            if setrlimit(Resource::NOFILE, hard, hard).is_ok() {
+                log::debug!("set open file descriptor limit to {}", limit);
+
+                log::trace!("exit: set_open_file_limit -> {}", true);
+                return true;
+            }
+        }
+    }
+
+    // failed to set a new limit, as limit adjustments are a 'nice to have', we'll just log
+    // and move along
+    log::warn!("could not set open file descriptor limit to {}", limit);
+
+    log::trace!("exit: set_open_file_limit -> {}", false);
+    false
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
 
+    #[test]
+    /// set_open_file_limit with a low requested limit succeeds
+    fn utils_set_open_file_limit_with_low_requested_limit() {
+        let (_, hard) = getrlimit(Resource::NOFILE).unwrap();
+        let lower_limit = hard.as_usize() - 1;
+        assert!(set_open_file_limit(lower_limit));
+    }
+
+    #[test]
+    /// set_open_file_limit with a high requested limit succeeds
+    fn utils_set_open_file_limit_with_high_requested_limit() {
+        let (_, hard) = getrlimit(Resource::NOFILE).unwrap();
+        let higher_limit = hard.as_usize() + 1;
+        // calculate a new soft to ensure soft != hard and hit that logic branch
+        let new_soft = Rlim::from_usize(hard.as_usize() - 1);
+        setrlimit(Resource::NOFILE, new_soft, hard).unwrap();
+        assert!(set_open_file_limit(higher_limit));
+    }
+
+    #[test]
+    /// set_open_file_limit should fail when hard == soft
+    fn utils_set_open_file_limit_with_fails_when_both_limits_are_equal() {
+        let (_, hard) = getrlimit(Resource::NOFILE).unwrap();
+        // calculate a new soft to ensure soft == hard and hit the failure logic branch
+        setrlimit(Resource::NOFILE, hard, hard).unwrap();
+        assert!(!set_open_file_limit(hard.as_usize())); // returns false
+    }
+
     #[test]
     /// base url returns 1
     fn get_current_depth_base_url_returns_1() {
@@ -352,6 +492,19 @@ mod tests {
         );
     }
 
+    #[test]
+    /// word that is a fully formed url, should return an error
+    fn format_url_word_that_is_a_url() {
+        let url = format_url(
+            "http://localhost",
+            "http://schmocalhost",
+            false,
+            &Vec::new(),
+            None,
+        );
+        assert!(url.is_err());
+    }
+
     #[test]
     /// status colorizer uses red for 500s
     fn status_colorizer_uses_red_for_500s() {

tests/test_banner.rs

@@ -23,7 +23,7 @@ fn banner_prints_proxy() -> Result<(), Box<dyn std::error::Error>> {
         .pipe_stdin(file)
         .unwrap()
         .assert()
-        .failure()
+        .success()
         .stderr(
             predicate::str::contains("─┬─")
                 .and(predicate::str::contains("Target Url"))
@@ -43,6 +43,46 @@ fn banner_prints_proxy() -> Result<(), Box<dyn std::error::Error>> {
     Ok(())
 }
 
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + replay proxy
+fn banner_prints_replay_proxy() -> Result<(), Box<dyn std::error::Error>> {
+    let urls = vec![
+        String::from("http://localhost"),
+        String::from("http://schmocalhost"),
+    ];
+    let (tmp_dir, file) = setup_tmp_directory(&urls, "wordlist")?;
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--stdin")
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--replay-proxy")
+        .arg("http://127.0.0.1:8081")
+        .pipe_stdin(file)
+        .unwrap()
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("http://schmocalhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Replay Proxy"))
+                .and(predicate::str::contains("http://127.0.0.1:8081"))
+                .and(predicate::str::contains("─┴─")),
+        );
+
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
 #[test]
 /// test allows non-existent wordlist to trigger the banner printing to stderr
 /// expect to see all mandatory prints + multiple headers
@@ -56,7 +96,7 @@ fn banner_prints_headers() -> Result<(), Box<dyn std::error::Error>> {
         .arg("-H")
         .arg("mostuff:mothings")
         .assert()
-        .failure()
+        .success()
         .stderr(
             predicate::str::contains("─┬─")
                 .and(predicate::str::contains("Target Url"))
@@ -77,7 +117,7 @@ fn banner_prints_headers() -> Result<(), Box<dyn std::error::Error>> {
 #[test]
 /// test allows non-existent wordlist to trigger the banner printing to stderr
 /// expect to see all mandatory prints + multiple size filters
-fn banner_prints_filter_sizes() -> Result<(), Box<dyn std::error::Error>> {
+fn banner_prints_filter_sizes() {
     Command::cargo_bin("feroxbuster")
         .unwrap()
         .arg("--url")
@@ -86,8 +126,16 @@ fn banner_prints_filter_sizes() -> Result<(), Box<dyn std::error::Error>> {
         .arg("789456123")
         .arg("--filter-size")
         .arg("44444444")
+        .arg("-N")
+        .arg("678")
+        .arg("--filter-lines")
+        .arg("679")
+        .arg("-W")
+        .arg("93")
+        .arg("--filter-words")
+        .arg("94")
         .assert()
-        .failure()
+        .success()
         .stderr(
             predicate::str::contains("─┬─")
                 .and(predicate::str::contains("Target Url"))
@@ -98,11 +146,16 @@ fn banner_prints_filter_sizes() -> Result<(), Box<dyn std::error::Error>> {
                 .and(predicate::str::contains("Timeout (secs)"))
                 .and(predicate::str::contains("User-Agent"))
                 .and(predicate::str::contains("Size Filter"))
+                .and(predicate::str::contains("Word Count Filter"))
+                .and(predicate::str::contains("Line Count Filter"))
                 .and(predicate::str::contains("789456123"))
                 .and(predicate::str::contains("44444444"))
+                .and(predicate::str::contains("93"))
+                .and(predicate::str::contains("94"))
+                .and(predicate::str::contains("678"))
+                .and(predicate::str::contains("679"))
                 .and(predicate::str::contains("─┴─")),
         );
-    Ok(())
 }
 
 #[test]
@@ -118,7 +171,7 @@ fn banner_prints_queries() -> Result<(), Box<dyn std::error::Error>> {
         .arg("--query")
         .arg("stuff=things")
         .assert()
-        .failure()
+        .success()
         .stderr(
             predicate::str::contains("─┬─")
                 .and(predicate::str::contains("Target Url"))
@@ -147,7 +200,7 @@ fn banner_prints_status_codes() -> Result<(), Box<dyn std::error::Error>> {
         .arg("-s")
         .arg("201,301,401")
         .assert()
-        .failure()
+        .success()
         .stderr(
             predicate::str::contains("─┬─")
                 .and(predicate::str::contains("Target Url"))
@@ -163,6 +216,37 @@ fn banner_prints_status_codes() -> Result<(), Box<dyn std::error::Error>> {
     Ok(())
 }
 
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + replay codes
+fn banner_prints_replay_codes() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--replay-codes")
+        .arg("200,302")
+        .arg("--replay-proxy")
+        .arg("http://localhost:8081")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Replay Proxy"))
+                .and(predicate::str::contains("http://localhost:8081"))
+                .and(predicate::str::contains("Replay Proxy Codes"))
+                .and(predicate::str::contains("[200, 302]"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
 #[test]
 /// test allows non-existent wordlist to trigger the banner printing to stderr
 /// expect to see all mandatory prints + output file
@@ -174,7 +258,7 @@ fn banner_prints_output_file() -> Result<(), Box<dyn std::error::Error>> {
         .arg("--output")
         .arg("/super/cool/path")
         .assert()
-        .failure()
+        .success()
         .stderr(
             predicate::str::contains("─┬─")
                 .and(predicate::str::contains("Target Url"))
@@ -201,7 +285,7 @@ fn banner_prints_insecure() -> Result<(), Box<dyn std::error::Error>> {
         .arg("http://localhost")
         .arg("-k")
         .assert()
-        .failure()
+        .success()
         .stderr(
             predicate::str::contains("─┬─")
                 .and(predicate::str::contains("Target Url"))
@@ -228,7 +312,7 @@ fn banner_prints_redirects() -> Result<(), Box<dyn std::error::Error>> {
         .arg("http://localhost")
         .arg("-r")
         .assert()
-        .failure()
+        .success()
         .stderr(
             predicate::str::contains("─┬─")
                 .and(predicate::str::contains("Target Url"))
@@ -258,7 +342,7 @@ fn banner_prints_extensions() -> Result<(), Box<dyn std::error::Error>> {
         .arg("--extensions")
         .arg("pdf")
         .assert()
-        .failure()
+        .success()
         .stderr(
             predicate::str::contains("─┬─")
                 .and(predicate::str::contains("Target Url"))
@@ -285,7 +369,7 @@ fn banner_prints_dont_filter() -> Result<(), Box<dyn std::error::Error>> {
         .arg("http://localhost")
         .arg("--dont-filter")
         .assert()
-        .failure()
+        .success()
         .stderr(
             predicate::str::contains("─┬─")
                 .and(predicate::str::contains("Target Url"))
@@ -312,7 +396,7 @@ fn banner_prints_verbosity_one() -> Result<(), Box<dyn std::error::Error>> {
         .arg("http://localhost")
         .arg("-v")
         .assert()
-        .failure()
+        .success()
         .stderr(
             predicate::str::contains("─┬─")
                 .and(predicate::str::contains("Target Url"))
@@ -339,7 +423,7 @@ fn banner_prints_verbosity_two() -> Result<(), Box<dyn std::error::Error>> {
         .arg("http://localhost")
         .arg("-vv")
         .assert()
-        .failure()
+        .success()
         .stderr(
             predicate::str::contains("─┬─")
                 .and(predicate::str::contains("Target Url"))
@@ -366,7 +450,7 @@ fn banner_prints_verbosity_three() -> Result<(), Box<dyn std::error::Error>> {
         .arg("http://localhost")
         .arg("-vvv")
         .assert()
-        .failure()
+        .success()
         .stderr(
             predicate::str::contains("─┬─")
                 .and(predicate::str::contains("Target Url"))
@@ -393,7 +477,7 @@ fn banner_prints_verbosity_four() -> Result<(), Box<dyn std::error::Error>> {
         .arg("http://localhost")
         .arg("-vvvv")
         .assert()
-        .failure()
+        .success()
         .stderr(
             predicate::str::contains("─┬─")
                 .and(predicate::str::contains("Target Url"))
@@ -420,7 +504,7 @@ fn banner_prints_add_slash() -> Result<(), Box<dyn std::error::Error>> {
         .arg("http://localhost")
         .arg("-f")
         .assert()
-        .failure()
+        .success()
         .stderr(
             predicate::str::contains("─┬─")
                 .and(predicate::str::contains("Target Url"))
@@ -448,7 +532,7 @@ fn banner_prints_infinite_depth() -> Result<(), Box<dyn std::error::Error>> {
         .arg("--depth")
         .arg("0")
         .assert()
-        .failure()
+        .success()
         .stderr(
             predicate::str::contains("─┬─")
                 .and(predicate::str::contains("Target Url"))
@@ -476,7 +560,7 @@ fn banner_prints_recursion_depth() -> Result<(), Box<dyn std::error::Error>> {
         .arg("--depth")
         .arg("343214")
         .assert()
-        .failure()
+        .success()
         .stderr(
             predicate::str::contains("─┬─")
                 .and(predicate::str::contains("Target Url"))
@@ -503,7 +587,7 @@ fn banner_prints_no_recursion() -> Result<(), Box<dyn std::error::Error>> {
         .arg("http://localhost")
         .arg("-n")
         .assert()
-        .failure()
+        .success()
         .stderr(
             predicate::str::contains("─┬─")
                 .and(predicate::str::contains("Target Url"))
@@ -522,7 +606,7 @@ fn banner_prints_no_recursion() -> Result<(), Box<dyn std::error::Error>> {
 
 #[test]
 /// test allows non-existent wordlist to trigger the banner printing to stderr
-/// expect to see only the error of could not connect
+/// expect to see nothing
 fn banner_doesnt_print() -> Result<(), Box<dyn std::error::Error>> {
     Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -530,10 +614,8 @@ fn banner_doesnt_print() -> Result<(), Box<dyn std::error::Error>> {
         .arg("http://localhost")
         .arg("-q")
         .assert()
-        .failure()
-        .stderr(predicate::str::contains(
-            "ERROR heuristics::connectivity_test Could not connect to any target provided",
-        ));
+        .success()
+        .stderr(predicate::str::is_empty());
     Ok(())
 }
 
@@ -547,7 +629,7 @@ fn banner_prints_extract_links() -> Result<(), Box<dyn std::error::Error>> {
         .arg("http://localhost")
         .arg("-e")
         .assert()
-        .failure()
+        .success()
         .stderr(
             predicate::str::contains("─┬─")
                 .and(predicate::str::contains("Target Url"))
@@ -575,7 +657,7 @@ fn banner_prints_scan_limit() -> Result<(), Box<dyn std::error::Error>> {
         .arg("-L")
         .arg("4")
         .assert()
-        .failure()
+        .success()
         .stderr(
             predicate::str::contains("─┬─")
                 .and(predicate::str::contains("Target Url"))
@@ -603,7 +685,7 @@ fn banner_prints_filter_status() -> Result<(), Box<dyn std::error::Error>> {
         .arg("-C")
         .arg("200")
         .assert()
-        .failure()
+        .success()
         .stderr(
             predicate::str::contains("─┬─")
                 .and(predicate::str::contains("Target Url"))

tests/test_config.rs

@@ -18,7 +18,7 @@ fn read_in_config_file_for_settings() -> Result<(), Box<dyn std::error::Error>>
         .arg(file.as_os_str())
         .arg("-vvvv")
         .assert()
-        .failure()
+        .success()
         .stderr(predicate::str::contains("│ 37"));
 
     teardown_tmp_directory(tmp_dir);

tests/test_heuristics.rs

@@ -19,11 +19,9 @@ fn test_single_target_cannot_connect() -> Result<(), Box<dyn std::error::Error>>
         .arg("--wordlist")
         .arg(file.as_os_str())
         .assert()
-        .failure()
-        .stderr(
-            predicate::str::contains("Could not connect to any target provided")
-                .and(predicate::str::contains("ERROR"))
-                .and(predicate::str::contains("heuristics::connectivity_test")),
+        .success()
+        .stdout(
+            predicate::str::contains("Could not connect to http://fjdksafjkdsajfkdsajkfdsajkfsdjkdsfdsafdsafdsajkr3l2ajfdskafdsjk, skipping...", )
         );
 
     teardown_tmp_directory(tmp_dir);
@@ -47,11 +45,9 @@ fn test_two_targets_cannot_connect() -> Result<(), Box<dyn std::error::Error>> {
         .pipe_stdin(file)
         .unwrap()
         .assert()
-        .failure()
-        .stderr(
-            predicate::str::contains("Could not connect to any target provided")
-                .and(predicate::str::contains("ERROR"))
-                .and(predicate::str::contains("heuristics::connectivity_test")),
+        .success()
+        .stdout(
+            predicate::str::contains("Could not connect to http://fjdksafjkdsajfkdsajkfdsajkfsdjkdsfdsafdsafdsajkr3l2ajfdskafdsjk, skipping...", )
         );
 
     teardown_tmp_directory(tmp_dir);

tests/test_main.rs

@@ -25,10 +25,8 @@ fn main_use_root_owned_file_as_wordlist() -> Result<(), Box<dyn std::error::Erro
         .arg("/etc/shadow")
         .arg("-vvvv")
         .assert()
-        .success()
-        .stderr(predicate::str::contains(
-            "ERROR main::get_unique_words_from_wordlist Permission denied (os error 13)",
-        ));
+        .failure()
+        .stdout(predicate::str::contains("Permission denied (os error 13)"));
 
     // connectivity test hits it once
     assert_eq!(mock.times_called(), 1);
@@ -57,9 +55,7 @@ fn main_use_empty_wordlist() -> Result<(), Box<dyn std::error::Error>> {
         .arg("-vvvv")
         .assert()
         .failure()
-        .stderr(predicate::str::contains(
-            "ERROR main::scan Did not find any words in",
-        ));
+        .stdout(predicate::str::contains("Did not find any words in"));
 
     assert_eq!(mock.times_called(), 1);
 
@@ -83,11 +79,9 @@ fn main_use_empty_stdin_targets() -> Result<(), Box<dyn std::error::Error>> {
         .pipe_stdin(file)
         .unwrap()
         .assert()
-        .failure()
+        .success()
         .stderr(
             predicate::str::contains("Could not connect to any target provided")
-                .and(predicate::str::contains("ERROR"))
-                .and(predicate::str::contains("heuristics::connectivity_test"))
                 .and(predicate::str::contains("Target Url"))
                 .not(), // no target url found
         );

tests/test_scanner.rs

@@ -411,3 +411,52 @@ fn scanner_single_request_scan_with_filtered_result() -> Result<(), Box<dyn std:
     teardown_tmp_directory(tmp_dir);
     Ok(())
 }
+
+#[test]
+/// send a single valid request, expect a 200 response that then gets routed to the replay
+/// proxy
+fn scanner_single_request_replayed_to_proxy() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let proxy = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/LICENSE")
+        .return_status(200)
+        .return_body("this is a test")
+        .create_on(&srv);
+
+    let mock_two = Mock::new()
+        .expect_method(GET)
+        .expect_path("/LICENSE")
+        .return_status(200)
+        .return_body("this is a test")
+        .create_on(&proxy);
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--replay-proxy")
+        .arg(format!("http://{}", proxy.address().to_string()))
+        .arg("--replay-codes")
+        .arg("200")
+        .unwrap();
+
+    cmd.assert()
+        .success()
+        .stdout(
+            predicate::str::contains("/LICENSE")
+                .and(predicate::str::contains("200"))
+                .and(predicate::str::contains("14")),
+        )
+        .stderr(predicate::str::contains("Replay Proxy Codes"));
+
+    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock_two.times_called(), 1);
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}