bumped version to 2.0.1

2.0.0 internal overhaul

.github/stale.yml

@@ -1,5 +1,5 @@
 # Number of days of inactivity before an issue becomes stale
-daysUntilStale: 21
+daysUntilStale: 14
 # Number of days of inactivity before a stale issue is closed
 daysUntilClose: 7
 # Issues with these labels will never be considered stale

.github/workflows/coverage.yml

@@ -23,9 +23,13 @@ jobs:
           RUSTFLAGS: '-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort'
           RUSTDOCFLAGS: '-Cpanic=abort'
       - uses: actions-rs/grcov@v0.1
+      - uses: actions/upload-artifact@v2
+        with:
+          name: lcov.info
+          path: lcov.info
       - name: Convert lcov to xml
         run: |
-          curl -O https://raw.githubusercontent.com/eriwen/lcov-to-cobertura-xml/master/lcov_cobertura/lcov_cobertura.py
+          curl -O https://raw.githubusercontent.com/epi052/lcov-to-cobertura-xml/master/lcov_cobertura/lcov_cobertura.py
           chmod +x lcov_cobertura.py
           ./lcov_cobertura.py ./lcov.info
       - uses: codecov/codecov-action@v1
@@ -34,3 +38,7 @@ jobs:
           file: ./coverage.xml
           name: codecov-umbrella
           fail_ci_if_error: true
+      - uses: actions/upload-artifact@v2
+        with:
+          name: coverage.xml
+          path: ./coverage.xml

.gitignore

@@ -3,10 +3,6 @@
 debug/
 target/
 
-# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
-# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
-Cargo.lock
-
 # These are backup files generated by rustfmt
 **/*.rs.bk
 
@@ -25,3 +21,9 @@ lcov_cobertura.py
 
 # dockerignore file that makes it so i can work on the docker config without copying a 4GB manifest or w/e it is
 .dockerignore
+
+# state file created during tests
+ferox-http*
+
+# python stuff cuz reasons
+Pipfile*

.rustfmt.toml

@@ -0,0 +1 @@
+reorder_modules = false

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "feroxbuster"
-version = "1.10.0"
+version = "2.0.1"
 authors = ["Ben 'epi' Risher <epibar052@gmail.com>"]
 license = "MIT"
 edition = "2018"
@@ -22,31 +22,34 @@ lazy_static = "1.4"
 
 [dependencies]
 futures = { version = "0.3"}
-tokio = { version = "0.2", features = ["full"] }
-tokio-util = {version = "0.3", features = ["codec"]}
+tokio = { version = "1.0", features = ["full"] }
+tokio-util = {version = "0.6.3", features = ["codec"]}
 log = "0.4"
 env_logger = "0.8"
-reqwest = { version = "0.10", features = ["socks"] }
+reqwest = { version = "0.11", features = ["socks"] }
 clap = "2.33"
 lazy_static = "1.4"
 toml = "0.5"
-serde = { version = "1.0", features = ["derive"] }
+serde = { version = "1.0", features = ["derive", "rc"] }
 serde_json = "1.0"
 uuid = { version = "0.8", features = ["v4"] }
 indicatif = "0.15"
-console = "0.12"
+console = "0.14"
 openssl = { version = "0.10", features = ["vendored"] }
 dirs = "3.0"
 regex = "1"
-crossterm = "0.18"
+crossterm = "0.19"
 rlimit = "0.5"
 ctrlc = "3.1"
+fuzzyhash = "0.2.1"
+anyhow = "1.0"
+leaky-bucket = "0.10.0"
 
 [dev-dependencies]
 tempfile = "3.1"
-httpmock = "0.4.5"
-assert_cmd = "1.0.1"
-predicates = "1.0.5"
+httpmock = "0.5.2"
+assert_cmd = "1.0.3"
+predicates = "1.0.7"
 
 [profile.release]
 lto = true

Makefile

@@ -0,0 +1,77 @@
+default_prefix = /usr/local
+prefix ?= $(default_prefix)
+exec_prefix = $(prefix)
+bindir = $(exec_prefix)/bin
+datarootdir = $(prefix)/share
+datadir = $(datarootdir)
+example_config = ferox-config.toml.example
+config_file = ferox-config.toml
+
+SHR_SOURCES = $(shell find src -type f -wholename '*src/*.rs') Cargo.toml Cargo.lock
+
+RELEASE = debug
+DEBUG ?= 0
+ifeq (0,$(DEBUG))
+	ARGS = --release
+	RELEASE = release
+endif
+
+VENDORED ?= 0
+ifeq (1,$(VENDORED))
+    ARGS += --frozen
+endif
+
+TARGET = target/$(RELEASE)
+
+.PHONY: all clean distclean install uninstall update
+
+BIN=feroxbuster
+DESKTOP=$(APPID).desktop
+
+all: cli
+
+cli: $(TARGET)/$(BIN) $(TARGET)/$(BIN).1.gz $(SHR_SOURCES)
+
+clean:
+	cargo clean
+
+distclean: clean
+	rm -rf .cargo vendor Cargo.lock vendor.tar
+
+vendor: vendor.tar
+
+vendor.tar:
+	mkdir -p .cargo
+	cargo vendor | head -n -1 > .cargo/config
+	echo 'directory = "vendor"' >> .cargo/config
+	tar pcf vendor.tar vendor
+	rm -rf vendor
+
+install-cli: cli
+	install -Dm 0755 "$(TARGET)/$(BIN)" "$(DESTDIR)$(bindir)/$(BIN)"
+	install -Dm 0644 "$(TARGET)/$(BIN).1.gz" "$(DESTDIR)$(datadir)/man/man1/$(BIN).1.gz"
+	install -Dm 0644 "$(example_config)" "/etc/$(BIN)/$(config_File)"
+
+install: all install-cli
+
+uninstall-cli:
+	rm -f "$(DESTDIR)$(bindir)/$(BIN)"
+	rm -f "$(DESTDIR)$(datadir)/man/man1/$(BIN).1.gz"
+	rm -rf "/etc/$(BIN)/"
+
+uninstall: uninstall-cli
+
+update:
+	cargo update
+
+extract:
+ifeq ($(VENDORED),1)
+	tar pxf vendor.tar
+endif
+
+$(TARGET)/$(BIN): extract
+	cargo build --manifest-path Cargo.toml $(ARGS)
+
+$(TARGET)/$(BIN).1.gz: $(TARGET)/$(BIN)
+	help2man --no-info $< | gzip -c > $@.partial
+	mv $@.partial $@

README.md

@@ -22,7 +22,7 @@
   <a href="https://crates.io/crates/feroxbuster">
     <img src="https://img.shields.io/crates/v/feroxbuster?color=blue&label=version&logo=rust">
   </a>
- 
+
   <a href="https://crates.io/crates/feroxbuster">
     <img src="https://img.shields.io/crates/d/feroxbuster?label=downloads&logo=rust&color=inactive">
   </a>
@@ -45,20 +45,26 @@
 
 ## 😕 What the heck is a ferox anyway?
 
-Ferox is short for Ferric Oxide. Ferric Oxide, simply put, is rust.  The name rustbuster was taken, so I decided on a variation.  🤷	
+Ferox is short for Ferric Oxide. Ferric Oxide, simply put, is rust. The name rustbuster was taken, so I decided on a
+variation. 🤷
 
-## 🤔 What's it do tho? 
+## 🤔 What's it do tho?
 
-`feroxbuster` is a tool designed to perform [Forced Browsing](https://owasp.org/www-community/attacks/Forced_browsing).  
+`feroxbuster` is a tool designed to perform [Forced Browsing](https://owasp.org/www-community/attacks/Forced_browsing).
 
-Forced browsing is an attack where the aim is to enumerate and access resources that are not referenced by the web application, but are still accessible by an attacker.
+Forced browsing is an attack where the aim is to enumerate and access resources that are not referenced by the web
+application, but are still accessible by an attacker.
 
-`feroxbuster` uses brute force combined with a wordlist to search for unlinked content in target directories. These resources may store sensitive information about web applications and operational systems, such as source code, credentials, internal network addressing, etc...
+`feroxbuster` uses brute force combined with a wordlist to search for unlinked content in target directories. These
+resources may store sensitive information about web applications and operational systems, such as source code,
+credentials, internal network addressing, etc...
 
-This attack is also known as Predictable Resource Location, File Enumeration, Directory Enumeration, and Resource Enumeration.
+This attack is also known as Predictable Resource Location, File Enumeration, Directory Enumeration, and Resource
+Enumeration.
 
 📖 Table of Contents
 -----------------
+
 - [Installation](#-installation)
     - [Download a Release](#download-a-release)
     - [Snap Install](#snap-install)
@@ -72,13 +78,17 @@ This attack is also known as Predictable Resource Location, File Enumeration, Di
     - [Threads and Connection Limits At A High-Level](#threads-and-connection-limits-at-a-high-level)
     - [ferox-config.toml](#ferox-configtoml)
     - [Command Line Parsing](#command-line-parsing)
+- [Scan's Display Explained](#-scans-display-explained)
+    - [Discovered Resource](#discovered-resource)
+    - [Overall Scan Progress Bar](#overall-scan-progress-bar)
+    - [Directory Scan Progress Bar](#directory-scan-progress-bar)
 - [Example Usage](#-example-usage)
     - [Multiple Values](#multiple-values)
     - [Include Headers](#include-headers)
     - [IPv6, Non-recursive scan with INFO logging enabled](#ipv6-non-recursive-scan-with-info-level-logging-enabled)
     - [Read urls from STDIN; pipe only resulting urls out to another tool](#read-urls-from-stdin-pipe-only-resulting-urls-out-to-another-tool)
     - [Proxy traffic through Burp](#proxy-traffic-through-burp)
-    - [Proxy traffic through a SOCKS proxy](#proxy-traffic-through-a-socks-proxy)
+    - [Proxy traffic through a SOCKS proxy (including DNS lookups)](#proxy-traffic-through-a-socks-proxy-including-dns-lookups)
     - [Pass auth token via query parameter](#pass-auth-token-via-query-parameter)
     - [Extract Links from Response Body (new in `v1.1.0`)](#extract-links-from-response-body-new-in-v110)
     - [Limit Total Number of Concurrent Scans (new in `v1.2.0`)](#limit-total-number-of-concurrent-scans-new-in-v120)
@@ -89,18 +99,28 @@ This attack is also known as Predictable Resource Location, File Enumeration, Di
     - [Filter Response Using a Regular Expression (new in `v1.8.0`)](#filter-response-using-a-regular-expression-new-in-v180)
     - [Stop and Resume Scans (save scan's state to disk) (new in `v1.9.0`)](#stop-and-resume-scans---resume-from-file-new-in-v190)
     - [Enforce a Time Limit on Your Scan (new in `v1.10.0`)](#enforce-a-time-limit-on-your-scan-new-in-v1100)
+    - [Extract Links from robots.txt (New in `v1.10.2`)](#extract-links-from-robotstxt-new-in-v1102)
+    - [Filter Response by Similarity to A Given Page (fuzzy filter) (new in `v1.11.0`)](#filter-response-by-similarity-to-a-given-page-fuzzy-filter-new-in-v1110)
+    - [Cancel a Recursive Scan Interactively (new in `v1.12.0`)](#cancel-a-recursive-scan-interactively-new-in-v1120)
+    - [Limit Number of Requests per Second (Rate Limiting) (new in `v2.0.0`)](#limit-number-of-requests-per-second-rate-limiting-new-in-v200)
+    - [Silence all Output or Be Kinda Quiet (new in `v2.0.0`)](#silence-all-output-or-be-kinda-quiet-new-in-v200)
 - [Comparison w/ Similar Tools](#-comparison-w-similar-tools)
 - [Common Problems/Issues (FAQ)](#-common-problemsissues-faq)
     - [No file descriptors available](#no-file-descriptors-available)
     - [Progress bars print one line at a time](#progress-bars-print-one-line-at-a-time)
+    - [What do each of the numbers beside the URL mean?](#what-do-each-of-the-numbers-beside-the-url-mean)
+    - [Connection closed before message completed](#connection-closed-before-message-completed)
+    - [SSL Error routines:tls_process_server_certificate:certificate verify failed](#ssl-error-routinestls_process_server_certificatecertificate-verify-failed)
 
 ## 💿 Installation
 
 ### Download a Release
 
-Releases for multiple architectures can be found in the [Releases](https://github.com/epi052/feroxbuster/releases) section.  The latest release for each of the following systems can be downloaded and executed as shown below.
+Releases for multiple architectures can be found in the [Releases](https://github.com/epi052/feroxbuster/releases)
+section. The latest release for each of the following systems can be downloaded and executed as shown below.
 
 #### Linux (32 and 64-bit) & MacOS
+
 ```
 curl -sL https://raw.githubusercontent.com/epi052/feroxbuster/master/install-nix.sh | bash
 ```
@@ -129,7 +149,7 @@ Install using `snap`
 sudo snap install feroxbuster
 ```
 
-The only gotcha here is that the snap package can only read wordlists from a few specific locations. There are a few 
+The only gotcha here is that the snap package can only read wordlists from a few specific locations. There are a few
 possible solutions, of which two are shown below.
 
 If the wordlist is on the same partition as your home directory, it can be hard-linked into `~/snap/feroxbuster/common`
@@ -139,7 +159,7 @@ ln /path/to/the/wordlist ~/snap/feroxbuster/common
 ./feroxbuster -u http://localhost -w ~/snap/feroxbuster/common/wordlist
 ``` 
 
-If the wordlist is on a separate partition, hard-linking won't work.  You'll need to copy it into the snap directory.
+If the wordlist is on a separate partition, hard-linking won't work. You'll need to copy it into the snap directory.
 
 ```
 cp /path/to/the/wordlist ~/snap/feroxbuster/common
@@ -174,7 +194,8 @@ cargo install feroxbuster
 
 ### apt Install
 
-Download `feroxbuster_amd64.deb` from the [Releases](https://github.com/epi052/feroxbuster/releases) section.  After that, use your favorite package manager to install the `.deb`.
+Download `feroxbuster_amd64.deb` from the [Releases](https://github.com/epi052/feroxbuster/releases) section. After
+that, use your favorite package manager to install the `.deb`.
 
 ```
 wget -sLO https://github.com/epi052/feroxbuster/releases/latest/download/feroxbuster_amd64.deb.zip
@@ -223,7 +244,9 @@ cat targets.txt | sudo docker run --net=host --init -i feroxbuster --stdin -x js
 
 #### Mount a volume to pass in `ferox-config.toml`
 
-You've got some options available if you want to pass in a config file.  [`ferox-buster.toml`](#ferox-configtoml) can live in multiple locations and still be valid, so it's up to you how you'd like to pass it in.  Below are a few valid examples:
+You've got some options available if you want to pass in a config file.  [`ferox-buster.toml`](#ferox-configtoml) can
+live in multiple locations and still be valid, so it's up to you how you'd like to pass it in. Below are a few valid
+examples:
 
 ```
 sudo docker run --init -v $(pwd)/ferox-config.toml:/etc/feroxbuster/ferox-config.toml -it feroxbuster -u http://example.com
@@ -246,7 +269,9 @@ alias feroxbuster="sudo docker run --init -v ~/.config/feroxbuster:/root/.config
 ```
 
 ## ⚙️ Configuration
+
 ### Default Values
+
 Configuration begins with with the following built-in default values baked into the binary:
 
 - timeout: `7` seconds
@@ -255,6 +280,7 @@ Configuration begins with with the following built-in default values baked into
 - threads: `50`
 - verbosity: `0` (no logging enabled)
 - scan_limit: `0` (no limit imposed on concurrent scans)
+- rate_limit: `0` (no limit imposed on requests per second)
 - status_codes: `200 204 301 302 307 308 401 403 405`
 - user_agent: `feroxbuster/VERSION`
 - recursion depth: `4`
@@ -264,11 +290,20 @@ Configuration begins with with the following built-in default values baked into
 
 ### Threads and Connection Limits At A High-Level
 
-This section explains how the `-t` and `-L` options work together to determine the overall aggressiveness of a scan. The combination of the two values set by these options determines how hard your target will get hit and to some extent also determines how many resources will be consumed on your local machine.
+This section explains how the `-t` and `-L` options work together to determine the overall aggressiveness of a scan. The
+combination of the two values set by these options determines how hard your target will get hit and to some extent also
+determines how many resources will be consumed on your local machine.
 
 #### A Note on Green Threads
 
-`feroxbuster` uses so-called [green threads](https://en.wikipedia.org/wiki/Green_threads) as opposed to traditional kernel/OS threads. This means (at a high-level) that the threads are implemented entirely in userspace, within a single running process. As a result, a scan with 30 green threads will appear to the OS to be a single process with no additional light-weight processes associated with it as far as the kernel is concerned. As such, there will not be any impact to process (`nproc`) limits when specifying larger values for `-t`. However, these threads will still consume file descriptors, so you will need to ensure that you have a suitable `nlimit` set when scaling up the amount of threads. More detailed documentation on setting appropriate `nlimit` values can be found in the [No File Descriptors Available](#no-file-descriptors-available) section of the FAQ
+`feroxbuster` uses so-called [green threads](https://en.wikipedia.org/wiki/Green_threads) as opposed to traditional
+kernel/OS threads. This means (at a high-level) that the threads are implemented entirely in userspace, within a single
+running process. As a result, a scan with 30 green threads will appear to the OS to be a single process with no
+additional light-weight processes associated with it as far as the kernel is concerned. As such, there will not be any
+impact to process (`nproc`) limits when specifying larger values for `-t`. However, these threads will still consume
+file descriptors, so you will need to ensure that you have a suitable `nlimit` set when scaling up the amount of
+threads. More detailed documentation on setting appropriate `nlimit` values can be found in
+the [No File Descriptors Available](#no-file-descriptors-available) section of the FAQ
 
 #### Threads and Connection Limits: The Implementation
 
@@ -277,15 +312,20 @@ This section explains how the `-t` and `-L` options work together to determine t
 
 #### Threads and Connection Limits: Examples
 
-To truly have only 30 active requests to a site at any given time, `-t 30 -L 1` is necessary. Using `-t 30 -L 2` will result in a maximum of 60 total requests being processed at any given time for that site. And so on. For a conversation on this, please see [Issue #126](https://github.com/epi052/feroxbuster/issues/126) which may provide more (or less) clarity :wink:
+To truly have only 30 active requests to a site at any given time, `-t 30 -L 1` is necessary. Using `-t 30 -L 2` will
+result in a maximum of 60 total requests being processed at any given time for that site. And so on. For a conversation
+on this, please see [Issue #126](https://github.com/epi052/feroxbuster/issues/126) which may provide more (or less)
+clarity :wink:
 
 ### ferox-config.toml
+
 After setting built-in default values, any values defined in a `ferox-config.toml` config file will override the
-built-in defaults.  
+built-in defaults.
 
 `feroxbuster` searches for `ferox-config.toml` in the following locations (in the order shown):
+
 - `/etc/feroxbuster/` (global)
-- `CONFIG_DIR/ferxobuster/` (per-user)
+- `CONFIG_DIR/feroxbuster/` (per-user)
 - The same directory as the `feroxbuster` executable (per-user)
 - The user's current working directory (per-target)
 
@@ -294,14 +334,15 @@ built-in defaults.
 > - MacOs: `$HOME/Library/Application Support` i.e. `/Users/bob/Library/Application Support`
 > - Windows: `{FOLDERID_RoamingAppData}` i.e. `C:\Users\Bob\AppData\Roaming`
 
-If more than one valid configuration file is found, each one overwrites the values found previously.  
+If more than one valid configuration file is found, each one overwrites the values found previously.
 
 If no configuration file is found, nothing happens at this stage.
 
-As an example, let's say that we prefer to use a different wordlist as our default when scanning; we can
-set the `wordlist` value in the config file to override the baked-in default.
+As an example, let's say that we prefer to use a different wordlist as our default when scanning; we can set
+the `wordlist` value in the config file to override the baked-in default.
 
 Notes of interest:
+
 - it's ok to only specify values you want to change without specifying anything else
 - variable names in `ferox-config.toml` must match their command-line counterpart
 
@@ -312,6 +353,7 @@ wordlist = "/wordlists/jhaddix/all.txt"
 ```
 
 A pre-made configuration file with examples of all available settings can be found in `ferox-config.toml.example`.
+
 ```toml
 # ferox-config.toml
 # Example configuration for feroxbuster
@@ -333,7 +375,9 @@ A pre-made configuration file with examples of all available settings can be fou
 # replay_codes = [200, 302]
 # verbosity = 1
 # scan_limit = 6
+# rate_limit = 250
 # quiet = true
+# silent = true
 # json = true
 # output = "/targets/ellingson_mineral_company/gibson.txt"
 # debug_log = "/var/log/find-the-derp.log"
@@ -349,6 +393,7 @@ A pre-made configuration file with examples of all available settings can be fou
 # depth = 1
 # filter_size = [5174]
 # filter_regex = ["^ignore me$"]
+# filter_similar = ["https://somesite.com/soft404"]
 # filter_word_count = [993]
 # filter_line_count = [35, 36]
 # queries = [["name","value"], ["rick", "astley"]]
@@ -370,7 +415,9 @@ A pre-made configuration file with examples of all available settings can be fou
 ```
 
 ### Command Line Parsing
-Finally, after parsing the available config file, any options/arguments given on the commandline will override any values that were set as a built-in or config-file value.
+
+Finally, after parsing the available config file, any options/arguments given on the commandline will override any
+values that were set as a built-in or config-file value.
 
 ```
 USAGE:
@@ -385,49 +432,91 @@ FLAGS:
     -k, --insecure         Disables TLS certificate validation
         --json             Emit JSON logs to --output and --debug-log instead of normal text
     -n, --no-recursion     Do not scan recursively
-    -q, --quiet            Only print URLs; Don't print status codes, response size, running config, etc...
+    -q, --quiet            Hide progress bars and banner (good for tmux windows w/ notifications)
     -r, --redirects        Follow redirects
+        --silent           Only print URLs + turn off logging (good for piping a list of urls to other commands)
         --stdin            Read url(s) from STDIN
     -V, --version          Prints version information
     -v, --verbosity        Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v's is probably
                            too much)
 
 OPTIONS:
-        --debug-log <FILE>                  Output file to write log entries (use w/ --json for JSON entries)
-    -d, --depth <RECURSION_DEPTH>           Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)
-    -x, --extensions <FILE_EXTENSION>...    File extension(s) to search for (ex: -x php -x pdf js)
-    -N, --filter-lines <LINES>...           Filter out messages of a particular line count (ex: -N 20 -N 31,30)
-    -X, --filter-regex <REGEX>...           Filter out messages via regular expression matching on the response's body
-                                            (ex: -X '^ignore me$')
-    -S, --filter-size <SIZE>...             Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)
-    -C, --filter-status <STATUS_CODE>...    Filter out status codes (deny list) (ex: -C 200 -C 401)
-    -W, --filter-words <WORDS>...           Filter out messages of a particular word count (ex: -W 312 -W 91,82)
-    -H, --headers <HEADER>...               Specify HTTP headers (ex: -H Header:val 'stuff: things')
-    -o, --output <FILE>                     Output file to write results to (use w/ --json for JSON entries)
-    -p, --proxy <PROXY>                     Proxy to use for requests (ex: http(s)://host:port, socks5://host:port)
-    -Q, --query <QUERY>...                  Specify URL query parameters (ex: -Q token=stuff -Q secret=key)
-    -R, --replay-codes <REPLAY_CODE>...     Status Codes to send through a Replay Proxy when found (default: --status-
-                                            codes value)
-    -P, --replay-proxy <REPLAY_PROXY>       Send only unfiltered requests through a Replay Proxy, instead of all
-                                            requests
-        --resume-from <STATE_FILE>          State file from which to resume a partially complete scan (ex. --resume-from
-                                            ferox-1606586780.state)
-    -L, --scan-limit <SCAN_LIMIT>           Limit total number of concurrent scans (default: 0, i.e. no limit)
-    -s, --status-codes <STATUS_CODE>...     Status Codes to include (allow list) (default: 200 204 301 302 307 308 401
-                                            403 405)
-    -t, --threads <THREADS>                 Number of concurrent threads (default: 50)
-        --time-limit <TIME_SPEC>            Limit total run time of all scans (ex: --time-limit 10m)
-    -T, --timeout <SECONDS>                 Number of seconds before a request times out (default: 7)
-    -u, --url <URL>...                      The target URL(s) (required, unless --stdin used)
-    -a, --user-agent <USER_AGENT>           Sets the User-Agent (default: feroxbuster/VERSION)
-    -w, --wordlist <FILE>                   Path to the wordlist
+        --debug-log <FILE>                        Output file to write log entries (use w/ --json for JSON entries)
+    -d, --depth <RECURSION_DEPTH>
+            Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)
+
+    -x, --extensions <FILE_EXTENSION>...          File extension(s) to search for (ex: -x php -x pdf js)
+    -N, --filter-lines <LINES>...                 Filter out messages of a particular line count (ex: -N 20 -N 31,30)
+    -X, --filter-regex <REGEX>...
+            Filter out messages via regular expression matching on the response's body (ex: -X '^ignore me$')
+
+        --filter-similar-to <UNWANTED_PAGE>...
+            Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)
+
+    -S, --filter-size <SIZE>...                   Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)
+    -C, --filter-status <STATUS_CODE>...          Filter out status codes (deny list) (ex: -C 200 -C 401)
+    -W, --filter-words <WORDS>...                 Filter out messages of a particular word count (ex: -W 312 -W 91,82)
+    -H, --headers <HEADER>...                     Specify HTTP headers (ex: -H Header:val 'stuff: things')
+    -o, --output <FILE>                           Output file to write results to (use w/ --json for JSON entries)
+    -p, --proxy <PROXY>
+            Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)
+
+    -Q, --query <QUERY>...                        Specify URL query parameters (ex: -Q token=stuff -Q secret=key)
+        --rate-limit <RATE_LIMIT>
+            Limit number of requests per second (per directory) (default: 0, i.e. no limit)
+
+    -R, --replay-codes <REPLAY_CODE>...
+            Status Codes to send through a Replay Proxy when found (default: --status-codes value)
+
+    -P, --replay-proxy <REPLAY_PROXY>
+            Send only unfiltered requests through a Replay Proxy, instead of all requests
+
+        --resume-from <STATE_FILE>
+            State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)
+
+    -L, --scan-limit <SCAN_LIMIT>                 Limit total number of concurrent scans (default: 0, i.e. no limit)
+    -s, --status-codes <STATUS_CODE>...
+            Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)
+
+    -t, --threads <THREADS>                       Number of concurrent threads (default: 50)
+        --time-limit <TIME_SPEC>                  Limit total run time of all scans (ex: --time-limit 10m)
+    -T, --timeout <SECONDS>                       Number of seconds before a request times out (default: 7)
+    -u, --url <URL>...                            The target URL(s) (required, unless --stdin used)
+    -a, --user-agent <USER_AGENT>                 Sets the User-Agent (default: feroxbuster/VERSION)
+    -w, --wordlist <FILE>                         Path to the wordlist
+
 ```
 
+## 📊 Scan's Display Explained
+
+`feroxbuster` attempts to be intuitive and easy to understand, however, if you are wondering about any of the scan's
+output and what it means, this is the section for you!  
+
+### Discovered Resource
+
+When `feroxbuster` finds a response that you haven't filtered out, it's reported above the progress bars and looks similar to what's pictured below.  
+
+The number of lines, words, and bytes shown here can be used to [filter those responses](#filter-response-by-word-count--line-count--new-in-v160)
+
+![response-bar-explained](img/response-bar-explained.png)
+
+### Overall Scan Progress Bar
+
+The top progress bar, colored yellow, tracks the overall scan status.  Its fields are described in the image below.
+
+![total-bar-explained](img/total-bar-explained.png)
+
+### Directory Scan Progress Bar
+
+All other progress bars, colored cyan, represent a scan of one particular directory and will look similar to what's below.   
+
+![dir-scan-bar-explained](img/dir-scan-bar-explained.png)
+
 ## 🧰 Example Usage
 
 ### Multiple Values
 
-Options that take multiple values are very flexible.  Consider the following ways of specifying extensions:
+Options that take multiple values are very flexible. Consider the following ways of specifying extensions:
 
 ```
 ./feroxbuster -u http://127.1 -x pdf -x js,html -x php txt json,docx
@@ -435,7 +524,8 @@ Options that take multiple values are very flexible.  Consider the following way
 
 The command above adds .pdf, .js, .html, .php, .txt, .json, and .docx to each url
 
-All of the methods above (multiple flags, space separated, comma separated, etc...) are valid and interchangeable.  The same goes for urls, headers, status codes, queries, and size filters.
+All of the methods above (multiple flags, space separated, comma separated, etc...) are valid and interchangeable. The
+same goes for urls, headers, status codes, queries, and size filters.
 
 ### Include Headers
 
@@ -452,7 +542,7 @@ All of the methods above (multiple flags, space separated, comma separated, etc.
 ### Read urls from STDIN; pipe only resulting urls out to another tool
 
 ```
-cat targets | ./feroxbuster --stdin --quiet -s 200 301 302 --redirects -x js | fff -s 200 -o js-files
+cat targets | ./feroxbuster --stdin --silent -s 200 301 302 --redirects -x js | fff -s 200 -o js-files
 ```
 
 ### Proxy traffic through Burp
@@ -461,24 +551,25 @@ cat targets | ./feroxbuster --stdin --quiet -s 200 301 302 --redirects -x js | f
 ./feroxbuster -u http://127.1 --insecure --proxy http://127.0.0.1:8080
 ```
 
-### Proxy traffic through a SOCKS proxy
+### Proxy traffic through a SOCKS proxy (including DNS lookups)
 
 ```
-./feroxbuster -u http://127.1 --proxy socks5://127.0.0.1:9050
+./feroxbuster -u http://127.1 --proxy socks5h://127.0.0.1:9050
 ```
 
-### Pass auth token via query parameter 
+### Pass auth token via query parameter
 
 ```
 ./feroxbuster -u http://127.1 --query token=0123456789ABCDEF
 ```
 
-### Extract Links from Response Body (New in `v1.1.0`) 
+### Extract Links from Response Body (New in `v1.1.0`)
 
 Search through the body of valid responses (html, javascript, etc...) for additional endpoints to scan. This turns
-`feroxbuster` into a hybrid that looks for both linked and unlinked content. 
+`feroxbuster` into a hybrid that looks for both linked and unlinked content.
 
 Example request/response with `--extract-links` enabled:
+
 - Make request to `http://example.com/index.html`
 - Receive, and read in, the `body` of the response
 - Search the `body` for absolute and relative links (i.e. `homepage/assets/img/icons/handshake.svg`)
@@ -493,7 +584,8 @@ Example request/response with `--extract-links` enabled:
 ./feroxbuster -u http://127.1 --extract-links
 ```
 
-Here's a comparison of a wordlist-only scan vs `--extract-links` using [Feline](https://www.hackthebox.eu/home/machines/profile/274) from Hack the Box:
+Here's a comparison of a wordlist-only scan vs `--extract-links`
+using [Feline](https://www.hackthebox.eu/home/machines/profile/274) from Hack the Box:
 
 Wordlist only
 
@@ -505,8 +597,8 @@ With `--extract-links`
 
 ### Limit Total Number of Concurrent Scans (new in `v1.2.0`)
 
-Limit the number of scans permitted to run at any given time.  Recursion will still identify new directories, but newly
-discovered directories can only begin scanning when the total number of active scans drops below the value passed to 
+Limit the number of scans permitted to run at any given time. Recursion will still identify new directories, but newly
+discovered directories can only begin scanning when the total number of active scans drops below the value passed to
 `--scan-limit`.
 
 ```
@@ -517,9 +609,9 @@ discovered directories can only begin scanning when the total number of active s
 
 ### Filter Response by Status Code  (new in `v1.3.0`)
 
-Version 1.3.0 included an overhaul to the filtering system which will allow for a wide array of filters to be added 
-with minimal effort. The first such filter is a Status Code Filter. As responses come back from the scanned server,
-each one is checked against a list of known filters and either displayed or not according to which filters are set.
+Version 1.3.0 included an overhaul to the filtering system which will allow for a wide array of filters to be added with
+minimal effort. The first such filter is a Status Code Filter. As responses come back from the scanned server, each one
+is checked against a list of known filters and either displayed or not according to which filters are set.
 
 ```
 ./feroxbuster -u http://127.1 --filter-status 301
@@ -527,48 +619,60 @@ each one is checked against a list of known filters and either displayed or not
 
 ### Pause an Active Scan (new in `v1.4.0`)
 
-Scans can be paused and resumed by pressing the ENTER key (shown below)
+**NOTE**: [v1.12.0](#cancel-a-recursive-scan-interactively-new-in-v1120) added an interactive menu to the pause/resume
+functionality.  Active scans can still be paused, however, now you're presented with the option to cancel a scan instead
+of simply seeing a spinner.
 
-![pause-resume-demo](img/pause-resume-demo.gif)
+Scans can be paused and resumed by pressing the ENTER key (~~shown below~~, please see [v1.12.0](#cancel-a-recursive-scan-interactively-new-in-v1120)'s entry for the latest visual representation)
 
 ### Replay Responses to a Proxy based on Status Code (new in `v1.5.0`)
 
-The `--replay-proxy` and `--replay-codes` options were added as a way to only send a select few responses to a proxy.  This is in stark contrast to `--proxy` which proxies EVERY request.  
+The `--replay-proxy` and `--replay-codes` options were added as a way to only send a select few responses to a proxy.
+This is in stark contrast to `--proxy` which proxies EVERY request.
 
-Imagine you only care about proxying responses that have either the status code `200` or `302` (or you just don't want to clutter up your Burp history).  These two options will allow you to fine-tune what gets proxied and what doesn't.  
+Imagine you only care about proxying responses that have either the status code `200` or `302` (or you just don't want
+to clutter up your Burp history). These two options will allow you to fine-tune what gets proxied and what doesn't.
 
 ```
 ./feroxbuster -u http://127.1 --replay-proxy http://localhost:8080 --replay-codes 200 302 --insecure
 ```
 
-Of note: this means that for every response that matches your replay criteria, you'll end up sending the request that generated that response a second time.  Depending on the target and your engagement terms (if any), it may not make sense from a traffic generated perspective.
+Of note: this means that for every response that matches your replay criteria, you'll end up sending the request that
+generated that response a second time. Depending on the target and your engagement terms (if any), it may not make sense
+from a traffic generated perspective.
 
 ![replay-proxy-demo](img/replay-proxy-demo.gif)
 
 ### Filter Response by Word Count & Line Count  (new in `v1.6.0`)
 
-In addition to filtering on the size of a response, version 1.6.0 added the ability to filter out responses based on the number of lines and/or words contained within the response body.  This change drove a change to the information displayed to the user as well. This section will detail the new information and how to make use of it with the new filters provided.
+In addition to filtering on the size of a response, version 1.6.0 added the ability to filter out responses based on the
+number of lines and/or words contained within the response body. This change drove a change to the information displayed
+to the user as well. This section will detail the new information and how to make use of it with the new filters
+provided.
 
 Example output:
+
 ```
 200        10l        212w       38437c https://example-site.com/index.html
 ```
 
 There are five columns of output above:
+
 - column 1: status code - can be filtered with `-C|--filter-status`
 - column 2: number of lines - can be filtered with `-N|--filter-lines`
 - column 3: number of words - can be filtered with `-W|--filter-words`
 - column 4: number of bytes (overall size) - can be filtered with `-S|--filter-size`
 - column 5: url to discovered resource
 
-### Filter Response Using a Regular Expression (new in `v1.8.0`) 
+### Filter Response Using a Regular Expression (new in `v1.8.0`)
 
-Version 1.3.0 included an overhaul to the filtering system which will allow for a wide array of filters to be added 
-with minimal effort. The latest addition is a Regular Expression Filter. As responses come back from the scanned server,
-the **body** of the response is checked against the filter's regular expression.  If the expression is found in the 
-body, then that response is filtered out.  
+Version 1.3.0 included an overhaul to the filtering system which will allow for a wide array of filters to be added with
+minimal effort. The latest addition is a Regular Expression Filter. As responses come back from the scanned server,
+the **body** of the response is checked against the filter's regular expression. If the expression is found in the body,
+then that response is filtered out.
 
-**NOTE: Using regular expressions to filter large responses or many regular expressions may negatively impact performance.**  
+**NOTE: Using regular expressions to filter large responses or many regular expressions may negatively impact
+performance.**
 
 ```
 ./feroxbuster -u http://127.1 --filter-regex '[aA]ccess [dD]enied.?' --output results.txt --json
@@ -576,7 +680,8 @@ body, then that response is filtered out.
 
 ### Stop and Resume Scans (`--resume-from FILE`) (new in `v1.9.0`)
 
-Version 1.9.0 adds a few features that allow for completely stopping a scan, and resuming that same scan from a file on disk. 
+Version 1.9.0 adds a few features that allow for completely stopping a scan, and resuming that same scan from a file on
+disk.
 
 A simple `Ctrl+C` during a scan will create a file that contains information about the scan that was cancelled.
 
@@ -586,48 +691,51 @@ A simple `Ctrl+C` during a scan will create a file that contains information abo
 // example snippet of state file
 
 {
-   "scans":[
-      {
-         "id":"057016a14769414aac9a7a62707598cb",
-         "url":"https://localhost.com",
-         "scan_type":"Directory",
-         "complete":true
-      },
-      {
-         "id":"400b2323a16f43468a04ffcbbeba34c6",
-         "url":"https://localhost.com/css",
-         "scan_type":"Directory",
-         "complete":false
+  "scans": [
+    {
+      "id": "057016a14769414aac9a7a62707598cb",
+      "url": "https://localhost.com",
+      "scan_type": "Directory",
+      "complete": true
+    },
+    {
+      "id": "400b2323a16f43468a04ffcbbeba34c6",
+      "url": "https://localhost.com/css",
+      "scan_type": "Directory",
+      "complete": false
+    }
+  ],
+  "config": {
+    "wordlist": "/wordlists/seclists/Discovery/Web-Content/common.txt",
+    "...": "..."
+  },
+  "responses": [
+    {
+      "type": "response",
+      "url": "https://localhost.com/Login",
+      "path": "/Login",
+      "wildcard": false,
+      "status": 302,
+      "content_length": 0,
+      "line_count": 0,
+      "word_count": 0,
+      "headers": {
+        "content-length": "0",
+        "server": "nginx/1.16.1"
       }
-   ],
-   "config":{
-      "wordlist":"/wordlists/seclists/Discovery/Web-Content/common.txt",
-      "...":"..."
-   },
-   "responses":[
-      {
-         "type":"response",
-         "url":"https://localhost.com/Login",
-         "path":"/Login",
-         "wildcard":false,
-         "status":302,
-         "content_length":0,
-         "line_count":0,
-         "word_count":0,
-         "headers":{
-            "content-length":"0",
-            "server":"nginx/1.16.1"
-         }
-      }
-   ]
+    }
+  ]
 },
 ```
 
-Based on the example image above, the same scan can be resumed by using `feroxbuster --resume-from ferox-http_localhost-1606947491.state`.  Directories that were already complete are not rescanned, however partially complete scans are started from the beginning.  
+Based on the example image above, the same scan can be resumed by
+using `feroxbuster --resume-from ferox-http_localhost-1606947491.state`. Directories that were already complete are not
+rescanned, however partially complete scans are started from the beginning.
 
 ![resumed-scan](img/resumed-scan.gif)
 
-In order to prevent state file creation when `Ctrl+C` is pressed, you can simply add the entry below to your `ferox-config.toml`.
+In order to prevent state file creation when `Ctrl+C` is pressed, you can simply add the entry below to
+your `ferox-config.toml`.
 
 ```toml
 # ferox-config.toml
@@ -637,9 +745,12 @@ save_state = false
 
 ### Enforce a Time Limit on Your Scan (new in `v1.10.0`)
 
-Version 1.10.0 adds the ability to set a maximum runtime, or time limit, on your scan.  The usage is pretty simple: a number followed directly by a single character representing seconds, minutes, hours, or days.  `feroxbuster` refers to this combination as a time_spec.  
+Version 1.10.0 adds the ability to set a maximum runtime, or time limit, on your scan. The usage is pretty simple: a
+number followed directly by a single character representing seconds, minutes, hours, or days.  `feroxbuster` refers to
+this combination as a time_spec.
 
 Examples of possible time_specs:
+
 - `30s` - 30 seconds
 - `20m` - 20 minutes
 - `1h`  - 1 hour
@@ -649,28 +760,138 @@ A valid time_spec can be passed to `--time-limit` in order to force a shutdown a
 
 ![time-limit](img/time-limit.gif)
 
+### Extract Links from robots.txt (New in `v1.10.2`)
+
+In addition to [extracting links from the response body](#extract-links-from-response-body-new-in-v110), using
+`--extract-links` makes a request to `/robots.txt` and examines all `Allow` and `Disallow` entries. Directory entries
+are added to the scan queue, while file entries are requested and then reported if appropriate.
+
+### Filter Response by Similarity to A Given Page (fuzzy filter) (new in `v1.11.0`)
+
+Version 1.11.0 adds the ability to specify an example page for filtering pages that are similar to the given example.
+
+For example, consider a site that attempts to redirect new users to a `/register` endpoint. The `/register` page has a
+CSRF token that alters the page's response slightly with each new request (sometimes affecting overall length). This
+means that a simple line/word/char filter won't be able to filter all responses. In order to filter those redirects out,
+one could use a command like this:
+
+```
+./feroxbuster -u https://somesite.xyz --filter-similar-to https://somesite.xyz/register
+```
+
+`--filter-similar-to` requests the page passed to it via CLI (`https://somesite.xyz/register`), after which it hashes 
+the response body using the [SSDeep algorithm](https://ssdeep-project.github.io/ssdeep/index.html).  All subsequent 
+pages are hashed and compared to the original request's hash. If the comparison of the two hashes meets a certain 
+percentage of similarity (currently 95%), then that request will be filtered out.
+
+SSDeep was selected as it does a good job of identifying near-duplicate pages once content-length reaches a certain 
+size, while remaining performant.  Other algorithms were tested but resulted in huge performance hits (orders of 
+magnitude slower on requests/second).
+
+**NOTE**
+- SSDeep/`--filter-similar-to` does not do well at detecting similarity of very small responses
+  - The lack of accuracy with very small responses is considered a fair trade-off for not negatively impacting performance
+- Using a bunch of `--filter-similar-to` values **may** negatively impact performance
+
+### Cancel a Recursive Scan Interactively (new in `v1.12.0`)
+
+Version 1.12.0 expanded the pause/resume functionality introduced in [v1.4.0](#pause-an-active-scan-new-in-v140) by 
+adding an interactive menu from which currently running recursive scans can be cancelled, without affecting the overall scan.  Scans can still be paused indefinitely by pressing `ENTER`, however, the   
+
+Scans that are started via `-u` or passed in through `--stdin` cannot be cancelled, only scans found via `--extract-links` or recursion are eligible.
+
+Below is an example of the Scan Cancel Menu™.
+
+![cancel-menu](img/cancel-menu.png)
+
+Using the menu is pretty simple:
+- Press `ENTER` to view the menu
+- Choose a scan to cancel by entering its scan index (`1`)
+  - more than one scan can be selected by using a comma-separated list (`1,2,3` ... etc)
+- Confirm selections, after which all non-cancelled scans will resume
+
+Here is a short demonstration of cancelling two in-progress scans found via recursion.
+
+![cancel-scan](img/cancel-scan.gif)
+
+### Limit Number of Requests per Second (Rate Limiting) (new in `v2.0.0`)
+
+Version 2.0.0 added the ability to limit the number of requests per second. One thing to note is that the limit is 
+enforced on a per-directory basis. 
+
+Limit number of requests per second, per directory, to 100 (requests per second will increase by 100 for each active 
+directory found during recursion)
+
+```
+./feroxbuster -u http://localhost --rate-limit 100
+```
+
+Limit number of requests per second to 100 to the target as a whole (only one directory at a time will be scanned, thus
+limiting the number of requests per second overall)
+
+```
+./feroxbuster -u http://localhost --rate-limit 100 --scan-limit 1
+```
+
+![rate-limit](img/rate-limit-demo.gif)
+
+### Silence all Output or Be Kinda Quiet (new in `v2.0.0`)
+
+Version 2.0.0 introduces `--silent` which is almost equivalent to version 1.x.x's `--quiet`.  
+
+#### `--silent`
+
+Good for piping a list of urls to other commands:
+  - disables logging (no error messages to screen)
+  - don't print banner
+  - only display urls during scan
+
+example output:
+```
+https://localhost.com/contact
+https://localhost.com/about
+https://localhost.com/terms
+```
+
+#### `--quiet`
+
+Good for tmux windows that have notifications enabled as the only updates shown by the scan are new valid responses
+and new directories found that are suitable for recursion.
+  - hide progress bars
+  - don't print banner
+
+example output:
+```
+302        0l        0w        0c https://localhost.com/Login
+200      126l      281w     4091c https://localhost.com/maintenance
+200      126l      281w     4092c https://localhost.com/terms
+... more individual entries, followed by the directories being scanned ...
+Scanning: https://localhost.com
+Scanning: https://localhost.com/homepage
+Scanning: https://localhost.com/api
+```
+
 ## 🧐 Comparison w/ Similar Tools
 
-There are quite a few similar tools for forced browsing/content discovery.  Burp Suite Pro, Dirb, Dirbuster, etc... 
-However, in my opinion, there are two that set the standard: [gobuster](https://github.com/OJ/gobuster) and 
-[ffuf](https://github.com/ffuf/ffuf).  Both are mature, feature-rich, and all-around incredible tools to use.
+There are quite a few similar tools for forced browsing/content discovery. Burp Suite Pro, Dirb, Dirbuster, etc...
+However, in my opinion, there are two that set the standard: [gobuster](https://github.com/OJ/gobuster) and
+[ffuf](https://github.com/ffuf/ffuf). Both are mature, feature-rich, and all-around incredible tools to use.
 
-So, why would you ever want to use feroxbuster over ffuf/gobuster?  In most cases, you probably won't.  ffuf in particular
-can do the vast majority of things that feroxbuster can, while still offering boatloads more functionality.  Here are
-a few of the use-cases in which feroxbuster may be a better fit:
+So, why would you ever want to use feroxbuster over ffuf/gobuster? In most cases, you probably won't. ffuf in particular
+can do the vast majority of things that feroxbuster can, while still offering boatloads more functionality. Here are a
+few of the use-cases in which feroxbuster may be a better fit:
 
 - You want a **simple** tool usage experience
 - You want to be able to run your content discovery as part of some crazy 12 command unix **pipeline extravaganza**
 - You want to scan through a **SOCKS** proxy
 - You want **auto-filtering** of Wildcard responses by default
-- You want an integrated **link extractor** to increase discovered endpoints
+- You want an integrated **link extractor/robots.txt parser** to increase discovered endpoints
 - You want **recursion** along with some other thing mentioned above (ffuf also does recursion)
 - You want a **configuration file** option for overriding built-in default values for your scans
 
 |                                                          | feroxbuster | gobuster | ffuf |
 |------------------------------------------------------------------------------|---|---|---|
 | fast                                                                         | ✔ | ✔ | ✔ |
-| easy to use                                                                  | ✔ | ✔ |   |
 | allows recursion                                                             | ✔ |   | ✔ |
 | can specify query parameters                                                 | ✔ |   | ✔ |
 | SOCKS proxy support                                                          | ✔ |   |   |
@@ -692,13 +913,18 @@ a few of the use-cases in which feroxbuster may be a better fit:
 | filter out responses by regular expression (`v1.8.0`)                        | ✔ |   | ✔ |
 | save scan's state to disk (can pick up where it left off) (`v1.9.0`)         | ✔ |   |   |
 | maximum run time limit (`v1.10.0`)                                           | ✔ |   | ✔ |
+| use robots.txt to increase scan coverage (`v1.10.2`)                         | ✔ |   |   |
+| use example page's response to fuzzily filter similar pages  (`v1.11.0`)     | ✔ |   |   |
+| cancel a recursive scan interactively (`v1.12.0`)                            | ✔ |   |   |
+| limit number of requests per second (`v2.0.0`)                               | ✔ | ✔ | ✔ |
+| hide progress bars or be silent (or some variation) (`v2.0.0`)               | ✔ | ✔ | ✔ |
 | **huge** number of other options                                             |   |   | ✔ |
 
-Of note, there's another written-in-rust content discovery tool, [rustbuster](https://github.com/phra/rustbuster). I 
-came across rustbuster when I was naming my tool (😢). I don't have any experience using it, but it appears to 
-be able to do POST requests with an HTTP body, has SOCKS support, and has an 8.3 shortname scanner (in addition to vhost
-dns, directory, etc...).  In short, it definitely looks interesting and may be what you're looking for as it has some 
-capability I haven't seen in similar tools.  
+Of note, there's another written-in-rust content discovery tool, [rustbuster](https://github.com/phra/rustbuster). I
+came across rustbuster when I was naming my tool (😢). I don't have any experience using it, but it appears to be able
+to do POST requests with an HTTP body, has SOCKS support, and has an 8.3 shortname scanner (in addition to vhost dns,
+directory, etc...). In short, it definitely looks interesting and may be what you're looking for as it has some
+capability I haven't seen in similar tools.
 
 ## 🤯 Common Problems/Issues (FAQ)
 
@@ -708,21 +934,24 @@ Why do I get a bunch of `No file descriptors available (os error 24)` errors?
 
 ---
 
-There are a few potential causes of this error.  The simplest is that your operating system sets an open file limit that is aggressively low.  Through personal testing, I've found that `4096` is a reasonable open file limit (this will vary based on your exact setup).
+There are a few potential causes of this error. The simplest is that your operating system sets an open file limit that
+is aggressively low. Through personal testing, I've found that `4096` is a reasonable open file limit (this will vary
+based on your exact setup).
 
-There are quite a few options to solve this particular problem, of which a handful are shown below.  
+There are quite a few options to solve this particular problem, of which a handful are shown below.
 
 #### Increase the Number of Open Files
 
-We'll start by increasing the number of open files the OS allows. On my Kali install, the default was `1024`, and I know some MacOS installs use `256` 😕.
+We'll start by increasing the number of open files the OS allows. On my Kali install, the default was `1024`, and I know
+some MacOS installs use `256` 😕.
 
 ##### Edit `/etc/security/limits.conf`
 
-One option to up the limit is to edit `/etc/security/limits.conf` so that it includes the two lines below.  
+One option to up the limit is to edit `/etc/security/limits.conf` so that it includes the two lines below.
 
 - `*` represents all users
-- `hard` and `soft` indicate the hard and soft limits for the OS 
-- `nofile` is the number of open files option. 
+- `hard` and `soft` indicate the hard and soft limits for the OS
+- `nofile` is the number of open files option.
 
 ```
 /etc/security/limits.conf
@@ -743,20 +972,25 @@ ulimit -n 4096
 
 #### Additional Tweaks (may not be needed)
 
-If you still find yourself hitting the file limit with the above changes, there are a few additional tweaks that may help.  
+If you still find yourself hitting the file limit with the above changes, there are a few additional tweaks that may
+help.
 
-> This section was shamelessly stolen from this [stackoverflow answer](https://stackoverflow.com/a/3923785).  More information is included in that post and is recommended reading if you end up needing to use this section.
+> This section was shamelessly stolen from this [stackoverflow answer](https://stackoverflow.com/a/3923785). More information is included in that post and is recommended reading if you end up needing to use this section.
 
-✨ Special thanks to HTB user [@sparkla](https://www.hackthebox.eu/home/users/profile/221599) for their help with identifying these additional tweaks ✨
+✨ Special thanks to HTB user [@sparkla](https://www.hackthebox.eu/home/users/profile/221599) for their help with
+identifying these additional tweaks ✨
 
 ##### Increase the ephemeral port range, and decrease the tcp_fin_timeout.
 
-The ephermal port range defines the maximum number of outbound sockets a host can create from a particular I.P. address. The fin_timeout defines the minimum time these sockets will stay in TIME_WAIT state (unusable after being used once). Usual system defaults are
+The ephermal port range defines the maximum number of outbound sockets a host can create from a particular I.P. address.
+The fin_timeout defines the minimum time these sockets will stay in TIME_WAIT state (unusable after being used once).
+Usual system defaults are
 
 - `net.ipv4.ip_local_port_range = 32768   61000`
 - `net.ipv4.tcp_fin_timeout = 60`
 
-This basically means your system cannot consistently guarantee more than `(61000 - 32768) / 60 = 470` sockets per second.
+This basically means your system cannot consistently guarantee more than `(61000 - 32768) / 60 = 470` sockets per
+second.
 
 ```
 sudo sysctl net.ipv4.ip_local_port_range="15000 61000"
@@ -765,7 +999,9 @@ sudo sysctl net.ipv4.tcp_fin_timeout=30
 
 ##### Allow socket reuse while in a `TIME_WAIT` status
 
-This allows fast cycling of sockets in time_wait state and re-using them. Make sure to read post [Coping with the TCP TIME-WAIT](https://vincent.bernat.ch/en/blog/2014-tcp-time-wait-state-linux) from Vincent Bernat to understand the implications.
+This allows fast cycling of sockets in time_wait state and re-using them. Make sure to read
+post [Coping with the TCP TIME-WAIT](https://vincent.bernat.ch/en/blog/2014-tcp-time-wait-state-linux) from Vincent
+Bernat to understand the implications.
 
 ```
 sudo sysctl net.ipv4.tcp_tw_reuse=1 
@@ -773,13 +1009,47 @@ sudo sysctl net.ipv4.tcp_tw_reuse=1
 
 ### Progress bars print one line at a time
 
-`feroxbuster` needs a terminal width of at least the size of what's being printed in order to do progress bar printing correctly.  If your width is too small, you may see output like what's shown below.
+`feroxbuster` needs a terminal width of at least the size of what's being printed in order to do progress bar printing
+correctly. If your width is too small, you may see output like what's shown below.
 
 ![small-term](img/small-term.png)
 
-If you can, simply make the terminal wider and rerun.  If you're unable to make your terminal wider
-consider using `-q` to suppress the progress bars.
+If you can, simply make the terminal wider and rerun. If you're unable to make your terminal wider consider using `-q`
+to suppress the progress bars.
 
 ### What do each of the numbers beside the URL mean?
 
-Please refer to [this section](#filter-response-by-word-count--line-count--new-in-v160) where each number's meaning and how to use it to filter responses is discussed.
+Please refer to [this section](#filter-response-by-word-count--line-count--new-in-v160) where each number's meaning and
+how to use it to filter responses is discussed.
+
+### Connection closed before message completed
+
+The error in question can be boiled down to 'networking stuff'. `feroxbuster`
+uses [reqwest](https://docs.rs/reqwest/latest/) which uses [hyper](https://docs.rs/hyper/latest/hyper/) to make requests
+to the server. [This issue report](https://github.com/hyperium/hyper/issues/2136#issuecomment-589345238) to the hyper
+project explains what is happening (quoted below to save you a click). This isn't a bug so much as it's a
+target-specific tuning issue. When lowering the `-t` value, the error doesn't occur (or happens much less frequently).
+
+This isn't a bug. Simply slow down the scan. A `-t` value of 50 was chosen as a sane default that's still quite fast out
+of the box. However, network related errors may occur when the client and/or server become over-saturated.
+The [Threads and Connection Limits At A High-Level](#threads-and-connection-limits-at-a-high-level) section details how
+to accomplish per-target tuning.
+
+> This is just due to the racy nature of networking.
+>
+> hyper has a connection pool of idle connections, and it selected one to send your request. Most of the time, hyper will receive the server's FIN and drop the dead connection from its pool. But occasionally, a connection will be selected from the pool and written to at the same time the server is deciding to close the connection. Since hyper already wrote some of the request, it can't really retry it automatically on a new connection, since the server may have acted already.
+
+### SSL Error routines:tls_process_server_certificate:certificate verify failed
+
+In the event you see an error similar to
+
+![self-signed](img/insecure.png)
+
+```
+error trying to connect: error:1416F086:SSL routines:tls_process_server_certificate:certificate verify failed:ssl/statem/statem_clnt.c:1913: (self signed certificate)
+```
+
+You just need to add the `-k|--insecure` flag to your command.
+
+`feroxbuster` rejects self-signed certs and other "insecure" certificates/site configurations by default. You can choose
+to scan these services anyway by telling `feroxbuster` to ignore insecure server certs.

build.rs

@@ -5,6 +5,12 @@ use clap::Shell;
 include!("src/parser.rs");
 
 fn main() {
+    println!("cargo:rerun-if-env-changed=src/parser.rs");
+
+    if std::env::var("DOCS_RS").is_ok() {
+        return; // only build when we're not generating docs
+    }
+
     let outdir = "shell_completions";
 
     let mut app = initialize();

ferox-config.toml.example

@@ -17,7 +17,9 @@
 # replay_codes = [200, 302]
 # verbosity = 1
 # scan_limit = 6
+# rate_limit = 250
 # quiet = true
+# silent = true
 # json = true
 # output = "/targets/ellingson_mineral_company/gibson.txt"
 # debug_log = "/var/log/find-the-derp.log"
@@ -33,6 +35,7 @@
 # depth = 1
 # filter_size = [5174]
 # filter_regex = ["^ignore me$"]
+# filter_similar = ["https://somesite.com/soft404"]
 # filter_word_count = [993]
 # filter_line_count = [35, 36]
 # queries = [["name","value"], ["rick", "astley"]]

install-nix.sh

@@ -36,18 +36,23 @@ elif [[ "$(expr substr $(uname -s) 1 5)" == "Linux" ]]; then
         rm "${LIN64_ZIP}"
     fi
 
-    echo "[=] Installing Noto Emoji Font"
-    mkdir -p ~/.fonts
-    pushd ~/.fonts 2>&1 >/dev/null
+    if [[ -e ~/.fonts/NotoColorEmoji.ttf ]]; then
+       echo "[=] Found Noto Emoji Font, skipping install" 
+    else 
+        echo "[=] Installing Noto Emoji Font"
+        mkdir -p ~/.fonts
+        pushd ~/.fonts 2>&1 >/dev/null
 
-    curl -sLO "${EMOJI_URL}"
+        curl -sLO "${EMOJI_URL}"
 
-    fc-cache -f -v >/dev/null
+        fc-cache -f -v >/dev/null
 
-    popd 2>&1 >/dev/null
-    echo "[+] Noto Emoji Font installed"
+        popd 2>&1 >/dev/null
+        echo "[+] Noto Emoji Font installed"
+    fi
 fi
 
+
 chmod +x ./feroxbuster
 
 echo "[+] Installed feroxbuster version $(./feroxbuster -V)"

shell_completions/_feroxbuster

@@ -25,8 +25,8 @@ _feroxbuster() {
 '--depth=[Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)]' \
 '-T+[Number of seconds before a request times out (default: 7)]' \
 '--timeout=[Number of seconds before a request times out (default: 7)]' \
-'-p+[Proxy to use for requests (ex: http(s)://host:port, socks5://host:port)]' \
-'--proxy=[Proxy to use for requests (ex: http(s)://host:port, socks5://host:port)]' \
+'-p+[Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)]' \
+'--proxy=[Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)]' \
 '-P+[Send only unfiltered requests through a Replay Proxy, instead of all requests]' \
 '--replay-proxy=[Send only unfiltered requests through a Replay Proxy, instead of all requests]' \
 '*-R+[Status Codes to send through a Replay Proxy when found (default: --status-codes value)]' \
@@ -55,13 +55,16 @@ _feroxbuster() {
 '*--filter-lines=[Filter out messages of a particular line count (ex: -N 20 -N 31,30)]' \
 '*-C+[Filter out status codes (deny list) (ex: -C 200 -C 401)]' \
 '*--filter-status=[Filter out status codes (deny list) (ex: -C 200 -C 401)]' \
+'*--filter-similar-to=[Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)]' \
 '-L+[Limit total number of concurrent scans (default: 0, i.e. no limit)]' \
 '--scan-limit=[Limit total number of concurrent scans (default: 0, i.e. no limit)]' \
+'--rate-limit=[Limit number of requests per second (per directory) (default: 0, i.e. no limit)]' \
 '--time-limit=[Limit total run time of all scans (ex: --time-limit 10m)]' \
-'*-v[Increase verbosity level (use -vv or more for greater effect. \[CAUTION\] 4 -v'\''s is probably too much)]' \
-'*--verbosity[Increase verbosity level (use -vv or more for greater effect. \[CAUTION\] 4 -v'\''s is probably too much)]' \
-'-q[Only print URLs; Don'\''t print status codes, response size, running config, etc...]' \
-'--quiet[Only print URLs; Don'\''t print status codes, response size, running config, etc...]' \
+'(--silent)*-v[Increase verbosity level (use -vv or more for greater effect. \[CAUTION\] 4 -v'\''s is probably too much)]' \
+'(--silent)*--verbosity[Increase verbosity level (use -vv or more for greater effect. \[CAUTION\] 4 -v'\''s is probably too much)]' \
+'(-q --quiet)--silent[Only print URLs + turn off logging (good for piping a list of urls to other commands)]' \
+'-q[Hide progress bars and banner (good for tmux windows w/ notifications)]' \
+'--quiet[Hide progress bars and banner (good for tmux windows w/ notifications)]' \
 '--json[Emit JSON logs to --output and --debug-log instead of normal text]' \
 '-D[Don'\''t auto-filter wildcard responses]' \
 '--dont-filter[Don'\''t auto-filter wildcard responses]' \

shell_completions/_feroxbuster.ps1

@@ -30,8 +30,8 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
             [CompletionResult]::new('--depth', 'depth', [CompletionResultType]::ParameterName, 'Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)')
             [CompletionResult]::new('-T', 'T', [CompletionResultType]::ParameterName, 'Number of seconds before a request times out (default: 7)')
             [CompletionResult]::new('--timeout', 'timeout', [CompletionResultType]::ParameterName, 'Number of seconds before a request times out (default: 7)')
-            [CompletionResult]::new('-p', 'p', [CompletionResultType]::ParameterName, 'Proxy to use for requests (ex: http(s)://host:port, socks5://host:port)')
-            [CompletionResult]::new('--proxy', 'proxy', [CompletionResultType]::ParameterName, 'Proxy to use for requests (ex: http(s)://host:port, socks5://host:port)')
+            [CompletionResult]::new('-p', 'p', [CompletionResultType]::ParameterName, 'Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)')
+            [CompletionResult]::new('--proxy', 'proxy', [CompletionResultType]::ParameterName, 'Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)')
             [CompletionResult]::new('-P', 'P', [CompletionResultType]::ParameterName, 'Send only unfiltered requests through a Replay Proxy, instead of all requests')
             [CompletionResult]::new('--replay-proxy', 'replay-proxy', [CompletionResultType]::ParameterName, 'Send only unfiltered requests through a Replay Proxy, instead of all requests')
             [CompletionResult]::new('-R', 'R', [CompletionResultType]::ParameterName, 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)')
@@ -60,13 +60,16 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
             [CompletionResult]::new('--filter-lines', 'filter-lines', [CompletionResultType]::ParameterName, 'Filter out messages of a particular line count (ex: -N 20 -N 31,30)')
             [CompletionResult]::new('-C', 'C', [CompletionResultType]::ParameterName, 'Filter out status codes (deny list) (ex: -C 200 -C 401)')
             [CompletionResult]::new('--filter-status', 'filter-status', [CompletionResultType]::ParameterName, 'Filter out status codes (deny list) (ex: -C 200 -C 401)')
+            [CompletionResult]::new('--filter-similar-to', 'filter-similar-to', [CompletionResultType]::ParameterName, 'Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)')
             [CompletionResult]::new('-L', 'L', [CompletionResultType]::ParameterName, 'Limit total number of concurrent scans (default: 0, i.e. no limit)')
             [CompletionResult]::new('--scan-limit', 'scan-limit', [CompletionResultType]::ParameterName, 'Limit total number of concurrent scans (default: 0, i.e. no limit)')
+            [CompletionResult]::new('--rate-limit', 'rate-limit', [CompletionResultType]::ParameterName, 'Limit number of requests per second (per directory) (default: 0, i.e. no limit)')
             [CompletionResult]::new('--time-limit', 'time-limit', [CompletionResultType]::ParameterName, 'Limit total run time of all scans (ex: --time-limit 10m)')
             [CompletionResult]::new('-v', 'v', [CompletionResultType]::ParameterName, 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v''s is probably too much)')
             [CompletionResult]::new('--verbosity', 'verbosity', [CompletionResultType]::ParameterName, 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v''s is probably too much)')
-            [CompletionResult]::new('-q', 'q', [CompletionResultType]::ParameterName, 'Only print URLs; Don''t print status codes, response size, running config, etc...')
-            [CompletionResult]::new('--quiet', 'quiet', [CompletionResultType]::ParameterName, 'Only print URLs; Don''t print status codes, response size, running config, etc...')
+            [CompletionResult]::new('--silent', 'silent', [CompletionResultType]::ParameterName, 'Only print URLs + turn off logging (good for piping a list of urls to other commands)')
+            [CompletionResult]::new('-q', 'q', [CompletionResultType]::ParameterName, 'Hide progress bars and banner (good for tmux windows w/ notifications)')
+            [CompletionResult]::new('--quiet', 'quiet', [CompletionResultType]::ParameterName, 'Hide progress bars and banner (good for tmux windows w/ notifications)')
             [CompletionResult]::new('--json', 'json', [CompletionResultType]::ParameterName, 'Emit JSON logs to --output and --debug-log instead of normal text')
             [CompletionResult]::new('-D', 'D', [CompletionResultType]::ParameterName, 'Don''t auto-filter wildcard responses')
             [CompletionResult]::new('--dont-filter', 'dont-filter', [CompletionResultType]::ParameterName, 'Don''t auto-filter wildcard responses')

shell_completions/feroxbuster.bash

@@ -20,7 +20,7 @@ _feroxbuster() {
 
     case "${cmd}" in
         feroxbuster)
-            opts=" -v -q -D -r -k -n -f -e -h -V -w -u -t -d -T -p -P -R -s -o -a -x -H -Q -S -X -W -N -C -L  --verbosity --quiet --json --dont-filter --redirects --insecure --no-recursion --add-slash --stdin --extract-links --help --version --wordlist --url --threads --depth --timeout --proxy --replay-proxy --replay-codes --status-codes --output --resume-from --debug-log --user-agent --extensions --headers --query --filter-size --filter-regex --filter-words --filter-lines --filter-status --scan-limit --time-limit  "
+            opts=" -v -q -D -r -k -n -f -e -h -V -w -u -t -d -T -p -P -R -s -o -a -x -H -Q -S -X -W -N -C -L  --verbosity --silent --quiet --json --dont-filter --redirects --insecure --no-recursion --add-slash --stdin --extract-links --help --version --wordlist --url --threads --depth --timeout --proxy --replay-proxy --replay-codes --status-codes --output --resume-from --debug-log --user-agent --extensions --headers --query --filter-size --filter-regex --filter-words --filter-lines --filter-status --filter-similar-to --scan-limit --rate-limit --time-limit  "
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 1 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -187,6 +187,10 @@ _feroxbuster() {
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
                     ;;
+                --filter-similar-to)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
                 --scan-limit)
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
@@ -195,6 +199,10 @@ _feroxbuster() {
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
                     ;;
+                --rate-limit)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
                 --time-limit)
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0

shell_completions/feroxbuster.fish

@@ -3,7 +3,7 @@ complete -c feroxbuster -n "__fish_use_subcommand" -s u -l url -d 'The target UR
 complete -c feroxbuster -n "__fish_use_subcommand" -s t -l threads -d 'Number of concurrent threads (default: 50)'
 complete -c feroxbuster -n "__fish_use_subcommand" -s d -l depth -d 'Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)'
 complete -c feroxbuster -n "__fish_use_subcommand" -s T -l timeout -d 'Number of seconds before a request times out (default: 7)'
-complete -c feroxbuster -n "__fish_use_subcommand" -s p -l proxy -d 'Proxy to use for requests (ex: http(s)://host:port, socks5://host:port)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s p -l proxy -d 'Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)'
 complete -c feroxbuster -n "__fish_use_subcommand" -s P -l replay-proxy -d 'Send only unfiltered requests through a Replay Proxy, instead of all requests'
 complete -c feroxbuster -n "__fish_use_subcommand" -s R -l replay-codes -d 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)'
 complete -c feroxbuster -n "__fish_use_subcommand" -s s -l status-codes -d 'Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)'
@@ -19,10 +19,13 @@ complete -c feroxbuster -n "__fish_use_subcommand" -s X -l filter-regex -d 'Filt
 complete -c feroxbuster -n "__fish_use_subcommand" -s W -l filter-words -d 'Filter out messages of a particular word count (ex: -W 312 -W 91,82)'
 complete -c feroxbuster -n "__fish_use_subcommand" -s N -l filter-lines -d 'Filter out messages of a particular line count (ex: -N 20 -N 31,30)'
 complete -c feroxbuster -n "__fish_use_subcommand" -s C -l filter-status -d 'Filter out status codes (deny list) (ex: -C 200 -C 401)'
+complete -c feroxbuster -n "__fish_use_subcommand" -l filter-similar-to -d 'Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)'
 complete -c feroxbuster -n "__fish_use_subcommand" -s L -l scan-limit -d 'Limit total number of concurrent scans (default: 0, i.e. no limit)'
+complete -c feroxbuster -n "__fish_use_subcommand" -l rate-limit -d 'Limit number of requests per second (per directory) (default: 0, i.e. no limit)'
 complete -c feroxbuster -n "__fish_use_subcommand" -l time-limit -d 'Limit total run time of all scans (ex: --time-limit 10m)'
 complete -c feroxbuster -n "__fish_use_subcommand" -s v -l verbosity -d 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v\'s is probably too much)'
-complete -c feroxbuster -n "__fish_use_subcommand" -s q -l quiet -d 'Only print URLs; Don\'t print status codes, response size, running config, etc...'
+complete -c feroxbuster -n "__fish_use_subcommand" -l silent -d 'Only print URLs + turn off logging (good for piping a list of urls to other commands)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s q -l quiet -d 'Hide progress bars and banner (good for tmux windows w/ notifications)'
 complete -c feroxbuster -n "__fish_use_subcommand" -l json -d 'Emit JSON logs to --output and --debug-log instead of normal text'
 complete -c feroxbuster -n "__fish_use_subcommand" -s D -l dont-filter -d 'Don\'t auto-filter wildcard responses'
 complete -c feroxbuster -n "__fish_use_subcommand" -s r -l redirects -d 'Follow redirects'

src/banner.rs

@@ -1,708 +0,0 @@
-use crate::config::{Configuration, CONFIGURATION};
-use crate::utils::{make_request, status_colorizer};
-use console::style;
-use reqwest::{Client, Url};
-use serde_json::Value;
-use std::io::Write;
-
-/// macro helper to abstract away repetitive string formatting
-macro_rules! format_banner_entry_helper {
-    // \u{0020} -> unicode space
-    // \u{2502} -> vertical box drawing character, i.e. │
-    ($rune:expr, $name:expr, $value:expr, $indent:expr, $col_width:expr) => {
-        format!(
-            "\u{0020}{:\u{0020}<indent$}{:\u{0020}<col_w$}\u{2502}\u{0020}{}",
-            $rune,
-            $name,
-            $value,
-            indent = $indent,
-            col_w = $col_width
-        )
-    };
-    ($rune:expr, $name:expr, $value:expr, $value2:expr, $indent:expr, $col_width:expr) => {
-        format!(
-            "\u{0020}{:\u{0020}<indent$}{:\u{0020}<col_w$}\u{2502}\u{0020}{}:\u{0020}{}",
-            $rune,
-            $name,
-            $value,
-            $value2,
-            indent = $indent,
-            col_w = $col_width
-        )
-    };
-}
-
-/// macro that wraps another macro helper to abstract away repetitive string formatting
-macro_rules! format_banner_entry {
-    // 4 -> unicode emoji padding width
-    // 22 -> column width (when unicode rune is 4 bytes wide, 23 when it's 3)
-    // hardcoded since macros don't allow let statements
-    ($rune:expr, $name:expr, $value:expr) => {
-        format_banner_entry_helper!($rune, $name, $value, 3, 22)
-    };
-    ($rune:expr, $name:expr, $value1:expr, $value2:expr) => {
-        format_banner_entry_helper!($rune, $name, $value1, $value2, 3, 22)
-    };
-}
-
-/// Url used to query github's api; specifically used to look for the latest tagged release name
-const UPDATE_URL: &str = "https://api.github.com/repos/epi052/feroxbuster/releases/latest";
-
-/// Simple enum to hold three different update states
-#[derive(Debug)]
-enum UpdateStatus {
-    /// this version and latest release are the same
-    UpToDate,
-
-    /// this version and latest release are not the same
-    OutOfDate,
-
-    /// some error occurred during version check
-    Unknown,
-}
-
-/// Makes a request to the given url, expecting to receive a JSON response that contains a field
-/// named `tag_name` that holds a value representing the latest tagged release of this tool.
-///
-/// ex: v1.1.0
-///
-/// Returns `UpdateStatus`
-async fn needs_update(client: &Client, url: &str, bin_version: &str) -> UpdateStatus {
-    log::trace!("enter: needs_update({:?}, {})", client, url);
-
-    let unknown = UpdateStatus::Unknown;
-
-    let api_url = match Url::parse(url) {
-        Ok(url) => url,
-        Err(e) => {
-            log::error!("{}", e);
-            log::trace!("exit: needs_update -> {:?}", unknown);
-            return unknown;
-        }
-    };
-
-    if let Ok(response) = make_request(&client, &api_url).await {
-        let body = response.text().await.unwrap_or_default();
-
-        let json_response: Value = serde_json::from_str(&body).unwrap_or_default();
-
-        if json_response.is_null() {
-            // unwrap_or_default above should result in a null value for the json_response variable
-            log::error!("Could not parse JSON from response body");
-            log::trace!("exit: needs_update -> {:?}", unknown);
-            return unknown;
-        }
-
-        let latest_version = match json_response["tag_name"].as_str() {
-            Some(tag) => tag.trim_start_matches('v'),
-            None => {
-                log::error!("Could not get version field from JSON response");
-                log::debug!("{}", json_response);
-                log::trace!("exit: needs_update -> {:?}", unknown);
-                return unknown;
-            }
-        };
-
-        // if we've gotten this far, we have a string in the form of X.X.X where X is a number
-        // all that's left is to compare the current version with the version found above
-
-        return if latest_version == bin_version {
-            // there's really only two possible outcomes if we accept that the tag conforms to
-            // the X.X.X pattern:
-            //   1. the version strings match, meaning we're up to date
-            //   2. the version strings do not match, meaning we're out of date
-            //
-            // except for developers working on this code, nobody should ever be in a situation
-            // where they have a version greater than the latest tagged release
-            log::trace!("exit: needs_update -> UpdateStatus::UpToDate");
-            UpdateStatus::UpToDate
-        } else {
-            log::trace!("exit: needs_update -> UpdateStatus::OutOfDate");
-            UpdateStatus::OutOfDate
-        };
-    }
-
-    log::trace!("exit: needs_update -> {:?}", unknown);
-    unknown
-}
-
-/// Prints the banner to stdout.
-///
-/// Only prints those settings which are either always present, or passed in by the user.
-pub async fn initialize<W>(targets: &[String], config: &Configuration, version: &str, mut writer: W)
-where
-    W: Write,
-{
-    let artwork = format!(
-        r#"
- ___  ___  __   __     __      __         __   ___
-|__  |__  |__) |__) | /  `    /  \ \_/ | |  \ |__
-|    |___ |  \ |  \ | \__,    \__/ / \ | |__/ |___
-by Ben "epi" Risher {}                  ver: {}"#,
-        '\u{1F913}', version
-    );
-
-    let status = needs_update(&CONFIGURATION.client, UPDATE_URL, version).await;
-
-    let top = "───────────────────────────┬──────────────────────";
-    let addl_section = "──────────────────────────────────────────────────";
-    let bottom = "───────────────────────────┴──────────────────────";
-
-    writeln!(&mut writer, "{}", artwork).unwrap_or_default();
-    writeln!(&mut writer, "{}", top).unwrap_or_default();
-
-    // begin with always printed items
-    for target in targets {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1F3af}", "Target Url", target)
-        )
-        .unwrap_or_default(); // 🎯
-    }
-
-    let mut codes = vec![];
-
-    for code in &config.status_codes {
-        codes.push(status_colorizer(&code.to_string()))
-    }
-
-    writeln!(
-        &mut writer,
-        "{}",
-        format_banner_entry!("\u{1F680}", "Threads", config.threads)
-    )
-    .unwrap_or_default(); // 🚀
-
-    writeln!(
-        &mut writer,
-        "{}",
-        format_banner_entry!("\u{1f4d6}", "Wordlist", config.wordlist)
-    )
-    .unwrap_or_default(); // 📖
-
-    writeln!(
-        &mut writer,
-        "{}",
-        format_banner_entry!(
-            "\u{1F197}",
-            "Status Codes",
-            format!("[{}]", codes.join(", "))
-        )
-    )
-    .unwrap_or_default(); // 🆗
-
-    if !config.filter_status.is_empty() {
-        // exception here for optional print due to me wanting the allows and denys to be printed
-        // one after the other
-        let mut code_filters = vec![];
-
-        for code in &config.filter_status {
-            code_filters.push(status_colorizer(&code.to_string()))
-        }
-
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!(
-                "\u{1f5d1}",
-                "Status Code Filters",
-                format!("[{}]", code_filters.join(", "))
-            )
-        )
-        .unwrap_or_default(); // 🗑
-    }
-
-    writeln!(
-        &mut writer,
-        "{}",
-        format_banner_entry!("\u{1f4a5}", "Timeout (secs)", config.timeout)
-    )
-    .unwrap_or_default(); // 💥
-
-    writeln!(
-        &mut writer,
-        "{}",
-        format_banner_entry!("\u{1F9a1}", "User-Agent", config.user_agent)
-    )
-    .unwrap_or_default(); // 🦡
-
-    // followed by the maybe printed or variably displayed values
-    if !config.config.is_empty() {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1f489}", "Config File", config.config)
-        )
-        .unwrap_or_default(); // 💉
-    }
-
-    if !config.proxy.is_empty() {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1f48e}", "Proxy", config.proxy)
-        )
-        .unwrap_or_default(); // 💎
-    }
-
-    if !config.replay_proxy.is_empty() {
-        // i include replay codes logic here because in config.rs, replay codes are set to the
-        // value in status codes, meaning it's never empty
-
-        let mut replay_codes = vec![];
-
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1f3a5}", "Replay Proxy", config.replay_proxy)
-        )
-        .unwrap_or_default(); // 🎥
-
-        for code in &config.replay_codes {
-            replay_codes.push(status_colorizer(&code.to_string()))
-        }
-
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!(
-                "\u{1f4fc}",
-                "Replay Proxy Codes",
-                format!("[{}]", replay_codes.join(", "))
-            )
-        )
-        .unwrap_or_default(); // 📼
-    }
-
-    if !config.headers.is_empty() {
-        for (name, value) in &config.headers {
-            writeln!(
-                &mut writer,
-                "{}",
-                format_banner_entry!("\u{1f92f}", "Header", name, value)
-            )
-            .unwrap_or_default(); // 🤯
-        }
-    }
-
-    if !config.filter_size.is_empty() {
-        for filter in &config.filter_size {
-            writeln!(
-                &mut writer,
-                "{}",
-                format_banner_entry!("\u{1f4a2}", "Size Filter", filter)
-            )
-            .unwrap_or_default(); // 💢
-        }
-    }
-
-    for filter in &config.filter_word_count {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1f4a2}", "Word Count Filter", filter)
-        )
-        .unwrap_or_default(); // 💢
-    }
-
-    for filter in &config.filter_line_count {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1f4a2}", "Line Count Filter", filter)
-        )
-        .unwrap_or_default(); // 💢
-    }
-
-    for filter in &config.filter_regex {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1f4a2}", "Regex Filter", filter)
-        )
-        .unwrap_or_default(); // 💢
-    }
-
-    if config.extract_links {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1F50E}", "Extract Links", config.extract_links)
-        )
-        .unwrap_or_default(); // 🔎
-    }
-
-    if config.json {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1F9d4}", "JSON Output", config.json)
-        )
-        .unwrap_or_default(); // 🧔
-    }
-
-    if !config.queries.is_empty() {
-        for query in &config.queries {
-            writeln!(
-                &mut writer,
-                "{}",
-                format_banner_entry!(
-                    "\u{1f914}",
-                    "Query Parameter",
-                    format!("{}={}", query.0, query.1)
-                )
-            )
-            .unwrap_or_default(); // 🤔
-        }
-    }
-
-    if !config.output.is_empty() {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1f4be}", "Output File", config.output)
-        )
-        .unwrap_or_default(); // 💾
-    }
-
-    if !config.debug_log.is_empty() {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1fab2}", "Debugging Log", config.debug_log)
-        )
-        .unwrap_or_default(); // 🪲
-    }
-
-    if !config.extensions.is_empty() {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!(
-                "\u{1f4b2}",
-                "Extensions",
-                format!("[{}]", config.extensions.join(", "))
-            )
-        )
-        .unwrap_or_default(); // 💲
-    }
-
-    if config.insecure {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1f513}", "Insecure", config.insecure)
-        )
-        .unwrap_or_default(); // 🔓
-    }
-
-    if config.redirects {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1f4cd}", "Follow Redirects", config.redirects)
-        )
-        .unwrap_or_default(); // 📍
-    }
-
-    if config.dont_filter {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1f92a}", "Filter Wildcards", !config.dont_filter)
-        )
-        .unwrap_or_default(); // 🤪
-    }
-
-    match config.verbosity {
-        //speaker medium volume (increasing with verbosity to loudspeaker)
-        1 => {
-            writeln!(
-                &mut writer,
-                "{}",
-                format_banner_entry!("\u{1f508}", "Verbosity", config.verbosity)
-            )
-            .unwrap_or_default(); // 🔈
-        }
-        2 => {
-            writeln!(
-                &mut writer,
-                "{}",
-                format_banner_entry!("\u{1f509}", "Verbosity", config.verbosity)
-            )
-            .unwrap_or_default(); // 🔉
-        }
-        3 => {
-            writeln!(
-                &mut writer,
-                "{}",
-                format_banner_entry!("\u{1f50a}", "Verbosity", config.verbosity)
-            )
-            .unwrap_or_default(); // 🔊
-        }
-        4 => {
-            writeln!(
-                &mut writer,
-                "{}",
-                format_banner_entry!("\u{1f4e2}", "Verbosity", config.verbosity)
-            )
-            .unwrap_or_default(); // 📢
-        }
-        _ => {}
-    }
-
-    if config.add_slash {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1fa93}", "Add Slash", config.add_slash)
-        )
-        .unwrap_or_default(); // 🪓
-    }
-
-    if !config.no_recursion {
-        if config.depth == 0 {
-            writeln!(
-                &mut writer,
-                "{}",
-                format_banner_entry!("\u{1f503}", "Recursion Depth", "INFINITE")
-            )
-            .unwrap_or_default(); // 🔃
-        } else {
-            writeln!(
-                &mut writer,
-                "{}",
-                format_banner_entry!("\u{1f503}", "Recursion Depth", config.depth)
-            )
-            .unwrap_or_default(); // 🔃
-        }
-    } else {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1f6ab}", "Do Not Recurse", config.no_recursion)
-        )
-        .unwrap_or_default(); // 🚫
-    }
-
-    if CONFIGURATION.scan_limit > 0 {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1f9a5}", "Concurrent Scan Limit", config.scan_limit)
-        )
-        .unwrap_or_default(); // 🦥
-    }
-
-    if !CONFIGURATION.time_limit.is_empty() {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1f556}", "Time Limit", config.time_limit)
-        )
-        .unwrap_or_default(); // 🕖
-    }
-
-    if matches!(status, UpdateStatus::OutOfDate) {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!(
-                "\u{1f389}",
-                "New Version Available",
-                "https://github.com/epi052/feroxbuster/releases/latest"
-            )
-        )
-        .unwrap_or_default(); // 🎉
-    }
-
-    writeln!(&mut writer, "{}", bottom).unwrap_or_default();
-    // ⏯
-    writeln!(
-        &mut writer,
-        " \u{23ef}   Press [{}] to {}|{} your scan",
-        style("ENTER").yellow(),
-        style("pause").red(),
-        style("resume").green()
-    )
-    .unwrap_or_default();
-    writeln!(&mut writer, "{}", addl_section).unwrap_or_default();
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::VERSION;
-    use httpmock::Method::GET;
-    use httpmock::{Mock, MockServer};
-    use std::fs::read_to_string;
-    use std::io::stderr;
-    use std::time::Duration;
-    use tempfile::NamedTempFile;
-
-    #[tokio::test(core_threads = 1)]
-    /// test to hit no execution of targets for loop in banner
-    async fn banner_intialize_without_targets() {
-        let config = Configuration::default();
-        initialize(&[], &config, VERSION, stderr()).await;
-    }
-
-    #[tokio::test(core_threads = 1)]
-    /// test to hit no execution of statuscode for loop in banner
-    async fn banner_intialize_without_status_codes() {
-        let mut config = Configuration::default();
-        config.status_codes = vec![];
-        initialize(
-            &[String::from("http://localhost")],
-            &config,
-            VERSION,
-            stderr(),
-        )
-        .await;
-    }
-
-    #[tokio::test(core_threads = 1)]
-    /// test to hit an empty config file
-    async fn banner_intialize_without_config_file() {
-        let mut config = Configuration::default();
-        config.config = String::new();
-        initialize(
-            &[String::from("http://localhost")],
-            &config,
-            VERSION,
-            stderr(),
-        )
-        .await;
-    }
-
-    #[tokio::test(core_threads = 1)]
-    /// test to hit an empty config file
-    async fn banner_intialize_without_queries() {
-        let mut config = Configuration::default();
-        config.queries = vec![(String::new(), String::new())];
-        initialize(
-            &[String::from("http://localhost")],
-            &config,
-            VERSION,
-            stderr(),
-        )
-        .await;
-    }
-
-    #[tokio::test(core_threads = 1)]
-    /// test to show that a new version is available for download
-    async fn banner_intialize_with_mismatched_version() {
-        let config = Configuration::default();
-        let file = NamedTempFile::new().unwrap();
-        initialize(
-            &[String::from("http://localhost")],
-            &config,
-            "mismatched-version",
-            &file,
-        )
-        .await;
-        let contents = read_to_string(file.path()).unwrap();
-        println!("contents: {}", contents);
-        assert!(contents.contains("New Version Available"));
-        assert!(contents.contains("https://github.com/epi052/feroxbuster/releases/latest"));
-    }
-
-    #[tokio::test(core_threads = 1)]
-    /// test that
-    async fn banner_needs_update_returns_unknown_with_bad_url() {
-        let result = needs_update(&CONFIGURATION.client, &"", VERSION).await;
-        assert!(matches!(result, UpdateStatus::Unknown));
-    }
-
-    #[tokio::test(core_threads = 1)]
-    /// test return value of good url to needs_update
-    async fn banner_needs_update_returns_up_to_date() {
-        let srv = MockServer::start();
-
-        let mock = Mock::new()
-            .expect_method(GET)
-            .expect_path("/latest")
-            .return_status(200)
-            .return_body("{\"tag_name\":\"v1.1.0\"}")
-            .create_on(&srv);
-
-        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.1.0").await;
-
-        assert_eq!(mock.times_called(), 1);
-        assert!(matches!(result, UpdateStatus::UpToDate));
-    }
-
-    #[tokio::test(core_threads = 1)]
-    /// test return value of good url to needs_update that returns a newer version than current
-    async fn banner_needs_update_returns_out_of_date() {
-        let srv = MockServer::start();
-
-        let mock = Mock::new()
-            .expect_method(GET)
-            .expect_path("/latest")
-            .return_status(200)
-            .return_body("{\"tag_name\":\"v1.1.0\"}")
-            .create_on(&srv);
-
-        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1").await;
-
-        assert_eq!(mock.times_called(), 1);
-        assert!(matches!(result, UpdateStatus::OutOfDate));
-    }
-
-    #[tokio::test(core_threads = 1)]
-    /// test return value of good url that times out
-    async fn banner_needs_update_returns_unknown_on_timeout() {
-        let srv = MockServer::start();
-
-        let mock = Mock::new()
-            .expect_method(GET)
-            .expect_path("/latest")
-            .return_status(200)
-            .return_body("{\"tag_name\":\"v1.1.0\"}")
-            .return_with_delay(Duration::from_secs(8))
-            .create_on(&srv);
-
-        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1").await;
-
-        assert_eq!(mock.times_called(), 1);
-        assert!(matches!(result, UpdateStatus::Unknown));
-    }
-
-    #[tokio::test(core_threads = 1)]
-    /// test return value of good url with bad json response
-    async fn banner_needs_update_returns_unknown_on_bad_json_response() {
-        let srv = MockServer::start();
-
-        let mock = Mock::new()
-            .expect_method(GET)
-            .expect_path("/latest")
-            .return_status(200)
-            .return_body("not json")
-            .create_on(&srv);
-
-        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1").await;
-
-        assert_eq!(mock.times_called(), 1);
-        assert!(matches!(result, UpdateStatus::Unknown));
-    }
-
-    #[tokio::test(core_threads = 1)]
-    /// test return value of good url with json response that lacks the tag_name field
-    async fn banner_needs_update_returns_unknown_on_json_without_correct_tag() {
-        let srv = MockServer::start();
-
-        let mock = Mock::new()
-            .expect_method(GET)
-            .expect_path("/latest")
-            .return_status(200)
-            .return_body("{\"no tag_name\": \"doesn't exist\"}")
-            .create_on(&srv);
-
-        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1").await;
-
-        assert_eq!(mock.times_called(), 1);
-        assert!(matches!(result, UpdateStatus::Unknown));
-    }
-}

src/banner/container.rs

@@ -0,0 +1,532 @@
+use super::entry::BannerEntry;
+use crate::event_handlers::Handles;
+use crate::{
+    config::Configuration,
+    utils::{make_request, status_colorizer},
+    VERSION,
+};
+use anyhow::{bail, Result};
+use console::{style, Emoji};
+use reqwest::Url;
+use serde_json::Value;
+use std::{io::Write, sync::Arc};
+
+/// Url used to query github's api; specifically used to look for the latest tagged release name
+pub const UPDATE_URL: &str = "https://api.github.com/repos/epi052/feroxbuster/releases/latest";
+
+/// Simple enum to hold three different update states
+#[derive(Debug)]
+pub(super) enum UpdateStatus {
+    /// this version and latest release are the same
+    UpToDate,
+
+    /// this version and latest release are not the same
+    OutOfDate,
+
+    /// some error occurred during version check
+    Unknown,
+}
+
+/// Banner object, contains multiple BannerEntry's and knows how to display itself
+pub struct Banner {
+    /// all live targets
+    targets: Vec<BannerEntry>,
+
+    /// represents Configuration.status_codes
+    status_codes: BannerEntry,
+
+    /// represents Configuration.filter_status
+    filter_status: BannerEntry,
+
+    /// represents Configuration.threads
+    threads: BannerEntry,
+
+    /// represents Configuration.wordlist
+    wordlist: BannerEntry,
+
+    /// represents Configuration.timeout
+    timeout: BannerEntry,
+
+    /// represents Configuration.user_agent
+    user_agent: BannerEntry,
+
+    /// represents Configuration.config
+    config: BannerEntry,
+
+    /// represents Configuration.proxy
+    proxy: BannerEntry,
+
+    /// represents Configuration.replay_proxy
+    replay_proxy: BannerEntry,
+
+    /// represents Configuration.replay_codes
+    replay_codes: BannerEntry,
+
+    /// represents Configuration.headers
+    headers: Vec<BannerEntry>,
+
+    /// represents Configuration.filter_size
+    filter_size: Vec<BannerEntry>,
+
+    /// represents Configuration.filter_similar
+    filter_similar: Vec<BannerEntry>,
+
+    /// represents Configuration.filter_word_count
+    filter_word_count: Vec<BannerEntry>,
+
+    /// represents Configuration.filter_line_count
+    filter_line_count: Vec<BannerEntry>,
+
+    /// represents Configuration.filter_regex
+    filter_regex: Vec<BannerEntry>,
+
+    /// represents Configuration.extract_links
+    extract_links: BannerEntry,
+
+    /// represents Configuration.json
+    json: BannerEntry,
+
+    /// represents Configuration.output
+    output: BannerEntry,
+
+    /// represents Configuration.debug_log
+    debug_log: BannerEntry,
+
+    /// represents Configuration.extensions
+    extensions: BannerEntry,
+
+    /// represents Configuration.insecure
+    insecure: BannerEntry,
+
+    /// represents Configuration.redirects
+    redirects: BannerEntry,
+
+    /// represents Configuration.dont_filter
+    dont_filter: BannerEntry,
+
+    /// represents Configuration.queries
+    queries: Vec<BannerEntry>,
+
+    /// represents Configuration.verbosity
+    verbosity: BannerEntry,
+
+    /// represents Configuration.add_slash
+    add_slash: BannerEntry,
+
+    /// represents Configuration.no_recursion
+    no_recursion: BannerEntry,
+
+    /// represents Configuration.scan_limit
+    scan_limit: BannerEntry,
+
+    /// represents Configuration.time_limit
+    time_limit: BannerEntry,
+
+    /// represents Configuration.rate_limit
+    rate_limit: BannerEntry,
+
+    /// current version of feroxbuster
+    pub(super) version: String,
+
+    /// whether or not there is a known new version
+    pub(super) update_status: UpdateStatus,
+}
+
+/// implementation of Banner
+impl Banner {
+    /// Create a new Banner from a Configuration and live targets
+    pub fn new(tgts: &[String], config: &Configuration) -> Self {
+        let mut targets = Vec::new();
+        let mut code_filters = Vec::new();
+        let mut replay_codes = Vec::new();
+        let mut headers = Vec::new();
+        let mut filter_size = Vec::new();
+        let mut filter_similar = Vec::new();
+        let mut filter_word_count = Vec::new();
+        let mut filter_line_count = Vec::new();
+        let mut filter_regex = Vec::new();
+        let mut queries = Vec::new();
+
+        for target in tgts {
+            targets.push(BannerEntry::new("🎯", "Target Url", target));
+        }
+
+        let mut codes = vec![];
+        for code in &config.status_codes {
+            codes.push(status_colorizer(&code.to_string()))
+        }
+        let status_codes =
+            BannerEntry::new("👌", "Status Codes", &format!("[{}]", codes.join(", ")));
+
+        for code in &config.filter_status {
+            code_filters.push(status_colorizer(&code.to_string()))
+        }
+        let filter_status = BannerEntry::new(
+            "🗑",
+            "Status Code Filters",
+            &format!("[{}]", code_filters.join(", ")),
+        );
+
+        for code in &config.replay_codes {
+            replay_codes.push(status_colorizer(&code.to_string()))
+        }
+        let replay_codes = BannerEntry::new(
+            "📼",
+            "Replay Proxy Codes",
+            &format!("[{}]", replay_codes.join(", ")),
+        );
+
+        for (name, value) in &config.headers {
+            headers.push(BannerEntry::new(
+                "🤯",
+                "Header",
+                &format!("{}: {}", name, value),
+            ));
+        }
+
+        for filter in &config.filter_size {
+            filter_size.push(BannerEntry::new("💢", "Size Filter", &filter.to_string()));
+        }
+
+        for filter in &config.filter_similar {
+            filter_similar.push(BannerEntry::new("💢", "Similarity Filter", filter));
+        }
+
+        for filter in &config.filter_word_count {
+            filter_word_count.push(BannerEntry::new(
+                "💢",
+                "Word Count Filter",
+                &filter.to_string(),
+            ));
+        }
+
+        for filter in &config.filter_line_count {
+            filter_line_count.push(BannerEntry::new(
+                "💢",
+                "Line Count Filter",
+                &filter.to_string(),
+            ));
+        }
+
+        for filter in &config.filter_regex {
+            filter_regex.push(BannerEntry::new("💢", "Regex Filter", filter));
+        }
+
+        for query in &config.queries {
+            queries.push(BannerEntry::new(
+                "🤔",
+                "Query Parameter",
+                &format!("{}={}", query.0, query.1),
+            ));
+        }
+
+        let volume = ["🔈", "🔉", "🔊", "📢"];
+        let verbosity = if let 1..=4 = config.verbosity {
+            //speaker medium volume (increasing with verbosity to loudspeaker)
+            BannerEntry::new(
+                volume[config.verbosity as usize - 1],
+                "Verbosity",
+                &config.verbosity.to_string(),
+            )
+        } else {
+            BannerEntry::default()
+        };
+
+        let no_recursion = if !config.no_recursion {
+            let depth = if config.depth == 0 {
+                "INFINITE".to_string()
+            } else {
+                config.depth.to_string()
+            };
+
+            BannerEntry::new("🔃", "Recursion Depth", &depth)
+        } else {
+            BannerEntry::new("🚫", "Do Not Recurse", &config.no_recursion.to_string())
+        };
+
+        let scan_limit = BannerEntry::new(
+            "🦥",
+            "Concurrent Scan Limit",
+            &config.scan_limit.to_string(),
+        );
+
+        let replay_proxy = BannerEntry::new("🎥", "Replay Proxy", &config.replay_proxy);
+        let cfg = BannerEntry::new("💉", "Config File", &config.config);
+        let proxy = BannerEntry::new("💎", "Proxy", &config.proxy);
+        let threads = BannerEntry::new("🚀", "Threads", &config.threads.to_string());
+        let wordlist = BannerEntry::new("📖", "Wordlist", &config.wordlist);
+        let timeout = BannerEntry::new("💥", "Timeout (secs)", &config.timeout.to_string());
+        let user_agent = BannerEntry::new("🦡", "User-Agent", &config.user_agent);
+        let extract_links =
+            BannerEntry::new("🔎", "Extract Links", &config.extract_links.to_string());
+        let json = BannerEntry::new("🧔", "JSON Output", &config.json.to_string());
+        let output = BannerEntry::new("💾", "Output File", &config.output);
+        let debug_log = BannerEntry::new("🪲", "Debugging Log", &config.debug_log);
+        let extensions = BannerEntry::new(
+            "💲",
+            "Extensions",
+            &format!("[{}]", config.extensions.join(", ")),
+        );
+        let insecure = BannerEntry::new("🔓", "Insecure", &config.insecure.to_string());
+        let redirects = BannerEntry::new("📍", "Follow Redirects", &config.redirects.to_string());
+        let dont_filter =
+            BannerEntry::new("🤪", "Filter Wildcards", &(!config.dont_filter).to_string());
+        let add_slash = BannerEntry::new("🪓", "Add Slash", &config.add_slash.to_string());
+        let time_limit = BannerEntry::new("🕖", "Time Limit", &config.time_limit);
+        let rate_limit =
+            BannerEntry::new("🚧", "Requests per Second", &config.rate_limit.to_string());
+
+        Self {
+            targets,
+            status_codes,
+            threads,
+            wordlist,
+            filter_status,
+            timeout,
+            user_agent,
+            proxy,
+            replay_codes,
+            replay_proxy,
+            headers,
+            filter_size,
+            filter_similar,
+            filter_word_count,
+            filter_line_count,
+            filter_regex,
+            extract_links,
+            json,
+            queries,
+            output,
+            debug_log,
+            extensions,
+            insecure,
+            dont_filter,
+            redirects,
+            verbosity,
+            add_slash,
+            no_recursion,
+            rate_limit,
+            scan_limit,
+            time_limit,
+            config: cfg,
+            version: VERSION.to_string(),
+            update_status: UpdateStatus::Unknown,
+        }
+    }
+
+    /// get a fancy header for the banner
+    fn header(&self) -> String {
+        let artwork = format!(
+            r#"
+ ___  ___  __   __     __      __         __   ___
+|__  |__  |__) |__) | /  `    /  \ \_/ | |  \ |__
+|    |___ |  \ |  \ | \__,    \__/ / \ | |__/ |___
+by Ben "epi" Risher {}                 ver: {}"#,
+            Emoji("🤓", &format!("{:<2}", "\u{0020}")),
+            self.version
+        );
+
+        let top = "───────────────────────────┬──────────────────────";
+
+        format!("{}\n{}", artwork, top)
+    }
+
+    /// get a fancy footer for the banner
+    fn footer(&self) -> String {
+        let addl_section = "──────────────────────────────────────────────────";
+        let bottom = "───────────────────────────┴──────────────────────";
+
+        let instructions = format!(
+            " 🏁  Press [{}] to use the {}™",
+            style("ENTER").yellow(),
+            style("Scan Cancel Menu").bright().yellow(),
+        );
+
+        format!("{}\n{}\n{}", bottom, instructions, addl_section)
+    }
+
+    /// Makes a request to the given url, expecting to receive a JSON response that contains a field
+    /// named `tag_name` that holds a value representing the latest tagged release of this tool.
+    ///
+    /// ex: v1.1.0
+    pub async fn check_for_updates(&mut self, url: &str, handles: Arc<Handles>) -> Result<()> {
+        log::trace!("enter: needs_update({}, {:?})", url, handles);
+
+        let api_url = Url::parse(url)?;
+
+        let response = make_request(
+            &handles.config.client,
+            &api_url,
+            handles.config.output_level,
+            handles.stats.tx.clone(),
+        )
+        .await?;
+
+        let body = response.text().await?;
+
+        let json_response: Value = serde_json::from_str(&body)?;
+
+        let latest_version = match json_response["tag_name"].as_str() {
+            Some(tag) => tag.trim_start_matches('v'),
+            None => {
+                bail!("JSON has no tag_name: {}", json_response);
+            }
+        };
+
+        // if we've gotten this far, we have a string in the form of X.X.X where X is a number
+        // all that's left is to compare the current version with the version found above
+
+        if latest_version == self.version {
+            // there's really only two possible outcomes if we accept that the tag conforms to
+            // the X.X.X pattern:
+            //   1. the version strings match, meaning we're up to date
+            //   2. the version strings do not match, meaning we're out of date
+            //
+            // except for developers working on this code, nobody should ever be in a situation
+            // where they have a version greater than the latest tagged release
+            self.update_status = UpdateStatus::UpToDate;
+        } else {
+            self.update_status = UpdateStatus::OutOfDate;
+        }
+
+        log::trace!("exit: check_for_updates -> {:?}", self.update_status);
+        Ok(())
+    }
+
+    /// display the banner on Write writer
+    pub fn print_to<W>(&self, mut writer: W, config: Arc<Configuration>) -> Result<()>
+    where
+        W: Write,
+    {
+        writeln!(&mut writer, "{}", self.header())?;
+
+        // begin with always printed items
+        for target in &self.targets {
+            writeln!(&mut writer, "{}", target)?;
+        }
+
+        writeln!(&mut writer, "{}", self.threads)?;
+        writeln!(&mut writer, "{}", self.wordlist)?;
+        writeln!(&mut writer, "{}", self.status_codes)?;
+
+        if !config.filter_status.is_empty() {
+            // exception here for an optional print in the middle of always printed values is due
+            // to me wanting the allows and denys to be printed one after the other
+            writeln!(&mut writer, "{}", self.filter_status)?;
+        }
+
+        writeln!(&mut writer, "{}", self.timeout)?;
+        writeln!(&mut writer, "{}", self.user_agent)?;
+
+        // followed by the maybe printed or variably displayed values
+        if !config.config.is_empty() {
+            writeln!(&mut writer, "{}", self.config)?;
+        }
+
+        if !config.proxy.is_empty() {
+            writeln!(&mut writer, "{}", self.proxy)?;
+        }
+
+        if !config.replay_proxy.is_empty() {
+            // i include replay codes logic here because in config.rs, replay codes are set to the
+            // value in status codes, meaning it's never empty
+            writeln!(&mut writer, "{}", self.replay_proxy)?;
+            writeln!(&mut writer, "{}", self.replay_codes)?;
+        }
+
+        for header in &self.headers {
+            writeln!(&mut writer, "{}", header)?;
+        }
+
+        for filter in &self.filter_size {
+            writeln!(&mut writer, "{}", filter)?;
+        }
+
+        for filter in &self.filter_similar {
+            writeln!(&mut writer, "{}", filter)?;
+        }
+
+        for filter in &self.filter_word_count {
+            writeln!(&mut writer, "{}", filter)?;
+        }
+
+        for filter in &self.filter_line_count {
+            writeln!(&mut writer, "{}", filter)?;
+        }
+
+        for filter in &self.filter_regex {
+            writeln!(&mut writer, "{}", filter)?;
+        }
+
+        if config.extract_links {
+            writeln!(&mut writer, "{}", self.extract_links)?;
+        }
+
+        if config.json {
+            writeln!(&mut writer, "{}", self.json)?;
+        }
+
+        for query in &self.queries {
+            writeln!(&mut writer, "{}", query)?;
+        }
+
+        if !config.output.is_empty() {
+            writeln!(&mut writer, "{}", self.output)?;
+        }
+
+        if !config.debug_log.is_empty() {
+            writeln!(&mut writer, "{}", self.debug_log)?;
+        }
+
+        if !config.extensions.is_empty() {
+            writeln!(&mut writer, "{}", self.extensions)?;
+        }
+
+        if config.insecure {
+            writeln!(&mut writer, "{}", self.insecure)?;
+        }
+
+        if config.redirects {
+            writeln!(&mut writer, "{}", self.redirects)?;
+        }
+
+        if config.dont_filter {
+            writeln!(&mut writer, "{}", self.dont_filter)?;
+        }
+
+        if let 1..=4 = config.verbosity {
+            writeln!(&mut writer, "{}", self.verbosity)?;
+        }
+
+        if config.add_slash {
+            writeln!(&mut writer, "{}", self.add_slash)?;
+        }
+
+        writeln!(&mut writer, "{}", self.no_recursion)?;
+
+        if config.scan_limit > 0 {
+            writeln!(&mut writer, "{}", self.scan_limit)?;
+        }
+
+        if config.rate_limit > 0 {
+            writeln!(&mut writer, "{}", self.rate_limit)?;
+        }
+
+        if !config.time_limit.is_empty() {
+            writeln!(&mut writer, "{}", self.time_limit)?;
+        }
+
+        if matches!(self.update_status, UpdateStatus::OutOfDate) {
+            let update = BannerEntry::new(
+                "🎉",
+                "New Version Available",
+                "https://github.com/epi052/feroxbuster/releases/latest",
+            );
+            writeln!(&mut writer, "{}", update)?;
+        }
+
+        writeln!(&mut writer, "{}", self.footer())?;
+
+        Ok(())
+    }
+}

src/banner/entry.rs

@@ -0,0 +1,57 @@
+use console::{measure_text_width, Emoji};
+use std::fmt;
+
+/// Initial visual indentation size used in formatting banner entries
+const INDENT: usize = 3;
+
+/// Column width used in formatting banner entries
+const COL_WIDTH: usize = 22;
+
+/// Represents a single line on the banner
+#[derive(Default)]
+pub(super) struct BannerEntry {
+    /// emoji used in the banner entry
+    emoji: String,
+
+    /// title used in the banner entry
+    title: String,
+
+    /// value passed in via config/cli/defaults
+    value: String,
+}
+
+/// implementation of a banner entry
+impl BannerEntry {
+    /// Create a new banner entry from given fields
+    pub fn new(emoji: &str, title: &str, value: &str) -> Self {
+        BannerEntry {
+            emoji: emoji.to_string(),
+            title: title.to_string(),
+            value: value.to_string(),
+        }
+    }
+
+    /// Simple wrapper for emoji or fallback when terminal doesn't support emoji
+    fn format_emoji(&self) -> String {
+        let width = measure_text_width(&self.emoji);
+        let pad_len = width * width;
+        let pad = format!("{:<pad_len$}", "\u{0020}", pad_len = pad_len);
+        Emoji(&self.emoji, &pad).to_string()
+    }
+}
+
+/// Display implementation for a banner entry
+impl fmt::Display for BannerEntry {
+    /// Display formatter for the given banner entry
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "\u{0020}{:\u{0020}<indent$}{:\u{0020}<width$}\u{2502}\u{0020}{}",
+            self.format_emoji(),
+            self.title,
+            self.value,
+            indent = INDENT,
+            width = COL_WIDTH
+        )
+    }
+}

src/banner/mod.rs

@@ -0,0 +1,8 @@
+//! all logic related to building/printing the banner seen when scans start
+mod container;
+mod entry;
+
+#[cfg(test)]
+mod tests;
+
+pub use self::container::{Banner, UPDATE_URL};

src/banner/tests.rs

@@ -0,0 +1,171 @@
+use super::container::UpdateStatus;
+use super::*;
+use crate::{config::Configuration, event_handlers::Handles};
+use httpmock::Method::GET;
+use httpmock::MockServer;
+use std::{io::stderr, sync::Arc, time::Duration};
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test to hit no execution of targets for loop in banner
+async fn banner_intialize_without_targets() {
+    let config = Configuration::new().unwrap();
+    let banner = Banner::new(&[], &config);
+    banner.print_to(stderr(), Arc::new(config)).unwrap();
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test to hit no execution of statuscode for loop in banner
+async fn banner_intialize_without_status_codes() {
+    let config = Configuration {
+        status_codes: vec![],
+        ..Default::default()
+    };
+
+    let banner = Banner::new(&[String::from("http://localhost")], &config);
+    banner.print_to(stderr(), Arc::new(config)).unwrap();
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test to hit an empty config file
+async fn banner_intialize_without_config_file() {
+    let config = Configuration {
+        config: String::new(),
+        ..Default::default()
+    };
+
+    let banner = Banner::new(&[String::from("http://localhost")], &config);
+    banner.print_to(stderr(), Arc::new(config)).unwrap();
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test to hit an empty queries
+async fn banner_intialize_without_queries() {
+    let config = Configuration {
+        queries: vec![(String::new(), String::new())],
+        ..Default::default()
+    };
+
+    let banner = Banner::new(&[String::from("http://localhost")], &config);
+    banner.print_to(stderr(), Arc::new(config)).unwrap();
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test that
+async fn banner_needs_update_returns_unknown_with_bad_url() {
+    let handles = Arc::new(Handles::for_testing(None, None).0);
+
+    let mut banner = Banner::new(
+        &[String::from("http://localhost")],
+        &Configuration::new().unwrap(),
+    );
+
+    let _ = banner.check_for_updates("", handles).await;
+
+    assert!(matches!(banner.update_status, UpdateStatus::Unknown));
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test return value of good url to needs_update
+async fn banner_needs_update_returns_up_to_date() {
+    let srv = MockServer::start();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/latest");
+        then.status(200).body("{\"tag_name\":\"v1.1.0\"}");
+    });
+
+    let handles = Arc::new(Handles::for_testing(None, None).0);
+
+    let mut banner = Banner::new(&[srv.url("")], &Configuration::new().unwrap());
+    banner.version = String::from("1.1.0");
+
+    let _ = banner.check_for_updates(&srv.url("/latest"), handles).await;
+
+    assert_eq!(mock.hits(), 1);
+    assert!(matches!(banner.update_status, UpdateStatus::UpToDate));
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test return value of good url to needs_update that returns a newer version than current
+async fn banner_needs_update_returns_out_of_date() {
+    let srv = MockServer::start();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/latest");
+        then.status(200).body("{\"tag_name\":\"v1.1.0\"}");
+    });
+
+    let handles = Arc::new(Handles::for_testing(None, None).0);
+
+    let mut banner = Banner::new(&[srv.url("")], &Configuration::new().unwrap());
+    banner.version = String::from("1.0.1");
+
+    let _ = banner.check_for_updates(&srv.url("/latest"), handles).await;
+
+    assert_eq!(mock.hits(), 1);
+    assert!(matches!(banner.update_status, UpdateStatus::OutOfDate));
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test return value of good url that times out
+async fn banner_needs_update_returns_unknown_on_timeout() {
+    let srv = MockServer::start();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/latest");
+        then.status(200)
+            .body("{\"tag_name\":\"v1.1.0\"}")
+            .delay(Duration::from_secs(8));
+    });
+
+    let handles = Arc::new(Handles::for_testing(None, None).0);
+
+    let mut banner = Banner::new(&[srv.url("")], &Configuration::new().unwrap());
+
+    let _ = banner.check_for_updates(&srv.url("/latest"), handles).await;
+
+    assert_eq!(mock.hits(), 1);
+    assert!(matches!(banner.update_status, UpdateStatus::Unknown));
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test return value of good url with bad json response
+async fn banner_needs_update_returns_unknown_on_bad_json_response() {
+    let srv = MockServer::start();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/latest");
+        then.status(200).body("not json");
+    });
+
+    let handles = Arc::new(Handles::for_testing(None, None).0);
+
+    let mut banner = Banner::new(&[srv.url("")], &Configuration::new().unwrap());
+
+    let _ = banner.check_for_updates(&srv.url("/latest"), handles).await;
+
+    assert_eq!(mock.hits(), 1);
+    assert!(matches!(banner.update_status, UpdateStatus::Unknown));
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test return value of good url with json response that lacks the tag_name field
+async fn banner_needs_update_returns_unknown_on_json_without_correct_tag() {
+    let srv = MockServer::start();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/latest");
+        then.status(200)
+            .body("{\"no tag_name\": \"doesn't exist\"}");
+    });
+
+    let handles = Arc::new(Handles::for_testing(None, None).0);
+
+    let mut banner = Banner::new(&[srv.url("")], &Configuration::new().unwrap());
+    banner.version = String::from("1.0.1");
+
+    let _ = banner.check_for_updates(&srv.url("/latest"), handles).await;
+
+    assert_eq!(mock.hits(), 1);
+    assert!(matches!(banner.update_status, UpdateStatus::Unknown));
+}

src/client.rs

@@ -1,10 +1,8 @@
-use crate::utils::{module_colorizer, status_colorizer};
+use anyhow::Result;
 use reqwest::header::HeaderMap;
 use reqwest::{redirect::Policy, Client, Proxy};
 use std::collections::HashMap;
 use std::convert::TryInto;
-#[cfg(not(test))]
-use std::process::exit;
 use std::time::Duration;
 
 /// Create and return an instance of [reqwest::Client](https://docs.rs/reqwest/latest/reqwest/struct.Client.html)
@@ -15,15 +13,14 @@ pub fn initialize(
     insecure: bool,
     headers: &HashMap<String, String>,
     proxy: Option<&str>,
-) -> Client {
+) -> Result<Client> {
     let policy = if redirects {
         Policy::limited(10)
     } else {
         Policy::none()
     };
 
-    // try_into returns infallible as its error, unwrap is safe here
-    let header_map: HeaderMap = headers.try_into().unwrap();
+    let header_map: HeaderMap = headers.try_into()?;
 
     let client = Client::builder()
         .timeout(Duration::new(timeout, 0))
@@ -32,51 +29,15 @@ pub fn initialize(
         .default_headers(header_map)
         .redirect(policy);
 
-    let client = match proxy {
-        // a proxy is specified, need to add it to the client
-        Some(some_proxy) => {
-            if !some_proxy.is_empty() {
-                // it's not an empty string
-                match Proxy::all(some_proxy) {
-                    Ok(proxy_obj) => client.proxy(proxy_obj),
-                    Err(e) => {
-                        eprintln!(
-                            "{} {} {}",
-                            status_colorizer("ERROR"),
-                            module_colorizer("Client::initialize"),
-                            e
-                        );
-
-                        #[cfg(test)]
-                        panic!();
-                        #[cfg(not(test))]
-                        exit(1);
-                    }
-                }
-            } else {
-                client // Some("") was used?
-            }
-        }
-        // no proxy specified
-        None => client,
-    };
-
-    match client.build() {
-        Ok(client) => client,
-        Err(e) => {
-            eprintln!(
-                "{} {} {}",
-                status_colorizer("ERROR"),
-                module_colorizer("Client::build"),
-                e
-            );
-
-            #[cfg(test)]
-            panic!();
-            #[cfg(not(test))]
-            exit(1);
+    if let Some(some_proxy) = proxy {
+        if !some_proxy.is_empty() {
+            // it's not an empty string; set the proxy
+            let proxy_obj = Proxy::all(some_proxy)?;
+            return Ok(client.proxy(proxy_obj).build()?);
         }
     }
+
+    Ok(client.build()?)
 }
 
 #[cfg(test)]
@@ -88,7 +49,7 @@ mod tests {
     /// create client with a bad proxy, expect panic
     fn client_with_bad_proxy() {
         let headers = HashMap::new();
-        initialize(0, "stuff", true, false, &headers, Some("not a valid proxy"));
+        initialize(0, "stuff", true, false, &headers, Some("not a valid proxy")).unwrap();
     }
 
     #[test]
@@ -96,6 +57,6 @@ mod tests {
     fn client_with_good_proxy() {
         let headers = HashMap::new();
         let proxy = "http://127.0.0.1:8080";
-        initialize(0, "stuff", true, true, &headers, Some(proxy));
+        initialize(0, "stuff", true, true, &headers, Some(proxy)).unwrap();
     }
 }

src/config/container.rs

@@ -1,29 +1,22 @@
-use crate::scan_manager::resume_scan;
-use crate::utils::{module_colorizer, status_colorizer};
-use crate::{client, parser, progress};
-use crate::{FeroxSerialize, DEFAULT_CONFIG_NAME, DEFAULT_STATUS_CODES, DEFAULT_WORDLIST, VERSION};
+use super::utils::{
+    depth, report_and_exit, save_state, serialized_type, status_codes, threads, timeout,
+    user_agent, wordlist, OutputLevel,
+};
+use crate::config::determine_output_level;
+use crate::{
+    client, parser, scan_manager::resume_scan, traits::FeroxSerialize, utils::fmt_err,
+    DEFAULT_CONFIG_NAME,
+};
+use anyhow::{anyhow, Context, Result};
 use clap::{value_t, ArgMatches};
-use indicatif::{MultiProgress, ProgressBar, ProgressDrawTarget};
-use lazy_static::lazy_static;
 use reqwest::{Client, StatusCode};
 use serde::{Deserialize, Serialize};
-use std::collections::HashMap;
-use std::env::{current_dir, current_exe};
-use std::fs::read_to_string;
-use std::path::PathBuf;
-#[cfg(not(test))]
-use std::process::exit;
-
-lazy_static! {
-    /// Global configuration state
-    pub static ref CONFIGURATION: Configuration = Configuration::new();
-
-    /// Global progress bar that houses other progress bars
-    pub static ref PROGRESS_BAR: MultiProgress = MultiProgress::with_draw_target(ProgressDrawTarget::stdout());
-
-    /// Global progress bar that is only used for printing messages that don't jack up other bars
-    pub static ref PROGRESS_PRINTER: ProgressBar = progress::add_bar("", 0, true, false);
-}
+use std::{
+    collections::HashMap,
+    env::{current_dir, current_exe},
+    fs::read_to_string,
+    path::PathBuf,
+};
 
 /// macro helper to abstract away repetitive configuration updates
 macro_rules! update_config_if_present {
@@ -51,21 +44,6 @@ macro_rules! update_if_not_default {
     };
 }
 
-/// simple helper to clean up some code reuse below; panics under test / exits in prod
-fn report_and_exit(err: &str) -> ! {
-    eprintln!(
-        "{} {}: {}",
-        status_colorizer("ERROR"),
-        module_colorizer("Configuration::new"),
-        err
-    );
-
-    #[cfg(test)]
-    panic!();
-    #[cfg(not(test))]
-    exit(1);
-}
-
 /// Represents the final, global configuration of the program.
 ///
 /// This struct is the combination of the following:
@@ -80,7 +58,7 @@ fn report_and_exit(err: &str) -> ! {
 pub struct Configuration {
     #[serde(rename = "type", default = "serialized_type")]
     /// Name of this type of struct, used for serialization, i.e. `{"type":"configuration"}`
-    kind: String,
+    pub kind: String,
 
     /// Path to the wordlist
     #[serde(default = "wordlist")]
@@ -90,11 +68,11 @@ pub struct Configuration {
     #[serde(default)]
     pub config: String,
 
-    /// Proxy to use for requests (ex: http(s)://host:port, socks5://host:port)
+    /// Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)
     #[serde(default)]
     pub proxy: String,
 
-    /// Replay Proxy to use for requests (ex: http(s)://host:port, socks5://host:port)
+    /// Replay Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)
     #[serde(default)]
     pub replay_proxy: String,
 
@@ -134,10 +112,18 @@ pub struct Configuration {
     #[serde(default)]
     pub verbosity: u8,
 
-    /// Only print URLs
+    /// Only print URLs (was --quiet in versions < 2.0.0)
+    #[serde(default)]
+    pub silent: bool,
+
+    /// No header, no status bars
     #[serde(default)]
     pub quiet: bool,
 
+    /// more easily differentiate between the three states of output levels
+    #[serde(skip)]
+    pub output_level: OutputLevel,
+
     /// Store log output as NDJSON
     #[serde(default)]
     pub json: bool,
@@ -199,6 +185,10 @@ pub struct Configuration {
     #[serde(default)]
     pub scan_limit: usize,
 
+    /// Number of requests per second permitted (per directory); a limit of 0 means no limit is imposed
+    #[serde(default)]
+    pub rate_limit: usize,
+
     /// Filter out messages of a particular size
     #[serde(default)]
     pub filter_size: Vec<u64>,
@@ -223,6 +213,10 @@ pub struct Configuration {
     #[serde(default)]
     pub resumed: bool,
 
+    /// Resume scan from this file
+    #[serde(default)]
+    pub resume_from: String,
+
     /// Whether or not a scan's current state should be saved when user presses Ctrl+C
     ///
     /// Not configurable from CLI; can only be set from a config file
@@ -233,53 +227,10 @@ pub struct Configuration {
     /// non-negative integer and the next character is either s, m, h, or d (case insensitive)
     #[serde(default)]
     pub time_limit: String,
-}
 
-// functions timeout, threads, status_codes, user_agent, wordlist, save_state, and depth are used to provide
-// defaults in the event that a ferox-config.toml is found but one or more of the values below
-// aren't listed in the config.  This way, we get the correct defaults upon Deserialization
-
-/// default Configuration type for use in json output
-fn serialized_type() -> String {
-    String::from("configuration")
-}
-
-/// default timeout value
-fn timeout() -> u64 {
-    7
-}
-
-/// default save_state value
-fn save_state() -> bool {
-    true
-}
-
-/// default threads value
-fn threads() -> usize {
-    50
-}
-
-/// default status codes
-fn status_codes() -> Vec<u16> {
-    DEFAULT_STATUS_CODES
-        .iter()
-        .map(|code| code.as_u16())
-        .collect()
-}
-
-/// default wordlist
-fn wordlist() -> String {
-    String::from(DEFAULT_WORDLIST)
-}
-
-/// default user-agent
-fn user_agent() -> String {
-    format!("feroxbuster/{}", VERSION)
-}
-
-/// default recursion depth
-fn depth() -> usize {
-    4
+    /// Filter out response bodies that meet a certain threshold of similarity
+    #[serde(default)]
+    pub filter_similar: Vec<String>,
 }
 
 impl Default for Configuration {
@@ -287,11 +238,13 @@ impl Default for Configuration {
     fn default() -> Self {
         let timeout = timeout();
         let user_agent = user_agent();
-        let client = client::initialize(timeout, &user_agent, false, false, &HashMap::new(), None);
+        let client = client::initialize(timeout, &user_agent, false, false, &HashMap::new(), None)
+            .expect("Could not build client");
         let replay_client = None;
         let status_codes = status_codes();
         let replay_codes = status_codes.clone();
         let kind = serialized_type();
+        let output_level = OutputLevel::Default;
 
         Configuration {
             kind,
@@ -302,12 +255,15 @@ impl Default for Configuration {
             status_codes,
             replay_client,
             dont_filter: false,
+            silent: false,
             quiet: false,
+            output_level,
             resumed: false,
             stdin: false,
             json: false,
             verbosity: 0,
             scan_limit: 0,
+            rate_limit: 0,
             add_slash: false,
             insecure: false,
             redirects: false,
@@ -320,6 +276,7 @@ impl Default for Configuration {
             debug_log: String::new(),
             target_url: String::new(),
             time_limit: String::new(),
+            resume_from: String::new(),
             replay_proxy: String::new(),
             queries: Vec::new(),
             extensions: Vec::new(),
@@ -328,6 +285,7 @@ impl Default for Configuration {
             filter_line_count: Vec::new(),
             filter_word_count: Vec::new(),
             filter_status: Vec::new(),
+            filter_similar: Vec::new(),
             headers: HashMap::new(),
             depth: depth(),
             threads: threads(),
@@ -354,11 +312,13 @@ impl Configuration {
     /// - **output**: `None` (print to stdout)
     /// - **debug_log**: `None`
     /// - **quiet**: `false`
+    /// - **silent**: `false`
     /// - **save_state**: `true`
     /// - **user_agent**: `feroxbuster/VERSION`
     /// - **insecure**: `false` (don't be insecure, i.e. don't allow invalid certs)
     /// - **extensions**: `None`
     /// - **filter_size**: `None`
+    /// - **filter_similar**: `None`
     /// - **filter_regex**: `None`
     /// - **filter_word_count**: `None`
     /// - **filter_line_count**: `None`
@@ -371,6 +331,7 @@ impl Configuration {
     /// - **dont_filter**: `false` (auto filter wildcard responses)
     /// - **depth**: `4` (maximum recursion depth)
     /// - **scan_limit**: `0` (no limit on concurrent scans imposed)
+    /// - **rate_limit**: `0` (no limit on concurrent scans imposed)
     /// - **time_limit**: `None` (no limit on length of scan imposed)
     /// - **replay_proxy**: `None` (no limit on concurrent scans imposed)
     /// - **replay_codes**: [`DEFAULT_RESPONSE_CODES`](constant.DEFAULT_RESPONSE_CODES.html)
@@ -392,12 +353,14 @@ impl Configuration {
     ///
     /// The resulting [Configuration](struct.Configuration.html) is a singleton with a `static`
     /// lifetime.
-    pub fn new() -> Self {
+    pub fn new() -> Result<Self> {
         // when compiling for test, we want to eliminate the runtime dependency of the parser
         if cfg!(test) {
-            let mut test_config = Configuration::default();
-            test_config.save_state = false; // don't clutter up junk when testing
-            return test_config;
+            let test_config = Configuration {
+                save_state: false, // don't clutter up junk when testing
+                ..Default::default()
+            };
+            return Ok(test_config);
         }
 
         let args = parser::initialize().get_matches();
@@ -407,11 +370,11 @@ impl Configuration {
         let mut config = Configuration::default();
 
         // read in all config files
-        Self::parse_config_files(&mut config);
+        Self::parse_config_files(&mut config)?;
 
         // read in the user provided options, this produces a separate instance of Configuration
         // in order to allow for potentially merging into a --resume-from Configuration
-        let cli_config = Self::parse_cli_args(&args);
+        let cli_config = Self::parse_cli_args(&args)?;
 
         // --resume-from used, need to first read the Configuration from disk, and then
         // merge the cli_config into the resumed config
@@ -438,7 +401,7 @@ impl Configuration {
             // clients aren't serialized, have to remake them from the previous config
             Self::try_rebuild_clients(&mut previous_config);
 
-            return previous_config;
+            return Ok(previous_config);
         }
 
         // if we've gotten to this point in the code, --resume-from was not used, so we need to
@@ -448,12 +411,12 @@ impl Configuration {
         // rebuild clients is the last step in either code branch
         Self::try_rebuild_clients(&mut config);
 
-        config
+        Ok(config)
     }
 
     /// Parse all possible versions of the ferox-config.toml file, adhering to the order of
     /// precedence outlined above
-    fn parse_config_files(mut config: &mut Self) {
+    fn parse_config_files(mut config: &mut Self) -> Result<()> {
         // Next, we parse the ferox-config.toml file, if present and set the values
         // therein to overwrite our default values. Deserialized defaults are specified
         // in the Configuration struct so that we don't change anything that isn't
@@ -469,46 +432,47 @@ impl Configuration {
         let config_file = PathBuf::new()
             .join("/etc/feroxbuster")
             .join(DEFAULT_CONFIG_NAME);
-        Self::parse_and_merge_config(config_file, &mut config);
+        Self::parse_and_merge_config(config_file, &mut config)?;
 
         // merge a config found at ~/.config/feroxbuster/ferox-config.toml
-        if let Some(config_dir) = dirs::config_dir() {
-            // config_dir() resolves to one of the following
-            //   - linux: $XDG_CONFIG_HOME or $HOME/.config
-            //   - macOS: $HOME/Library/Application Support
-            //   - windows: {FOLDERID_RoamingAppData}
-
-            let config_file = config_dir.join("feroxbuster").join(DEFAULT_CONFIG_NAME);
-            Self::parse_and_merge_config(config_file, &mut config);
-        };
+        // config_dir() resolves to one of the following
+        //   - linux: $XDG_CONFIG_HOME or $HOME/.config
+        //   - macOS: $HOME/Library/Application Support
+        //   - windows: {FOLDERID_RoamingAppData}
+        let config_dir = dirs::config_dir().ok_or_else(|| anyhow!("Couldn't load config"))?;
+        let config_file = config_dir.join("feroxbuster").join(DEFAULT_CONFIG_NAME);
+        Self::parse_and_merge_config(config_file, &mut config)?;
 
         // merge a config found in same the directory as feroxbuster executable
-        if let Ok(exe_path) = current_exe() {
-            if let Some(bin_dir) = exe_path.parent() {
-                let config_file = bin_dir.join(DEFAULT_CONFIG_NAME);
-                Self::parse_and_merge_config(config_file, &mut config);
-            };
-        };
+        let exe_path = current_exe()?;
+        let bin_dir = exe_path
+            .parent()
+            .ok_or_else(|| anyhow!("Couldn't load config"))?;
+        let config_file = bin_dir.join(DEFAULT_CONFIG_NAME);
+        Self::parse_and_merge_config(config_file, &mut config)?;
 
         // merge a config found in the user's current working directory
-        if let Ok(cwd) = current_dir() {
-            let config_file = cwd.join(DEFAULT_CONFIG_NAME);
-            Self::parse_and_merge_config(config_file, &mut config);
-        }
+        let cwd = current_dir()?;
+        let config_file = cwd.join(DEFAULT_CONFIG_NAME);
+        Self::parse_and_merge_config(config_file, &mut config)?;
+
+        Ok(())
     }
 
     /// Given a set of ArgMatches read from the CLI, update and return the default Configuration
     /// settings
-    fn parse_cli_args(args: &ArgMatches) -> Self {
+    fn parse_cli_args(args: &ArgMatches) -> Result<Self> {
         let mut config = Configuration::default();
 
         update_config_if_present!(&mut config.threads, args, "threads", usize);
         update_config_if_present!(&mut config.depth, args, "depth", usize);
         update_config_if_present!(&mut config.scan_limit, args, "scan_limit", usize);
+        update_config_if_present!(&mut config.rate_limit, args, "rate_limit", usize);
         update_config_if_present!(&mut config.wordlist, args, "wordlist", String);
         update_config_if_present!(&mut config.output, args, "output", String);
         update_config_if_present!(&mut config.debug_log, args, "debug_log", String);
         update_config_if_present!(&mut config.time_limit, args, "time_limit", String);
+        update_config_if_present!(&mut config.resume_from, args, "resume_from", String);
 
         if let Some(arg) = args.values_of("status_codes") {
             config.status_codes = arg
@@ -552,6 +516,10 @@ impl Configuration {
             config.filter_regex = arg.map(|val| val.to_string()).collect();
         }
 
+        if let Some(arg) = args.values_of("filter_similar") {
+            config.filter_similar = arg.map(|val| val.to_string()).collect();
+        }
+
         if let Some(arg) = args.values_of("filter_size") {
             config.filter_size = arg
                 .map(|size| {
@@ -579,12 +547,18 @@ impl Configuration {
                 .collect();
         }
 
-        if args.is_present("quiet") {
+        if args.is_present("silent") {
             // the reason this is protected by an if statement:
-            // consider a user specifying quiet = true in ferox-config.toml
+            // consider a user specifying silent = true in ferox-config.toml
             // if the line below is outside of the if, we'd overwrite true with
-            // false if no -q is used on the command line
+            // false if no --silent is used on the command line
+            config.silent = true;
+            config.output_level = OutputLevel::Silent;
+        }
+
+        if args.is_present("quiet") {
             config.quiet = true;
+            config.output_level = OutputLevel::Quiet;
         }
 
         if args.is_present("dont_filter") {
@@ -662,7 +636,7 @@ impl Configuration {
             }
         }
 
-        config
+        Ok(config)
     }
 
     /// this function determines if we've gotten a Client configuration change from
@@ -686,6 +660,7 @@ impl Configuration {
                     &configuration.headers,
                     None,
                 )
+                .expect("Could not rebuild client")
             } else {
                 configuration.client = client::initialize(
                     configuration.timeout,
@@ -695,40 +670,41 @@ impl Configuration {
                     &configuration.headers,
                     Some(&configuration.proxy),
                 )
+                .expect("Could not rebuild client")
             }
         }
 
         if !configuration.replay_proxy.is_empty() {
             // only set replay_client when replay_proxy is set
-            configuration.replay_client = Some(client::initialize(
-                configuration.timeout,
-                &configuration.user_agent,
-                configuration.redirects,
-                configuration.insecure,
-                &configuration.headers,
-                Some(&configuration.replay_proxy),
-            ));
+            configuration.replay_client = Some(
+                client::initialize(
+                    configuration.timeout,
+                    &configuration.user_agent,
+                    configuration.redirects,
+                    configuration.insecure,
+                    &configuration.headers,
+                    Some(&configuration.replay_proxy),
+                )
+                .expect("Could not rebuild client"),
+            );
         }
     }
 
     /// Given a configuration file's location and an instance of `Configuration`, read in
     /// the config file if found and update the current settings with the settings found therein
-    fn parse_and_merge_config(config_file: PathBuf, mut config: &mut Self) {
+    fn parse_and_merge_config(config_file: PathBuf, mut config: &mut Self) -> Result<()> {
         if config_file.exists() {
             // save off a string version of the path before it goes out of scope
-            let conf_str = match config_file.to_str() {
-                Some(cs) => String::from(cs),
-                None => String::new(),
-            };
+            let conf_str = config_file.to_str().unwrap_or("").to_string();
+            let settings = Self::parse_config(config_file)?;
 
-            if let Some(settings) = Self::parse_config(config_file) {
-                // set the config used for viewing in the banner
-                config.config = conf_str;
+            // set the config used for viewing in the banner
+            config.config = conf_str;
 
-                // update the settings
-                Self::merge_config(&mut config, settings);
-            }
+            // update the settings
+            Self::merge_config(&mut config, settings);
         }
+        Ok(())
     }
 
     /// Given two Configurations, overwrite `settings` with the fields found in `settings_to_merge`
@@ -743,7 +719,10 @@ impl Configuration {
         update_if_not_default!(&mut conf.time_limit, new.time_limit, "");
         update_if_not_default!(&mut conf.proxy, new.proxy, "");
         update_if_not_default!(&mut conf.verbosity, new.verbosity, 0);
+        update_if_not_default!(&mut conf.silent, new.silent, false);
         update_if_not_default!(&mut conf.quiet, new.quiet, false);
+        // use updated quiet/silent values to determin output level
+        conf.output_level = determine_output_level(conf.quiet, conf.silent);
         update_if_not_default!(&mut conf.output, new.output, "");
         update_if_not_default!(&mut conf.redirects, new.redirects, false);
         update_if_not_default!(&mut conf.insecure, new.insecure, false);
@@ -760,6 +739,11 @@ impl Configuration {
             new.filter_regex,
             Vec::<String>::new()
         );
+        update_if_not_default!(
+            &mut conf.filter_similar,
+            new.filter_similar,
+            Vec::<String>::new()
+        );
         update_if_not_default!(
             &mut conf.filter_word_count,
             new.filter_word_count,
@@ -777,8 +761,10 @@ impl Configuration {
         );
         update_if_not_default!(&mut conf.dont_filter, new.dont_filter, false);
         update_if_not_default!(&mut conf.scan_limit, new.scan_limit, 0);
+        update_if_not_default!(&mut conf.rate_limit, new.rate_limit, 0);
         update_if_not_default!(&mut conf.replay_proxy, new.replay_proxy, "");
         update_if_not_default!(&mut conf.debug_log, new.debug_log, "");
+        update_if_not_default!(&mut conf.resume_from, new.resume_from, "");
         update_if_not_default!(&mut conf.json, new.json, false);
 
         update_if_not_default!(&mut conf.timeout, new.timeout, timeout());
@@ -795,23 +781,10 @@ impl Configuration {
     /// If present, read in `DEFAULT_CONFIG_NAME` and deserialize the specified values
     ///
     /// uses serde to deserialize the toml into a `Configuration` struct
-    fn parse_config(config_file: PathBuf) -> Option<Self> {
-        if let Ok(content) = read_to_string(config_file) {
-            match toml::from_str(content.as_str()) {
-                Ok(config) => {
-                    return Some(config);
-                }
-                Err(e) => {
-                    println!(
-                        "{} {} {}",
-                        status_colorizer("ERROR"),
-                        module_colorizer("config::parse_config"),
-                        e
-                    );
-                }
-            }
-        }
-        None
+    pub(super) fn parse_config(config_file: PathBuf) -> Result<Self> {
+        let content = read_to_string(config_file)?;
+        let config: Self = toml::from_str(content.as_str())?;
+        Ok(config)
     }
 }
 
@@ -846,357 +819,10 @@ impl FeroxSerialize for Configuration {
     ///    ],
     /// ...
     /// }\n
-    fn as_json(&self) -> String {
-        if let Ok(mut json) = serde_json::to_string(&self) {
-            json.push('\n');
-            json
-        } else {
-            String::from("{\"error\":\"could not Configuration convert to json\"}")
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use std::fs::write;
-    use tempfile::TempDir;
-
-    /// creates a dummy configuration file for testing
-    fn setup_config_test() -> Configuration {
-        let data = r#"
-            wordlist = "/some/path"
-            status_codes = [201, 301, 401]
-            replay_codes = [201, 301]
-            threads = 40
-            timeout = 5
-            proxy = "http://127.0.0.1:8080"
-            replay_proxy = "http://127.0.0.1:8081"
-            quiet = true
-            verbosity = 1
-            scan_limit = 6
-            time_limit = "10m"
-            output = "/some/otherpath"
-            debug_log = "/yet/anotherpath"
-            redirects = true
-            insecure = true
-            extensions = ["html", "php", "js"]
-            headers = {stuff = "things", mostuff = "mothings"}
-            queries = [["name","value"], ["rick", "astley"]]
-            no_recursion = true
-            add_slash = true
-            stdin = true
-            dont_filter = true
-            extract_links = true
-            json = true
-            save_state = false
-            depth = 1
-            filter_size = [4120]
-            filter_regex = ["^ignore me$"]
-            filter_word_count = [994, 992]
-            filter_line_count = [34]
-            filter_status = [201]
-        "#;
-        let tmp_dir = TempDir::new().unwrap();
-        let file = tmp_dir.path().join(DEFAULT_CONFIG_NAME);
-        write(&file, data).unwrap();
-        Configuration::parse_config(file).unwrap()
-    }
-
-    #[test]
-    /// test that all default config values meet expectations
-    fn default_configuration() {
-        let config = Configuration::default();
-        assert_eq!(config.wordlist, wordlist());
-        assert_eq!(config.proxy, String::new());
-        assert_eq!(config.target_url, String::new());
-        assert_eq!(config.time_limit, String::new());
-        assert_eq!(config.debug_log, String::new());
-        assert_eq!(config.config, String::new());
-        assert_eq!(config.replay_proxy, String::new());
-        assert_eq!(config.status_codes, status_codes());
-        assert_eq!(config.replay_codes, config.status_codes);
-        assert!(config.replay_client.is_none());
-        assert_eq!(config.threads, threads());
-        assert_eq!(config.depth, depth());
-        assert_eq!(config.timeout, timeout());
-        assert_eq!(config.verbosity, 0);
-        assert_eq!(config.scan_limit, 0);
-        assert_eq!(config.quiet, false);
-        assert_eq!(config.dont_filter, false);
-        assert_eq!(config.no_recursion, false);
-        assert_eq!(config.json, false);
-        assert_eq!(config.save_state, true);
-        assert_eq!(config.stdin, false);
-        assert_eq!(config.add_slash, false);
-        assert_eq!(config.redirects, false);
-        assert_eq!(config.extract_links, false);
-        assert_eq!(config.insecure, false);
-        assert_eq!(config.queries, Vec::new());
-        assert_eq!(config.extensions, Vec::<String>::new());
-        assert_eq!(config.filter_size, Vec::<u64>::new());
-        assert_eq!(config.filter_regex, Vec::<String>::new());
-        assert_eq!(config.filter_word_count, Vec::<usize>::new());
-        assert_eq!(config.filter_line_count, Vec::<usize>::new());
-        assert_eq!(config.filter_status, Vec::<u16>::new());
-        assert_eq!(config.headers, HashMap::new());
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_wordlist() {
-        let config = setup_config_test();
-        assert_eq!(config.wordlist, "/some/path");
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_debug_log() {
-        let config = setup_config_test();
-        assert_eq!(config.debug_log, "/yet/anotherpath");
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_status_codes() {
-        let config = setup_config_test();
-        assert_eq!(config.status_codes, vec![201, 301, 401]);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_replay_codes() {
-        let config = setup_config_test();
-        assert_eq!(config.replay_codes, vec![201, 301]);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_threads() {
-        let config = setup_config_test();
-        assert_eq!(config.threads, 40);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_depth() {
-        let config = setup_config_test();
-        assert_eq!(config.depth, 1);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_scan_limit() {
-        let config = setup_config_test();
-        assert_eq!(config.scan_limit, 6);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_timeout() {
-        let config = setup_config_test();
-        assert_eq!(config.timeout, 5);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_proxy() {
-        let config = setup_config_test();
-        assert_eq!(config.proxy, "http://127.0.0.1:8080");
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_replay_proxy() {
-        let config = setup_config_test();
-        assert_eq!(config.replay_proxy, "http://127.0.0.1:8081");
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_quiet() {
-        let config = setup_config_test();
-        assert_eq!(config.quiet, true);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_json() {
-        let config = setup_config_test();
-        assert_eq!(config.json, true);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_verbosity() {
-        let config = setup_config_test();
-        assert_eq!(config.verbosity, 1);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_output() {
-        let config = setup_config_test();
-        assert_eq!(config.output, "/some/otherpath");
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_redirects() {
-        let config = setup_config_test();
-        assert_eq!(config.redirects, true);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_insecure() {
-        let config = setup_config_test();
-        assert_eq!(config.insecure, true);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_no_recursion() {
-        let config = setup_config_test();
-        assert_eq!(config.no_recursion, true);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_stdin() {
-        let config = setup_config_test();
-        assert_eq!(config.stdin, true);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_dont_filter() {
-        let config = setup_config_test();
-        assert_eq!(config.dont_filter, true);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_add_slash() {
-        let config = setup_config_test();
-        assert_eq!(config.add_slash, true);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_extract_links() {
-        let config = setup_config_test();
-        assert_eq!(config.extract_links, true);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_extensions() {
-        let config = setup_config_test();
-        assert_eq!(config.extensions, vec!["html", "php", "js"]);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_filter_regex() {
-        let config = setup_config_test();
-        assert_eq!(config.filter_regex, vec!["^ignore me$"]);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_filter_size() {
-        let config = setup_config_test();
-        assert_eq!(config.filter_size, vec![4120]);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_filter_word_count() {
-        let config = setup_config_test();
-        assert_eq!(config.filter_word_count, vec![994, 992]);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_filter_line_count() {
-        let config = setup_config_test();
-        assert_eq!(config.filter_line_count, vec![34]);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_filter_status() {
-        let config = setup_config_test();
-        assert_eq!(config.filter_status, vec![201]);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_save_state() {
-        let config = setup_config_test();
-        assert_eq!(config.save_state, false);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_time_limit() {
-        let config = setup_config_test();
-        assert_eq!(config.time_limit, "10m");
-    }
-
-    #[test]
-    /// parse the test config and see that the values parsed are correct
-    fn config_reads_headers() {
-        let config = setup_config_test();
-        let mut headers = HashMap::new();
-        headers.insert("stuff".to_string(), "things".to_string());
-        headers.insert("mostuff".to_string(), "mothings".to_string());
-        assert_eq!(config.headers, headers);
-    }
-
-    #[test]
-    /// parse the test config and see that the values parsed are correct
-    fn config_reads_queries() {
-        let config = setup_config_test();
-        let mut queries = vec![];
-        queries.push(("name".to_string(), "value".to_string()));
-        queries.push(("rick".to_string(), "astley".to_string()));
-        assert_eq!(config.queries, queries);
-    }
-
-    #[test]
-    #[should_panic]
-    /// test that an error message is printed and panic is called when report_and_exit is called
-    fn config_report_and_exit_works() {
-        report_and_exit("some message");
-    }
-
-    #[test]
-    /// test as_str method of Configuration
-    fn as_str_returns_string_with_newline() {
-        let config = Configuration::new();
-        let config_str = config.as_str();
-        println!("{}", config_str);
-        assert!(config_str.starts_with("Configuration {"));
-        assert!(config_str.ends_with("}\n"));
-        assert!(config_str.contains("replay_codes:"));
-        assert!(config_str.contains("client: Client {"));
-        assert!(config_str.contains("user_agent: \"feroxbuster"));
-    }
-
-    #[test]
-    /// test as_json method of Configuration
-    fn as_json_returns_json_representation_of_configuration_with_newline() {
-        let mut config = Configuration::new();
-        config.timeout = 12;
-        config.depth = 2;
-        let config_str = config.as_json();
-        let json: Configuration = serde_json::from_str(&config_str).unwrap();
-        assert_eq!(json.config, config.config);
-        assert_eq!(json.wordlist, config.wordlist);
-        assert_eq!(json.replay_codes, config.replay_codes);
-        assert_eq!(json.timeout, config.timeout);
-        assert_eq!(json.depth, config.depth);
+    fn as_json(&self) -> Result<String> {
+        let mut json = serde_json::to_string(&self)
+            .with_context(|| fmt_err("Could not convert Configuration to JSON"))?;
+        json.push('\n');
+        Ok(json)
     }
 }

src/config/mod.rs

@@ -0,0 +1,9 @@
+//! all logic related to instantiating a running configuration
+
+mod container;
+mod utils;
+#[cfg(test)]
+mod tests;
+
+pub use self::container::Configuration;
+pub use self::utils::{determine_output_level, OutputLevel};

src/config/tests.rs

@@ -0,0 +1,378 @@
+use super::utils::*;
+use super::*;
+use crate::{traits::FeroxSerialize, DEFAULT_CONFIG_NAME};
+use std::{collections::HashMap, fs::write};
+use tempfile::TempDir;
+
+/// creates a dummy configuration file for testing
+fn setup_config_test() -> Configuration {
+    let data = r#"
+            wordlist = "/some/path"
+            status_codes = [201, 301, 401]
+            replay_codes = [201, 301]
+            threads = 40
+            timeout = 5
+            proxy = "http://127.0.0.1:8080"
+            replay_proxy = "http://127.0.0.1:8081"
+            quiet = true
+            silent = true
+            verbosity = 1
+            scan_limit = 6
+            rate_limit = 250
+            time_limit = "10m"
+            output = "/some/otherpath"
+            debug_log = "/yet/anotherpath"
+            resume_from = "/some/state/file"
+            redirects = true
+            insecure = true
+            extensions = ["html", "php", "js"]
+            headers = {stuff = "things", mostuff = "mothings"}
+            queries = [["name","value"], ["rick", "astley"]]
+            no_recursion = true
+            add_slash = true
+            stdin = true
+            dont_filter = true
+            extract_links = true
+            json = true
+            save_state = false
+            depth = 1
+            filter_size = [4120]
+            filter_regex = ["^ignore me$"]
+            filter_similar = ["https://somesite.com/soft404"]
+            filter_word_count = [994, 992]
+            filter_line_count = [34]
+            filter_status = [201]
+        "#;
+    let tmp_dir = TempDir::new().unwrap();
+    let file = tmp_dir.path().join(DEFAULT_CONFIG_NAME);
+    write(&file, data).unwrap();
+    Configuration::parse_config(file).unwrap()
+}
+
+#[test]
+/// test that all default config values meet expectations
+fn default_configuration() {
+    let config = Configuration::default();
+    assert_eq!(config.wordlist, wordlist());
+    assert_eq!(config.proxy, String::new());
+    assert_eq!(config.target_url, String::new());
+    assert_eq!(config.time_limit, String::new());
+    assert_eq!(config.resume_from, String::new());
+    assert_eq!(config.debug_log, String::new());
+    assert_eq!(config.config, String::new());
+    assert_eq!(config.replay_proxy, String::new());
+    assert_eq!(config.status_codes, status_codes());
+    assert_eq!(config.replay_codes, config.status_codes);
+    assert!(config.replay_client.is_none());
+    assert_eq!(config.threads, threads());
+    assert_eq!(config.depth, depth());
+    assert_eq!(config.timeout, timeout());
+    assert_eq!(config.verbosity, 0);
+    assert_eq!(config.scan_limit, 0);
+    assert_eq!(config.silent, false);
+    assert_eq!(config.quiet, false);
+    assert_eq!(config.dont_filter, false);
+    assert_eq!(config.no_recursion, false);
+    assert_eq!(config.json, false);
+    assert_eq!(config.save_state, true);
+    assert_eq!(config.stdin, false);
+    assert_eq!(config.add_slash, false);
+    assert_eq!(config.redirects, false);
+    assert_eq!(config.extract_links, false);
+    assert_eq!(config.insecure, false);
+    assert_eq!(config.queries, Vec::new());
+    assert_eq!(config.extensions, Vec::<String>::new());
+    assert_eq!(config.filter_size, Vec::<u64>::new());
+    assert_eq!(config.filter_regex, Vec::<String>::new());
+    assert_eq!(config.filter_similar, Vec::<String>::new());
+    assert_eq!(config.filter_word_count, Vec::<usize>::new());
+    assert_eq!(config.filter_line_count, Vec::<usize>::new());
+    assert_eq!(config.filter_status, Vec::<u16>::new());
+    assert_eq!(config.headers, HashMap::new());
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_wordlist() {
+    let config = setup_config_test();
+    assert_eq!(config.wordlist, "/some/path");
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_debug_log() {
+    let config = setup_config_test();
+    assert_eq!(config.debug_log, "/yet/anotherpath");
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_status_codes() {
+    let config = setup_config_test();
+    assert_eq!(config.status_codes, vec![201, 301, 401]);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_replay_codes() {
+    let config = setup_config_test();
+    assert_eq!(config.replay_codes, vec![201, 301]);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_threads() {
+    let config = setup_config_test();
+    assert_eq!(config.threads, 40);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_depth() {
+    let config = setup_config_test();
+    assert_eq!(config.depth, 1);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_scan_limit() {
+    let config = setup_config_test();
+    assert_eq!(config.scan_limit, 6);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_rate_limit() {
+    let config = setup_config_test();
+    assert_eq!(config.rate_limit, 250);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_timeout() {
+    let config = setup_config_test();
+    assert_eq!(config.timeout, 5);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_proxy() {
+    let config = setup_config_test();
+    assert_eq!(config.proxy, "http://127.0.0.1:8080");
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_replay_proxy() {
+    let config = setup_config_test();
+    assert_eq!(config.replay_proxy, "http://127.0.0.1:8081");
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_silent() {
+    let config = setup_config_test();
+    assert_eq!(config.silent, true);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_quiet() {
+    let config = setup_config_test();
+    assert_eq!(config.quiet, true);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_json() {
+    let config = setup_config_test();
+    assert_eq!(config.json, true);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_verbosity() {
+    let config = setup_config_test();
+    assert_eq!(config.verbosity, 1);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_output() {
+    let config = setup_config_test();
+    assert_eq!(config.output, "/some/otherpath");
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_redirects() {
+    let config = setup_config_test();
+    assert_eq!(config.redirects, true);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_insecure() {
+    let config = setup_config_test();
+    assert_eq!(config.insecure, true);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_no_recursion() {
+    let config = setup_config_test();
+    assert_eq!(config.no_recursion, true);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_stdin() {
+    let config = setup_config_test();
+    assert_eq!(config.stdin, true);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_dont_filter() {
+    let config = setup_config_test();
+    assert_eq!(config.dont_filter, true);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_add_slash() {
+    let config = setup_config_test();
+    assert_eq!(config.add_slash, true);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_extract_links() {
+    let config = setup_config_test();
+    assert_eq!(config.extract_links, true);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_extensions() {
+    let config = setup_config_test();
+    assert_eq!(config.extensions, vec!["html", "php", "js"]);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_filter_regex() {
+    let config = setup_config_test();
+    assert_eq!(config.filter_regex, vec!["^ignore me$"]);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_filter_similar() {
+    let config = setup_config_test();
+    assert_eq!(config.filter_similar, vec!["https://somesite.com/soft404"]);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_filter_size() {
+    let config = setup_config_test();
+    assert_eq!(config.filter_size, vec![4120]);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_filter_word_count() {
+    let config = setup_config_test();
+    assert_eq!(config.filter_word_count, vec![994, 992]);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_filter_line_count() {
+    let config = setup_config_test();
+    assert_eq!(config.filter_line_count, vec![34]);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_filter_status() {
+    let config = setup_config_test();
+    assert_eq!(config.filter_status, vec![201]);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_save_state() {
+    let config = setup_config_test();
+    assert_eq!(config.save_state, false);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_time_limit() {
+    let config = setup_config_test();
+    assert_eq!(config.time_limit, "10m");
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_resume_from() {
+    let config = setup_config_test();
+    assert_eq!(config.resume_from, "/some/state/file");
+}
+
+#[test]
+/// parse the test config and see that the values parsed are correct
+fn config_reads_headers() {
+    let config = setup_config_test();
+    let mut headers = HashMap::new();
+    headers.insert("stuff".to_string(), "things".to_string());
+    headers.insert("mostuff".to_string(), "mothings".to_string());
+    assert_eq!(config.headers, headers);
+}
+
+#[test]
+/// parse the test config and see that the values parsed are correct
+fn config_reads_queries() {
+    let config = setup_config_test();
+    let mut queries = vec![];
+    queries.push(("name".to_string(), "value".to_string()));
+    queries.push(("rick".to_string(), "astley".to_string()));
+    assert_eq!(config.queries, queries);
+}
+
+#[test]
+#[should_panic]
+/// test that an error message is printed and panic is called when report_and_exit is called
+fn config_report_and_exit_works() {
+    report_and_exit("some message");
+}
+
+#[test]
+/// test as_str method of Configuration
+fn as_str_returns_string_with_newline() {
+    let config = Configuration::new().unwrap();
+    let config_str = config.as_str();
+    println!("{}", config_str);
+    assert!(config_str.starts_with("Configuration {"));
+    assert!(config_str.ends_with("}\n"));
+    assert!(config_str.contains("replay_codes:"));
+    assert!(config_str.contains("client: Client {"));
+    assert!(config_str.contains("user_agent: \"feroxbuster"));
+}
+
+#[test]
+/// test as_json method of Configuration
+fn as_json_returns_json_representation_of_configuration_with_newline() {
+    let mut config = Configuration::new().unwrap();
+    config.timeout = 12;
+    config.depth = 2;
+    let config_str = config.as_json().unwrap();
+    let json: Configuration = serde_json::from_str(&config_str).unwrap();
+    assert_eq!(json.config, config.config);
+    assert_eq!(json.wordlist, config.wordlist);
+    assert_eq!(json.replay_codes, config.replay_codes);
+    assert_eq!(json.timeout, config.timeout);
+    assert_eq!(json.depth, config.depth);
+}

src/config/utils.rs

@@ -0,0 +1,131 @@
+use crate::{
+    utils::{module_colorizer, status_colorizer},
+    DEFAULT_STATUS_CODES, DEFAULT_WORDLIST, VERSION,
+};
+#[cfg(not(test))]
+use std::process::exit;
+
+/// simple helper to clean up some code reuse below; panics under test / exits in prod
+pub(super) fn report_and_exit(err: &str) -> ! {
+    eprintln!(
+        "{} {}: {}",
+        status_colorizer("ERROR"),
+        module_colorizer("Configuration::new"),
+        err
+    );
+
+    #[cfg(test)]
+    panic!();
+    #[cfg(not(test))]
+    exit(1);
+}
+
+// functions timeout, threads, status_codes, user_agent, wordlist, save_state, and depth are used to provide
+// defaults in the event that a ferox-config.toml is found but one or more of the values below
+// aren't listed in the config.  This way, we get the correct defaults upon Deserialization
+
+/// default Configuration type for use in json output
+pub(super) fn serialized_type() -> String {
+    String::from("configuration")
+}
+
+/// default timeout value
+pub(super) fn timeout() -> u64 {
+    7
+}
+
+/// default save_state value
+pub(super) fn save_state() -> bool {
+    true
+}
+
+/// default threads value
+pub(super) fn threads() -> usize {
+    50
+}
+
+/// default status codes
+pub(super) fn status_codes() -> Vec<u16> {
+    DEFAULT_STATUS_CODES
+        .iter()
+        .map(|code| code.as_u16())
+        .collect()
+}
+
+/// default wordlist
+pub(super) fn wordlist() -> String {
+    String::from(DEFAULT_WORDLIST)
+}
+
+/// default user-agent
+pub(super) fn user_agent() -> String {
+    format!("feroxbuster/{}", VERSION)
+}
+
+/// default recursion depth
+pub(super) fn depth() -> usize {
+    4
+}
+
+/// enum representing the three possible states for informational output (not logging verbosity)
+#[derive(Debug, Copy, Clone, PartialEq)]
+pub enum OutputLevel {
+    /// normal scan, no --quiet|--silent
+    Default,
+
+    /// quiet scan, print some information, but not all (new in versions >= 2.0.0)
+    Quiet,
+
+    /// silent scan, only print urls (used to be --quiet in versions 1.x.x)
+    Silent,
+}
+
+/// implement a default for OutputLevel
+impl Default for OutputLevel {
+    /// return Default
+    fn default() -> Self {
+        Self::Default
+    }
+}
+
+/// given the current settings for quiet and silent, determine output_level (DRY helper)
+pub fn determine_output_level(quiet: bool, silent: bool) -> OutputLevel {
+    if quiet && silent {
+        // user COULD have both as true in config file, take the more quiet of the two
+        OutputLevel::Silent
+    } else if quiet {
+        OutputLevel::Quiet
+    } else if silent {
+        OutputLevel::Silent
+    } else {
+        OutputLevel::Default
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    /// test determine_output_level returns higher of the two levels if both given values are true
+    fn determine_output_level_returns_correct_results() {
+        let mut level = determine_output_level(true, true);
+        assert_eq!(level, OutputLevel::Silent);
+
+        level = determine_output_level(false, true);
+        assert_eq!(level, OutputLevel::Silent);
+
+        level = determine_output_level(false, false);
+        assert_eq!(level, OutputLevel::Default);
+
+        level = determine_output_level(true, false);
+        assert_eq!(level, OutputLevel::Quiet);
+    }
+
+    #[test]
+    #[should_panic]
+    /// report_and_exit should panic/exit when called
+    fn report_and_exit_panics_under_test() {
+        report_and_exit("test");
+    }
+}

src/event_handlers/command.rs

@@ -0,0 +1,66 @@
+use std::collections::HashSet;
+use std::sync::Arc;
+
+use reqwest::StatusCode;
+use tokio::sync::oneshot::Sender;
+
+use crate::response::FeroxResponse;
+use crate::{
+    statistics::{StatError, StatField},
+    traits::FeroxFilter,
+};
+
+/// Protocol definition for updating an event handler via mpsc
+#[derive(Debug)]
+pub enum Command {
+    /// Add one to the total number of requests
+    AddRequest,
+
+    /// Add one to the proper field(s) based on the given `StatError`
+    AddError(StatError),
+
+    /// Add one to the proper field(s) based on the given `StatusCode`
+    AddStatus(StatusCode),
+
+    /// Create the progress bar (`BarType::Total`) that is updated from the stats thread
+    CreateBar,
+
+    /// Update a `Stats` field that corresponds to the given `StatField` by the given `usize` value
+    UpdateUsizeField(StatField, usize),
+
+    /// Update a `Stats` field that corresponds to the given `StatField` by the given `f64` value
+    UpdateF64Field(StatField, f64),
+
+    /// Save a `Stats` object to disk using `reporter::get_cached_file_handle`
+    Save,
+
+    /// Load a `Stats` object from disk
+    LoadStats(String),
+
+    /// Add a `FeroxFilter` implementor to `FilterHandler`'s instance of `FeroxFilters`
+    AddFilter(Box<dyn FeroxFilter>),
+
+    /// Send a `FeroxResponse` to the output handler for reporting
+    Report(Box<FeroxResponse>),
+
+    /// Send a group of urls to be scanned (only used for the urls passed in explicitly by the user)
+    ScanInitialUrls(Vec<String>),
+
+    /// Determine whether or not recursion is appropriate, given a FeroxResponse, if so start a scan
+    TryRecursion(Box<FeroxResponse>),
+
+    /// Send a pointer to the wordlist to the recursion handler
+    UpdateWordlist(Arc<HashSet<String>>),
+
+    /// Instruct the ScanHandler to join on all known scans, use sender to notify main when done
+    JoinTasks(Sender<bool>),
+
+    /// Command used to test that a spawned task succeeded in initialization
+    Ping,
+
+    /// Just receive a sender and reply, used for slowing down the main thread
+    Sync(Sender<bool>),
+
+    /// Break out of the (infinite) mpsc receive loop
+    Exit,
+}

src/event_handlers/container.rs

@@ -0,0 +1,135 @@
+use super::*;
+use crate::config::Configuration;
+use crate::event_handlers::scans::ScanHandle;
+use crate::scan_manager::FeroxScans;
+use crate::Joiner;
+#[cfg(test)]
+use crate::{filters::FeroxFilters, statistics::Stats, Command};
+use anyhow::{bail, Result};
+use std::sync::{Arc, RwLock};
+#[cfg(test)]
+use tokio::sync::mpsc::{self, UnboundedReceiver};
+
+#[derive(Debug)]
+/// Simple container for multiple JoinHandles
+pub struct Tasks {
+    /// JoinHandle for terminal handler
+    pub terminal: Joiner,
+
+    /// JoinHandle for statistics handler
+    pub stats: Joiner,
+
+    /// JoinHandle for filters handler
+    pub filters: Joiner,
+
+    /// JoinHandle for scans handler
+    pub scans: Joiner,
+}
+
+/// Tasks implementation
+impl Tasks {
+    /// Given JoinHandles for terminal, statistics, and filters create a new Tasks object
+    pub fn new(terminal: Joiner, stats: Joiner, filters: Joiner, scans: Joiner) -> Self {
+        Self {
+            terminal,
+            stats,
+            filters,
+            scans,
+        }
+    }
+}
+
+#[derive(Debug)]
+/// Container for the different *Handles that will be shared across modules
+pub struct Handles {
+    /// Handle for statistics
+    pub stats: StatsHandle,
+
+    /// Handle for filters
+    pub filters: FiltersHandle,
+
+    /// Handle for output (terminal/file)
+    pub output: TermOutHandle,
+
+    /// Handle for Configuration
+    pub config: Arc<Configuration>,
+
+    /// Handle for recursion
+    pub scans: RwLock<Option<ScanHandle>>,
+}
+
+/// implementation of Handles
+impl Handles {
+    /// Given a StatsHandle, FiltersHandle, and OutputHandle, create a Handles object
+    pub fn new(
+        stats: StatsHandle,
+        filters: FiltersHandle,
+        output: TermOutHandle,
+        config: Arc<Configuration>,
+    ) -> Self {
+        Self {
+            stats,
+            filters,
+            output,
+            config,
+            scans: RwLock::new(None),
+        }
+    }
+
+    /// create a Handles object suitable for unit testing (non-functional)
+    #[cfg(test)]
+    pub fn for_testing(
+        scanned_urls: Option<Arc<FeroxScans>>,
+        config: Option<Arc<Configuration>>,
+    ) -> (Self, UnboundedReceiver<Command>) {
+        let configuration = config.unwrap_or_else(|| Arc::new(Configuration::new().unwrap()));
+        let (tx, rx) = mpsc::unbounded_channel::<Command>();
+        let terminal_handle = TermOutHandle::new(tx.clone(), tx.clone());
+        let stats_handle = StatsHandle::new(
+            Arc::new(Stats::new(
+                configuration.extensions.len(),
+                configuration.json,
+            )),
+            tx.clone(),
+        );
+        let filters_handle = FiltersHandle::new(Arc::new(FeroxFilters::default()), tx.clone());
+        let handles = Self::new(stats_handle, filters_handle, terminal_handle, configuration);
+        if let Some(sh) = scanned_urls {
+            let scan_handle = ScanHandle::new(sh, tx);
+            handles.set_scan_handle(scan_handle);
+        }
+        (handles, rx)
+    }
+
+    /// Set the ScanHandle object
+    pub fn set_scan_handle(&self, handle: ScanHandle) {
+        if let Ok(mut guard) = self.scans.write() {
+            if guard.is_none() {
+                let _ = std::mem::replace(&mut *guard, Some(handle));
+            }
+        }
+    }
+
+    /// Helper to easily send a Command over the (locked) underlying CommandSender object
+    pub fn send_scan_command(&self, command: Command) -> Result<()> {
+        if let Ok(guard) = self.scans.read().as_ref() {
+            if let Some(handle) = guard.as_ref() {
+                handle.send(command)?;
+                return Ok(());
+            }
+        }
+
+        bail!("Could not get underlying CommandSender object")
+    }
+
+    /// Helper to easily get the (locked) underlying FeroxScans object
+    pub fn ferox_scans(&self) -> Result<Arc<FeroxScans>> {
+        if let Ok(guard) = self.scans.read().as_ref() {
+            if let Some(handle) = guard.as_ref() {
+                return Ok(handle.data.clone());
+            }
+        }
+
+        bail!("Could not get underlying FeroxScans")
+    }
+}

src/event_handlers/filters.rs

@@ -0,0 +1,101 @@
+use super::*;
+use crate::{filters::FeroxFilters, CommandSender, FeroxChannel, Joiner};
+use anyhow::Result;
+use std::sync::Arc;
+use tokio::sync::{
+    mpsc::{self, UnboundedReceiver},
+    oneshot,
+};
+
+#[derive(Debug)]
+/// Container for filters transmitter and FeroxFilters object
+pub struct FiltersHandle {
+    /// FeroxFilters object used across modules to track active filters
+    pub data: Arc<FeroxFilters>,
+
+    /// transmitter used to update `data`
+    pub tx: CommandSender,
+}
+
+/// implementation of FiltersHandle
+impl FiltersHandle {
+    /// Given an Arc-wrapped FeroxFilters and CommandSender, create a new FiltersHandle
+    pub fn new(data: Arc<FeroxFilters>, tx: CommandSender) -> Self {
+        Self { data, tx }
+    }
+
+    /// Send the given Command over `tx`
+    pub fn send(&self, command: Command) -> Result<()> {
+        self.tx.send(command)?;
+        Ok(())
+    }
+
+    /// Sync the handle with the handler
+    pub async fn sync(&self) -> Result<()> {
+        let (tx, rx) = oneshot::channel::<bool>();
+        self.send(Command::Sync(tx))?;
+        rx.await?;
+        Ok(())
+    }
+}
+
+/// event handler for updating a single data structure of all active filters
+#[derive(Debug)]
+pub struct FiltersHandler {
+    /// collection of FeroxFilters
+    data: Arc<FeroxFilters>,
+
+    /// Receiver half of mpsc from which `Command`s are processed
+    receiver: UnboundedReceiver<Command>,
+}
+
+/// implementation of event handler for filters
+impl FiltersHandler {
+    /// create new event handler
+    pub fn new(data: Arc<FeroxFilters>, receiver: UnboundedReceiver<Command>) -> Self {
+        Self { data, receiver }
+    }
+
+    /// Initialize new `FeroxFilters` and the sc side of an mpsc channel that is responsible for
+    /// updates to the aforementioned object.
+    pub fn initialize() -> (Joiner, FiltersHandle) {
+        log::trace!("enter: initialize");
+
+        let data = Arc::new(FeroxFilters::default());
+        let (tx, rx): FeroxChannel<Command> = mpsc::unbounded_channel();
+
+        let mut handler = Self::new(data.clone(), rx);
+
+        let task = tokio::spawn(async move { handler.start().await });
+
+        let event_handle = FiltersHandle::new(data, tx);
+
+        log::trace!("exit: initialize -> ({:?}, {:?})", task, event_handle);
+
+        (task, event_handle)
+    }
+
+    /// Start a single consumer task (sc side of mpsc)
+    ///
+    /// The consumer simply receives `Command` and acts accordingly
+    pub async fn start(&mut self) -> Result<()> {
+        log::trace!("enter: start({:?})", self);
+
+        while let Some(command) = self.receiver.recv().await {
+            match command {
+                Command::AddFilter(filter) => {
+                    self.data.push(filter)?;
+                }
+                Command::Sync(sender) => {
+                    log::debug!("filters: {:?}", self);
+                    sender.send(true).unwrap_or_default();
+                }
+                Command::Exit => break,
+                _ => {} // no other commands needed for FilterHandler
+            }
+        }
+
+        log::trace!("exit: start");
+        Ok(())
+    }
+}

src/event_handlers/inputs.rs

@@ -0,0 +1,152 @@
+use super::*;
+use crate::{
+    progress::PROGRESS_PRINTER,
+    scan_manager::{FeroxState, PAUSE_SCAN},
+    scanner::RESPONSES,
+    statistics::StatError,
+    utils::{open_file, write_to},
+    SLEEP_DURATION,
+};
+use anyhow::Result;
+use console::style;
+use crossterm::event::{self, Event, KeyCode};
+use std::{
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        Arc,
+    },
+    thread::sleep,
+    time::Duration,
+    time::{SystemTime, UNIX_EPOCH},
+};
+
+/// Atomic boolean flag, used to determine whether or not the terminal input handler should exit
+pub static SCAN_COMPLETE: AtomicBool = AtomicBool::new(false);
+
+/// Container for filters transmitter and FeroxFilters object
+pub struct TermInputHandler {
+    /// handles to other handlers
+    handles: Arc<Handles>,
+}
+
+/// implementation of event handler for terminal input
+///
+/// kicks off the following handlers related to terminal input:
+///     ctrl+c handler that saves scan state to disk
+///     enter handler that listens for enter during scans to drop into interactive scan cancel menu
+impl TermInputHandler {
+    /// Create new event handler
+    pub fn new(handles: Arc<Handles>) -> Self {
+        Self { handles }
+    }
+
+    /// Initialize the sigint and enter handlers that are responsible for handling initial user
+    /// interaction during scans
+    pub fn initialize(handles: Arc<Handles>) {
+        log::trace!("enter: initialize({:?})", handles);
+
+        let handler = Self::new(handles);
+        handler.start();
+
+        log::trace!("exit: initialize");
+    }
+
+    /// wrapper around sigint_handler and enter_handler
+    fn start(&self) {
+        tokio::task::spawn_blocking(Self::enter_handler);
+
+        if self.handles.config.save_state {
+            // start the ctrl+c handler
+            let cloned = self.handles.clone();
+
+            let result = ctrlc::set_handler(move || {
+                let _ = Self::sigint_handler(cloned.clone());
+            });
+
+            if result.is_err() {
+                log::warn!("Could not set Ctrl+c handler; scan state will not be saved");
+                self.handles
+                    .stats
+                    .send(Command::AddError(StatError::Other))
+                    .unwrap_or_default();
+            }
+        }
+    }
+
+    /// Writes the current state of the program to disk (if save_state is true) and then exits
+    pub fn sigint_handler(handles: Arc<Handles>) -> Result<()> {
+        log::trace!("enter: sigint_handler({:?})", handles);
+
+        let ts = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs();
+
+        let slug = if !handles.config.target_url.is_empty() {
+            // target url populated
+            handles
+                .config
+                .target_url
+                .replace("://", "_")
+                .replace("/", "_")
+                .replace(".", "_")
+        } else {
+            // stdin used
+            "stdin".to_string()
+        };
+
+        let filename = format!("ferox-{}-{}.state", slug, ts);
+        let warning = format!(
+            "🚨 Caught {} 🚨 saving scan state to {} ...",
+            style("ctrl+c").yellow(),
+            filename
+        );
+
+        PROGRESS_PRINTER.println(warning);
+
+        let state = FeroxState::new(
+            handles.ferox_scans()?,
+            handles.config.clone(),
+            &RESPONSES,
+            handles.stats.data.clone(),
+        );
+
+        let state_file = open_file(&filename);
+
+        let mut buffered_file = state_file?;
+        write_to(&state, &mut buffered_file, true)?;
+
+        log::trace!("exit: sigint_handler (end of program)");
+        std::process::exit(1);
+    }
+
+    /// Handles specific key events triggered by the user over stdin
+    fn enter_handler() {
+        // todo eventually move away from atomics, the blocking recv is the problem
+        log::trace!("enter: start_enter_handler");
+
+        loop {
+            if PAUSE_SCAN.load(Ordering::Relaxed) {
+                // if the scan is already paused, we don't want this event poller fighting the user
+                // over stdin
+                sleep(Duration::from_millis(SLEEP_DURATION));
+            } else if event::poll(Duration::from_millis(SLEEP_DURATION)).unwrap_or(false) {
+                // It's guaranteed that the `read()` won't block when the `poll()`
+                // function returns `true`
+
+                if let Ok(key_pressed) = event::read() {
+                    // ignore any other keys
+                    if key_pressed == Event::Key(KeyCode::Enter.into()) {
+                        // if the user presses Enter, set PAUSE_SCAN to true. The interactive menu
+                        // will be triggered and will handle setting PAUSE_SCAN to false
+                        PAUSE_SCAN.store(true, Ordering::Release);
+                    }
+                }
+            } else {
+                // Timeout expired and no `Event` is available; use the timeout to check SCAN_COMPLETE
+                if SCAN_COMPLETE.load(Ordering::Relaxed) {
+                    // scan has been marked complete by main, time to exit the loop
+                    break;
+                }
+            }
+        }
+        log::trace!("exit: start_enter_handler");
+    }
+}

src/event_handlers/mod.rs

@@ -0,0 +1,16 @@
+//! collection of event handlers (typically long-running tokio spawned tasks)
+mod statistics;
+mod filters;
+mod container;
+mod command;
+mod outputs;
+mod scans;
+mod inputs;
+
+pub use self::command::Command;
+pub use self::container::{Handles, Tasks};
+pub use self::filters::{FiltersHandle, FiltersHandler};
+pub use self::inputs::{TermInputHandler, SCAN_COMPLETE};
+pub use self::outputs::{TermOutHandle, TermOutHandler};
+pub use self::scans::{ScanHandle, ScanHandler};
+pub use self::statistics::{StatsHandle, StatsHandler};

src/event_handlers/outputs.rs

@@ -0,0 +1,286 @@
+use super::Command::UpdateUsizeField;
+use super::*;
+
+use anyhow::{Context, Result};
+use tokio::sync::{mpsc, oneshot};
+
+use crate::{
+    config::Configuration,
+    progress::PROGRESS_PRINTER,
+    scanner::RESPONSES,
+    send_command, skip_fail,
+    statistics::StatField::ResourcesDiscovered,
+    traits::FeroxSerialize,
+    utils::{ferox_print, fmt_err, make_request, open_file, write_to},
+    CommandReceiver, CommandSender, Joiner,
+};
+use std::sync::Arc;
+
+#[derive(Debug)]
+/// Container for terminal output transmitter
+pub struct TermOutHandle {
+    /// Transmitter that sends to the TermOutHandler handler
+    pub tx: CommandSender,
+
+    /// Transmitter that sends to the FileOutHandler handler
+    pub tx_file: CommandSender,
+}
+
+/// implementation of OutputHandle
+impl TermOutHandle {
+    /// Given a CommandSender, create a new OutputHandle
+    pub fn new(tx: CommandSender, tx_file: CommandSender) -> Self {
+        Self { tx, tx_file }
+    }
+
+    /// Send the given Command over `tx`
+    pub fn send(&self, command: Command) -> Result<()> {
+        self.tx.send(command)?;
+        Ok(())
+    }
+
+    /// Sync the handle with the handler
+    pub async fn sync(&self, send_to_file: bool) -> Result<()> {
+        let (tx, rx) = oneshot::channel::<bool>();
+        self.send(Command::Sync(tx))?;
+
+        if send_to_file {
+            let (tx, rx) = oneshot::channel::<bool>();
+            self.tx_file.send(Command::Sync(tx))?;
+            rx.await?;
+        }
+
+        rx.await?;
+        Ok(())
+    }
+}
+
+#[derive(Debug)]
+/// Event handler for files
+pub struct FileOutHandler {
+    /// file output handler's receiver
+    receiver: CommandReceiver,
+
+    /// pointer to "global" configuration struct
+    config: Arc<Configuration>,
+}
+
+impl FileOutHandler {
+    /// Given a file tx/rx pair along with a filename and awaitable task, create
+    /// a FileOutHandler
+    fn new(rx: CommandReceiver, config: Arc<Configuration>) -> Self {
+        Self {
+            receiver: rx,
+            config,
+        }
+    }
+
+    /// Spawn a single consumer task (sc side of mpsc)
+    ///
+    /// The consumer simply receives responses from the terminal handler and writes them to disk
+    async fn start(&mut self, tx_stats: CommandSender) -> Result<()> {
+        log::trace!("enter: start_file_handler({:?})", tx_stats);
+
+        let mut file = open_file(&self.config.output)?;
+
+        log::info!("Writing scan results to {}", self.config.output);
+
+        while let Some(command) = self.receiver.recv().await {
+            match command {
+                Command::Report(response) => {
+                    skip_fail!(write_to(&*response, &mut file, self.config.json));
+                }
+                Command::Exit => {
+                    break;
+                }
+                Command::Sync(sender) => {
+                    skip_fail!(sender.send(true));
+                }
+                _ => {} // no more needed
+            }
+        }
+
+        // close the file before we tell statistics to save current data to the same file
+        drop(file);
+
+        send_command!(tx_stats, Command::Save);
+
+        log::trace!("exit: start_file_handler");
+        Ok(())
+    }
+}
+
+#[derive(Debug)]
+/// Event handler for terminal
+pub struct TermOutHandler {
+    /// terminal output handler's receiver
+    receiver: CommandReceiver,
+
+    /// file handler
+    tx_file: CommandSender,
+
+    /// optional file handler task
+    file_task: Option<Joiner>,
+
+    /// pointer to "global" configuration struct
+    config: Arc<Configuration>,
+}
+
+/// implementation of TermOutHandler
+impl TermOutHandler {
+    /// Given a terminal receiver along with a file transmitter and filename, create
+    /// an OutputHandler
+    fn new(
+        receiver: CommandReceiver,
+        tx_file: CommandSender,
+        file_task: Option<Joiner>,
+        config: Arc<Configuration>,
+    ) -> Self {
+        Self {
+            receiver,
+            tx_file,
+            config,
+            file_task,
+        }
+    }
+
+    /// Creates all required output handlers (terminal, file) and updates the given Handles/Tasks
+    pub fn initialize(
+        config: Arc<Configuration>,
+        tx_stats: CommandSender,
+    ) -> (Joiner, TermOutHandle) {
+        log::trace!("enter: initialize({:?}, {:?})", config, tx_stats);
+
+        let (tx_term, rx_term) = mpsc::unbounded_channel::<Command>();
+        let (tx_file, rx_file) = mpsc::unbounded_channel::<Command>();
+
+        let mut file_handler = FileOutHandler::new(rx_file, config.clone());
+
+        let tx_stats_clone = tx_stats.clone();
+
+        let file_task = if !config.output.is_empty() {
+            // -o used, need to spawn the thread for writing to disk
+            Some(tokio::spawn(async move {
+                file_handler.start(tx_stats_clone).await
+            }))
+        } else {
+            None
+        };
+
+        let mut term_handler = Self::new(rx_term, tx_file.clone(), file_task, config);
+        let term_task = tokio::spawn(async move { term_handler.start(tx_stats).await });
+
+        let event_handle = TermOutHandle::new(tx_term, tx_file);
+
+        log::trace!("exit: initialize -> ({:?}, {:?})", term_task, event_handle);
+
+        (term_task, event_handle)
+    }
+
+    /// Start a single consumer task (sc side of mpsc)
+    ///
+    /// The consumer simply receives `Command` and acts accordingly
+    async fn start(&mut self, tx_stats: CommandSender) -> Result<()> {
+        log::trace!("enter: start({:?})", tx_stats);
+
+        while let Some(command) = self.receiver.recv().await {
+            match command {
+                Command::Report(mut resp) => {
+                    let contains_sentry =
+                        self.config.status_codes.contains(&resp.status().as_u16());
+                    let unknown_sentry = !RESPONSES.contains(&resp); // !contains == unknown
+                    let should_process_response = contains_sentry && unknown_sentry;
+
+                    if should_process_response {
+                        // print to stdout
+                        ferox_print(&resp.as_str(), &PROGRESS_PRINTER);
+
+                        send_command!(tx_stats, UpdateUsizeField(ResourcesDiscovered, 1));
+
+                        if self.file_task.is_some() {
+                            // -o used, need to send the report to be written out to disk
+                            self.tx_file
+                                .send(Command::Report(resp.clone()))
+                                .with_context(|| {
+                                    fmt_err(&format!("Could not send {} to file handler", resp))
+                                })?;
+                        }
+                    }
+                    log::trace!("report complete: {}", resp.url());
+
+                    if self.config.replay_client.is_some() && should_process_response {
+                        // replay proxy specified/client created and this response's status code is one that
+                        // should be replayed
+                        make_request(
+                            self.config.replay_client.as_ref().unwrap(),
+                            &resp.url(),
+                            self.config.output_level,
+                            tx_stats.clone(),
+                        )
+                        .await
+                        .with_context(|| "Could not replay request through replay proxy")?;
+                    }
+
+                    if should_process_response {
+                        // add response to RESPONSES for serialization in case of ctrl+c
+                        // placed all by its lonesome like this so that RESPONSES can take ownership
+                        // of the FeroxResponse
+
+                        // before ownership is transferred, there's no real reason to keep the body anymore
+                        // so we can free that piece of data, reducing memory usage
+                        resp.drop_text();
+
+                        RESPONSES.insert(*resp);
+                    }
+                }
+                Command::Sync(sender) => {
+                    sender.send(true).unwrap_or_default();
+                }
+                Command::Exit => {
+                    if self.file_task.is_some() && self.tx_file.send(Command::Exit).is_ok() {
+                        self.file_task.as_mut().unwrap().await??; // wait for death
+                    }
+                    break;
+                }
+                _ => {} // no more commands needed
+            }
+        }
+        log::trace!("exit: start");
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    /// try to hit struct field coverage of FileOutHandler
+    fn struct_fields_of_file_out_handler() {
+        let (_, rx) = mpsc::unbounded_channel::<Command>();
+        let config = Arc::new(Configuration::new().unwrap());
+        let foh = FileOutHandler {
+            config,
+            receiver: rx,
+        };
+        println!("{:?}", foh);
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// try to hit struct field coverage of TermOutHandler
+    async fn struct_fields_of_term_out_handler() {
+        let (tx, rx) = mpsc::unbounded_channel::<Command>();
+        let (tx_file, _) = mpsc::unbounded_channel::<Command>();
+        let config = Arc::new(Configuration::new().unwrap());
+
+        let toh = TermOutHandler {
+            config,
+            file_task: None,
+            receiver: rx,
+            tx_file,
+        };
+
+        println!("{:?}", toh);
+        tx.send(Command::Exit).unwrap();
+    }
+}

src/event_handlers/scans.rs

@@ -0,0 +1,274 @@
+use std::collections::HashSet;
+use std::sync::Arc;
+
+use anyhow::{bail, Result};
+use tokio::sync::{mpsc, Semaphore};
+
+use crate::response::FeroxResponse;
+use crate::url::FeroxUrl;
+use crate::{
+    scan_manager::{FeroxScan, FeroxScans, ScanOrder},
+    scanner::FeroxScanner,
+    statistics::StatField::TotalScans,
+    CommandReceiver, CommandSender, FeroxChannel, Joiner,
+};
+
+use super::command::Command::UpdateUsizeField;
+use super::*;
+
+#[derive(Debug)]
+/// Container for recursion transmitter and FeroxScans object
+pub struct ScanHandle {
+    /// FeroxScans object used across modules to track scans
+    pub data: Arc<FeroxScans>,
+
+    /// transmitter used to update `data`
+    pub tx: CommandSender,
+}
+
+/// implementation of RecursionHandle
+impl ScanHandle {
+    /// Given an Arc-wrapped FeroxScans and CommandSender, create a new RecursionHandle
+    pub fn new(data: Arc<FeroxScans>, tx: CommandSender) -> Self {
+        Self { data, tx }
+    }
+
+    /// Send the given Command over `tx`
+    pub fn send(&self, command: Command) -> Result<()> {
+        self.tx.send(command)?;
+        Ok(())
+    }
+}
+
+/// event handler for updating a single data structure of all FeroxScans
+#[derive(Debug)]
+pub struct ScanHandler {
+    /// collection of FeroxScans
+    data: Arc<FeroxScans>,
+
+    /// handles to other handlers needed to kick off a scan while already past main
+    handles: Arc<Handles>,
+
+    /// Receiver half of mpsc from which `Command`s are processed
+    receiver: CommandReceiver,
+
+    /// wordlist (re)used for each scan
+    wordlist: std::sync::Mutex<Option<Arc<HashSet<String>>>>,
+
+    /// group of scans that need to be joined
+    tasks: Vec<Arc<FeroxScan>>,
+
+    /// Maximum recursion depth, a depth of 0 is infinite recursion
+    max_depth: usize,
+
+    /// depths associated with the initial targets provided by the user
+    depths: Vec<(String, usize)>,
+
+    /// Bounded semaphore used as a barrier to limit concurrent scans
+    limiter: Arc<Semaphore>,
+}
+
+/// implementation of event handler for filters
+impl ScanHandler {
+    /// create new event handler
+    pub fn new(
+        data: Arc<FeroxScans>,
+        handles: Arc<Handles>,
+        max_depth: usize,
+        receiver: CommandReceiver,
+    ) -> Self {
+        let limit = handles.config.scan_limit;
+        let limiter = Semaphore::new(limit);
+
+        if limit == 0 {
+            // scan_limit == 0 means no limit should be imposed... however, scoping the Semaphore
+            // permit is tricky, so as a workaround, we'll add a ridiculous number of permits to
+            // the semaphore (1,152,921,504,606,846,975 to be exact) and call that 'unlimited'
+
+            // note to self: the docs say max is usize::MAX >> 3, however, threads will panic if
+            // that value is used (says adding (1) will overflow the semaphore, even though none
+            // are being added...)
+            limiter.add_permits(usize::MAX >> 4);
+        }
+
+        Self {
+            data,
+            handles,
+            receiver,
+            max_depth,
+            tasks: Vec::new(),
+            depths: Vec::new(),
+            limiter: Arc::new(limiter),
+            wordlist: std::sync::Mutex::new(None),
+        }
+    }
+
+    /// Set the wordlist
+    fn wordlist(&self, wordlist: Arc<HashSet<String>>) {
+        if let Ok(mut guard) = self.wordlist.lock() {
+            if guard.is_none() {
+                let _ = std::mem::replace(&mut *guard, Some(wordlist));
+            }
+        }
+    }
+
+    /// Initialize new `FeroxScans` and the sc side of an mpsc channel that is responsible for
+    /// updates to the aforementioned object.
+    pub fn initialize(handles: Arc<Handles>) -> (Joiner, ScanHandle) {
+        log::trace!("enter: initialize");
+
+        let data = Arc::new(FeroxScans::new(handles.config.output_level));
+        let (tx, rx): FeroxChannel<Command> = mpsc::unbounded_channel();
+
+        let max_depth = handles.config.depth;
+
+        let mut handler = Self::new(data.clone(), handles, max_depth, rx);
+
+        let task = tokio::spawn(async move { handler.start().await });
+
+        let event_handle = ScanHandle::new(data, tx);
+
+        log::trace!("exit: initialize -> ({:?}, {:?})", task, event_handle);
+
+        (task, event_handle)
+    }
+
+    /// Start a single consumer task (sc side of mpsc)
+    ///
+    /// The consumer simply receives `Command` and acts accordingly
+    pub async fn start(&mut self) -> Result<()> {
+        log::trace!("enter: start({:?})", self);
+
+        while let Some(command) = self.receiver.recv().await {
+            match command {
+                Command::ScanInitialUrls(targets) => {
+                    self.ordered_scan_url(targets, ScanOrder::Initial).await?;
+                }
+                Command::UpdateWordlist(wordlist) => {
+                    self.wordlist(wordlist);
+                }
+                Command::JoinTasks(sender) => {
+                    let ferox_scans = self.handles.ferox_scans().unwrap_or_default();
+                    let limiter_clone = self.limiter.clone();
+
+                    tokio::spawn(async move {
+                        while ferox_scans.has_active_scans() {
+                            for scan in ferox_scans.get_active_scans() {
+                                scan.join().await;
+                            }
+                        }
+                        limiter_clone.close();
+                        sender.send(true).expect("oneshot channel failed");
+                    });
+                }
+                Command::TryRecursion(response) => {
+                    self.try_recursion(response).await?;
+                }
+                Command::Sync(sender) => {
+                    sender.send(true).unwrap_or_default();
+                }
+                _ => {} // no other commands needed for RecursionHandler
+            }
+        }
+
+        log::trace!("exit: start");
+        Ok(())
+    }
+
+    /// Helper to easily get the (locked) underlying wordlist
+    pub fn get_wordlist(&self) -> Result<Arc<HashSet<String>>> {
+        if let Ok(guard) = self.wordlist.lock().as_ref() {
+            if let Some(list) = guard.as_ref() {
+                return Ok(list.clone());
+            }
+        }
+
+        bail!("Could not get underlying wordlist")
+    }
+
+    /// wrapper around scanning a url to stay DRY
+    async fn ordered_scan_url(&mut self, targets: Vec<String>, order: ScanOrder) -> Result<()> {
+        log::trace!("enter: ordered_scan_url({:?}, {:?})", targets, order);
+
+        for target in targets {
+            if self.data.contains(&target) && matches!(order, ScanOrder::Latest) {
+                // FeroxScans knows about this url and scan isn't an Initial scan
+                // initial scans are skipped because when resuming from a .state file, the scans
+                // will already be populated in FeroxScans, so we need to not skip kicking off
+                // their scans
+                continue;
+            }
+
+            let scan = if let Some(ferox_scan) = self.data.get_scan_by_url(&target) {
+                ferox_scan // scan already known
+            } else {
+                self.data.add_directory_scan(&target, order).1 // add the new target; return FeroxScan
+            };
+
+            let list = self.get_wordlist()?;
+
+            log::info!("scan handler received {} - beginning scan", target);
+
+            if matches!(order, ScanOrder::Initial) {
+                // keeps track of the initial targets' scan depths in order to enforce the
+                // maximum recursion depth on any identified sub-directories
+                let url = FeroxUrl::from_string(&target, self.handles.clone());
+                let depth = url.depth().unwrap_or(0);
+                self.depths.push((target.clone(), depth));
+            }
+
+            let scanner = FeroxScanner::new(
+                &target,
+                order,
+                list,
+                self.limiter.clone(),
+                self.handles.clone(),
+            );
+
+            let task = tokio::spawn(async move {
+                if let Err(e) = scanner.scan_url().await {
+                    log::warn!("{}", e);
+                }
+            });
+
+            self.handles.stats.send(UpdateUsizeField(TotalScans, 1))?;
+
+            scan.set_task(task).await?;
+
+            self.tasks.push(scan.clone());
+        }
+
+        log::trace!("exit: ordered_scan_url");
+        Ok(())
+    }
+
+    async fn try_recursion(&mut self, response: Box<FeroxResponse>) -> Result<()> {
+        log::trace!("enter: try_recursion({:?})", response,);
+
+        let mut base_depth = 1_usize;
+
+        for (base_url, base_url_depth) in &self.depths {
+            if response.url().as_str().starts_with(base_url) {
+                base_depth = *base_url_depth;
+            }
+        }
+
+        if response.reached_max_depth(base_depth, self.max_depth, self.handles.clone()) {
+            // at or past recursion depth
+            return Ok(());
+        }
+
+        if !response.is_directory() {
+            // not a directory
+            return Ok(());
+        }
+
+        let targets = vec![response.url().to_string()];
+        self.ordered_scan_url(targets, ScanOrder::Latest).await?;
+
+        log::info!("Added new directory to recursive scan: {}", response.url());
+
+        log::trace!("exit: try_recursion");
+        Ok(())
+    }
+}

src/event_handlers/statistics.rs

@@ -0,0 +1,163 @@
+use super::*;
+use crate::{
+    config::Configuration,
+    progress::{add_bar, BarType},
+    statistics::{StatField, Stats},
+    CommandSender, FeroxChannel, Joiner,
+};
+use anyhow::Result;
+use console::style;
+use indicatif::ProgressBar;
+use std::{sync::Arc, time::Instant};
+use tokio::sync::{
+    mpsc::{self, UnboundedReceiver},
+    oneshot,
+};
+
+#[derive(Debug)]
+/// Container for statistics transmitter and Stats object
+pub struct StatsHandle {
+    /// Stats object used across modules to track statistics
+    pub data: Arc<Stats>,
+
+    /// transmitter used to update `data`
+    pub tx: CommandSender,
+}
+
+/// implementation of StatsHandle
+impl StatsHandle {
+    /// Given an Arc-wrapped Stats and CommandSender, create a new StatsHandle
+    pub fn new(data: Arc<Stats>, tx: CommandSender) -> Self {
+        Self { data, tx }
+    }
+
+    /// Send the given Command over `tx`
+    pub fn send(&self, command: Command) -> Result<()> {
+        self.tx.send(command)?;
+        Ok(())
+    }
+
+    /// Sync the handle with the handler
+    pub async fn sync(&self) -> Result<()> {
+        let (tx, rx) = oneshot::channel::<bool>();
+        self.send(Command::Sync(tx))?;
+        rx.await?;
+        Ok(())
+    }
+}
+
+/// event handler struct for updating statistics
+#[derive(Debug)]
+pub struct StatsHandler {
+    /// overall scan's progress bar
+    bar: ProgressBar,
+
+    /// Receiver half of mpsc from which `StatCommand`s are processed
+    receiver: UnboundedReceiver<Command>,
+
+    /// data class that stores all statistics updates
+    stats: Arc<Stats>,
+}
+
+/// implementation of event handler for statistics
+impl StatsHandler {
+    /// create new event handler
+    fn new(stats: Arc<Stats>, rx_stats: UnboundedReceiver<Command>) -> Self {
+        // will be updated later via StatCommand; delay is for banner to print first
+        let bar = ProgressBar::hidden();
+
+        Self {
+            bar,
+            stats,
+            receiver: rx_stats,
+        }
+    }
+
+    /// Start a single consumer task (sc side of mpsc)
+    ///
+    /// The consumer simply receives `StatCommands` and updates the given `Stats` object as appropriate
+    async fn start(&mut self, output_file: &str) -> Result<()> {
+        log::trace!("enter: start({:?})", self);
+
+        let start = Instant::now();
+
+        while let Some(command) = self.receiver.recv().await {
+            match command as Command {
+                Command::AddError(err) => {
+                    self.stats.add_error(err);
+                    self.increment_bar();
+                }
+                Command::AddStatus(status) => {
+                    self.stats.add_status_code(status);
+                    self.increment_bar();
+                }
+                Command::AddRequest => {
+                    self.stats.add_request();
+                    self.increment_bar();
+                }
+                Command::Save => {
+                    self.stats
+                        .save(start.elapsed().as_secs_f64(), output_file)?;
+                }
+                Command::UpdateUsizeField(field, value) => {
+                    self.stats.update_usize_field(field, value);
+
+                    if matches!(field, StatField::TotalScans) {
+                        self.bar.set_length(self.stats.total_expected() as u64);
+                    }
+                }
+                Command::UpdateF64Field(field, value) => self.stats.update_f64_field(field, value),
+                Command::CreateBar => {
+                    self.bar = add_bar("", self.stats.total_expected() as u64, BarType::Total);
+                }
+                Command::LoadStats(filename) => {
+                    self.stats.merge_from(&filename)?;
+                }
+                Command::Sync(sender) => {
+                    sender.send(true).unwrap_or_default();
+                }
+                Command::Exit => break,
+                _ => {} // no more commands needed
+            }
+        }
+
+        self.bar.finish();
+
+        log::debug!("{:#?}", *self.stats);
+        log::trace!("exit: start");
+        Ok(())
+    }
+
+    /// Wrapper around incrementing the overall scan's progress bar
+    fn increment_bar(&self) {
+        let msg = format!(
+            "{}:{:<7} {}:{:<7}",
+            style("found").green(),
+            self.stats.resources_discovered(),
+            style("errors").red(),
+            self.stats.errors(),
+        );
+
+        self.bar.set_message(&msg);
+        self.bar.inc(1);
+    }
+
+    /// Initialize new `Stats` object and the sc side of an mpsc channel that is responsible for
+    /// updates to the aforementioned object.
+    pub fn initialize(config: Arc<Configuration>) -> (Joiner, StatsHandle) {
+        log::trace!("enter: initialize");
+
+        let data = Arc::new(Stats::new(config.extensions.len(), config.json));
+        let (tx, rx): FeroxChannel<Command> = mpsc::unbounded_channel();
+
+        let mut handler = StatsHandler::new(data.clone(), rx);
+
+        let task = tokio::spawn(async move { handler.start(&config.output).await });
+
+        let event_handle = StatsHandle::new(data, tx);
+
+        log::trace!("exit: initialize -> ({:?}, {:?})", task, event_handle);
+
+        (task, event_handle)
+    }
+}

src/extractor.rs

@@ -1,269 +0,0 @@
-use crate::FeroxResponse;
-use lazy_static::lazy_static;
-use regex::Regex;
-use reqwest::Url;
-use std::collections::HashSet;
-
-/// Regular expression used in [LinkFinder](https://github.com/GerbenJavado/LinkFinder)
-///
-/// Incorporates change from this [Pull Request](https://github.com/GerbenJavado/LinkFinder/pull/66/files)
-const LINKFINDER_REGEX: &str = r#"(?:"|')(((?:[a-zA-Z]{1,10}://|//)[^"'/]{1,}\.[a-zA-Z]{2,}[^"']{0,})|((?:/|\.\./|\./)[^"'><,;| *()(%%$^/\\\[\]][^"'><,;|()]{1,})|([a-zA-Z0-9_\-/]{1,}/[a-zA-Z0-9_\-/]{1,}\.(?:[a-zA-Z]{1,4}|action)(?:[\?|#][^"|']{0,}|))|([a-zA-Z0-9_\-/]{1,}/[a-zA-Z0-9_\-/]{3,}(?:[\?|#][^"|']{0,}|))|([a-zA-Z0-9_\-.]{1,}\.(?:php|asp|aspx|jsp|json|action|html|js|txt|xml)(?:[\?|#][^"|']{0,}|)))(?:"|')"#;
-
-lazy_static! {
-    /// `LINKFINDER_REGEX` as a regex::Regex type
-    static ref REGEX: Regex = Regex::new(LINKFINDER_REGEX).unwrap();
-}
-
-/// Iterate over a given path, return a list of every sub-path found
-///
-/// example: `path` contains a link fragment `homepage/assets/img/icons/handshake.svg`
-/// the following fragments would be returned:
-///   - homepage/assets/img/icons/handshake.svg
-///   - homepage/assets/img/icons/
-///   - homepage/assets/img/
-///   - homepage/assets/
-///   - homepage/
-fn get_sub_paths_from_path(path: &str) -> Vec<String> {
-    log::trace!("enter: get_sub_paths_from_path({})", path);
-    let mut paths = vec![];
-
-    // filter out any empty strings caused by .split
-    let mut parts: Vec<&str> = path.split('/').filter(|s| !s.is_empty()).collect();
-
-    let length = parts.len();
-
-    for _ in 0..length {
-        // iterate over all parts of the path
-        if parts.is_empty() {
-            // pop left us with an empty vector, we're done
-            break;
-        }
-
-        let possible_path = parts.join("/");
-
-        if possible_path.is_empty() {
-            // .join can result in an empty string, which we don't need, ignore
-            continue;
-        }
-
-        paths.push(possible_path); // good sub-path found
-        parts.pop(); // use .pop() to remove the last part of the path and continue iteration
-    }
-
-    log::trace!("exit: get_sub_paths_from_path -> {:?}", paths);
-    paths
-}
-
-/// simple helper to stay DRY, trys to join a url + fragment and add it to the `links` HashSet
-fn add_link_to_set_of_links(link: &str, url: &Url, links: &mut HashSet<String>) {
-    log::trace!(
-        "enter: add_link_to_set_of_links({}, {}, {:?})",
-        link,
-        url.to_string(),
-        links
-    );
-    match url.join(&link) {
-        Ok(new_url) => {
-            links.insert(new_url.to_string());
-        }
-        Err(e) => {
-            log::error!("Could not join given url to the base url: {}", e);
-        }
-    }
-    log::trace!("exit: add_link_to_set_of_links");
-}
-
-/// Given a `reqwest::Response`, perform the following actions
-///   - parse the response's text for links using the linkfinder regex
-///   - for every link found take its url path and parse each sub-path
-///     - example: Response contains a link fragment `homepage/assets/img/icons/handshake.svg`
-///       with a base url of http://localhost, the following urls would be returned:
-///         - homepage/assets/img/icons/handshake.svg
-///         - homepage/assets/img/icons/
-///         - homepage/assets/img/
-///         - homepage/assets/
-///         - homepage/
-pub async fn get_links(response: &FeroxResponse) -> HashSet<String> {
-    log::trace!("enter: get_links({})", response.url().as_str());
-
-    let mut links = HashSet::<String>::new();
-
-    let body = response.text();
-
-    for capture in REGEX.captures_iter(&body) {
-        // remove single & double quotes from both ends of the capture
-        // capture[0] is the entire match, additional capture groups start at [1]
-        let link = capture[0].trim_matches(|c| c == '\'' || c == '"');
-
-        match Url::parse(link) {
-            Ok(absolute) => {
-                if absolute.domain() != response.url().domain()
-                    || absolute.host() != response.url().host()
-                {
-                    // domains/ips are not the same, don't scan things that aren't part of the original
-                    // target url
-                    continue;
-                }
-
-                for sub_path in get_sub_paths_from_path(absolute.path()) {
-                    // take a url fragment like homepage/assets/img/icons/handshake.svg and
-                    // incrementally add
-                    //     - homepage/assets/img/icons/
-                    //     - homepage/assets/img/
-                    //     - homepage/assets/
-                    //     - homepage/
-                    log::debug!("Adding {} to {:?}", sub_path, links);
-                    add_link_to_set_of_links(&sub_path, &response.url(), &mut links);
-                }
-            }
-            Err(e) => {
-                // this is the expected error that happens when we try to parse a url fragment
-                //     ex: Url::parse("/login") -> Err("relative URL without a base")
-                // while this is technically an error, these are good results for us
-                if e.to_string().contains("relative URL without a base") {
-                    for sub_path in get_sub_paths_from_path(link) {
-                        // incrementally save all sub-paths that led to the relative url's resource
-                        log::debug!("Adding {} to {:?}", sub_path, links);
-                        add_link_to_set_of_links(&sub_path, &response.url(), &mut links);
-                    }
-                } else {
-                    // unexpected error has occurred
-                    log::error!("Could not parse given url: {}", e);
-                }
-            }
-        }
-    }
-
-    log::trace!("exit: get_links -> {:?}", links);
-    links
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::utils::make_request;
-    use httpmock::Method::GET;
-    use httpmock::{Mock, MockServer};
-    use reqwest::Client;
-
-    #[test]
-    /// extract sub paths from the given url fragment; expect 4 sub paths and that all are
-    /// in the expected array
-    fn extractor_get_sub_paths_from_path_with_multiple_paths() {
-        let path = "homepage/assets/img/icons/handshake.svg";
-        let paths = get_sub_paths_from_path(&path);
-        let expected = vec![
-            "homepage",
-            "homepage/assets",
-            "homepage/assets/img",
-            "homepage/assets/img/icons",
-            "homepage/assets/img/icons/handshake.svg",
-        ];
-
-        assert_eq!(paths.len(), expected.len());
-        for expected_path in expected {
-            assert_eq!(paths.contains(&expected_path.to_string()), true);
-        }
-    }
-
-    #[test]
-    /// extract sub paths from the given url fragment; expect 2 sub paths and that all are
-    /// in the expected array. the fragment is wrapped in slashes to ensure no empty strings are
-    /// returned
-    fn extractor_get_sub_paths_from_path_with_enclosing_slashes() {
-        let path = "/homepage/assets/";
-        let paths = get_sub_paths_from_path(&path);
-        let expected = vec!["homepage", "homepage/assets"];
-
-        assert_eq!(paths.len(), expected.len());
-        for expected_path in expected {
-            assert_eq!(paths.contains(&expected_path.to_string()), true);
-        }
-    }
-
-    #[test]
-    /// extract sub paths from the given url fragment; expect 1 sub path, no forward slashes are
-    /// included
-    fn extractor_get_sub_paths_from_path_with_only_a_word() {
-        let path = "homepage";
-        let paths = get_sub_paths_from_path(&path);
-        let expected = vec!["homepage"];
-
-        assert_eq!(paths.len(), expected.len());
-        for expected_path in expected {
-            assert_eq!(paths.contains(&expected_path.to_string()), true);
-        }
-    }
-
-    #[test]
-    /// extract sub paths from the given url fragment; expect 1 sub path, forward slash removed
-    fn extractor_get_sub_paths_from_path_with_an_absolute_word() {
-        let path = "/homepage";
-        let paths = get_sub_paths_from_path(&path);
-        let expected = vec!["homepage"];
-
-        assert_eq!(paths.len(), expected.len());
-        for expected_path in expected {
-            assert_eq!(paths.contains(&expected_path.to_string()), true);
-        }
-    }
-
-    #[test]
-    /// test that a full url and fragment are joined correctly, then added to the given list
-    /// i.e. the happy path
-    fn extractor_add_link_to_set_of_links_happy_path() {
-        let url = Url::parse("https://localhost").unwrap();
-        let mut links = HashSet::<String>::new();
-        let link = "admin";
-
-        assert_eq!(links.len(), 0);
-        add_link_to_set_of_links(link, &url, &mut links);
-
-        assert_eq!(links.len(), 1);
-        assert!(links.contains("https://localhost/admin"));
-    }
-
-    #[test]
-    /// test that an invalid path fragment doesn't add anything to the set of links
-    fn extractor_add_link_to_set_of_links_with_non_base_url() {
-        let url = Url::parse("https://localhost").unwrap();
-        let mut links = HashSet::<String>::new();
-        let link = "\\\\\\\\";
-
-        assert_eq!(links.len(), 0);
-        add_link_to_set_of_links(link, &url, &mut links);
-
-        assert_eq!(links.len(), 0);
-        assert!(links.is_empty());
-    }
-
-    #[tokio::test(core_threads = 1)]
-    /// use make_request to generate a Response, and use the Response to test get_links;
-    /// the response will contain an absolute path to a domain that is not part of the scanned
-    /// domain; expect an empty set returned
-    async fn extractor_get_links_with_absolute_url_that_differs_from_target_domain(
-    ) -> Result<(), Box<dyn std::error::Error>> {
-        let srv = MockServer::start();
-
-        let mock = Mock::new()
-            .expect_method(GET)
-            .expect_path("/some-path")
-            .return_status(200)
-            .return_body("\"http://defintely.not.a.thing.probably.com/homepage/assets/img/icons/handshake.svg\"")
-            .create_on(&srv);
-
-        let client = Client::new();
-        let url = Url::parse(&srv.url("/some-path")).unwrap();
-
-        let response = make_request(&client, &url).await.unwrap();
-
-        let ferox_response = FeroxResponse::from(response, true).await;
-
-        let links = get_links(&ferox_response).await;
-
-        assert!(links.is_empty());
-
-        assert_eq!(mock.times_called(), 1);
-        Ok(())
-    }
-}

src/extractor/builder.rs

@@ -0,0 +1,100 @@
+use super::*;
+use crate::event_handlers::Handles;
+use anyhow::{bail, Result};
+
+/// Regular expression used in [LinkFinder](https://github.com/GerbenJavado/LinkFinder)
+///
+/// Incorporates change from this [Pull Request](https://github.com/GerbenJavado/LinkFinder/pull/66/files)
+pub(super) const LINKFINDER_REGEX: &str = r#"(?:"|')(((?:[a-zA-Z]{1,10}://|//)[^"'/]{1,}\.[a-zA-Z]{2,}[^"']{0,})|((?:/|\.\./|\./)[^"'><,;| *()(%%$^/\\\[\]][^"'><,;|()]{1,})|([a-zA-Z0-9_\-/]{1,}/[a-zA-Z0-9_\-/]{1,}\.(?:[a-zA-Z]{1,4}|action)(?:[\?|#][^"|']{0,}|))|([a-zA-Z0-9_\-/]{1,}/[a-zA-Z0-9_\-/]{3,}(?:[\?|#][^"|']{0,}|))|([a-zA-Z0-9_\-.]{1,}\.(?:php|asp|aspx|jsp|json|action|html|js|txt|xml)(?:[\?|#][^"|']{0,}|)))(?:"|')"#;
+
+/// Regular expression to pull url paths from robots.txt
+///
+/// ref: https://developers.google.com/search/reference/robots_txt
+pub(super) const ROBOTS_TXT_REGEX: &str =
+    r#"(?m)^ *(Allow|Disallow): *(?P<url_path>[a-zA-Z0-9._/?#@!&'()+,;%=-]+?)$"#; // multi-line (?m)
+
+/// Which type of extraction should be performed
+#[derive(Debug, Copy, Clone)]
+pub enum ExtractionTarget {
+    /// Examine a response body and extract links
+    ResponseBody,
+
+    /// Examine robots.txt (specifically) and extract links
+    RobotsTxt,
+}
+
+/// responsible for building an `Extractor`
+pub struct ExtractorBuilder<'a> {
+    /// Response from which to extract links
+    response: Option<&'a FeroxResponse>,
+
+    /// Response from which to extract links
+    url: String,
+
+    /// Handles object to house the underlying mpsc transmitters
+    handles: Option<Arc<Handles>>,
+
+    /// type of extraction to be performed
+    target: ExtractionTarget,
+}
+
+/// ExtractorBuilder implementation
+impl<'a> Default for ExtractorBuilder<'a> {
+    fn default() -> Self {
+        Self {
+            response: None,
+            url: "".to_string(),
+            handles: None,
+            target: ExtractionTarget::ResponseBody,
+        }
+    }
+}
+
+/// ExtractorBuilder implementation
+impl<'a> ExtractorBuilder<'a> {
+    /// builder call to set `handles`
+    pub fn handles(&mut self, handles: Arc<Handles>) -> &mut Self {
+        self.handles = Some(handles);
+        self
+    }
+
+    /// builder call to set `url`
+    pub fn url(&mut self, url: &str) -> &mut Self {
+        self.url = url.to_string();
+        self
+    }
+
+    /// builder call to set `target`
+    pub fn target(&mut self, target: ExtractionTarget) -> &mut Self {
+        self.target = target;
+        self
+    }
+
+    /// builder call to set `response`
+    pub fn response(&mut self, response: &'a FeroxResponse) -> &mut Self {
+        self.response = Some(response);
+        self
+    }
+
+    /// finalize configuration of ExtratorBuilder and return an Extractor
+    ///
+    /// requires either with_url or with_response to have been used in the build process
+    pub fn build(&self) -> Result<Extractor<'a>> {
+        if (self.url.is_empty() && self.response.is_none()) || self.handles.is_none() {
+            bail!("Extractor requires a URL or a FeroxResponse be specified as well as a Handles object")
+        }
+
+        Ok(Extractor {
+            links_regex: Regex::new(LINKFINDER_REGEX).unwrap(),
+            robots_regex: Regex::new(ROBOTS_TXT_REGEX).unwrap(),
+            response: if self.response.is_some() {
+                Some(self.response.unwrap())
+            } else {
+                None
+            },
+            url: self.url.to_owned(),
+            handles: self.handles.as_ref().unwrap().clone(),
+            target: self.target,
+        })
+    }
+}

src/extractor/container.rs

@@ -0,0 +1,414 @@
+use super::*;
+use crate::{
+    client,
+    event_handlers::{
+        Command,
+        Command::{AddError, UpdateUsizeField},
+        Handles,
+    },
+    scan_manager::ScanOrder,
+    statistics::{
+        StatError::Other,
+        StatField::{LinksExtracted, TotalExpected},
+    },
+    url::FeroxUrl,
+    utils::make_request,
+};
+use anyhow::{bail, Context, Result};
+use reqwest::{StatusCode, Url};
+use std::collections::HashSet;
+use tokio::sync::oneshot;
+
+/// Whether an active scan is recursive or not
+#[derive(Debug)]
+enum RecursionStatus {
+    /// Scan is recursive
+    Recursive,
+
+    /// Scan is not recursive
+    NotRecursive,
+}
+
+/// Handles all logic related to extracting links from requested source code
+#[derive(Debug)]
+pub struct Extractor<'a> {
+    /// `LINKFINDER_REGEX` as a regex::Regex type
+    pub(super) links_regex: Regex,
+
+    /// `ROBOTS_TXT_REGEX` as a regex::Regex type
+    pub(super) robots_regex: Regex,
+
+    /// Response from which to extract links
+    pub(super) response: Option<&'a FeroxResponse>,
+
+    /// Response from which to extract links
+    pub(super) url: String,
+
+    /// Handles object to house the underlying mpsc transmitters
+    pub(super) handles: Arc<Handles>,
+
+    /// type of extraction to be performed
+    pub(super) target: ExtractionTarget,
+}
+
+/// Extractor implementation
+impl<'a> Extractor<'a> {
+    /// business logic that handles getting links from a normal http body response
+    pub async fn extract(&self) -> Result<()> {
+        let links = match self.target {
+            ExtractionTarget::ResponseBody => self.extract_from_body().await?,
+            ExtractionTarget::RobotsTxt => self.extract_from_robots().await?,
+        };
+
+        let recursive = if self.handles.config.no_recursion {
+            RecursionStatus::NotRecursive
+        } else {
+            RecursionStatus::Recursive
+        };
+
+        let scanned_urls = self.handles.ferox_scans()?;
+
+        for link in links {
+            let mut resp = match self.request_link(&link).await {
+                Ok(resp) => resp,
+                Err(_) => continue,
+            };
+
+            // filter if necessary
+            if self
+                .handles
+                .filters
+                .data
+                .should_filter_response(&resp, self.handles.stats.tx.clone())
+            {
+                continue;
+            }
+
+            if resp.is_file() {
+                // very likely a file, simply request and report
+                log::debug!("Extracted file: {}", resp);
+
+                scanned_urls.add_file_scan(&resp.url().to_string(), ScanOrder::Latest);
+
+                if let Err(e) = resp.send_report(self.handles.output.tx.clone()) {
+                    log::warn!("Could not send FeroxResponse to output handler: {}", e);
+                }
+
+                continue;
+            }
+
+            if matches!(recursive, RecursionStatus::Recursive) {
+                log::debug!("Extracted Directory: {}", resp);
+
+                if !resp.url().as_str().ends_with('/')
+                    && (resp.status().is_success()
+                        || matches!(resp.status(), &StatusCode::FORBIDDEN))
+                {
+                    // if the url doesn't end with a /
+                    // and the response code is either a 2xx or 403
+
+                    // since all of these are 2xx or 403, recursion is only attempted if the
+                    // url ends in a /. I am actually ok with adding the slash and not
+                    // adding it, as both have merit.  Leaving it in for now to see how
+                    // things turn out (current as of: v1.1.0)
+                    resp.set_url(&format!("{}/", resp.url()));
+                }
+
+                self.handles
+                    .send_scan_command(Command::TryRecursion(Box::new(resp)))?;
+                let (tx, rx) = oneshot::channel::<bool>();
+                self.handles.send_scan_command(Command::Sync(tx))?;
+                rx.await?;
+            }
+        }
+        Ok(())
+    }
+
+    /// Given a `reqwest::Response`, perform the following actions
+    ///   - parse the response's text for links using the linkfinder regex
+    ///   - for every link found take its url path and parse each sub-path
+    ///     - example: Response contains a link fragment `homepage/assets/img/icons/handshake.svg`
+    ///       with a base url of http://localhost, the following urls would be returned:
+    ///         - homepage/assets/img/icons/handshake.svg
+    ///         - homepage/assets/img/icons/
+    ///         - homepage/assets/img/
+    ///         - homepage/assets/
+    ///         - homepage/
+    pub(super) async fn extract_from_body(&self) -> Result<HashSet<String>> {
+        log::trace!("enter: get_links");
+
+        let mut links = HashSet::<String>::new();
+
+        let body = self.response.unwrap().text();
+
+        for capture in self.links_regex.captures_iter(&body) {
+            // remove single & double quotes from both ends of the capture
+            // capture[0] is the entire match, additional capture groups start at [1]
+            let link = capture[0].trim_matches(|c| c == '\'' || c == '"');
+
+            match Url::parse(link) {
+                Ok(absolute) => {
+                    if absolute.domain() != self.response.unwrap().url().domain()
+                        || absolute.host() != self.response.unwrap().url().host()
+                    {
+                        // domains/ips are not the same, don't scan things that aren't part of the original
+                        // target url
+                        continue;
+                    }
+
+                    if self.add_all_sub_paths(absolute.path(), &mut links).is_err() {
+                        log::warn!("could not add sub-paths from {} to {:?}", absolute, links);
+                    }
+                }
+                Err(e) => {
+                    // this is the expected error that happens when we try to parse a url fragment
+                    //     ex: Url::parse("/login") -> Err("relative URL without a base")
+                    // while this is technically an error, these are good results for us
+                    if e.to_string().contains("relative URL without a base") {
+                        if self.add_all_sub_paths(link, &mut links).is_err() {
+                            log::warn!("could not add sub-paths from {} to {:?}", link, links);
+                        }
+                    } else {
+                        // unexpected error has occurred
+                        log::warn!("Could not parse given url: {}", e);
+                        self.handles.stats.send(AddError(Other)).unwrap_or_default();
+                    }
+                }
+            }
+        }
+
+        self.update_stats(links.len())?;
+
+        log::trace!("exit: get_links -> {:?}", links);
+
+        Ok(links)
+    }
+
+    /// take a url fragment like homepage/assets/img/icons/handshake.svg and
+    /// incrementally add
+    ///     - homepage/assets/img/icons/
+    ///     - homepage/assets/img/
+    ///     - homepage/assets/
+    ///     - homepage/
+    fn add_all_sub_paths(&self, url_path: &str, mut links: &mut HashSet<String>) -> Result<()> {
+        log::trace!("enter: add_all_sub_paths({}, {:?})", url_path, links);
+
+        for sub_path in self.get_sub_paths_from_path(url_path) {
+            self.add_link_to_set_of_links(&sub_path, &mut links)?;
+        }
+
+        log::trace!("exit: add_all_sub_paths");
+        Ok(())
+    }
+
+    /// Iterate over a given path, return a list of every sub-path found
+    ///
+    /// example: `path` contains a link fragment `homepage/assets/img/icons/handshake.svg`
+    /// the following fragments would be returned:
+    ///   - homepage/assets/img/icons/handshake.svg
+    ///   - homepage/assets/img/icons/
+    ///   - homepage/assets/img/
+    ///   - homepage/assets/
+    ///   - homepage/
+    pub(super) fn get_sub_paths_from_path(&self, path: &str) -> Vec<String> {
+        log::trace!("enter: get_sub_paths_from_path({})", path);
+        let mut paths = vec![];
+
+        // filter out any empty strings caused by .split
+        let mut parts: Vec<&str> = path.split('/').filter(|s| !s.is_empty()).collect();
+
+        let length = parts.len();
+
+        for i in 0..length {
+            // iterate over all parts of the path
+            if parts.is_empty() {
+                // pop left us with an empty vector, we're done
+                break;
+            }
+
+            let mut possible_path = parts.join("/");
+
+            if possible_path.is_empty() {
+                // .join can result in an empty string, which we don't need, ignore
+                continue;
+            }
+
+            if i > 0 {
+                // this isn't the last index of the parts array
+                // ex: /buried/misc/stupidfile.php
+                // this block skips the file but sees all parent folders
+                possible_path = format!("{}/", possible_path);
+            }
+
+            paths.push(possible_path); // good sub-path found
+            parts.pop(); // use .pop() to remove the last part of the path and continue iteration
+        }
+
+        log::trace!("exit: get_sub_paths_from_path -> {:?}", paths);
+        paths
+    }
+
+    /// simple helper to stay DRY, trys to join a url + fragment and add it to the `links` HashSet
+    pub(super) fn add_link_to_set_of_links(
+        &self,
+        link: &str,
+        links: &mut HashSet<String>,
+    ) -> Result<()> {
+        log::trace!("enter: add_link_to_set_of_links({}, {:?})", link, links);
+
+        let old_url = match self.target {
+            ExtractionTarget::ResponseBody => self.response.unwrap().url().clone(),
+            ExtractionTarget::RobotsTxt => match Url::parse(&self.url) {
+                Ok(u) => u,
+                Err(e) => {
+                    bail!("Could not parse {}: {}", self.url, e);
+                }
+            },
+        };
+
+        let new_url = old_url
+            .join(&link)
+            .with_context(|| format!("Could not join {} with {}", old_url, link))?;
+
+        links.insert(new_url.to_string());
+
+        log::trace!("exit: add_link_to_set_of_links");
+
+        Ok(())
+    }
+
+    /// Wrapper around link extraction logic
+    /// currently used in two places:
+    ///   - links from response bodies
+    ///   - links from robots.txt responses
+    ///
+    /// general steps taken:
+    ///   - create a new Url object based on cli options/args
+    ///   - check if the new Url has already been seen/scanned -> None
+    ///   - make a request to the new Url ? -> Some(response) : None
+    pub(super) async fn request_link(&self, url: &str) -> Result<FeroxResponse> {
+        log::trace!("enter: request_link({})", url);
+
+        let ferox_url = FeroxUrl::from_string(&url, self.handles.clone());
+
+        // create a url based on the given command line options
+        let new_url = ferox_url.format(&"", None)?;
+
+        let scanned_urls = self.handles.ferox_scans()?;
+
+        if scanned_urls.get_scan_by_url(&new_url.to_string()).is_some() {
+            //we've seen the url before and don't need to scan again
+            log::trace!("exit: request_link -> None");
+            bail!("previously seen url");
+        }
+
+        // make the request and store the response
+        let new_response = make_request(
+            &self.handles.config.client,
+            &new_url,
+            self.handles.config.output_level,
+            self.handles.stats.tx.clone(),
+        )
+        .await?;
+
+        let new_ferox_response =
+            FeroxResponse::from(new_response, true, self.handles.config.output_level).await;
+
+        log::trace!("exit: request_link -> {:?}", new_ferox_response);
+
+        Ok(new_ferox_response)
+    }
+
+    /// Entry point to perform link extraction from robots.txt
+    ///
+    /// `base_url` can have paths and subpaths, however robots.txt will be requested from the
+    /// root of the url
+    /// given the url:
+    ///     http://localhost/stuff/things
+    /// this function requests:
+    ///     http://localhost/robots.txt
+    pub(super) async fn extract_from_robots(&self) -> Result<HashSet<String>> {
+        log::trace!("enter: extract_robots_txt");
+
+        let mut links: HashSet<String> = HashSet::new();
+
+        let response = self.request_robots_txt().await?;
+
+        for capture in self.robots_regex.captures_iter(response.text()) {
+            if let Some(new_path) = capture.name("url_path") {
+                let mut new_url = Url::parse(&self.url)?;
+                new_url.set_path(new_path.as_str());
+                if self.add_all_sub_paths(&new_url.path(), &mut links).is_err() {
+                    log::warn!("could not add sub-paths from {} to {:?}", new_url, links);
+                }
+            }
+        }
+
+        self.update_stats(links.len())?;
+
+        log::trace!("exit: extract_robots_txt -> {:?}", links);
+        Ok(links)
+    }
+
+    /// helper function that simply requests /robots.txt on the given url's base url
+    ///
+    /// example:
+    ///     http://localhost/api/users -> http://localhost/robots.txt
+    ///     
+    /// The length of the given path has no effect on what's requested; it's always
+    /// base url + /robots.txt
+    pub(super) async fn request_robots_txt(&self) -> Result<FeroxResponse> {
+        log::trace!("enter: get_robots_file");
+
+        // more often than not, domain/robots.txt will redirect to www.domain/robots.txt or something
+        // similar; to account for that, create a client that will follow redirects, regardless of
+        // what the user specified for the scanning client. Other than redirects, it will respect
+        // all other user specified settings
+        let follow_redirects = true;
+
+        let proxy = if self.handles.config.proxy.is_empty() {
+            None
+        } else {
+            Some(self.handles.config.proxy.as_str())
+        };
+
+        let client = client::initialize(
+            self.handles.config.timeout,
+            &self.handles.config.user_agent,
+            follow_redirects,
+            self.handles.config.insecure,
+            &self.handles.config.headers,
+            proxy,
+        )?;
+
+        let mut url = Url::parse(&self.url)?;
+        url.set_path("/robots.txt"); // overwrite existing path with /robots.txt
+
+        let response = make_request(
+            &client,
+            &url,
+            self.handles.config.output_level,
+            self.handles.stats.tx.clone(),
+        )
+        .await?;
+        let ferox_response =
+            FeroxResponse::from(response, true, self.handles.config.output_level).await;
+
+        log::trace!("exit: get_robots_file -> {}", ferox_response);
+        return Ok(ferox_response);
+    }
+
+    /// update total number of links extracted and expected responses
+    fn update_stats(&self, num_links: usize) -> Result<()> {
+        let multiplier = self.handles.config.extensions.len().max(1);
+
+        self.handles
+            .stats
+            .send(UpdateUsizeField(LinksExtracted, num_links))?;
+        self.handles
+            .stats
+            .send(UpdateUsizeField(TotalExpected, num_links * multiplier))?;
+
+        Ok(())
+    }
+}

src/extractor/mod.rs

@@ -0,0 +1,13 @@
+//! extract links from html source and robots.txt
+mod builder;
+mod container;
+#[cfg(test)]
+mod tests;
+
+pub use self::builder::ExtractionTarget;
+pub use self::builder::ExtractorBuilder;
+pub use self::container::Extractor;
+
+use crate::response::FeroxResponse;
+use regex::Regex;
+use std::sync::Arc;

src/extractor/tests.rs

@@ -0,0 +1,354 @@
+use super::builder::{LINKFINDER_REGEX, ROBOTS_TXT_REGEX};
+use super::*;
+use crate::config::{Configuration, OutputLevel};
+use crate::scan_manager::ScanOrder;
+use crate::{
+    event_handlers::Handles, scan_manager::FeroxScans, utils::make_request, Command, FeroxChannel,
+};
+use anyhow::Result;
+use httpmock::{Method::GET, MockServer};
+use lazy_static::lazy_static;
+use reqwest::{Client, StatusCode, Url};
+use std::collections::HashSet;
+use tokio::sync::mpsc;
+
+lazy_static! {
+    /// Extractor for testing robots.txt
+    static ref ROBOTS_EXT: Extractor<'static> = setup_extractor(ExtractionTarget::RobotsTxt, Arc::new(FeroxScans::default()));
+
+    /// Extractor for testing response bodies
+    static ref BODY_EXT: Extractor<'static> = setup_extractor(ExtractionTarget::ResponseBody, Arc::new(FeroxScans::default()));
+
+    /// FeroxResponse for Extractor
+    static ref RESPONSE: FeroxResponse = get_test_response();
+}
+
+/// constructor for the default FeroxResponse used during testing
+fn get_test_response() -> FeroxResponse {
+    let mut resp = FeroxResponse::default();
+    resp.set_text("nulla pharetra diam sit amet nisl suscipit adipiscing bibendum est");
+    resp
+}
+
+/// creates a single extractor that can be used to test standalone functions
+fn setup_extractor(target: ExtractionTarget, scanned_urls: Arc<FeroxScans>) -> Extractor<'static> {
+    let mut builder = ExtractorBuilder::default();
+
+    let builder = match target {
+        ExtractionTarget::ResponseBody => builder
+            .target(ExtractionTarget::ResponseBody)
+            .response(&RESPONSE),
+        ExtractionTarget::RobotsTxt => builder
+            .url("http://localhost")
+            .target(ExtractionTarget::RobotsTxt),
+    };
+
+    let config = Arc::new(Configuration::new().unwrap());
+    let handles = Arc::new(Handles::for_testing(Some(scanned_urls), Some(config)).0);
+
+    builder.handles(handles).build().unwrap()
+}
+
+#[test]
+/// extract sub paths from the given url fragment; expect 4 sub paths and that all are
+/// in the expected array
+fn extractor_get_sub_paths_from_path_with_multiple_paths() {
+    let path = "homepage/assets/img/icons/handshake.svg";
+    let r_paths = ROBOTS_EXT.get_sub_paths_from_path(&path);
+    let b_paths = BODY_EXT.get_sub_paths_from_path(&path);
+    let expected = vec![
+        "homepage/",
+        "homepage/assets/",
+        "homepage/assets/img/",
+        "homepage/assets/img/icons/",
+        "homepage/assets/img/icons/handshake.svg",
+    ];
+
+    assert_eq!(r_paths.len(), expected.len());
+    assert_eq!(b_paths.len(), expected.len());
+    for expected_path in expected {
+        assert_eq!(r_paths.contains(&expected_path.to_string()), true);
+        assert_eq!(b_paths.contains(&expected_path.to_string()), true);
+    }
+}
+
+#[test]
+/// extract sub paths from the given url fragment; expect 2 sub paths and that all are
+/// in the expected array. the fragment is wrapped in slashes to ensure no empty strings are
+/// returned
+fn extractor_get_sub_paths_from_path_with_enclosing_slashes() {
+    let path = "/homepage/assets/";
+    let r_paths = ROBOTS_EXT.get_sub_paths_from_path(&path);
+    let b_paths = BODY_EXT.get_sub_paths_from_path(&path);
+    let expected = vec!["homepage/", "homepage/assets"];
+
+    assert_eq!(r_paths.len(), expected.len());
+    assert_eq!(b_paths.len(), expected.len());
+    for expected_path in expected {
+        assert_eq!(r_paths.contains(&expected_path.to_string()), true);
+        assert_eq!(b_paths.contains(&expected_path.to_string()), true);
+    }
+}
+
+#[test]
+/// extract sub paths from the given url fragment; expect 1 sub path, no forward slashes are
+/// included
+fn extractor_get_sub_paths_from_path_with_only_a_word() {
+    let path = "homepage";
+    let r_paths = ROBOTS_EXT.get_sub_paths_from_path(&path);
+    let b_paths = BODY_EXT.get_sub_paths_from_path(&path);
+    let expected = vec!["homepage"];
+
+    assert_eq!(r_paths.len(), expected.len());
+    assert_eq!(b_paths.len(), expected.len());
+    for expected_path in expected {
+        assert_eq!(r_paths.contains(&expected_path.to_string()), true);
+        assert_eq!(b_paths.contains(&expected_path.to_string()), true);
+    }
+}
+
+#[test]
+/// extract sub paths from the given url fragment; expect 1 sub path, forward slash removed
+fn extractor_get_sub_paths_from_path_with_an_absolute_word() {
+    let path = "/homepage";
+    let r_paths = ROBOTS_EXT.get_sub_paths_from_path(&path);
+    let b_paths = BODY_EXT.get_sub_paths_from_path(&path);
+    let expected = vec!["homepage"];
+
+    assert_eq!(r_paths.len(), expected.len());
+    assert_eq!(b_paths.len(), expected.len());
+    for expected_path in expected {
+        assert_eq!(r_paths.contains(&expected_path.to_string()), true);
+        assert_eq!(b_paths.contains(&expected_path.to_string()), true);
+    }
+}
+
+#[test]
+/// test that an ExtractorBuilder without a FeroxResponse and without a URL bails
+fn extractor_builder_bails_when_neither_required_field_is_set() {
+    let handles = Arc::new(Handles::for_testing(None, None).0);
+
+    let extractor = ExtractorBuilder::default()
+        .url("")
+        .target(ExtractionTarget::RobotsTxt)
+        .handles(handles)
+        .build();
+
+    assert!(extractor.is_err());
+}
+
+#[test]
+/// Extractor with a non-base url bails
+fn extractor_with_non_base_url_bails() -> Result<()> {
+    let mut links = HashSet::<String>::new();
+    let link = "admin";
+    let handles = Arc::new(Handles::for_testing(None, None).0);
+
+    let extractor = ExtractorBuilder::default()
+        .url("\\\\\\")
+        .handles(handles)
+        .target(ExtractionTarget::RobotsTxt)
+        .build()?;
+
+    let result = extractor.add_link_to_set_of_links(link, &mut links);
+
+    assert!(result.is_err());
+    Ok(())
+}
+
+#[test]
+/// test that a full url and fragment are joined correctly, then added to the given list
+/// i.e. the happy path
+fn extractor_add_link_to_set_of_links_happy_path() {
+    let mut r_links = HashSet::<String>::new();
+    let r_link = "admin";
+    let mut b_links = HashSet::<String>::new();
+    let b_link = "shmadmin";
+
+    assert_eq!(r_links.len(), 0);
+    ROBOTS_EXT
+        .add_link_to_set_of_links(r_link, &mut r_links)
+        .unwrap();
+
+    assert_eq!(r_links.len(), 1);
+    assert!(r_links.contains("http://localhost/admin"));
+
+    assert_eq!(b_links.len(), 0);
+
+    BODY_EXT
+        .add_link_to_set_of_links(b_link, &mut b_links)
+        .unwrap();
+
+    assert_eq!(b_links.len(), 1);
+    assert!(b_links.contains("http://localhost/shmadmin"));
+}
+
+#[test]
+/// test that an invalid path fragment doesn't add anything to the set of links
+fn extractor_add_link_to_set_of_links_with_non_base_url() {
+    let mut links = HashSet::<String>::new();
+    let link = "\\\\\\\\";
+
+    assert_eq!(links.len(), 0);
+    assert!(ROBOTS_EXT
+        .add_link_to_set_of_links(link, &mut links)
+        .is_err());
+    assert!(BODY_EXT.add_link_to_set_of_links(link, &mut links).is_err());
+
+    assert_eq!(links.len(), 0);
+    assert!(links.is_empty());
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// use make_request to generate a Response, and use the Response to test get_links;
+/// the response will contain an absolute path to a domain that is not part of the scanned
+/// domain; expect an empty set returned
+async fn extractor_get_links_with_absolute_url_that_differs_from_target_domain() -> Result<()> {
+    let (tx_stats, _): FeroxChannel<Command> = mpsc::unbounded_channel();
+
+    let srv = MockServer::start();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/some-path");
+        then.status(200).body(
+            "\"http://defintely.not.a.thing.probably.com/homepage/assets/img/icons/handshake.svg\"",
+        );
+    });
+
+    let client = Client::new();
+    let url = Url::parse(&srv.url("/some-path")).unwrap();
+
+    let response = make_request(&client, &url, OutputLevel::Default, tx_stats.clone())
+        .await
+        .unwrap();
+    let (handles, _rx) = Handles::for_testing(None, None);
+
+    let handles = Arc::new(handles);
+    let ferox_response = FeroxResponse::from(response, true, OutputLevel::Default).await;
+
+    let extractor = Extractor {
+        links_regex: Regex::new(LINKFINDER_REGEX).unwrap(),
+        robots_regex: Regex::new(ROBOTS_TXT_REGEX).unwrap(),
+        response: Some(&ferox_response),
+        url: String::new(),
+        target: ExtractionTarget::ResponseBody,
+        handles: handles.clone(),
+    };
+
+    let links = extractor.extract_from_body().await?;
+
+    assert!(links.is_empty());
+    assert_eq!(mock.hits(), 1);
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test that /robots.txt is correctly requested given a base url (happy path)
+async fn request_robots_txt_without_proxy() -> Result<()> {
+    let handles = Arc::new(Handles::for_testing(None, None).0);
+
+    let srv = MockServer::start();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/robots.txt");
+        then.status(200).body("this is a test");
+    });
+
+    let extractor = Extractor {
+        links_regex: Regex::new(LINKFINDER_REGEX).unwrap(),
+        robots_regex: Regex::new(ROBOTS_TXT_REGEX).unwrap(),
+        response: None,
+        url: srv.url("/api/users/stuff/things"),
+        target: ExtractionTarget::RobotsTxt,
+        handles,
+    };
+
+    let resp = extractor.request_robots_txt().await?;
+
+    assert!(matches!(resp.status(), &StatusCode::OK));
+    println!("{}", resp);
+    assert_eq!(resp.content_length(), 14);
+    assert_eq!(mock.hits(), 1);
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test that /robots.txt is correctly requested given a base url (happy path) when a proxy is used
+async fn request_robots_txt_with_proxy() -> Result<()> {
+    let handles = Arc::new(Handles::for_testing(None, None).0);
+    let mut config = Configuration::new()?;
+
+    let srv = MockServer::start();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/robots.txt");
+        then.status(200).body("this is also a test");
+    });
+
+    // note: the proxy doesn't actually do anything other than hit a different code branch
+    // in this unit test; it would however have an effect on an integration test
+    config.proxy = srv.url("/ima-proxy");
+    config.no_recursion = true;
+
+    let extractor = ExtractorBuilder::default()
+        .url(&srv.url("/api/different/path"))
+        .target(ExtractionTarget::RobotsTxt)
+        .handles(handles)
+        .build()?;
+
+    let resp = extractor.request_robots_txt().await?;
+
+    assert!(matches!(resp.status(), &StatusCode::OK));
+    assert_eq!(resp.content_length(), 19);
+    assert_eq!(mock.hits(), 1);
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// request_link's happy path, expect back a FeroxResponse
+async fn request_link_happy_path() -> Result<()> {
+    let srv = MockServer::start();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/login.php");
+        then.status(200).body("this is a test");
+    });
+
+    let r_resp = ROBOTS_EXT.request_link(&srv.url("/login.php")).await?;
+    let b_resp = BODY_EXT.request_link(&srv.url("/login.php")).await?;
+
+    assert!(matches!(r_resp.status(), &StatusCode::OK));
+    assert!(matches!(b_resp.status(), &StatusCode::OK));
+    assert_eq!(r_resp.content_length(), 14);
+    assert_eq!(b_resp.content_length(), 14);
+    assert_eq!(mock.hits(), 2);
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// request_link should bail in the event that the url is already in scanned_urls
+async fn request_link_bails_on_seen_url() -> Result<()> {
+    let url = "/unique-for-this-test.php";
+    let srv = MockServer::start();
+    let served = srv.url(url);
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path(url);
+        then.status(200)
+            .body("this is a unique test, don't reuse the endpoint");
+    });
+
+    let scans = Arc::new(FeroxScans::default());
+    scans.add_file_scan(&served, ScanOrder::Latest);
+
+    let robots = setup_extractor(ExtractionTarget::RobotsTxt, scans.clone());
+    let body = setup_extractor(ExtractionTarget::ResponseBody, scans);
+
+    let r_resp = robots.request_link(&served).await;
+    let b_resp = body.request_link(&served).await;
+
+    assert!(r_resp.is_err());
+    assert!(b_resp.is_err());
+    assert_eq!(mock.hits(), 0); // function exits before requests can happen
+    Ok(())
+}

src/filters.rs

@@ -1,422 +0,0 @@
-use crate::config::CONFIGURATION;
-use crate::utils::get_url_path_length;
-use crate::FeroxResponse;
-use regex::Regex;
-use std::any::Any;
-use std::fmt::Debug;
-
-// references:
-//   https://dev.to/magnusstrale/rust-trait-objects-in-a-vector-non-trivial-4co5
-//   https://stackoverflow.com/questions/25339603/how-to-test-for-equality-between-trait-objects
-
-/// FeroxFilter trait; represents different types of possible filters that can be applied to
-/// responses
-pub trait FeroxFilter: Debug + Send + Sync {
-    /// Determine whether or not this particular filter should be applied or not
-    fn should_filter_response(&self, response: &FeroxResponse) -> bool;
-
-    /// delegates to the FeroxFilter-implementing type which gives us the actual type of self
-    fn box_eq(&self, other: &dyn Any) -> bool;
-
-    /// gives us `other` as Any in box_eq
-    fn as_any(&self) -> &dyn Any;
-}
-
-/// implementation of PartialEq, necessary long-form due to "trait cannot be made into an object"
-/// error when attempting to derive PartialEq on the trait itself
-impl PartialEq for Box<dyn FeroxFilter> {
-    /// Perform a comparison of two implementors of the FeroxFilter trait
-    fn eq(&self, other: &Box<dyn FeroxFilter>) -> bool {
-        self.box_eq(other.as_any())
-    }
-}
-
-/// Data holder for two pieces of data needed when auto-filtering out wildcard responses
-///
-/// `dynamic` is the size of the response that will later be combined with the length
-/// of the path of the url requested and used to determine interesting pages from custom
-/// 404s where the requested url is reflected back in the response
-///
-/// `size` is size of the response that should be included with filters passed via runtime
-/// configuration and any static wildcard lengths.
-#[derive(Debug, Default, Clone, PartialEq)]
-pub struct WildcardFilter {
-    /// size of the response that will later be combined with the length of the path of the url
-    /// requested
-    pub dynamic: u64,
-
-    /// size of the response that should be included with filters passed via runtime configuration
-    pub size: u64,
-}
-
-/// implementation of FeroxFilter for WildcardFilter
-impl FeroxFilter for WildcardFilter {
-    /// Examine size, dynamic, and content_len to determine whether or not the response received
-    /// is a wildcard response and therefore should be filtered out
-    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
-        log::trace!("enter: should_filter_response({:?} {})", self, response);
-
-        // quick return if dont_filter is set
-        if CONFIGURATION.dont_filter {
-            // --dont-filter applies specifically to wildcard filters, it is not a 100% catch all
-            // for not filtering anything.  As such, it should live in the implementation of
-            // a wildcard filter
-            return false;
-        }
-
-        if self.size > 0 && self.size == response.content_length() {
-            // static wildcard size found during testing
-            // size isn't default, size equals response length, and auto-filter is on
-            log::debug!("static wildcard: filtered out {}", response.url());
-            log::trace!("exit: should_filter_response -> true");
-            return true;
-        }
-
-        if self.dynamic > 0 {
-            // dynamic wildcard offset found during testing
-
-            // I'm about to manually split this url path instead of using reqwest::Url's
-            // builtin parsing. The reason is that they call .split() on the url path
-            // except that I don't want an empty string taking up the last index in the
-            // event that the url ends with a forward slash.  It's ugly enough to be split
-            // into its own function for readability.
-            let url_len = get_url_path_length(&response.url());
-
-            if url_len + self.dynamic == response.content_length() {
-                log::debug!("dynamic wildcard: filtered out {}", response.url());
-                log::trace!("exit: should_filter_response -> true");
-                return true;
-            }
-        }
-        log::trace!("exit: should_filter_response -> false");
-        false
-    }
-
-    /// Compare one WildcardFilter to another
-    fn box_eq(&self, other: &dyn Any) -> bool {
-        other.downcast_ref::<Self>().map_or(false, |a| self == a)
-    }
-
-    /// Return self as Any for dynamic dispatch purposes
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-}
-
-/// Simple implementor of FeroxFilter; used to filter out status codes specified using
-/// -C|--filter-status
-#[derive(Default, Debug, PartialEq)]
-pub struct StatusCodeFilter {
-    /// Status code that should not be displayed to the user
-    pub filter_code: u16,
-}
-
-/// implementation of FeroxFilter for StatusCodeFilter
-impl FeroxFilter for StatusCodeFilter {
-    /// Check `filter_code` against what was passed in via -C|--filter-status
-    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
-        log::trace!("enter: should_filter_response({:?} {})", self, response);
-
-        if response.status().as_u16() == self.filter_code {
-            log::debug!(
-                "filtered out {} based on --filter-status of {}",
-                response.url(),
-                self.filter_code
-            );
-            log::trace!("exit: should_filter_response -> true");
-            return true;
-        }
-
-        log::trace!("exit: should_filter_response -> false");
-        false
-    }
-
-    /// Compare one StatusCodeFilter to another
-    fn box_eq(&self, other: &dyn Any) -> bool {
-        other.downcast_ref::<Self>().map_or(false, |a| self == a)
-    }
-
-    /// Return self as Any for dynamic dispatch purposes
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-}
-
-/// Simple implementor of FeroxFilter; used to filter out responses based on the number of lines
-/// in a Response body; specified using -N|--filter-lines
-#[derive(Default, Debug, PartialEq)]
-pub struct LinesFilter {
-    /// Number of lines in a Response's body that should be filtered
-    pub line_count: usize,
-}
-
-/// implementation of FeroxFilter for LinesFilter
-impl FeroxFilter for LinesFilter {
-    /// Check `line_count` against what was passed in via -N|--filter-lines
-    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
-        log::trace!("enter: should_filter_response({:?} {})", self, response);
-
-        let result = response.line_count() == self.line_count;
-
-        log::trace!("exit: should_filter_response -> {}", result);
-
-        result
-    }
-
-    /// Compare one LinesFilter to another
-    fn box_eq(&self, other: &dyn Any) -> bool {
-        other.downcast_ref::<Self>().map_or(false, |a| self == a)
-    }
-
-    /// Return self as Any for dynamic dispatch purposes
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-}
-
-/// Simple implementor of FeroxFilter; used to filter out responses based on the number of words
-/// in a Response body; specified using -W|--filter-words
-#[derive(Default, Debug, PartialEq)]
-pub struct WordsFilter {
-    /// Number of words in a Response's body that should be filtered
-    pub word_count: usize,
-}
-
-/// implementation of FeroxFilter for WordsFilter
-impl FeroxFilter for WordsFilter {
-    /// Check `word_count` against what was passed in via -W|--filter-words
-    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
-        log::trace!("enter: should_filter_response({:?} {})", self, response);
-
-        let result = response.word_count() == self.word_count;
-
-        log::trace!("exit: should_filter_response -> {}", result);
-
-        result
-    }
-
-    /// Compare one WordsFilter to another
-    fn box_eq(&self, other: &dyn Any) -> bool {
-        other.downcast_ref::<Self>().map_or(false, |a| self == a)
-    }
-
-    /// Return self as Any for dynamic dispatch purposes
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-}
-
-/// Simple implementor of FeroxFilter; used to filter out responses based on the length of a
-/// Response body; specified using -S|--filter-size
-#[derive(Default, Debug, PartialEq)]
-pub struct SizeFilter {
-    /// Overall length of a Response's body that should be filtered
-    pub content_length: u64,
-}
-
-/// implementation of FeroxFilter for SizeFilter
-impl FeroxFilter for SizeFilter {
-    /// Check `content_length` against what was passed in via -S|--filter-size
-    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
-        log::trace!("enter: should_filter_response({:?} {})", self, response);
-
-        let result = response.content_length() == self.content_length;
-
-        log::trace!("exit: should_filter_response -> {}", result);
-
-        result
-    }
-
-    /// Compare one SizeFilter to another
-    fn box_eq(&self, other: &dyn Any) -> bool {
-        other.downcast_ref::<Self>().map_or(false, |a| self == a)
-    }
-
-    /// Return self as Any for dynamic dispatch purposes
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-}
-
-/// Simple implementor of FeroxFilter; used to filter out responses based on a given regular
-/// expression; specified using -X|--filter-regex
-#[derive(Debug)]
-pub struct RegexFilter {
-    /// Regular expression to be applied to the response body for filtering, compiled
-    pub compiled: Regex,
-
-    /// Regular expression as passed in on the command line, not compiled
-    pub raw_string: String,
-}
-
-/// implementation of FeroxFilter for RegexFilter
-impl FeroxFilter for RegexFilter {
-    /// Check `expression` against the response body, if the expression matches, the response
-    /// should be filtered out
-    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
-        log::trace!("enter: should_filter_response({:?} {})", self, response);
-
-        let result = self.compiled.is_match(response.text());
-
-        log::trace!("exit: should_filter_response -> {}", result);
-
-        result
-    }
-
-    /// Compare one SizeFilter to another
-    fn box_eq(&self, other: &dyn Any) -> bool {
-        other.downcast_ref::<Self>().map_or(false, |a| self == a)
-    }
-
-    /// Return self as Any for dynamic dispatch purposes
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-}
-
-/// PartialEq implementation for RegexFilter
-impl PartialEq for RegexFilter {
-    /// Simple comparison of the raw string passed in via the command line
-    fn eq(&self, other: &RegexFilter) -> bool {
-        self.raw_string == other.raw_string
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use reqwest::Url;
-
-    #[test]
-    /// just a simple test to increase code coverage by hitting as_any and the inner value
-    fn lines_filter_as_any() {
-        let filter = LinesFilter { line_count: 1 };
-
-        assert_eq!(filter.line_count, 1);
-        assert_eq!(
-            *filter.as_any().downcast_ref::<LinesFilter>().unwrap(),
-            filter
-        );
-    }
-
-    #[test]
-    /// just a simple test to increase code coverage by hitting as_any and the inner value
-    fn words_filter_as_any() {
-        let filter = WordsFilter { word_count: 1 };
-
-        assert_eq!(filter.word_count, 1);
-        assert_eq!(
-            *filter.as_any().downcast_ref::<WordsFilter>().unwrap(),
-            filter
-        );
-    }
-
-    #[test]
-    /// just a simple test to increase code coverage by hitting as_any and the inner value
-    fn size_filter_as_any() {
-        let filter = SizeFilter { content_length: 1 };
-
-        assert_eq!(filter.content_length, 1);
-        assert_eq!(
-            *filter.as_any().downcast_ref::<SizeFilter>().unwrap(),
-            filter
-        );
-    }
-
-    #[test]
-    /// just a simple test to increase code coverage by hitting as_any and the inner value
-    fn status_code_filter_as_any() {
-        let filter = StatusCodeFilter { filter_code: 200 };
-
-        assert_eq!(filter.filter_code, 200);
-        assert_eq!(
-            *filter.as_any().downcast_ref::<StatusCodeFilter>().unwrap(),
-            filter
-        );
-    }
-
-    #[test]
-    /// just a simple test to increase code coverage by hitting as_any and the inner value
-    fn regex_filter_as_any() {
-        let raw = r".*\.txt$";
-        let compiled = Regex::new(raw).unwrap();
-        let filter = RegexFilter {
-            compiled,
-            raw_string: raw.to_string(),
-        };
-
-        assert_eq!(filter.raw_string, r".*\.txt$");
-        assert_eq!(
-            *filter.as_any().downcast_ref::<RegexFilter>().unwrap(),
-            filter
-        );
-    }
-
-    #[test]
-    /// test should_filter on WilcardFilter where static logic matches
-    fn wildcard_should_filter_when_static_wildcard_found() {
-        let resp = FeroxResponse {
-            text: String::new(),
-            wildcard: true,
-            url: Url::parse("http://localhost").unwrap(),
-            content_length: 100,
-            word_count: 50,
-            line_count: 25,
-            headers: reqwest::header::HeaderMap::new(),
-            status: reqwest::StatusCode::OK,
-        };
-
-        let filter = WildcardFilter {
-            size: 100,
-            dynamic: 0,
-        };
-
-        assert!(filter.should_filter_response(&resp));
-    }
-
-    #[test]
-    /// test should_filter on WilcardFilter where dynamic logic matches
-    fn wildcard_should_filter_when_dynamic_wildcard_found() {
-        let resp = FeroxResponse {
-            text: String::new(),
-            wildcard: true,
-            url: Url::parse("http://localhost/stuff").unwrap(),
-            content_length: 100,
-            word_count: 50,
-            line_count: 25,
-            headers: reqwest::header::HeaderMap::new(),
-            status: reqwest::StatusCode::OK,
-        };
-
-        let filter = WildcardFilter {
-            size: 0,
-            dynamic: 95,
-        };
-
-        assert!(filter.should_filter_response(&resp));
-    }
-
-    #[test]
-    /// test should_filter on RegexFilter where regex matches body
-    fn regexfilter_should_filter_when_regex_matches_on_response_body() {
-        let resp = FeroxResponse {
-            text: String::from("im a body response hurr durr!"),
-            wildcard: false,
-            url: Url::parse("http://localhost/stuff").unwrap(),
-            content_length: 100,
-            word_count: 50,
-            line_count: 25,
-            headers: reqwest::header::HeaderMap::new(),
-            status: reqwest::StatusCode::OK,
-        };
-
-        let raw = r"response...rr";
-
-        let filter = RegexFilter {
-            raw_string: raw.to_string(),
-            compiled: Regex::new(raw).unwrap(),
-        };
-
-        assert!(filter.should_filter_response(&resp));
-    }
-}

src/filters/container.rs

@@ -0,0 +1,56 @@
+use std::sync::Mutex;
+
+use anyhow::Result;
+
+use crate::response::FeroxResponse;
+use crate::{
+    event_handlers::Command::UpdateUsizeField, statistics::StatField::WildcardsFiltered,
+    CommandSender,
+};
+
+use super::{FeroxFilter, WildcardFilter};
+
+/// Container around a collection of `FeroxFilters`s
+#[derive(Debug, Default)]
+pub struct FeroxFilters {
+    /// collection of `FeroxFilters`
+    pub filters: Mutex<Vec<Box<dyn FeroxFilter>>>,
+}
+
+/// implementation of FeroxFilter collection
+impl FeroxFilters {
+    /// add a single FeroxFilter to the collection
+    pub fn push(&self, filter: Box<dyn FeroxFilter>) -> Result<()> {
+        if let Ok(mut guard) = self.filters.lock() {
+            if guard.contains(&filter) {
+                return Ok(());
+            }
+
+            guard.push(filter)
+        }
+        Ok(())
+    }
+
+    /// Simple helper to stay DRY; determines whether or not a given `FeroxResponse` should be reported
+    /// to the user or not.
+    pub fn should_filter_response(
+        &self,
+        response: &FeroxResponse,
+        tx_stats: CommandSender,
+    ) -> bool {
+        if let Ok(filters) = self.filters.lock() {
+            for filter in filters.iter() {
+                // wildcard.should_filter goes here
+                if filter.should_filter_response(&response) {
+                    if filter.as_any().downcast_ref::<WildcardFilter>().is_some() {
+                        tx_stats
+                            .send(UpdateUsizeField(WildcardsFiltered, 1))
+                            .unwrap_or_default();
+                    }
+                    return true;
+                }
+            }
+        }
+        false
+    }
+}

src/filters/init.rs

@@ -0,0 +1,102 @@
+use super::{
+    LinesFilter, RegexFilter, SimilarityFilter, SizeFilter, StatusCodeFilter, WordsFilter,
+};
+use crate::{
+    event_handlers::Handles,
+    response::FeroxResponse,
+    skip_fail,
+    utils::{fmt_err, make_request},
+    Command::AddFilter,
+    SIMILARITY_THRESHOLD,
+};
+use anyhow::Result;
+use fuzzyhash::FuzzyHash;
+use regex::Regex;
+use reqwest::Url;
+use std::sync::Arc;
+
+/// add all user-supplied filters to the (already started) filters handler
+pub async fn initialize(handles: Arc<Handles>) -> Result<()> {
+    // add any status code filters to filters handler's FeroxFilters  (-C|--filter-status)
+    for code_filter in &handles.config.filter_status {
+        let filter = StatusCodeFilter {
+            filter_code: *code_filter,
+        };
+        let boxed_filter = Box::new(filter);
+        skip_fail!(handles.filters.send(AddFilter(boxed_filter)));
+    }
+
+    // add any line count filters to filters handler's FeroxFilters  (-N|--filter-lines)
+    for lines_filter in &handles.config.filter_line_count {
+        let filter = LinesFilter {
+            line_count: *lines_filter,
+        };
+        let boxed_filter = Box::new(filter);
+        skip_fail!(handles.filters.send(AddFilter(boxed_filter)));
+    }
+
+    // add any line count filters to filters handler's FeroxFilters  (-W|--filter-words)
+    for words_filter in &handles.config.filter_word_count {
+        let filter = WordsFilter {
+            word_count: *words_filter,
+        };
+        let boxed_filter = Box::new(filter);
+        skip_fail!(handles.filters.send(AddFilter(boxed_filter)));
+    }
+
+    // add any line count filters to filters handler's FeroxFilters  (-S|--filter-size)
+    for size_filter in &handles.config.filter_size {
+        let filter = SizeFilter {
+            content_length: *size_filter,
+        };
+        let boxed_filter = Box::new(filter);
+        skip_fail!(handles.filters.send(AddFilter(boxed_filter)));
+    }
+
+    // add any regex filters to filters handler's FeroxFilters  (-X|--filter-regex)
+    for regex_filter in &handles.config.filter_regex {
+        let raw = regex_filter;
+        let compiled = skip_fail!(Regex::new(&raw));
+
+        let filter = RegexFilter {
+            raw_string: raw.to_owned(),
+            compiled,
+        };
+        let boxed_filter = Box::new(filter);
+        skip_fail!(handles.filters.send(AddFilter(boxed_filter)));
+    }
+
+    // add any similarity filters to filters handler's FeroxFilters  (--filter-similar-to)
+    for similarity_filter in &handles.config.filter_similar {
+        // url as-is based on input, ignores user-specified url manipulation options (add-slash etc)
+        let url = skip_fail!(Url::parse(&similarity_filter));
+
+        // attempt to request the given url
+        let resp = skip_fail!(
+            make_request(
+                &handles.config.client,
+                &url,
+                handles.config.output_level,
+                handles.stats.tx.clone()
+            )
+            .await
+        );
+
+        // if successful, create a filter based on the response's body
+        let fr = FeroxResponse::from(resp, true, handles.config.output_level).await;
+
+        // hash the response body and store the resulting hash in the filter object
+        let hash = FuzzyHash::new(&fr.text()).to_string();
+
+        let filter = SimilarityFilter {
+            text: hash,
+            threshold: SIMILARITY_THRESHOLD,
+        };
+
+        let boxed_filter = Box::new(filter);
+        skip_fail!(handles.filters.send(AddFilter(boxed_filter)));
+    }
+
+    handles.filters.sync().await?;
+    Ok(())
+}

src/filters/lines.rs

@@ -0,0 +1,33 @@
+use super::*;
+
+/// Simple implementor of FeroxFilter; used to filter out responses based on the number of lines
+/// in a Response body; specified using -N|--filter-lines
+#[derive(Default, Debug, PartialEq)]
+pub struct LinesFilter {
+    /// Number of lines in a Response's body that should be filtered
+    pub line_count: usize,
+}
+
+/// implementation of FeroxFilter for LinesFilter
+impl FeroxFilter for LinesFilter {
+    /// Check `line_count` against what was passed in via -N|--filter-lines
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        let result = response.line_count() == self.line_count;
+
+        log::trace!("exit: should_filter_response -> {}", result);
+
+        result
+    }
+
+    /// Compare one LinesFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/filters/mod.rs

@@ -0,0 +1,28 @@
+//! contains all of feroxbuster's filters
+use std::any::Any;
+use std::fmt::Debug;
+
+use crate::response::FeroxResponse;
+use crate::traits::{FeroxFilter, FeroxSerialize};
+
+pub use self::container::FeroxFilters;
+pub use self::init::initialize;
+pub use self::lines::LinesFilter;
+pub use self::regex::RegexFilter;
+pub use self::similarity::SimilarityFilter;
+pub use self::size::SizeFilter;
+pub use self::status_code::StatusCodeFilter;
+pub use self::wildcard::WildcardFilter;
+pub use self::words::WordsFilter;
+
+mod wildcard;
+mod status_code;
+mod words;
+mod lines;
+mod size;
+mod regex;
+mod similarity;
+mod container;
+#[cfg(test)]
+mod tests;
+mod init;

src/filters/regex.rs

@@ -0,0 +1,46 @@
+use super::*;
+use ::regex::Regex;
+
+/// Simple implementor of FeroxFilter; used to filter out responses based on a given regular
+/// expression; specified using -X|--filter-regex
+#[derive(Debug)]
+pub struct RegexFilter {
+    /// Regular expression to be applied to the response body for filtering, compiled
+    pub compiled: Regex,
+
+    /// Regular expression as passed in on the command line, not compiled
+    pub raw_string: String,
+}
+
+/// implementation of FeroxFilter for RegexFilter
+impl FeroxFilter for RegexFilter {
+    /// Check `expression` against the response body, if the expression matches, the response
+    /// should be filtered out
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        let result = self.compiled.is_match(response.text());
+
+        log::trace!("exit: should_filter_response -> {}", result);
+
+        result
+    }
+
+    /// Compare one SizeFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
+
+/// PartialEq implementation for RegexFilter
+impl PartialEq for RegexFilter {
+    /// Simple comparison of the raw string passed in via the command line
+    fn eq(&self, other: &RegexFilter) -> bool {
+        self.raw_string == other.raw_string
+    }
+}

src/filters/similarity.rs

@@ -0,0 +1,40 @@
+use super::*;
+use fuzzyhash::FuzzyHash;
+
+/// Simple implementor of FeroxFilter; used to filter out responses based on the similarity of a
+/// Response body with a known response; specified using --filter-similar-to
+#[derive(Default, Debug, PartialEq)]
+pub struct SimilarityFilter {
+    /// Response's body to be used for comparison for similarity
+    pub text: String,
+
+    /// Percentage of similarity at which a page is determined to be a near-duplicate of another
+    pub threshold: u32,
+}
+
+/// implementation of FeroxFilter for SimilarityFilter
+impl FeroxFilter for SimilarityFilter {
+    /// Check `FeroxResponse::text` against what was requested from the site passed in via
+    /// --filter-similar-to
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        let other = FuzzyHash::new(&response.text());
+
+        if let Ok(result) = FuzzyHash::compare(&self.text, &other.to_string()) {
+            return result >= self.threshold;
+        }
+
+        // couldn't hash the response, don't filter
+        log::warn!("Could not hash body from {}", response.as_str());
+        false
+    }
+
+    /// Compare one SimilarityFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/filters/size.rs

@@ -0,0 +1,33 @@
+use super::*;
+
+/// Simple implementor of FeroxFilter; used to filter out responses based on the length of a
+/// Response body; specified using -S|--filter-size
+#[derive(Default, Debug, PartialEq)]
+pub struct SizeFilter {
+    /// Overall length of a Response's body that should be filtered
+    pub content_length: u64,
+}
+
+/// implementation of FeroxFilter for SizeFilter
+impl FeroxFilter for SizeFilter {
+    /// Check `content_length` against what was passed in via -S|--filter-size
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        let result = response.content_length() == self.content_length;
+
+        log::trace!("exit: should_filter_response -> {}", result);
+
+        result
+    }
+
+    /// Compare one SizeFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/filters/status_code.rs

@@ -0,0 +1,40 @@
+use super::*;
+
+/// Simple implementor of FeroxFilter; used to filter out status codes specified using
+/// -C|--filter-status
+#[derive(Default, Debug, PartialEq)]
+pub struct StatusCodeFilter {
+    /// Status code that should not be displayed to the user
+    pub filter_code: u16,
+}
+
+/// implementation of FeroxFilter for StatusCodeFilter
+impl FeroxFilter for StatusCodeFilter {
+    /// Check `filter_code` against what was passed in via -C|--filter-status
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        if response.status().as_u16() == self.filter_code {
+            log::debug!(
+                "filtered out {} based on --filter-status of {}",
+                response.url(),
+                self.filter_code
+            );
+            log::trace!("exit: should_filter_response -> true");
+            return true;
+        }
+
+        log::trace!("exit: should_filter_response -> false");
+        false
+    }
+
+    /// Compare one StatusCodeFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/filters/tests.rs

@@ -0,0 +1,215 @@
+use super::*;
+use ::fuzzyhash::FuzzyHash;
+use ::regex::Regex;
+
+#[test]
+/// simply test the default values for wildcardfilter, expect 0, 0
+fn wildcard_filter_default() {
+    let wcf = WildcardFilter::default();
+    assert_eq!(wcf.size, u64::MAX);
+    assert_eq!(wcf.dynamic, u64::MAX);
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn wildcard_filter_as_any() {
+    let filter = WildcardFilter::default();
+    let filter2 = WildcardFilter::default();
+
+    assert!(filter.box_eq(filter2.as_any()));
+
+    assert_eq!(
+        *filter.as_any().downcast_ref::<WildcardFilter>().unwrap(),
+        filter
+    );
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn lines_filter_as_any() {
+    let filter = LinesFilter { line_count: 1 };
+    let filter2 = LinesFilter { line_count: 1 };
+
+    assert!(filter.box_eq(filter2.as_any()));
+
+    assert_eq!(filter.line_count, 1);
+    assert_eq!(
+        *filter.as_any().downcast_ref::<LinesFilter>().unwrap(),
+        filter
+    );
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn words_filter_as_any() {
+    let filter = WordsFilter { word_count: 1 };
+    let filter2 = WordsFilter { word_count: 1 };
+
+    assert!(filter.box_eq(filter2.as_any()));
+
+    assert_eq!(filter.word_count, 1);
+    assert_eq!(
+        *filter.as_any().downcast_ref::<WordsFilter>().unwrap(),
+        filter
+    );
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn size_filter_as_any() {
+    let filter = SizeFilter { content_length: 1 };
+    let filter2 = SizeFilter { content_length: 1 };
+
+    assert!(filter.box_eq(filter2.as_any()));
+
+    assert_eq!(filter.content_length, 1);
+    assert_eq!(
+        *filter.as_any().downcast_ref::<SizeFilter>().unwrap(),
+        filter
+    );
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn status_code_filter_as_any() {
+    let filter = StatusCodeFilter { filter_code: 200 };
+    let filter2 = StatusCodeFilter { filter_code: 200 };
+
+    assert!(filter.box_eq(filter2.as_any()));
+
+    assert_eq!(filter.filter_code, 200);
+    assert_eq!(
+        *filter.as_any().downcast_ref::<StatusCodeFilter>().unwrap(),
+        filter
+    );
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn regex_filter_as_any() {
+    let raw = r".*\.txt$";
+    let compiled = Regex::new(raw).unwrap();
+    let compiled2 = Regex::new(raw).unwrap();
+    let filter = RegexFilter {
+        compiled,
+        raw_string: raw.to_string(),
+    };
+    let filter2 = RegexFilter {
+        compiled: compiled2,
+        raw_string: raw.to_string(),
+    };
+
+    assert!(filter.box_eq(filter2.as_any()));
+
+    assert_eq!(filter.raw_string, r".*\.txt$");
+    assert_eq!(
+        *filter.as_any().downcast_ref::<RegexFilter>().unwrap(),
+        filter
+    );
+}
+
+#[test]
+/// test should_filter on WilcardFilter where static logic matches
+fn wildcard_should_filter_when_static_wildcard_found() {
+    let mut resp = FeroxResponse::default();
+    resp.set_wildcard(true);
+    resp.set_url("http://localhost");
+    resp.set_text(
+        "pellentesque diam volutpat commodo sed egestas egestas fringilla phasellus faucibus",
+    );
+
+    let filter = WildcardFilter {
+        size: 83,
+        dynamic: 0,
+        dont_filter: false,
+    };
+
+    assert!(filter.should_filter_response(&resp));
+}
+
+#[test]
+/// test should_filter on WilcardFilter where dynamic logic matches
+fn wildcard_should_filter_when_dynamic_wildcard_found() {
+    let mut resp = FeroxResponse::default();
+    resp.set_wildcard(true);
+    resp.set_url("http://localhost/stuff");
+    resp.set_text("pellentesque diam volutpat commodo sed egestas egestas fringilla");
+
+    let filter = WildcardFilter {
+        size: 0,
+        dynamic: 59, // content-length - 5 (len('stuff'))
+        dont_filter: false,
+    };
+
+    println!("resp: {:?}: filter: {:?}", resp, filter);
+
+    assert!(filter.should_filter_response(&resp));
+}
+
+#[test]
+/// test should_filter on RegexFilter where regex matches body
+fn regexfilter_should_filter_when_regex_matches_on_response_body() {
+    let mut resp = FeroxResponse::default();
+    resp.set_url("http://localhost/stuff");
+    resp.set_text("im a body response hurr durr!");
+
+    let raw = r"response...rr";
+
+    let filter = RegexFilter {
+        raw_string: raw.to_string(),
+        compiled: Regex::new(raw).unwrap(),
+    };
+
+    assert!(filter.should_filter_response(&resp));
+}
+
+#[test]
+/// a few simple tests for similarity filter
+fn similarity_filter_is_accurate() {
+    let mut resp = FeroxResponse::default();
+    resp.set_url("http://localhost/stuff");
+    resp.set_text("sitting");
+
+    let mut filter = SimilarityFilter {
+        text: FuzzyHash::new("kitten").to_string(),
+        threshold: 95,
+    };
+
+    // kitten/sitting is 57% similar, so a threshold of 95 should not be filtered
+    assert!(!filter.should_filter_response(&resp));
+
+    resp.set_text("");
+    filter.text = String::new();
+    filter.threshold = 100;
+
+    // two empty strings are the same, however ssdeep doesn't accept empty strings, expect false
+    assert!(!filter.should_filter_response(&resp));
+
+    resp.set_text("some data to hash for the purposes of running a test");
+    filter.text = FuzzyHash::new("some data to hash for the purposes of running a te").to_string();
+    filter.threshold = 17;
+
+    assert!(filter.should_filter_response(&resp));
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn similarity_filter_as_any() {
+    let filter = SimilarityFilter {
+        text: String::from("stuff"),
+        threshold: 95,
+    };
+
+    let filter2 = SimilarityFilter {
+        text: String::from("stuff"),
+        threshold: 95,
+    };
+
+    assert!(filter.box_eq(filter2.as_any()));
+
+    assert_eq!(filter.text, "stuff");
+    assert_eq!(
+        *filter.as_any().downcast_ref::<SimilarityFilter>().unwrap(),
+        filter
+    );
+}

src/filters/wildcard.rs

@@ -0,0 +1,100 @@
+use super::*;
+use crate::url::FeroxUrl;
+
+/// Data holder for two pieces of data needed when auto-filtering out wildcard responses
+///
+/// `dynamic` is the size of the response that will later be combined with the length
+/// of the path of the url requested and used to determine interesting pages from custom
+/// 404s where the requested url is reflected back in the response
+///
+/// `size` is size of the response that should be included with filters passed via runtime
+/// configuration and any static wildcard lengths.
+#[derive(Debug, Clone, PartialEq)]
+pub struct WildcardFilter {
+    /// size of the response that will later be combined with the length of the path of the url
+    /// requested
+    pub dynamic: u64,
+
+    /// size of the response that should be included with filters passed via runtime configuration
+    pub size: u64,
+
+    /// whether or not the user passed -D on the command line
+    pub(super) dont_filter: bool,
+}
+
+/// implementation of WildcardFilter
+impl WildcardFilter {
+    /// given a boolean representing whether -D was used or not, create a new WildcardFilter
+    pub fn new(dont_filter: bool) -> Self {
+        Self {
+            dont_filter,
+            ..Default::default()
+        }
+    }
+}
+
+/// implement default that populates both values with u64::MAX
+impl Default for WildcardFilter {
+    /// populate both values with u64::MAX
+    fn default() -> Self {
+        Self {
+            dont_filter: false,
+            size: u64::MAX,
+            dynamic: u64::MAX,
+        }
+    }
+}
+
+/// implementation of FeroxFilter for WildcardFilter
+impl FeroxFilter for WildcardFilter {
+    /// Examine size, dynamic, and content_len to determine whether or not the response received
+    /// is a wildcard response and therefore should be filtered out
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        // quick return if dont_filter is set
+        if self.dont_filter {
+            // --dont-filter applies specifically to wildcard filters, it is not a 100% catch all
+            // for not filtering anything.  As such, it should live in the implementation of
+            // a wildcard filter
+            return false;
+        }
+
+        if self.size != u64::MAX && self.size == response.content_length() {
+            // static wildcard size found during testing
+            // size isn't default, size equals response length, and auto-filter is on
+            log::debug!("static wildcard: filtered out {}", response.url());
+            log::trace!("exit: should_filter_response -> true");
+            return true;
+        }
+
+        if self.dynamic != u64::MAX {
+            // dynamic wildcard offset found during testing
+
+            // I'm about to manually split this url path instead of using reqwest::Url's
+            // builtin parsing. The reason is that they call .split() on the url path
+            // except that I don't want an empty string taking up the last index in the
+            // event that the url ends with a forward slash.  It's ugly enough to be split
+            // into its own function for readability.
+            let url_len = FeroxUrl::path_length_of_url(&response.url());
+
+            if url_len + self.dynamic == response.content_length() {
+                log::debug!("dynamic wildcard: filtered out {}", response.url());
+                log::trace!("exit: should_filter_response -> true");
+                return true;
+            }
+        }
+        log::trace!("exit: should_filter_response -> false");
+        false
+    }
+
+    /// Compare one WildcardFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/filters/words.rs

@@ -0,0 +1,33 @@
+use super::*;
+
+/// Simple implementor of FeroxFilter; used to filter out responses based on the number of words
+/// in a Response body; specified using -W|--filter-words
+#[derive(Default, Debug, PartialEq)]
+pub struct WordsFilter {
+    /// Number of words in a Response's body that should be filtered
+    pub word_count: usize,
+}
+
+/// implementation of FeroxFilter for WordsFilter
+impl FeroxFilter for WordsFilter {
+    /// Check `word_count` against what was passed in via -W|--filter-words
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        let result = response.word_count() == self.word_count;
+
+        log::trace!("exit: should_filter_response -> {}", result);
+
+        result
+    }
+
+    /// Compare one WordsFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/heuristics.rs

@@ -1,247 +1,254 @@
-use crate::{
-    config::{CONFIGURATION, PROGRESS_PRINTER},
-    filters::WildcardFilter,
-    scanner::should_filter_response,
-    utils::{ferox_print, format_url, get_url_path_length, make_request, status_colorizer},
-    FeroxResponse,
-};
+use std::sync::Arc;
+
+use anyhow::{bail, Result};
 use console::style;
-use indicatif::ProgressBar;
-use tokio::sync::mpsc::UnboundedSender;
 use uuid::Uuid;
 
+use crate::{
+    config::OutputLevel,
+    event_handlers::{Command, Handles},
+    filters::WildcardFilter,
+    progress::PROGRESS_PRINTER,
+    response::FeroxResponse,
+    skip_fail,
+    url::FeroxUrl,
+    utils::{ferox_print, fmt_err, make_request, status_colorizer},
+};
+
 /// length of a standard UUID, used when determining wildcard responses
 const UUID_LENGTH: u64 = 32;
 
-/// Simple helper to return a uuid, formatted as lowercase without hyphens
-///
-/// `length` determines the number of uuids to string together. Each uuid
-/// is 32 characters long. So, a length of 1 return a 32 character string,
-/// a length of 2 returns a 64 character string, and so on...
-fn unique_string(length: usize) -> String {
-    log::trace!("enter: unique_string({})", length);
-    let mut ids = vec![];
-
-    for _ in 0..length {
-        ids.push(Uuid::new_v4().to_simple().to_string());
-    }
-
-    let unique_id = ids.join("");
-
-    log::trace!("exit: unique_string -> {}", unique_id);
-    unique_id
+/// wrapper around ugly string formatting
+macro_rules! format_template {
+    ($template:expr, $length:expr) => {
+        format!(
+            $template,
+            status_colorizer("WLD"),
+            "-",
+            "-",
+            "-",
+            style("auto-filtering").yellow(),
+            style($length).cyan(),
+            style("--dont-filter").yellow()
+        )
+    };
 }
 
-/// Tests the given url to see if it issues a wildcard response
-///
-/// In the event that url returns a wildcard response, a
-/// [WildcardFilter](struct.WildcardFilter.html) is created and returned to the caller.
-pub async fn wildcard_test(
-    target_url: &str,
-    bar: ProgressBar,
-    tx_term: UnboundedSender<FeroxResponse>,
-) -> Option<WildcardFilter> {
-    log::trace!(
-        "enter: wildcard_test({:?}, {:?}, {:?})",
-        target_url,
-        bar,
-        tx_term
-    );
+/// container for heuristics related info
+pub struct HeuristicTests {
+    /// Handles object for event handler interaction
+    handles: Arc<Handles>,
+}
 
-    if CONFIGURATION.dont_filter {
-        // early return, dont_filter scans don't need tested
-        log::trace!("exit: wildcard_test -> None");
-        return None;
+/// HeuristicTests implementation
+impl HeuristicTests {
+    /// create a new HeuristicTests struct
+    pub fn new(handles: Arc<Handles>) -> Self {
+        Self { handles }
     }
 
-    let tx_clone_one = tx_term.clone();
-    let tx_clone_two = tx_term.clone();
+    /// Simple helper to return a uuid, formatted as lowercase without hyphens
+    ///
+    /// `length` determines the number of uuids to string together. Each uuid
+    /// is 32 characters long. So, a length of 1 return a 32 character string,
+    /// a length of 2 returns a 64 character string, and so on...
+    fn unique_string(&self, length: usize) -> String {
+        log::trace!("enter: unique_string({})", length);
+        let mut ids = vec![];
 
-    if let Some(ferox_response) = make_wildcard_request(&target_url, 1, tx_clone_one).await {
-        bar.inc(1);
+        for _ in 0..length {
+            ids.push(Uuid::new_v4().to_simple().to_string());
+        }
+
+        let unique_id = ids.join("");
+
+        log::trace!("exit: unique_string -> {}", unique_id);
+        unique_id
+    }
+
+    /// wrapper for sending a filter to the filters event handler
+    fn send_filter(&self, filter: WildcardFilter) -> Result<()> {
+        self.handles
+            .filters
+            .send(Command::AddFilter(Box::new(filter)))
+    }
+
+    /// Tests the given url to see if it issues a wildcard response
+    ///
+    /// In the event that url returns a wildcard response, a
+    /// [WildcardFilter](struct.WildcardFilter.html) is created and sent to the filters event
+    /// handler.
+    ///
+    /// Returns the number of times to increment the caller's progress bar
+    pub async fn wildcard(&self, target_url: &str) -> Result<u64> {
+        log::trace!("enter: wildcard_test({:?})", target_url);
+
+        if self.handles.config.dont_filter {
+            // early return, dont_filter scans don't need tested
+            log::trace!("exit: wildcard_test -> 0");
+            return Ok(0);
+        }
+
+        let ferox_url = FeroxUrl::from_string(target_url, self.handles.clone());
+
+        let ferox_response = self.make_wildcard_request(&ferox_url, 1).await?;
 
         // found a wildcard response
-        let mut wildcard = WildcardFilter::default();
+        let mut wildcard = WildcardFilter::new(self.handles.config.dont_filter);
 
         let wc_length = ferox_response.content_length();
 
         if wc_length == 0 {
-            log::trace!("exit: wildcard_test -> Some({:?})", wildcard);
-            return Some(wildcard);
+            log::trace!("exit: wildcard_test -> 1");
+            self.send_filter(wildcard)?;
+            return Ok(1);
         }
 
         // content length of wildcard is non-zero, perform additional tests:
         //   make a second request, with a known-sized (64) longer request
-        if let Some(resp_two) = make_wildcard_request(&target_url, 3, tx_clone_two).await {
-            bar.inc(1);
+        let resp_two = self.make_wildcard_request(&ferox_url, 3).await?;
 
-            let wc2_length = resp_two.content_length();
+        let wc2_length = resp_two.content_length();
 
-            if wc2_length == wc_length + (UUID_LENGTH * 2) {
-                // second length is what we'd expect to see if the requested url is
-                // reflected in the response along with some static content; aka custom 404
-                let url_len = get_url_path_length(&ferox_response.url());
+        if wc2_length == wc_length + (UUID_LENGTH * 2) {
+            // second length is what we'd expect to see if the requested url is
+            // reflected in the response along with some static content; aka custom 404
+            let url_len = ferox_url.path_length()?;
 
-                wildcard.dynamic = wc_length - url_len;
+            wildcard.dynamic = wc_length - url_len;
 
-                if !CONFIGURATION.quiet {
-                    let msg = format!(
-                            "{} {:>9} {:>9} {:>9} Wildcard response is dynamic; {} ({} + url length) responses; toggle this behavior by using {}\n",
-                            status_colorizer("WLD"),
-                            "-",
-                            "-",
-                            "-",
-                            style("auto-filtering").yellow(),
-                            style(wc_length - url_len).cyan(),
-                            style("--dont-filter").yellow()
-                    );
-
-                    ferox_print(&msg, &PROGRESS_PRINTER);
-                }
-            } else if wc_length == wc2_length {
-                wildcard.size = wc_length;
-
-                if !CONFIGURATION.quiet {
-                    let msg = format!(
-                        "{} {:>9} {:>9} {:>9} Wildcard response is static; {} {} responses; toggle this behavior by using {}\n",
-                        status_colorizer("WLD"),
-                        "-",
-                        "-",
-                        "-",
-                        style("auto-filtering").yellow(),
-                        style(wc_length).cyan(),
-                        style("--dont-filter").yellow()
-                    );
-
-                    ferox_print(&msg, &PROGRESS_PRINTER);
-                }
+            if matches!(
+                self.handles.config.output_level,
+                OutputLevel::Default | OutputLevel::Quiet
+            ) {
+                let msg = format_template!("{} {:>9} {:>9} {:>9} Wildcard response is dynamic; {} ({} + url length) responses; toggle this behavior by using {}\n", wildcard.dynamic);
+                ferox_print(&msg, &PROGRESS_PRINTER);
+            }
+        } else if wc_length == wc2_length {
+            wildcard.size = wc_length;
+
+            if matches!(
+                self.handles.config.output_level,
+                OutputLevel::Default | OutputLevel::Quiet
+            ) {
+                let msg = format_template!("{} {:>9} {:>9} {:>9} Wildcard response is static; {} {} responses; toggle this behavior by using {}\n", wildcard.size);
+                ferox_print(&msg, &PROGRESS_PRINTER);
             }
-        } else {
-            bar.inc(2);
         }
 
-        log::trace!("exit: wildcard_test -> Some({:?})", wildcard);
-        return Some(wildcard);
+        self.send_filter(wildcard)?;
+
+        log::trace!("exit: wildcard_test");
+        Ok(2)
     }
 
-    log::trace!("exit: wildcard_test -> None");
-    None
-}
+    /// Generates a uuid and appends it to the given target url. The reasoning is that the randomly
+    /// generated unique string should not exist on and be served by the target web server.
+    ///
+    /// Once the unique url is created, the request is sent to the server. If the server responds
+    /// back with a valid status code, the response is considered to be a wildcard response. If that
+    /// wildcard response has a 3xx status code, that redirection location is displayed to the user.
+    async fn make_wildcard_request(
+        &self,
+        target: &FeroxUrl,
+        length: usize,
+    ) -> Result<FeroxResponse> {
+        log::trace!("enter: make_wildcard_request({}, {})", target, length);
 
-/// Generates a uuid and appends it to the given target url. The reasoning is that the randomly
-/// generated unique string should not exist on and be served by the target web server.
-///
-/// Once the unique url is created, the request is sent to the server. If the server responds
-/// back with a valid status code, the response is considered to be a wildcard response. If that
-/// wildcard response has a 3xx status code, that redirection location is displayed to the user.
-async fn make_wildcard_request(
-    target_url: &str,
-    length: usize,
-    tx_file: UnboundedSender<FeroxResponse>,
-) -> Option<FeroxResponse> {
-    log::trace!(
-        "enter: make_wildcard_request({}, {}, {:?})",
-        target_url,
-        length,
-        tx_file
-    );
+        let unique_str = self.unique_string(length);
+        let nonexistent_url = target.format(&unique_str, None)?;
 
-    let unique_str = unique_string(length);
+        let response = make_request(
+            &self.handles.config.client,
+            &nonexistent_url.to_owned(),
+            self.handles.config.output_level,
+            self.handles.stats.tx.clone(),
+        )
+        .await?;
 
-    let nonexistent = match format_url(
-        target_url,
-        &unique_str,
-        CONFIGURATION.add_slash,
-        &CONFIGURATION.queries,
-        None,
-    ) {
-        Ok(url) => url,
-        Err(e) => {
-            log::error!("{}", e);
-            log::trace!("exit: make_wildcard_request -> None");
-            return None;
-        }
-    };
+        if self
+            .handles
+            .config
+            .status_codes
+            .contains(&response.status().as_u16())
+        {
+            // found a wildcard response
+            let mut ferox_response =
+                FeroxResponse::from(response, true, self.handles.config.output_level).await;
+            ferox_response.set_wildcard(true);
 
-    match make_request(&CONFIGURATION.client, &nonexistent.to_owned()).await {
-        Ok(response) => {
-            if CONFIGURATION
-                .status_codes
-                .contains(&response.status().as_u16())
+            if self
+                .handles
+                .filters
+                .data
+                .should_filter_response(&ferox_response, self.handles.stats.tx.clone())
             {
-                // found a wildcard response
-                let mut ferox_response = FeroxResponse::from(response, true).await;
-                ferox_response.wildcard = true;
+                bail!("filtered response")
+            }
 
-                if !CONFIGURATION.quiet
-                    && !should_filter_response(&ferox_response)
-                    && tx_file.send(ferox_response.clone()).is_err()
-                {
-                    return None;
+            if matches!(
+                self.handles.config.output_level,
+                OutputLevel::Default | OutputLevel::Quiet
+            ) {
+                let boxed = Box::new(ferox_response.clone());
+                self.handles.output.send(Command::Report(boxed))?;
+            }
+
+            log::trace!("exit: make_wildcard_request -> {}", ferox_response);
+            return Ok(ferox_response);
+        }
+
+        log::trace!("exit: make_wildcard_request -> Err");
+        bail!("uninteresting status code")
+    }
+
+    /// Simply tries to connect to all given sites before starting to scan
+    ///
+    /// In the event that no sites can be reached, the program will exit.
+    ///
+    /// Any urls that are found to be alive are returned to the caller.
+    pub async fn connectivity(&self, target_urls: &[String]) -> Result<Vec<String>> {
+        log::trace!("enter: connectivity_test({:?})", target_urls);
+
+        let mut good_urls = vec![];
+
+        for target_url in target_urls {
+            let url = FeroxUrl::from_string(&target_url, self.handles.clone());
+            let request = skip_fail!(url.format("", None));
+            let result = make_request(
+                &self.handles.config.client,
+                &request,
+                self.handles.config.output_level,
+                self.handles.stats.tx.clone(),
+            )
+            .await;
+
+            match result {
+                Ok(_) => {
+                    good_urls.push(target_url.to_owned());
                 }
-
-                log::trace!("exit: make_wildcard_request -> {}", ferox_response);
-                return Some(ferox_response);
-            }
-        }
-        Err(e) => {
-            log::warn!("{}", e);
-            log::trace!("exit: make_wildcard_request -> None");
-            return None;
-        }
-    }
-    log::trace!("exit: make_wildcard_request -> None");
-    None
-}
-
-/// Simply tries to connect to all given sites before starting to scan
-///
-/// In the event that no sites can be reached, the program will exit.
-///
-/// Any urls that are found to be alive are returned to the caller.
-pub async fn connectivity_test(target_urls: &[String]) -> Vec<String> {
-    log::trace!("enter: connectivity_test({:?})", target_urls);
-
-    let mut good_urls = vec![];
-
-    for target_url in target_urls {
-        let request = match format_url(
-            target_url,
-            "",
-            CONFIGURATION.add_slash,
-            &CONFIGURATION.queries,
-            None,
-        ) {
-            Ok(url) => url,
-            Err(e) => {
-                log::error!("{}", e);
-                continue;
-            }
-        };
-
-        match make_request(&CONFIGURATION.client, &request).await {
-            Ok(_) => {
-                good_urls.push(target_url.to_owned());
-            }
-            Err(e) => {
-                if !CONFIGURATION.quiet {
-                    ferox_print(
-                        &format!("Could not connect to {}, skipping...", target_url),
-                        &PROGRESS_PRINTER,
-                    );
+                Err(e) => {
+                    if matches!(
+                        self.handles.config.output_level,
+                        OutputLevel::Default | OutputLevel::Quiet
+                    ) {
+                        ferox_print(
+                            &format!("Could not connect to {}, skipping...", target_url),
+                            &PROGRESS_PRINTER,
+                        );
+                    }
+                    log::warn!("{}", e);
                 }
-                log::error!("{}", e);
             }
         }
+
+        if good_urls.is_empty() {
+            bail!("Could not connect to any target provided");
+        }
+
+        log::trace!("exit: connectivity_test -> {:?}", good_urls);
+        Ok(good_urls)
     }
-
-    if good_urls.is_empty() {
-        log::error!("Could not connect to any target provided, exiting.");
-    }
-
-    log::trace!("exit: connectivity_test -> {:?}", good_urls);
-
-    good_urls
 }
 
 #[cfg(test)]
@@ -251,16 +258,10 @@ mod tests {
     #[test]
     /// request a unique string of 32bytes * a value returns correct result
     fn heuristics_unique_string_returns_correct_length() {
+        let (handles, _) = Handles::for_testing(None, None);
+        let tester = HeuristicTests::new(Arc::new(handles));
         for i in 0..10 {
-            assert_eq!(unique_string(i).len(), i * 32);
+            assert_eq!(tester.unique_string(i).len(), i * 32);
         }
     }
-
-    #[test]
-    /// simply test the default values for wildcardfilter, expect 0, 0
-    fn heuristics_wildcardfilter_dafaults() {
-        let wcf = WildcardFilter::default();
-        assert_eq!(wcf.size, 0);
-        assert_eq!(wcf.dynamic, 0);
-    }
 }

src/lib.rs

@@ -1,56 +1,53 @@
+use anyhow::Result;
+use reqwest::StatusCode;
+use tokio::{
+    sync::mpsc::{UnboundedReceiver, UnboundedSender},
+    task::JoinHandle,
+};
+
+use crate::event_handlers::Command;
+
 pub mod banner;
-pub mod client;
 pub mod config;
-pub mod extractor;
+mod client;
+pub mod event_handlers;
 pub mod filters;
 pub mod heuristics;
 pub mod logger;
-pub mod parser;
+mod parser;
 pub mod progress;
-pub mod reporter;
 pub mod scan_manager;
 pub mod scanner;
+pub mod statistics;
+mod traits;
 pub mod utils;
+mod extractor;
+mod macros;
+mod url;
+mod response;
+mod message;
 
-use crate::utils::{get_url_path_length, status_colorizer};
-use console::{style, Color};
-use reqwest::header::{HeaderName, HeaderValue};
-use reqwest::{header::HeaderMap, Response, StatusCode, Url};
-use serde::{ser::SerializeStruct, Deserialize, Deserializer, Serialize, Serializer};
-use serde_json::Value;
-use std::collections::HashMap;
-use std::convert::{TryFrom, TryInto};
-use std::str::FromStr;
-use std::{error, fmt};
-use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender};
+/// Alias for tokio::sync::mpsc::UnboundedSender<Command>
+pub(crate) type CommandSender = UnboundedSender<Command>;
 
-/// Generic Result type to ease error handling in async contexts
-pub type FeroxResult<T> = std::result::Result<T, Box<dyn error::Error + Send + Sync + 'static>>;
+/// Alias for tokio::sync::mpsc::UnboundedSender<Command>
+pub(crate) type CommandReceiver = UnboundedReceiver<Command>;
 
-/// Simple Error implementation to allow for custom error returns
-#[derive(Debug, Default)]
-pub struct FeroxError {
-    /// fancy string that can be printed via Display
-    pub message: String,
-}
-
-impl error::Error for FeroxError {}
-
-impl fmt::Display for FeroxError {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "{}", &self.message)
-    }
-}
+/// Alias for tokio::task::JoinHandle<anyhow::Result<()>>
+pub(crate) type Joiner = JoinHandle<Result<()>>;
 
 /// Generic mpsc::unbounded_channel type to tidy up some code
-pub type FeroxChannel<T> = (UnboundedSender<T>, UnboundedReceiver<T>);
+pub(crate) type FeroxChannel<T> = (UnboundedSender<T>, UnboundedReceiver<T>);
 
 /// Version pulled from Cargo.toml at compile time
-pub const VERSION: &str = env!("CARGO_PKG_VERSION");
+pub(crate) const VERSION: &str = env!("CARGO_PKG_VERSION");
 
 /// Maximum number of file descriptors that can be opened during a scan
 pub const DEFAULT_OPEN_FILE_LIMIT: usize = 8192;
 
+/// Default value used to determine near-duplicate web pages (equivalent to 95%)
+pub const SIMILARITY_THRESHOLD: u32 = 95;
+
 /// Default wordlist to use when `-w|--wordlist` isn't specified and not `wordlist` isn't set
 /// in a [ferox-config.toml](constant.DEFAULT_CONFIG_NAME.html) config file.
 ///
@@ -60,7 +57,7 @@ pub const DEFAULT_WORDLIST: &str =
     "/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt";
 
 /// Number of milliseconds to wait between polls of `PAUSE_SCAN` when user pauses a scan
-pub static SLEEP_DURATION: u64 = 500;
+pub(crate) static SLEEP_DURATION: u64 = 500;
 
 /// Default list of status codes to report
 ///
@@ -90,443 +87,6 @@ pub const DEFAULT_STATUS_CODES: [StatusCode; 9] = [
 /// Expected location is in the same directory as the feroxbuster binary.
 pub const DEFAULT_CONFIG_NAME: &str = "ferox-config.toml";
 
-/// FeroxSerialize trait; represents different types that are Serialize and also implement
-/// as_str / as_json methods
-pub trait FeroxSerialize: Serialize {
-    /// Return a String representation of the object, generally the human readable version of the
-    /// implementor
-    fn as_str(&self) -> String;
-
-    /// Return an NDJSON representation of the object
-    fn as_json(&self) -> String;
-}
-
-/// A `FeroxResponse`, derived from a `Response` to a submitted `Request`
-#[derive(Debug, Clone)]
-pub struct FeroxResponse {
-    /// The final `Url` of this `FeroxResponse`
-    url: Url,
-
-    /// The `StatusCode` of this `FeroxResponse`
-    status: StatusCode,
-
-    /// The full response text
-    text: String,
-
-    /// The content-length of this response, if known
-    content_length: u64,
-
-    /// The number of lines contained in the body of this response, if known
-    line_count: usize,
-
-    /// The number of words contained in the body of this response, if known
-    word_count: usize,
-
-    /// The `Headers` of this `FeroxResponse`
-    headers: HeaderMap,
-
-    /// Wildcard response status
-    wildcard: bool,
-}
-
-/// Implement Display for FeroxResponse
-impl fmt::Display for FeroxResponse {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(
-            f,
-            "FeroxResponse {{ url: {}, status: {}, content-length: {} }}",
-            self.url(),
-            self.status(),
-            self.content_length()
-        )
-    }
-}
-
-/// `FeroxResponse` implementation
-impl FeroxResponse {
-    /// Get the `StatusCode` of this `FeroxResponse`
-    pub fn status(&self) -> &StatusCode {
-        &self.status
-    }
-
-    /// Get the final `Url` of this `FeroxResponse`.
-    pub fn url(&self) -> &Url {
-        &self.url
-    }
-
-    /// Get the full response text
-    pub fn text(&self) -> &str {
-        &self.text
-    }
-
-    /// Get the `Headers` of this `FeroxResponse`
-    pub fn headers(&self) -> &HeaderMap {
-        &self.headers
-    }
-
-    /// Get the content-length of this response, if known
-    pub fn content_length(&self) -> u64 {
-        self.content_length
-    }
-
-    /// Set `FeroxResponse`'s `url` attribute, has no affect if an error occurs
-    pub fn set_url(&mut self, url: &str) {
-        match Url::parse(&url) {
-            Ok(url) => {
-                self.url = url;
-            }
-            Err(e) => {
-                log::error!("Could not parse {} into a Url: {}", url, e);
-            }
-        };
-    }
-
-    /// Make a reasonable guess at whether the response is a file or not
-    ///
-    /// Examines the last part of a path to determine if it has an obvious extension
-    /// i.e. http://localhost/some/path/stuff.js where stuff.js indicates a file
-    ///
-    /// Additionally, inspects query parameters, as they're also often indicative of a file
-    pub fn is_file(&self) -> bool {
-        let has_extension = match self.url.path_segments() {
-            Some(path) => {
-                if let Some(last) = path.last() {
-                    last.contains('.') // last segment has some sort of extension, probably
-                } else {
-                    false
-                }
-            }
-            None => false,
-        };
-
-        self.url.query_pairs().count() > 0 || has_extension
-    }
-
-    /// Returns line count of the response text.
-    pub fn line_count(&self) -> usize {
-        self.line_count
-    }
-
-    /// Returns word count of the response text.
-    pub fn word_count(&self) -> usize {
-        self.word_count
-    }
-
-    /// Create a new `FeroxResponse` from the given `Response`
-    pub async fn from(response: Response, read_body: bool) -> Self {
-        let url = response.url().clone();
-        let status = response.status();
-        let headers = response.headers().clone();
-        let content_length = response.content_length().unwrap_or(0);
-
-        let text = if read_body {
-            // .text() consumes the response, must be called last
-            // additionally, --extract-links is currently the only place we use the body of the
-            // response, so we forego the processing if not performing extraction
-            match response.text().await {
-                // await the response's body
-                Ok(text) => text,
-                Err(e) => {
-                    log::error!("Could not parse body from response: {}", e);
-                    String::new()
-                }
-            }
-        } else {
-            String::new()
-        };
-
-        let line_count = text.lines().count();
-        let word_count = text.lines().map(|s| s.split_whitespace().count()).sum();
-
-        FeroxResponse {
-            url,
-            status,
-            content_length,
-            text,
-            headers,
-            line_count,
-            word_count,
-            wildcard: false,
-        }
-    }
-}
-
-/// Implement FeroxSerialusize::from(ize for FeroxRespons)e
-impl FeroxSerialize for FeroxResponse {
-    /// Simple wrapper around create_report_string
-    fn as_str(&self) -> String {
-        let lines = self.line_count().to_string();
-        let words = self.word_count().to_string();
-        let chars = self.content_length().to_string();
-        let status = self.status().as_str();
-        let wild_status = status_colorizer("WLD");
-
-        if self.wildcard {
-            // response is a wildcard, special messages abound when this is the case...
-
-            // create the base message
-            let mut message = format!(
-                "{} {:>8}l {:>8}w {:>8}c Got {} for {} (url length: {})\n",
-                wild_status,
-                lines,
-                words,
-                chars,
-                status_colorizer(&status),
-                self.url(),
-                get_url_path_length(&self.url())
-            );
-
-            if self.status().is_redirection() {
-                // when it's a redirect, show where it goes, if possible
-                if let Some(next_loc) = self.headers().get("Location") {
-                    let next_loc_str = next_loc.to_str().unwrap_or("Unknown");
-
-                    let redirect_msg = format!(
-                        "{} {:>9} {:>9} {:>9} {} redirects to => {}\n",
-                        wild_status,
-                        "-",
-                        "-",
-                        "-",
-                        self.url(),
-                        next_loc_str
-                    );
-
-                    message.push_str(&redirect_msg);
-                }
-            }
-
-            // base message + redirection message (if appropriate)
-            message
-        } else {
-            // not a wildcard, just create a normal entry
-            utils::create_report_string(
-                self.status.as_str(),
-                &lines,
-                &words,
-                &chars,
-                self.url().as_str(),
-            )
-        }
-    }
-
-    /// Create an NDJSON representation of the FeroxResponse
-    ///
-    /// (expanded for clarity)
-    /// ex:
-    /// {
-    ///    "type":"response",
-    ///    "url":"https://localhost.com/images",
-    ///    "path":"/images",
-    ///    "status":301,
-    ///    "content_length":179,
-    ///    "line_count":10,
-    ///    "word_count":16,
-    ///    "headers":{
-    ///       "x-content-type-options":"nosniff",
-    ///       "strict-transport-security":"max-age=31536000; includeSubDomains",
-    ///       "x-frame-options":"SAMEORIGIN",
-    ///       "connection":"keep-alive",
-    ///       "server":"nginx/1.16.1",
-    ///       "content-type":"text/html; charset=UTF-8",
-    ///       "referrer-policy":"origin-when-cross-origin",
-    ///       "content-security-policy":"default-src 'none'",
-    ///       "access-control-allow-headers":"X-Requested-With",
-    ///       "x-xss-protection":"1; mode=block",
-    ///       "content-length":"179",
-    ///       "date":"Mon, 23 Nov 2020 15:33:24 GMT",
-    ///       "location":"/images/",
-    ///       "access-control-allow-origin":"https://localhost.com"
-    ///    }
-    /// }\n
-    fn as_json(&self) -> String {
-        if let Ok(mut json) = serde_json::to_string(&self) {
-            json.push('\n');
-            json
-        } else {
-            format!("{{\"error\":\"could not convert {} to json\"}}", self.url())
-        }
-    }
-}
-
-/// Serialize implementation for FeroxResponse
-impl Serialize for FeroxResponse {
-    /// Function that handles serialization of a FeroxResponse to NDJSON
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        let mut headers = HashMap::new();
-        let mut state = serializer.serialize_struct("FeroxResponse", 7)?;
-
-        // need to convert the HeaderMap to a HashMap in order to pass it to the serializer
-        for (key, value) in &self.headers {
-            let k = key.as_str().to_owned();
-            let v = String::from_utf8_lossy(value.as_bytes());
-            headers.insert(k, v);
-        }
-
-        state.serialize_field("type", "response")?;
-        state.serialize_field("url", self.url.as_str())?;
-        state.serialize_field("path", self.url.path())?;
-        state.serialize_field("wildcard", &self.wildcard)?;
-        state.serialize_field("status", &self.status.as_u16())?;
-        state.serialize_field("content_length", &self.content_length)?;
-        state.serialize_field("line_count", &self.line_count)?;
-        state.serialize_field("word_count", &self.word_count)?;
-        state.serialize_field("headers", &headers)?;
-
-        state.end()
-    }
-}
-
-/// Deserialize implementation for FeroxResponse
-impl<'de> Deserialize<'de> for FeroxResponse {
-    /// Deserialize a FeroxResponse from a serde_json::Value
-    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
-    where
-        D: Deserializer<'de>,
-    {
-        let mut response = Self {
-            url: Url::parse("http://localhost").unwrap(),
-            status: StatusCode::OK,
-            text: String::new(),
-            content_length: 0,
-            headers: HeaderMap::new(),
-            wildcard: false,
-            line_count: 0,
-            word_count: 0,
-        };
-
-        let map: HashMap<String, Value> = HashMap::deserialize(deserializer)?;
-
-        for (key, value) in &map {
-            match key.as_str() {
-                "url" => {
-                    if let Some(url) = value.as_str() {
-                        if let Ok(parsed) = Url::parse(url) {
-                            response.url = parsed;
-                        }
-                    }
-                }
-                "status" => {
-                    if let Some(num) = value.as_u64() {
-                        if let Ok(smaller) = u16::try_from(num) {
-                            if let Ok(status) = StatusCode::from_u16(smaller) {
-                                response.status = status;
-                            }
-                        }
-                    }
-                }
-                "content_length" => {
-                    if let Some(num) = value.as_u64() {
-                        response.content_length = num;
-                    }
-                }
-                "line_count" => {
-                    if let Some(num) = value.as_u64() {
-                        response.line_count = num.try_into().unwrap_or_default();
-                    }
-                }
-                "word_count" => {
-                    if let Some(num) = value.as_u64() {
-                        response.word_count = num.try_into().unwrap_or_default();
-                    }
-                }
-                "headers" => {
-                    let mut headers = HeaderMap::<HeaderValue>::default();
-
-                    if let Some(map_headers) = value.as_object() {
-                        for (h_key, h_value) in map_headers {
-                            let h_value_str = h_value.as_str().unwrap_or("");
-                            let h_name = HeaderName::from_str(h_key)
-                                .unwrap_or_else(|_| HeaderName::from_str("Unknown").unwrap());
-                            let h_value_parsed = HeaderValue::from_str(h_value_str)
-                                .unwrap_or_else(|_| HeaderValue::from_str("Unknown").unwrap());
-                            headers.insert(h_name, h_value_parsed);
-                        }
-                    }
-
-                    response.headers = headers;
-                }
-                "wildcard" => {
-                    if let Some(result) = value.as_bool() {
-                        response.wildcard = result;
-                    }
-                }
-                _ => {}
-            }
-        }
-
-        Ok(response)
-    }
-}
-
-#[derive(Serialize, Deserialize, Default)]
-/// Representation of a log entry, can be represented as a human readable string or JSON
-pub struct FeroxMessage {
-    #[serde(rename = "type")]
-    /// Name of this type of struct, used for serialization, i.e. `{"type":"log"}`
-    kind: String,
-
-    /// The log message
-    pub message: String,
-
-    /// The log level
-    pub level: String,
-
-    /// The number of seconds elapsed since the scan started
-    pub time_offset: f32,
-
-    /// The module from which log::* was called
-    pub module: String,
-}
-
-/// Implementation of FeroxMessage
-impl FeroxSerialize for FeroxMessage {
-    /// Create an NDJSON representation of the log message
-    ///
-    /// (expanded for clarity)
-    /// ex:
-    /// {
-    ///   "type": "log",
-    ///   "message": "Sent https://localhost/api to file handler",
-    ///   "level": "DEBUG",
-    ///   "time_offset": 0.86333454,
-    ///   "module": "feroxbuster::reporter"
-    /// }\n
-    fn as_json(&self) -> String {
-        if let Ok(mut json) = serde_json::to_string(&self) {
-            json.push('\n');
-            json
-        } else {
-            String::from("{\"error\":\"could not convert to json\"}")
-        }
-    }
-
-    /// Create a string representation of the log message
-    ///
-    /// ex:  301       10l       16w      173c https://localhost/api
-    fn as_str(&self) -> String {
-        let (level_name, level_color) = match self.level.as_str() {
-            "ERROR" => ("ERR", Color::Red),
-            "WARN" => ("WRN", Color::Red),
-            "INFO" => ("INF", Color::Cyan),
-            "DEBUG" => ("DBG", Color::Yellow),
-            "TRACE" => ("TRC", Color::Magenta),
-            "WILDCARD" => ("WLD", Color::Cyan),
-            _ => ("UNK", Color::White),
-        };
-
-        format!(
-            "{} {:10.03} {} {}\n",
-            style(level_name).bg(level_color).black(),
-            style(self.time_offset).dim(),
-            self.module,
-            style(&self.message).dim(),
-        )
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -551,46 +111,4 @@ mod tests {
     fn default_version() {
         assert_eq!(VERSION, env!("CARGO_PKG_VERSION"));
     }
-
-    #[test]
-    /// test as_str method of FeroxMessage
-    fn ferox_message_as_str_returns_string_with_newline() {
-        let message = FeroxMessage {
-            message: "message".to_string(),
-            module: "utils".to_string(),
-            time_offset: 1.0,
-            level: "INFO".to_string(),
-            kind: "log".to_string(),
-        };
-        let message_str = message.as_str();
-
-        assert!(message_str.contains("INF"));
-        assert!(message_str.contains("1.000"));
-        assert!(message_str.contains("utils"));
-        assert!(message_str.contains("message"));
-        assert!(message_str.ends_with('\n'));
-    }
-
-    #[test]
-    /// test as_json method of FeroxMessage
-    fn ferox_message_as_json_returns_json_representation_of_ferox_message_with_newline() {
-        let message = FeroxMessage {
-            message: "message".to_string(),
-            module: "utils".to_string(),
-            time_offset: 1.0,
-            level: "INFO".to_string(),
-            kind: "log".to_string(),
-        };
-
-        let message_str = message.as_json();
-
-        let error_margin = f32::EPSILON;
-
-        let json: FeroxMessage = serde_json::from_str(&message_str).unwrap();
-        assert_eq!(json.module, message.module);
-        assert_eq!(json.message, message.message);
-        assert!((json.time_offset - message.time_offset).abs() < error_margin);
-        assert_eq!(json.level, message.level);
-        assert_eq!(json.kind, message.kind);
-    }
 }

src/logger.rs

@@ -1,24 +1,31 @@
-use crate::{
-    config::{CONFIGURATION, PROGRESS_PRINTER},
-    reporter::safe_file_write,
-    utils::open_file,
-    FeroxMessage, FeroxSerialize,
-};
-use env_logger::Builder;
 use std::env;
+use std::fs::OpenOptions;
+use std::io::BufWriter;
+use std::sync::{Arc, RwLock};
 use std::time::Instant;
 
+use anyhow::{Context, Result};
+use env_logger::Builder;
+
+use crate::{
+    config::Configuration,
+    message::FeroxMessage,
+    progress::PROGRESS_PRINTER,
+    traits::FeroxSerialize,
+    utils::{fmt_err, write_to},
+};
+
 /// Create a customized instance of
 /// [env_logger::Logger](https://docs.rs/env_logger/latest/env_logger/struct.Logger.html)
 /// with timer offset/color and set the log level based on `verbosity`
-pub fn initialize(verbosity: u8) {
+pub fn initialize(config: Arc<Configuration>) -> Result<()> {
     // use occurrences of -v on commandline to or verbosity = N in feroxconfig.toml to set
     // log level for the application; respects already specified RUST_LOG environment variable
     match env::var("RUST_LOG") {
         Ok(_) => {} // RUST_LOG found, don't override
         Err(_) => {
             // only set log level based on verbosity when RUST_LOG variable doesn't exist
-            match verbosity {
+            match config.verbosity {
                 0 => (),
                 1 => env::set_var("RUST_LOG", "warn"),
                 2 => env::set_var("RUST_LOG", "info"),
@@ -31,12 +38,22 @@ pub fn initialize(verbosity: u8) {
     let start = Instant::now();
     let mut builder = Builder::from_default_env();
 
-    let debug_file = open_file(&CONFIGURATION.debug_log);
+    let file = if !config.debug_log.is_empty() {
+        let f = OpenOptions::new() // std fs
+            .create(true)
+            .append(true)
+            .open(&config.debug_log)
+            .with_context(|| fmt_err(&format!("Could not open {}", &config.debug_log)))?;
+
+        let mut writer = BufWriter::new(f);
 
-    if let Some(buffered_file) = debug_file.clone() {
         // write out the configuration to the debug file if it exists
-        safe_file_write(&*CONFIGURATION, buffered_file, CONFIGURATION.json);
-    }
+        write_to(&*config, &mut writer, config.json)?;
+
+        Some(Arc::new(RwLock::new(writer)))
+    } else {
+        None
+    };
 
     builder
         .format(move |_, record| {
@@ -50,11 +67,15 @@ pub fn initialize(verbosity: u8) {
 
             PROGRESS_PRINTER.println(&log_entry.as_str());
 
-            if let Some(buffered_file) = debug_file.clone() {
-                safe_file_write(&log_entry, buffered_file, CONFIGURATION.json);
+            if let Some(buffered_file) = file.clone() {
+                if let Ok(mut unlocked) = buffered_file.write() {
+                    let _ = write_to(&log_entry, &mut unlocked, config.json);
+                }
             }
 
             Ok(())
         })
         .init();
+
+    Ok(())
 }

src/macros.rs

@@ -0,0 +1,23 @@
+#![macro_use]
+
+#[macro_export]
+/// wrapper to improve code readability
+macro_rules! send_command {
+    ($tx:expr, $value:expr) => {
+        $tx.send($value).unwrap_or_default();
+    };
+}
+
+#[macro_export]
+/// while looping, check for a Result, if Ok return the value, if Err, continue
+macro_rules! skip_fail {
+    ($res:expr) => {
+        match $res {
+            Ok(val) => val,
+            Err(e) => {
+                log::warn!("{}", fmt_err(&format!("{}; skipping...", e)));
+                continue;
+            }
+        }
+    };
+}

src/main.rs

@@ -1,84 +1,47 @@
-use crossterm::event::{self, Event, KeyCode};
-use feroxbuster::progress::add_bar;
-use feroxbuster::{
-    banner,
-    config::{CONFIGURATION, PROGRESS_BAR, PROGRESS_PRINTER},
-    heuristics, logger, reporter,
-    scan_manager::{self, PAUSE_SCAN},
-    scanner::{self, scan_url, RESPONSES, SCANNED_URLS},
-    utils::{ferox_print, get_current_depth, module_colorizer, status_colorizer},
-    FeroxError, FeroxResponse, FeroxResult, FeroxSerialize, SLEEP_DURATION, VERSION,
-};
-#[cfg(not(target_os = "windows"))]
-use feroxbuster::{utils::set_open_file_limit, DEFAULT_OPEN_FILE_LIMIT};
-use futures::StreamExt;
-use std::convert::TryInto;
 use std::{
     collections::HashSet,
     fs::File,
     io::{stderr, BufRead, BufReader},
-    process,
-    sync::{
-        atomic::{AtomicBool, Ordering},
-        Arc,
-    },
-    time::Duration,
+    sync::{atomic::Ordering, Arc},
 };
-use tokio::{io, sync::mpsc::UnboundedSender, task::JoinHandle};
+
+use anyhow::{bail, Context, Result};
+use futures::StreamExt;
+use tokio::{io, sync::oneshot};
 use tokio_util::codec::{FramedRead, LinesCodec};
 
-/// Atomic boolean flag, used to determine whether or not the terminal input handler should exit
-pub static SCAN_COMPLETE: AtomicBool = AtomicBool::new(false);
-
-/// Handles specific key events triggered by the user over stdin
-fn terminal_input_handler() {
-    log::trace!("enter: terminal_input_handler");
-
-    loop {
-        if event::poll(Duration::from_millis(SLEEP_DURATION)).unwrap_or(false) {
-            // It's guaranteed that the `read()` won't block when the `poll()`
-            // function returns `true`
-
-            if let Ok(key_pressed) = event::read() {
-                if key_pressed == Event::Key(KeyCode::Enter.into()) {
-                    // if the user presses Enter, toggle the value stored in PAUSE_SCAN
-                    // ignore any other keys
-                    let current = PAUSE_SCAN.load(Ordering::Acquire);
-
-                    PAUSE_SCAN.store(!current, Ordering::Release);
-                }
-            }
-        } else {
-            // Timeout expired and no `Event` is available; use the timeout to check SCAN_COMPLETE
-            if SCAN_COMPLETE.load(Ordering::Relaxed) {
-                // scan has been marked complete by main, time to exit the loop
-                break;
-            }
-        }
-    }
-    log::trace!("exit: terminal_input_handler");
-}
+use feroxbuster::{
+    banner::{Banner, UPDATE_URL},
+    config::{Configuration, OutputLevel},
+    event_handlers::{
+        Command::{CreateBar, Exit, JoinTasks, LoadStats, ScanInitialUrls, UpdateWordlist},
+        FiltersHandler, Handles, ScanHandler, StatsHandler, Tasks, TermInputHandler,
+        TermOutHandler, SCAN_COMPLETE,
+    },
+    filters, heuristics, logger,
+    progress::{PROGRESS_BAR, PROGRESS_PRINTER},
+    scan_manager::{self},
+    scanner,
+    utils::fmt_err,
+};
+#[cfg(not(target_os = "windows"))]
+use feroxbuster::{utils::set_open_file_limit, DEFAULT_OPEN_FILE_LIMIT};
 
 /// Create a HashSet of Strings from the given wordlist then stores it inside an Arc
-fn get_unique_words_from_wordlist(path: &str) -> FeroxResult<Arc<HashSet<String>>> {
+fn get_unique_words_from_wordlist(path: &str) -> Result<Arc<HashSet<String>>> {
     log::trace!("enter: get_unique_words_from_wordlist({})", path);
 
-    let file = match File::open(&path) {
-        Ok(f) => f,
-        Err(e) => {
-            log::error!("Could not open wordlist: {}", e);
-            log::trace!("exit: get_unique_words_from_wordlist -> {}", e);
-
-            return Err(Box::new(e));
-        }
-    };
+    let file = File::open(&path).with_context(|| format!("Could not open {}", path))?;
 
     let reader = BufReader::new(file);
 
     let mut words = HashSet::new();
 
     for line in reader.lines() {
-        let result = line?;
+        let result = match line {
+            Ok(read_line) => read_line,
+            Err(_) => continue,
+        };
 
         if result.starts_with('#') || result.is_empty() {
             continue;
@@ -96,81 +59,63 @@ fn get_unique_words_from_wordlist(path: &str) -> FeroxResult<Arc<HashSet<String>
 }
 
 /// Determine whether it's a single url scan or urls are coming from stdin, then scan as needed
-async fn scan(
-    targets: Vec<String>,
-    tx_term: UnboundedSender<FeroxResponse>,
-    tx_file: UnboundedSender<FeroxResponse>,
-) -> FeroxResult<()> {
-    log::trace!("enter: scan({:?}, {:?}, {:?})", targets, tx_term, tx_file);
+async fn scan(targets: Vec<String>, handles: Arc<Handles>) -> Result<()> {
+    log::trace!("enter: scan({:?}, {:?})", targets, handles);
     // cloning an Arc is cheap (it's basically a pointer into the heap)
     // so that will allow for cheap/safe sharing of a single wordlist across multi-target scans
     // as well as additional directories found as part of recursion
-    let words =
-        tokio::spawn(async move { get_unique_words_from_wordlist(&CONFIGURATION.wordlist) })
-            .await??;
+
+    let words = {
+        let words_handles = handles.clone();
+        tokio::spawn(async move { get_unique_words_from_wordlist(&words_handles.config.wordlist) })
+            .await??
+    };
 
     if words.len() == 0 {
-        let mut err = FeroxError::default();
-        err.message = format!("Did not find any words in {}", CONFIGURATION.wordlist);
-        return Err(Box::new(err));
+        bail!("Did not find any words in {}", handles.config.wordlist);
     }
 
-    scanner::initialize(words.len(), &CONFIGURATION);
+    let scanned_urls = handles.ferox_scans()?;
 
-    if CONFIGURATION.resumed {
-        if let Ok(scans) = SCANNED_URLS.scans.lock() {
-            for scan in scans.iter() {
-                if let Ok(locked_scan) = scan.lock() {
-                    if locked_scan.complete {
-                        // these scans are complete, and just need to be shown to the user
-                        let pb = add_bar(
-                            &locked_scan.url,
-                            words.len().try_into().unwrap_or_default(),
-                            false,
-                            true,
-                        );
-                        pb.finish();
-                    }
-                }
-            }
-        }
+    handles.send_scan_command(UpdateWordlist(words.clone()))?;
 
-        if let Ok(responses) = RESPONSES.responses.read() {
-            for response in responses.iter() {
-                PROGRESS_PRINTER.println(response.as_str());
-            }
-        }
+    scanner::initialize(words.len(), handles.clone()).await?;
+
+    // at this point, the stat thread's progress bar can be created; things that needed to happen
+    // first:
+    // - banner gets printed
+    // - scanner initialized (this sent expected requests per directory to the stats thread, which
+    //   having been set, makes it so the progress bar doesn't flash as full before anything has
+    //   even happened
+    if matches!(handles.config.output_level, OutputLevel::Default) {
+        // only create the bar if no --silent|--quiet
+        handles.stats.send(CreateBar)?;
+
+        // blocks until the bar is created / avoids race condition in first two bars
+        handles.stats.sync().await?;
     }
 
-    let mut tasks = vec![];
-
-    for target in targets {
-        let word_clone = words.clone();
-        let term_clone = tx_term.clone();
-        let file_clone = tx_file.clone();
-
-        let task = tokio::spawn(async move {
-            let base_depth = get_current_depth(&target);
-            scan_url(&target, word_clone, base_depth, term_clone, file_clone).await;
-        });
-
-        tasks.push(task);
+    if handles.config.resumed {
+        // display what has already been completed
+        scanned_urls.print_known_responses();
+        scanned_urls.print_completed_bars(words.len())?;
     }
 
-    // drive execution of all accumulated futures
-    futures::future::join_all(tasks).await;
+    log::debug!("sending {:?} to be scanned as initial targets", targets);
+    handles.send_scan_command(ScanInitialUrls(targets))?;
+
     log::trace!("exit: scan");
 
     Ok(())
 }
 
 /// Get targets from either commandline or stdin, pass them back to the caller as a Result<Vec>
-async fn get_targets() -> FeroxResult<Vec<String>> {
-    log::trace!("enter: get_targets");
+async fn get_targets(handles: Arc<Handles>) -> Result<Vec<String>> {
+    log::trace!("enter: get_targets({:?})", handles);
 
     let mut targets = vec![];
 
-    if CONFIGURATION.stdin {
+    if handles.config.stdin {
         // got targets from stdin, i.e. cat sites | ./feroxbuster ...
         // just need to read the targets from stdin and spawn a future for each target found
         let stdin = io::stdin(); // tokio's stdin, not std
@@ -179,24 +124,25 @@ async fn get_targets() -> FeroxResult<Vec<String>> {
         while let Some(line) = reader.next().await {
             targets.push(line?);
         }
-    } else if CONFIGURATION.resumed {
+    } else if handles.config.resumed {
         // resume-from can't be used with --url, and --stdin is marked false for every resumed
         // scan, making it mutually exclusive from either of the other two options
-        if let Ok(scans) = SCANNED_URLS.scans.lock() {
+        let ferox_scans = handles.ferox_scans()?;
+
+        if let Ok(scans) = ferox_scans.scans.read() {
             for scan in scans.iter() {
-                // SCANNED_URLS gets deserialized scans added to it at program start if --resume-from
+                // ferox_scans gets deserialized scans added to it at program start if --resume-from
                 // is used, so scans that aren't marked complete still need to be scanned
-                if let Ok(locked_scan) = scan.lock() {
-                    if locked_scan.complete {
-                        // this one's already done, ignore it
-                        continue;
-                    }
-                    targets.push(locked_scan.url.to_owned());
+                if scan.is_complete() {
+                    // this one's already done, ignore it
+                    continue;
                 }
+
+                targets.push(scan.url().to_owned());
             }
-        }
+        };
     } else {
-        targets.push(CONFIGURATION.target_url.clone());
+        targets.push(handles.config.target_url.clone());
     }
 
     log::trace!("exit: get_targets -> {:?}", targets);
@@ -206,7 +152,7 @@ async fn get_targets() -> FeroxResult<Vec<String>> {
 
 /// async main called from real main, broken out in this way to allow for some synchronous code
 /// to be executed before bringing the tokio runtime online
-async fn wrapped_main() {
+async fn wrapped_main(config: Arc<Configuration>) -> Result<()> {
     // join can only be called once, otherwise it causes the thread to panic
     tokio::task::spawn_blocking(move || {
         // ok, lazy_static! uses (unsurprisingly in retrospect) a lazy loading model where the
@@ -220,12 +166,34 @@ async fn wrapped_main() {
         PROGRESS_BAR.join().unwrap();
     });
 
-    if !CONFIGURATION.time_limit.is_empty() {
+    // spawn all event handlers, expect back a JoinHandle and a *Handle to the specific event
+    let (stats_task, stats_handle) = StatsHandler::initialize(config.clone());
+    let (filters_task, filters_handle) = FiltersHandler::initialize();
+    let (out_task, out_handle) =
+        TermOutHandler::initialize(config.clone(), stats_handle.tx.clone());
+
+    // bundle up all the disparate handles and JoinHandles (tasks)
+    let handles = Arc::new(Handles::new(
+        stats_handle,
+        filters_handle,
+        out_handle,
+        config.clone(),
+    ));
+
+    let (scan_task, scan_handle) = ScanHandler::initialize(handles.clone());
+
+    handles.set_scan_handle(scan_handle); // must be done after Handles initialization
+
+    filters::initialize(handles.clone()).await?; // send user-supplied filters to the handler
+
+    // create new Tasks object, each of these handles is one that will be joined on later
+    let tasks = Tasks::new(out_task, stats_task, filters_task, scan_task);
+
+    if !config.time_limit.is_empty() {
         // --time-limit value not an empty string, need to kick off the thread that enforces
         // the limit
-        tokio::spawn(async move {
-            scan_manager::start_max_time_thread(&CONFIGURATION.time_limit).await
-        });
+        let time_handles = handles.clone();
+        tokio::spawn(async move { scan_manager::start_max_time_thread(time_handles).await });
     }
 
     // can't trace main until after logger is initialized and the above task is started
@@ -234,106 +202,113 @@ async fn wrapped_main() {
     // spawn a thread that listens for keyboard input on stdin, when a user presses enter
     // the input handler will toggle PAUSE_SCAN, which in turn is used to pause and resume
     // scans that are already running
-    tokio::task::spawn_blocking(terminal_input_handler);
+    // also starts ctrl+c handler
+    TermInputHandler::initialize(handles.clone());
 
-    let save_output = !CONFIGURATION.output.is_empty(); // was -o used?
+    if config.resumed {
+        let scanned_urls = handles.ferox_scans()?;
+        let from_here = config.resume_from.clone();
 
-    let (tx_term, tx_file, term_handle, file_handle) =
-        reporter::initialize(&CONFIGURATION.output, save_output);
+        // populate FeroxScans object with previously seen scans
+        scanned_urls.add_serialized_scans(&from_here)?;
+
+        // populate Stats object with previously known statistics
+        handles.stats.send(LoadStats(from_here))?;
+    }
 
     // get targets from command line or stdin
-    let targets = match get_targets().await {
+    let targets = match get_targets(handles.clone()).await {
         Ok(t) => t,
         Err(e) => {
             // should only happen in the event that there was an error reading from stdin
-            log::error!("{} {}", module_colorizer("main::get_targets"), e);
-            clean_up(tx_term, term_handle, tx_file, file_handle, save_output).await;
-            return;
+            clean_up(handles, tasks).await?;
+            bail!("Could not get determine initial targets: {}", e);
         }
     };
 
-    if !CONFIGURATION.quiet {
-        // only print banner if -q isn't used
+    if matches!(config.output_level, OutputLevel::Default) {
+        // only print banner if output level is default (no banner on --quiet|--silent)
         let std_stderr = stderr(); // std::io::stderr
-        banner::initialize(&targets, &CONFIGURATION, &VERSION, std_stderr).await;
+
+        let mut banner = Banner::new(&targets, &config);
+
+        // only interested in the side-effect that sets banner.update_status
+        let _ = banner.check_for_updates(UPDATE_URL, handles.clone()).await;
+
+        if banner.print_to(std_stderr, config.clone()).is_err() {
+            clean_up(handles, tasks).await?;
+            bail!(fmt_err("Could not print banner"));
+        }
+    }
+
+    {
+        let send_to_file = !config.output.is_empty();
+
+        // The TermOutHandler spawns a FileOutHandler, so errors in the FileOutHandler never bubble
+        // up due to the TermOutHandler never awaiting the result of FileOutHandler::start (that's
+        // done later here in main). Ping checks that the tx/rx connection to the file handler works
+        if send_to_file && handles.output.sync(send_to_file).await.is_err() {
+            // output file specified and file handler could not initialize
+            clean_up(handles, tasks).await?;
+            let msg = format!("Couldn't start {} file handler", config.output);
+            bail!(fmt_err(&msg));
+        }
     }
 
     // discard non-responsive targets
-    let live_targets = heuristics::connectivity_test(&targets).await;
+    let live_targets = {
+        let test = heuristics::HeuristicTests::new(handles.clone());
+        let result = test.connectivity(&targets).await;
+        if result.is_err() {
+            clean_up(handles, tasks).await?;
+            bail!(fmt_err(&result.unwrap_err().to_string()));
+        }
+        result?
+    };
 
     if live_targets.is_empty() {
-        clean_up(tx_term, term_handle, tx_file, file_handle, save_output).await;
-        return;
+        clean_up(handles, tasks).await?;
+        bail!(fmt_err("Could not find any live targets to scan"));
     }
 
     // kick off a scan against any targets determined to be responsive
-    match scan(live_targets, tx_term.clone(), tx_file.clone()).await {
-        Ok(_) => {
-            log::info!("All scans complete!");
-        }
+    match scan(live_targets, handles.clone()).await {
+        Ok(_) => {}
         Err(e) => {
-            ferox_print(
-                &format!("{} while scanning: {}", status_colorizer("Error"), e),
-                &PROGRESS_PRINTER,
-            );
-            clean_up(tx_term, term_handle, tx_file, file_handle, save_output).await;
-            process::exit(1);
+            clean_up(handles, tasks).await?;
+            bail!(fmt_err(&format!("Failed while scanning: {}", e)));
         }
-    };
+    }
 
-    clean_up(tx_term, term_handle, tx_file, file_handle, save_output).await;
+    clean_up(handles, tasks).await?;
 
-    log::trace!("exit: main");
+    log::trace!("exit: wrapped_main");
+    Ok(())
 }
 
 /// Single cleanup function that handles all the necessary drops/finishes etc required to gracefully
 /// shutdown the program
-async fn clean_up(
-    tx_term: UnboundedSender<FeroxResponse>,
-    term_handle: JoinHandle<()>,
-    tx_file: UnboundedSender<FeroxResponse>,
-    file_handle: Option<JoinHandle<()>>,
-    save_output: bool,
-) {
-    log::trace!(
-        "enter: clean_up({:?}, {:?}, {:?}, {:?}, {})",
-        tx_term,
-        term_handle,
-        tx_file,
-        file_handle,
-        save_output
-    );
+async fn clean_up(handles: Arc<Handles>, tasks: Tasks) -> Result<()> {
+    log::trace!("enter: clean_up({:?}, {:?})", handles, tasks);
 
-    drop(tx_term);
-    log::trace!("dropped terminal output handler's transmitter");
+    let (tx, rx) = oneshot::channel::<bool>();
+    handles.send_scan_command(JoinTasks(tx))?;
+    rx.await?;
 
-    log::trace!("awaiting terminal output handler's receiver");
-    // after dropping tx, we can await the future where rx lived
-    match term_handle.await {
-        Ok(_) => {}
-        Err(e) => {
-            log::error!("error awaiting terminal output handler's receiver: {}", e);
-        }
-    }
-    log::trace!("done awaiting terminal output handler's receiver");
+    log::info!("All scans complete!");
 
-    log::trace!("tx_file: {:?}", tx_file);
-    // the same drop/await process used on the terminal handler is repeated for the file handler
-    // we drop the file transmitter every time, because it's created no matter what
-    drop(tx_file);
+    // terminal handler closes file handler if one is in use
+    handles.output.send(Exit)?;
+    tasks.terminal.await??;
+    log::trace!("terminal handler closed");
 
-    log::trace!("dropped file output handler's transmitter");
-    if save_output {
-        // but we only await if -o was specified
-        log::trace!("awaiting file output handler's receiver");
-        match file_handle.unwrap().await {
-            Ok(_) => {}
-            Err(e) => {
-                log::error!("error awaiting file output handler's receiver: {}", e);
-            }
-        }
-        log::trace!("done awaiting file output handler's receiver");
-    }
+    handles.filters.send(Exit)?;
+    tasks.filters.await??;
+    log::trace!("filters handler closed");
+
+    handles.stats.send(Exit)?;
+    tasks.stats.await??;
+    log::trace!("stats handler closed");
 
     // mark all scans complete so the terminal input handler will exit cleanly
     SCAN_COMPLETE.store(true, Ordering::Relaxed);
@@ -343,23 +318,36 @@ async fn clean_up(
     PROGRESS_PRINTER.finish();
 
     log::trace!("exit: clean_up");
+    Ok(())
 }
 
-fn main() {
-    // setup logging based on the number of -v's used
-    logger::initialize(CONFIGURATION.verbosity);
+fn main() -> Result<()> {
+    let config = Arc::new(Configuration::new().with_context(|| "Could not create Configuration")?);
 
-    if CONFIGURATION.save_state {
-        // start the ctrl+c handler
-        scan_manager::initialize();
+    // setup logging based on the number of -v's used
+    if matches!(
+        config.output_level,
+        OutputLevel::Default | OutputLevel::Quiet
+    ) {
+        // don't log on --silent
+        logger::initialize(config.clone())?;
     }
 
     // this function uses rlimit, which is not supported on windows
     #[cfg(not(target_os = "windows"))]
     set_open_file_limit(DEFAULT_OPEN_FILE_LIMIT);
 
-    if let Ok(mut runtime) = tokio::runtime::Runtime::new() {
-        let future = wrapped_main();
-        runtime.block_on(future);
+    if let Ok(runtime) = tokio::runtime::Builder::new_multi_thread()
+        .enable_all()
+        .build()
+    {
+        let future = wrapped_main(config);
+        if let Err(e) = runtime.block_on(future) {
+            eprintln!("{}", e);
+        };
     }
+
+    log::trace!("exit: main");
+
+    Ok(())
 }

src/message.rs

@@ -0,0 +1,121 @@
+use anyhow::Context;
+use console::{style, Color};
+
+use crate::traits::FeroxSerialize;
+use crate::utils::fmt_err;
+use serde::{Deserialize, Serialize};
+
+#[derive(Serialize, Deserialize, Default)]
+/// Representation of a log entry, can be represented as a human readable string or JSON
+pub struct FeroxMessage {
+    #[serde(rename = "type")]
+    /// Name of this type of struct, used for serialization, i.e. `{"type":"log"}`
+    pub(crate) kind: String,
+
+    /// The log message
+    pub(crate) message: String,
+
+    /// The log level
+    pub(crate) level: String,
+
+    /// The number of seconds elapsed since the scan started
+    pub(crate) time_offset: f32,
+
+    /// The module from which log::* was called
+    pub(crate) module: String,
+}
+
+/// Implementation of FeroxMessage
+impl FeroxSerialize for FeroxMessage {
+    /// Create a string representation of the log message
+    ///
+    /// ex:  301       10l       16w      173c https://localhost/api
+    fn as_str(&self) -> String {
+        let (level_name, level_color) = match self.level.as_str() {
+            "ERROR" => ("ERR", Color::Red),
+            "WARN" => ("WRN", Color::Red),
+            "INFO" => ("INF", Color::Cyan),
+            "DEBUG" => ("DBG", Color::Yellow),
+            "TRACE" => ("TRC", Color::Magenta),
+            "WILDCARD" => ("WLD", Color::Cyan),
+            _ => ("UNK", Color::White),
+        };
+
+        format!(
+            "{} {:10.03} {} {}\n",
+            style(level_name).bg(level_color).black(),
+            style(self.time_offset).dim(),
+            self.module,
+            style(&self.message).dim(),
+        )
+    }
+
+    /// Create an NDJSON representation of the log message
+    ///
+    /// (expanded for clarity)
+    /// ex:
+    /// {
+    ///   "type": "log",
+    ///   "message": "Sent https://localhost/api to file handler",
+    ///   "level": "DEBUG",
+    ///   "time_offset": 0.86333454,
+    ///   "module": "feroxbuster::reporter"
+    /// }\n
+    fn as_json(&self) -> anyhow::Result<String> {
+        let mut json = serde_json::to_string(&self).with_context(|| {
+            fmt_err(&format!(
+                "Could not convert {}:{} to JSON",
+                self.level, self.message
+            ))
+        })?;
+        json.push('\n');
+        Ok(json)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    /// test as_str method of FeroxMessage
+    fn ferox_message_as_str_returns_string_with_newline() {
+        let message = FeroxMessage {
+            message: "message".to_string(),
+            module: "utils".to_string(),
+            time_offset: 1.0,
+            level: "INFO".to_string(),
+            kind: "log".to_string(),
+        };
+        let message_str = message.as_str();
+
+        assert!(message_str.contains("INF"));
+        assert!(message_str.contains("1.000"));
+        assert!(message_str.contains("utils"));
+        assert!(message_str.contains("message"));
+        assert!(message_str.ends_with('\n'));
+    }
+
+    #[test]
+    /// test as_json method of FeroxMessage
+    fn ferox_message_as_json_returns_json_representation_of_ferox_message_with_newline() {
+        let message = FeroxMessage {
+            message: "message".to_string(),
+            module: "utils".to_string(),
+            time_offset: 1.0,
+            level: "INFO".to_string(),
+            kind: "log".to_string(),
+        };
+
+        let message_str = message.as_json().unwrap();
+
+        let error_margin = f32::EPSILON;
+
+        let json: FeroxMessage = serde_json::from_str(&message_str).unwrap();
+        assert_eq!(json.module, message.module);
+        assert_eq!(json.message, message.message);
+        assert!((json.time_offset - message.time_offset).abs() < error_margin);
+        assert_eq!(json.level, message.level);
+        assert_eq!(json.kind, message.kind);
+    }
+}

src/parser.rs

@@ -68,6 +68,7 @@ pub fn initialize() -> App<'static, 'static> {
                 .long("verbosity")
                 .takes_value(false)
                 .multiple(true)
+                .conflicts_with("silent")
                 .help("Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v's is probably too much)"),
         )
         .arg(
@@ -77,7 +78,7 @@ pub fn initialize() -> App<'static, 'static> {
                 .takes_value(true)
                 .value_name("PROXY")
                 .help(
-                    "Proxy to use for requests (ex: http(s)://host:port, socks5://host:port)",
+                    "Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)",
                 ),
         )
         .arg(
@@ -115,12 +116,19 @@ pub fn initialize() -> App<'static, 'static> {
                     "Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)",
                 ),
         )
+        .arg(
+            Arg::with_name("silent")
+                .long("silent")
+                .takes_value(false)
+                .conflicts_with("quiet")
+                .help("Only print URLs + turn off logging (good for piping a list of urls to other commands)")
+        )
         .arg(
             Arg::with_name("quiet")
                 .short("q")
                 .long("quiet")
                 .takes_value(false)
-                .help("Only print URLs; Don't print status codes, response size, running config, etc...")
+                .help("Hide progress bars and banner (good for tmux windows w/ notifications)")
         )
         .arg(
             Arg::with_name("json")
@@ -301,6 +309,17 @@ pub fn initialize() -> App<'static, 'static> {
                     "Filter out status codes (deny list) (ex: -C 200 -C 401)",
                 ),
         )
+        .arg(
+            Arg::with_name("filter_similar")
+                .long("filter-similar-to")
+                .value_name("UNWANTED_PAGE")
+                .takes_value(true)
+                .multiple(true)
+                .use_delimiter(true)
+                .help(
+                    "Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)",
+                ),
+        )
         .arg(
             Arg::with_name("extract_links")
                 .short("e")
@@ -316,6 +335,13 @@ pub fn initialize() -> App<'static, 'static> {
                 .takes_value(true)
                 .help("Limit total number of concurrent scans (default: 0, i.e. no limit)")
         )
+        .arg(
+            Arg::with_name("rate_limit")
+                .long("rate-limit")
+                .value_name("RATE_LIMIT")
+                .takes_value(true)
+                .help("Limit number of requests per second (per directory) (default: 0, i.e. no limit)")
+        )
         .arg(
             Arg::with_name("time_limit")
                 .long("time-limit")
@@ -346,7 +372,7 @@ EXAMPLES:
         ./feroxbuster -u http://[::1] --no-recursion -vv
 
     Read urls from STDIN; pipe only resulting urls out to another tool
-        cat targets | ./feroxbuster --stdin --quiet -s 200 301 302 --redirects -x js | fff -s 200 -o js-files
+        cat targets | ./feroxbuster --stdin --silent -s 200 301 302 --redirects -x js | fff -s 200 -o js-files
 
     Proxy traffic through Burp
         ./feroxbuster -u http://127.1 --insecure --proxy http://127.0.0.1:8080

src/progress.rs

@@ -1,22 +1,50 @@
-use crate::config::{CONFIGURATION, PROGRESS_BAR};
-use indicatif::{ProgressBar, ProgressStyle};
+use indicatif::{MultiProgress, ProgressBar, ProgressDrawTarget, ProgressStyle};
+use lazy_static::lazy_static;
+
+lazy_static! {
+    /// Global progress bar that houses other progress bars
+    pub static ref PROGRESS_BAR: MultiProgress = MultiProgress::with_draw_target(ProgressDrawTarget::stdout());
+
+    /// Global progress bar that is only used for printing messages that don't jack up other bars
+    pub static ref PROGRESS_PRINTER: ProgressBar = add_bar("", 0, BarType::Hidden);
+}
+
+/// Types of ProgressBars that can be added to `PROGRESS_BAR`
+#[derive(Copy, Clone)]
+pub enum BarType {
+    /// no template used / not visible
+    Hidden,
+
+    /// normal directory status bar (reqs/sec shown)
+    Default,
+
+    /// similar to `Default`, except `-` is used in place of line/word/char count
+    Message,
+
+    /// bar used to show overall scan metrics
+    Total,
+
+    /// simpler output bar that shows only the directory being scanned (no updating info)
+    Quiet,
+}
 
 /// Add an [indicatif::ProgressBar](https://docs.rs/indicatif/latest/indicatif/struct.ProgressBar.html)
 /// to the global [PROGRESS_BAR](../config/struct.PROGRESS_BAR.html)
-pub fn add_bar(prefix: &str, length: u64, hidden: bool, hide_per_sec: bool) -> ProgressBar {
-    let style = if hidden || CONFIGURATION.quiet {
-        ProgressStyle::default_bar().template("")
-    } else if hide_per_sec {
-        ProgressStyle::default_bar()
-            .template(&format!(
-                "[{{bar:.cyan/blue}}] - {{elapsed:<4}} {{pos:>7}}/{{len:7}} {:7} {{prefix}}",
-                "-"
-            ))
-            .progress_chars("#>-")
-    } else {
-        ProgressStyle::default_bar()
-            .template("[{bar:.cyan/blue}] - {elapsed:<4} {pos:>7}/{len:7} {per_sec:7} {prefix}")
-            .progress_chars("#>-")
+pub fn add_bar(prefix: &str, length: u64, bar_type: BarType) -> ProgressBar {
+    let mut style = ProgressStyle::default_bar().progress_chars("#>-");
+
+    style = match bar_type {
+        BarType::Hidden => style.template(""),
+        BarType::Default => style
+            .template("[{bar:.cyan/blue}] - {elapsed:<4} {pos:>7}/{len:7} {per_sec:7} {prefix}"),
+        BarType::Message => style.template(&format!(
+            "[{{bar:.cyan/blue}}] - {{elapsed:<4}} {{pos:>7}}/{{len:7}} {:7} {{prefix}}",
+            "-"
+        )),
+        BarType::Total => {
+            style.template("[{bar:.yellow/blue}] - {elapsed:<4} {pos:>7}/{len:7} {eta:7} {msg}")
+        }
+        BarType::Quiet => style.template("Scanning: {prefix}"),
     };
 
     let progress_bar = PROGRESS_BAR.add(ProgressBar::new(length));
@@ -35,16 +63,19 @@ mod tests {
     #[test]
     /// hit all code branches for add_bar
     fn add_bar_with_all_configurations() {
-        let p1 = add_bar("prefix", 2, true, false); // hidden
-        let p2 = add_bar("prefix", 2, false, true); // no per second field
-        let p3 = add_bar("prefix", 2, false, false); // normal bar
+        let p1 = add_bar("prefix", 2, BarType::Hidden); // hidden
+        let p2 = add_bar("prefix", 2, BarType::Message); // no per second field
+        let p3 = add_bar("prefix", 2, BarType::Default); // normal bar
+        let p4 = add_bar("prefix", 2, BarType::Total); // totals bar
 
         p1.finish();
         p2.finish();
         p3.finish();
+        p4.finish();
 
         assert!(p1.is_finished());
         assert!(p2.is_finished());
         assert!(p3.is_finished());
+        assert!(p4.is_finished());
     }
 }

src/reporter.rs

@@ -1,232 +0,0 @@
-use crate::{
-    config::{CONFIGURATION, PROGRESS_PRINTER},
-    scanner::RESPONSES,
-    utils::{ferox_print, make_request, open_file},
-    FeroxChannel, FeroxResponse, FeroxSerialize,
-};
-use console::strip_ansi_codes;
-use std::io::Write;
-use std::sync::{Arc, Once, RwLock};
-use std::{fs, io};
-use tokio::sync::mpsc::{self, UnboundedReceiver, UnboundedSender};
-use tokio::task::JoinHandle;
-
-/// Singleton buffered file behind an Arc/RwLock; used for file writes from two locations:
-///     - [logger::initialize](../logger/fn.initialize.html) (specifically a closure on the global logger instance)
-///     - `reporter::spawn_file_handler`
-pub static mut LOCKED_FILE: Option<Arc<RwLock<io::BufWriter<fs::File>>>> = None;
-
-/// An initializer Once variable used to create `LOCKED_FILE`
-static INIT: Once = Once::new();
-
-// Accessing a `static mut` is unsafe much of the time, but if we do so
-// in a synchronized fashion (e.g., write once or read all) then we're
-// good to go!
-//
-// This function will only call `open_file` once, and will
-// otherwise always return the value returned from the first invocation.
-pub fn get_cached_file_handle(filename: &str) -> Option<Arc<RwLock<io::BufWriter<fs::File>>>> {
-    unsafe {
-        INIT.call_once(|| {
-            LOCKED_FILE = open_file(&filename);
-        });
-        LOCKED_FILE.clone()
-    }
-}
-
-/// Creates all required output handlers (terminal, file) and returns
-/// the transmitter sides of each mpsc along with each receiver's future's JoinHandle to be awaited
-///
-/// Any other module that needs to write a Response to stdout or output results to a file should
-/// be passed a clone of the appropriate returned transmitter
-pub fn initialize(
-    output_file: &str,
-    save_output: bool,
-) -> (
-    UnboundedSender<FeroxResponse>,
-    UnboundedSender<FeroxResponse>,
-    JoinHandle<()>,
-    Option<JoinHandle<()>>,
-) {
-    log::trace!("enter: initialize({}, {})", output_file, save_output);
-
-    let (tx_rpt, rx_rpt): FeroxChannel<FeroxResponse> = mpsc::unbounded_channel();
-    let (tx_file, rx_file): FeroxChannel<FeroxResponse> = mpsc::unbounded_channel();
-
-    let file_clone = tx_file.clone();
-
-    let term_reporter =
-        tokio::spawn(async move { spawn_terminal_reporter(rx_rpt, file_clone, save_output).await });
-
-    let file_reporter = if save_output {
-        // -o used, need to spawn the thread for writing to disk
-        let file_clone = output_file.to_string();
-        Some(tokio::spawn(async move {
-            spawn_file_reporter(rx_file, &file_clone).await
-        }))
-    } else {
-        None
-    };
-
-    log::trace!(
-        "exit: initialize -> ({:?}, {:?}, {:?}, {:?})",
-        tx_rpt,
-        tx_file,
-        term_reporter,
-        file_reporter
-    );
-    (tx_rpt, tx_file, term_reporter, file_reporter)
-}
-
-/// Spawn a single consumer task (sc side of mpsc)
-///
-/// The consumer simply receives responses and prints them if they meet the given
-/// reporting criteria
-async fn spawn_terminal_reporter(
-    mut resp_chan: UnboundedReceiver<FeroxResponse>,
-    file_chan: UnboundedSender<FeroxResponse>,
-    save_output: bool,
-) {
-    log::trace!(
-        "enter: spawn_terminal_reporter({:?}, {:?}, {})",
-        resp_chan,
-        file_chan,
-        save_output
-    );
-
-    while let Some(mut resp) = resp_chan.recv().await {
-        log::trace!("received {} on reporting channel", resp.url());
-
-        let contains_sentry = CONFIGURATION.status_codes.contains(&resp.status().as_u16());
-        let unknown_sentry = !RESPONSES.contains(&resp); // !contains == unknown
-        let should_process_response = contains_sentry && unknown_sentry;
-
-        if should_process_response {
-            // print to stdout
-            ferox_print(&resp.as_str(), &PROGRESS_PRINTER);
-
-            if save_output {
-                // -o used, need to send the report to be written out to disk
-                match file_chan.send(resp.clone()) {
-                    Ok(_) => {
-                        log::debug!("Sent {} to file handler", resp.url());
-                    }
-                    Err(e) => {
-                        log::error!("Could not send {} to file handler: {}", resp.url(), e);
-                    }
-                }
-            }
-        }
-        log::trace!("report complete: {}", resp.url());
-
-        if CONFIGURATION.replay_client.is_some() && should_process_response {
-            // replay proxy specified/client created and this response's status code is one that
-            // should be replayed
-            match make_request(CONFIGURATION.replay_client.as_ref().unwrap(), &resp.url()).await {
-                Ok(_) => {}
-                Err(e) => {
-                    log::error!("{}", e);
-                }
-            }
-        }
-
-        if should_process_response {
-            // add response to RESPONSES for serialization in case of ctrl+c
-            // placed all by its lonesome like this so that RESPONSES can take ownership
-            // of the FeroxResponse
-
-            // before ownership is transferred, there's no real reason to keep the body anymore
-            // so we can free that piece of data, reducing memory usage
-            resp.text = String::new();
-
-            RESPONSES.insert(resp);
-        }
-    }
-    log::trace!("exit: spawn_terminal_reporter");
-}
-
-/// Spawn a single consumer task (sc side of mpsc)
-///
-/// The consumer simply receives responses and writes them to the given output file if they meet
-/// the given reporting criteria
-async fn spawn_file_reporter(
-    mut report_channel: UnboundedReceiver<FeroxResponse>,
-    output_file: &str,
-) {
-    let buffered_file = match get_cached_file_handle(&CONFIGURATION.output) {
-        Some(file) => file,
-        None => {
-            log::trace!("exit: spawn_file_reporter");
-            return;
-        }
-    };
-
-    log::trace!(
-        "enter: spawn_file_reporter({:?}, {})",
-        report_channel,
-        output_file
-    );
-
-    log::info!("Writing scan results to {}", output_file);
-
-    while let Some(response) = report_channel.recv().await {
-        safe_file_write(&response, buffered_file.clone(), CONFIGURATION.json);
-    }
-
-    log::trace!("exit: spawn_file_reporter");
-}
-
-/// Given a string and a reference to a locked buffered file, write the contents and flush
-/// the buffer to disk.
-pub fn safe_file_write<T>(
-    value: &T,
-    locked_file: Arc<RwLock<io::BufWriter<fs::File>>>,
-    convert_to_json: bool,
-) where
-    T: FeroxSerialize,
-{
-    // note to future self: adding logging of anything other than error to this function
-    // is a bad idea. we call this function while processing records generated by the logger.
-    // If we then call log::... while already processing some logging output, it results in
-    // the second log entry being injected into the first.
-
-    let contents = if convert_to_json {
-        value.as_json()
-    } else {
-        value.as_str()
-    };
-
-    let contents = strip_ansi_codes(&contents);
-
-    if let Ok(mut handle) = locked_file.write() {
-        // write lock acquired
-        match handle.write(contents.as_bytes()) {
-            Ok(_) => {}
-            Err(e) => {
-                log::error!("could not write report to disk: {}", e);
-            }
-        }
-
-        match handle.flush() {
-            // this function is used within async functions/loops, so i'm flushing so that in
-            // the event of a ctrl+c or w/e results seen so far are saved instead of left lying
-            // around in the buffer
-            Ok(_) => {}
-            Err(e) => {
-                log::error!("error writing to file: {}", e);
-            }
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    #[should_panic]
-    /// asserts that an empty string for a filename returns None
-    fn reporter_get_cached_file_handle_without_filename_returns_none() {
-        let _used = get_cached_file_handle(&"").unwrap();
-    }
-}

src/response.rs

@@ -0,0 +1,640 @@
+use std::{
+    collections::HashMap,
+    convert::{TryFrom, TryInto},
+    fmt,
+    str::FromStr,
+    sync::Arc,
+};
+
+use anyhow::{Context, Result};
+use reqwest::{
+    header::{HeaderMap, HeaderName, HeaderValue},
+    Response, StatusCode, Url,
+};
+use serde::ser::SerializeStruct;
+use serde::{Deserialize, Deserializer, Serialize, Serializer};
+use serde_json::Value;
+
+use crate::{
+    config::OutputLevel,
+    event_handlers::{Command, Handles},
+    traits::FeroxSerialize,
+    url::FeroxUrl,
+    utils::{self, fmt_err, status_colorizer},
+    CommandSender,
+};
+
+/// A `FeroxResponse`, derived from a `Response` to a submitted `Request`
+#[derive(Debug, Clone)]
+pub struct FeroxResponse {
+    /// The final `Url` of this `FeroxResponse`
+    url: Url,
+
+    /// The `StatusCode` of this `FeroxResponse`
+    status: StatusCode,
+
+    /// The full response text
+    text: String,
+
+    /// The content-length of this response, if known
+    content_length: u64,
+
+    /// The number of lines contained in the body of this response, if known
+    line_count: usize,
+
+    /// The number of words contained in the body of this response, if known
+    word_count: usize,
+
+    /// The `Headers` of this `FeroxResponse`
+    headers: HeaderMap,
+
+    /// Wildcard response status
+    wildcard: bool,
+
+    /// whether the user passed --quiet|--silent on the command line
+    pub(crate) output_level: OutputLevel,
+}
+
+/// implement Default trait for FeroxResponse
+impl Default for FeroxResponse {
+    /// return a default reqwest::Url and then normal defaults after that
+    fn default() -> Self {
+        Self {
+            url: Url::parse("http://localhost").unwrap(),
+            status: Default::default(),
+            text: "".to_string(),
+            content_length: 0,
+            line_count: 0,
+            word_count: 0,
+            headers: Default::default(),
+            wildcard: false,
+            output_level: Default::default(),
+        }
+    }
+}
+
+/// Implement Display for FeroxResponse
+impl fmt::Display for FeroxResponse {
+    /// formatter for Display
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(
+            f,
+            "FeroxResponse {{ url: {}, status: {}, content-length: {} }}",
+            self.url(),
+            self.status(),
+            self.content_length()
+        )
+    }
+}
+
+/// `FeroxResponse` implementation
+impl FeroxResponse {
+    /// Get the `StatusCode` of this `FeroxResponse`
+    pub fn status(&self) -> &StatusCode {
+        &self.status
+    }
+
+    /// Get the `wildcard` of this `FeroxResponse`
+    pub fn wildcard(&self) -> bool {
+        self.wildcard
+    }
+
+    /// Get the final `Url` of this `FeroxResponse`.
+    pub fn url(&self) -> &Url {
+        &self.url
+    }
+
+    /// Get the full response text
+    pub fn text(&self) -> &str {
+        &self.text
+    }
+
+    /// Get the `Headers` of this `FeroxResponse`
+    pub fn headers(&self) -> &HeaderMap {
+        &self.headers
+    }
+
+    /// Get the content-length of this response, if known
+    pub fn content_length(&self) -> u64 {
+        self.content_length
+    }
+
+    /// Set `FeroxResponse`'s `url` attribute, has no affect if an error occurs
+    pub fn set_url(&mut self, url: &str) {
+        match Url::parse(&url) {
+            Ok(url) => {
+                self.url = url;
+            }
+            Err(e) => {
+                log::warn!("Could not parse {} into a Url: {}", url, e);
+            }
+        };
+    }
+
+    /// set `wildcard` attribute
+    pub fn set_wildcard(&mut self, is_wildcard: bool) {
+        self.wildcard = is_wildcard;
+    }
+
+    /// set `text` attribute; update words/lines/content_length
+    #[cfg(test)]
+    pub fn set_text(&mut self, text: &str) {
+        self.text = String::from(text);
+        self.content_length = self.text.len() as u64;
+        self.line_count = self.text.lines().count();
+        self.word_count = self
+            .text
+            .lines()
+            .map(|s| s.split_whitespace().count())
+            .sum();
+    }
+
+    /// free the `text` data, reducing memory usage
+    pub fn drop_text(&mut self) {
+        self.text = String::new();
+    }
+
+    /// Make a reasonable guess at whether the response is a file or not
+    ///
+    /// Examines the last part of a path to determine if it has an obvious extension
+    /// i.e. http://localhost/some/path/stuff.js where stuff.js indicates a file
+    ///
+    /// Additionally, inspects query parameters, as they're also often indicative of a file
+    pub fn is_file(&self) -> bool {
+        let has_extension = match self.url.path_segments() {
+            Some(path) => {
+                if let Some(last) = path.last() {
+                    last.contains('.') // last segment has some sort of extension, probably
+                } else {
+                    false
+                }
+            }
+            None => false,
+        };
+
+        self.url.query_pairs().count() > 0 || has_extension
+    }
+
+    /// Returns line count of the response text.
+    pub fn line_count(&self) -> usize {
+        self.line_count
+    }
+
+    /// Returns word count of the response text.
+    pub fn word_count(&self) -> usize {
+        self.word_count
+    }
+
+    /// Create a new `FeroxResponse` from the given `Response`
+    pub async fn from(response: Response, read_body: bool, output_level: OutputLevel) -> Self {
+        let url = response.url().clone();
+        let status = response.status();
+        let headers = response.headers().clone();
+        let content_length = response.content_length().unwrap_or(0);
+
+        let text = if read_body {
+            // .text() consumes the response, must be called last
+            // additionally, --extract-links is currently the only place we use the body of the
+            // response, so we forego the processing if not performing extraction
+            match response.text().await {
+                // await the response's body
+                Ok(text) => text,
+                Err(e) => {
+                    log::warn!("Could not parse body from response: {}", e);
+                    String::new()
+                }
+            }
+        } else {
+            String::new()
+        };
+
+        let line_count = text.lines().count();
+        let word_count = text.lines().map(|s| s.split_whitespace().count()).sum();
+
+        FeroxResponse {
+            url,
+            status,
+            content_length,
+            text,
+            headers,
+            line_count,
+            word_count,
+            output_level,
+            wildcard: false,
+        }
+    }
+
+    /// Helper function that determines if the configured maximum recursion depth has been reached
+    ///
+    /// Essentially looks at the Url path and determines how many directories are present in the
+    /// given Url
+    pub(crate) fn reached_max_depth(
+        &self,
+        base_depth: usize,
+        max_depth: usize,
+        handles: Arc<Handles>,
+    ) -> bool {
+        log::trace!(
+            "enter: reached_max_depth({}, {}, {:?})",
+            base_depth,
+            max_depth,
+            handles
+        );
+
+        if max_depth == 0 {
+            // early return, as 0 means recurse forever; no additional processing needed
+            log::trace!("exit: reached_max_depth -> false");
+            return false;
+        }
+        let url = FeroxUrl::from_url(&self.url, handles);
+        let depth = url.depth().unwrap_or_default(); // 0 on error
+
+        if depth - base_depth >= max_depth {
+            return true;
+        }
+
+        log::trace!("exit: reached_max_depth -> false");
+        false
+    }
+
+    /// Helper function to determine suitability for recursion
+    ///
+    /// handles 2xx and 3xx responses by either checking if the url ends with a / (2xx)
+    /// or if the Location header is present and matches the base url + / (3xx)
+    pub fn is_directory(&self) -> bool {
+        log::trace!("enter: is_directory({})", self);
+
+        if self.status().is_redirection() {
+            // status code is 3xx
+            match self.headers().get("Location") {
+                // and has a Location header
+                Some(loc) => {
+                    // get absolute redirect Url based on the already known base url
+                    log::debug!("Location header: {:?}", loc);
+
+                    if let Ok(loc_str) = loc.to_str() {
+                        if let Ok(abs_url) = self.url().join(loc_str) {
+                            if format!("{}/", self.url()) == abs_url.as_str() {
+                                // if current response's Url + / == the absolute redirection
+                                // location, we've found a directory suitable for recursion
+                                log::debug!(
+                                    "found directory suitable for recursion: {}",
+                                    self.url()
+                                );
+                                log::trace!("exit: is_directory -> true");
+                                return true;
+                            }
+                        }
+                    }
+                }
+                None => {
+                    log::debug!("expected Location header, but none was found: {}", self);
+                    log::trace!("exit: is_directory -> false");
+                    return false;
+                }
+            }
+        } else if self.status().is_success() || matches!(self.status(), &StatusCode::FORBIDDEN) {
+            // status code is 2xx or 403, need to check if it ends in /
+
+            if self.url().as_str().ends_with('/') {
+                log::debug!("{} is directory suitable for recursion", self.url());
+                log::trace!("exit: is_directory -> true");
+                return true;
+            }
+        }
+
+        log::trace!("exit: is_directory -> false");
+        false
+    }
+
+    /// Simple helper to send a `FeroxResponse` over the tx side of an `mpsc::unbounded_channel`
+    pub fn send_report(self, report_sender: CommandSender) -> Result<()> {
+        log::trace!("enter: send_report({:?}", report_sender);
+
+        report_sender.send(Command::Report(Box::new(self)))?;
+
+        log::trace!("exit: send_report");
+        Ok(())
+    }
+}
+
+/// Implement FeroxSerialize for FeroxResponse
+impl FeroxSerialize for FeroxResponse {
+    /// Simple wrapper around create_report_string
+    fn as_str(&self) -> String {
+        let lines = self.line_count().to_string();
+        let words = self.word_count().to_string();
+        let chars = self.content_length().to_string();
+        let status = self.status().as_str();
+        let wild_status = status_colorizer("WLD");
+
+        if self.wildcard && matches!(self.output_level, OutputLevel::Default | OutputLevel::Quiet) {
+            // --silent was not used and response is a wildcard, special messages abound when
+            // this is the case...
+
+            // create the base message
+            let mut message = format!(
+                "{} {:>8}l {:>8}w {:>8}c Got {} for {} (url length: {})\n",
+                wild_status,
+                lines,
+                words,
+                chars,
+                status_colorizer(&status),
+                self.url(),
+                FeroxUrl::path_length_of_url(&self.url)
+            );
+
+            if self.status().is_redirection() {
+                // when it's a redirect, show where it goes, if possible
+                if let Some(next_loc) = self.headers().get("Location") {
+                    let next_loc_str = next_loc.to_str().unwrap_or("Unknown");
+
+                    let redirect_msg = format!(
+                        "{} {:>9} {:>9} {:>9} {} redirects to => {}\n",
+                        wild_status,
+                        "-",
+                        "-",
+                        "-",
+                        self.url(),
+                        next_loc_str
+                    );
+
+                    message.push_str(&redirect_msg);
+                }
+            }
+
+            // base message + redirection message (if appropriate)
+            message
+        } else {
+            // not a wildcard, just create a normal entry
+            utils::create_report_string(
+                self.status.as_str(),
+                &lines,
+                &words,
+                &chars,
+                self.url().as_str(),
+                self.output_level,
+            )
+        }
+    }
+
+    /// Create an NDJSON representation of the FeroxResponse
+    ///
+    /// (expanded for clarity)
+    /// ex:
+    /// {
+    ///    "type":"response",
+    ///    "url":"https://localhost.com/images",
+    ///    "path":"/images",
+    ///    "status":301,
+    ///    "content_length":179,
+    ///    "line_count":10,
+    ///    "word_count":16,
+    ///    "headers":{
+    ///       "x-content-type-options":"nosniff",
+    ///       "strict-transport-security":"max-age=31536000; includeSubDomains",
+    ///       "x-frame-options":"SAMEORIGIN",
+    ///       "connection":"keep-alive",
+    ///       "server":"nginx/1.16.1",
+    ///       "content-type":"text/html; charset=UTF-8",
+    ///       "referrer-policy":"origin-when-cross-origin",
+    ///       "content-security-policy":"default-src 'none'",
+    ///       "access-control-allow-headers":"X-Requested-With",
+    ///       "x-xss-protection":"1; mode=block",
+    ///       "content-length":"179",
+    ///       "date":"Mon, 23 Nov 2020 15:33:24 GMT",
+    ///       "location":"/images/",
+    ///       "access-control-allow-origin":"https://localhost.com"
+    ///    }
+    /// }\n
+    fn as_json(&self) -> anyhow::Result<String> {
+        let mut json = serde_json::to_string(&self)
+            .with_context(|| fmt_err(&format!("Could not convert {} to JSON", self.url())))?;
+        json.push('\n');
+        Ok(json)
+    }
+}
+
+/// Serialize implementation for FeroxResponse
+impl Serialize for FeroxResponse {
+    /// Function that handles serialization of a FeroxResponse to NDJSON
+    fn serialize<S>(&self, serializer: S) -> anyhow::Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let mut headers = HashMap::new();
+        let mut state = serializer.serialize_struct("FeroxResponse", 7)?;
+
+        // need to convert the HeaderMap to a HashMap in order to pass it to the serializer
+        for (key, value) in &self.headers {
+            let k = key.as_str().to_owned();
+            let v = String::from_utf8_lossy(value.as_bytes());
+            headers.insert(k, v);
+        }
+
+        state.serialize_field("type", "response")?;
+        state.serialize_field("url", self.url.as_str())?;
+        state.serialize_field("path", self.url.path())?;
+        state.serialize_field("wildcard", &self.wildcard)?;
+        state.serialize_field("status", &self.status.as_u16())?;
+        state.serialize_field("content_length", &self.content_length)?;
+        state.serialize_field("line_count", &self.line_count)?;
+        state.serialize_field("word_count", &self.word_count)?;
+        state.serialize_field("headers", &headers)?;
+
+        state.end()
+    }
+}
+
+/// Deserialize implementation for FeroxResponse
+impl<'de> Deserialize<'de> for FeroxResponse {
+    /// Deserialize a FeroxResponse from a serde_json::Value
+    fn deserialize<D>(deserializer: D) -> anyhow::Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let mut response = Self {
+            url: Url::parse("http://localhost").unwrap(),
+            status: StatusCode::OK,
+            text: String::new(),
+            content_length: 0,
+            headers: HeaderMap::new(),
+            wildcard: false,
+            output_level: Default::default(),
+            line_count: 0,
+            word_count: 0,
+        };
+
+        let map: HashMap<String, Value> = HashMap::deserialize(deserializer)?;
+
+        for (key, value) in &map {
+            match key.as_str() {
+                "url" => {
+                    if let Some(url) = value.as_str() {
+                        if let Ok(parsed) = Url::parse(url) {
+                            response.url = parsed;
+                        }
+                    }
+                }
+                "status" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(smaller) = u16::try_from(num) {
+                            if let Ok(status) = StatusCode::from_u16(smaller) {
+                                response.status = status;
+                            }
+                        }
+                    }
+                }
+                "content_length" => {
+                    if let Some(num) = value.as_u64() {
+                        response.content_length = num;
+                    }
+                }
+                "line_count" => {
+                    if let Some(num) = value.as_u64() {
+                        response.line_count = num.try_into().unwrap_or_default();
+                    }
+                }
+                "word_count" => {
+                    if let Some(num) = value.as_u64() {
+                        response.word_count = num.try_into().unwrap_or_default();
+                    }
+                }
+                "headers" => {
+                    let mut headers = HeaderMap::<HeaderValue>::default();
+
+                    if let Some(map_headers) = value.as_object() {
+                        for (h_key, h_value) in map_headers {
+                            let h_value_str = h_value.as_str().unwrap_or("");
+                            let h_name = HeaderName::from_str(h_key)
+                                .unwrap_or_else(|_| HeaderName::from_str("Unknown").unwrap());
+                            let h_value_parsed = HeaderValue::from_str(h_value_str)
+                                .unwrap_or_else(|_| HeaderValue::from_str("Unknown").unwrap());
+                            headers.insert(h_name, h_value_parsed);
+                        }
+                    }
+
+                    response.headers = headers;
+                }
+                "wildcard" => {
+                    if let Some(result) = value.as_bool() {
+                        response.wildcard = result;
+                    }
+                }
+                _ => {}
+            }
+        }
+
+        Ok(response)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    /// call reached_max_depth with max depth of zero, which is infinite recursion, expect false
+    fn reached_max_depth_returns_early_on_zero() {
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+        let url = Url::parse("http://localhost").unwrap();
+        let response = FeroxResponse {
+            url,
+            status: Default::default(),
+            text: "".to_string(),
+            content_length: 0,
+            line_count: 0,
+            word_count: 0,
+            headers: Default::default(),
+            wildcard: false,
+            output_level: Default::default(),
+        };
+        let result = response.reached_max_depth(0, 0, handles);
+        assert!(!result);
+    }
+
+    #[test]
+    /// call reached_max_depth with url depth equal to max depth, expect true
+    fn reached_max_depth_current_depth_equals_max() {
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+
+        let url = Url::parse("http://localhost/one/two").unwrap();
+        let response = FeroxResponse {
+            url,
+            status: Default::default(),
+            text: "".to_string(),
+            content_length: 0,
+            line_count: 0,
+            word_count: 0,
+            headers: Default::default(),
+            wildcard: false,
+            output_level: Default::default(),
+        };
+
+        let result = response.reached_max_depth(0, 2, handles);
+        assert!(result);
+    }
+
+    #[test]
+    /// call reached_max_depth with url dpeth less than max depth, expect false
+    fn reached_max_depth_current_depth_less_than_max() {
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+        let url = Url::parse("http://localhost").unwrap();
+        let response = FeroxResponse {
+            url,
+            status: Default::default(),
+            text: "".to_string(),
+            content_length: 0,
+            line_count: 0,
+            word_count: 0,
+            headers: Default::default(),
+            wildcard: false,
+            output_level: Default::default(),
+        };
+
+        let result = response.reached_max_depth(0, 2, handles);
+        assert!(!result);
+    }
+
+    #[test]
+    /// call reached_max_depth with url of 2, base depth of 2, and max depth of 2, expect false
+    fn reached_max_depth_base_depth_equals_max_depth() {
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+        let url = Url::parse("http://localhost/one/two").unwrap();
+        let response = FeroxResponse {
+            url,
+            status: Default::default(),
+            text: "".to_string(),
+            content_length: 0,
+            line_count: 0,
+            word_count: 0,
+            headers: Default::default(),
+            wildcard: false,
+            output_level: Default::default(),
+        };
+
+        let result = response.reached_max_depth(2, 2, handles);
+        assert!(!result);
+    }
+
+    #[test]
+    /// call reached_max_depth with url depth greater than max depth, expect true
+    fn reached_max_depth_current_greater_than_max() {
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+        let url = Url::parse("http://localhost/one/two/three").unwrap();
+        let response = FeroxResponse {
+            url,
+            status: Default::default(),
+            text: "".to_string(),
+            content_length: 0,
+            line_count: 0,
+            word_count: 0,
+            headers: Default::default(),
+            wildcard: false,
+            output_level: Default::default(),
+        };
+
+        let result = response.reached_max_depth(0, 2, handles);
+        assert!(result);
+    }
+}

src/scan_manager/menu.rs

@@ -0,0 +1,135 @@
+use crate::progress::PROGRESS_BAR;
+use console::{measure_text_width, pad_str, style, Alignment, Term};
+use indicatif::ProgressDrawTarget;
+
+/// Interactive scan cancellation menu
+#[derive(Debug)]
+pub(super) struct Menu {
+    /// character to use as visual separator of lines
+    separator: String,
+
+    /// name of menu
+    name: String,
+
+    /// header: name surrounded by separators
+    header: String,
+
+    /// instructions
+    instructions: String,
+
+    /// footer: instructions surrounded by separators
+    footer: String,
+
+    /// target for output
+    term: Term,
+}
+
+/// Implementation of Menu
+impl Menu {
+    /// Creates new Menu
+    pub(super) fn new() -> Self {
+        let separator = "─".to_string();
+
+        let instructions = format!(
+            "Enter a {} list of indexes to {} (ex: 2,3)",
+            style("comma-separated").yellow(),
+            style("cancel").red(),
+        );
+
+        let name = format!(
+            "{} {} {}",
+            "💀",
+            style("Scan Cancel Menu").bright().yellow(),
+            "💀"
+        );
+
+        let longest = measure_text_width(&instructions).max(measure_text_width(&name));
+
+        let border = separator.repeat(longest);
+
+        let padded_name = pad_str(&name, longest, Alignment::Center, None);
+
+        let header = format!("{}\n{}\n{}", border, padded_name, border);
+        let footer = format!("{}\n{}\n{}", border, instructions, border);
+
+        Self {
+            separator,
+            name,
+            header,
+            instructions,
+            footer,
+            term: Term::stderr(),
+        }
+    }
+
+    /// print menu header
+    pub(super) fn print_header(&self) {
+        self.println(&self.header);
+    }
+
+    /// print menu footer
+    pub(super) fn print_footer(&self) {
+        self.println(&self.footer);
+    }
+
+    /// set PROGRESS_BAR bar target to hidden
+    pub(super) fn hide_progress_bars(&self) {
+        PROGRESS_BAR.set_draw_target(ProgressDrawTarget::hidden());
+    }
+
+    /// set PROGRESS_BAR bar target to hidden
+    pub(super) fn show_progress_bars(&self) {
+        PROGRESS_BAR.set_draw_target(ProgressDrawTarget::stdout());
+    }
+
+    /// Wrapper around console's Term::clear_screen and flush
+    pub(super) fn clear_screen(&self) {
+        self.term.clear_screen().unwrap_or_default();
+        self.term.flush().unwrap_or_default();
+    }
+
+    /// Wrapper around console's Term::write_line
+    pub(super) fn println(&self, msg: &str) {
+        self.term.write_line(msg).unwrap_or_default();
+    }
+
+    /// split a string into vec of usizes
+    pub(super) fn split_to_nums(&self, line: &str) -> Vec<usize> {
+        line.split(',')
+            .map(|s| {
+                s.trim().to_string().parse::<usize>().unwrap_or_else(|e| {
+                    self.println(&format!("Found non-numeric input: {}", e));
+                    0
+                })
+            })
+            .filter(|m| *m != 0)
+            .collect()
+    }
+
+    /// get comma-separated list of scan indexes from the user
+    pub(super) fn get_scans_from_user(&self) -> Option<Vec<usize>> {
+        if let Ok(line) = self.term.read_line() {
+            Some(self.split_to_nums(&line))
+        } else {
+            None
+        }
+    }
+
+    /// Given a url, confirm with user that we should cancel
+    pub(super) fn confirm_cancellation(&self, url: &str) -> char {
+        self.println(&format!(
+            "You sure you wanna cancel this scan: {}? [Y/n]",
+            url
+        ));
+
+        self.term.read_char().unwrap_or('n')
+    }
+}
+
+/// Default implementation for Menu
+impl Default for Menu {
+    /// return Menu::new as default
+    fn default() -> Menu {
+        Menu::new()
+    }
+}

src/scan_manager/mod.rs

@@ -0,0 +1,17 @@
+mod scan_container;
+mod response_container;
+mod scan;
+mod menu;
+mod utils;
+mod order;
+mod state;
+#[cfg(test)]
+mod tests;
+
+pub(self) use menu::Menu;
+pub use order::ScanOrder;
+pub use response_container::FeroxResponses;
+pub use scan::{FeroxScan, ScanStatus, ScanType};
+pub use scan_container::{FeroxScans, PAUSE_SCAN};
+pub use state::FeroxState;
+pub use utils::{resume_scan, start_max_time_thread};

src/scan_manager/order.rs

@@ -0,0 +1,10 @@
+#[derive(Debug, Copy, Clone)]
+/// Simple enum to designate whether a URL was passed in by the user (Initial) or found during
+/// scanning (Latest)
+pub enum ScanOrder {
+    /// Url was passed in by the user
+    Initial,
+
+    /// Url was found during scanning
+    Latest,
+}

src/scan_manager/response_container.rs

@@ -0,0 +1,55 @@
+use crate::response::FeroxResponse;
+use serde::{ser::SerializeSeq, Serialize, Serializer};
+use std::sync::{Arc, RwLock};
+
+/// Container around a locked vector of `FeroxResponse`s, adds wrappers for insertion and search
+#[derive(Debug, Default)]
+pub struct FeroxResponses {
+    /// Internal structure: locked hashset of `FeroxScan`s
+    pub responses: Arc<RwLock<Vec<FeroxResponse>>>,
+}
+
+/// Serialize implementation for FeroxResponses
+impl Serialize for FeroxResponses {
+    /// Function that handles serialization of FeroxResponses
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        if let Ok(responses) = self.responses.read() {
+            let mut seq = serializer.serialize_seq(Some(responses.len()))?;
+
+            for response in responses.iter() {
+                seq.serialize_element(response)?;
+            }
+
+            seq.end()
+        } else {
+            // if for some reason we can't unlock the mutex, just write an empty list
+            let seq = serializer.serialize_seq(Some(0))?;
+            seq.end()
+        }
+    }
+}
+
+/// Implementation of `FeroxResponses`
+impl FeroxResponses {
+    /// Add a `FeroxResponse` to the internal container
+    pub fn insert(&self, response: FeroxResponse) {
+        if let Ok(mut responses) = self.responses.write() {
+            responses.push(response);
+        }
+    }
+
+    /// Simple check for whether or not a FeroxResponse is contained within the inner container
+    pub fn contains(&self, other: &FeroxResponse) -> bool {
+        if let Ok(responses) = self.responses.read() {
+            for response in responses.iter() {
+                if response.url() == other.url() {
+                    return true;
+                }
+            }
+        }
+        false
+    }
+}

src/scan_manager/scan.rs

@@ -0,0 +1,362 @@
+use super::*;
+use crate::{
+    config::OutputLevel,
+    progress::{add_bar, BarType},
+};
+use anyhow::Result;
+use console::style;
+use indicatif::ProgressBar;
+use serde::{ser::SerializeStruct, Deserialize, Deserializer, Serialize, Serializer};
+use serde_json::Value;
+use std::{
+    collections::HashMap,
+    fmt,
+    sync::{Arc, Mutex},
+};
+
+use tokio::{sync, task::JoinHandle};
+use uuid::Uuid;
+
+/// Struct to hold scan-related state
+///
+/// The purpose of this container is to open up the pathway to aborting currently running tasks and
+/// serialization of all scan state into a state file in order to resume scans that were cut short
+#[derive(Debug)]
+pub struct FeroxScan {
+    /// UUID that uniquely ID's the scan
+    pub(super) id: String,
+
+    /// The URL that to be scanned
+    pub(super) url: String,
+
+    /// The type of scan
+    pub(super) scan_type: ScanType,
+
+    /// The order in which the scan was received
+    pub(super) scan_order: ScanOrder,
+
+    /// Number of requests to populate the progress bar with
+    pub(super) num_requests: u64,
+
+    /// Status of this scan
+    pub(super) status: Mutex<ScanStatus>,
+
+    /// The spawned tokio task performing this scan (uses tokio::sync::Mutex)
+    pub(super) task: sync::Mutex<Option<JoinHandle<()>>>,
+
+    /// The progress bar associated with this scan
+    pub(super) progress_bar: Mutex<Option<ProgressBar>>,
+
+    /// whether or not the user passed --silent|--quiet on the command line
+    pub(super) output_level: OutputLevel,
+}
+
+/// Default implementation for FeroxScan
+impl Default for FeroxScan {
+    /// Create a default FeroxScan, populates ID with a new UUID
+    fn default() -> Self {
+        let new_id = Uuid::new_v4().to_simple().to_string();
+
+        FeroxScan {
+            id: new_id,
+            task: sync::Mutex::new(None), // tokio mutex
+            status: Mutex::new(ScanStatus::default()),
+            num_requests: 0,
+            scan_order: ScanOrder::Latest,
+            url: String::new(),
+            progress_bar: Mutex::new(None),
+            scan_type: ScanType::File,
+            output_level: Default::default(),
+        }
+    }
+}
+
+/// Implementation of FeroxScan
+impl FeroxScan {
+    /// Stop a currently running scan
+    pub async fn abort(&self) -> Result<()> {
+        let mut guard = self.task.lock().await;
+
+        if guard.is_some() {
+            if let Some(task) = std::mem::replace(&mut *guard, None) {
+                task.abort();
+                self.set_status(ScanStatus::Cancelled)?;
+                self.stop_progress_bar();
+            }
+        }
+
+        Ok(())
+    }
+
+    /// getter for url
+    pub fn url(&self) -> &str {
+        &self.url
+    }
+
+    /// small wrapper to set the JoinHandle
+    pub async fn set_task(&self, task: JoinHandle<()>) -> Result<()> {
+        let mut guard = self.task.lock().await;
+        let _ = std::mem::replace(&mut *guard, Some(task));
+        Ok(())
+    }
+
+    /// small wrapper to set ScanStatus
+    pub fn set_status(&self, status: ScanStatus) -> Result<()> {
+        if let Ok(mut guard) = self.status.lock() {
+            let _ = std::mem::replace(&mut *guard, status);
+        }
+        Ok(())
+    }
+
+    /// Simple helper to call .finish on the scan's progress bar
+    pub(super) fn stop_progress_bar(&self) {
+        if let Ok(guard) = self.progress_bar.lock() {
+            if guard.is_some() {
+                (*guard).as_ref().unwrap().finish_at_current_pos()
+            }
+        }
+    }
+
+    /// Simple helper get a progress bar
+    pub fn progress_bar(&self) -> ProgressBar {
+        match self.progress_bar.lock() {
+            Ok(mut guard) => {
+                if guard.is_some() {
+                    (*guard).as_ref().unwrap().clone()
+                } else {
+                    let bar_type = match self.output_level {
+                        OutputLevel::Default => BarType::Default,
+                        OutputLevel::Quiet => BarType::Quiet,
+                        OutputLevel::Silent => BarType::Hidden,
+                    };
+
+                    let pb = add_bar(&self.url, self.num_requests, bar_type);
+                    pb.reset_elapsed();
+
+                    let _ = std::mem::replace(&mut *guard, Some(pb.clone()));
+                    pb
+                }
+            }
+            Err(_) => {
+                log::warn!("Could not unlock progress bar on {:?}", self);
+
+                let bar_type = match self.output_level {
+                    OutputLevel::Default => BarType::Default,
+                    OutputLevel::Quiet => BarType::Quiet,
+                    OutputLevel::Silent => BarType::Hidden,
+                };
+
+                let pb = add_bar(&self.url, self.num_requests, bar_type);
+                pb.reset_elapsed();
+
+                pb
+            }
+        }
+    }
+
+    /// Given a URL and ProgressBar, create a new FeroxScan, wrap it in an Arc and return it
+    pub fn new(
+        url: &str,
+        scan_type: ScanType,
+        scan_order: ScanOrder,
+        num_requests: u64,
+        output_level: OutputLevel,
+        pb: Option<ProgressBar>,
+    ) -> Arc<Self> {
+        Arc::new(Self {
+            url: url.to_string(),
+            scan_type,
+            scan_order,
+            num_requests,
+            output_level,
+            progress_bar: Mutex::new(pb),
+            ..Default::default()
+        })
+    }
+
+    /// Mark the scan as complete and stop the scan's progress bar
+    pub fn finish(&self) -> Result<()> {
+        self.set_status(ScanStatus::Complete)?;
+        self.stop_progress_bar();
+        Ok(())
+    }
+
+    /// small wrapper to inspect ScanType and ScanStatus to see if a Directory scan is running or
+    /// in the queue to be run
+    pub fn is_active(&self) -> bool {
+        if let Ok(guard) = self.status.lock() {
+            return matches!(
+                (self.scan_type, *guard),
+                (ScanType::Directory, ScanStatus::Running)
+                    | (ScanType::Directory, ScanStatus::NotStarted)
+            );
+        }
+        false
+    }
+
+    /// small wrapper to inspect ScanStatus and see if it's Complete
+    pub fn is_complete(&self) -> bool {
+        if let Ok(guard) = self.status.lock() {
+            return matches!(*guard, ScanStatus::Complete);
+        }
+        false
+    }
+
+    /// await a task's completion, similar to a thread's join; perform necessary bookkeeping
+    pub async fn join(&self) {
+        log::trace!("enter join({:?})", self);
+        let mut guard = self.task.lock().await;
+
+        if guard.is_some() {
+            if let Some(task) = std::mem::replace(&mut *guard, None) {
+                task.await.unwrap();
+                self.set_status(ScanStatus::Complete)
+                    .unwrap_or_else(|e| log::warn!("Could not mark scan complete: {}", e))
+            }
+        }
+
+        log::trace!("exit join({:?})", self);
+    }
+}
+
+/// Display implementation
+impl fmt::Display for FeroxScan {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let status = if let Ok(guard) = self.status.lock() {
+            match *guard {
+                ScanStatus::NotStarted => style("not started").bright().blue(),
+                ScanStatus::Complete => style("complete").green(),
+                ScanStatus::Cancelled => style("cancelled").red(),
+                ScanStatus::Running => style("running").bright().yellow(),
+            }
+        } else {
+            style("unknown").red()
+        };
+
+        write!(f, "{:12} {}", status, self.url)
+    }
+}
+
+/// PartialEq implementation; uses FeroxScan.id for comparison
+impl PartialEq for FeroxScan {
+    fn eq(&self, other: &Self) -> bool {
+        self.id == other.id
+    }
+}
+
+/// Serialize implementation for FeroxScan
+impl Serialize for FeroxScan {
+    /// Function that handles serialization of a FeroxScan
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let mut state = serializer.serialize_struct("FeroxScan", 4)?;
+
+        state.serialize_field("id", &self.id)?;
+        state.serialize_field("url", &self.url)?;
+        state.serialize_field("scan_type", &self.scan_type)?;
+        state.serialize_field("status", &self.status)?;
+        state.serialize_field("num_requests", &self.num_requests)?;
+
+        state.end()
+    }
+}
+
+/// Deserialize implementation for FeroxScan
+impl<'de> Deserialize<'de> for FeroxScan {
+    /// Deserialize a FeroxScan from a serde_json::Value
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let mut scan = Self::default();
+
+        let map: HashMap<String, Value> = HashMap::deserialize(deserializer)?;
+
+        for (key, value) in &map {
+            match key.as_str() {
+                "id" => {
+                    if let Some(id) = value.as_str() {
+                        scan.id = id.to_string();
+                    }
+                }
+                "scan_type" => {
+                    if let Some(scan_type) = value.as_str() {
+                        scan.scan_type = match scan_type {
+                            "File" => ScanType::File,
+                            "Directory" => ScanType::Directory,
+                            _ => ScanType::File,
+                        }
+                    }
+                }
+                "status" => {
+                    if let Some(status) = value.as_str() {
+                        scan.status = Mutex::new(match status {
+                            "NotStarted" => ScanStatus::NotStarted,
+                            "Running" => ScanStatus::Running,
+                            "Complete" => ScanStatus::Complete,
+                            "Cancelled" => ScanStatus::Cancelled,
+                            _ => ScanStatus::default(),
+                        })
+                    }
+                }
+                "url" => {
+                    if let Some(url) = value.as_str() {
+                        scan.url = url.to_string();
+                    }
+                }
+                "num_requests" => {
+                    if let Some(num_requests) = value.as_u64() {
+                        scan.num_requests = num_requests;
+                    }
+                }
+                _ => {}
+            }
+        }
+
+        Ok(scan)
+    }
+}
+
+/// Simple enum used to flag a `FeroxScan` as likely a directory or file
+#[derive(Debug, Serialize, Deserialize, Copy, Clone)]
+pub enum ScanType {
+    /// Just a file being requested
+    File,
+
+    /// A an entire directory that might be scanned
+    Directory,
+}
+
+/// Default implementation for ScanType
+impl Default for ScanType {
+    /// Return ScanType::File as default
+    fn default() -> Self {
+        Self::File
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, Copy, Clone)]
+/// Simple enum to represent a scan's current status ([in]complete, cancelled)
+pub enum ScanStatus {
+    /// Scan hasn't started yet
+    NotStarted,
+
+    /// Scan finished normally
+    Complete,
+
+    /// Scan was cancelled by the user
+    Cancelled,
+
+    /// Scan has started, but hasn't finished, nor been cancelled
+    Running,
+}
+
+/// Default implementation for ScanStatus
+impl Default for ScanStatus {
+    /// Default variant for ScanStatus is NotStarted
+    fn default() -> Self {
+        Self::NotStarted
+    }
+}

src/scan_manager/scan_container.rs

@@ -0,0 +1,428 @@
+use super::scan::ScanType;
+use super::*;
+use crate::{
+    config::OutputLevel,
+    progress::PROGRESS_PRINTER,
+    progress::{add_bar, BarType},
+    scanner::RESPONSES,
+    traits::FeroxSerialize,
+    SLEEP_DURATION,
+};
+use anyhow::Result;
+use serde::{ser::SerializeSeq, Serialize, Serializer};
+use std::{
+    convert::TryInto,
+    fs::File,
+    io::BufReader,
+    ops::Index,
+    sync::{
+        atomic::{AtomicBool, AtomicUsize, Ordering},
+        Arc, Mutex, RwLock,
+    },
+    thread::sleep,
+};
+use tokio::time::{self, Duration};
+
+/// Single atomic number that gets incremented once, used to track first thread to interact with
+/// when pausing a scan
+static INTERACTIVE_BARRIER: AtomicUsize = AtomicUsize::new(0);
+
+/// Atomic boolean flag, used to determine whether or not a scan should pause or resume
+pub static PAUSE_SCAN: AtomicBool = AtomicBool::new(false);
+
+/// Container around a locked hashset of `FeroxScan`s, adds wrappers for insertion and searching
+#[derive(Debug, Default)]
+pub struct FeroxScans {
+    /// Internal structure: locked hashset of `FeroxScan`s
+    pub scans: RwLock<Vec<Arc<FeroxScan>>>,
+
+    /// menu used for providing a way for users to cancel a scan
+    menu: Menu,
+
+    /// number of requests expected per scan (mirrors the same on Stats); used for initializing
+    /// progress bars and feroxscans
+    bar_length: Mutex<u64>,
+
+    /// whether or not the user passed --silent|--quiet on the command line
+    output_level: OutputLevel,
+}
+
+/// Serialize implementation for FeroxScans
+///
+/// purposefully skips menu attribute
+impl Serialize for FeroxScans {
+    /// Function that handles serialization of FeroxScans
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        if let Ok(scans) = self.scans.read() {
+            let mut seq = serializer.serialize_seq(Some(scans.len()))?;
+            for scan in scans.iter() {
+                seq.serialize_element(&*scan).unwrap_or_default();
+            }
+
+            seq.end()
+        } else {
+            // if for some reason we can't unlock the RwLock, just write an empty list
+            let seq = serializer.serialize_seq(Some(0))?;
+            seq.end()
+        }
+    }
+}
+
+/// Implementation of `FeroxScans`
+impl FeroxScans {
+    /// given an OutputLevel, create a new FeroxScans object
+    pub fn new(output_level: OutputLevel) -> Self {
+        Self {
+            output_level,
+            ..Default::default()
+        }
+    }
+
+    /// Add a `FeroxScan` to the internal container
+    ///
+    /// If the internal container did NOT contain the scan, true is returned; else false
+    pub fn insert(&self, scan: Arc<FeroxScan>) -> bool {
+        // If the container did contain the scan, set sentry to false
+        // If the container did not contain the scan, set sentry to true
+        let sentry = !self.contains(&scan.url);
+
+        if sentry {
+            // can't update the internal container while the scan itself is locked, so first
+            // lock the scan and check the container for the scan's presence, then add if
+            // not found
+            match self.scans.write() {
+                Ok(mut scans) => {
+                    scans.push(scan);
+                }
+                Err(e) => {
+                    log::warn!("FeroxScans' container's mutex is poisoned: {}", e);
+                    return false;
+                }
+            }
+        }
+
+        sentry
+    }
+
+    /// load serialized FeroxScan(s) into this FeroxScans  
+    pub fn add_serialized_scans(&self, filename: &str) -> Result<()> {
+        log::trace!("enter: add_serialized_scans({})", filename);
+        let file = File::open(filename)?;
+
+        let reader = BufReader::new(file);
+        let state: serde_json::Value = serde_json::from_reader(reader)?;
+
+        if let Some(scans) = state.get("scans") {
+            if let Some(arr_scans) = scans.as_array() {
+                for scan in arr_scans {
+                    let mut deser_scan: FeroxScan =
+                        serde_json::from_value(scan.clone()).unwrap_or_default();
+                    // FeroxScans gets -q value from config as usual; the FeroxScans themselves
+                    // rely on that value being passed in. If the user starts a scan without -q
+                    // and resumes the scan but adds -q, FeroxScan will not have the proper value
+                    // without the line below
+                    deser_scan.output_level = self.output_level;
+
+                    log::debug!("added: {}", deser_scan);
+                    self.insert(Arc::new(deser_scan));
+                }
+            }
+        }
+
+        log::trace!("exit: add_serialized_scans");
+        Ok(())
+    }
+
+    /// Simple check for whether or not a FeroxScan is contained within the inner container based
+    /// on the given URL
+    pub fn contains(&self, url: &str) -> bool {
+        if let Ok(scans) = self.scans.read() {
+            for scan in scans.iter() {
+                if scan.url == url {
+                    return true;
+                }
+            }
+        }
+        false
+    }
+
+    /// Find and return a `FeroxScan` based on the given URL
+    pub fn get_scan_by_url(&self, url: &str) -> Option<Arc<FeroxScan>> {
+        if let Ok(guard) = self.scans.read() {
+            for scan in guard.iter() {
+                if scan.url == url {
+                    return Some(scan.clone());
+                }
+            }
+        }
+        None
+    }
+
+    /// Print all FeroxScans of type Directory
+    ///
+    /// Example:
+    ///   0: complete   https://10.129.45.20
+    ///   9: complete   https://10.129.45.20/images
+    ///  10: complete   https://10.129.45.20/assets
+    pub async fn display_scans(&self) {
+        let scans = {
+            // written this way in order to grab the vector and drop the lock immediately
+            // otherwise the spawned task that this is a part of is no longer Send due to
+            // the scan.task.lock().await below while the lock is held (RwLock is not Send)
+            self.scans
+                .read()
+                .expect("Could not acquire lock in display_scans")
+                .clone()
+        };
+
+        for (i, scan) in scans.iter().enumerate() {
+            if matches!(scan.scan_order, ScanOrder::Initial) || scan.task.try_lock().is_err() {
+                // original target passed in via either -u or --stdin
+                continue;
+            }
+
+            if matches!(scan.scan_type, ScanType::Directory) {
+                // we're only interested in displaying directory scans, as those are
+                // the only ones that make sense to be stopped
+                let scan_msg = format!("{:3}: {}", i, scan);
+                self.menu.println(&scan_msg);
+            }
+        }
+    }
+
+    /// Given a list of indexes, cancel their associated FeroxScans
+    async fn cancel_scans(&self, indexes: Vec<usize>) {
+        let menu_pause_duration = Duration::from_millis(SLEEP_DURATION);
+
+        for num in indexes {
+            let selected = match self.scans.read() {
+                Ok(u_scans) => {
+                    // check if number provided is out of range
+                    if num >= u_scans.len() {
+                        // usize can't be negative, just need to handle exceeding bounds
+                        self.menu
+                            .println(&format!("The number {} is not a valid choice.", num));
+                        sleep(menu_pause_duration);
+                        continue;
+                    }
+                    u_scans.index(num).clone()
+                }
+                Err(..) => continue,
+            };
+
+            let input = self.menu.confirm_cancellation(&selected.url);
+
+            if input == 'y' || input == '\n' {
+                self.menu.println(&format!("Stopping {}...", selected.url));
+                selected
+                    .abort()
+                    .await
+                    .unwrap_or_else(|e| log::warn!("Could not cancel task: {}", e));
+            } else {
+                self.menu.println("Ok, doing nothing...");
+            }
+
+            sleep(menu_pause_duration);
+        }
+    }
+
+    /// CLI menu that allows for interactive cancellation of recursed-into directories
+    async fn interactive_menu(&self) {
+        self.menu.hide_progress_bars();
+        self.menu.clear_screen();
+        self.menu.print_header();
+        self.display_scans().await;
+        self.menu.print_footer();
+
+        if let Some(input) = self.menu.get_scans_from_user() {
+            self.cancel_scans(input).await
+        };
+
+        self.menu.clear_screen();
+        self.menu.show_progress_bars();
+    }
+
+    /// prints all known responses that the scanner has already seen
+    pub fn print_known_responses(&self) {
+        if let Ok(mut responses) = RESPONSES.responses.write() {
+            for response in responses.iter_mut() {
+                if self.output_level != response.output_level {
+                    // set the output_level prior to printing the response to ensure that the
+                    // response's setting aligns with the overall configuration (since we're
+                    // calling this from a resumed state)
+                    response.output_level = self.output_level;
+                }
+                PROGRESS_PRINTER.println(response.as_str());
+            }
+        }
+    }
+
+    /// if a resumed scan is already complete, display a completed progress bar to the user
+    pub fn print_completed_bars(&self, bar_length: usize) -> Result<()> {
+        let bar_type = match self.output_level {
+            OutputLevel::Default => BarType::Message,
+            OutputLevel::Quiet => BarType::Quiet,
+            OutputLevel::Silent => return Ok(()), // fast exit when --silent was used
+        };
+
+        if let Ok(scans) = self.scans.read() {
+            for scan in scans.iter() {
+                if scan.is_complete() {
+                    // these scans are complete, and just need to be shown to the user
+                    let pb = add_bar(
+                        &scan.url,
+                        bar_length.try_into().unwrap_or_default(),
+                        bar_type,
+                    );
+                    pb.finish();
+                }
+            }
+        }
+        Ok(())
+    }
+
+    /// Forced the calling thread into a busy loop
+    ///
+    /// Every `SLEEP_DURATION` milliseconds, the function examines the result stored in `PAUSE_SCAN`
+    ///
+    /// When the value stored in `PAUSE_SCAN` becomes `false`, the function returns, exiting the busy
+    /// loop
+    pub async fn pause(&self, get_user_input: bool) {
+        // function uses tokio::time, not std
+
+        // local testing showed a pretty slow increase (less than linear) in CPU usage as # of
+        // concurrent scans rose when SLEEP_DURATION was set to 500, using that as the default for now
+        let mut interval = time::interval(time::Duration::from_millis(SLEEP_DURATION));
+
+        if INTERACTIVE_BARRIER.load(Ordering::Relaxed) == 0 {
+            INTERACTIVE_BARRIER.fetch_add(1, Ordering::Relaxed);
+
+            if get_user_input {
+                self.interactive_menu().await;
+                PAUSE_SCAN.store(false, Ordering::Relaxed);
+                self.print_known_responses();
+            }
+        }
+
+        loop {
+            // first tick happens immediately, all others wait the specified duration
+            interval.tick().await;
+
+            if !PAUSE_SCAN.load(Ordering::Acquire) {
+                // PAUSE_SCAN is false, so we can exit the busy loop
+
+                if INTERACTIVE_BARRIER.load(Ordering::Relaxed) == 1 {
+                    INTERACTIVE_BARRIER.fetch_sub(1, Ordering::Relaxed);
+                }
+
+                log::trace!("exit: pause_scan");
+                return;
+            }
+        }
+    }
+
+    /// set the bar length of FeroxScans
+    pub fn set_bar_length(&self, bar_length: u64) {
+        if let Ok(mut guard) = self.bar_length.lock() {
+            *guard = bar_length;
+        }
+    }
+
+    /// Given a url, create a new `FeroxScan` and add it to `FeroxScans`
+    ///
+    /// If `FeroxScans` did not already contain the scan, return true; otherwise return false
+    ///
+    /// Also return a reference to the new `FeroxScan`
+    pub(super) fn add_scan(
+        &self,
+        url: &str,
+        scan_type: ScanType,
+        scan_order: ScanOrder,
+    ) -> (bool, Arc<FeroxScan>) {
+        let bar_length = if let Ok(guard) = self.bar_length.lock() {
+            *guard
+        } else {
+            0
+        };
+
+        let bar = match scan_type {
+            ScanType::Directory => {
+                let bar_type = match self.output_level {
+                    OutputLevel::Default => BarType::Default,
+                    OutputLevel::Quiet => BarType::Quiet,
+                    OutputLevel::Silent => BarType::Hidden,
+                };
+
+                let progress_bar = add_bar(&url, bar_length, bar_type);
+
+                progress_bar.reset_elapsed();
+
+                Some(progress_bar)
+            }
+            ScanType::File => None,
+        };
+
+        let ferox_scan = FeroxScan::new(
+            &url,
+            scan_type,
+            scan_order,
+            bar_length,
+            self.output_level,
+            bar,
+        );
+
+        // If the set did not contain the scan, true is returned.
+        // If the set did contain the scan, false is returned.
+        let response = self.insert(ferox_scan.clone());
+
+        (response, ferox_scan)
+    }
+
+    /// Given a url, create a new `FeroxScan` and add it to `FeroxScans` as a Directory Scan
+    ///
+    /// If `FeroxScans` did not already contain the scan, return true; otherwise return false
+    ///
+    /// Also return a reference to the new `FeroxScan`
+    pub fn add_directory_scan(&self, url: &str, scan_order: ScanOrder) -> (bool, Arc<FeroxScan>) {
+        self.add_scan(&url, ScanType::Directory, scan_order)
+    }
+
+    /// Given a url, create a new `FeroxScan` and add it to `FeroxScans` as a File Scan
+    ///
+    /// If `FeroxScans` did not already contain the scan, return true; otherwise return false
+    ///
+    /// Also return a reference to the new `FeroxScan`
+    pub fn add_file_scan(&self, url: &str, scan_order: ScanOrder) -> (bool, Arc<FeroxScan>) {
+        self.add_scan(&url, ScanType::File, scan_order)
+    }
+
+    /// small helper to determine whether any scans are active or not
+    pub fn has_active_scans(&self) -> bool {
+        if let Ok(guard) = self.scans.read() {
+            for scan in guard.iter() {
+                if scan.is_active() {
+                    return true;
+                }
+            }
+        }
+        false
+    }
+
+    /// Retrieve all active scans
+    pub fn get_active_scans(&self) -> Vec<Arc<FeroxScan>> {
+        let mut scans = vec![];
+
+        if let Ok(guard) = self.scans.read() {
+            for scan in guard.iter() {
+                if !scan.is_active() {
+                    continue;
+                }
+                scans.push(scan.clone());
+            }
+        }
+        scans
+    }
+}

src/scan_manager/state.rs

@@ -0,0 +1,53 @@
+use super::*;
+use crate::{config::Configuration, statistics::Stats, traits::FeroxSerialize, utils::fmt_err};
+use anyhow::{Context, Result};
+use serde::Serialize;
+use std::sync::Arc;
+
+/// Data container for (de)?serialization of multiple items
+#[derive(Serialize, Debug)]
+pub struct FeroxState {
+    /// Known scans
+    scans: Arc<FeroxScans>,
+
+    /// Current running config
+    config: Arc<Configuration>,
+
+    /// Known responses
+    responses: &'static FeroxResponses,
+
+    /// Gathered statistics
+    statistics: Arc<Stats>,
+}
+
+/// implementation of FeroxState
+impl FeroxState {
+    /// create new FeroxState object
+    pub fn new(
+        scans: Arc<FeroxScans>,
+        config: Arc<Configuration>,
+        responses: &'static FeroxResponses,
+        statistics: Arc<Stats>,
+    ) -> Self {
+        Self {
+            scans,
+            config,
+            responses,
+            statistics,
+        }
+    }
+}
+
+/// FeroxSerialize implementation for FeroxState
+impl FeroxSerialize for FeroxState {
+    /// Simply return debug format of FeroxState to satisfy as_str
+    fn as_str(&self) -> String {
+        format!("{:?}", self)
+    }
+
+    /// Simple call to produce a JSON string using the given FeroxState
+    fn as_json(&self) -> Result<String> {
+        Ok(serde_json::to_string(&self)
+            .with_context(|| fmt_err("Could not convert scan's running state to JSON"))?)
+    }
+}

src/scan_manager/tests.rs

@@ -0,0 +1,518 @@
+use super::*;
+use crate::{
+    config::{Configuration, OutputLevel},
+    event_handlers::Handles,
+    response::FeroxResponse,
+    scanner::RESPONSES,
+    statistics::Stats,
+    traits::FeroxSerialize,
+    SLEEP_DURATION, VERSION,
+};
+use indicatif::ProgressBar;
+use predicates::prelude::*;
+use std::sync::{atomic::Ordering, Arc};
+use std::thread::sleep;
+use tokio::time::{self, Duration};
+
+#[test]
+/// test that ScanType's default is File
+fn default_scantype_is_file() {
+    match ScanType::default() {
+        ScanType::File => {}
+        ScanType::Directory => panic!(),
+    }
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// tests that pause_scan pauses execution and releases execution when PAUSE_SCAN is toggled
+/// the spinner used during the test has had .finish_and_clear called on it, meaning that
+/// a new one will be created, taking the if branch within the function
+async fn scanner_pause_scan_with_finished_spinner() {
+    let now = time::Instant::now();
+    let urls = FeroxScans::default();
+
+    PAUSE_SCAN.store(true, Ordering::Relaxed);
+
+    let expected = time::Duration::from_secs(2);
+
+    tokio::spawn(async move {
+        time::sleep(expected).await;
+        PAUSE_SCAN.store(false, Ordering::Relaxed);
+    });
+
+    urls.pause(false).await;
+
+    assert!(now.elapsed() > expected);
+}
+
+#[test]
+/// add an unknown url to the hashset, expect true
+fn add_url_to_list_of_scanned_urls_with_unknown_url() {
+    let urls = FeroxScans::default();
+    let url = "http://unknown_url";
+    let (result, _scan) = urls.add_scan(url, ScanType::Directory, ScanOrder::Latest);
+    assert_eq!(result, true);
+}
+
+#[test]
+/// add a known url to the hashset, with a trailing slash, expect false
+fn add_url_to_list_of_scanned_urls_with_known_url() {
+    let urls = FeroxScans::default();
+    let pb = ProgressBar::new(1);
+    let url = "http://unknown_url/";
+
+    let scan = FeroxScan::new(
+        url,
+        ScanType::Directory,
+        ScanOrder::Latest,
+        pb.length(),
+        OutputLevel::Default,
+        Some(pb),
+    );
+
+    assert_eq!(urls.insert(scan), true);
+
+    let (result, _scan) = urls.add_scan(url, ScanType::Directory, ScanOrder::Latest);
+
+    assert_eq!(result, false);
+}
+
+#[test]
+/// stop_progress_bar should stop the progress bar
+fn stop_progress_bar_stops_bar() {
+    let pb = ProgressBar::new(1);
+    let url = "http://unknown_url/";
+
+    let scan = FeroxScan::new(
+        url,
+        ScanType::Directory,
+        ScanOrder::Latest,
+        pb.length(),
+        OutputLevel::Default,
+        Some(pb),
+    );
+
+    assert_eq!(
+        scan.progress_bar
+            .lock()
+            .unwrap()
+            .as_ref()
+            .unwrap()
+            .is_finished(),
+        false
+    );
+
+    scan.stop_progress_bar();
+
+    assert_eq!(
+        scan.progress_bar
+            .lock()
+            .unwrap()
+            .as_ref()
+            .unwrap()
+            .is_finished(),
+        true
+    );
+}
+
+#[test]
+/// add a known url to the hashset, without a trailing slash, expect false
+fn add_url_to_list_of_scanned_urls_with_known_url_without_slash() {
+    let urls = FeroxScans::default();
+    let url = "http://unknown_url";
+
+    let scan = FeroxScan::new(
+        url,
+        ScanType::File,
+        ScanOrder::Latest,
+        0,
+        OutputLevel::Default,
+        None,
+    );
+
+    assert_eq!(urls.insert(scan), true);
+
+    let (result, _scan) = urls.add_scan(url, ScanType::File, ScanOrder::Latest);
+
+    assert_eq!(result, false);
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// just increasing coverage, no real expectations
+async fn call_display_scans() {
+    let urls = FeroxScans::default();
+    let pb = ProgressBar::new(1);
+    let pb_two = ProgressBar::new(2);
+    let url = "http://unknown_url/";
+    let url_two = "http://unknown_url/fa";
+    let scan = FeroxScan::new(
+        url,
+        ScanType::Directory,
+        ScanOrder::Latest,
+        pb.length(),
+        OutputLevel::Default,
+        Some(pb),
+    );
+    let scan_two = FeroxScan::new(
+        url_two,
+        ScanType::Directory,
+        ScanOrder::Latest,
+        pb_two.length(),
+        OutputLevel::Default,
+        Some(pb_two),
+    );
+
+    scan_two.finish().unwrap(); // one complete, one incomplete
+    scan_two
+        .set_task(tokio::spawn(async move {
+            sleep(Duration::from_millis(SLEEP_DURATION));
+        }))
+        .await
+        .unwrap();
+
+    assert_eq!(urls.insert(scan), true);
+    assert_eq!(urls.insert(scan_two), true);
+
+    urls.display_scans().await;
+}
+
+#[test]
+/// ensure that PartialEq compares FeroxScan.id fields
+fn partial_eq_compares_the_id_field() {
+    let url = "http://unknown_url/";
+    let scan = FeroxScan::new(
+        url,
+        ScanType::Directory,
+        ScanOrder::Latest,
+        0,
+        OutputLevel::Default,
+        None,
+    );
+    let scan_two = FeroxScan::new(
+        url,
+        ScanType::Directory,
+        ScanOrder::Latest,
+        0,
+        OutputLevel::Default,
+        None,
+    );
+
+    assert!(!scan.eq(&scan_two));
+
+    let scan_two = scan.clone();
+
+    assert!(scan.eq(&scan_two));
+}
+
+#[test]
+/// show that a new progress bar is created if one doesn't exist
+fn ferox_scan_get_progress_bar_when_none_is_set() {
+    let scan = FeroxScan::default();
+
+    assert!(scan.progress_bar.lock().unwrap().is_none()); // no pb exists
+
+    let pb = scan.progress_bar();
+
+    assert!(scan.progress_bar.lock().unwrap().is_some()); // new pb created
+    assert!(!pb.is_finished()) // not finished
+}
+
+#[test]
+/// given a JSON entry representing a FeroxScan, test that it deserializes into the proper type
+/// with the right attributes
+fn ferox_scan_deserialize() {
+    let fs_json = r#"{"id":"057016a14769414aac9a7a62707598cb","url":"https://spiritanimal.com","scan_type":"Directory","status":"Complete"}"#;
+    let fs_json_two = r#"{"id":"057016a14769414aac9a7a62707598cb","url":"https://spiritanimal.com","scan_type":"Not Correct","status":"Cancelled"}"#;
+    let fs_json_three = r#"{"id":"057016a14769414aac9a7a62707598cb","url":"https://spiritanimal.com","scan_type":"Not Correct","status":"","num_requests":42}"#;
+
+    let fs: FeroxScan = serde_json::from_str(fs_json).unwrap();
+    let fs_two: FeroxScan = serde_json::from_str(fs_json_two).unwrap();
+    let fs_three: FeroxScan = serde_json::from_str(fs_json_three).unwrap();
+    assert_eq!(fs.url, "https://spiritanimal.com");
+
+    match fs.scan_type {
+        ScanType::Directory => {}
+        ScanType::File => {
+            panic!();
+        }
+    }
+    match fs_two.scan_type {
+        ScanType::Directory => {
+            panic!();
+        }
+        ScanType::File => {}
+    }
+
+    match *fs.progress_bar.lock().unwrap() {
+        None => {}
+        Some(_) => {
+            panic!();
+        }
+    }
+    assert!(matches!(*fs.status.lock().unwrap(), ScanStatus::Complete));
+    assert!(matches!(
+        *fs_two.status.lock().unwrap(),
+        ScanStatus::Cancelled
+    ));
+    assert!(matches!(
+        *fs_three.status.lock().unwrap(),
+        ScanStatus::NotStarted
+    ));
+    assert_eq!(fs_three.num_requests, 42);
+    assert_eq!(fs.id, "057016a14769414aac9a7a62707598cb");
+}
+
+#[test]
+/// given a FeroxScan, test that it serializes into the proper JSON entry
+fn ferox_scan_serialize() {
+    let fs = FeroxScan::new(
+        "https://spiritanimal.com",
+        ScanType::Directory,
+        ScanOrder::Latest,
+        0,
+        OutputLevel::Default,
+        None,
+    );
+    let fs_json = format!(
+        r#"{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","status":"NotStarted","num_requests":0}}"#,
+        fs.id
+    );
+    assert_eq!(fs_json, serde_json::to_string(&*fs).unwrap());
+}
+
+#[test]
+/// given a FeroxScans, test that it serializes into the proper JSON entry
+fn ferox_scans_serialize() {
+    let ferox_scan = FeroxScan::new(
+        "https://spiritanimal.com",
+        ScanType::Directory,
+        ScanOrder::Latest,
+        0,
+        OutputLevel::Default,
+        None,
+    );
+    let ferox_scans = FeroxScans::default();
+    let ferox_scans_json = format!(
+        r#"[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","status":"NotStarted","num_requests":0}}]"#,
+        ferox_scan.id
+    );
+    ferox_scans.scans.write().unwrap().push(ferox_scan);
+    assert_eq!(
+        ferox_scans_json,
+        serde_json::to_string(&ferox_scans).unwrap()
+    );
+}
+
+#[test]
+/// given a FeroxResponses, test that it serializes into the proper JSON entry
+fn ferox_responses_serialize() {
+    let json_response = r#"{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{"server":"nginx/1.16.1"}}"#;
+    let response: FeroxResponse = serde_json::from_str(json_response).unwrap();
+
+    let responses = FeroxResponses::default();
+    responses.insert(response);
+    // responses has a response now
+
+    // serialized should be a list of responses
+    let expected = format!("[{}]", json_response);
+
+    let serialized = serde_json::to_string(&responses).unwrap();
+    assert_eq!(expected, serialized);
+}
+
+#[test]
+/// given a FeroxResponse, test that it serializes into the proper JSON entry
+fn ferox_response_serialize_and_deserialize() {
+    // deserialize
+    let json_response = r#"{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{"server":"nginx/1.16.1"}}"#;
+    let response: FeroxResponse = serde_json::from_str(json_response).unwrap();
+
+    assert_eq!(response.url().as_str(), "https://nerdcore.com/css");
+    assert_eq!(response.url().path(), "/css");
+    assert_eq!(response.wildcard(), true);
+    assert_eq!(response.status().as_u16(), 301);
+    assert_eq!(response.content_length(), 173);
+    assert_eq!(response.line_count(), 10);
+    assert_eq!(response.word_count(), 16);
+    assert_eq!(response.headers().get("server").unwrap(), "nginx/1.16.1");
+
+    // serialize, however, this can fail when headers are out of order
+    let new_json = serde_json::to_string(&response).unwrap();
+    assert_eq!(json_response, new_json);
+}
+
+#[test]
+/// test FeroxSerialize implementation of FeroxState
+fn feroxstates_feroxserialize_implementation() {
+    let ferox_scan = FeroxScan::new(
+        "https://spiritanimal.com",
+        ScanType::Directory,
+        ScanOrder::Latest,
+        0,
+        OutputLevel::Default,
+        None,
+    );
+    let ferox_scans = FeroxScans::default();
+    let saved_id = ferox_scan.id.clone();
+    ferox_scans.insert(ferox_scan);
+
+    let config = Configuration::new().unwrap();
+    let stats = Arc::new(Stats::new(config.extensions.len(), config.json));
+
+    let json_response = r#"{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{"server":"nginx/1.16.1"}}"#;
+    let response: FeroxResponse = serde_json::from_str(json_response).unwrap();
+    RESPONSES.insert(response);
+
+    let ferox_state = FeroxState::new(
+        Arc::new(ferox_scans),
+        Arc::new(Configuration::new().unwrap()),
+        &RESPONSES,
+        stats,
+    );
+
+    let expected_strs = predicates::str::contains("scans: FeroxScans").and(
+        predicate::str::contains("config: Configuration")
+            .and(predicate::str::contains("responses: FeroxResponses"))
+            .and(predicate::str::contains("nerdcore.com"))
+            .and(predicate::str::contains("/css"))
+            .and(predicate::str::contains("https://spiritanimal.com")),
+    );
+
+    assert!(expected_strs.eval(&ferox_state.as_str()));
+
+    let json_state = ferox_state.as_json().unwrap();
+    let expected = format!(
+        r#"{{"scans":[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","status":"NotStarted","num_requests":0}}],"config":{{"type":"configuration","wordlist":"/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt","config":"","proxy":"","replay_proxy":"","target_url":"","status_codes":[200,204,301,302,307,308,401,403,405],"replay_codes":[200,204,301,302,307,308,401,403,405],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"silent":false,"quiet":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/{}","redirects":false,"insecure":false,"extensions":[],"headers":{{}},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":false,"depth":4,"scan_limit":0,"rate_limit":0,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false,"resumed":false,"resume_from":"","save_state":false,"time_limit":"","filter_similar":[]}},"responses":[{{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{{"server":"nginx/1.16.1"}}}}]"#,
+        saved_id, VERSION
+    );
+    println!("{}\n{}", expected, json_state);
+    assert!(predicates::str::contains(expected).eval(&json_state));
+}
+
+#[should_panic]
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// call start_max_time_thread with a valid timespec, expect a panic, but only after a certain
+/// number of seconds
+async fn start_max_time_thread_panics_after_delay() {
+    let now = time::Instant::now();
+    let delay = time::Duration::new(3, 0);
+
+    let config = Configuration {
+        time_limit: String::from("3s"),
+        ..Default::default()
+    };
+
+    let handles = Arc::new(Handles::for_testing(None, Some(Arc::new(config))).0);
+
+    start_max_time_thread(handles).await;
+
+    assert!(now.elapsed() > delay);
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// call start_max_time_thread with a timespec that's too large to be parsed correctly, expect
+/// immediate return and no panic, as the sigint handler is never called
+async fn start_max_time_thread_returns_immediately_with_too_large_input() {
+    let now = time::Instant::now();
+    let delay = time::Duration::new(1, 0);
+    let config = Configuration {
+        time_limit: String::from("18446744073709551616m"),
+        ..Default::default()
+    };
+
+    let handles = Arc::new(Handles::for_testing(None, Some(Arc::new(config))).0);
+
+    // pub const MAX: usize = usize::MAX; // 18_446_744_073_709_551_615usize
+    start_max_time_thread(handles).await; // can't fit in dest u64
+
+    assert!(now.elapsed() < delay); // assuming function call will take less than 1second
+}
+
+#[test]
+/// coverage for FeroxScan's Display implementation
+fn feroxscan_display() {
+    let scan = FeroxScan {
+        id: "".to_string(),
+        url: String::from("http://localhost"),
+        scan_order: ScanOrder::Latest,
+        scan_type: Default::default(),
+        num_requests: 0,
+        output_level: OutputLevel::Default,
+        status: Default::default(),
+        task: tokio::sync::Mutex::new(None),
+        progress_bar: std::sync::Mutex::new(None),
+    };
+
+    let not_started = format!("{}", scan);
+
+    assert!(predicate::str::contains("not started")
+        .and(predicate::str::contains("localhost"))
+        .eval(&not_started));
+
+    scan.set_status(ScanStatus::Complete).unwrap();
+    let complete = format!("{}", scan);
+    assert!(predicate::str::contains("complete")
+        .and(predicate::str::contains("localhost"))
+        .eval(&complete));
+
+    scan.set_status(ScanStatus::Cancelled).unwrap();
+    let cancelled = format!("{}", scan);
+    assert!(predicate::str::contains("cancelled")
+        .and(predicate::str::contains("localhost"))
+        .eval(&cancelled));
+
+    scan.set_status(ScanStatus::Running).unwrap();
+    let running = format!("{}", scan);
+    assert!(predicate::str::contains("running")
+        .and(predicate::str::contains("localhost"))
+        .eval(&running));
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// call FeroxScan::abort, ensure status becomes cancelled
+async fn ferox_scan_abort() {
+    let scan = FeroxScan {
+        id: "".to_string(),
+        url: String::from("http://localhost"),
+        scan_order: ScanOrder::Latest,
+        scan_type: Default::default(),
+        num_requests: 0,
+        output_level: OutputLevel::Default,
+        status: std::sync::Mutex::new(ScanStatus::Running),
+        task: tokio::sync::Mutex::new(Some(tokio::spawn(async move {
+            sleep(Duration::from_millis(SLEEP_DURATION * 2));
+        }))),
+        progress_bar: std::sync::Mutex::new(None),
+    };
+
+    scan.abort().await.unwrap();
+
+    assert!(matches!(
+        *scan.status.lock().unwrap(),
+        ScanStatus::Cancelled
+    ));
+}
+
+#[test]
+/// call a few menu functions for coverage's sake
+///
+/// there's not a trivial way to test these programmatically (at least i'm too lazy rn to do it)
+/// and their correctness can be verified easily manually; just calling for now
+fn menu_print_header_and_footer() {
+    let menu = Menu::new();
+    menu.clear_screen();
+    menu.print_header();
+    menu.print_footer();
+    menu.hide_progress_bars();
+    menu.show_progress_bars();
+}
+
+#[test]
+/// ensure spaces are trimmed and numbers are returned from split_to_nums
+fn split_to_nums_is_correct() {
+    let menu = Menu::new();
+
+    let nums = menu.split_to_nums("1, 3,      4");
+
+    assert_eq!(nums, vec![1, 3, 4]);
+}

src/scan_manager/utils.rs

@@ -0,0 +1,92 @@
+#[cfg(not(test))]
+use crate::event_handlers::TermInputHandler;
+use crate::{
+    config::Configuration, event_handlers::Handles, parser::TIMESPEC_REGEX, scanner::RESPONSES,
+};
+
+use std::{fs::File, io::BufReader, sync::Arc};
+use tokio::time;
+
+/// Given a string representing some number of seconds, minutes, hours, or days, convert
+/// that representation to seconds and then wait for those seconds to elapse.  Once that period
+/// of time has elapsed, kill all currently running scans and dump a state file to disk that can
+/// be used to resume any unfinished scan.
+pub async fn start_max_time_thread(handles: Arc<Handles>) {
+    log::trace!("enter: start_max_time_thread({:?})", handles);
+
+    // as this function has already made it through the parser, which calls is_match on
+    // the value passed to --time-limit using TIMESPEC_REGEX; we can safely assume that
+    // the capture groups are populated; can expect something like 10m, 30s, 1h, etc...
+    let captures = TIMESPEC_REGEX.captures(&handles.config.time_limit).unwrap();
+    let length_match = captures.get(1).unwrap();
+    let measurement_match = captures.get(2).unwrap();
+
+    if let Ok(length) = length_match.as_str().parse::<u64>() {
+        let length_in_secs = match measurement_match.as_str().to_ascii_lowercase().as_str() {
+            "s" => length,
+            "m" => length * 60,           // minutes
+            "h" => length * 60 * 60,      // hours
+            "d" => length * 60 * 60 * 24, // days
+            _ => length,
+        };
+
+        log::debug!(
+            "max time limit as string: {} and as seconds: {}",
+            handles.config.time_limit,
+            length_in_secs
+        );
+
+        time::sleep(time::Duration::new(length_in_secs, 0)).await;
+
+        log::trace!("exit: start_max_time_thread");
+
+        #[cfg(test)]
+        panic!(handles);
+        #[cfg(not(test))]
+        let _ = TermInputHandler::sigint_handler(handles.clone());
+    }
+
+    log::warn!(
+        "Could not parse the value provided ({}), can't enforce time limit",
+        handles.config.time_limit
+    );
+}
+
+/// Primary logic used to load a Configuration from disk and populate the appropriate data
+/// structures
+pub fn resume_scan(filename: &str) -> Configuration {
+    log::trace!("enter: resume_scan({})", filename);
+
+    let file = File::open(filename).unwrap_or_else(|e| {
+        log::error!("{}", e);
+        log::error!("Could not open state file, exiting");
+        std::process::exit(1);
+    });
+
+    let reader = BufReader::new(file);
+    let state: serde_json::Value = serde_json::from_reader(reader).unwrap();
+
+    let conf = state.get("config").unwrap_or_else(|| {
+        log::error!("Could not load configuration from state file, exiting");
+        std::process::exit(1);
+    });
+
+    let config = serde_json::from_value(conf.clone()).unwrap_or_else(|e| {
+        log::error!("{}", e);
+        log::error!("Could not deserialize configuration found in state file, exiting");
+        std::process::exit(1);
+    });
+
+    if let Some(responses) = state.get("responses") {
+        if let Some(arr_responses) = responses.as_array() {
+            for response in arr_responses {
+                if let Ok(deser_resp) = serde_json::from_value(response.clone()) {
+                    RESPONSES.insert(deser_resp);
+                }
+            }
+        }
+    }
+
+    log::trace!("exit: resume_scan -> {:?}", config);
+    config
+}

src/statistics/container.rs

@@ -0,0 +1,620 @@
+use std::{
+    fs::File,
+    io::BufReader,
+    sync::{
+        atomic::{AtomicUsize, Ordering},
+        Mutex,
+    },
+};
+
+use anyhow::{Context, Result};
+use reqwest::StatusCode;
+use serde::{Deserialize, Serialize};
+
+use crate::{
+    traits::FeroxSerialize,
+    utils::{fmt_err, open_file, write_to},
+};
+
+use super::{error::StatError, field::StatField};
+
+/// Data collection of statistics related to a scan
+#[derive(Default, Deserialize, Debug, Serialize)]
+pub struct Stats {
+    #[serde(rename = "type")]
+    /// Name of this type of struct, used for serialization, i.e. `{"type":"statistics"}`
+    kind: String,
+
+    /// tracker for number of timeouts seen by the client
+    timeouts: AtomicUsize,
+
+    /// tracker for total number of requests sent by the client
+    requests: AtomicUsize,
+
+    /// tracker for total number of requests expected to send if the scan runs to completion
+    ///
+    /// Note: this is a per-scan expectation; `expected_requests * current # of scans` would be
+    /// indicative of the current expectation at any given time, but is a moving target.  
+    expected_per_scan: AtomicUsize,
+
+    /// tracker for accumulating total number of requests expected (i.e. as a new scan is started
+    /// this value should increase by `expected_requests`
+    total_expected: AtomicUsize,
+
+    /// tracker for total number of errors encountered by the client
+    errors: AtomicUsize,
+
+    /// tracker for overall number of 2xx status codes seen by the client
+    successes: AtomicUsize,
+
+    /// tracker for overall number of 3xx status codes seen by the client
+    redirects: AtomicUsize,
+
+    /// tracker for overall number of 4xx status codes seen by the client
+    client_errors: AtomicUsize,
+
+    /// tracker for overall number of 5xx status codes seen by the client
+    server_errors: AtomicUsize,
+
+    /// tracker for number of scans performed, this directly equates to number of directories
+    /// recursed into and affects the total number of expected requests
+    total_scans: AtomicUsize,
+
+    /// tracker for initial number of requested targets
+    initial_targets: AtomicUsize,
+
+    /// tracker for number of links extracted when `--extract-links` is used; sources are
+    /// response bodies and robots.txt as of v1.11.0
+    links_extracted: AtomicUsize,
+
+    /// tracker for overall number of 200s seen by the client
+    status_200s: AtomicUsize,
+
+    /// tracker for overall number of 301s seen by the client
+    status_301s: AtomicUsize,
+
+    /// tracker for overall number of 302s seen by the client
+    status_302s: AtomicUsize,
+
+    /// tracker for overall number of 401s seen by the client
+    status_401s: AtomicUsize,
+
+    /// tracker for overall number of 403s seen by the client
+    status_403s: AtomicUsize,
+
+    /// tracker for overall number of 429s seen by the client
+    status_429s: AtomicUsize,
+
+    /// tracker for overall number of 500s seen by the client
+    status_500s: AtomicUsize,
+
+    /// tracker for overall number of 503s seen by the client
+    status_503s: AtomicUsize,
+
+    /// tracker for overall number of 504s seen by the client
+    status_504s: AtomicUsize,
+
+    /// tracker for overall number of 508s seen by the client
+    status_508s: AtomicUsize,
+
+    /// tracker for overall number of wildcard urls filtered out by the client
+    wildcards_filtered: AtomicUsize,
+
+    /// tracker for overall number of all filtered responses
+    responses_filtered: AtomicUsize,
+
+    /// tracker for number of files found
+    resources_discovered: AtomicUsize,
+
+    /// tracker for number of errors triggered during URL formatting
+    url_format_errors: AtomicUsize,
+
+    /// tracker for number of errors triggered by the `reqwest::RedirectPolicy`
+    redirection_errors: AtomicUsize,
+
+    /// tracker for number of errors related to the connecting
+    connection_errors: AtomicUsize,
+
+    /// tracker for number of errors related to the request used
+    request_errors: AtomicUsize,
+
+    /// tracker for each directory's total scan time in seconds as a float
+    directory_scan_times: Mutex<Vec<f64>>,
+
+    /// tracker for total runtime
+    total_runtime: Mutex<Vec<f64>>,
+
+    /// tracker for the number of extensions the user specified
+    #[serde(skip)]
+    num_extensions: usize,
+
+    /// tracker for whether to use json during serialization or not
+    #[serde(skip)]
+    json: bool,
+}
+
+/// FeroxSerialize implementation for Stats
+impl FeroxSerialize for Stats {
+    /// Simply return empty string here to disable serializing this to the output file as a string
+    /// due to it looking like garbage
+    fn as_str(&self) -> String {
+        String::new()
+    }
+
+    /// Simple call to produce a JSON string using the given Stats object
+    fn as_json(&self) -> Result<String> {
+        Ok(serde_json::to_string(&self)?)
+    }
+}
+
+/// implementation of statistics data collection struct
+impl Stats {
+    /// Small wrapper for default to set `kind` to "statistics" and `total_runtime` to have at least
+    /// one value
+    pub fn new(num_extensions: usize, is_json: bool) -> Self {
+        Self {
+            num_extensions,
+            json: is_json,
+            kind: String::from("statistics"),
+            total_runtime: Mutex::new(vec![0.0]),
+            ..Default::default()
+        }
+    }
+
+    /// public getter for expected_per_scan
+    pub fn expected_per_scan(&self) -> usize {
+        atomic_load!(self.expected_per_scan)
+    }
+
+    /// public getter for resources_discovered
+    pub fn resources_discovered(&self) -> usize {
+        atomic_load!(self.resources_discovered)
+    }
+
+    /// public getter for errors
+    pub fn errors(&self) -> usize {
+        atomic_load!(self.errors)
+    }
+
+    /// public getter for total_expected
+    pub fn total_expected(&self) -> usize {
+        atomic_load!(self.total_expected)
+    }
+
+    /// public getter for initial_targets
+    pub fn initial_targets(&self) -> usize {
+        atomic_load!(self.initial_targets)
+    }
+
+    /// increment `requests` field by one
+    pub fn add_request(&self) {
+        atomic_increment!(self.requests);
+    }
+
+    /// given an `Instant` update total runtime
+    fn update_runtime(&self, seconds: f64) {
+        if let Ok(mut runtime) = self.total_runtime.lock() {
+            runtime[0] = seconds;
+        }
+    }
+
+    /// save an instance of `Stats` to disk after updating the total runtime for the scan
+    pub fn save(&self, seconds: f64, location: &str) -> Result<()> {
+        let mut file = open_file(location)?;
+
+        self.update_runtime(seconds);
+
+        write_to(self, &mut file, self.json)?;
+
+        Ok(())
+    }
+
+    /// Inspect the given `StatError` and increment the appropriate fields
+    ///
+    /// Implies incrementing:
+    ///     - requests
+    ///     - errors
+    pub fn add_error(&self, error: StatError) {
+        self.add_request();
+        atomic_increment!(self.errors);
+
+        match error {
+            StatError::Timeout => {
+                atomic_increment!(self.timeouts);
+            }
+            StatError::Status403 => {
+                atomic_increment!(self.status_403s);
+                atomic_increment!(self.client_errors);
+            }
+            StatError::UrlFormat => {
+                atomic_increment!(self.url_format_errors);
+            }
+            StatError::Redirection => {
+                atomic_increment!(self.redirection_errors);
+            }
+            StatError::Connection => {
+                atomic_increment!(self.connection_errors);
+            }
+            StatError::Request => {
+                atomic_increment!(self.request_errors);
+            }
+            StatError::Other => {
+                atomic_increment!(self.errors);
+            }
+        }
+    }
+
+    /// Inspect the given `StatusCode` and increment the appropriate fields
+    ///
+    /// Implies incrementing:
+    ///     - requests
+    ///     - status_403s (when code is 403)
+    ///     - errors (when code is [45]xx)
+    pub fn add_status_code(&self, status: StatusCode) {
+        self.add_request();
+
+        if status.is_success() {
+            atomic_increment!(self.successes);
+        } else if status.is_redirection() {
+            atomic_increment!(self.redirects);
+        } else if status.is_client_error() {
+            atomic_increment!(self.client_errors);
+        } else if status.is_server_error() {
+            atomic_increment!(self.server_errors);
+        }
+
+        match status {
+            StatusCode::FORBIDDEN => {
+                atomic_increment!(self.status_403s);
+            }
+            StatusCode::OK => {
+                atomic_increment!(self.status_200s);
+            }
+            StatusCode::MOVED_PERMANENTLY => {
+                atomic_increment!(self.status_301s);
+            }
+            StatusCode::FOUND => {
+                atomic_increment!(self.status_302s);
+            }
+            StatusCode::UNAUTHORIZED => {
+                atomic_increment!(self.status_401s);
+            }
+            StatusCode::TOO_MANY_REQUESTS => {
+                atomic_increment!(self.status_429s);
+            }
+            StatusCode::INTERNAL_SERVER_ERROR => {
+                atomic_increment!(self.status_500s);
+            }
+            StatusCode::SERVICE_UNAVAILABLE => {
+                atomic_increment!(self.status_503s);
+            }
+            StatusCode::GATEWAY_TIMEOUT => {
+                atomic_increment!(self.status_504s);
+            }
+            StatusCode::LOOP_DETECTED => {
+                atomic_increment!(self.status_508s);
+            }
+            _ => {} // other status codes ignored for stat gathering
+        }
+    }
+
+    /// Update a `Stats` field of type f64
+    pub fn update_f64_field(&self, field: StatField, value: f64) {
+        if let StatField::DirScanTimes = field {
+            if let Ok(mut locked_times) = self.directory_scan_times.lock() {
+                locked_times.push(value);
+            }
+        }
+    }
+
+    /// Update a `Stats` field of type usize
+    pub fn update_usize_field(&self, field: StatField, value: usize) {
+        match field {
+            StatField::ExpectedPerScan => {
+                atomic_increment!(self.expected_per_scan, value);
+            }
+            StatField::TotalScans => {
+                let multiplier = self.num_extensions.max(1);
+
+                atomic_increment!(self.total_scans, value);
+                atomic_increment!(
+                    self.total_expected,
+                    value * self.expected_per_scan.load(Ordering::Relaxed) * multiplier
+                );
+            }
+            StatField::TotalExpected => {
+                atomic_increment!(self.total_expected, value);
+            }
+            StatField::LinksExtracted => {
+                atomic_increment!(self.links_extracted, value);
+            }
+            StatField::WildcardsFiltered => {
+                atomic_increment!(self.wildcards_filtered, value);
+                atomic_increment!(self.responses_filtered, value);
+            }
+            StatField::ResponsesFiltered => {
+                atomic_increment!(self.responses_filtered, value);
+            }
+            StatField::ResourcesDiscovered => {
+                atomic_increment!(self.resources_discovered, value);
+            }
+            StatField::InitialTargets => {
+                atomic_increment!(self.initial_targets, value);
+            }
+            _ => {} // f64 fields
+        }
+    }
+
+    /// Merge a given `Stats` object from a json entry written to disk when handling a Ctrl+c
+    ///
+    /// This is only ever called when resuming a scan from disk
+    pub fn merge_from(&self, filename: &str) -> Result<()> {
+        let file = File::open(filename)
+            .with_context(|| fmt_err(&format!("Could not open {}", filename)))?;
+        let reader = BufReader::new(file);
+        let state: serde_json::Value = serde_json::from_reader(reader)?;
+
+        if let Some(state_stats) = state.get("statistics") {
+            let d_stats = serde_json::from_value::<Stats>(state_stats.clone())?;
+            atomic_increment!(self.successes, atomic_load!(d_stats.successes));
+            atomic_increment!(self.timeouts, atomic_load!(d_stats.timeouts));
+            atomic_increment!(self.requests, atomic_load!(d_stats.requests));
+            atomic_increment!(self.errors, atomic_load!(d_stats.errors));
+            atomic_increment!(self.redirects, atomic_load!(d_stats.redirects));
+            atomic_increment!(self.client_errors, atomic_load!(d_stats.client_errors));
+            atomic_increment!(self.server_errors, atomic_load!(d_stats.server_errors));
+            atomic_increment!(self.links_extracted, atomic_load!(d_stats.links_extracted));
+            atomic_increment!(self.status_200s, atomic_load!(d_stats.status_200s));
+            atomic_increment!(self.status_301s, atomic_load!(d_stats.status_301s));
+            atomic_increment!(self.status_302s, atomic_load!(d_stats.status_302s));
+            atomic_increment!(self.status_401s, atomic_load!(d_stats.status_401s));
+            atomic_increment!(self.status_403s, atomic_load!(d_stats.status_403s));
+            atomic_increment!(self.status_429s, atomic_load!(d_stats.status_429s));
+            atomic_increment!(self.status_500s, atomic_load!(d_stats.status_500s));
+            atomic_increment!(self.status_503s, atomic_load!(d_stats.status_503s));
+            atomic_increment!(self.status_504s, atomic_load!(d_stats.status_504s));
+            atomic_increment!(self.status_508s, atomic_load!(d_stats.status_508s));
+            atomic_increment!(
+                self.wildcards_filtered,
+                atomic_load!(d_stats.wildcards_filtered)
+            );
+            atomic_increment!(
+                self.responses_filtered,
+                atomic_load!(d_stats.responses_filtered)
+            );
+            atomic_increment!(
+                self.resources_discovered,
+                atomic_load!(d_stats.resources_discovered)
+            );
+            atomic_increment!(
+                self.url_format_errors,
+                atomic_load!(d_stats.url_format_errors)
+            );
+            atomic_increment!(
+                self.connection_errors,
+                atomic_load!(d_stats.connection_errors)
+            );
+            atomic_increment!(
+                self.redirection_errors,
+                atomic_load!(d_stats.redirection_errors)
+            );
+            atomic_increment!(self.request_errors, atomic_load!(d_stats.request_errors));
+
+            if let Ok(scan_times) = d_stats.directory_scan_times.lock() {
+                for scan_time in scan_times.iter() {
+                    self.update_f64_field(StatField::DirScanTimes, *scan_time);
+                }
+            };
+        }
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::{config::Configuration, Command};
+    use std::fs::write;
+    use tempfile::NamedTempFile;
+
+    use super::super::*;
+    use super::*;
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// when sent StatCommand::AddRequest, stats object should reflect the change
+    async fn statistics_handler_increments_requests() -> Result<()> {
+        let (task, handle) = setup_stats_test();
+
+        handle.tx.send(Command::AddRequest)?;
+        handle.tx.send(Command::AddRequest)?;
+        handle.tx.send(Command::AddRequest)?;
+
+        teardown_stats_test(handle.tx.clone(), task).await;
+
+        assert_eq!(handle.data.requests.load(Ordering::Relaxed), 3);
+
+        Ok(())
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// when sent StatCommand::AddRequest, stats object should reflect the change
+    ///
+    /// incrementing a 403 (tracked in status_403s) should also increment:
+    ///     - errors
+    ///     - requests
+    ///     - client_errors
+    async fn statistics_handler_increments_403() {
+        let (task, handle) = setup_stats_test();
+
+        let err = Command::AddError(StatError::Status403);
+        let err2 = Command::AddError(StatError::Status403);
+
+        handle.tx.send(err).unwrap_or_default();
+        handle.tx.send(err2).unwrap_or_default();
+
+        teardown_stats_test(handle.tx.clone(), task).await;
+
+        assert_eq!(handle.data.errors.load(Ordering::Relaxed), 2);
+        assert_eq!(handle.data.requests.load(Ordering::Relaxed), 2);
+        assert_eq!(handle.data.status_403s.load(Ordering::Relaxed), 2);
+        assert_eq!(handle.data.client_errors.load(Ordering::Relaxed), 2);
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// when sent StatCommand::AddRequest, stats object should reflect the change
+    ///
+    /// incrementing a 403 (tracked in status_403s) should also increment:
+    ///     - requests
+    ///     - client_errors
+    async fn statistics_handler_increments_403_via_status_code() {
+        let (task, handle) = setup_stats_test();
+
+        let err = Command::AddStatus(reqwest::StatusCode::FORBIDDEN);
+        let err2 = Command::AddStatus(reqwest::StatusCode::FORBIDDEN);
+
+        handle.tx.send(err).unwrap_or_default();
+        handle.tx.send(err2).unwrap_or_default();
+
+        teardown_stats_test(handle.tx.clone(), task).await;
+
+        assert_eq!(handle.data.requests.load(Ordering::Relaxed), 2);
+        assert_eq!(handle.data.status_403s.load(Ordering::Relaxed), 2);
+        assert_eq!(handle.data.client_errors.load(Ordering::Relaxed), 2);
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// when sent StatCommand::AddStatus, stats object should reflect the change
+    ///
+    /// incrementing a 500 (tracked in server_errors) should also increment:
+    ///     - requests
+    async fn statistics_handler_increments_500_via_status_code() -> Result<()> {
+        let (task, handle) = setup_stats_test();
+
+        let err = Command::AddStatus(reqwest::StatusCode::INTERNAL_SERVER_ERROR);
+        let err2 = Command::AddStatus(reqwest::StatusCode::INTERNAL_SERVER_ERROR);
+
+        handle.tx.send(err)?;
+        handle.tx.send(err2)?;
+
+        teardown_stats_test(handle.tx.clone(), task).await;
+
+        assert_eq!(handle.data.requests.load(Ordering::Relaxed), 2);
+        assert_eq!(handle.data.server_errors.load(Ordering::Relaxed), 2);
+
+        Ok(())
+    }
+
+    #[test]
+    /// when Stats::add_error receives StatError::Timeout, it should increment the following:
+    ///     - timeouts
+    ///     - requests
+    ///     - errors
+    fn stats_increments_timeouts() {
+        let config = Configuration::new().unwrap();
+        let stats = Stats::new(config.extensions.len(), config.json);
+
+        stats.add_error(StatError::Timeout);
+        stats.add_error(StatError::Timeout);
+        stats.add_error(StatError::Timeout);
+        stats.add_error(StatError::Timeout);
+
+        assert_eq!(stats.errors.load(Ordering::Relaxed), 4);
+        assert_eq!(stats.requests.load(Ordering::Relaxed), 4);
+        assert_eq!(stats.timeouts.load(Ordering::Relaxed), 4);
+    }
+
+    #[test]
+    /// when Stats::update_usize_field receives StatField::WildcardsFiltered, it should increment
+    /// the following:
+    ///     - responses_filtered
+    fn stats_increments_wildcards() {
+        let config = Configuration::new().unwrap();
+        let stats = Stats::new(config.extensions.len(), config.json);
+
+        assert_eq!(stats.responses_filtered.load(Ordering::Relaxed), 0);
+        assert_eq!(stats.wildcards_filtered.load(Ordering::Relaxed), 0);
+
+        stats.update_usize_field(StatField::WildcardsFiltered, 1);
+        stats.update_usize_field(StatField::WildcardsFiltered, 1);
+
+        assert_eq!(stats.responses_filtered.load(Ordering::Relaxed), 2);
+        assert_eq!(stats.wildcards_filtered.load(Ordering::Relaxed), 2);
+    }
+
+    #[test]
+    /// when Stats::update_usize_field receives StatField::ResponsesFiltered, it should increment
+    fn stats_increments_responses_filtered() {
+        let config = Configuration::new().unwrap();
+        let stats = Stats::new(config.extensions.len(), config.json);
+
+        assert_eq!(stats.responses_filtered.load(Ordering::Relaxed), 0);
+
+        stats.update_usize_field(StatField::ResponsesFiltered, 1);
+        stats.update_usize_field(StatField::ResponsesFiltered, 1);
+        stats.update_usize_field(StatField::ResponsesFiltered, 1);
+
+        assert_eq!(stats.responses_filtered.load(Ordering::Relaxed), 3);
+    }
+
+    #[test]
+    /// Stats::merge_from should properly increment expected fields and ignore others
+    fn stats_merge_from_alters_correct_fields() {
+        let contents = r#"{"statistics":{"type":"statistics","timeouts":1,"requests":9207,"expected_per_scan":707,"total_expected":9191,"errors":3,"successes":720,"redirects":13,"client_errors":8474,"server_errors":2,"total_scans":13,"initial_targets":1,"links_extracted":51,"status_403s":3,"status_200s":720,"status_301s":12,"status_302s":1,"status_401s":4,"status_429s":2,"status_500s":5,"status_503s":9,"status_504s":6,"status_508s":7,"wildcards_filtered":707,"responses_filtered":707,"resources_discovered":27,"directory_scan_times":[2.211973078,1.989015505,1.898675839,3.9714468910000003,4.938152838,5.256073528,6.021986595,6.065740734,6.42633762,7.095142125,7.336982137,5.319785619,4.843649778],"total_runtime":[11.556575456000001],"url_format_errors":17,"redirection_errors":12,"connection_errors":21,"request_errors":4}}"#;
+        let config = Configuration::new().unwrap();
+        let stats = Stats::new(config.extensions.len(), config.json);
+
+        let tfile = NamedTempFile::new().unwrap();
+        write(&tfile, contents).unwrap();
+
+        stats.merge_from(tfile.path().to_str().unwrap()).unwrap();
+
+        // as of 1.11.1; all Stats fields are accounted for whether they're updated in merge_from
+        // or not
+        assert_eq!(atomic_load!(stats.timeouts), 1);
+        assert_eq!(atomic_load!(stats.requests), 9207);
+        assert_eq!(atomic_load!(stats.expected_per_scan), 0); // not updated in merge_from
+        assert_eq!(atomic_load!(stats.total_expected), 0); // not updated in merge_from
+        assert_eq!(atomic_load!(stats.errors), 3);
+        assert_eq!(atomic_load!(stats.successes), 720);
+        assert_eq!(atomic_load!(stats.redirects), 13);
+        assert_eq!(atomic_load!(stats.client_errors), 8474);
+        assert_eq!(atomic_load!(stats.server_errors), 2);
+        assert_eq!(atomic_load!(stats.total_scans), 0); // not updated in merge_from
+        assert_eq!(atomic_load!(stats.initial_targets), 0); // not updated in merge_from
+        assert_eq!(atomic_load!(stats.links_extracted), 51);
+        assert_eq!(atomic_load!(stats.status_200s), 720);
+        assert_eq!(atomic_load!(stats.status_301s), 12);
+        assert_eq!(atomic_load!(stats.status_302s), 1);
+        assert_eq!(atomic_load!(stats.status_401s), 4);
+        assert_eq!(atomic_load!(stats.status_403s), 3);
+        assert_eq!(atomic_load!(stats.status_429s), 2);
+        assert_eq!(atomic_load!(stats.status_500s), 5);
+        assert_eq!(atomic_load!(stats.status_503s), 9);
+        assert_eq!(atomic_load!(stats.status_504s), 6);
+        assert_eq!(atomic_load!(stats.status_508s), 7);
+        assert_eq!(atomic_load!(stats.wildcards_filtered), 707);
+        assert_eq!(atomic_load!(stats.responses_filtered), 707);
+        assert_eq!(atomic_load!(stats.resources_discovered), 27);
+        assert_eq!(atomic_load!(stats.url_format_errors), 17);
+        assert_eq!(atomic_load!(stats.redirection_errors), 12);
+        assert_eq!(atomic_load!(stats.connection_errors), 21);
+        assert_eq!(atomic_load!(stats.request_errors), 4);
+        assert_eq!(stats.directory_scan_times.lock().unwrap().len(), 13);
+        for scan in stats.directory_scan_times.lock().unwrap().iter() {
+            assert!(scan.max(0.0) > 0.0); // all scans are non-zero
+        }
+        // total_runtime not updated in merge_from
+        assert_eq!(stats.total_runtime.lock().unwrap().len(), 1);
+        assert!((stats.total_runtime.lock().unwrap()[0] - 0.0).abs() < f64::EPSILON);
+    }
+
+    #[test]
+    /// ensure update runtime overwrites the default 0th entry
+    fn update_runtime_works() {
+        let config = Configuration::new().unwrap();
+        let stats = Stats::new(config.extensions.len(), config.json);
+
+        assert!((stats.total_runtime.lock().unwrap()[0] - 0.0).abs() < f64::EPSILON);
+        stats.update_runtime(20.2);
+        assert!((stats.total_runtime.lock().unwrap()[0] - 20.2).abs() < f64::EPSILON);
+    }
+}

src/statistics/error.rs

@@ -0,0 +1,24 @@
+#[derive(Debug, Copy, Clone)]
+/// Enum variants used to inform the `StatCommand` protocol what `Stats` fields should be updated
+pub enum StatError {
+    /// Represents a 403 response code
+    Status403,
+
+    /// Represents a timeout error
+    Timeout,
+
+    /// Represents a URL formatting error
+    UrlFormat,
+
+    /// Represents an error encountered during redirection
+    Redirection,
+
+    /// Represents an error encountered during connection
+    Connection,
+
+    /// Represents an error resulting from the client's request
+    Request,
+
+    /// Represents any other error not explicitly defined above
+    Other,
+}

src/statistics/field.rs

@@ -0,0 +1,33 @@
+/// Enum representing fields whose updates need to be performed in batches instead of one at
+/// a time
+#[derive(Debug, Copy, Clone)]
+pub enum StatField {
+    /// Due to the necessary order of events, the number of requests expected to be sent isn't
+    /// known until after `statistics::initialize` is called. This command allows for updating
+    /// the `expected_per_scan` field after initialization
+    ExpectedPerScan,
+
+    /// Translates to `total_scans`
+    TotalScans,
+
+    /// Translates to `links_extracted`
+    LinksExtracted,
+
+    /// Translates to `total_expected`
+    TotalExpected,
+
+    /// Translates to `wildcards_filtered`
+    WildcardsFiltered,
+
+    /// Translates to `responses_filtered`
+    ResponsesFiltered,
+
+    /// Translates to `resources_discovered`
+    ResourcesDiscovered,
+
+    /// Translates to `initial_targets`
+    InitialTargets,
+
+    /// Translates to `directory_scan_times`; assumes a single append to the vector
+    DirScanTimes,
+}

src/statistics/init.rs

@@ -0,0 +1 @@
+

src/statistics/macros.rs

@@ -0,0 +1,23 @@
+#![macro_use]
+
+/// Wrapper `Atomic*.fetch_add` to save me from writing Ordering::Relaxed a bajillion times
+///
+/// default is to increment by 1, second arg can be used to increment by a different value
+#[macro_export]
+macro_rules! atomic_increment {
+    ($metric:expr) => {
+        $metric.fetch_add(1, Ordering::Relaxed);
+    };
+
+    ($metric:expr, $value:expr) => {
+        $metric.fetch_add($value, Ordering::Relaxed);
+    };
+}
+
+/// Wrapper around `Atomic*.load` to save me from writing Ordering::Relaxed a bajillion times
+#[macro_export]
+macro_rules! atomic_load {
+    ($metric:expr) => {
+        $metric.load(Ordering::Relaxed);
+    };
+}

src/statistics/mod.rs

@@ -0,0 +1,13 @@
+mod error;
+mod macros;
+mod container;
+mod field;
+#[cfg(test)]
+mod tests;
+
+pub use self::container::Stats;
+pub use self::error::StatError;
+pub use self::field::StatField;
+
+#[cfg(test)]
+use self::tests::{setup_stats_test, teardown_stats_test};

src/statistics/tests.rs

@@ -0,0 +1,66 @@
+use super::*;
+use crate::{
+    config::Configuration,
+    event_handlers::{Command, StatsHandle, StatsHandler},
+    traits::FeroxSerialize,
+    CommandSender, Joiner,
+};
+use anyhow::Result;
+use reqwest::StatusCode;
+use std::sync::Arc;
+use tempfile::NamedTempFile;
+
+/// simple helper to reduce code reuse
+pub fn setup_stats_test() -> (Joiner, StatsHandle) {
+    let config = Arc::new(Configuration::new().unwrap());
+    StatsHandler::initialize(config)
+}
+
+/// another helper to stay DRY; must be called after any sent commands and before any checks
+/// performed against the Stats object
+pub async fn teardown_stats_test(sender: CommandSender, task: Joiner) {
+    // send exit and await, once the await completes, stats should be updated
+    sender.send(Command::Exit).unwrap_or_default();
+    task.await.unwrap().unwrap();
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// when sent StatCommand::Exit, function should exit its while loop (runs forever otherwise)
+async fn statistics_handler_exits() -> Result<()> {
+    let (task, handle) = setup_stats_test();
+
+    handle.tx.send(Command::Exit)?;
+
+    task.await??; // blocks on the handler's while loop
+
+    // if we've made it here, the test has succeeded
+    Ok(())
+}
+
+#[test]
+/// Stats::save should write contents of Stats to disk
+fn save_writes_stats_object_to_disk() {
+    let config = Configuration::new().unwrap();
+    let stats = Stats::new(config.extensions.len(), config.json);
+
+    stats.add_request();
+    stats.add_request();
+    stats.add_request();
+    stats.add_request();
+    stats.add_error(StatError::Timeout);
+    stats.add_error(StatError::Timeout);
+    stats.add_error(StatError::Timeout);
+    stats.add_error(StatError::Timeout);
+    stats.add_status_code(StatusCode::OK);
+    stats.add_status_code(StatusCode::OK);
+    stats.add_status_code(StatusCode::OK);
+    let outfile = NamedTempFile::new().unwrap();
+    if stats
+        .save(174.33, &outfile.path().to_str().unwrap())
+        .is_ok()
+    {}
+
+    assert!(stats.as_json().unwrap().contains("statistics"));
+    assert!(stats.as_json().unwrap().contains("11")); // requests made
+    assert!(stats.as_str().is_empty());
+}

src/traits.rs

@@ -0,0 +1,43 @@
+//! collection of all traits used
+use crate::response::FeroxResponse;
+use anyhow::Result;
+use serde::Serialize;
+use std::any::Any;
+use std::fmt::Debug;
+
+// references:
+//   https://dev.to/magnusstrale/rust-trait-objects-in-a-vector-non-trivial-4co5
+//   https://stackoverflow.com/questions/25339603/how-to-test-for-equality-between-trait-objects
+
+/// FeroxFilter trait; represents different types of possible filters that can be applied to
+/// responses
+pub trait FeroxFilter: Debug + Send + Sync {
+    /// Determine whether or not this particular filter should be applied or not
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool;
+
+    /// delegates to the FeroxFilter-implementing type which gives us the actual type of self
+    fn box_eq(&self, other: &dyn Any) -> bool;
+
+    /// gives us `other` as Any in box_eq
+    fn as_any(&self) -> &dyn Any;
+}
+
+/// implementation of PartialEq, necessary long-form due to "trait cannot be made into an object"
+/// error when attempting to derive PartialEq on the trait itself
+impl PartialEq for Box<dyn FeroxFilter> {
+    /// Perform a comparison of two implementors of the FeroxFilter trait
+    fn eq(&self, other: &Box<dyn FeroxFilter>) -> bool {
+        self.box_eq(other.as_any())
+    }
+}
+
+/// FeroxSerialize trait; represents different types that are Serialize and also implement
+/// as_str / as_json methods
+pub trait FeroxSerialize: Serialize {
+    /// Return a String representation of the object, generally the human readable version of the
+    /// implementor
+    fn as_str(&self) -> String;
+
+    /// Return an NDJSON representation of the object
+    fn as_json(&self) -> Result<String>;
+}

src/url.rs

@@ -0,0 +1,463 @@
+use crate::{event_handlers::Handles, statistics::StatError::UrlFormat, Command::AddError};
+use anyhow::{anyhow, bail, Result};
+use reqwest::Url;
+use std::{convert::TryInto, fmt, sync::Arc};
+
+/// abstraction around target urls; collects all Url related shenanigans in one place
+#[derive(Debug)]
+pub struct FeroxUrl {
+    /// string representation of the target url
+    target: String,
+
+    /// Handles object for grabbing config values
+    handles: Arc<Handles>,
+}
+
+/// implementation of FeroxUrl
+impl FeroxUrl {
+    /// Create new FeroxUrl given a target url as a string
+    pub fn from_string(target: &str, handles: Arc<Handles>) -> Self {
+        Self {
+            handles,
+            target: String::from(target),
+        }
+    }
+
+    /// Create new FeroxUrl given a target url as a reqwest::Url
+    pub fn from_url(target: &Url, handles: Arc<Handles>) -> Self {
+        Self {
+            handles,
+            target: target.as_str().to_string(),
+        }
+    }
+
+    /// Creates a vector of formatted Urls
+    ///
+    /// At least one value will be returned (base_url + word)
+    ///
+    /// If any extensions were passed to the program, each extension will add a
+    /// (base_url + word + ext) Url to the vector
+    pub fn formatted_urls(&self, word: &str) -> Result<Vec<Url>> {
+        log::trace!("enter: formatted_urls({})", word);
+
+        let mut urls = vec![];
+
+        match self.format(word, None) {
+            // default request, i.e. no extension
+            Ok(url) => urls.push(url),
+            Err(_) => self.handles.stats.send(AddError(UrlFormat))?,
+        }
+
+        for ext in self.handles.config.extensions.iter() {
+            match self.format(word, Some(ext)) {
+                // any extensions passed in
+                Ok(url) => urls.push(url),
+                Err(_) => self.handles.stats.send(AddError(UrlFormat))?,
+            }
+        }
+
+        log::trace!("exit: formatted_urls -> {:?}", urls);
+        Ok(urls)
+    }
+
+    /// Simple helper to generate a `Url`
+    ///
+    /// Errors during parsing `url` or joining `word` are propagated up the call stack
+    pub fn format(&self, word: &str, extension: Option<&str>) -> Result<Url> {
+        log::trace!("enter: format({}, {:?})", word, extension);
+
+        if Url::parse(&word).is_ok() {
+            // when a full url is passed in as a word to be joined to a base url using
+            // reqwest::Url::join, the result is that the word (url) completely overwrites the base
+            // url, potentially resulting in requests to places that aren't actually the target
+            // specified.
+            //
+            // in order to resolve the issue, we check if the word from the wordlist is a parsable URL
+            // and if so, don't do any further processing
+            let message = format!("word ({}) from wordlist is a URL, skipping...", word);
+            log::warn!("{}", message);
+            log::trace!("exit: format -> Err({})", message);
+            bail!(message);
+        }
+
+        // from reqwest::Url::join
+        //   Note: a trailing slash is significant. Without it, the last path component
+        //   is considered to be a “file” name to be removed to get at the “directory”
+        //   that is used as the base
+        //
+        // the transforms that occur here will need to keep this in mind, i.e. add a slash to preserve
+        // the current directory sent as part of the url
+        let url = if word.is_empty() {
+            // v1.0.6: added during --extract-links feature implementation to support creating urls
+            // that were extracted from response bodies, i.e. http://localhost/some/path/js/main.js
+            self.target.to_string()
+        } else if !self.target.ends_with('/') {
+            format!("{}/", self.target)
+        } else {
+            self.target.to_string()
+        };
+
+        // extensions and slashes are mutually exclusive cases
+        let word = if extension.is_some() {
+            format!("{}.{}", word, extension.unwrap())
+        } else if self.handles.config.add_slash && !word.ends_with('/') {
+            // -f used, and word doesn't already end with a /
+            format!("{}/", word)
+        } else if word.starts_with("//") {
+            // bug ID'd by @Sicks3c, when a wordlist contains words that begin with 2 forward slashes
+            // i.e. //1_40_0/static/js, it gets joined onto the base url in a surprising way
+            // ex: https://localhost/ + //1_40_0/static/js -> https://1_40_0/static/js
+            // this is due to the fact that //... is a valid url. The fix is introduced here in 1.12.2
+            // and simply removes prefixed forward slashes if there are two of them. Additionally,
+            // trim_start_matches will trim the pattern until it's gone, so even if there are more than
+            // 2 /'s, they'll still be trimmed
+            word.trim_start_matches('/').to_string()
+        } else {
+            String::from(word)
+        };
+
+        let base_url = Url::parse(&url)?;
+        let joined = base_url.join(&word)?;
+
+        if self.handles.config.queries.is_empty() {
+            // no query params to process
+            log::trace!("exit: format -> {}", joined);
+            Ok(joined)
+        } else {
+            let with_params =
+                Url::parse_with_params(joined.as_str(), &self.handles.config.queries)?;
+            log::trace!("exit: format_url -> {}", with_params);
+            Ok(with_params) // request with params attached
+        }
+    }
+
+    /// Gets the length of a url's path
+    pub fn path_length(&self) -> Result<u64> {
+        let parsed = Url::parse(&self.target)?;
+        Ok(FeroxUrl::path_length_of_url(&parsed))
+    }
+
+    /// Gets the length of a url's path
+    ///
+    /// example: http://localhost/stuff -> 5
+    pub fn path_length_of_url(url: &Url) -> u64 {
+        log::trace!("enter: get_path_length({})", url);
+
+        let path = url.path();
+
+        let segments = if let Some(split) = path.strip_prefix('/') {
+            split.split_terminator('/')
+        } else {
+            log::trace!("exit: get_path_length -> 0");
+            return 0;
+        };
+
+        if let Some(last) = segments.last() {
+            // failure on conversion should be very unlikely. While a usize can absolutely overflow a
+            // u64, the generally accepted maximum for the length of a url is ~2000.  so the value we're
+            // putting into the u64 should never realistically be anywhere close to producing an
+            // overflow.
+            // usize max: 18,446,744,073,709,551,615
+            // u64 max:   9,223,372,036,854,775,807
+            let url_len: u64 = last
+                .len()
+                .try_into()
+                .expect("Failed usize -> u64 conversion");
+
+            log::trace!("exit: get_path_length -> {}", url_len);
+            return url_len;
+        }
+
+        log::trace!("exit: get_path_length -> 0");
+        0
+    }
+
+    /// Simple helper to abstract away adding a forward-slash to a url if not present
+    ///
+    /// used mostly for deduplication purposes and url state tracking
+    pub fn normalize(&self) -> String {
+        log::trace!("enter: normalize");
+
+        let normalized = if self.target.ends_with('/') {
+            self.target.to_string()
+        } else {
+            format!("{}/", self.target)
+        };
+
+        log::trace!("exit: normalize -> {}", normalized);
+        normalized
+    }
+
+    /// Helper function that determines the current depth of a given url
+    ///
+    /// Essentially looks at the Url path and determines how many directories are present in the
+    /// given Url
+    ///
+    /// http://localhost -> 1
+    /// http://localhost/ -> 1
+    /// http://localhost/stuff -> 2
+    /// ...
+    ///
+    /// returns 0 on error and relative urls
+    pub fn depth(&self) -> Result<usize> {
+        log::trace!("enter: get_depth");
+
+        let target = self.normalize();
+
+        let parsed = Url::parse(&target)?;
+        let parts = parsed
+            .path_segments()
+            .ok_or_else(|| anyhow!("No path segments found"))?;
+
+        // at least an empty string returned by the Split, meaning top-level urls
+        let mut depth = 0;
+
+        for _ in parts {
+            depth += 1;
+        }
+
+        log::trace!("exit: get_depth -> {}", depth);
+        Ok(depth)
+    }
+}
+
+/// Display implementation for a FeroxUrl
+impl fmt::Display for FeroxUrl {
+    /// formatter for FeroxUrl
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}", &self.target)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::config::Configuration;
+
+    #[test]
+    /// sending url + word without any extensions should get back one url with the joined word
+    fn formatted_urls_no_extension_returns_base_url_with_word() {
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+        let url = FeroxUrl::from_string("http://localhost", handles);
+        let urls = url.formatted_urls("turbo").unwrap();
+        assert_eq!(urls, [Url::parse("http://localhost/turbo").unwrap()])
+    }
+
+    #[test]
+    /// sending url + word + 1 extension should get back two urls, one base and one with extension
+    fn formatted_urls_one_extension_returns_two_urls() {
+        let config = Configuration {
+            extensions: vec![String::from("js")],
+            ..Default::default()
+        };
+
+        let handles = Arc::new(Handles::for_testing(None, Some(Arc::new(config))).0);
+        let url = FeroxUrl::from_string("http://localhost", handles);
+        let urls = url.formatted_urls("turbo").unwrap();
+
+        assert_eq!(
+            urls,
+            [
+                Url::parse("http://localhost/turbo").unwrap(),
+                Url::parse("http://localhost/turbo.js").unwrap()
+            ]
+        )
+    }
+
+    #[test]
+    /// sending url + word + multiple extensions should get back n+1 urls
+    fn formatted_urls_multiple_extensions_returns_n_plus_one_urls() {
+        let ext_vec = vec![
+            vec![String::from("js")],
+            vec![String::from("js"), String::from("php")],
+            vec![String::from("js"), String::from("php"), String::from("pdf")],
+            vec![
+                String::from("js"),
+                String::from("php"),
+                String::from("pdf"),
+                String::from("tar.gz"),
+            ],
+        ];
+        let base = Url::parse("http://localhost/turbo").unwrap();
+        let js = Url::parse("http://localhost/turbo.js").unwrap();
+        let php = Url::parse("http://localhost/turbo.php").unwrap();
+        let pdf = Url::parse("http://localhost/turbo.pdf").unwrap();
+        let tar = Url::parse("http://localhost/turbo.tar.gz").unwrap();
+
+        let expected = vec![
+            vec![base.clone(), js.clone()],
+            vec![base.clone(), js.clone(), php.clone()],
+            vec![base.clone(), js.clone(), php.clone(), pdf.clone()],
+            vec![base, js, php, pdf, tar],
+        ];
+
+        for (i, ext_set) in ext_vec.into_iter().enumerate() {
+            let config = Configuration {
+                extensions: ext_set,
+                ..Default::default()
+            };
+
+            let handles = Arc::new(Handles::for_testing(None, Some(Arc::new(config))).0);
+            let url = FeroxUrl::from_string("http://localhost", handles);
+
+            let urls = url.formatted_urls("turbo").unwrap();
+            assert_eq!(urls, expected[i]);
+        }
+    }
+
+    #[test]
+    /// base url returns 1
+    fn depth_base_url_returns_1() {
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+        let url = FeroxUrl::from_string("http://localhost", handles);
+
+        let depth = url.depth().unwrap();
+        assert_eq!(depth, 1);
+    }
+
+    #[test]
+    /// base url with slash returns 1
+    fn depth_base_url_with_slash_returns_1() {
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+        let url = FeroxUrl::from_string("http://localhost/", handles);
+
+        let depth = url.depth().unwrap();
+        assert_eq!(depth, 1);
+    }
+
+    #[test]
+    /// base url + 1 dir returns 2
+    fn depth_one_dir_returns_2() {
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+        let url = FeroxUrl::from_string("http://localhost/src", handles);
+
+        let depth = url.depth().unwrap();
+        assert_eq!(depth, 2);
+    }
+
+    #[test]
+    /// base url + 1 dir and slash returns 2
+    fn depth_one_dir_with_slash_returns_2() {
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+        let url = FeroxUrl::from_string("http://localhost/src/", handles);
+
+        let depth = url.depth().unwrap();
+        assert_eq!(depth, 2);
+    }
+
+    #[test]
+    /// base url + 1 word + no slash + no extension
+    fn format_url_normal() {
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+        let url = FeroxUrl::from_string("http://localhost", handles);
+        let formatted = url.format("stuff", None).unwrap();
+
+        assert_eq!(
+            formatted,
+            reqwest::Url::parse("http://localhost/stuff").unwrap()
+        );
+    }
+
+    #[test]
+    /// base url + no word + no slash + no extension
+    fn format_url_no_word() {
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+        let url = FeroxUrl::from_string("http://localhost", handles);
+        let formatted = url.format("", None).unwrap();
+        assert_eq!(formatted, reqwest::Url::parse("http://localhost").unwrap());
+    }
+
+    #[test]
+    /// base url + word + no slash + no extension + queries
+    fn format_url_joins_queries() {
+        let config = Configuration {
+            queries: vec![(String::from("stuff"), String::from("things"))],
+            ..Default::default()
+        };
+
+        let handles = Arc::new(Handles::for_testing(None, Some(Arc::new(config))).0);
+        let url = FeroxUrl::from_string("http://localhost", handles);
+        let formatted = url.format("lazer", None).unwrap();
+
+        assert_eq!(
+            formatted,
+            reqwest::Url::parse("http://localhost/lazer?stuff=things").unwrap()
+        );
+    }
+
+    #[test]
+    /// base url + no word + no slash + no extension + queries
+    fn format_url_without_word_joins_queries() {
+        let config = Configuration {
+            queries: vec![(String::from("stuff"), String::from("things"))],
+            ..Default::default()
+        };
+
+        let handles = Arc::new(Handles::for_testing(None, Some(Arc::new(config))).0);
+        let url = FeroxUrl::from_string("http://localhost", handles);
+        let formatted = url.format("", None).unwrap();
+
+        assert_eq!(
+            formatted,
+            reqwest::Url::parse("http://localhost/?stuff=things").unwrap()
+        );
+    }
+
+    #[test]
+    #[should_panic]
+    /// no base url is an error
+    fn format_url_no_url() {
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+        let url = FeroxUrl::from_string("", handles);
+        url.format("stuff", None).unwrap();
+    }
+
+    #[test]
+    /// word prepended with slash is adjusted for correctness
+    fn format_url_word_with_preslash() {
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+        let url = FeroxUrl::from_string("http://localhost", handles);
+        let formatted = url.format("/stuff", None).unwrap();
+
+        assert_eq!(
+            formatted,
+            reqwest::Url::parse("http://localhost/stuff").unwrap()
+        );
+    }
+
+    #[test]
+    /// word with appended slash allows the slash to persist
+    fn format_url_word_with_postslash() {
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+        let url = FeroxUrl::from_string("http://localhost", handles);
+        let formatted = url.format("stuff/", None).unwrap();
+
+        assert_eq!(
+            formatted,
+            reqwest::Url::parse("http://localhost/stuff/").unwrap()
+        );
+    }
+
+    #[test]
+    /// word with two prepended slashes doesn't discard the entire domain
+    fn format_url_word_with_two_prepended_slashes() {
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+        let url = FeroxUrl::from_string("http://localhost", handles);
+        let formatted = url.format("//upload/img", None).unwrap();
+
+        assert_eq!(
+            formatted,
+            reqwest::Url::parse("http://localhost/upload/img").unwrap()
+        );
+    }
+
+    #[test]
+    /// word that is a fully formed url, should return an error
+    fn format_url_word_that_is_a_url() {
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+        let url = FeroxUrl::from_string("http://localhost", handles);
+        let formatted = url.format("http://schmocalhost", None);
+
+        assert!(formatted.is_err());
+    }
+}

src/utils.rs

@@ -1,88 +1,39 @@
-use crate::{
-    config::{CONFIGURATION, PROGRESS_PRINTER},
-    FeroxError, FeroxResult,
-};
+use anyhow::{bail, Context, Result};
 use console::{strip_ansi_codes, style, user_attended};
 use indicatif::ProgressBar;
 use reqwest::{Client, Response, Url};
 #[cfg(not(target_os = "windows"))]
 use rlimit::{getrlimit, setrlimit, Resource, Rlim};
-use std::convert::TryInto;
-use std::sync::{Arc, RwLock};
-use std::{fs, io};
+use std::{
+    fs,
+    io::{self, BufWriter, Write},
+};
+use tokio::sync::mpsc::UnboundedSender;
+
+use crate::{
+    config::OutputLevel,
+    event_handlers::Command::{self, AddError, AddStatus},
+    progress::PROGRESS_PRINTER,
+    send_command,
+    statistics::StatError::{Connection, Other, Redirection, Request, Timeout},
+    traits::FeroxSerialize,
+};
 
 /// Given the path to a file, open the file in append mode (create it if it doesn't exist) and
-/// return a reference to the file that is buffered and locked
-pub fn open_file(filename: &str) -> Option<Arc<RwLock<io::BufWriter<fs::File>>>> {
+/// return a reference to the buffered file
+pub fn open_file(filename: &str) -> Result<BufWriter<fs::File>> {
     log::trace!("enter: open_file({})", filename);
 
-    match fs::OpenOptions::new() // std fs
+    let file = fs::OpenOptions::new() // std fs
         .create(true)
         .append(true)
         .open(filename)
-    {
-        Ok(file) => {
-            let writer = io::BufWriter::new(file); // std io
+        .with_context(|| fmt_err(&format!("Could not open {}", filename)))?;
 
-            let locked_file = Some(Arc::new(RwLock::new(writer)));
+    let writer = BufWriter::new(file); // std io
 
-            log::trace!("exit: open_file -> {:?}", locked_file);
-            locked_file
-        }
-        Err(e) => {
-            log::error!("{}", e);
-            log::trace!("exit: open_file -> None");
-            None
-        }
-    }
-}
-
-/// Helper function that determines the current depth of a given url
-///
-/// Essentially looks at the Url path and determines how many directories are present in the
-/// given Url
-///
-/// http://localhost -> 1
-/// http://localhost/ -> 1
-/// http://localhost/stuff -> 2
-/// ...
-///
-/// returns 0 on error and relative urls
-pub fn get_current_depth(target: &str) -> usize {
-    log::trace!("enter: get_current_depth({})", target);
-
-    let target = normalize_url(target);
-
-    match Url::parse(&target) {
-        Ok(url) => {
-            if let Some(parts) = url.path_segments() {
-                // at least an empty string returned by the Split, meaning top-level urls
-                let mut depth = 0;
-
-                for _ in parts {
-                    depth += 1;
-                }
-
-                let return_val = depth;
-
-                log::trace!("exit: get_current_depth -> {}", return_val);
-                return return_val;
-            };
-
-            log::debug!(
-                "get_current_depth called on a Url that cannot be a base: {}",
-                url
-            );
-            log::trace!("exit: get_current_depth -> 0");
-
-            0
-        }
-        Err(e) => {
-            log::error!("could not parse to url: {}", e);
-            log::trace!("exit: get_current_depth -> 0");
-            0
-        }
-    }
+    log::trace!("exit: open_file -> {:?}", writer);
+    Ok(writer)
 }
 
 /// Takes in a string and examines the first character to return a color version of the same string
@@ -99,6 +50,11 @@ pub fn status_colorizer(status: &str) -> String {
     }
 }
 
+/// simple wrapper to stay DRY
+pub fn fmt_err(msg: &str) -> String {
+    format!("{}: {}", status_colorizer("ERROR"), msg)
+}
+
 /// Takes in a string and colors it using console::style
 ///
 /// mainly putting this here in case i want to change the color later, making any changes easy
@@ -106,41 +62,6 @@ pub fn module_colorizer(modname: &str) -> String {
     style(modname).cyan().to_string()
 }
 
-/// Gets the length of a url's path
-///
-/// example: http://localhost/stuff -> 5
-pub fn get_url_path_length(url: &Url) -> u64 {
-    log::trace!("enter: get_url_path_length({})", url);
-
-    let path = url.path();
-
-    let segments = if let Some(split) = path.strip_prefix('/') {
-        split.split_terminator('/')
-    } else {
-        log::trace!("exit: get_url_path_length -> 0");
-        return 0;
-    };
-
-    if let Some(last) = segments.last() {
-        // failure on conversion should be very unlikely. While a usize can absolutely overflow a
-        // u64, the generally accepted maximum for the length of a url is ~2000.  so the value we're
-        // putting into the u64 should never realistically be anywhere close to producing an
-        // overflow.
-        // usize max: 18,446,744,073,709,551,615
-        // u64 max:   9,223,372,036,854,775,807
-        let url_len: u64 = last
-            .len()
-            .try_into()
-            .expect("Failed usize -> u64 conversion");
-
-        log::trace!("exit: get_url_path_length -> {}", url_len);
-        return url_len;
-    }
-
-    log::trace!("exit: get_url_path_length -> 0");
-    0
-}
-
 /// Simple helper to abstract away the check for an attached terminal.
 ///
 /// If a terminal is attached, progress bars are visible and the progress bar is used to print
@@ -160,139 +81,64 @@ pub fn ferox_print(msg: &str, bar: &ProgressBar) {
     }
 }
 
-/// Simple helper to generate a `Url`
-///
-/// Errors during parsing `url` or joining `word` are propagated up the call stack
-pub fn format_url(
-    url: &str,
-    word: &str,
-    add_slash: bool,
-    queries: &[(String, String)],
-    extension: Option<&str>,
-) -> FeroxResult<Url> {
+/// Initiate request to the given `Url` using `Client`
+pub async fn make_request(
+    client: &Client,
+    url: &Url,
+    output_level: OutputLevel,
+    tx_stats: UnboundedSender<Command>,
+) -> Result<Response> {
     log::trace!(
-        "enter: format_url({}, {}, {}, {:?} {:?})",
+        "enter: make_request(Configuration::Client, {}, {:?}, {:?})",
         url,
-        word,
-        add_slash,
-        queries,
-        extension
+        output_level,
+        tx_stats
     );
 
-    if Url::parse(&word).is_ok() {
-        // when a full url is passed in as a word to be joined to a base url using
-        // reqwest::Url::join, the result is that the word (url) completely overwrites the base
-        // url, potentially resulting in requests to places that aren't actually the target
-        // specified.
-        //
-        // in order to resolve the issue, we check if the word from the wordlist is a parsable URL
-        // and if so, don't do any further processing
-        let message = format!(
-            "word ({}) from the wordlist is actually a URL, skipping...",
-            word
-        );
-        log::warn!("{}", message);
-
-        let mut err = FeroxError::default();
-        err.message = message;
-
-        log::trace!("exit: format_url -> {}", err);
-        return Err(Box::new(err));
-    }
-
-    // from reqwest::Url::join
-    //   Note: a trailing slash is significant. Without it, the last path component
-    //   is considered to be a “file” name to be removed to get at the “directory”
-    //   that is used as the base
-    //
-    // the transforms that occur here will need to keep this in mind, i.e. add a slash to preserve
-    // the current directory sent as part of the url
-    let url = if word.is_empty() {
-        // v1.0.6: added during --extract-links feature inplementation to support creating urls
-        // that were extracted from response bodies, i.e. http://localhost/some/path/js/main.js
-        url.to_string()
-    } else if !url.ends_with('/') {
-        format!("{}/", url)
-    } else {
-        url.to_string()
-    };
-
-    let base_url = reqwest::Url::parse(&url)?;
-
-    // extensions and slashes are mutually exclusive cases
-    let word = if extension.is_some() {
-        format!("{}.{}", word, extension.unwrap())
-    } else if add_slash && !word.ends_with('/') {
-        // -f used, and word doesn't already end with a /
-        format!("{}/", word)
-    } else {
-        String::from(word)
-    };
-
-    match base_url.join(&word) {
-        Ok(request) => {
-            if queries.is_empty() {
-                // no query params to process
-                log::trace!("exit: format_url -> {}", request);
-                Ok(request)
-            } else {
-                match reqwest::Url::parse_with_params(request.as_str(), queries) {
-                    Ok(req_w_params) => {
-                        log::trace!("exit: format_url -> {}", req_w_params);
-                        Ok(req_w_params) // request with params attached
-                    }
-                    Err(e) => {
-                        log::error!(
-                            "Could not add query params {:?} to {}: {}",
-                            queries,
-                            request,
-                            e
-                        );
-                        log::trace!("exit: format_url -> {}", request);
-                        Ok(request) // couldn't process params, return initially ok url
-                    }
-                }
-            }
-        }
-        Err(e) => {
-            log::trace!("exit: format_url -> {}", e);
-            log::error!("Could not join {} with {}", word, base_url);
-            Err(Box::new(e))
-        }
-    }
-}
-
-/// Initiate request to the given `Url` using `Client`
-pub async fn make_request(client: &Client, url: &Url) -> FeroxResult<Response> {
-    log::trace!("enter: make_request(CONFIGURATION.Client, {})", url);
-
     match client.get(url.to_owned()).send().await {
-        Ok(resp) => {
-            log::trace!("exit: make_request -> {:?}", resp);
-            Ok(resp)
-        }
         Err(e) => {
             log::trace!("exit: make_request -> {}", e);
-            if e.to_string().contains("operation timed out") {
-                // only warn for timeouts, while actual errors are still left as errors
-                log::warn!("Error while making request: {}", e);
+
+            if e.is_timeout() {
+                send_command!(tx_stats, AddError(Timeout));
             } else if e.is_redirect() {
                 if let Some(last_redirect) = e.url() {
                     // get where we were headed (last_redirect) and where we came from (url)
                     let fancy_message = format!("{} !=> {}", url, last_redirect);
 
                     let report = if let Some(msg_status) = e.status() {
-                        create_report_string(msg_status.as_str(), "-1", "-1", "-1", &fancy_message)
+                        send_command!(tx_stats, AddStatus(msg_status));
+                        create_report_string(
+                            msg_status.as_str(),
+                            "-1",
+                            "-1",
+                            "-1",
+                            &fancy_message,
+                            output_level,
+                        )
                     } else {
-                        create_report_string("UNK", "-1", "-1", "-1", &fancy_message)
+                        create_report_string("UNK", "-1", "-1", "-1", &fancy_message, output_level)
                     };
 
+                    send_command!(tx_stats, AddError(Redirection));
+
                     ferox_print(&report, &PROGRESS_PRINTER)
                 };
+            } else if e.is_connect() {
+                send_command!(tx_stats, AddError(Connection));
+            } else if e.is_request() {
+                send_command!(tx_stats, AddError(Request));
             } else {
-                log::error!("Error while making request: {}", e);
+                send_command!(tx_stats, AddError(Other));
             }
-            Err(Box::new(e))
+
+            log::warn!("Error while making request: {}", e);
+            bail!("{}", e)
+        }
+        Ok(resp) => {
+            log::trace!("exit: make_request -> {:?}", resp);
+            send_command!(tx_stats, AddStatus(resp.status()));
+            Ok(resp)
         }
     }
 }
@@ -307,9 +153,10 @@ pub fn create_report_string(
     word_count: &str,
     content_length: &str,
     url: &str,
+    output_level: OutputLevel,
 ) -> String {
-    if CONFIGURATION.quiet {
-        // -q used, just need the url
+    if matches!(output_level, OutputLevel::Silent) {
+        // --silent used, just need the url
         format!("{}\n", url)
     } else {
         // normal printing with status and sizes
@@ -372,20 +219,39 @@ pub fn set_open_file_limit(limit: usize) -> bool {
     false
 }
 
-/// Simple helper to abstract away adding a forward-slash to a url if not present
-///
-/// used mostly for deduplication purposes and url state tracking
-pub fn normalize_url(url: &str) -> String {
-    log::trace!("enter: normalize_url({})", url);
+/// Given a string and a reference to a locked buffered file, write the contents and flush
+/// the buffer to disk.
+pub fn write_to<T>(
+    value: &T,
+    file: &mut io::BufWriter<fs::File>,
+    convert_to_json: bool,
+) -> Result<()>
+where
+    T: FeroxSerialize,
+{
+    // note to future self: adding logging of anything other than error to this function
+    // is a bad idea. we call this function while processing records generated by the logger.
+    // If we then call log::... while already processing some logging output, it results in
+    // the second log entry being injected into the first.
 
-    let normalized = if url.ends_with('/') {
-        url.to_string()
+    let contents = if convert_to_json {
+        value.as_json()?
     } else {
-        format!("{}/", url)
+        value.as_str()
     };
 
-    log::trace!("exit: normalize_url -> {}", normalized);
-    normalized
+    let contents = strip_ansi_codes(&contents);
+
+    let written = file.write(contents.as_bytes())?;
+
+    if written > 0 {
+        // this function is used within async functions/loops, so i'm flushing so that in
+        // the event of a ctrl+c or w/e results seen so far are saved instead of left lying
+        // around in the buffer
+        file.flush()?;
+    }
+
+    Ok(())
 }
 
 #[cfg(test)]
@@ -420,129 +286,6 @@ mod tests {
         assert!(!set_open_file_limit(hard.as_usize())); // returns false
     }
 
-    #[test]
-    /// base url returns 1
-    fn get_current_depth_base_url_returns_1() {
-        let depth = get_current_depth("http://localhost");
-        assert_eq!(depth, 1);
-    }
-
-    #[test]
-    /// base url with slash returns 1
-    fn get_current_depth_base_url_with_slash_returns_1() {
-        let depth = get_current_depth("http://localhost/");
-        assert_eq!(depth, 1);
-    }
-
-    #[test]
-    /// base url + 1 dir returns 2
-    fn get_current_depth_one_dir_returns_2() {
-        let depth = get_current_depth("http://localhost/src");
-        assert_eq!(depth, 2);
-    }
-
-    #[test]
-    /// base url + 1 dir and slash returns 2
-    fn get_current_depth_one_dir_with_slash_returns_2() {
-        let depth = get_current_depth("http://localhost/src/");
-        assert_eq!(depth, 2);
-    }
-
-    #[test]
-    /// base url + 1 dir and slash returns 2
-    fn get_current_depth_single_forward_slash_is_zero() {
-        let depth = get_current_depth("");
-        assert_eq!(depth, 0);
-    }
-
-    #[test]
-    /// base url + 1 word + no slash + no extension
-    fn format_url_normal() {
-        assert_eq!(
-            format_url("http://localhost", "stuff", false, &Vec::new(), None).unwrap(),
-            reqwest::Url::parse("http://localhost/stuff").unwrap()
-        );
-    }
-
-    #[test]
-    /// base url + no word + no slash + no extension
-    fn format_url_no_word() {
-        assert_eq!(
-            format_url("http://localhost", "", false, &Vec::new(), None).unwrap(),
-            reqwest::Url::parse("http://localhost").unwrap()
-        );
-    }
-
-    #[test]
-    /// base url + word + no slash + no extension + queries
-    fn format_url_joins_queries() {
-        assert_eq!(
-            format_url(
-                "http://localhost",
-                "lazer",
-                false,
-                &[(String::from("stuff"), String::from("things"))],
-                None
-            )
-            .unwrap(),
-            reqwest::Url::parse("http://localhost/lazer?stuff=things").unwrap()
-        );
-    }
-
-    #[test]
-    /// base url + no word + no slash + no extension + queries
-    fn format_url_without_word_joins_queries() {
-        assert_eq!(
-            format_url(
-                "http://localhost",
-                "",
-                false,
-                &[(String::from("stuff"), String::from("things"))],
-                None
-            )
-            .unwrap(),
-            reqwest::Url::parse("http://localhost/?stuff=things").unwrap()
-        );
-    }
-
-    #[test]
-    #[should_panic]
-    /// no base url is an error
-    fn format_url_no_url() {
-        format_url("", "stuff", false, &Vec::new(), None).unwrap();
-    }
-
-    #[test]
-    /// word prepended with slash is adjusted for correctness
-    fn format_url_word_with_preslash() {
-        assert_eq!(
-            format_url("http://localhost", "/stuff", false, &Vec::new(), None).unwrap(),
-            reqwest::Url::parse("http://localhost/stuff").unwrap()
-        );
-    }
-
-    #[test]
-    /// word with appended slash allows the slash to persist
-    fn format_url_word_with_postslash() {
-        assert_eq!(
-            format_url("http://localhost", "stuff/", false, &Vec::new(), None).unwrap(),
-            reqwest::Url::parse("http://localhost/stuff/").unwrap()
-        );
-    }
-
-    #[test]
-    /// word that is a fully formed url, should return an error
-    fn format_url_word_that_is_a_url() {
-        let url = format_url(
-            "http://localhost",
-            "http://schmocalhost",
-            false,
-            &Vec::new(),
-            None,
-        );
-        assert!(url.is_err());
-    }
-
     #[test]
     /// status colorizer uses red for 500s
     fn status_colorizer_uses_red_for_500s() {

tests/test_banner.rs

@@ -270,6 +270,9 @@ fn banner_prints_output_file() -> Result<(), Box<dyn std::error::Error>> {
                 .and(predicate::str::contains("User-Agent"))
                 .and(predicate::str::contains("Output File"))
                 .and(predicate::str::contains("/super/cool/path"))
+                .and(predicate::str::contains(
+                    "ERROR: Couldn't start /super/cool/path file handler",
+                ))
                 .and(predicate::str::contains("─┴─")),
         );
     Ok(())
@@ -615,7 +618,9 @@ fn banner_doesnt_print() -> Result<(), Box<dyn std::error::Error>> {
         .arg("-q")
         .assert()
         .success()
-        .stderr(predicate::str::is_empty());
+        .stderr(predicate::str::contains(
+            "Could not connect to any target provided",
+        ));
     Ok(())
 }
 
@@ -810,3 +815,105 @@ fn banner_prints_time_limit() {
                 .and(predicate::str::contains("─┴─")),
         );
 }
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + similarity filter
+fn banner_prints_similarity_filter() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--filter-similar-to")
+        .arg("https://somesite.com")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Similarity Filter"))
+                .and(predicate::str::contains("│ https://somesite.com"))
+                .and(predicate::str::contains("─┴─")),
+        );
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + rate limit
+fn banner_prints_rate_limit() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--rate-limit")
+        .arg("6735")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Requests per Second"))
+                .and(predicate::str::contains("│ 6735"))
+                .and(predicate::str::contains("─┴─")),
+        );
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see no banner output
+fn banner_doesnt_print_when_silent() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--silent")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .not()
+                .and(predicate::str::contains("Target Url").not())
+                .and(predicate::str::contains("http://localhost").not())
+                .and(predicate::str::contains("Threads").not())
+                .and(predicate::str::contains("Wordlist").not())
+                .and(predicate::str::contains("Status Codes").not())
+                .and(predicate::str::contains("Timeout (secs)").not())
+                .and(predicate::str::contains("User-Agent").not()),
+        );
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see no banner output
+fn banner_doesnt_print_when_quiet() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--quiet")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .not()
+                .and(predicate::str::contains("Target Url").not())
+                .and(predicate::str::contains("http://localhost").not())
+                .and(predicate::str::contains("Threads").not())
+                .and(predicate::str::contains("Wordlist").not())
+                .and(predicate::str::contains("Status Codes").not())
+                .and(predicate::str::contains("Timeout (secs)").not())
+                .and(predicate::str::contains("User-Agent").not()),
+        );
+}

tests/test_extractor.rs

@@ -1,7 +1,7 @@
 mod utils;
 use assert_cmd::prelude::*;
 use httpmock::Method::GET;
-use httpmock::{Mock, MockServer};
+use httpmock::MockServer;
 use predicates::prelude::*;
 use std::process::Command;
 use utils::{setup_tmp_directory, teardown_tmp_directory};
@@ -13,18 +13,17 @@ fn extractor_finds_absolute_url() -> Result<(), Box<dyn std::error::Error>> {
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/LICENSE")
-        .return_status(200)
-        .return_body(&srv.url("'/homepage/assets/img/icons/handshake.svg'"))
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200)
+            .body(&srv.url("'/homepage/assets/img/icons/handshake.svg'"));
+    });
 
-    let mock_two = Mock::new()
-        .expect_method(GET)
-        .expect_path("/homepage/assets/img/icons/handshake.svg")
-        .return_status(200)
-        .create_on(&srv);
+    let mock_two = srv.mock(|when, then| {
+        when.method(GET)
+            .path("/homepage/assets/img/icons/handshake.svg");
+        then.status(200);
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -43,8 +42,8 @@ fn extractor_finds_absolute_url() -> Result<(), Box<dyn std::error::Error>> {
             )),
     );
 
-    assert_eq!(mock.times_called(), 1);
-    assert_eq!(mock_two.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_two.hits(), 1);
     teardown_tmp_directory(tmp_dir);
     Ok(())
 }
@@ -56,12 +55,11 @@ fn extractor_finds_absolute_url_to_different_domain() -> Result<(), Box<dyn std:
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/LICENSE")
-        .return_status(200)
-        .return_body("\"http://localhost/homepage/assets/img/icons/handshake.svg\"")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200)
+            .body("\"http://localhost/homepage/assets/img/icons/handshake.svg\"");
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -81,7 +79,7 @@ fn extractor_finds_absolute_url_to_different_domain() -> Result<(), Box<dyn std:
             .not(),
     );
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
     teardown_tmp_directory(tmp_dir);
     Ok(())
 }
@@ -92,18 +90,17 @@ fn extractor_finds_relative_url() -> Result<(), Box<dyn std::error::Error>> {
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/LICENSE")
-        .return_status(200)
-        .return_body("\"/homepage/assets/img/icons/handshake.svg\"")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200)
+            .body("\"/homepage/assets/img/icons/handshake.svg\"");
+    });
 
-    let mock_two = Mock::new()
-        .expect_method(GET)
-        .expect_path("/homepage/assets/img/icons/handshake.svg")
-        .return_status(200)
-        .create_on(&srv);
+    let mock_two = srv.mock(|when, then| {
+        when.method(GET)
+            .path("/homepage/assets/img/icons/handshake.svg");
+        then.status(200);
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -122,8 +119,8 @@ fn extractor_finds_relative_url() -> Result<(), Box<dyn std::error::Error>> {
             )),
     );
 
-    assert_eq!(mock.times_called(), 1);
-    assert_eq!(mock_two.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_two.hits(), 1);
     teardown_tmp_directory(tmp_dir);
     Ok(())
 }
@@ -136,25 +133,23 @@ fn extractor_finds_same_relative_url_twice() {
     let (tmp_dir, file) =
         setup_tmp_directory(&["LICENSE".to_string(), "README".to_string()], "wordlist").unwrap();
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/LICENSE")
-        .return_status(200)
-        .return_body(&srv.url("\"/homepage/assets/img/icons/handshake.svg\""))
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200)
+            .body(&srv.url("\"/homepage/assets/img/icons/handshake.svg\""));
+    });
 
-    let mock_two = Mock::new()
-        .expect_method(GET)
-        .expect_path("/README")
-        .return_body(&srv.url("\"/homepage/assets/img/icons/handshake.svg\""))
-        .return_status(200)
-        .create_on(&srv);
+    let mock_two = srv.mock(|when, then| {
+        when.method(GET).path("/README");
+        then.status(200)
+            .body(&srv.url("\"/homepage/assets/img/icons/handshake.svg\""));
+    });
 
-    let mock_three = Mock::new()
-        .expect_method(GET)
-        .expect_path("/homepage/assets/img/icons/handshake.svg")
-        .return_status(200)
-        .create_on(&srv);
+    let mock_three = srv.mock(|when, then| {
+        when.method(GET)
+            .path("/homepage/assets/img/icons/handshake.svg");
+        then.status(200);
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -168,15 +163,14 @@ fn extractor_finds_same_relative_url_twice() {
     cmd.assert().success().stdout(
         predicate::str::contains("/LICENSE")
             .and(predicate::str::contains("200"))
-            .and(predicate::str::contains(
-                "/homepage/assets/img/icons/handshake.svg",
-            )),
+            // .count(1) asserts that we only see the endpoint reported once, even though there
+            // is the potential to request the same url twice
+            .and(predicate::str::contains("/homepage/assets/img/icons/handshake.svg").count(1)),
     );
 
-    assert_eq!(mock.times_called(), 1);
-    assert_eq!(mock_two.times_called(), 1);
-    assert!(mock_three.times_called() <= 2); // todo: sometimes this is 2 instead of 1
-                                             // the expectation is one, suggesting a race condition... investigate and fix
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_two.hits(), 1);
+    assert!(mock_three.hits() <= 2);
     teardown_tmp_directory(tmp_dir);
 }
 
@@ -188,19 +182,17 @@ fn extractor_finds_filtered_content() -> Result<(), Box<dyn std::error::Error>>
     let (tmp_dir, file) =
         setup_tmp_directory(&["LICENSE".to_string(), "README".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/LICENSE")
-        .return_status(200)
-        .return_body(&srv.url("\"/homepage/assets/img/icons/handshake.svg\""))
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200)
+            .body(&srv.url("\"/homepage/assets/img/icons/handshake.svg\""));
+    });
 
-    let mock_two = Mock::new()
-        .expect_method(GET)
-        .expect_path("/homepage/assets/img/icons/handshake.svg")
-        .return_body("im a little teapot")
-        .return_status(200)
-        .create_on(&srv);
+    let mock_two = srv.mock(|when, then| {
+        when.method(GET)
+            .path("/homepage/assets/img/icons/handshake.svg");
+        then.status(200).body("im a little teapot");
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -222,8 +214,209 @@ fn extractor_finds_filtered_content() -> Result<(), Box<dyn std::error::Error>>
             .not(),
     );
 
-    assert_eq!(mock.times_called(), 1);
-    assert_eq!(mock_two.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_two.hits(), 1);
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// serve a robots.txt with a file and and a folder link contained within it. ferox should
+/// find both links and request each one. Additionally, a scan should start with the directory
+/// link found, meaning the wordlist will be thrown at the sub directory
+fn extractor_finds_robots_txt_links_and_displays_files_or_scans_directories() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("im a little teapot"); // 18
+    });
+
+    let mock_two = srv.mock(|when, then| {
+        when.method(GET).path("/robots.txt");
+        then.status(200).body(
+            r#"
+            User-agent: *
+            Crawl-delay: 10
+            # CSS, JS, Images
+            Allow: /misc/*.css$
+            Disallow: /misc/stupidfile.php
+               Disallow: /disallowed-subdir/
+            "#,
+        );
+    });
+
+    let mock_file = srv.mock(|when, then| {
+        when.method(GET).path("/misc/stupidfile.php");
+        then.status(200).body("im a little teapot too"); // 22
+    });
+
+    let mock_scanned_file = srv.mock(|when, then| {
+        when.method(GET).path("/misc/LICENSE");
+        then.status(200).body("i too, am a container for tea"); // 29
+    });
+
+    let mock_dir = srv.mock(|when, _| {
+        when.method(GET).path("/misc/");
+    });
+
+    let mock_disallowed = srv.mock(|when, then| {
+        when.method(GET).path("/disallowed-subdir");
+        then.status(403);
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--extract-links")
+        .arg("-vvvv")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE") // 2 directories contain LICENSE
+            .count(2)
+            .and(predicate::str::contains("18c"))
+            .and(predicate::str::contains("/misc/stupidfile.php"))
+            .and(predicate::str::contains("22c"))
+            .and(predicate::str::contains("/misc/LICENSE"))
+            .and(predicate::str::contains("29c"))
+            .and(predicate::str::contains("200").count(3)),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_dir.hits(), 1);
+    assert_eq!(mock_two.hits(), 1);
+    assert_eq!(mock_file.hits(), 1);
+    assert_eq!(mock_disallowed.hits(), 1);
+    assert_eq!(mock_scanned_file.hits(), 1);
+    teardown_tmp_directory(tmp_dir);
+}
+
+#[test]
+/// serve a robots.txt with a file and and a folder link contained within it. ferox should
+/// find both links and request each one. This is the non-recursive version of the test above
+fn extractor_finds_robots_txt_links_and_displays_files_non_recursive() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("im a little teapot"); // 18
+    });
+
+    let mock_two = srv.mock(|when, then| {
+        when.method(GET).path("/robots.txt");
+        then.status(200).body(
+            r#"
+            User-agent: *
+            Crawl-delay: 10
+            # CSS, JS, Images
+            Allow: /misc/*.css$
+            Disallow: /misc/stupidfile.php
+               Disallow: /disallowed-subdir/
+            "#,
+        );
+    });
+
+    let mock_file = srv.mock(|when, then| {
+        when.method(GET).path("/misc/stupidfile.php");
+        then.status(200).body("im a little teapot too"); // 22
+    });
+
+    let mock_scanned_file = srv.mock(|when, then| {
+        when.method(GET).path("/misc/LICENSE");
+        then.status(200).body("i too, am a container for tea"); // 29
+    });
+
+    let mock_dir = srv.mock(|when, _| {
+        when.method(GET).path("/misc/");
+    });
+
+    let mock_disallowed = srv.mock(|when, then| {
+        when.method(GET).path("/disallowed-subdir");
+        then.status(404);
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--extract-links")
+        .arg("--no-recursion")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .and(predicate::str::contains("18c"))
+            .and(predicate::str::contains("/misc/stupidfile.php"))
+            .and(predicate::str::contains("22c"))
+            .and(predicate::str::contains("/misc/LICENSE").not())
+            .and(predicate::str::contains("29c").not())
+            .and(predicate::str::contains("200").count(2)),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_dir.hits(), 1);
+    assert_eq!(mock_two.hits(), 1);
+    assert_eq!(mock_file.hits(), 1);
+    assert_eq!(mock_disallowed.hits(), 1);
+    assert_eq!(mock_scanned_file.hits(), 0);
+    teardown_tmp_directory(tmp_dir);
+}
+
+#[test]
+/// send a request to a page that contains a link that contains a directory that returns a 403
+/// --extract-links should find the link and make recurse into the 403 directory, finding LICENSE
+fn extractor_recurses_into_403_directories() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200)
+            .body(&srv.url("'/homepage/assets/img/icons/handshake.svg'"));
+    });
+
+    let mock_two = srv.mock(|when, then| {
+        when.method(GET).path("/homepage/assets/img/icons/LICENSE");
+        then.status(200).body("that's just like, your opinion man");
+    });
+
+    let forbidden_dir = srv.mock(|when, then| {
+        when.method(GET).path("/homepage/assets/img/icons/");
+        then.status(403);
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--extract-links")
+        .arg("--depth") // need to go past default 4 directories
+        .arg("0")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .count(2)
+            .and(predicate::str::contains("1w")) // link in /LICENSE
+            .and(predicate::str::contains("34c")) // recursed LICENSE
+            .and(predicate::str::contains(
+                "/homepage/assets/img/icons/LICENSE",
+            )),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_two.hits(), 1);
+    assert_eq!(forbidden_dir.hits(), 1);
     teardown_tmp_directory(tmp_dir);
     Ok(())
 }

tests/test_filters.rs

@@ -1,7 +1,7 @@
 mod utils;
 use assert_cmd::prelude::*;
 use httpmock::Method::GET;
-use httpmock::{Mock, MockServer};
+use httpmock::MockServer;
 use predicates::prelude::*;
 use std::process::Command;
 use utils::{setup_tmp_directory, teardown_tmp_directory};
@@ -14,19 +14,15 @@ fn filters_status_code_should_filter_response() {
     let (tmp_dir, file) =
         setup_tmp_directory(&["LICENSE".to_string(), "file.js".to_string()], "wordlist").unwrap();
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/LICENSE")
-        .return_status(302)
-        .return_body("this is a test")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(302).body("this is a test");
+    });
 
-    let mock_two = Mock::new()
-        .expect_method(GET)
-        .expect_path("/file.js")
-        .return_status(200)
-        .return_body("this is also a test of some import")
-        .create_on(&srv);
+    let mock_two = srv.mock(|when, then| {
+        when.method(GET).path("/file.js");
+        then.status(200).body("this is also a test of some import");
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -51,8 +47,8 @@ fn filters_status_code_should_filter_response() {
             .and(predicate::str::contains("34c")),
     );
 
-    assert_eq!(mock.times_called(), 1);
-    assert_eq!(mock_two.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_two.hits(), 1);
     teardown_tmp_directory(tmp_dir);
 }
 
@@ -64,19 +60,16 @@ fn filters_lines_should_filter_response() {
     let (tmp_dir, file) =
         setup_tmp_directory(&["LICENSE".to_string(), "file.js".to_string()], "wordlist").unwrap();
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/LICENSE")
-        .return_status(302)
-        .return_body("this is a test")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(302).body("this is a test");
+    });
 
-    let mock_two = Mock::new()
-        .expect_method(GET)
-        .expect_path("/file.js")
-        .return_status(200)
-        .return_body("this is also a test of some import\nwith 2 lines, no less")
-        .create_on(&srv);
+    let mock_two = srv.mock(|when, then| {
+        when.method(GET).path("/file.js");
+        then.status(200)
+            .body("this is also a test of some import\nwith 2 lines, no less");
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -100,8 +93,8 @@ fn filters_lines_should_filter_response() {
             .not(),
     );
 
-    assert_eq!(mock.times_called(), 1);
-    assert_eq!(mock_two.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_two.hits(), 1);
     teardown_tmp_directory(tmp_dir);
 }
 
@@ -113,19 +106,16 @@ fn filters_words_should_filter_response() {
     let (tmp_dir, file) =
         setup_tmp_directory(&["LICENSE".to_string(), "file.js".to_string()], "wordlist").unwrap();
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/LICENSE")
-        .return_status(302)
-        .return_body("this is a test")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(302).body("this is a test");
+    });
 
-    let mock_two = Mock::new()
-        .expect_method(GET)
-        .expect_path("/file.js")
-        .return_status(200)
-        .return_body("this is also a test of some import\nwith 2 lines, no less")
-        .create_on(&srv);
+    let mock_two = srv.mock(|when, then| {
+        when.method(GET).path("/file.js");
+        then.status(200)
+            .body("this is also a test of some import\nwith 2 lines, no less");
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -149,8 +139,8 @@ fn filters_words_should_filter_response() {
             .not(),
     );
 
-    assert_eq!(mock.times_called(), 1);
-    assert_eq!(mock_two.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_two.hits(), 1);
     teardown_tmp_directory(tmp_dir);
 }
 
@@ -162,19 +152,16 @@ fn filters_size_should_filter_response() {
     let (tmp_dir, file) =
         setup_tmp_directory(&["LICENSE".to_string(), "file.js".to_string()], "wordlist").unwrap();
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/LICENSE")
-        .return_status(302)
-        .return_body("this is a test")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(302).body("this is a test");
+    });
 
-    let mock_two = Mock::new()
-        .expect_method(GET)
-        .expect_path("/file.js")
-        .return_status(200)
-        .return_body("this is also a test of some import\nwith 2 lines, no less")
-        .create_on(&srv);
+    let mock_two = srv.mock(|when, then| {
+        when.method(GET).path("/file.js");
+        then.status(200)
+            .body("this is also a test of some import\nwith 2 lines, no less");
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -198,7 +185,65 @@ fn filters_size_should_filter_response() {
             .not(),
     );
 
-    assert_eq!(mock.times_called(), 1);
-    assert_eq!(mock_two.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_two.hits(), 1);
+    teardown_tmp_directory(tmp_dir);
+}
+
+#[test]
+/// create a FeroxResponse that should elicit a true from
+/// SimilarityFilter::should_filter_response
+fn filters_similar_should_filter_response() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(
+        &["not-similar".to_string(), "similar".to_string()],
+        "wordlist",
+    )
+    .unwrap();
+
+    // ''.join(random.choices(string.ascii_letters + string.digits + string.whitespace, k=4096))
+    let content = "VCiYFr0HKsEIK6r\r1hJLYnOr90Aji\rDWAjQA3LVAzrluN48FuSPrRpm\n \tV\x0cx\nSCc5sX\nTB\x0c6Of7ns\t2HDwQCduKTqG8gG\x0beszazwljW01H60HMOLziOKwQwEYV7CbrLWQiLeCWKVxX\rvag\nAAEOhjER7gURuGXw\nMyY\t8mSw\x0b\x0bK0Z9G0Pt\x0bJZItAIqAq FxeaoOeLqWVFvxtDFfko0YVYt1I\rNmSXZ4lnOoiBCLbu6TLb80lClhY\tPN7Lp36F786I\nglwRK2oD45EtN SWW IF6uqKdf\x0czAcVycf\x0cBzHYnn1HAkU2Jluos0qwMGJ2m74z\nLd3\x0cIUVZmnRmHHWQGd1u2xmsZR\x0bfnml10ur6J\x0ba8xOZatiY 15Aq3KOGWdD3xQwqo\r5SKnnxH5tqU\rO\rZpJ\n7t7UUgfE\niWFgqWDpMeOG 1248M I\ro5B9Yed\r2aq2\tXxLn31s3hCV WEfQd60DKp6eFhUeUSeXDq6qjgTnWigoCZQERf\rXp7s2L37 iOEMl3\r41\nBShOjLfD8Kj0\rbu0ENreRjP\nY77jsrsaYgOsUrEzw\x0bw3OLi\n8fkddcaOvJeutTy B\rsDMkK\x0cnx2S0N\x0cDaY\x0c9iyo6p4IL\tOC1qgNlWP4VLg\tWmPG46ZMCirth5h4FwkS\nD2WsiEA2Z\n0xbLd7Uww hUQC6 3V\r1SsWem4UcQxG\rfuVvWl\nD9\nDpZQFFgiqhQiq1I0LMAR\r\rKBmj4iurrxaoMHTl9oj\x0b0N3AfD17gyqZiJ67bgizvecsRGeB1f\x0c\nYRvieJqIVHDKOOR\ruhqnVZz4BQ5FFBusz\x0cZl5\x0bt\tbdOUhAAAKyA6Jwl 7OjzojiRHGD6dl ncsgndsKURhFv4\tV5d\n73iPzbT\t8v6IrJtnq\nJuFl7A\x0b\rVnnsjTW0Y4QB1BgCy3B\x0cma7\tpPt5jmcJH7v5J\tYKEXh UqRChBFY5nbFbmXjJYxevPYJmSHC\rDQ4j9de\rTMZ\rtWaPAzkJjH\x0c\nyrEuf9WaMM\trFlKo9r9w\r\nQkQqIEu8Gfr\t aRzvN\r2oZhCyB4fa\np37\tXQi4Wa\no7gHUDQLoRvkK1dy2K3ydrI0O6\rFTGS7oHA\x0bajFOd\rcS5W25tFGhocwxM0\nuugNGDLjBQ\tWGdJV0\x0c\r7bNLs\x0cr deAWt35A4co\x0bPCuYmQ ExxtK\rvpckCyJxLrK5xULK\x0cvqtiGyovHQW8aDjV3rhXhR\nmQvmK\x0czLx\x0cECSYSF5jP35zN VkaRzQ lZ4 l06X4HHpsVn 8y8fGbIP\tRWFUAeFI24\rqN\x0cBW7u7WPMv36BmkgzQ\x0c2\x0cyLf\tYo8iRjE7zMsceym4ZnWg7EsOedh2cES\rz2n\x0cJi52uIPfSkAPzW\rEekjgWdb8y 285F4xae8\n8AiIkT4l3AOy\rT4yeXgaRMCI4t3PkHeFZ\rEb6R4FNCE \nbVil\x0c6qxSVPnU\nh\ttFMNE4\x0c\rwF\t\x0cW5vebbRWG\x0biVZLP\x0ct\x0c5gQ4CJ9KJl\x0cwyIfSIYaCvi4m1r\tJbYqmI0NVO36A\t8BSPNlaKbR73l9mxZxoqD4yca\n5h\r7a0z\tVm34aTy\tnLj5nSrh8er5lN0J7hcjmUk2DL\nyWEVNXTF8RWfC\x0bpcgBQXOQzidyYO\x0bh76UyUPAjELmNoECgGq06hiFGDI LiPZcofhcm\r62fEixIoyG\tmI\x0cYLQvBCbCluGgbm\x0c7GI6\n19il8PdPqss2uQqA5KgkHMIb hh211YuqV9kdmVnwyD63pz3p t58q6kHX\r\teYBrg6eDh\x0bx8\x0cI1SOV3Gt5qubmixHR\rApbgkTQJQ\tX0t11IP55hys2d\x0bF dh7j7G0Ac\x0bQMNvkSU9AV\x0b8mcIPHy9d\x0cyINf5qu\x0cdiBFrhiNRmCZ4r\tSx4N5VOm6KCp2T8bOVEjOR6otPAN5e\n\x0csyJ3giBjkgg 9dYQKq5P75AG5\x0bfD6zZO4DxQ44uX7Kz50dv4ncXQA\rqgHT\rLRcsRl\rW\t7We\tpAEJHMChxwVK\x0cprVvINvolf7hj\tUrob\rW3pXlqKIEQT8t7\x0bGODJanb328OiQCxE\rPfW4j\rl3p\x0cRXDB55u0MN7isBL\ty3UvE1 7I\nfuoZVPzk7az1\rMzA2FROXu0k\rFq pby6pHMqfTQT7iTw izlk0CUpyoUaq5w3UPFK7\rMOPw2cZ6FsVITbCoPhT\rIvuImCFGqmYpE hNevWkPCtwwnx2sX\x0c7oKzBExp32ZpdY\tstuDjSzfalsO1M\x0bNMUegnBDr3Liv3Lv\x0b\n37VZT2LEJ9fNYDi9r\t\x0bYC\rHSt0oJbk\x0b\x0bUdS8eB\nMXBPDEppZjHR7vGZYqX7yFm t1i682AXWf VPTzYTvm6mhOre8\x0bk0spJNYuI\tk\tC1B1N0 AYYDWH\t\tX1TjinXdkXcbFTlIiBLzx\rmUoyx9b7paJSVMX\tfLo8hU1Dmuluyk8R8\x0c4\x0cBe\nCrIMlyek4i\x0bFwuE9\nXUqpVxikH0PZspopUwPM9Kcue\rBh2Mf\rme3h4qelC\x0bEH\x0bkkxi6U\x0cE\x0ctqBgN93 V4ovmocLrK6\ngCQlf\x0cshRVvrPq\x0cOjgbjhSEK8PIx8OYqjjDDkJ0AgLhfbdGw2\nLMv2M0E08PGXnqUyVsjN\t C 4\n80 Fia g\x0b5dEFvyl5Y80U6sMAdHgk2nzC5ElDBhgcBprXC\x0bIMKXyt\x0ce5SkYcRartfblLqD1 A5\nre\x0bj67lJYCs\t8b50xA69eMHqGDLLP8sJceN19kkonjLj\t\rS\tk9sMOeewQHbT \x0cp53aMX9\x0bDYCZWAtdA6h\rAFHDEYFBE1MzdOxMO\x0cvDE7QfLb3jq4s\tI3aVTmDDOQAnuvWb2AGUUP\rf2HinUAiF13LKEfpqcD06S8aQC0Kyl729L7a6CbuoB0GRlJx tD yuTVqD62HuXpfKrDsbejEdp3\rxjc\x0bn4lLNaViizec\rWR\x0cTT5aZ\ny9\rO1qB1XGQPnES\nUhJtU Ll7t3Zglj1IAEx 8Rh3V\x0bfmUSC4\x0bVR9l33LS3bPAJpLbH3Q2\nv2fqMeIt3nGR\x0cgCixM4qzVSx7Yb192a1HWx8nnuWQIEK7QHL6p\x0cD3d0Y1FoZqsmY2U\rspvt3gwKOHR6RaZlmhX\n3bmIEF6\x0b\x0bMXJKOnXPgjkdhun4aGDBw\x0cOEW\repDYTcc48oZ4lg7PukNq7TU\tWP0ZJbzVKK\rxAMaZujwTqQXsXODiE2DdwnstAa6CMYfzj7J\x0c2Q\tY2764IYCy 3Fqm0\x0ckbe7VvfqWUh0\tUlubxZ\rX59MfNSfCfcH8GFZIGIRPt\rZVXfra1 H7VI2yJ\x0cspGDCi\rcgHfZa8528CP9tilUx0ifWPGqskLVDPLJP\nciNxodMQSrJXp\ro\r9aBFHCV\x0cR\rrp\x0bmMfxg5rG\tSuWonbJQlmHQ\ri34w8S\x0cN9Ezj2k2OmLH\x0cEcVUDjXNZIFCtlA843I44p GZyhlOctwpd7 OZnUxk4uacN\r8NihNGO\n9eXy5l6gQe5srySxxvuX5jtCzuJ35xvCfEXYa\x0b2lTDBOAaSYpnl v9L\x0cY8RLg2oE7xeCUbD\tSHKZgeXHZIzYAmA7bsmiZUfzmo5ZZUhtBh4F\x0bTx1\x0bz zQov5mYwfpWJTR2Q\x0bLRXMuBzj\x0bZC\x0b pFNPj8ixWJQggQlr9eNW6SHLJk731nc\x0cBn\x0ckQxg2BdRT\x0bp6lf7G\x0bnIMDeY8w6fUf\x0cjGE1Pfsekv7EYEIHsOAsZb3lBfBPO9\tXpHPBMRmRtzMc5WoX6C5cc\x0cBuTPtPOgXnap1Y3xq7pcMcgu55xblsXEAJKsojjR7aDB\tU84kUKRNEj\n8mcqEyOmvq1WA\na6bhzYf9VQv2aj9KLfByVqUKNFVIc4Mkha\x0c0aCPQSKe0GGwPlSfbtNXhdhxAb3RLf1J\x0cshJzjQe4DCmlRmjt\tlB0BwzBpkg2hTYM\r S\x0cux\x0bj6IcEZ\n\ngQ\rKKgg \rrv4sUMy5sfY1aatjK1MmUyXR\rRHk\x0cqq\x0cD1fy4C0\n\x0byd4SFKOyKJqx2mzI74vPxLLo\x0c0OamjXuUu\nWGkiA70nuf0PGRfwLEBPCMeyneJI1HcIXH\nCTFEIMiAq6fT\rmJgC hXEU\rriAhCm3OzgbcDgvQgDSyUw5jl\x0cTaLOPuFseq\x0cj2npTd57itktTdWBY7sqlOGKNSc\x0ctx2mUoHi31EF3l5lvYPDeG6bIPFwIn7\tG6G \x0bgNkSn89flvqcvI73RA";
+    let mutated = "VCiYFr0HKsEIK6r\r1hJLYnOr90Aji\rDWAjQA3LVAzrluN484327FuSPrRpm\n \tV\x0cx\nSCc5sX\nTB\x0c6Of7ns\t2HDwQCduKTqG8gG\x0beszazwljW01H60HMOLziOKwQwEYV7CbrLWQiLeCWKVxX\rvag\nAAEOhjER7gURuGXw\nMyY\t8mSw\x0b\x0bK0Z9G0Pt\x0bJZItAIqAq FxeaoOeLqWVFvxtDFfko0YVYt1I\rNmSXZ4lnOoiBCLbu6TLb80lClhY\tPN7Lp36F786I\nglwRK2oD45EtN SWW IF6uqKdf\x0czAcVycf\x0cBzHYnn1HAkU2Jluos0qwMGJ2m74z\nLd3\x0cIUVZmnRmHHWQGd1u2xmsZR\x0bfnml10ur6J\x0ba8xOZatiY 15Aq3KOGWdD3xQwqo\r5SKnnxH5tqU\rO\rZpJ\n7t7UUgfE\niWFgqWDpMeOG 1248M I\ro5B9Yed\r2aq2\tXxLn31s3hCV WEfQd60DKp6eFhUeUSeXDq6qjgTnWigoCZQERf\rXp7s2L37 iOEMl3\r41\nBShOjLfD8Kj0\rbu0ENreRjP\nY77jsrsaYgOsUrEzw\x0bw3OLi\n8fkddcaOvJeutTy B\rsDMkK\x0cnx2S0N\x0cDaY\x0c9iyo6p4IL\tOC1qgNlWP4VLg\tWmPG46ZMCirth5h4FwkS\nD2WsiEA2Z\n0xbLd7Uww hUQC6 3V\r1SsWem4UcQxG\rfuVvWl\nD9\nDpZQFFgiqhQiq1I0LMAR\r\rKBmj4iurrxaoMHTl9oj\x0b0N3AfD17gyqZiJ67bgizvecsRGeB1f\x0c\nYRvieJqIVHDKOOR\ruhqnVZz4BQ5FFBusz\x0cZl5\x0bt\tbdOUhAAAKyA6Jwl 7OjzojiRHGD6dl ncsgndsKURhFv4\tV5d\n73iPzbT\t8v6IrJtnq\nJuFl7A\x0b\rVnnsjTW0Y4QB1BgCy3B\x0cma7\tpPt5jmcJH7v5J\tYKEXh UqRChBFY5nbFbmXjJYxevPYJmSHC\rDQ4j9de\rTMZ\rtWaPAzkJjH\x0c\nyrEuf9WaMM\trFlKo9r9w\r\nQkQqIEu8Gfr\t aRzvN\r2oZhCyB4fa\np37\tXQi4Wa\no7gHUDQLoRvkK1dy2K3ydrI0O6\rFTGS7oHA\x0bajFOd\rcS5W25tFGhocwxM0\nuugNGDLjBQ\tWGdJV0\x0c\r7bNLs\x0cr deAWt35A4co\x0bPCuYmQ ExxtK\rvpckCyJxLrK5xULK\x0cvqtiGyovHQW8aDjV3rhXhR\nmQvmK\x0czLx\x0cECSYSF5jP35zN VkaRzQ lZ4 l06X4HHpsVn 8y8fGbIP\tRWFUAeFI24\rqN\x0cBW7u7WPMv36BmkgzQ\x0c2\x0cyLf\tYo8iRjE7zMsceym4ZnWg7EsOedh2cES\rz2n\x0cJi52uIPfSkAPzW\rEekjgWdb8y 285F4xae8\n8AiIkT4l3AOy\rT4yeXgaRMCI4t3PkHeFZ\rEb6R4FNCE \nbVil\x0c6qxSVPnU\nh\ttFMNE4\x0c\rwF\t\x0cW5vebbRWG\x0biVZLP\x0ct\x0c5gQ4CJ9KJl\x0cwyIfSIYaCvi4m1r\tJbYqmI0NVO36A\t8BSPNlaKbR73l9mxZxoqD4yca\n5h\r7a0z\tVm34aTy\tnLj5nSrh8er5lN0J7hcjmUk2DL\nyWEVNXTF8RWfC\x0bpcgBQXOQzidyYO\x0bh76UyUPAjELmNoECgGq06hiFGDI LiPZcofhcm\r62fEixIoyG\tmI\x0cYLQvBCbCluGgbm\x0c7GI6\n19il8PdPqss2uQqA5KgkHMIb hh211YuqV9kdmVnwyD63pz3p t58q6kHX\r\teYBrg6eDh\x0bx8\x0cI1SOV3Gt5qubmixHR\rApbgkTQJQ\tX0t11IP55hys2d\x0bF dh7j7G0Ac\x0bQMNvkSU9AV\x0b8mcIPHy9d\x0cyINf5qu\x0cdiBFrhiNRmCZ4r\tSx4N5VOm6KCp2T8bOVEjOR6otPAN5e\n\x0csyJ3giBjkgg 9dYQKq5P75AG5\x0bfD6zZO4DxQ44uX7Kz50dv4ncXQA\rqgHT\rLRcsRl\rW\t7We\tpAEJHMChxwVK\x0cprVvINvolf7hj\tUrob\rW3pXlqKIEQT8t7\x0bGODJanb328OiQCxE\rPfW4j\rl3p\x0cRXDB55u0MN7isBL\ty3UvE1 7I\nfuoZVPzk7az1\rMzA2FROXu0k\rFq pby6pHMqfTQT7iTw izlk0CUpyoUaq5w3UPFK7\rMOPw2cZ6FsVITbCoPhT\rIvuImCFGqmYpE hNevWkPCtwwnx2sX\x0c7oKzBExp32ZpdY\tstuDjSzfalsO1M\x0bNMUegnBDr3Liv3Lv\x0b\n37VZT2LEJ9fNYDi9r\t\x0bYC\rHSt0oJbk\x0b\x0bUdS8eB\nMXBPDEppZjHR7vGZYqX7yFm t1i682AXWf VPTzYTvm6mhOre8\x0bk0spJNYuI\tk\tC1B1N0 AYYDWH\t\tX1TjinXdkXcbFTlIiBLzx\rmUoyx9b7paJSVMX\tfLo8hU1Dmuluyk8R8\x0c4\x0cBe\nCrIMlyek4i\x0bFwuE9\nXUqpVxikH0PZspopUwPM9Kcue\rBh2Mf\rme3h4qelC\x0bEH\x0bkkxi6U\x0cE\x0ctqBgN93 V4ovmocLrK6\ngCQlf\x0cshRVvrPq\x0cOjgbjhSEK8PIx8OYqjjDDkJ0AgLhfbdGw2\nLMv2M0E08PGXnqUyVsjN\t C 4\n80 Fia g\x0b5dEFvyl5Y80U6sMAdHgk2nzC5ElDBhgcBprXC\x0bIMKXyt\x0ce5SkYcRartfblLqD1 A5\nre\x0bj67lJYCs\t8b50xA69eMHqGDLLP8sJceN19kkonjLj\t\rS\tk9sMOeewQHbT \x0cp53aMX9\x0bDYCZWAtdA6h\rAFHDEYFBE1MzdOxMO\x0cvDE7QfLb3jq4s\tI3aVTmDDOQAnuvWb2AGUUP\rf2HinUAiF13LKEfpqcD06S8aQC0Kyl729L7a6CbuoB0GRlJx tD yuTVqD62HuXpfKrDsbejEdp3\rxjc\x0bn4lLNaViizec\rWR\x0cTT5aZ\ny9\rO1qB1XGQPnES\nUhJtU Ll7t3Zglj1IAEx 8Rh3V\x0bfmUSC4\x0bVR9l33LS3bPAJpLbH3Q2\nv2fqMeIt3nGR\x0cgCixM4qzVSx7Yb192a1HWx8nnuWQIEK7QHL6p\x0cD3d0Y1FoZqsmY2U\rspvt3gwKOHR6RaZlmhX\n3bmIEF6\x0b\x0bMXJKOnXPgjkdhun4aGDBw\x0cOEW\repDYTcc48oZ4lg7PukNq7TU\tWP0ZJbzVKK\rxAMaZujwTqQXsXODiE2DdwnstAa6CMYfzj7J\x0c2Q\tY2764IYCy 3Fqm0\x0ckbe7VvfqWUh0\tUlubxZ\rX59MfNSfCfcH8GFZIGIRPt\rZVXfra1 H7VI2yJ\x0cspGDCi\rcgHfZa8528CP9tilUx0ifWPGqskLVDPLJP\nciNxodMQSrJXp\ro\r9aBFHCV\x0cR\rrp\x0bmMfxg5rG\tSuWonbJQlmHQ\ri34w8S\x0cN9Ezj2k2OmLH\x0cEcVUDjXNZIFCtlA843I44p GZyhlOctwpd7 OZnUxk4uacN\r8NihNGO\n9eXy5l6gQe5srySxxvuX5jtCzuJ35xvCfEXYa\x0b2lTDBOAaSYpnl v9L\x0cY8RLg2oE7xeCUbD\tSHKZgeXHZIzYAmA7bsmiZUfzmo5ZZUhtBh4F\x0bTx1\x0bz zQov5mYwfpWJTR2Q\x0bLRXMuBzj\x0bZC\x0b pFNPj8ixWJQggQlr9eNW6SHLJk731nc\x0cBn\x0ckQxg2BdRT\x0bp6lf7G\x0bnIMDeY8w6fUf\x0cjGE1Pfsekv7EYEIHsOAsZb3lBfBPO9\tXpHPBMRmRtzMc5WoX6C5cc\x0cBuTPtPOgXnap1Y3xq7pcMcgu55xblsXEAJKsojjR7aDB\tU84kUKRNEj\n8mcqEyOmvq1WA\na6bhzYf9VQv2aj9KLfByVqUKNFVIc4Mkha\x0c0aCPQSKe0GGwPlSfbtNXhdhxAb3RLf1J\x0cshJzjQe4DCmlRmjt\tlB0BwzBpkg2hTYM\r S\x0cux\x0bj6IcEZ\n\ngQ\rKKgg \rrv4sUMy5sfY1aatjK1MmUyXR\rRHk\x0cqq\x0cD1fy4C0\n\x0byd4SFKOyKJqx2mzI74vPxLLo\x0c0OamjXuUu\nWGkiA70nuf0PGRfwLEBPCMeyneJI1HcIXH\nCTFEIMiAq6fT\rmJgC hXEU\rriAhCm3OzgbcDgvQgDSyUw5jl\x0cTaimauFseq\x0cj2npTd57itktTdWBY7sqlOGKNSc\x0ctx2mUoHi31EF3l5lvYPDeG6bIPFwIn7\tG6G \x0bgNkSn89flvqcvI73RA";
+
+    let canary = srv.mock(|when, then| {
+        when.method(GET).path("/canary");
+        then.status(200).body(content);
+    });
+
+    // not similar, should see results in output
+    let not_similar = srv.mock(|when, then| {
+        when.method(GET).path("/not-similar");
+        then.status(302).body("this is a test");
+    });
+
+    // similar, should not see results
+    let similar = srv.mock(|when, then| {
+        when.method(GET).path("/similar");
+        then.status(200).body(mutated);
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--filter-similar-to")
+        .arg(srv.url("/canary"))
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICfdafdsafdsafadsENSE")
+            .and(predicate::str::contains("302"))
+            .and(predicate::str::contains("14c"))
+            .and(predicate::str::contains("/similar"))
+            .not()
+            .and(predicate::str::contains("4100c"))
+            .not(),
+    );
+
+    assert_eq!(canary.hits(), 1);
+    assert_eq!(similar.hits(), 1);
+    assert_eq!(not_similar.hits(), 1);
     teardown_tmp_directory(tmp_dir);
 }

tests/test_heuristics.rs

@@ -2,7 +2,7 @@ mod utils;
 use assert_cmd::prelude::*;
 use assert_cmd::Command;
 use httpmock::Method::GET;
-use httpmock::{Mock, MockServer, Regex};
+use httpmock::{MockServer, Regex};
 use predicates::prelude::*;
 use utils::{setup_tmp_directory, teardown_tmp_directory};
 
@@ -65,12 +65,10 @@ fn test_one_good_and_one_bad_target_scan_succeeds() -> Result<(), Box<dyn std::e
     let urls = vec![not_real, srv.url("/"), String::from("LICENSE")];
     let (tmp_dir, file) = setup_tmp_directory(&urls, "wordlist")?;
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/LICENSE")
-        .return_status(200)
-        .return_body("this is a test")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
 
     let mut cmd = Command::cargo_bin("feroxbuster").unwrap();
 
@@ -86,7 +84,51 @@ fn test_one_good_and_one_bad_target_scan_succeeds() -> Result<(), Box<dyn std::e
                 .and(predicate::str::contains("200"))
                 .and(predicate::str::contains("14")),
         );
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
+
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// test pipes two good targets to the scanner, expected result is that both targets
+/// are scanned successfully and no error is reported (result of issue #169)
+fn test_two_good_targets_scan_succeeds() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let srv2 = MockServer::start();
+
+    let urls = vec![srv.url("/"), srv2.url("/"), String::from("LICENSE")];
+    let (tmp_dir, file) = setup_tmp_directory(&urls, "wordlist")?;
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
+
+    let mock2 = srv2.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(403).body("this also is a test");
+    });
+
+    let mut cmd = Command::cargo_bin("feroxbuster").unwrap();
+
+    cmd.arg("--stdin")
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .pipe_stdin(file)
+        .unwrap()
+        .assert()
+        .success()
+        .stdout(
+            predicate::str::contains("/LICENSE")
+                .and(predicate::str::contains("200"))
+                .and(predicate::str::contains("403"))
+                .and(predicate::str::contains("14c"))
+                .and(predicate::str::contains("19c")),
+        );
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock2.hits(), 1);
 
     teardown_tmp_directory(tmp_dir);
     Ok(())
@@ -98,12 +140,11 @@ fn test_static_wildcard_request_found() -> Result<(), Box<dyn std::error::Error>
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap())
-        .return_status(200)
-        .return_body("this is a test")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap());
+        then.status(200).body("this is a test");
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -123,7 +164,7 @@ fn test_static_wildcard_request_found() -> Result<(), Box<dyn std::error::Error>
             .and(predicate::str::contains("(url length: 32)")),
     );
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
     Ok(())
 }
 
@@ -134,19 +175,17 @@ fn test_dynamic_wildcard_request_found() {
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
     let outfile = tmp_dir.path().join("outfile");
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap())
-        .return_status(200)
-        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap());
+        then.status(200)
+            .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+    });
 
-    let mock2 = Mock::new()
-        .expect_method(GET)
-        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap())
-        .return_status(200)
-        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
-        .create_on(&srv);
+    let mock2 = srv.mock(|when, then| {
+        when.method(GET).path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap());
+        then.status(200).body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -177,8 +216,8 @@ fn test_dynamic_wildcard_request_found() {
             .and(predicate::str::contains("(url length: 96)")),
     );
 
-    assert_eq!(mock.times_called(), 1);
-    assert_eq!(mock2.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock2.hits(), 1);
 }
 
 #[test]
@@ -187,12 +226,11 @@ fn heuristics_static_wildcard_request_with_dont_filter() -> Result<(), Box<dyn s
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap())
-        .return_status(200)
-        .return_body("this is a test")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap());
+        then.status(200).body("this is a test");
+    });
 
     Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -205,7 +243,7 @@ fn heuristics_static_wildcard_request_with_dont_filter() -> Result<(), Box<dyn s
 
     teardown_tmp_directory(tmp_dir);
 
-    assert_eq!(mock.times_called(), 0);
+    assert_eq!(mock.hits(), 0);
     Ok(())
 }
 
@@ -215,19 +253,19 @@ fn heuristics_wildcard_test_with_two_static_wildcards() {
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap())
-        .return_status(200)
-        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap());
+        then.status(200)
+            .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+    });
 
-    let mock2 = Mock::new()
-        .expect_method(GET)
-        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap())
-        .return_status(200)
-        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
-        .create_on(&srv);
+    let mock2 = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap());
+        then.status(200)
+            .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -251,30 +289,30 @@ fn heuristics_wildcard_test_with_two_static_wildcards() {
             )),
     );
 
-    assert_eq!(mock.times_called(), 1);
-    assert_eq!(mock2.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock2.hits(), 1);
 }
 
 #[test]
 /// test finds a static wildcard and reports nothing to stdout
-fn heuristics_wildcard_test_with_two_static_wildcards_with_quiet_enabled(
+fn heuristics_wildcard_test_with_two_static_wildcards_with_silent_enabled(
 ) -> Result<(), Box<dyn std::error::Error>> {
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap())
-        .return_status(200)
-        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap());
+        then.status(200)
+            .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+    });
 
-    let mock2 = Mock::new()
-        .expect_method(GET)
-        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap())
-        .return_status(200)
-        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
-        .create_on(&srv);
+    let mock2 = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap());
+        then.status(200)
+            .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -283,15 +321,15 @@ fn heuristics_wildcard_test_with_two_static_wildcards_with_quiet_enabled(
         .arg("--wordlist")
         .arg(file.as_os_str())
         .arg("--add-slash")
-        .arg("-q")
+        .arg("--silent")
         .unwrap();
 
     teardown_tmp_directory(tmp_dir);
 
     cmd.assert().success().stdout(predicate::str::is_empty());
 
-    assert_eq!(mock.times_called(), 1);
-    assert_eq!(mock2.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock2.hits(), 1);
     Ok(())
 }
 
@@ -302,19 +340,19 @@ fn heuristics_wildcard_test_with_two_static_wildcards_and_output_to_file() {
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
     let outfile = tmp_dir.path().join("outfile");
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap())
-        .return_status(200)
-        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap());
+        then.status(200)
+            .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+    });
 
-    let mock2 = Mock::new()
-        .expect_method(GET)
-        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap())
-        .return_status(200)
-        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
-        .create_on(&srv);
+    let mock2 = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap());
+        then.status(200)
+            .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -348,8 +386,8 @@ fn heuristics_wildcard_test_with_two_static_wildcards_and_output_to_file() {
             )),
     );
 
-    assert_eq!(mock.times_called(), 1);
-    assert_eq!(mock2.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock2.hits(), 1);
 }
 
 #[test]
@@ -361,20 +399,20 @@ fn heuristics_wildcard_test_with_redirect_as_response_code(
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
     let outfile = tmp_dir.path().join("outfile");
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap())
-        .return_status(301)
-        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap());
+        then.status(301)
+            .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+    });
 
-    let mock2 = Mock::new()
-        .expect_method(GET)
-        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap())
-        .return_status(301)
-        .return_header("Location", &srv.url("/some-redirect"))
-        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
-        .create_on(&srv);
+    let mock2 = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap());
+        then.status(301)
+            .header("Location", &srv.url("/some-redirect"))
+            .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -407,7 +445,7 @@ fn heuristics_wildcard_test_with_redirect_as_response_code(
             .and(predicate::str::contains("WLD")),
     );
 
-    assert_eq!(mock.times_called(), 1);
-    assert_eq!(mock2.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock2.hits(), 1);
     Ok(())
 }

tests/test_main.rs

@@ -1,7 +1,7 @@
 mod utils;
 use assert_cmd::Command;
 use httpmock::Method::GET;
-use httpmock::{Mock, MockServer};
+use httpmock::MockServer;
 use predicates::prelude::*;
 use utils::{setup_tmp_directory, teardown_tmp_directory};
 
@@ -10,12 +10,10 @@ use utils::{setup_tmp_directory, teardown_tmp_directory};
 fn main_use_root_owned_file_as_wordlist() -> Result<(), Box<dyn std::error::Error>> {
     let srv = MockServer::start();
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/")
-        .return_status(200)
-        .return_body("this is a test")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/");
+        then.status(200).body("this is a test");
+    });
 
     Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -25,11 +23,13 @@ fn main_use_root_owned_file_as_wordlist() -> Result<(), Box<dyn std::error::Erro
         .arg("/etc/shadow")
         .arg("-vvvv")
         .assert()
-        .failure()
-        .stdout(predicate::str::contains("Permission denied (os error 13)"));
+        .success()
+        .stderr(predicate::str::contains(
+            "Failed while scanning: Could not open /etc/shadow",
+        ));
 
     // connectivity test hits it once
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
     Ok(())
 }
 
@@ -39,12 +39,10 @@ fn main_use_empty_wordlist() -> Result<(), Box<dyn std::error::Error>> {
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&[], "wordlist")?;
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/")
-        .return_status(200)
-        .return_body("this is a test")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/");
+        then.status(200).body("this is a test");
+    });
 
     Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -54,10 +52,12 @@ fn main_use_empty_wordlist() -> Result<(), Box<dyn std::error::Error>> {
         .arg(file.as_os_str())
         .arg("-vvvv")
         .assert()
-        .failure()
-        .stdout(predicate::str::contains("Did not find any words in"));
+        .success()
+        .stderr(predicate::str::contains(
+            "Failed while scanning: Did not find any words in",
+        ));
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
 
     teardown_tmp_directory(tmp_dir);
     Ok(())

tests/test_scan_manager.rs

@@ -1,7 +1,7 @@
 mod utils;
 use assert_cmd::Command;
 use httpmock::Method::GET;
-use httpmock::{Mock, MockServer};
+use httpmock::MockServer;
 use predicates::prelude::*;
 use std::fs::{read_to_string, write};
 use std::path::Path;
@@ -20,17 +20,17 @@ fn resume_scan_works() {
     // localhost:PORT/ <- complete
     // localhost:PORT/js <- will get scanned with /css and /stuff
     let complete_scan = format!(
-        r#"{{"id":"057016a14769414aac9a7a62707598cb","url":"{}","scan_type":"Directory","complete":true}}"#,
+        r#"{{"id":"057016a14769414aac9a7a62707598cb","url":"{}","scan_type":"Directory","status":"Complete"}}"#,
         srv.url("/")
     );
     let incomplete_scan = format!(
-        r#"{{"id":"400b2323a16f43468a04ffcbbeba34c6","url":"{}","scan_type":"Directory","complete":false}}"#,
+        r#"{{"id":"400b2323a16f43468a04ffcbbeba34c6","url":"{}","scan_type":"Directory","status":"NotStarted"}}"#,
         srv.url("/js")
     );
     let scans = format!(r#""scans":[{},{}]"#, complete_scan, incomplete_scan);
 
     let config = format!(
-        r#""config": {{"type":"configuration","wordlist":"{}","config":"","proxy":"","replay_proxy":"","target_url":"{}","status_codes":[200,204,301,302,307,308,401,403,405],"replay_codes":[200,204,301,302,307,308,401,403,405],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"quiet":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/1.9.0","redirects":false,"insecure":false,"extensions":[],"headers":{{}},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":false,"depth":2,"scan_limit":1,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false}}"#,
+        r#""config": {{"type":"configuration","wordlist":"{}","config":"","proxy":"","replay_proxy":"","target_url":"{}","status_codes":[200,204,301,302,307,308,401,403,405],"replay_codes":[200,204,301,302,307,308,401,403,405],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"silent":false,"quiet":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/1.9.0","redirects":false,"insecure":false,"extensions":[],"headers":{{}},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":false,"depth":2,"scan_limit":1,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false}}"#,
         file.to_string_lossy(),
         srv.url("/")
     );
@@ -43,28 +43,23 @@ fn resume_scan_works() {
     let responses = format!(r#""responses":[{}]"#, response);
 
     // not scanned because /js is not complete, and /js/stuff response is not known
-    let not_scanned_yet = Mock::new()
-        .expect_method(GET)
-        .expect_path("/js/stuff")
-        .return_status(200)
-        .return_body("i expect to be scanned")
-        .create_on(&srv);
+    let not_scanned_yet = srv.mock(|when, then| {
+        when.method(GET).path("/js/stuff");
+        then.status(200).body("i expect to be scanned");
+    });
 
     // will get scanned because /js is not complete, but because response of /js/css is known, the
     // response will not be in stdout
-    let already_scanned = Mock::new()
-        .expect_method(GET)
-        .expect_path("/js/css")
-        .return_status(200)
-        .create_on(&srv);
+    let already_scanned = srv.mock(|when, then| {
+        when.method(GET).path("/js/css");
+        then.status(200);
+    });
 
     // already scanned because scan on / is complete
-    let also_already_scanned = Mock::new()
-        .expect_method(GET)
-        .expect_path("/css")
-        .return_status(200)
-        .return_body("two words")
-        .create_on(&srv);
+    let also_already_scanned = srv.mock(|when, then| {
+        when.method(GET).path("/css");
+        then.status(200).body("two words");
+    });
 
     let state_file_contents = format!("{{{},{},{}}}", scans, config, responses);
     let (tmp_dir2, state_file) = setup_tmp_directory(&[state_file_contents], "state-file").unwrap();
@@ -90,9 +85,9 @@ fn resume_scan_works() {
     teardown_tmp_directory(tmp_dir);
     teardown_tmp_directory(tmp_dir2);
 
-    assert_eq!(already_scanned.times_called(), 1);
-    assert_eq!(also_already_scanned.times_called(), 0);
-    assert_eq!(not_scanned_yet.times_called(), 1);
+    assert_eq!(already_scanned.hits(), 1);
+    assert_eq!(also_already_scanned.hits(), 0);
+    assert_eq!(not_scanned_yet.hits(), 1);
 }
 
 #[test]

tests/test_scanner.rs

@@ -1,9 +1,9 @@
 mod utils;
 use assert_cmd::prelude::*;
 use httpmock::Method::GET;
-use httpmock::{Mock, MockServer};
+use httpmock::MockServer;
 use predicates::prelude::*;
-use std::process::Command;
+use std::{process::Command, time};
 use utils::{setup_tmp_directory, teardown_tmp_directory};
 
 #[test]
@@ -12,12 +12,10 @@ fn scanner_single_request_scan() -> Result<(), Box<dyn std::error::Error>> {
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/LICENSE")
-        .return_status(200)
-        .return_body("this is a test")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -34,7 +32,7 @@ fn scanner_single_request_scan() -> Result<(), Box<dyn std::error::Error>> {
             .and(predicate::str::contains("14")),
     );
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
     teardown_tmp_directory(tmp_dir);
     Ok(())
 }
@@ -51,33 +49,26 @@ fn scanner_recursive_request_scan() -> Result<(), Box<dyn std::error::Error>> {
     ];
     let (tmp_dir, file) = setup_tmp_directory(&urls, "wordlist")?;
 
-    let js_mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/js")
-        .return_status(301)
-        .return_header("Location", &srv.url("/js/"))
-        .create_on(&srv);
+    let js_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js");
+        then.status(301).header("Location", &srv.url("/js/"));
+    });
 
-    let js_prod_mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/js/prod")
-        .return_status(301)
-        .return_header("Location", &srv.url("/js/prod/"))
-        .create_on(&srv);
+    let js_prod_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js/prod");
+        then.status(301).header("Location", &srv.url("/js/prod/"));
+    });
 
-    let js_dev_mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/js/dev")
-        .return_status(301)
-        .return_header("Location", &srv.url("/js/dev/"))
-        .create_on(&srv);
+    let js_dev_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js/dev");
+        then.status(301).header("Location", &srv.url("/js/dev/"));
+    });
 
-    let js_dev_file_mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/js/dev/file.js")
-        .return_status(200)
-        .return_body("this is a test and is more bytes than other ones")
-        .create_on(&srv);
+    let js_dev_file_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js/dev/file.js");
+        then.status(200)
+            .body("this is a test and is more bytes than other ones");
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -98,10 +89,10 @@ fn scanner_recursive_request_scan() -> Result<(), Box<dyn std::error::Error>> {
             .and(predicate::str::is_match("200.*js/dev/file.js").unwrap()),
     );
 
-    assert_eq!(js_mock.times_called(), 1);
-    assert_eq!(js_prod_mock.times_called(), 1);
-    assert_eq!(js_dev_mock.times_called(), 1);
-    assert_eq!(js_dev_file_mock.times_called(), 1);
+    assert_eq!(js_mock.hits(), 1);
+    assert_eq!(js_prod_mock.hits(), 1);
+    assert_eq!(js_dev_mock.hits(), 1);
+    assert_eq!(js_dev_file_mock.hits(), 1);
 
     teardown_tmp_directory(tmp_dir);
 
@@ -121,33 +112,26 @@ fn scanner_recursive_request_scan_using_only_success_responses(
     ];
     let (tmp_dir, file) = setup_tmp_directory(&urls, "wordlist")?;
 
-    let js_mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/js/")
-        .return_status(200)
-        .return_header("Location", &srv.url("/js/"))
-        .create_on(&srv);
+    let js_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js/");
+        then.status(200).header("Location", &srv.url("/js/"));
+    });
 
-    let js_prod_mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/js/prod/")
-        .return_status(200)
-        .return_header("Location", &srv.url("/js/prod/"))
-        .create_on(&srv);
+    let js_prod_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js/prod/");
+        then.status(200).header("Location", &srv.url("/js/prod/"));
+    });
 
-    let js_dev_mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/js/dev/")
-        .return_status(200)
-        .return_header("Location", &srv.url("/js/dev/"))
-        .create_on(&srv);
+    let js_dev_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js/dev/");
+        then.status(200).header("Location", &srv.url("/js/dev/"));
+    });
 
-    let js_dev_file_mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/js/dev/file.js")
-        .return_status(200)
-        .return_body("this is a test and is more bytes than other ones")
-        .create_on(&srv);
+    let js_dev_file_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js/dev/file.js");
+        then.status(200)
+            .body("this is a test and is more bytes than other ones");
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -169,10 +153,10 @@ fn scanner_recursive_request_scan_using_only_success_responses(
             .and(predicate::str::is_match("200.*js/dev/file.js").unwrap()),
     );
 
-    assert_eq!(js_mock.times_called(), 1);
-    assert_eq!(js_prod_mock.times_called(), 1);
-    assert_eq!(js_dev_mock.times_called(), 1);
-    assert_eq!(js_dev_file_mock.times_called(), 1);
+    assert_eq!(js_mock.hits(), 1);
+    assert_eq!(js_prod_mock.hits(), 1);
+    assert_eq!(js_dev_mock.hits(), 1);
+    assert_eq!(js_dev_file_mock.hits(), 1);
 
     teardown_tmp_directory(tmp_dir);
 
@@ -185,12 +169,10 @@ fn scanner_single_request_scan_with_file_output() -> Result<(), Box<dyn std::err
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/LICENSE")
-        .return_status(200)
-        .return_body("this is a test")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
 
     let outfile = tmp_dir.path().join("output");
 
@@ -211,7 +193,7 @@ fn scanner_single_request_scan_with_file_output() -> Result<(), Box<dyn std::err
     assert!(contents.contains("200"));
     assert!(contents.contains("14"));
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
     teardown_tmp_directory(tmp_dir);
     Ok(())
 }
@@ -223,12 +205,10 @@ fn scanner_single_request_scan_with_file_output_and_tack_q(
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/LICENSE")
-        .return_status(200)
-        .return_body("this is a test")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
 
     let outfile = tmp_dir.path().join("output");
 
@@ -249,24 +229,22 @@ fn scanner_single_request_scan_with_file_output_and_tack_q(
     let url = srv.url("/LICENSE");
     assert!(contents.contains(&url));
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
     teardown_tmp_directory(tmp_dir);
     Ok(())
 }
 
 #[test]
-/// send an invalid output file, expect nothing to be written to disk
+/// send an invalid output file, expect scan to fail
 fn scanner_single_request_scan_with_invalid_file_output() -> Result<(), Box<dyn std::error::Error>>
 {
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/LICENSE")
-        .return_status(200)
-        .return_body("this is a test")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
 
     let outfile = tmp_dir.path(); // outfile is a directory
 
@@ -285,7 +263,7 @@ fn scanner_single_request_scan_with_invalid_file_output() -> Result<(), Box<dyn
     let contents = std::fs::read_to_string(outfile);
     assert!(contents.is_err());
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 0);
     teardown_tmp_directory(tmp_dir);
     Ok(())
 }
@@ -296,12 +274,10 @@ fn scanner_single_request_quiet_scan() -> Result<(), Box<dyn std::error::Error>>
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/LICENSE")
-        .return_status(200)
-        .return_body("this is a test")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -321,7 +297,7 @@ fn scanner_single_request_quiet_scan() -> Result<(), Box<dyn std::error::Error>>
             .not(),
     );
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
     teardown_tmp_directory(tmp_dir);
     Ok(())
 }
@@ -334,12 +310,10 @@ fn scanner_single_request_returns_301_without_location_header(
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/LICENSE")
-        .return_body("this is a test")
-        .return_status(301)
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(301).body("this is a test");
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -359,7 +333,7 @@ fn scanner_single_request_returns_301_without_location_header(
             .and(predicate::str::contains("14")),
     );
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
     teardown_tmp_directory(tmp_dir);
     Ok(())
 }
@@ -372,19 +346,15 @@ fn scanner_single_request_replayed_to_proxy() -> Result<(), Box<dyn std::error::
     let proxy = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/LICENSE")
-        .return_status(200)
-        .return_body("this is a test")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
 
-    let mock_two = Mock::new()
-        .expect_method(GET)
-        .expect_path("/LICENSE")
-        .return_status(200)
-        .return_body("this is a test")
-        .create_on(&proxy);
+    let mock_two = proxy.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -407,8 +377,8 @@ fn scanner_single_request_replayed_to_proxy() -> Result<(), Box<dyn std::error::
         )
         .stderr(predicate::str::contains("Replay Proxy Codes"));
 
-    assert_eq!(mock.times_called(), 1);
-    assert_eq!(mock_two.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_two.hits(), 1);
     teardown_tmp_directory(tmp_dir);
     Ok(())
 }
@@ -420,19 +390,15 @@ fn scanner_single_request_scan_with_filtered_result() -> Result<(), Box<dyn std:
     let (tmp_dir, file) =
         setup_tmp_directory(&["LICENSE".to_string(), "ignored".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/LICENSE")
-        .return_status(200)
-        .return_body("this is a not a test")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a not a test");
+    });
 
-    let filtered_mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/ignored")
-        .return_status(200)
-        .return_body("this is a test")
-        .create_on(&srv);
+    let filtered_mock = srv.mock(|when, then| {
+        when.method(GET).path("/ignored");
+        then.status(200).body("this is a test");
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -455,8 +421,8 @@ fn scanner_single_request_scan_with_filtered_result() -> Result<(), Box<dyn std:
             .not(),
     );
 
-    assert_eq!(mock.times_called(), 1);
-    assert_eq!(filtered_mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(filtered_mock.hits(), 1);
     teardown_tmp_directory(tmp_dir);
     Ok(())
 }
@@ -467,12 +433,10 @@ fn scanner_single_request_scan_with_debug_logging() {
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/LICENSE")
-        .return_status(200)
-        .return_body("this is a test")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
 
     let outfile = tmp_dir.path().join("debug.log");
 
@@ -494,9 +458,9 @@ fn scanner_single_request_scan_with_debug_logging() {
     assert!(contents.contains("DBG"));
     assert!(contents.contains("INF"));
     assert!(contents.contains("feroxbuster All scans complete!"));
-    assert!(contents.contains("feroxbuster exit: terminal_input_handler"));
+    assert!(contents.contains("feroxbuster::event_handlers::inputs exit: start_enter_handler"));
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
     teardown_tmp_directory(tmp_dir);
 }
 
@@ -506,12 +470,10 @@ fn scanner_single_request_scan_with_debug_logging_as_json() {
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/LICENSE")
-        .return_status(200)
-        .return_body("this is a test")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
 
     let outfile = tmp_dir.path().join("debug.log");
 
@@ -535,10 +497,11 @@ fn scanner_single_request_scan_with_debug_logging_as_json() {
     assert!(contents.contains("\"level\":\"INFO\""));
     assert!(contents.contains("time_offset"));
     assert!(contents.contains("\"module\":\"feroxbuster::scanner\""));
+    assert!(contents.contains("\"module\":\"feroxbuster::event_handlers::inputs\""));
+    assert!(contents.contains("exit: start_enter_handler"));
     assert!(contents.contains("All scans complete!"));
-    assert!(contents.contains("exit: terminal_input_handler"));
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
     teardown_tmp_directory(tmp_dir);
 }
 
@@ -549,19 +512,16 @@ fn scanner_single_request_scan_with_regex_filtered_result() {
     let (tmp_dir, file) =
         setup_tmp_directory(&["LICENSE".to_string(), "ignored".to_string()], "wordlist").unwrap();
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/LICENSE")
-        .return_status(200)
-        .return_body("this is a not a test")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
 
-    let filtered_mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/ignored")
-        .return_status(200)
-        .return_body("this is a test\nThat rug really tied the room together")
-        .create_on(&srv);
+    let filtered_mock = srv.mock(|when, then| {
+        when.method(GET).path("/ignored");
+        then.status(200)
+            .body("this is a test\nThat rug really tied the room together");
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -583,7 +543,95 @@ fn scanner_single_request_scan_with_regex_filtered_result() {
             .not(),
     );
 
-    assert_eq!(mock.times_called(), 1);
-    assert_eq!(filtered_mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(filtered_mock.hits(), 1);
+    teardown_tmp_directory(tmp_dir);
+}
+
+#[test]
+/// send a request to a 403 directory, expect recursion to work into the 403
+fn scanner_recursion_works_with_403_directories() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) =
+        setup_tmp_directory(&["LICENSE".to_string(), "ignored/".to_string()], "wordlist").unwrap();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
+
+    let forbidden_dir = srv.mock(|when, then| {
+        when.method(GET).path("/ignored/");
+        then.status(403);
+    });
+
+    let found_anyway = srv.mock(|when, then| {
+        when.method(GET).path("/ignored/LICENSE");
+        then.status(200)
+            .body("this is a test\nThat rug really tied the room together");
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .count(2)
+            .and(predicate::str::contains("200").count(2))
+            .and(predicate::str::contains("403"))
+            .and(predicate::str::contains("53c"))
+            .and(predicate::str::contains("14c"))
+            .and(predicate::str::contains("0c"))
+            .and(predicate::str::contains("ignored").count(2))
+            .and(predicate::str::contains("/ignored/LICENSE")),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(found_anyway.hits(), 1);
+    assert_eq!(forbidden_dir.hits(), 1);
+
+    teardown_tmp_directory(tmp_dir);
+}
+
+#[test]
+/// kick off scan with a time limit;  
+fn rate_limit_enforced_when_specified() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(
+        &[
+            "css".to_string(),
+            "stuff".to_string(),
+            "css1".to_string(),
+            "css2".to_string(),
+            "css3".to_string(),
+            "css4".to_string(),
+        ],
+        "wordlist",
+    )
+    .unwrap();
+
+    let now = time::Instant::now();
+    let lower_bound = time::Duration::new(5, 0);
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--rate-limit")
+        .arg("1")
+        .assert()
+        .success();
+
+    // --rate-limit is 1, so the test should take roughly 5 seconds, so elapsed should be at least
+    // 5 seconds. If not rate-limited, this test takes about 500ms without rate limiting
+    assert!(now.elapsed() > lower_bound);
+
     teardown_tmp_directory(tmp_dir);
 }