changed filter status emoji

add --parallel option

.github/pull_request_template.md

@@ -7,11 +7,11 @@ Long form explanations of most of the items below can be found in the [CONTRIBUT
 - [ ] Your PR description references the associated issue (i.e. fixes #123456)
 - [ ] Code is in its own branch
 - [ ] Branch name is related to the PR contents
-- [ ] PR targets master
+- [ ] PR targets main
 
 ## Static analysis checks
 - [ ] All rust files are formatted using `cargo fmt`
-- [ ] All `clippy` checks pass when running `cargo clippy --all-targets --all-features -- -D warnings -A clippy::deref_addrof`
+- [ ] All `clippy` checks pass when running `cargo clippy --all-targets --all-features -- -D warnings -A clippy::deref_addrof -A clippy::mutex-atomic`
 - [ ] All existing tests pass
 
 ## Documentation

.github/workflows/build.yml

@@ -5,7 +5,7 @@ on: [push]
 jobs:
   build-nix:
     runs-on: ${{ matrix.os }}
-    if: github.ref == 'refs/heads/master'
+    if: github.ref == 'refs/heads/main'
     strategy:
       matrix:
         type: [ubuntu-x64, ubuntu-x86]
@@ -73,7 +73,7 @@ jobs:
 
   build-macos:
     runs-on: macos-latest
-    if: github.ref == 'refs/heads/master'
+    if: github.ref == 'refs/heads/main'
     steps:
       - uses: actions/checkout@v2
       - uses: actions-rs/toolchain@v1
@@ -103,7 +103,7 @@ jobs:
 
   build-windows:
     runs-on: ${{ matrix.os }}
-    if: github.ref == 'refs/heads/master'
+    if: github.ref == 'refs/heads/main'
     strategy:
       matrix:
         type: [windows-x64, windows-x86]

.github/workflows/check.yml

@@ -61,4 +61,4 @@ jobs:
       - uses: actions-rs/cargo@v1
         with:
           command: clippy
-          args: --all-targets --all-features -- -D warnings -A clippy::deref_addrof
+          args: --all-targets --all-features -- -D warnings -A clippy::deref_addrof -A clippy::mutex-atomic

.github/workflows/coverage.yml

@@ -23,9 +23,13 @@ jobs:
           RUSTFLAGS: '-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort'
           RUSTDOCFLAGS: '-Cpanic=abort'
       - uses: actions-rs/grcov@v0.1
+      - uses: actions/upload-artifact@v2
+        with:
+          name: lcov.info
+          path: lcov.info
       - name: Convert lcov to xml
         run: |
-          curl -O https://raw.githubusercontent.com/eriwen/lcov-to-cobertura-xml/master/lcov_cobertura/lcov_cobertura.py
+          curl -O https://raw.githubusercontent.com/epi052/lcov-to-cobertura-xml/master/lcov_cobertura/lcov_cobertura.py
           chmod +x lcov_cobertura.py
           ./lcov_cobertura.py ./lcov.info
       - uses: codecov/codecov-action@v1
@@ -34,3 +38,7 @@ jobs:
           file: ./coverage.xml
           name: codecov-umbrella
           fail_ci_if_error: true
+      - uses: actions/upload-artifact@v2
+        with:
+          name: coverage.xml
+          path: ./coverage.xml

.gitignore

@@ -3,10 +3,6 @@
 debug/
 target/
 
-# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
-# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
-Cargo.lock
-
 # These are backup files generated by rustfmt
 **/*.rs.bk
 
@@ -25,3 +21,9 @@ lcov_cobertura.py
 
 # dockerignore file that makes it so i can work on the docker config without copying a 4GB manifest or w/e it is
 .dockerignore
+
+# state file created during tests
+ferox-http*
+
+# python stuff cuz reasons
+Pipfile*

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "feroxbuster"
-version = "1.12.2"
+version = "2.2.1"
 authors = ["Ben 'epi' Risher <epibar052@gmail.com>"]
 license = "MIT"
 edition = "2018"
@@ -22,16 +22,16 @@ lazy_static = "1.4"
 
 [dependencies]
 futures = { version = "0.3"}
-tokio = { version = "1.0", features = ["full"] }
-tokio-util = {version = "0.6", features = ["codec"]}
+tokio = { version = "1.2.0", features = ["full"] }
+tokio-util = {version = "0.6.3", features = ["codec"]}
 log = "0.4"
-env_logger = "0.8"
-reqwest = { version = "0.11", features = ["socks"] }
+env_logger = "0.8.3"
+reqwest = { version = "0.11.1", features = ["socks"] }
 clap = "2.33"
 lazy_static = "1.4"
 toml = "0.5"
 serde = { version = "1.0", features = ["derive", "rc"] }
-serde_json = "1.0"
+serde_json = "1.0.62"
 uuid = { version = "0.8", features = ["v4"] }
 indicatif = "0.15"
 console = "0.14"
@@ -42,12 +42,14 @@ crossterm = "0.19"
 rlimit = "0.5"
 ctrlc = "3.1"
 fuzzyhash = "0.2.1"
+anyhow = "1.0"
+leaky-bucket = "0.10.0"
 
 [dev-dependencies]
 tempfile = "3.1"
 httpmock = "0.5.2"
-assert_cmd = "1.0.1"
-predicates = "1.0.5"
+assert_cmd = "1.0.3"
+predicates = "1.0.7"
 
 [profile.release]
 lto = true

Makefile

@@ -0,0 +1,77 @@
+default_prefix = /usr/local
+prefix ?= $(default_prefix)
+exec_prefix = $(prefix)
+bindir = $(exec_prefix)/bin
+datarootdir = $(prefix)/share
+datadir = $(datarootdir)
+example_config = ferox-config.toml.example
+config_file = ferox-config.toml
+
+SHR_SOURCES = $(shell find src -type f -wholename '*src/*.rs') Cargo.toml Cargo.lock
+
+RELEASE = debug
+DEBUG ?= 0
+ifeq (0,$(DEBUG))
+	ARGS = --release
+	RELEASE = release
+endif
+
+VENDORED ?= 0
+ifeq (1,$(VENDORED))
+    ARGS += --frozen
+endif
+
+TARGET = target/$(RELEASE)
+
+.PHONY: all clean distclean install uninstall update
+
+BIN=feroxbuster
+DESKTOP=$(APPID).desktop
+
+all: cli
+
+cli: $(TARGET)/$(BIN) $(TARGET)/$(BIN).1.gz $(SHR_SOURCES)
+
+clean:
+	cargo clean
+
+distclean: clean
+	rm -rf .cargo vendor Cargo.lock vendor.tar
+
+vendor: vendor.tar
+
+vendor.tar:
+	mkdir -p .cargo
+	cargo vendor | head -n -1 > .cargo/config
+	echo 'directory = "vendor"' >> .cargo/config
+	tar pcf vendor.tar vendor
+	rm -rf vendor
+
+install-cli: cli
+	install -Dm 0755 "$(TARGET)/$(BIN)" "$(DESTDIR)$(bindir)/$(BIN)"
+	install -Dm 0644 "$(TARGET)/$(BIN).1.gz" "$(DESTDIR)$(datadir)/man/man1/$(BIN).1.gz"
+	install -Dm 0644 "$(example_config)" "/etc/$(BIN)/$(config_File)"
+
+install: all install-cli
+
+uninstall-cli:
+	rm -f "$(DESTDIR)$(bindir)/$(BIN)"
+	rm -f "$(DESTDIR)$(datadir)/man/man1/$(BIN).1.gz"
+	rm -rf "/etc/$(BIN)/"
+
+uninstall: uninstall-cli
+
+update:
+	cargo update
+
+extract:
+ifeq ($(VENDORED),1)
+	tar pxf vendor.tar
+endif
+
+$(TARGET)/$(BIN): extract
+	cargo build --manifest-path Cargo.toml $(ARGS)
+
+$(TARGET)/$(BIN).1.gz: $(TARGET)/$(BIN)
+	help2man --no-info $< | gzip -c > $@.partial
+	mv $@.partial $@

README.md

@@ -8,7 +8,7 @@
 
 <p align="center">
   <a href="https://github.com/epi052/feroxbuster/actions?query=workflow%3A%22CI+Pipeline%22">
-    <img src="https://img.shields.io/github/workflow/status/epi052/feroxbuster/CI%20Pipeline/master?logo=github">
+    <img src="https://img.shields.io/github/workflow/status/epi052/feroxbuster/CI%20Pipeline/main?logo=github">
   </a>
 
   <a href="https://github.com/epi052/feroxbuster/releases">
@@ -102,6 +102,10 @@ Enumeration.
     - [Extract Links from robots.txt (New in `v1.10.2`)](#extract-links-from-robotstxt-new-in-v1102)
     - [Filter Response by Similarity to A Given Page (fuzzy filter) (new in `v1.11.0`)](#filter-response-by-similarity-to-a-given-page-fuzzy-filter-new-in-v1110)
     - [Cancel a Recursive Scan Interactively (new in `v1.12.0`)](#cancel-a-recursive-scan-interactively-new-in-v1120)
+    - [Limit Number of Requests per Second (Rate Limiting) (new in `v2.0.0`)](#limit-number-of-requests-per-second-rate-limiting-new-in-v200)
+    - [Silence all Output or Be Kinda Quiet (new in `v2.0.0`)](#silence-all-output-or-be-kinda-quiet-new-in-v200)
+    - [Auto-tune or Auto-bail from Scans (new in `v2.1.0`)](#auto-tune-or-auto-bail-from-scans-new-in-v210)
+    - [Run Scans in Parallel (new in `v2.2.0`)](#run-scans-in-parallel-new-in-v220)
 - [Comparison w/ Similar Tools](#-comparison-w-similar-tools)
 - [Common Problems/Issues (FAQ)](#-common-problemsissues-faq)
     - [No file descriptors available](#no-file-descriptors-available)
@@ -196,9 +200,9 @@ Download `feroxbuster_amd64.deb` from the [Releases](https://github.com/epi052/f
 that, use your favorite package manager to install the `.deb`.
 
 ```
-wget -sLO https://github.com/epi052/feroxbuster/releases/latest/download/feroxbuster_amd64.deb.zip
+curl -sLO https://github.com/epi052/feroxbuster/releases/latest/download/feroxbuster_amd64.deb.zip
 unzip feroxbuster_amd64.deb.zip
-sudo apt install ./feroxbuster_amd64.deb
+sudo apt install ./feroxbuster_*_amd64.deb
 ```
 
 ### AUR Install
@@ -278,6 +282,7 @@ Configuration begins with with the following built-in default values baked into
 - threads: `50`
 - verbosity: `0` (no logging enabled)
 - scan_limit: `0` (no limit imposed on concurrent scans)
+- rate_limit: `0` (no limit imposed on requests per second)
 - status_codes: `200 204 301 302 307 308 401 403 405`
 - user_agent: `feroxbuster/VERSION`
 - recursion depth: `4`
@@ -322,7 +327,7 @@ built-in defaults.
 `feroxbuster` searches for `ferox-config.toml` in the following locations (in the order shown):
 
 - `/etc/feroxbuster/` (global)
-- `CONFIG_DIR/ferxobuster/` (per-user)
+- `CONFIG_DIR/feroxbuster/` (per-user)
 - The same directory as the `feroxbuster` executable (per-user)
 - The user's current working directory (per-target)
 
@@ -366,13 +371,18 @@ A pre-made configuration file with examples of all available settings can be fou
 # status_codes = [200, 500]
 # filter_status = [301]
 # threads = 1
+# parallel = 2
 # timeout = 5
+# auto_tune = true
+# auto_bail = true
 # proxy = "http://127.0.0.1:8080"
 # replay_proxy = "http://127.0.0.1:8081"
 # replay_codes = [200, 302]
 # verbosity = 1
 # scan_limit = 6
+# rate_limit = 250
 # quiet = true
+# silent = true
 # json = true
 # output = "/targets/ellingson_mineral_company/gibson.txt"
 # debug_log = "/var/log/find-the-derp.log"
@@ -420,6 +430,8 @@ USAGE:
 
 FLAGS:
     -f, --add-slash        Append / to each request
+        --auto-bail        Automatically stop scanning when an excessive amount of errors are encountered
+        --auto-tune        Automatically lower scan rate when an excessive amount of errors are encountered
     -D, --dont-filter      Don't auto-filter wildcard responses
     -e, --extract-links    Extract links from response body (html, javascript, etc...); make new requests based on
                            findings (default: false)
@@ -427,42 +439,61 @@ FLAGS:
     -k, --insecure         Disables TLS certificate validation
         --json             Emit JSON logs to --output and --debug-log instead of normal text
     -n, --no-recursion     Do not scan recursively
-    -q, --quiet            Only print URLs; Don't print status codes, response size, running config, etc...
+    -q, --quiet            Hide progress bars and banner (good for tmux windows w/ notifications)
     -r, --redirects        Follow redirects
+        --silent           Only print URLs + turn off logging (good for piping a list of urls to other commands)
         --stdin            Read url(s) from STDIN
     -V, --version          Prints version information
     -v, --verbosity        Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v's is probably
                            too much)
 
 OPTIONS:
-        --debug-log <FILE>                  Output file to write log entries (use w/ --json for JSON entries)
-    -d, --depth <RECURSION_DEPTH>           Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)
-    -x, --extensions <FILE_EXTENSION>...    File extension(s) to search for (ex: -x php -x pdf js)
-    -N, --filter-lines <LINES>...           Filter out messages of a particular line count (ex: -N 20 -N 31,30)
-    -X, --filter-regex <REGEX>...           Filter out messages via regular expression matching on the response's body
-                                            (ex: -X '^ignore me$')
-    -S, --filter-size <SIZE>...             Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)
-    -C, --filter-status <STATUS_CODE>...    Filter out status codes (deny list) (ex: -C 200 -C 401)
-    -W, --filter-words <WORDS>...           Filter out messages of a particular word count (ex: -W 312 -W 91,82)
-    -H, --headers <HEADER>...               Specify HTTP headers (ex: -H Header:val 'stuff: things')
-    -o, --output <FILE>                     Output file to write results to (use w/ --json for JSON entries)
-    -p, --proxy <PROXY>                     Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)
-    -Q, --query <QUERY>...                  Specify URL query parameters (ex: -Q token=stuff -Q secret=key)
-    -R, --replay-codes <REPLAY_CODE>...     Status Codes to send through a Replay Proxy when found (default: --status-
-                                            codes value)
-    -P, --replay-proxy <REPLAY_PROXY>       Send only unfiltered requests through a Replay Proxy, instead of all
-                                            requests
-        --resume-from <STATE_FILE>          State file from which to resume a partially complete scan (ex. --resume-from
-                                            ferox-1606586780.state)
-    -L, --scan-limit <SCAN_LIMIT>           Limit total number of concurrent scans (default: 0, i.e. no limit)
-    -s, --status-codes <STATUS_CODE>...     Status Codes to include (allow list) (default: 200 204 301 302 307 308 401
-                                            403 405)
-    -t, --threads <THREADS>                 Number of concurrent threads (default: 50)
-        --time-limit <TIME_SPEC>            Limit total run time of all scans (ex: --time-limit 10m)
-    -T, --timeout <SECONDS>                 Number of seconds before a request times out (default: 7)
-    -u, --url <URL>...                      The target URL(s) (required, unless --stdin used)
-    -a, --user-agent <USER_AGENT>           Sets the User-Agent (default: feroxbuster/VERSION)
-    -w, --wordlist <FILE>                   Path to the wordlist
+        --debug-log <FILE>                        Output file to write log entries (use w/ --json for JSON entries)
+    -d, --depth <RECURSION_DEPTH>
+            Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)
+
+    -x, --extensions <FILE_EXTENSION>...          File extension(s) to search for (ex: -x php -x pdf js)
+    -N, --filter-lines <LINES>...                 Filter out messages of a particular line count (ex: -N 20 -N 31,30)
+    -X, --filter-regex <REGEX>...
+            Filter out messages via regular expression matching on the response's body (ex: -X '^ignore me$')
+
+        --filter-similar-to <UNWANTED_PAGE>...
+            Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)
+
+    -S, --filter-size <SIZE>...                   Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)
+    -C, --filter-status <STATUS_CODE>...          Filter out status codes (deny list) (ex: -C 200 -C 401)
+    -W, --filter-words <WORDS>...                 Filter out messages of a particular word count (ex: -W 312 -W 91,82)
+    -H, --headers <HEADER>...                     Specify HTTP headers (ex: -H Header:val 'stuff: things')
+    -o, --output <FILE>                           Output file to write results to (use w/ --json for JSON entries)
+        --parallel <PARALLEL_SCANS>
+            Run parallel feroxbuster instances (one child process per url passed via stdin)
+
+    -p, --proxy <PROXY>
+            Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)
+
+    -Q, --query <QUERY>...                        Specify URL query parameters (ex: -Q token=stuff -Q secret=key)
+        --rate-limit <RATE_LIMIT>
+            Limit number of requests per second (per directory) (default: 0, i.e. no limit)
+
+    -R, --replay-codes <REPLAY_CODE>...
+            Status Codes to send through a Replay Proxy when found (default: --status-codes value)
+
+    -P, --replay-proxy <REPLAY_PROXY>
+            Send only unfiltered requests through a Replay Proxy, instead of all requests
+
+        --resume-from <STATE_FILE>
+            State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)
+
+    -L, --scan-limit <SCAN_LIMIT>                 Limit total number of concurrent scans (default: 0, i.e. no limit)
+    -s, --status-codes <STATUS_CODE>...
+            Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)
+
+    -t, --threads <THREADS>                       Number of concurrent threads (default: 50)
+        --time-limit <TIME_SPEC>                  Limit total run time of all scans (ex: --time-limit 10m)
+    -T, --timeout <SECONDS>                       Number of seconds before a request times out (default: 7)
+    -u, --url <URL>...                            The target URL(s) (required, unless --stdin used)
+    -a, --user-agent <USER_AGENT>                 Sets the User-Agent (default: feroxbuster/VERSION)
+    -w, --wordlist <FILE>                         Path to the wordlist
 ```
 
 ## 📊 Scan's Display Explained
@@ -520,7 +551,7 @@ same goes for urls, headers, status codes, queries, and size filters.
 ### Read urls from STDIN; pipe only resulting urls out to another tool
 
 ```
-cat targets | ./feroxbuster --stdin --quiet -s 200 301 302 --redirects -x js | fff -s 200 -o js-files
+cat targets | ./feroxbuster --stdin --silent -s 200 301 302 --redirects -x js | fff -s 200 -o js-files
 ```
 
 ### Proxy traffic through Burp
@@ -792,6 +823,109 @@ Here is a short demonstration of cancelling two in-progress scans found via recu
 
 ![cancel-scan](img/cancel-scan.gif)
 
+### Limit Number of Requests per Second (Rate Limiting) (new in `v2.0.0`)
+
+Version 2.0.0 added the ability to limit the number of requests per second. One thing to note is that the limit is 
+enforced on a per-directory basis. 
+
+Limit number of requests per second, per directory, to 100 (requests per second will increase by 100 for each active 
+directory found during recursion)
+
+```
+./feroxbuster -u http://localhost --rate-limit 100
+```
+
+Limit number of requests per second to 100 to the target as a whole (only one directory at a time will be scanned, thus
+limiting the number of requests per second overall)
+
+```
+./feroxbuster -u http://localhost --rate-limit 100 --scan-limit 1
+```
+
+![rate-limit](img/rate-limit-demo.gif)
+
+### Silence all Output or Be Kinda Quiet (new in `v2.0.0`)
+
+Version 2.0.0 introduces `--silent` which is almost equivalent to version 1.x.x's `--quiet`.  
+
+#### `--silent`
+
+Good for piping a list of urls to other commands:
+  - disables logging (no error messages to screen)
+  - don't print banner
+  - only display urls during scan
+
+example output:
+```
+https://localhost.com/contact
+https://localhost.com/about
+https://localhost.com/terms
+```
+
+#### `--quiet`
+
+Good for tmux windows that have notifications enabled as the only updates shown by the scan are new valid responses
+and new directories found that are suitable for recursion.
+  - hide progress bars
+  - don't print banner
+
+example output:
+```
+302        0l        0w        0c https://localhost.com/Login
+200      126l      281w     4091c https://localhost.com/maintenance
+200      126l      281w     4092c https://localhost.com/terms
+... more individual entries, followed by the directories being scanned ...
+Scanning: https://localhost.com
+Scanning: https://localhost.com/homepage
+Scanning: https://localhost.com/api
+```
+
+### Auto-tune or Auto-bail from scans (new in `v2.1.0`)
+
+Version 2.1.0 introduces the `--auto-tune` and `--auto-bail` flags. You can think of these flags as Policies. Both actions (tuning and bailing) are triggered by the same criteria (below).  Policies are only enforced after at least 50 requests have been made (or # of threads, if that's > 50).
+
+Policy Enforcement Criteria:
+  - number of general errors (timeouts, etc) is higher than half the number of threads (or at least 25 if threads are lower) (per directory scanned)
+  - 90% of responses are `403|Forbidden` (per directory scanned)
+  - 30% of requests are `429|Too Many Requests` (per directory scanned)
+
+> both demo gifs below use --timeout to overload a single-threaded python web server and elicit timeouts
+
+#### --auto-tune
+
+The AutoTune policy enforces a rate limit on individual directory scans when one of the criteria above is met.  The rate limit self-adjusts every (`timeout / 2`) seconds. If the number of errors have increased during that time, the allowed rate of requests is lowered.  On the other hand, if the number of errors hasn't moved, the allowed rate of requests is increased.  If no additional errors are found after a certain number of checks, the rate limit will be removed completely. 
+
+![auto-tune](img/auto-tune-demo.gif)
+
+#### --auto-bail
+
+The AutoBail policy aborts individual directory scans when one of the criteria above is met.  They just stop getting scanned, no muss, no fuss. 
+
+![auto-bail](img/auto-bail-demo.gif)
+
+### Run Scans in Parallel (new in `v2.2.0`)
+
+Version 2.2.0 introduces the `--parallel` option.  If you're one of those people who use `feroxbuster` to scan 100s of hosts at a time, this is the option for you! `--parallel` spawns a child process per target passed in over stdin (recursive directories are still async within each child).
+
+The number of parallel scans is limited to whatever you pass to `--parallel`. When one child finishes its scan, the next child will be spawned.  
+
+Unfortunately, using `--parallel` limits terminal output such that only discovered URLs are shown. No amount of `-v`'s will help you here. I imagine this isn't too big of a deal, as folks that need `--parallel` probably aren't sitting there watching the output... 🙃
+
+Example Command:
+```
+cat large-target-list | ./feroxbuster --stdin --parallel 10 --extract-links --auto-bail
+```
+
+Resuling Process List (illustrative):
+```
+feroxbuster --stdin --parallel 10
+ \_ feroxbuster --silent --extract-links --auto-bail -u https://target-one
+ \_ feroxbuster --silent --extract-links --auto-bail -u https://target-two
+ \_ feroxbuster --silent --extract-links --auto-bail -u https://target-three
+ \_ ...
+ \_ feroxbuster --silent --extract-links --auto-bail -u https://target-ten
+```
+
 ## 🧐 Comparison w/ Similar Tools
 
 There are quite a few similar tools for forced browsing/content discovery. Burp Suite Pro, Dirb, Dirbuster, etc...
@@ -813,7 +947,6 @@ few of the use-cases in which feroxbuster may be a better fit:
 |                                                          | feroxbuster | gobuster | ffuf |
 |------------------------------------------------------------------------------|---|---|---|
 | fast                                                                         | ✔ | ✔ | ✔ |
-| easy to use                                                                  | ✔ | ✔ |   |
 | allows recursion                                                             | ✔ |   | ✔ |
 | can specify query parameters                                                 | ✔ |   | ✔ |
 | SOCKS proxy support                                                          | ✔ |   |   |
@@ -838,6 +971,11 @@ few of the use-cases in which feroxbuster may be a better fit:
 | use robots.txt to increase scan coverage (`v1.10.2`)                         | ✔ |   |   |
 | use example page's response to fuzzily filter similar pages  (`v1.11.0`)     | ✔ |   |   |
 | cancel a recursive scan interactively (`v1.12.0`)                            | ✔ |   |   |
+| limit number of requests per second (`v2.0.0`)                               | ✔ | ✔ | ✔ |
+| hide progress bars or be silent (or some variation) (`v2.0.0`)               | ✔ | ✔ | ✔ |
+| automatically tune scans based on errors/403s/429s  (`v2.1.0`)               | ✔ |   |   |
+| automatically stop scans based on errors/403s/429s  (`v2.1.0`)               | ✔ |   | ✔ |
+| run scans in parallel (1 process per target) (`v2.2.0`)                      | ✔ |   |   |
 | **huge** number of other options                                             |   |   | ✔ |
 
 Of note, there's another written-in-rust content discovery tool, [rustbuster](https://github.com/phra/rustbuster). I

ferox-config.toml.example

@@ -16,8 +16,13 @@
 # replay_proxy = "http://127.0.0.1:8081"
 # replay_codes = [200, 302]
 # verbosity = 1
+# parallel = 8
 # scan_limit = 6
+# rate_limit = 250
 # quiet = true
+# silent = true
+# auto_tune = true
+# auto_bail = true
 # json = true
 # output = "/targets/ellingson_mineral_company/gibson.txt"
 # debug_log = "/var/log/find-the-derp.log"

shell_completions/_feroxbuster

@@ -58,11 +58,16 @@ _feroxbuster() {
 '*--filter-similar-to=[Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)]' \
 '-L+[Limit total number of concurrent scans (default: 0, i.e. no limit)]' \
 '--scan-limit=[Limit total number of concurrent scans (default: 0, i.e. no limit)]' \
+'--parallel=[Run parallel feroxbuster instances (one child process per url passed via stdin)]' \
+'(--auto-tune)--rate-limit=[Limit number of requests per second (per directory) (default: 0, i.e. no limit)]' \
 '--time-limit=[Limit total run time of all scans (ex: --time-limit 10m)]' \
-'*-v[Increase verbosity level (use -vv or more for greater effect. \[CAUTION\] 4 -v'\''s is probably too much)]' \
-'*--verbosity[Increase verbosity level (use -vv or more for greater effect. \[CAUTION\] 4 -v'\''s is probably too much)]' \
-'-q[Only print URLs; Don'\''t print status codes, response size, running config, etc...]' \
-'--quiet[Only print URLs; Don'\''t print status codes, response size, running config, etc...]' \
+'(--silent)*-v[Increase verbosity level (use -vv or more for greater effect. \[CAUTION\] 4 -v'\''s is probably too much)]' \
+'(--silent)*--verbosity[Increase verbosity level (use -vv or more for greater effect. \[CAUTION\] 4 -v'\''s is probably too much)]' \
+'(-q --quiet)--silent[Only print URLs + turn off logging (good for piping a list of urls to other commands)]' \
+'-q[Hide progress bars and banner (good for tmux windows w/ notifications)]' \
+'--quiet[Hide progress bars and banner (good for tmux windows w/ notifications)]' \
+'(--auto-bail)--auto-tune[Automatically lower scan rate when an excessive amount of errors are encountered]' \
+'--auto-bail[Automatically stop scanning when an excessive amount of errors are encountered]' \
 '--json[Emit JSON logs to --output and --debug-log instead of normal text]' \
 '-D[Don'\''t auto-filter wildcard responses]' \
 '--dont-filter[Don'\''t auto-filter wildcard responses]' \

shell_completions/_feroxbuster.ps1

@@ -63,11 +63,16 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
             [CompletionResult]::new('--filter-similar-to', 'filter-similar-to', [CompletionResultType]::ParameterName, 'Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)')
             [CompletionResult]::new('-L', 'L', [CompletionResultType]::ParameterName, 'Limit total number of concurrent scans (default: 0, i.e. no limit)')
             [CompletionResult]::new('--scan-limit', 'scan-limit', [CompletionResultType]::ParameterName, 'Limit total number of concurrent scans (default: 0, i.e. no limit)')
+            [CompletionResult]::new('--parallel', 'parallel', [CompletionResultType]::ParameterName, 'Run parallel feroxbuster instances (one child process per url passed via stdin)')
+            [CompletionResult]::new('--rate-limit', 'rate-limit', [CompletionResultType]::ParameterName, 'Limit number of requests per second (per directory) (default: 0, i.e. no limit)')
             [CompletionResult]::new('--time-limit', 'time-limit', [CompletionResultType]::ParameterName, 'Limit total run time of all scans (ex: --time-limit 10m)')
             [CompletionResult]::new('-v', 'v', [CompletionResultType]::ParameterName, 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v''s is probably too much)')
             [CompletionResult]::new('--verbosity', 'verbosity', [CompletionResultType]::ParameterName, 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v''s is probably too much)')
-            [CompletionResult]::new('-q', 'q', [CompletionResultType]::ParameterName, 'Only print URLs; Don''t print status codes, response size, running config, etc...')
-            [CompletionResult]::new('--quiet', 'quiet', [CompletionResultType]::ParameterName, 'Only print URLs; Don''t print status codes, response size, running config, etc...')
+            [CompletionResult]::new('--silent', 'silent', [CompletionResultType]::ParameterName, 'Only print URLs + turn off logging (good for piping a list of urls to other commands)')
+            [CompletionResult]::new('-q', 'q', [CompletionResultType]::ParameterName, 'Hide progress bars and banner (good for tmux windows w/ notifications)')
+            [CompletionResult]::new('--quiet', 'quiet', [CompletionResultType]::ParameterName, 'Hide progress bars and banner (good for tmux windows w/ notifications)')
+            [CompletionResult]::new('--auto-tune', 'auto-tune', [CompletionResultType]::ParameterName, 'Automatically lower scan rate when an excessive amount of errors are encountered')
+            [CompletionResult]::new('--auto-bail', 'auto-bail', [CompletionResultType]::ParameterName, 'Automatically stop scanning when an excessive amount of errors are encountered')
             [CompletionResult]::new('--json', 'json', [CompletionResultType]::ParameterName, 'Emit JSON logs to --output and --debug-log instead of normal text')
             [CompletionResult]::new('-D', 'D', [CompletionResultType]::ParameterName, 'Don''t auto-filter wildcard responses')
             [CompletionResult]::new('--dont-filter', 'dont-filter', [CompletionResultType]::ParameterName, 'Don''t auto-filter wildcard responses')

shell_completions/feroxbuster.bash

@@ -20,7 +20,7 @@ _feroxbuster() {
 
     case "${cmd}" in
         feroxbuster)
-            opts=" -v -q -D -r -k -n -f -e -h -V -w -u -t -d -T -p -P -R -s -o -a -x -H -Q -S -X -W -N -C -L  --verbosity --quiet --json --dont-filter --redirects --insecure --no-recursion --add-slash --stdin --extract-links --help --version --wordlist --url --threads --depth --timeout --proxy --replay-proxy --replay-codes --status-codes --output --resume-from --debug-log --user-agent --extensions --headers --query --filter-size --filter-regex --filter-words --filter-lines --filter-status --filter-similar-to --scan-limit --time-limit  "
+            opts=" -v -q -D -r -k -n -f -e -h -V -w -u -t -d -T -p -P -R -s -o -a -x -H -Q -S -X -W -N -C -L  --verbosity --silent --quiet --auto-tune --auto-bail --json --dont-filter --redirects --insecure --no-recursion --add-slash --stdin --extract-links --help --version --wordlist --url --threads --depth --timeout --proxy --replay-proxy --replay-codes --status-codes --output --resume-from --debug-log --user-agent --extensions --headers --query --filter-size --filter-regex --filter-words --filter-lines --filter-status --filter-similar-to --scan-limit --parallel --rate-limit --time-limit  "
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 1 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -199,6 +199,14 @@ _feroxbuster() {
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
                     ;;
+                --parallel)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --rate-limit)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
                 --time-limit)
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0

shell_completions/feroxbuster.fish

@@ -21,9 +21,14 @@ complete -c feroxbuster -n "__fish_use_subcommand" -s N -l filter-lines -d 'Filt
 complete -c feroxbuster -n "__fish_use_subcommand" -s C -l filter-status -d 'Filter out status codes (deny list) (ex: -C 200 -C 401)'
 complete -c feroxbuster -n "__fish_use_subcommand" -l filter-similar-to -d 'Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)'
 complete -c feroxbuster -n "__fish_use_subcommand" -s L -l scan-limit -d 'Limit total number of concurrent scans (default: 0, i.e. no limit)'
+complete -c feroxbuster -n "__fish_use_subcommand" -l parallel -d 'Run parallel feroxbuster instances (one child process per url passed via stdin)'
+complete -c feroxbuster -n "__fish_use_subcommand" -l rate-limit -d 'Limit number of requests per second (per directory) (default: 0, i.e. no limit)'
 complete -c feroxbuster -n "__fish_use_subcommand" -l time-limit -d 'Limit total run time of all scans (ex: --time-limit 10m)'
 complete -c feroxbuster -n "__fish_use_subcommand" -s v -l verbosity -d 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v\'s is probably too much)'
-complete -c feroxbuster -n "__fish_use_subcommand" -s q -l quiet -d 'Only print URLs; Don\'t print status codes, response size, running config, etc...'
+complete -c feroxbuster -n "__fish_use_subcommand" -l silent -d 'Only print URLs + turn off logging (good for piping a list of urls to other commands)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s q -l quiet -d 'Hide progress bars and banner (good for tmux windows w/ notifications)'
+complete -c feroxbuster -n "__fish_use_subcommand" -l auto-tune -d 'Automatically lower scan rate when an excessive amount of errors are encountered'
+complete -c feroxbuster -n "__fish_use_subcommand" -l auto-bail -d 'Automatically stop scanning when an excessive amount of errors are encountered'
 complete -c feroxbuster -n "__fish_use_subcommand" -l json -d 'Emit JSON logs to --output and --debug-log instead of normal text'
 complete -c feroxbuster -n "__fish_use_subcommand" -s D -l dont-filter -d 'Don\'t auto-filter wildcard responses'
 complete -c feroxbuster -n "__fish_use_subcommand" -s r -l redirects -d 'Follow redirects'

src/banner.rs

@@ -1,753 +0,0 @@
-use crate::{
-    config::{Configuration, CONFIGURATION},
-    statistics::StatCommand,
-    utils::{make_request, status_colorizer},
-};
-use console::{style, Emoji};
-use reqwest::{Client, Url};
-use serde_json::Value;
-use std::io::Write;
-use tokio::sync::mpsc::UnboundedSender;
-
-/// macro helper to abstract away repetitive string formatting
-macro_rules! format_banner_entry_helper {
-    // \u{0020} -> unicode space
-    // \u{2502} -> vertical box drawing character, i.e. │
-    ($rune:expr, $name:expr, $value:expr, $indent:expr, $col_width:expr) => {
-        format!(
-            "\u{0020}{:\u{0020}<indent$}{:\u{0020}<col_w$}\u{2502}\u{0020}{}",
-            $rune,
-            $name,
-            $value,
-            indent = $indent,
-            col_w = $col_width
-        )
-    };
-    ($rune:expr, $name:expr, $value:expr, $value2:expr, $indent:expr, $col_width:expr) => {
-        format!(
-            "\u{0020}{:\u{0020}<indent$}{:\u{0020}<col_w$}\u{2502}\u{0020}{}:\u{0020}{}",
-            $rune,
-            $name,
-            $value,
-            $value2,
-            indent = $indent,
-            col_w = $col_width
-        )
-    };
-}
-
-/// macro that wraps another macro helper to abstract away repetitive string formatting
-macro_rules! format_banner_entry {
-    // 4 -> unicode emoji padding width
-    // 22 -> column width (when unicode rune is 4 bytes wide, 23 when it's 3)
-    // hardcoded since macros don't allow let statements
-    ($rune:expr, $name:expr, $value:expr) => {
-        format_banner_entry_helper!($rune, $name, $value, 3, 22)
-    };
-    ($rune:expr, $name:expr, $value1:expr, $value2:expr) => {
-        format_banner_entry_helper!($rune, $name, $value1, $value2, 3, 22)
-    };
-}
-
-/// Url used to query github's api; specifically used to look for the latest tagged release name
-const UPDATE_URL: &str = "https://api.github.com/repos/epi052/feroxbuster/releases/latest";
-
-/// Simple enum to hold three different update states
-#[derive(Debug)]
-enum UpdateStatus {
-    /// this version and latest release are the same
-    UpToDate,
-
-    /// this version and latest release are not the same
-    OutOfDate,
-
-    /// some error occurred during version check
-    Unknown,
-}
-
-/// Makes a request to the given url, expecting to receive a JSON response that contains a field
-/// named `tag_name` that holds a value representing the latest tagged release of this tool.
-///
-/// ex: v1.1.0
-///
-/// Returns `UpdateStatus`
-async fn needs_update(
-    client: &Client,
-    url: &str,
-    bin_version: &str,
-    tx_stats: UnboundedSender<StatCommand>,
-) -> UpdateStatus {
-    log::trace!("enter: needs_update({:?}, {}, {:?})", client, url, tx_stats);
-
-    let unknown = UpdateStatus::Unknown;
-
-    let api_url = match Url::parse(url) {
-        Ok(url) => url,
-        Err(e) => {
-            log::error!("{}", e);
-            log::trace!("exit: needs_update -> {:?}", unknown);
-            return unknown;
-        }
-    };
-
-    if let Ok(response) = make_request(&client, &api_url, tx_stats.clone()).await {
-        let body = response.text().await.unwrap_or_default();
-
-        let json_response: Value = serde_json::from_str(&body).unwrap_or_default();
-
-        if json_response.is_null() {
-            // unwrap_or_default above should result in a null value for the json_response variable
-            log::error!("Could not parse JSON from response body");
-            log::trace!("exit: needs_update -> {:?}", unknown);
-            return unknown;
-        }
-
-        let latest_version = match json_response["tag_name"].as_str() {
-            Some(tag) => tag.trim_start_matches('v'),
-            None => {
-                log::error!("Could not get version field from JSON response");
-                log::debug!("{}", json_response);
-                log::trace!("exit: needs_update -> {:?}", unknown);
-                return unknown;
-            }
-        };
-
-        // if we've gotten this far, we have a string in the form of X.X.X where X is a number
-        // all that's left is to compare the current version with the version found above
-
-        return if latest_version == bin_version {
-            // there's really only two possible outcomes if we accept that the tag conforms to
-            // the X.X.X pattern:
-            //   1. the version strings match, meaning we're up to date
-            //   2. the version strings do not match, meaning we're out of date
-            //
-            // except for developers working on this code, nobody should ever be in a situation
-            // where they have a version greater than the latest tagged release
-            log::trace!("exit: needs_update -> UpdateStatus::UpToDate");
-            UpdateStatus::UpToDate
-        } else {
-            log::trace!("exit: needs_update -> UpdateStatus::OutOfDate");
-            UpdateStatus::OutOfDate
-        };
-    }
-
-    log::trace!("exit: needs_update -> {:?}", unknown);
-    unknown
-}
-
-/// Simple wrapper for emoji or fallback when terminal doesn't support emoji
-fn format_emoji(emoji: &str) -> String {
-    let width = console::measure_text_width(emoji);
-    let pad_len = width * width;
-    let pad = format!("{:<pad_len$}", "\u{0020}", pad_len = pad_len);
-    Emoji(emoji, &pad).to_string()
-}
-
-/// Prints the banner to stdout.
-///
-/// Only prints those settings which are either always present, or passed in by the user.
-pub async fn initialize<W>(
-    targets: &[String],
-    config: &Configuration,
-    version: &str,
-    mut writer: W,
-    tx_stats: UnboundedSender<StatCommand>,
-) where
-    W: Write,
-{
-    let artwork = format!(
-        r#"
- ___  ___  __   __     __      __         __   ___
-|__  |__  |__) |__) | /  `    /  \ \_/ | |  \ |__
-|    |___ |  \ |  \ | \__,    \__/ / \ | |__/ |___
-by Ben "epi" Risher {}                 ver: {}"#,
-        Emoji("🤓", &format!("{:<2}", "\u{0020}")),
-        version
-    );
-    let status = needs_update(&CONFIGURATION.client, UPDATE_URL, version, tx_stats).await;
-
-    let top = "───────────────────────────┬──────────────────────";
-    let addl_section = "──────────────────────────────────────────────────";
-    let bottom = "───────────────────────────┴──────────────────────";
-
-    writeln!(&mut writer, "{}", artwork).unwrap_or_default();
-    writeln!(&mut writer, "{}", top).unwrap_or_default();
-
-    // begin with always printed items
-    for target in targets {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!(format_emoji("🎯"), "Target Url", target)
-        )
-        .unwrap_or_default(); // 🎯
-    }
-
-    let mut codes = vec![];
-
-    for code in &config.status_codes {
-        codes.push(status_colorizer(&code.to_string()))
-    }
-
-    writeln!(
-        &mut writer,
-        "{}",
-        format_banner_entry!(format_emoji("🚀"), "Threads", config.threads)
-    )
-    .unwrap_or_default(); // 🚀
-
-    writeln!(
-        &mut writer,
-        "{}",
-        format_banner_entry!(format_emoji("📖"), "Wordlist", config.wordlist)
-    )
-    .unwrap_or_default(); // 📖
-
-    writeln!(
-        &mut writer,
-        "{}",
-        format_banner_entry!(
-            format_emoji("🆗"),
-            "Status Codes",
-            format!("[{}]", codes.join(", "))
-        )
-    )
-    .unwrap_or_default(); // 🆗
-
-    if !config.filter_status.is_empty() {
-        // exception here for optional print due to me wanting the allows and denys to be printed
-        // one after the other
-        let mut code_filters = vec![];
-
-        for code in &config.filter_status {
-            code_filters.push(status_colorizer(&code.to_string()))
-        }
-
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!(
-                format_emoji("🗑"),
-                "Status Code Filters",
-                format!("[{}]", code_filters.join(", "))
-            )
-        )
-        .unwrap_or_default(); // 🗑
-    }
-
-    writeln!(
-        &mut writer,
-        "{}",
-        format_banner_entry!(format_emoji("💥"), "Timeout (secs)", config.timeout)
-    )
-    .unwrap_or_default(); // 💥
-
-    writeln!(
-        &mut writer,
-        "{}",
-        format_banner_entry!(format_emoji("🦡"), "User-Agent", config.user_agent)
-    )
-    .unwrap_or_default(); // 🦡
-
-    // followed by the maybe printed or variably displayed values
-    if !config.config.is_empty() {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!(format_emoji("💉"), "Config File", config.config)
-        )
-        .unwrap_or_default(); // 💉
-    }
-
-    if !config.proxy.is_empty() {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!(format_emoji("💎"), "Proxy", config.proxy)
-        )
-        .unwrap_or_default(); // 💎
-    }
-
-    if !config.replay_proxy.is_empty() {
-        // i include replay codes logic here because in config.rs, replay codes are set to the
-        // value in status codes, meaning it's never empty
-
-        let mut replay_codes = vec![];
-
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!(format_emoji("🎥"), "Replay Proxy", config.replay_proxy)
-        )
-        .unwrap_or_default(); // 🎥
-
-        for code in &config.replay_codes {
-            replay_codes.push(status_colorizer(&code.to_string()))
-        }
-
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!(
-                format_emoji("📼"),
-                "Replay Proxy Codes",
-                format!("[{}]", replay_codes.join(", "))
-            )
-        )
-        .unwrap_or_default(); // 📼
-    }
-
-    if !config.headers.is_empty() {
-        for (name, value) in &config.headers {
-            writeln!(
-                &mut writer,
-                "{}",
-                format_banner_entry!(format_emoji("🤯"), "Header", name, value)
-            )
-            .unwrap_or_default(); // 🤯
-        }
-    }
-
-    if !config.filter_size.is_empty() {
-        for filter in &config.filter_size {
-            writeln!(
-                &mut writer,
-                "{}",
-                format_banner_entry!(format_emoji("💢"), "Size Filter", filter)
-            )
-            .unwrap_or_default(); // 💢
-        }
-    }
-
-    if !config.filter_similar.is_empty() {
-        for filter in &config.filter_similar {
-            writeln!(
-                &mut writer,
-                "{}",
-                format_banner_entry!(format_emoji("💢"), "Similarity Filter", filter)
-            )
-            .unwrap_or_default(); // 💢
-        }
-    }
-
-    for filter in &config.filter_word_count {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!(format_emoji("💢"), "Word Count Filter", filter)
-        )
-        .unwrap_or_default(); // 💢
-    }
-
-    for filter in &config.filter_line_count {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!(format_emoji("💢"), "Line Count Filter", filter)
-        )
-        .unwrap_or_default(); // 💢
-    }
-
-    for filter in &config.filter_regex {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!(format_emoji("💢"), "Regex Filter", filter)
-        )
-        .unwrap_or_default(); // 💢
-    }
-
-    if config.extract_links {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!(format_emoji("🔎"), "Extract Links", config.extract_links)
-        )
-        .unwrap_or_default(); // 🔎
-    }
-
-    if config.json {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!(format_emoji("🧔"), "JSON Output", config.json)
-        )
-        .unwrap_or_default(); // 🧔
-    }
-
-    if !config.queries.is_empty() {
-        for query in &config.queries {
-            writeln!(
-                &mut writer,
-                "{}",
-                format_banner_entry!(
-                    format_emoji("🤔"),
-                    "Query Parameter",
-                    format!("{}={}", query.0, query.1)
-                )
-            )
-            .unwrap_or_default(); // 🤔
-        }
-    }
-
-    if !config.output.is_empty() {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!(format_emoji("💾"), "Output File", config.output)
-        )
-        .unwrap_or_default(); // 💾
-    }
-
-    if !config.debug_log.is_empty() {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!(format_emoji("🪲"), "Debugging Log", config.debug_log)
-        )
-        .unwrap_or_default(); // 🪲
-    }
-
-    if !config.extensions.is_empty() {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!(
-                format_emoji("💲"),
-                "Extensions",
-                format!("[{}]", config.extensions.join(", "))
-            )
-        )
-        .unwrap_or_default(); // 💲
-    }
-
-    if config.insecure {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!(format_emoji("🔓"), "Insecure", config.insecure)
-        )
-        .unwrap_or_default(); // 🔓
-    }
-
-    if config.redirects {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!(format_emoji("📍"), "Follow Redirects", config.redirects)
-        )
-        .unwrap_or_default(); // 📍
-    }
-
-    if config.dont_filter {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!(format_emoji("🤪"), "Filter Wildcards", !config.dont_filter)
-        )
-        .unwrap_or_default(); // 🤪
-    }
-
-    let volume = ["🔈", "🔉", "🔊", "📢"];
-    if let 1..=4 = config.verbosity {
-        //speaker medium volume (increasing with verbosity to loudspeaker)
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!(
-                format_emoji(volume[config.verbosity as usize - 1]),
-                "Verbosity",
-                config.verbosity
-            )
-        )
-        .unwrap_or_default();
-    }
-
-    if config.add_slash {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!(format_emoji("🪓"), "Add Slash", config.add_slash)
-        )
-        .unwrap_or_default(); // 🪓
-    }
-
-    if !config.no_recursion {
-        if config.depth == 0 {
-            writeln!(
-                &mut writer,
-                "{}",
-                format_banner_entry!(format_emoji("🔃"), "Recursion Depth", "INFINITE")
-            )
-            .unwrap_or_default(); // 🔃
-        } else {
-            writeln!(
-                &mut writer,
-                "{}",
-                format_banner_entry!(format_emoji("🔃"), "Recursion Depth", config.depth)
-            )
-            .unwrap_or_default(); // 🔃
-        }
-    } else {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!(format_emoji("🚫"), "Do Not Recurse", config.no_recursion)
-        )
-        .unwrap_or_default(); // 🚫
-    }
-
-    if CONFIGURATION.scan_limit > 0 {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!(
-                format_emoji("🦥"),
-                "Concurrent Scan Limit",
-                config.scan_limit
-            )
-        )
-        .unwrap_or_default(); // 🦥
-    }
-
-    if !CONFIGURATION.time_limit.is_empty() {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!(format_emoji("🕖"), "Time Limit", config.time_limit)
-        )
-        .unwrap_or_default(); // 🕖
-    }
-
-    if matches!(status, UpdateStatus::OutOfDate) {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!(
-                format_emoji("🎉"),
-                "New Version Available",
-                "https://github.com/epi052/feroxbuster/releases/latest"
-            )
-        )
-        .unwrap_or_default(); // 🎉
-    }
-
-    writeln!(&mut writer, "{}", bottom).unwrap_or_default();
-    // ⏯
-    writeln!(
-        &mut writer,
-        " {}  Press [{}] to use the {}™",
-        format_emoji("🏁"),
-        style("ENTER").yellow(),
-        style("Scan Cancel Menu").bright().yellow(),
-    )
-    .unwrap_or_default();
-
-    writeln!(&mut writer, "{}", addl_section).unwrap_or_default();
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::{FeroxChannel, VERSION};
-    use httpmock::Method::GET;
-    use httpmock::MockServer;
-    use std::fs::read_to_string;
-    use std::io::stderr;
-    use std::time::Duration;
-    use tempfile::NamedTempFile;
-    use tokio::sync::mpsc;
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
-    /// test to hit no execution of targets for loop in banner
-    async fn banner_intialize_without_targets() {
-        let config = Configuration::default();
-        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
-
-        initialize(&[], &config, VERSION, stderr(), tx).await;
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
-    /// test to hit no execution of statuscode for loop in banner
-    async fn banner_intialize_without_status_codes() {
-        let config = Configuration {
-            status_codes: vec![],
-            ..Default::default()
-        };
-
-        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
-
-        initialize(
-            &[String::from("http://localhost")],
-            &config,
-            VERSION,
-            stderr(),
-            tx,
-        )
-        .await;
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
-    /// test to hit an empty config file
-    async fn banner_intialize_without_config_file() {
-        let config = Configuration {
-            config: String::new(),
-            ..Default::default()
-        };
-
-        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
-
-        initialize(
-            &[String::from("http://localhost")],
-            &config,
-            VERSION,
-            stderr(),
-            tx,
-        )
-        .await;
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
-    /// test to hit an empty config file
-    async fn banner_intialize_without_queries() {
-        let config = Configuration {
-            queries: vec![(String::new(), String::new())],
-            ..Default::default()
-        };
-
-        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
-
-        initialize(
-            &[String::from("http://localhost")],
-            &config,
-            VERSION,
-            stderr(),
-            tx,
-        )
-        .await;
-    }
-
-    #[ignore]
-    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
-    /// test to show that a new version is available for download
-    async fn banner_intialize_with_mismatched_version() {
-        let config = Configuration::default();
-        let file = NamedTempFile::new().unwrap();
-        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
-
-        initialize(
-            &[String::from("http://localhost")],
-            &config,
-            "mismatched-version",
-            &file,
-            tx,
-        )
-        .await;
-        let contents = read_to_string(file.path()).unwrap();
-        println!("contents: {}", contents);
-        assert!(contents.contains("New Version Available"));
-        assert!(contents.contains("https://github.com/epi052/feroxbuster/releases/latest"));
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
-    /// test that
-    async fn banner_needs_update_returns_unknown_with_bad_url() {
-        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
-
-        let result = needs_update(&CONFIGURATION.client, &"", VERSION, tx).await;
-        assert!(matches!(result, UpdateStatus::Unknown));
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
-    /// test return value of good url to needs_update
-    async fn banner_needs_update_returns_up_to_date() {
-        let srv = MockServer::start();
-
-        let mock = srv.mock(|when, then| {
-            when.method(GET).path("/latest");
-            then.status(200).body("{\"tag_name\":\"v1.1.0\"}");
-        });
-
-        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
-
-        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.1.0", tx).await;
-
-        assert_eq!(mock.hits(), 1);
-        assert!(matches!(result, UpdateStatus::UpToDate));
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
-    /// test return value of good url to needs_update that returns a newer version than current
-    async fn banner_needs_update_returns_out_of_date() {
-        let srv = MockServer::start();
-
-        let mock = srv.mock(|when, then| {
-            when.method(GET).path("/latest");
-            then.status(200).body("{\"tag_name\":\"v1.1.0\"}");
-        });
-
-        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
-
-        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1", tx).await;
-
-        assert_eq!(mock.hits(), 1);
-        assert!(matches!(result, UpdateStatus::OutOfDate));
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
-    /// test return value of good url that times out
-    async fn banner_needs_update_returns_unknown_on_timeout() {
-        let srv = MockServer::start();
-
-        let mock = srv.mock(|when, then| {
-            when.method(GET).path("/latest");
-            then.status(200)
-                .body("{\"tag_name\":\"v1.1.0\"}")
-                .delay(Duration::from_secs(8));
-        });
-
-        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
-
-        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1", tx).await;
-
-        assert_eq!(mock.hits(), 1);
-        assert!(matches!(result, UpdateStatus::Unknown));
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
-    /// test return value of good url with bad json response
-    async fn banner_needs_update_returns_unknown_on_bad_json_response() {
-        let srv = MockServer::start();
-
-        let mock = srv.mock(|when, then| {
-            when.method(GET).path("/latest");
-            then.status(200).body("not json");
-        });
-
-        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
-
-        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1", tx).await;
-
-        assert_eq!(mock.hits(), 1);
-        assert!(matches!(result, UpdateStatus::Unknown));
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
-    /// test return value of good url with json response that lacks the tag_name field
-    async fn banner_needs_update_returns_unknown_on_json_without_correct_tag() {
-        let srv = MockServer::start();
-
-        let mock = srv.mock(|when, then| {
-            when.method(GET).path("/latest");
-            then.status(200)
-                .body("{\"no tag_name\": \"doesn't exist\"}");
-        });
-
-        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
-
-        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1", tx).await;
-
-        assert_eq!(mock.hits(), 1);
-        assert!(matches!(result, UpdateStatus::Unknown));
-    }
-}

src/banner/container.rs

@@ -0,0 +1,551 @@
+use super::entry::BannerEntry;
+use crate::{
+    config::Configuration,
+    event_handlers::Handles,
+    utils::{logged_request, status_colorizer},
+    VERSION,
+};
+use anyhow::{bail, Result};
+use console::{style, Emoji};
+use reqwest::Url;
+use serde_json::Value;
+use std::{io::Write, sync::Arc};
+
+/// Url used to query github's api; specifically used to look for the latest tagged release name
+pub const UPDATE_URL: &str = "https://api.github.com/repos/epi052/feroxbuster/releases/latest";
+
+/// Simple enum to hold three different update states
+#[derive(Debug)]
+pub(super) enum UpdateStatus {
+    /// this version and latest release are the same
+    UpToDate,
+
+    /// this version and latest release are not the same
+    OutOfDate,
+
+    /// some error occurred during version check
+    Unknown,
+}
+
+/// Banner object, contains multiple BannerEntry's and knows how to display itself
+pub struct Banner {
+    /// all live targets
+    targets: Vec<BannerEntry>,
+
+    /// represents Configuration.status_codes
+    status_codes: BannerEntry,
+
+    /// represents Configuration.filter_status
+    filter_status: BannerEntry,
+
+    /// represents Configuration.threads
+    threads: BannerEntry,
+
+    /// represents Configuration.wordlist
+    wordlist: BannerEntry,
+
+    /// represents Configuration.timeout
+    timeout: BannerEntry,
+
+    /// represents Configuration.user_agent
+    user_agent: BannerEntry,
+
+    /// represents Configuration.config
+    config: BannerEntry,
+
+    /// represents Configuration.proxy
+    proxy: BannerEntry,
+
+    /// represents Configuration.replay_proxy
+    replay_proxy: BannerEntry,
+
+    /// represents Configuration.replay_codes
+    replay_codes: BannerEntry,
+
+    /// represents Configuration.headers
+    headers: Vec<BannerEntry>,
+
+    /// represents Configuration.filter_size
+    filter_size: Vec<BannerEntry>,
+
+    /// represents Configuration.filter_similar
+    filter_similar: Vec<BannerEntry>,
+
+    /// represents Configuration.filter_word_count
+    filter_word_count: Vec<BannerEntry>,
+
+    /// represents Configuration.filter_line_count
+    filter_line_count: Vec<BannerEntry>,
+
+    /// represents Configuration.filter_regex
+    filter_regex: Vec<BannerEntry>,
+
+    /// represents Configuration.extract_links
+    extract_links: BannerEntry,
+
+    /// represents Configuration.json
+    json: BannerEntry,
+
+    /// represents Configuration.output
+    output: BannerEntry,
+
+    /// represents Configuration.debug_log
+    debug_log: BannerEntry,
+
+    /// represents Configuration.extensions
+    extensions: BannerEntry,
+
+    /// represents Configuration.insecure
+    insecure: BannerEntry,
+
+    /// represents Configuration.redirects
+    redirects: BannerEntry,
+
+    /// represents Configuration.dont_filter
+    dont_filter: BannerEntry,
+
+    /// represents Configuration.queries
+    queries: Vec<BannerEntry>,
+
+    /// represents Configuration.verbosity
+    verbosity: BannerEntry,
+
+    /// represents Configuration.add_slash
+    add_slash: BannerEntry,
+
+    /// represents Configuration.no_recursion
+    no_recursion: BannerEntry,
+
+    /// represents Configuration.scan_limit
+    scan_limit: BannerEntry,
+
+    /// represents Configuration.time_limit
+    time_limit: BannerEntry,
+
+    /// represents Configuration.rate_limit
+    rate_limit: BannerEntry,
+
+    /// represents Configuration.parallel
+    parallel: BannerEntry,
+
+    /// represents Configuration.auto_tune
+    auto_tune: BannerEntry,
+
+    /// represents Configuration.auto_bail
+    auto_bail: BannerEntry,
+
+    /// current version of feroxbuster
+    pub(super) version: String,
+
+    /// whether or not there is a known new version
+    pub(super) update_status: UpdateStatus,
+}
+
+/// implementation of Banner
+impl Banner {
+    /// Create a new Banner from a Configuration and live targets
+    pub fn new(tgts: &[String], config: &Configuration) -> Self {
+        let mut targets = Vec::new();
+        let mut code_filters = Vec::new();
+        let mut replay_codes = Vec::new();
+        let mut headers = Vec::new();
+        let mut filter_size = Vec::new();
+        let mut filter_similar = Vec::new();
+        let mut filter_word_count = Vec::new();
+        let mut filter_line_count = Vec::new();
+        let mut filter_regex = Vec::new();
+        let mut queries = Vec::new();
+
+        for target in tgts {
+            targets.push(BannerEntry::new("🎯", "Target Url", target));
+        }
+
+        let mut codes = vec![];
+        for code in &config.status_codes {
+            codes.push(status_colorizer(&code.to_string()))
+        }
+        let status_codes =
+            BannerEntry::new("👌", "Status Codes", &format!("[{}]", codes.join(", ")));
+
+        for code in &config.filter_status {
+            code_filters.push(status_colorizer(&code.to_string()))
+        }
+        let filter_status = BannerEntry::new(
+            "💢",
+            "Status Code Filters",
+            &format!("[{}]", code_filters.join(", ")),
+        );
+
+        for code in &config.replay_codes {
+            replay_codes.push(status_colorizer(&code.to_string()))
+        }
+        let replay_codes = BannerEntry::new(
+            "📼",
+            "Replay Proxy Codes",
+            &format!("[{}]", replay_codes.join(", ")),
+        );
+
+        for (name, value) in &config.headers {
+            headers.push(BannerEntry::new(
+                "🤯",
+                "Header",
+                &format!("{}: {}", name, value),
+            ));
+        }
+
+        for filter in &config.filter_size {
+            filter_size.push(BannerEntry::new("💢", "Size Filter", &filter.to_string()));
+        }
+
+        for filter in &config.filter_similar {
+            filter_similar.push(BannerEntry::new("💢", "Similarity Filter", filter));
+        }
+
+        for filter in &config.filter_word_count {
+            filter_word_count.push(BannerEntry::new(
+                "💢",
+                "Word Count Filter",
+                &filter.to_string(),
+            ));
+        }
+
+        for filter in &config.filter_line_count {
+            filter_line_count.push(BannerEntry::new(
+                "💢",
+                "Line Count Filter",
+                &filter.to_string(),
+            ));
+        }
+
+        for filter in &config.filter_regex {
+            filter_regex.push(BannerEntry::new("💢", "Regex Filter", filter));
+        }
+
+        for query in &config.queries {
+            queries.push(BannerEntry::new(
+                "🤔",
+                "Query Parameter",
+                &format!("{}={}", query.0, query.1),
+            ));
+        }
+
+        let volume = ["🔈", "🔉", "🔊", "📢"];
+        let verbosity = if let 1..=4 = config.verbosity {
+            //speaker medium volume (increasing with verbosity to loudspeaker)
+            BannerEntry::new(
+                volume[config.verbosity as usize - 1],
+                "Verbosity",
+                &config.verbosity.to_string(),
+            )
+        } else {
+            BannerEntry::default()
+        };
+
+        let no_recursion = if !config.no_recursion {
+            let depth = if config.depth == 0 {
+                "INFINITE".to_string()
+            } else {
+                config.depth.to_string()
+            };
+
+            BannerEntry::new("🔃", "Recursion Depth", &depth)
+        } else {
+            BannerEntry::new("🚫", "Do Not Recurse", &config.no_recursion.to_string())
+        };
+
+        let scan_limit = BannerEntry::new(
+            "🦥",
+            "Concurrent Scan Limit",
+            &config.scan_limit.to_string(),
+        );
+
+        let replay_proxy = BannerEntry::new("🎥", "Replay Proxy", &config.replay_proxy);
+        let auto_tune = BannerEntry::new("🎶", "Auto Tune", &config.auto_tune.to_string());
+        let auto_bail = BannerEntry::new("🪣", "Auto Bail", &config.auto_bail.to_string());
+        let cfg = BannerEntry::new("💉", "Config File", &config.config);
+        let proxy = BannerEntry::new("💎", "Proxy", &config.proxy);
+        let threads = BannerEntry::new("🚀", "Threads", &config.threads.to_string());
+        let wordlist = BannerEntry::new("📖", "Wordlist", &config.wordlist);
+        let timeout = BannerEntry::new("💥", "Timeout (secs)", &config.timeout.to_string());
+        let user_agent = BannerEntry::new("🦡", "User-Agent", &config.user_agent);
+        let extract_links =
+            BannerEntry::new("🔎", "Extract Links", &config.extract_links.to_string());
+        let json = BannerEntry::new("🧔", "JSON Output", &config.json.to_string());
+        let output = BannerEntry::new("💾", "Output File", &config.output);
+        let debug_log = BannerEntry::new("🪲", "Debugging Log", &config.debug_log);
+        let extensions = BannerEntry::new(
+            "💲",
+            "Extensions",
+            &format!("[{}]", config.extensions.join(", ")),
+        );
+        let insecure = BannerEntry::new("🔓", "Insecure", &config.insecure.to_string());
+        let redirects = BannerEntry::new("📍", "Follow Redirects", &config.redirects.to_string());
+        let dont_filter =
+            BannerEntry::new("🤪", "Filter Wildcards", &(!config.dont_filter).to_string());
+        let add_slash = BannerEntry::new("🪓", "Add Slash", &config.add_slash.to_string());
+        let time_limit = BannerEntry::new("🕖", "Time Limit", &config.time_limit);
+        let parallel = BannerEntry::new("🛤", "Parallel Scans", &config.parallel.to_string());
+        let rate_limit =
+            BannerEntry::new("🚧", "Requests per Second", &config.rate_limit.to_string());
+
+        Self {
+            targets,
+            status_codes,
+            threads,
+            wordlist,
+            filter_status,
+            timeout,
+            user_agent,
+            auto_bail,
+            auto_tune,
+            proxy,
+            replay_codes,
+            replay_proxy,
+            headers,
+            filter_size,
+            filter_similar,
+            filter_word_count,
+            filter_line_count,
+            filter_regex,
+            extract_links,
+            parallel,
+            json,
+            queries,
+            output,
+            debug_log,
+            extensions,
+            insecure,
+            dont_filter,
+            redirects,
+            verbosity,
+            add_slash,
+            no_recursion,
+            rate_limit,
+            scan_limit,
+            time_limit,
+            config: cfg,
+            version: VERSION.to_string(),
+            update_status: UpdateStatus::Unknown,
+        }
+    }
+
+    /// get a fancy header for the banner
+    fn header(&self) -> String {
+        let artwork = format!(
+            r#"
+ ___  ___  __   __     __      __         __   ___
+|__  |__  |__) |__) | /  `    /  \ \_/ | |  \ |__
+|    |___ |  \ |  \ | \__,    \__/ / \ | |__/ |___
+by Ben "epi" Risher {}                 ver: {}"#,
+            Emoji("🤓", &format!("{:<2}", "\u{0020}")),
+            self.version
+        );
+
+        let top = "───────────────────────────┬──────────────────────";
+
+        format!("{}\n{}", artwork, top)
+    }
+
+    /// get a fancy footer for the banner
+    fn footer(&self) -> String {
+        let addl_section = "──────────────────────────────────────────────────";
+        let bottom = "───────────────────────────┴──────────────────────";
+
+        let instructions = format!(
+            " 🏁  Press [{}] to use the {}™",
+            style("ENTER").yellow(),
+            style("Scan Cancel Menu").bright().yellow(),
+        );
+
+        format!("{}\n{}\n{}", bottom, instructions, addl_section)
+    }
+
+    /// Makes a request to the given url, expecting to receive a JSON response that contains a field
+    /// named `tag_name` that holds a value representing the latest tagged release of this tool.
+    ///
+    /// ex: v1.1.0
+    pub async fn check_for_updates(&mut self, url: &str, handles: Arc<Handles>) -> Result<()> {
+        log::trace!("enter: needs_update({}, {:?})", url, handles);
+
+        let api_url = Url::parse(url)?;
+
+        let result = logged_request(&api_url, handles.clone()).await?;
+        let body = result.text().await?;
+
+        let json_response: Value = serde_json::from_str(&body)?;
+
+        let latest_version = match json_response["tag_name"].as_str() {
+            Some(tag) => tag.trim_start_matches('v'),
+            None => {
+                bail!("JSON has no tag_name: {}", json_response);
+            }
+        };
+
+        // if we've gotten this far, we have a string in the form of X.X.X where X is a number
+        // all that's left is to compare the current version with the version found above
+
+        if latest_version == self.version {
+            // there's really only two possible outcomes if we accept that the tag conforms to
+            // the X.X.X pattern:
+            //   1. the version strings match, meaning we're up to date
+            //   2. the version strings do not match, meaning we're out of date
+            //
+            // except for developers working on this code, nobody should ever be in a situation
+            // where they have a version greater than the latest tagged release
+            self.update_status = UpdateStatus::UpToDate;
+        } else {
+            self.update_status = UpdateStatus::OutOfDate;
+        }
+
+        log::trace!("exit: check_for_updates -> {:?}", self.update_status);
+        Ok(())
+    }
+
+    /// display the banner on Write writer
+    pub fn print_to<W>(&self, mut writer: W, config: Arc<Configuration>) -> Result<()>
+    where
+        W: Write,
+    {
+        writeln!(&mut writer, "{}", self.header())?;
+
+        // begin with always printed items
+        for target in &self.targets {
+            writeln!(&mut writer, "{}", target)?;
+        }
+
+        writeln!(&mut writer, "{}", self.threads)?;
+        writeln!(&mut writer, "{}", self.wordlist)?;
+        writeln!(&mut writer, "{}", self.status_codes)?;
+
+        if !config.filter_status.is_empty() {
+            // exception here for an optional print in the middle of always printed values is due
+            // to me wanting the allows and denys to be printed one after the other
+            writeln!(&mut writer, "{}", self.filter_status)?;
+        }
+
+        writeln!(&mut writer, "{}", self.timeout)?;
+        writeln!(&mut writer, "{}", self.user_agent)?;
+
+        // followed by the maybe printed or variably displayed values
+        if !config.config.is_empty() {
+            writeln!(&mut writer, "{}", self.config)?;
+        }
+
+        if !config.proxy.is_empty() {
+            writeln!(&mut writer, "{}", self.proxy)?;
+        }
+
+        if !config.replay_proxy.is_empty() {
+            // i include replay codes logic here because in config.rs, replay codes are set to the
+            // value in status codes, meaning it's never empty
+            writeln!(&mut writer, "{}", self.replay_proxy)?;
+            writeln!(&mut writer, "{}", self.replay_codes)?;
+        }
+
+        for header in &self.headers {
+            writeln!(&mut writer, "{}", header)?;
+        }
+
+        for filter in &self.filter_size {
+            writeln!(&mut writer, "{}", filter)?;
+        }
+
+        for filter in &self.filter_similar {
+            writeln!(&mut writer, "{}", filter)?;
+        }
+
+        for filter in &self.filter_word_count {
+            writeln!(&mut writer, "{}", filter)?;
+        }
+
+        for filter in &self.filter_line_count {
+            writeln!(&mut writer, "{}", filter)?;
+        }
+
+        for filter in &self.filter_regex {
+            writeln!(&mut writer, "{}", filter)?;
+        }
+
+        if config.extract_links {
+            writeln!(&mut writer, "{}", self.extract_links)?;
+        }
+
+        if config.json {
+            writeln!(&mut writer, "{}", self.json)?;
+        }
+
+        for query in &self.queries {
+            writeln!(&mut writer, "{}", query)?;
+        }
+
+        if !config.output.is_empty() {
+            writeln!(&mut writer, "{}", self.output)?;
+        }
+
+        if !config.debug_log.is_empty() {
+            writeln!(&mut writer, "{}", self.debug_log)?;
+        }
+
+        if !config.extensions.is_empty() {
+            writeln!(&mut writer, "{}", self.extensions)?;
+        }
+
+        if config.insecure {
+            writeln!(&mut writer, "{}", self.insecure)?;
+        }
+
+        if config.auto_bail {
+            writeln!(&mut writer, "{}", self.auto_bail)?;
+        }
+        if config.auto_tune {
+            writeln!(&mut writer, "{}", self.auto_tune)?;
+        }
+
+        if config.redirects {
+            writeln!(&mut writer, "{}", self.redirects)?;
+        }
+
+        if config.dont_filter {
+            writeln!(&mut writer, "{}", self.dont_filter)?;
+        }
+
+        if let 1..=4 = config.verbosity {
+            writeln!(&mut writer, "{}", self.verbosity)?;
+        }
+
+        if config.add_slash {
+            writeln!(&mut writer, "{}", self.add_slash)?;
+        }
+
+        writeln!(&mut writer, "{}", self.no_recursion)?;
+
+        if config.scan_limit > 0 {
+            writeln!(&mut writer, "{}", self.scan_limit)?;
+        }
+
+        if config.parallel > 0 {
+            writeln!(&mut writer, "{}", self.parallel)?;
+        }
+
+        if config.rate_limit > 0 {
+            writeln!(&mut writer, "{}", self.rate_limit)?;
+        }
+
+        if !config.time_limit.is_empty() {
+            writeln!(&mut writer, "{}", self.time_limit)?;
+        }
+
+        if matches!(self.update_status, UpdateStatus::OutOfDate) {
+            let update = BannerEntry::new(
+                "🎉",
+                "New Version Available",
+                "https://github.com/epi052/feroxbuster/releases/latest",
+            );
+            writeln!(&mut writer, "{}", update)?;
+        }
+
+        writeln!(&mut writer, "{}", self.footer())?;
+
+        Ok(())
+    }
+}

src/banner/entry.rs

@@ -0,0 +1,57 @@
+use console::{measure_text_width, Emoji};
+use std::fmt;
+
+/// Initial visual indentation size used in formatting banner entries
+const INDENT: usize = 3;
+
+/// Column width used in formatting banner entries
+const COL_WIDTH: usize = 22;
+
+/// Represents a single line on the banner
+#[derive(Default)]
+pub(super) struct BannerEntry {
+    /// emoji used in the banner entry
+    emoji: String,
+
+    /// title used in the banner entry
+    title: String,
+
+    /// value passed in via config/cli/defaults
+    value: String,
+}
+
+/// implementation of a banner entry
+impl BannerEntry {
+    /// Create a new banner entry from given fields
+    pub fn new(emoji: &str, title: &str, value: &str) -> Self {
+        BannerEntry {
+            emoji: emoji.to_string(),
+            title: title.to_string(),
+            value: value.to_string(),
+        }
+    }
+
+    /// Simple wrapper for emoji or fallback when terminal doesn't support emoji
+    fn format_emoji(&self) -> String {
+        let width = measure_text_width(&self.emoji);
+        let pad_len = width * width;
+        let pad = format!("{:<pad_len$}", "\u{0020}", pad_len = pad_len);
+        Emoji(&self.emoji, &pad).to_string()
+    }
+}
+
+/// Display implementation for a banner entry
+impl fmt::Display for BannerEntry {
+    /// Display formatter for the given banner entry
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "\u{0020}{:\u{0020}<indent$}{:\u{0020}<width$}\u{2502}\u{0020}{}",
+            self.format_emoji(),
+            self.title,
+            self.value,
+            indent = INDENT,
+            width = COL_WIDTH
+        )
+    }
+}

src/banner/mod.rs

@@ -0,0 +1,8 @@
+//! all logic related to building/printing the banner seen when scans start
+mod container;
+mod entry;
+
+#[cfg(test)]
+mod tests;
+
+pub use self::container::{Banner, UPDATE_URL};

src/banner/tests.rs

@@ -0,0 +1,174 @@
+use super::container::UpdateStatus;
+use super::*;
+use crate::{config::Configuration, event_handlers::Handles, scan_manager::FeroxScans};
+use httpmock::Method::GET;
+use httpmock::MockServer;
+use std::{io::stderr, sync::Arc, time::Duration};
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test to hit no execution of targets for loop in banner
+async fn banner_intialize_without_targets() {
+    let config = Configuration::new().unwrap();
+    let banner = Banner::new(&[], &config);
+    banner.print_to(stderr(), Arc::new(config)).unwrap();
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test to hit no execution of statuscode for loop in banner
+async fn banner_intialize_without_status_codes() {
+    let config = Configuration {
+        status_codes: vec![],
+        ..Default::default()
+    };
+
+    let banner = Banner::new(&[String::from("http://localhost")], &config);
+    banner.print_to(stderr(), Arc::new(config)).unwrap();
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test to hit an empty config file
+async fn banner_intialize_without_config_file() {
+    let config = Configuration {
+        config: String::new(),
+        ..Default::default()
+    };
+
+    let banner = Banner::new(&[String::from("http://localhost")], &config);
+    banner.print_to(stderr(), Arc::new(config)).unwrap();
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test to hit an empty queries
+async fn banner_intialize_without_queries() {
+    let config = Configuration {
+        queries: vec![(String::new(), String::new())],
+        ..Default::default()
+    };
+
+    let banner = Banner::new(&[String::from("http://localhost")], &config);
+    banner.print_to(stderr(), Arc::new(config)).unwrap();
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test that
+async fn banner_needs_update_returns_unknown_with_bad_url() {
+    let handles = Arc::new(Handles::for_testing(None, None).0);
+
+    let mut banner = Banner::new(
+        &[String::from("http://localhost")],
+        &Configuration::new().unwrap(),
+    );
+
+    let _ = banner.check_for_updates("", handles).await;
+
+    assert!(matches!(banner.update_status, UpdateStatus::Unknown));
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test return value of good url to needs_update
+async fn banner_needs_update_returns_up_to_date() {
+    let srv = MockServer::start();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/latest");
+        then.status(200).body("{\"tag_name\":\"v1.1.0\"}");
+    });
+    let scans = Arc::new(FeroxScans::default());
+
+    let handles = Arc::new(Handles::for_testing(Some(scans), None).0);
+
+    let mut banner = Banner::new(&[srv.url("")], &Configuration::new().unwrap());
+    banner.version = String::from("1.1.0");
+
+    let _ = banner.check_for_updates(&srv.url("/latest"), handles).await;
+
+    assert_eq!(mock.hits(), 1);
+    assert!(matches!(banner.update_status, UpdateStatus::UpToDate));
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test return value of good url to needs_update that returns a newer version than current
+async fn banner_needs_update_returns_out_of_date() {
+    let srv = MockServer::start();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/latest");
+        then.status(200).body("{\"tag_name\":\"v1.1.0\"}");
+    });
+
+    let scans = Arc::new(FeroxScans::default());
+
+    let handles = Arc::new(Handles::for_testing(Some(scans), None).0);
+
+    let mut banner = Banner::new(&[srv.url("")], &Configuration::new().unwrap());
+    banner.version = String::from("1.0.1");
+
+    let _ = banner.check_for_updates(&srv.url("/latest"), handles).await;
+
+    assert_eq!(mock.hits(), 1);
+    assert!(matches!(banner.update_status, UpdateStatus::OutOfDate));
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test return value of good url that times out
+async fn banner_needs_update_returns_unknown_on_timeout() {
+    let srv = MockServer::start();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/latest");
+        then.status(200)
+            .body("{\"tag_name\":\"v1.1.0\"}")
+            .delay(Duration::from_secs(8));
+    });
+
+    let handles = Arc::new(Handles::for_testing(None, None).0);
+
+    let mut banner = Banner::new(&[srv.url("")], &Configuration::new().unwrap());
+
+    let _ = banner.check_for_updates(&srv.url("/latest"), handles).await;
+
+    assert_eq!(mock.hits(), 1);
+    assert!(matches!(banner.update_status, UpdateStatus::Unknown));
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test return value of good url with bad json response
+async fn banner_needs_update_returns_unknown_on_bad_json_response() {
+    let srv = MockServer::start();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/latest");
+        then.status(200).body("not json");
+    });
+
+    let handles = Arc::new(Handles::for_testing(None, None).0);
+
+    let mut banner = Banner::new(&[srv.url("")], &Configuration::new().unwrap());
+
+    let _ = banner.check_for_updates(&srv.url("/latest"), handles).await;
+
+    assert_eq!(mock.hits(), 1);
+    assert!(matches!(banner.update_status, UpdateStatus::Unknown));
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test return value of good url with json response that lacks the tag_name field
+async fn banner_needs_update_returns_unknown_on_json_without_correct_tag() {
+    let srv = MockServer::start();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/latest");
+        then.status(200)
+            .body("{\"no tag_name\": \"doesn't exist\"}");
+    });
+
+    let handles = Arc::new(Handles::for_testing(None, None).0);
+
+    let mut banner = Banner::new(&[srv.url("")], &Configuration::new().unwrap());
+    banner.version = String::from("1.0.1");
+
+    let _ = banner.check_for_updates(&srv.url("/latest"), handles).await;
+
+    assert_eq!(mock.hits(), 1);
+    assert!(matches!(banner.update_status, UpdateStatus::Unknown));
+}

src/client.rs

@@ -1,10 +1,8 @@
-use crate::utils::{module_colorizer, status_colorizer};
+use anyhow::Result;
 use reqwest::header::HeaderMap;
 use reqwest::{redirect::Policy, Client, Proxy};
 use std::collections::HashMap;
 use std::convert::TryInto;
-#[cfg(not(test))]
-use std::process::exit;
 use std::time::Duration;
 
 /// Create and return an instance of [reqwest::Client](https://docs.rs/reqwest/latest/reqwest/struct.Client.html)
@@ -15,15 +13,14 @@ pub fn initialize(
     insecure: bool,
     headers: &HashMap<String, String>,
     proxy: Option<&str>,
-) -> Client {
+) -> Result<Client> {
     let policy = if redirects {
         Policy::limited(10)
     } else {
         Policy::none()
     };
 
-    // try_into returns infallible as its error, unwrap is safe here
-    let header_map: HeaderMap = headers.try_into().unwrap();
+    let header_map: HeaderMap = headers.try_into()?;
 
     let client = Client::builder()
         .timeout(Duration::new(timeout, 0))
@@ -32,51 +29,15 @@ pub fn initialize(
         .default_headers(header_map)
         .redirect(policy);
 
-    let client = match proxy {
-        // a proxy is specified, need to add it to the client
-        Some(some_proxy) => {
-            if !some_proxy.is_empty() {
-                // it's not an empty string
-                match Proxy::all(some_proxy) {
-                    Ok(proxy_obj) => client.proxy(proxy_obj),
-                    Err(e) => {
-                        eprintln!(
-                            "{} {} {}",
-                            status_colorizer("ERROR"),
-                            module_colorizer("Client::initialize"),
-                            e
-                        );
-
-                        #[cfg(test)]
-                        panic!();
-                        #[cfg(not(test))]
-                        exit(1);
-                    }
-                }
-            } else {
-                client // Some("") was used?
-            }
-        }
-        // no proxy specified
-        None => client,
-    };
-
-    match client.build() {
-        Ok(client) => client,
-        Err(e) => {
-            eprintln!(
-                "{} {} {}",
-                status_colorizer("ERROR"),
-                module_colorizer("Client::build"),
-                e
-            );
-
-            #[cfg(test)]
-            panic!();
-            #[cfg(not(test))]
-            exit(1);
+    if let Some(some_proxy) = proxy {
+        if !some_proxy.is_empty() {
+            // it's not an empty string; set the proxy
+            let proxy_obj = Proxy::all(some_proxy)?;
+            return Ok(client.proxy(proxy_obj).build()?);
         }
     }
+
+    Ok(client.build()?)
 }
 
 #[cfg(test)]
@@ -88,7 +49,7 @@ mod tests {
     /// create client with a bad proxy, expect panic
     fn client_with_bad_proxy() {
         let headers = HashMap::new();
-        initialize(0, "stuff", true, false, &headers, Some("not a valid proxy"));
+        initialize(0, "stuff", true, false, &headers, Some("not a valid proxy")).unwrap();
     }
 
     #[test]
@@ -96,6 +57,6 @@ mod tests {
     fn client_with_good_proxy() {
         let headers = HashMap::new();
         let proxy = "http://127.0.0.1:8080";
-        initialize(0, "stuff", true, true, &headers, Some(proxy));
+        initialize(0, "stuff", true, true, &headers, Some(proxy)).unwrap();
     }
 }

src/config/container.rs

@@ -1,17 +1,17 @@
-use crate::{
-    client, parser,
-    progress::{add_bar, BarType},
-    scan_manager::resume_scan,
-    utils::{module_colorizer, status_colorizer},
-    FeroxSerialize, DEFAULT_CONFIG_NAME, DEFAULT_STATUS_CODES, DEFAULT_WORDLIST, VERSION,
+use super::utils::{
+    depth, report_and_exit, save_state, serialized_type, status_codes, threads, timeout,
+    user_agent, wordlist, OutputLevel, RequesterPolicy,
 };
+use crate::config::determine_output_level;
+use crate::config::utils::determine_requester_policy;
+use crate::{
+    client, parser, scan_manager::resume_scan, traits::FeroxSerialize, utils::fmt_err,
+    DEFAULT_CONFIG_NAME,
+};
+use anyhow::{anyhow, Context, Result};
 use clap::{value_t, ArgMatches};
-use indicatif::{MultiProgress, ProgressBar, ProgressDrawTarget};
-use lazy_static::lazy_static;
 use reqwest::{Client, StatusCode};
 use serde::{Deserialize, Serialize};
-#[cfg(not(test))]
-use std::process::exit;
 use std::{
     collections::HashMap,
     env::{current_dir, current_exe},
@@ -19,17 +19,6 @@ use std::{
     path::PathBuf,
 };
 
-lazy_static! {
-    /// Global configuration state
-    pub static ref CONFIGURATION: Configuration = Configuration::new();
-
-    /// Global progress bar that houses other progress bars
-    pub static ref PROGRESS_BAR: MultiProgress = MultiProgress::with_draw_target(ProgressDrawTarget::stdout());
-
-    /// Global progress bar that is only used for printing messages that don't jack up other bars
-    pub static ref PROGRESS_PRINTER: ProgressBar = add_bar("", 0, BarType::Hidden);
-}
-
 /// macro helper to abstract away repetitive configuration updates
 macro_rules! update_config_if_present {
     ($c:expr, $m:ident, $v:expr, $t:ty) => {
@@ -56,21 +45,6 @@ macro_rules! update_if_not_default {
     };
 }
 
-/// simple helper to clean up some code reuse below; panics under test / exits in prod
-fn report_and_exit(err: &str) -> ! {
-    eprintln!(
-        "{} {}: {}",
-        status_colorizer("ERROR"),
-        module_colorizer("Configuration::new"),
-        err
-    );
-
-    #[cfg(test)]
-    panic!();
-    #[cfg(not(test))]
-    exit(1);
-}
-
 /// Represents the final, global configuration of the program.
 ///
 /// This struct is the combination of the following:
@@ -139,10 +113,30 @@ pub struct Configuration {
     #[serde(default)]
     pub verbosity: u8,
 
-    /// Only print URLs
+    /// Only print URLs (was --quiet in versions < 2.0.0)
+    #[serde(default)]
+    pub silent: bool,
+
+    /// No header, no status bars
     #[serde(default)]
     pub quiet: bool,
 
+    /// more easily differentiate between the three states of output levels
+    #[serde(skip)]
+    pub output_level: OutputLevel,
+
+    /// automatically bail at certain error thresholds
+    #[serde(default)]
+    pub auto_bail: bool,
+
+    /// automatically try to lower request rate in order to reduce errors
+    #[serde(default)]
+    pub auto_tune: bool,
+
+    /// more easily differentiate between the three requester policies
+    #[serde(skip)]
+    pub requester_policy: RequesterPolicy,
+
     /// Store log output as NDJSON
     #[serde(default)]
     pub json: bool,
@@ -204,6 +198,14 @@ pub struct Configuration {
     #[serde(default)]
     pub scan_limit: usize,
 
+    /// Number of parallel scans permitted; a limit of 0 means no limit is imposed
+    #[serde(default)]
+    pub parallel: usize,
+
+    /// Number of requests per second permitted (per directory); a limit of 0 means no limit is imposed
+    #[serde(default)]
+    pub rate_limit: usize,
+
     /// Filter out messages of a particular size
     #[serde(default)]
     pub filter_size: Vec<u64>,
@@ -248,63 +250,19 @@ pub struct Configuration {
     pub filter_similar: Vec<String>,
 }
 
-// functions timeout, threads, status_codes, user_agent, wordlist, save_state, and depth are used to provide
-// defaults in the event that a ferox-config.toml is found but one or more of the values below
-// aren't listed in the config.  This way, we get the correct defaults upon Deserialization
-
-/// default Configuration type for use in json output
-fn serialized_type() -> String {
-    String::from("configuration")
-}
-
-/// default timeout value
-fn timeout() -> u64 {
-    7
-}
-
-/// default save_state value
-fn save_state() -> bool {
-    true
-}
-
-/// default threads value
-fn threads() -> usize {
-    50
-}
-
-/// default status codes
-fn status_codes() -> Vec<u16> {
-    DEFAULT_STATUS_CODES
-        .iter()
-        .map(|code| code.as_u16())
-        .collect()
-}
-
-/// default wordlist
-fn wordlist() -> String {
-    String::from(DEFAULT_WORDLIST)
-}
-
-/// default user-agent
-fn user_agent() -> String {
-    format!("feroxbuster/{}", VERSION)
-}
-
-/// default recursion depth
-fn depth() -> usize {
-    4
-}
-
 impl Default for Configuration {
     /// Builds the default Configuration for feroxbuster
     fn default() -> Self {
         let timeout = timeout();
         let user_agent = user_agent();
-        let client = client::initialize(timeout, &user_agent, false, false, &HashMap::new(), None);
+        let client = client::initialize(timeout, &user_agent, false, false, &HashMap::new(), None)
+            .expect("Could not build client");
         let replay_client = None;
         let status_codes = status_codes();
         let replay_codes = status_codes.clone();
         let kind = serialized_type();
+        let output_level = OutputLevel::Default;
+        let requester_policy = RequesterPolicy::Default;
 
         Configuration {
             kind,
@@ -314,13 +272,20 @@ impl Default for Configuration {
             replay_codes,
             status_codes,
             replay_client,
+            requester_policy,
             dont_filter: false,
+            auto_bail: false,
+            auto_tune: false,
+            silent: false,
             quiet: false,
+            output_level,
             resumed: false,
             stdin: false,
             json: false,
             verbosity: 0,
             scan_limit: 0,
+            parallel: 0,
+            rate_limit: 0,
             add_slash: false,
             insecure: false,
             redirects: false,
@@ -369,6 +334,9 @@ impl Configuration {
     /// - **output**: `None` (print to stdout)
     /// - **debug_log**: `None`
     /// - **quiet**: `false`
+    /// - **silent**: `false`
+    /// - **auto_tune**: `false`
+    /// - **auto_bail**: `false`
     /// - **save_state**: `true`
     /// - **user_agent**: `feroxbuster/VERSION`
     /// - **insecure**: `false` (don't be insecure, i.e. don't allow invalid certs)
@@ -387,6 +355,8 @@ impl Configuration {
     /// - **dont_filter**: `false` (auto filter wildcard responses)
     /// - **depth**: `4` (maximum recursion depth)
     /// - **scan_limit**: `0` (no limit on concurrent scans imposed)
+    /// - **parallel**: `0` (no limit on parallel scans imposed)
+    /// - **rate_limit**: `0` (no limit on requests per second imposed)
     /// - **time_limit**: `None` (no limit on length of scan imposed)
     /// - **replay_proxy**: `None` (no limit on concurrent scans imposed)
     /// - **replay_codes**: [`DEFAULT_RESPONSE_CODES`](constant.DEFAULT_RESPONSE_CODES.html)
@@ -408,14 +378,14 @@ impl Configuration {
     ///
     /// The resulting [Configuration](struct.Configuration.html) is a singleton with a `static`
     /// lifetime.
-    pub fn new() -> Self {
+    pub fn new() -> Result<Self> {
         // when compiling for test, we want to eliminate the runtime dependency of the parser
         if cfg!(test) {
             let test_config = Configuration {
                 save_state: false, // don't clutter up junk when testing
                 ..Default::default()
             };
-            return test_config;
+            return Ok(test_config);
         }
 
         let args = parser::initialize().get_matches();
@@ -425,7 +395,7 @@ impl Configuration {
         let mut config = Configuration::default();
 
         // read in all config files
-        Self::parse_config_files(&mut config);
+        Self::parse_config_files(&mut config)?;
 
         // read in the user provided options, this produces a separate instance of Configuration
         // in order to allow for potentially merging into a --resume-from Configuration
@@ -456,7 +426,7 @@ impl Configuration {
             // clients aren't serialized, have to remake them from the previous config
             Self::try_rebuild_clients(&mut previous_config);
 
-            return previous_config;
+            return Ok(previous_config);
         }
 
         // if we've gotten to this point in the code, --resume-from was not used, so we need to
@@ -466,12 +436,12 @@ impl Configuration {
         // rebuild clients is the last step in either code branch
         Self::try_rebuild_clients(&mut config);
 
-        config
+        Ok(config)
     }
 
     /// Parse all possible versions of the ferox-config.toml file, adhering to the order of
     /// precedence outlined above
-    fn parse_config_files(mut config: &mut Self) {
+    fn parse_config_files(mut config: &mut Self) -> Result<()> {
         // Next, we parse the ferox-config.toml file, if present and set the values
         // therein to overwrite our default values. Deserialized defaults are specified
         // in the Configuration struct so that we don't change anything that isn't
@@ -487,32 +457,31 @@ impl Configuration {
         let config_file = PathBuf::new()
             .join("/etc/feroxbuster")
             .join(DEFAULT_CONFIG_NAME);
-        Self::parse_and_merge_config(config_file, &mut config);
+        Self::parse_and_merge_config(config_file, &mut config)?;
 
         // merge a config found at ~/.config/feroxbuster/ferox-config.toml
-        if let Some(config_dir) = dirs::config_dir() {
-            // config_dir() resolves to one of the following
-            //   - linux: $XDG_CONFIG_HOME or $HOME/.config
-            //   - macOS: $HOME/Library/Application Support
-            //   - windows: {FOLDERID_RoamingAppData}
-
-            let config_file = config_dir.join("feroxbuster").join(DEFAULT_CONFIG_NAME);
-            Self::parse_and_merge_config(config_file, &mut config);
-        };
+        // config_dir() resolves to one of the following
+        //   - linux: $XDG_CONFIG_HOME or $HOME/.config
+        //   - macOS: $HOME/Library/Application Support
+        //   - windows: {FOLDERID_RoamingAppData}
+        let config_dir = dirs::config_dir().ok_or_else(|| anyhow!("Couldn't load config"))?;
+        let config_file = config_dir.join("feroxbuster").join(DEFAULT_CONFIG_NAME);
+        Self::parse_and_merge_config(config_file, &mut config)?;
 
         // merge a config found in same the directory as feroxbuster executable
-        if let Ok(exe_path) = current_exe() {
-            if let Some(bin_dir) = exe_path.parent() {
-                let config_file = bin_dir.join(DEFAULT_CONFIG_NAME);
-                Self::parse_and_merge_config(config_file, &mut config);
-            };
-        };
+        let exe_path = current_exe()?;
+        let bin_dir = exe_path
+            .parent()
+            .ok_or_else(|| anyhow!("Couldn't load config"))?;
+        let config_file = bin_dir.join(DEFAULT_CONFIG_NAME);
+        Self::parse_and_merge_config(config_file, &mut config)?;
 
         // merge a config found in the user's current working directory
-        if let Ok(cwd) = current_dir() {
-            let config_file = cwd.join(DEFAULT_CONFIG_NAME);
-            Self::parse_and_merge_config(config_file, &mut config);
-        }
+        let cwd = current_dir()?;
+        let config_file = cwd.join(DEFAULT_CONFIG_NAME);
+        Self::parse_and_merge_config(config_file, &mut config)?;
+
+        Ok(())
     }
 
     /// Given a set of ArgMatches read from the CLI, update and return the default Configuration
@@ -523,6 +492,8 @@ impl Configuration {
         update_config_if_present!(&mut config.threads, args, "threads", usize);
         update_config_if_present!(&mut config.depth, args, "depth", usize);
         update_config_if_present!(&mut config.scan_limit, args, "scan_limit", usize);
+        update_config_if_present!(&mut config.parallel, args, "parallel", usize);
+        update_config_if_present!(&mut config.rate_limit, args, "rate_limit", usize);
         update_config_if_present!(&mut config.wordlist, args, "wordlist", String);
         update_config_if_present!(&mut config.output, args, "output", String);
         update_config_if_present!(&mut config.debug_log, args, "debug_log", String);
@@ -602,12 +573,28 @@ impl Configuration {
                 .collect();
         }
 
-        if args.is_present("quiet") {
+        if args.is_present("silent") {
             // the reason this is protected by an if statement:
-            // consider a user specifying quiet = true in ferox-config.toml
+            // consider a user specifying silent = true in ferox-config.toml
             // if the line below is outside of the if, we'd overwrite true with
-            // false if no -q is used on the command line
+            // false if no --silent is used on the command line
+            config.silent = true;
+            config.output_level = OutputLevel::Silent;
+        }
+
+        if args.is_present("quiet") {
             config.quiet = true;
+            config.output_level = OutputLevel::Quiet;
+        }
+
+        if args.is_present("auto_tune") {
+            config.auto_tune = true;
+            config.requester_policy = RequesterPolicy::AutoTune;
+        }
+
+        if args.is_present("auto_bail") {
+            config.auto_bail = true;
+            config.requester_policy = RequesterPolicy::AutoBail;
         }
 
         if args.is_present("dont_filter") {
@@ -709,6 +696,7 @@ impl Configuration {
                     &configuration.headers,
                     None,
                 )
+                .expect("Could not rebuild client")
             } else {
                 configuration.client = client::initialize(
                     configuration.timeout,
@@ -718,40 +706,41 @@ impl Configuration {
                     &configuration.headers,
                     Some(&configuration.proxy),
                 )
+                .expect("Could not rebuild client")
             }
         }
 
         if !configuration.replay_proxy.is_empty() {
             // only set replay_client when replay_proxy is set
-            configuration.replay_client = Some(client::initialize(
-                configuration.timeout,
-                &configuration.user_agent,
-                configuration.redirects,
-                configuration.insecure,
-                &configuration.headers,
-                Some(&configuration.replay_proxy),
-            ));
+            configuration.replay_client = Some(
+                client::initialize(
+                    configuration.timeout,
+                    &configuration.user_agent,
+                    configuration.redirects,
+                    configuration.insecure,
+                    &configuration.headers,
+                    Some(&configuration.replay_proxy),
+                )
+                .expect("Could not rebuild client"),
+            );
         }
     }
 
     /// Given a configuration file's location and an instance of `Configuration`, read in
     /// the config file if found and update the current settings with the settings found therein
-    fn parse_and_merge_config(config_file: PathBuf, mut config: &mut Self) {
+    fn parse_and_merge_config(config_file: PathBuf, mut config: &mut Self) -> Result<()> {
         if config_file.exists() {
             // save off a string version of the path before it goes out of scope
-            let conf_str = match config_file.to_str() {
-                Some(cs) => String::from(cs),
-                None => String::new(),
-            };
+            let conf_str = config_file.to_str().unwrap_or("").to_string();
+            let settings = Self::parse_config(config_file)?;
 
-            if let Some(settings) = Self::parse_config(config_file) {
-                // set the config used for viewing in the banner
-                config.config = conf_str;
+            // set the config used for viewing in the banner
+            config.config = conf_str;
 
-                // update the settings
-                Self::merge_config(&mut config, settings);
-            }
+            // update the settings
+            Self::merge_config(&mut config, settings);
         }
+        Ok(())
     }
 
     /// Given two Configurations, overwrite `settings` with the fields found in `settings_to_merge`
@@ -766,7 +755,13 @@ impl Configuration {
         update_if_not_default!(&mut conf.time_limit, new.time_limit, "");
         update_if_not_default!(&mut conf.proxy, new.proxy, "");
         update_if_not_default!(&mut conf.verbosity, new.verbosity, 0);
+        update_if_not_default!(&mut conf.silent, new.silent, false);
         update_if_not_default!(&mut conf.quiet, new.quiet, false);
+        update_if_not_default!(&mut conf.auto_bail, new.auto_bail, false);
+        update_if_not_default!(&mut conf.auto_tune, new.auto_tune, false);
+        // use updated quiet/silent values to determine output level; same for requester policy
+        conf.output_level = determine_output_level(conf.quiet, conf.silent);
+        conf.requester_policy = determine_requester_policy(conf.auto_tune, conf.auto_bail);
         update_if_not_default!(&mut conf.output, new.output, "");
         update_if_not_default!(&mut conf.redirects, new.redirects, false);
         update_if_not_default!(&mut conf.insecure, new.insecure, false);
@@ -805,6 +800,8 @@ impl Configuration {
         );
         update_if_not_default!(&mut conf.dont_filter, new.dont_filter, false);
         update_if_not_default!(&mut conf.scan_limit, new.scan_limit, 0);
+        update_if_not_default!(&mut conf.parallel, new.parallel, 0);
+        update_if_not_default!(&mut conf.rate_limit, new.rate_limit, 0);
         update_if_not_default!(&mut conf.replay_proxy, new.replay_proxy, "");
         update_if_not_default!(&mut conf.debug_log, new.debug_log, "");
         update_if_not_default!(&mut conf.resume_from, new.resume_from, "");
@@ -824,23 +821,10 @@ impl Configuration {
     /// If present, read in `DEFAULT_CONFIG_NAME` and deserialize the specified values
     ///
     /// uses serde to deserialize the toml into a `Configuration` struct
-    fn parse_config(config_file: PathBuf) -> Option<Self> {
-        if let Ok(content) = read_to_string(config_file) {
-            match toml::from_str(content.as_str()) {
-                Ok(config) => {
-                    return Some(config);
-                }
-                Err(e) => {
-                    println!(
-                        "{} {} {}",
-                        status_colorizer("ERROR"),
-                        module_colorizer("config::parse_config"),
-                        e
-                    );
-                }
-            }
-        }
-        None
+    pub(super) fn parse_config(config_file: PathBuf) -> Result<Self> {
+        let content = read_to_string(config_file)?;
+        let config: Self = toml::from_str(content.as_str())?;
+        Ok(config)
     }
 }
 
@@ -875,375 +859,10 @@ impl FeroxSerialize for Configuration {
     ///    ],
     /// ...
     /// }\n
-    fn as_json(&self) -> String {
-        if let Ok(mut json) = serde_json::to_string(&self) {
-            json.push('\n');
-            json
-        } else {
-            String::from("{\"error\":\"could not Configuration convert to json\"}")
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use std::fs::write;
-    use tempfile::TempDir;
-
-    /// creates a dummy configuration file for testing
-    fn setup_config_test() -> Configuration {
-        let data = r#"
-            wordlist = "/some/path"
-            status_codes = [201, 301, 401]
-            replay_codes = [201, 301]
-            threads = 40
-            timeout = 5
-            proxy = "http://127.0.0.1:8080"
-            replay_proxy = "http://127.0.0.1:8081"
-            quiet = true
-            verbosity = 1
-            scan_limit = 6
-            time_limit = "10m"
-            output = "/some/otherpath"
-            debug_log = "/yet/anotherpath"
-            resume_from = "/some/state/file"
-            redirects = true
-            insecure = true
-            extensions = ["html", "php", "js"]
-            headers = {stuff = "things", mostuff = "mothings"}
-            queries = [["name","value"], ["rick", "astley"]]
-            no_recursion = true
-            add_slash = true
-            stdin = true
-            dont_filter = true
-            extract_links = true
-            json = true
-            save_state = false
-            depth = 1
-            filter_size = [4120]
-            filter_regex = ["^ignore me$"]
-            filter_similar = ["https://somesite.com/soft404"]
-            filter_word_count = [994, 992]
-            filter_line_count = [34]
-            filter_status = [201]
-        "#;
-        let tmp_dir = TempDir::new().unwrap();
-        let file = tmp_dir.path().join(DEFAULT_CONFIG_NAME);
-        write(&file, data).unwrap();
-        Configuration::parse_config(file).unwrap()
-    }
-
-    #[test]
-    /// test that all default config values meet expectations
-    fn default_configuration() {
-        let config = Configuration::default();
-        assert_eq!(config.wordlist, wordlist());
-        assert_eq!(config.proxy, String::new());
-        assert_eq!(config.target_url, String::new());
-        assert_eq!(config.time_limit, String::new());
-        assert_eq!(config.resume_from, String::new());
-        assert_eq!(config.debug_log, String::new());
-        assert_eq!(config.config, String::new());
-        assert_eq!(config.replay_proxy, String::new());
-        assert_eq!(config.status_codes, status_codes());
-        assert_eq!(config.replay_codes, config.status_codes);
-        assert!(config.replay_client.is_none());
-        assert_eq!(config.threads, threads());
-        assert_eq!(config.depth, depth());
-        assert_eq!(config.timeout, timeout());
-        assert_eq!(config.verbosity, 0);
-        assert_eq!(config.scan_limit, 0);
-        assert_eq!(config.quiet, false);
-        assert_eq!(config.dont_filter, false);
-        assert_eq!(config.no_recursion, false);
-        assert_eq!(config.json, false);
-        assert_eq!(config.save_state, true);
-        assert_eq!(config.stdin, false);
-        assert_eq!(config.add_slash, false);
-        assert_eq!(config.redirects, false);
-        assert_eq!(config.extract_links, false);
-        assert_eq!(config.insecure, false);
-        assert_eq!(config.queries, Vec::new());
-        assert_eq!(config.extensions, Vec::<String>::new());
-        assert_eq!(config.filter_size, Vec::<u64>::new());
-        assert_eq!(config.filter_regex, Vec::<String>::new());
-        assert_eq!(config.filter_similar, Vec::<String>::new());
-        assert_eq!(config.filter_word_count, Vec::<usize>::new());
-        assert_eq!(config.filter_line_count, Vec::<usize>::new());
-        assert_eq!(config.filter_status, Vec::<u16>::new());
-        assert_eq!(config.headers, HashMap::new());
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_wordlist() {
-        let config = setup_config_test();
-        assert_eq!(config.wordlist, "/some/path");
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_debug_log() {
-        let config = setup_config_test();
-        assert_eq!(config.debug_log, "/yet/anotherpath");
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_status_codes() {
-        let config = setup_config_test();
-        assert_eq!(config.status_codes, vec![201, 301, 401]);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_replay_codes() {
-        let config = setup_config_test();
-        assert_eq!(config.replay_codes, vec![201, 301]);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_threads() {
-        let config = setup_config_test();
-        assert_eq!(config.threads, 40);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_depth() {
-        let config = setup_config_test();
-        assert_eq!(config.depth, 1);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_scan_limit() {
-        let config = setup_config_test();
-        assert_eq!(config.scan_limit, 6);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_timeout() {
-        let config = setup_config_test();
-        assert_eq!(config.timeout, 5);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_proxy() {
-        let config = setup_config_test();
-        assert_eq!(config.proxy, "http://127.0.0.1:8080");
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_replay_proxy() {
-        let config = setup_config_test();
-        assert_eq!(config.replay_proxy, "http://127.0.0.1:8081");
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_quiet() {
-        let config = setup_config_test();
-        assert_eq!(config.quiet, true);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_json() {
-        let config = setup_config_test();
-        assert_eq!(config.json, true);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_verbosity() {
-        let config = setup_config_test();
-        assert_eq!(config.verbosity, 1);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_output() {
-        let config = setup_config_test();
-        assert_eq!(config.output, "/some/otherpath");
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_redirects() {
-        let config = setup_config_test();
-        assert_eq!(config.redirects, true);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_insecure() {
-        let config = setup_config_test();
-        assert_eq!(config.insecure, true);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_no_recursion() {
-        let config = setup_config_test();
-        assert_eq!(config.no_recursion, true);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_stdin() {
-        let config = setup_config_test();
-        assert_eq!(config.stdin, true);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_dont_filter() {
-        let config = setup_config_test();
-        assert_eq!(config.dont_filter, true);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_add_slash() {
-        let config = setup_config_test();
-        assert_eq!(config.add_slash, true);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_extract_links() {
-        let config = setup_config_test();
-        assert_eq!(config.extract_links, true);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_extensions() {
-        let config = setup_config_test();
-        assert_eq!(config.extensions, vec!["html", "php", "js"]);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_filter_regex() {
-        let config = setup_config_test();
-        assert_eq!(config.filter_regex, vec!["^ignore me$"]);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_filter_similar() {
-        let config = setup_config_test();
-        assert_eq!(config.filter_similar, vec!["https://somesite.com/soft404"]);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_filter_size() {
-        let config = setup_config_test();
-        assert_eq!(config.filter_size, vec![4120]);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_filter_word_count() {
-        let config = setup_config_test();
-        assert_eq!(config.filter_word_count, vec![994, 992]);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_filter_line_count() {
-        let config = setup_config_test();
-        assert_eq!(config.filter_line_count, vec![34]);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_filter_status() {
-        let config = setup_config_test();
-        assert_eq!(config.filter_status, vec![201]);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_save_state() {
-        let config = setup_config_test();
-        assert_eq!(config.save_state, false);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_time_limit() {
-        let config = setup_config_test();
-        assert_eq!(config.time_limit, "10m");
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_resume_from() {
-        let config = setup_config_test();
-        assert_eq!(config.resume_from, "/some/state/file");
-    }
-
-    #[test]
-    /// parse the test config and see that the values parsed are correct
-    fn config_reads_headers() {
-        let config = setup_config_test();
-        let mut headers = HashMap::new();
-        headers.insert("stuff".to_string(), "things".to_string());
-        headers.insert("mostuff".to_string(), "mothings".to_string());
-        assert_eq!(config.headers, headers);
-    }
-
-    #[test]
-    /// parse the test config and see that the values parsed are correct
-    fn config_reads_queries() {
-        let config = setup_config_test();
-        let mut queries = vec![];
-        queries.push(("name".to_string(), "value".to_string()));
-        queries.push(("rick".to_string(), "astley".to_string()));
-        assert_eq!(config.queries, queries);
-    }
-
-    #[test]
-    #[should_panic]
-    /// test that an error message is printed and panic is called when report_and_exit is called
-    fn config_report_and_exit_works() {
-        report_and_exit("some message");
-    }
-
-    #[test]
-    /// test as_str method of Configuration
-    fn as_str_returns_string_with_newline() {
-        let config = Configuration::new();
-        let config_str = config.as_str();
-        println!("{}", config_str);
-        assert!(config_str.starts_with("Configuration {"));
-        assert!(config_str.ends_with("}\n"));
-        assert!(config_str.contains("replay_codes:"));
-        assert!(config_str.contains("client: Client {"));
-        assert!(config_str.contains("user_agent: \"feroxbuster"));
-    }
-
-    #[test]
-    /// test as_json method of Configuration
-    fn as_json_returns_json_representation_of_configuration_with_newline() {
-        let mut config = Configuration::new();
-        config.timeout = 12;
-        config.depth = 2;
-        let config_str = config.as_json();
-        let json: Configuration = serde_json::from_str(&config_str).unwrap();
-        assert_eq!(json.config, config.config);
-        assert_eq!(json.wordlist, config.wordlist);
-        assert_eq!(json.replay_codes, config.replay_codes);
-        assert_eq!(json.timeout, config.timeout);
-        assert_eq!(json.depth, config.depth);
+    fn as_json(&self) -> Result<String> {
+        let mut json = serde_json::to_string(&self)
+            .with_context(|| fmt_err("Could not convert Configuration to JSON"))?;
+        json.push('\n');
+        Ok(json)
     }
 }

src/config/mod.rs

@@ -0,0 +1,9 @@
+//! all logic related to instantiating a running configuration
+
+mod container;
+mod utils;
+#[cfg(test)]
+mod tests;
+
+pub use self::container::Configuration;
+pub use self::utils::{determine_output_level, OutputLevel, RequesterPolicy};

src/config/tests.rs

@@ -0,0 +1,406 @@
+use super::utils::*;
+use super::*;
+use crate::{traits::FeroxSerialize, DEFAULT_CONFIG_NAME};
+use std::{collections::HashMap, fs::write};
+use tempfile::TempDir;
+
+/// creates a dummy configuration file for testing
+fn setup_config_test() -> Configuration {
+    let data = r#"
+            wordlist = "/some/path"
+            status_codes = [201, 301, 401]
+            replay_codes = [201, 301]
+            threads = 40
+            timeout = 5
+            proxy = "http://127.0.0.1:8080"
+            replay_proxy = "http://127.0.0.1:8081"
+            quiet = true
+            silent = true
+            auto_tune = true
+            auto_bail = true
+            verbosity = 1
+            scan_limit = 6
+            parallel = 14
+            rate_limit = 250
+            time_limit = "10m"
+            output = "/some/otherpath"
+            debug_log = "/yet/anotherpath"
+            resume_from = "/some/state/file"
+            redirects = true
+            insecure = true
+            extensions = ["html", "php", "js"]
+            headers = {stuff = "things", mostuff = "mothings"}
+            queries = [["name","value"], ["rick", "astley"]]
+            no_recursion = true
+            add_slash = true
+            stdin = true
+            dont_filter = true
+            extract_links = true
+            json = true
+            save_state = false
+            depth = 1
+            filter_size = [4120]
+            filter_regex = ["^ignore me$"]
+            filter_similar = ["https://somesite.com/soft404"]
+            filter_word_count = [994, 992]
+            filter_line_count = [34]
+            filter_status = [201]
+        "#;
+    let tmp_dir = TempDir::new().unwrap();
+    let file = tmp_dir.path().join(DEFAULT_CONFIG_NAME);
+    write(&file, data).unwrap();
+    Configuration::parse_config(file).unwrap()
+}
+
+#[test]
+/// test that all default config values meet expectations
+fn default_configuration() {
+    let config = Configuration::default();
+    assert_eq!(config.wordlist, wordlist());
+    assert_eq!(config.proxy, String::new());
+    assert_eq!(config.target_url, String::new());
+    assert_eq!(config.time_limit, String::new());
+    assert_eq!(config.resume_from, String::new());
+    assert_eq!(config.debug_log, String::new());
+    assert_eq!(config.config, String::new());
+    assert_eq!(config.replay_proxy, String::new());
+    assert_eq!(config.status_codes, status_codes());
+    assert_eq!(config.replay_codes, config.status_codes);
+    assert!(config.replay_client.is_none());
+    assert_eq!(config.threads, threads());
+    assert_eq!(config.depth, depth());
+    assert_eq!(config.timeout, timeout());
+    assert_eq!(config.verbosity, 0);
+    assert_eq!(config.scan_limit, 0);
+    assert_eq!(config.silent, false);
+    assert_eq!(config.quiet, false);
+    assert_eq!(config.output_level, OutputLevel::Default);
+    assert_eq!(config.dont_filter, false);
+    assert_eq!(config.auto_tune, false);
+    assert_eq!(config.auto_bail, false);
+    assert_eq!(config.requester_policy, RequesterPolicy::Default);
+    assert_eq!(config.no_recursion, false);
+    assert_eq!(config.json, false);
+    assert_eq!(config.save_state, true);
+    assert_eq!(config.stdin, false);
+    assert_eq!(config.add_slash, false);
+    assert_eq!(config.redirects, false);
+    assert_eq!(config.extract_links, false);
+    assert_eq!(config.insecure, false);
+    assert_eq!(config.queries, Vec::new());
+    assert_eq!(config.extensions, Vec::<String>::new());
+    assert_eq!(config.filter_size, Vec::<u64>::new());
+    assert_eq!(config.filter_regex, Vec::<String>::new());
+    assert_eq!(config.filter_similar, Vec::<String>::new());
+    assert_eq!(config.filter_word_count, Vec::<usize>::new());
+    assert_eq!(config.filter_line_count, Vec::<usize>::new());
+    assert_eq!(config.filter_status, Vec::<u16>::new());
+    assert_eq!(config.headers, HashMap::new());
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_wordlist() {
+    let config = setup_config_test();
+    assert_eq!(config.wordlist, "/some/path");
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_debug_log() {
+    let config = setup_config_test();
+    assert_eq!(config.debug_log, "/yet/anotherpath");
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_status_codes() {
+    let config = setup_config_test();
+    assert_eq!(config.status_codes, vec![201, 301, 401]);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_replay_codes() {
+    let config = setup_config_test();
+    assert_eq!(config.replay_codes, vec![201, 301]);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_threads() {
+    let config = setup_config_test();
+    assert_eq!(config.threads, 40);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_depth() {
+    let config = setup_config_test();
+    assert_eq!(config.depth, 1);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_scan_limit() {
+    let config = setup_config_test();
+    assert_eq!(config.scan_limit, 6);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_parallel() {
+    let config = setup_config_test();
+    assert_eq!(config.parallel, 14);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_rate_limit() {
+    let config = setup_config_test();
+    assert_eq!(config.rate_limit, 250);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_timeout() {
+    let config = setup_config_test();
+    assert_eq!(config.timeout, 5);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_proxy() {
+    let config = setup_config_test();
+    assert_eq!(config.proxy, "http://127.0.0.1:8080");
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_replay_proxy() {
+    let config = setup_config_test();
+    assert_eq!(config.replay_proxy, "http://127.0.0.1:8081");
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_silent() {
+    let config = setup_config_test();
+    assert_eq!(config.silent, true);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_quiet() {
+    let config = setup_config_test();
+    assert_eq!(config.quiet, true);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_json() {
+    let config = setup_config_test();
+    assert_eq!(config.json, true);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_auto_bail() {
+    let config = setup_config_test();
+    assert_eq!(config.auto_bail, true);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_auto_tune() {
+    let config = setup_config_test();
+    assert_eq!(config.auto_tune, true);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_verbosity() {
+    let config = setup_config_test();
+    assert_eq!(config.verbosity, 1);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_output() {
+    let config = setup_config_test();
+    assert_eq!(config.output, "/some/otherpath");
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_redirects() {
+    let config = setup_config_test();
+    assert_eq!(config.redirects, true);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_insecure() {
+    let config = setup_config_test();
+    assert_eq!(config.insecure, true);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_no_recursion() {
+    let config = setup_config_test();
+    assert_eq!(config.no_recursion, true);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_stdin() {
+    let config = setup_config_test();
+    assert_eq!(config.stdin, true);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_dont_filter() {
+    let config = setup_config_test();
+    assert_eq!(config.dont_filter, true);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_add_slash() {
+    let config = setup_config_test();
+    assert_eq!(config.add_slash, true);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_extract_links() {
+    let config = setup_config_test();
+    assert_eq!(config.extract_links, true);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_extensions() {
+    let config = setup_config_test();
+    assert_eq!(config.extensions, vec!["html", "php", "js"]);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_filter_regex() {
+    let config = setup_config_test();
+    assert_eq!(config.filter_regex, vec!["^ignore me$"]);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_filter_similar() {
+    let config = setup_config_test();
+    assert_eq!(config.filter_similar, vec!["https://somesite.com/soft404"]);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_filter_size() {
+    let config = setup_config_test();
+    assert_eq!(config.filter_size, vec![4120]);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_filter_word_count() {
+    let config = setup_config_test();
+    assert_eq!(config.filter_word_count, vec![994, 992]);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_filter_line_count() {
+    let config = setup_config_test();
+    assert_eq!(config.filter_line_count, vec![34]);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_filter_status() {
+    let config = setup_config_test();
+    assert_eq!(config.filter_status, vec![201]);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_save_state() {
+    let config = setup_config_test();
+    assert_eq!(config.save_state, false);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_time_limit() {
+    let config = setup_config_test();
+    assert_eq!(config.time_limit, "10m");
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_resume_from() {
+    let config = setup_config_test();
+    assert_eq!(config.resume_from, "/some/state/file");
+}
+
+#[test]
+/// parse the test config and see that the values parsed are correct
+fn config_reads_headers() {
+    let config = setup_config_test();
+    let mut headers = HashMap::new();
+    headers.insert("stuff".to_string(), "things".to_string());
+    headers.insert("mostuff".to_string(), "mothings".to_string());
+    assert_eq!(config.headers, headers);
+}
+
+#[test]
+/// parse the test config and see that the values parsed are correct
+fn config_reads_queries() {
+    let config = setup_config_test();
+    let mut queries = vec![];
+    queries.push(("name".to_string(), "value".to_string()));
+    queries.push(("rick".to_string(), "astley".to_string()));
+    assert_eq!(config.queries, queries);
+}
+
+#[test]
+#[should_panic]
+/// test that an error message is printed and panic is called when report_and_exit is called
+fn config_report_and_exit_works() {
+    report_and_exit("some message");
+}
+
+#[test]
+/// test as_str method of Configuration
+fn as_str_returns_string_with_newline() {
+    let config = Configuration::new().unwrap();
+    let config_str = config.as_str();
+    println!("{}", config_str);
+    assert!(config_str.starts_with("Configuration {"));
+    assert!(config_str.ends_with("}\n"));
+    assert!(config_str.contains("replay_codes:"));
+    assert!(config_str.contains("client: Client {"));
+    assert!(config_str.contains("user_agent: \"feroxbuster"));
+}
+
+#[test]
+/// test as_json method of Configuration
+fn as_json_returns_json_representation_of_configuration_with_newline() {
+    let mut config = Configuration::new().unwrap();
+    config.timeout = 12;
+    config.depth = 2;
+    let config_str = config.as_json().unwrap();
+    let json: Configuration = serde_json::from_str(&config_str).unwrap();
+    assert_eq!(json.config, config.config);
+    assert_eq!(json.wordlist, config.wordlist);
+    assert_eq!(json.replay_codes, config.replay_codes);
+    assert_eq!(json.timeout, config.timeout);
+    assert_eq!(json.depth, config.depth);
+}

src/config/utils.rs

@@ -0,0 +1,182 @@
+use crate::{
+    utils::{module_colorizer, status_colorizer},
+    DEFAULT_STATUS_CODES, DEFAULT_WORDLIST, VERSION,
+};
+#[cfg(not(test))]
+use std::process::exit;
+
+/// simple helper to clean up some code reuse below; panics under test / exits in prod
+pub(super) fn report_and_exit(err: &str) -> ! {
+    eprintln!(
+        "{} {}: {}",
+        status_colorizer("ERROR"),
+        module_colorizer("Configuration::new"),
+        err
+    );
+
+    #[cfg(test)]
+    panic!();
+    #[cfg(not(test))]
+    exit(1);
+}
+
+// functions timeout, threads, status_codes, user_agent, wordlist, save_state, and depth are used to provide
+// defaults in the event that a ferox-config.toml is found but one or more of the values below
+// aren't listed in the config.  This way, we get the correct defaults upon Deserialization
+
+/// default Configuration type for use in json output
+pub(super) fn serialized_type() -> String {
+    String::from("configuration")
+}
+
+/// default timeout value
+pub(super) fn timeout() -> u64 {
+    7
+}
+
+/// default save_state value
+pub(super) fn save_state() -> bool {
+    true
+}
+
+/// default threads value
+pub(super) fn threads() -> usize {
+    50
+}
+
+/// default status codes
+pub(super) fn status_codes() -> Vec<u16> {
+    DEFAULT_STATUS_CODES
+        .iter()
+        .map(|code| code.as_u16())
+        .collect()
+}
+
+/// default wordlist
+pub(super) fn wordlist() -> String {
+    String::from(DEFAULT_WORDLIST)
+}
+
+/// default user-agent
+pub(super) fn user_agent() -> String {
+    format!("feroxbuster/{}", VERSION)
+}
+
+/// default recursion depth
+pub(super) fn depth() -> usize {
+    4
+}
+
+/// enum representing the three possible states for informational output (not logging verbosity)
+#[derive(Debug, Copy, Clone, PartialEq)]
+pub enum OutputLevel {
+    /// normal scan, no --quiet|--silent
+    Default,
+
+    /// quiet scan, print some information, but not all (new in versions >= 2.0.0)
+    Quiet,
+
+    /// silent scan, only print urls (used to be --quiet in versions 1.x.x)
+    Silent,
+}
+
+/// implement a default for OutputLevel
+impl Default for OutputLevel {
+    /// return Default
+    fn default() -> Self {
+        Self::Default
+    }
+}
+
+/// given the current settings for quiet and silent, determine output_level (DRY helper)
+pub fn determine_output_level(quiet: bool, silent: bool) -> OutputLevel {
+    if quiet && silent {
+        // user COULD have both as true in config file, take the more quiet of the two
+        OutputLevel::Silent
+    } else if quiet {
+        OutputLevel::Quiet
+    } else if silent {
+        OutputLevel::Silent
+    } else {
+        OutputLevel::Default
+    }
+}
+
+/// represents actions the Requester should take in certain situations
+#[derive(Debug, PartialEq, Copy, Clone)]
+pub enum RequesterPolicy {
+    /// automatically try to lower request rate in order to reduce errors
+    AutoTune,
+
+    /// automatically bail at certain error thresholds
+    AutoBail,
+
+    /// just let that junk run super natural
+    Default,
+}
+
+/// default implementation for RequesterPolicy
+impl Default for RequesterPolicy {
+    /// Default as default
+    fn default() -> Self {
+        Self::Default
+    }
+}
+
+/// given the current settings for quiet and silent, determine output_level (DRY helper)
+pub fn determine_requester_policy(auto_tune: bool, auto_bail: bool) -> RequesterPolicy {
+    if auto_tune && auto_bail {
+        // user COULD have both as true in config file, take the more aggressive of the two
+        RequesterPolicy::AutoBail
+    } else if auto_tune {
+        RequesterPolicy::AutoTune
+    } else if auto_bail {
+        RequesterPolicy::AutoBail
+    } else {
+        RequesterPolicy::Default
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    /// test determine_output_level returns higher of the two levels if both given values are true
+    fn determine_output_level_returns_correct_results() {
+        let mut level = determine_output_level(true, true);
+        assert_eq!(level, OutputLevel::Silent);
+
+        level = determine_output_level(false, true);
+        assert_eq!(level, OutputLevel::Silent);
+
+        level = determine_output_level(false, false);
+        assert_eq!(level, OutputLevel::Default);
+
+        level = determine_output_level(true, false);
+        assert_eq!(level, OutputLevel::Quiet);
+    }
+
+    #[test]
+    /// test determine_requester_policy returns higher of the two levels if both given values are true
+    fn determine_requester_policy_returns_correct_results() {
+        let mut level = determine_requester_policy(true, true);
+        assert_eq!(level, RequesterPolicy::AutoBail);
+
+        level = determine_requester_policy(false, true);
+        assert_eq!(level, RequesterPolicy::AutoBail);
+
+        level = determine_requester_policy(false, false);
+        assert_eq!(level, RequesterPolicy::Default);
+
+        level = determine_requester_policy(true, false);
+        assert_eq!(level, RequesterPolicy::AutoTune);
+    }
+
+    #[test]
+    #[should_panic]
+    /// report_and_exit should panic/exit when called
+    fn report_and_exit_panics_under_test() {
+        report_and_exit("test");
+    }
+}

src/event_handlers/command.rs

@@ -0,0 +1,69 @@
+use std::collections::HashSet;
+use std::sync::Arc;
+
+use reqwest::StatusCode;
+use tokio::sync::oneshot::Sender;
+
+use crate::response::FeroxResponse;
+use crate::{
+    statistics::{StatError, StatField},
+    traits::FeroxFilter,
+};
+
+/// Protocol definition for updating an event handler via mpsc
+#[derive(Debug)]
+pub enum Command {
+    /// Add one to the total number of requests
+    AddRequest,
+
+    /// Add one to the proper field(s) based on the given `StatError`
+    AddError(StatError),
+
+    /// Add one to the proper field(s) based on the given `StatusCode`
+    AddStatus(StatusCode),
+
+    /// Create the progress bar (`BarType::Total`) that is updated from the stats thread
+    CreateBar,
+
+    /// Add to a `Stats` field that corresponds to the given `StatField` by the given `usize` value
+    AddToUsizeField(StatField, usize),
+
+    /// Subtract from a `Stats` field that corresponds to the given `StatField` by the given `usize` value
+    SubtractFromUsizeField(StatField, usize),
+
+    /// Update a `Stats` field that corresponds to the given `StatField` by the given `f64` value
+    AddToF64Field(StatField, f64),
+
+    /// Save a `Stats` object to disk using `reporter::get_cached_file_handle`
+    Save,
+
+    /// Load a `Stats` object from disk
+    LoadStats(String),
+
+    /// Add a `FeroxFilter` implementor to `FilterHandler`'s instance of `FeroxFilters`
+    AddFilter(Box<dyn FeroxFilter>),
+
+    /// Send a `FeroxResponse` to the output handler for reporting
+    Report(Box<FeroxResponse>),
+
+    /// Send a group of urls to be scanned (only used for the urls passed in explicitly by the user)
+    ScanInitialUrls(Vec<String>),
+
+    /// Determine whether or not recursion is appropriate, given a FeroxResponse, if so start a scan
+    TryRecursion(Box<FeroxResponse>),
+
+    /// Send a pointer to the wordlist to the recursion handler
+    UpdateWordlist(Arc<HashSet<String>>),
+
+    /// Instruct the ScanHandler to join on all known scans, use sender to notify main when done
+    JoinTasks(Sender<bool>),
+
+    /// Command used to test that a spawned task succeeded in initialization
+    Ping,
+
+    /// Just receive a sender and reply, used for slowing down the main thread
+    Sync(Sender<bool>),
+
+    /// Break out of the (infinite) mpsc receive loop
+    Exit,
+}

src/event_handlers/container.rs

@@ -0,0 +1,135 @@
+use super::*;
+use crate::config::Configuration;
+use crate::event_handlers::scans::ScanHandle;
+use crate::scan_manager::FeroxScans;
+use crate::Joiner;
+#[cfg(test)]
+use crate::{filters::FeroxFilters, statistics::Stats, Command};
+use anyhow::{bail, Result};
+use std::sync::{Arc, RwLock};
+#[cfg(test)]
+use tokio::sync::mpsc::{self, UnboundedReceiver};
+
+#[derive(Debug)]
+/// Simple container for multiple JoinHandles
+pub struct Tasks {
+    /// JoinHandle for terminal handler
+    pub terminal: Joiner,
+
+    /// JoinHandle for statistics handler
+    pub stats: Joiner,
+
+    /// JoinHandle for filters handler
+    pub filters: Joiner,
+
+    /// JoinHandle for scans handler
+    pub scans: Joiner,
+}
+
+/// Tasks implementation
+impl Tasks {
+    /// Given JoinHandles for terminal, statistics, and filters create a new Tasks object
+    pub fn new(terminal: Joiner, stats: Joiner, filters: Joiner, scans: Joiner) -> Self {
+        Self {
+            terminal,
+            stats,
+            filters,
+            scans,
+        }
+    }
+}
+
+#[derive(Debug)]
+/// Container for the different *Handles that will be shared across modules
+pub struct Handles {
+    /// Handle for statistics
+    pub stats: StatsHandle,
+
+    /// Handle for filters
+    pub filters: FiltersHandle,
+
+    /// Handle for output (terminal/file)
+    pub output: TermOutHandle,
+
+    /// Handle for Configuration
+    pub config: Arc<Configuration>,
+
+    /// Handle for recursion
+    pub scans: RwLock<Option<ScanHandle>>,
+}
+
+/// implementation of Handles
+impl Handles {
+    /// Given a StatsHandle, FiltersHandle, and OutputHandle, create a Handles object
+    pub fn new(
+        stats: StatsHandle,
+        filters: FiltersHandle,
+        output: TermOutHandle,
+        config: Arc<Configuration>,
+    ) -> Self {
+        Self {
+            stats,
+            filters,
+            output,
+            config,
+            scans: RwLock::new(None),
+        }
+    }
+
+    /// create a Handles object suitable for unit testing (non-functional)
+    #[cfg(test)]
+    pub fn for_testing(
+        scanned_urls: Option<Arc<FeroxScans>>,
+        config: Option<Arc<Configuration>>,
+    ) -> (Self, UnboundedReceiver<Command>) {
+        let configuration = config.unwrap_or_else(|| Arc::new(Configuration::new().unwrap()));
+        let (tx, rx) = mpsc::unbounded_channel::<Command>();
+        let terminal_handle = TermOutHandle::new(tx.clone(), tx.clone());
+        let stats_handle = StatsHandle::new(
+            Arc::new(Stats::new(
+                configuration.extensions.len(),
+                configuration.json,
+            )),
+            tx.clone(),
+        );
+        let filters_handle = FiltersHandle::new(Arc::new(FeroxFilters::default()), tx.clone());
+        let handles = Self::new(stats_handle, filters_handle, terminal_handle, configuration);
+        if let Some(sh) = scanned_urls {
+            let scan_handle = ScanHandle::new(sh, tx);
+            handles.set_scan_handle(scan_handle);
+        }
+        (handles, rx)
+    }
+
+    /// Set the ScanHandle object
+    pub fn set_scan_handle(&self, handle: ScanHandle) {
+        if let Ok(mut guard) = self.scans.write() {
+            if guard.is_none() {
+                let _ = std::mem::replace(&mut *guard, Some(handle));
+            }
+        }
+    }
+
+    /// Helper to easily send a Command over the (locked) underlying CommandSender object
+    pub fn send_scan_command(&self, command: Command) -> Result<()> {
+        if let Ok(guard) = self.scans.read().as_ref() {
+            if let Some(handle) = guard.as_ref() {
+                handle.send(command)?;
+                return Ok(());
+            }
+        }
+
+        bail!("Could not get underlying CommandSender object")
+    }
+
+    /// Helper to easily get the (locked) underlying FeroxScans object
+    pub fn ferox_scans(&self) -> Result<Arc<FeroxScans>> {
+        if let Ok(guard) = self.scans.read().as_ref() {
+            if let Some(handle) = guard.as_ref() {
+                return Ok(handle.data.clone());
+            }
+        }
+
+        bail!("Could not get underlying FeroxScans")
+    }
+}

src/event_handlers/filters.rs

@@ -0,0 +1,101 @@
+use super::*;
+use crate::{filters::FeroxFilters, CommandSender, FeroxChannel, Joiner};
+use anyhow::Result;
+use std::sync::Arc;
+use tokio::sync::{
+    mpsc::{self, UnboundedReceiver},
+    oneshot,
+};
+
+#[derive(Debug)]
+/// Container for filters transmitter and FeroxFilters object
+pub struct FiltersHandle {
+    /// FeroxFilters object used across modules to track active filters
+    pub data: Arc<FeroxFilters>,
+
+    /// transmitter used to update `data`
+    pub tx: CommandSender,
+}
+
+/// implementation of FiltersHandle
+impl FiltersHandle {
+    /// Given an Arc-wrapped FeroxFilters and CommandSender, create a new FiltersHandle
+    pub fn new(data: Arc<FeroxFilters>, tx: CommandSender) -> Self {
+        Self { data, tx }
+    }
+
+    /// Send the given Command over `tx`
+    pub fn send(&self, command: Command) -> Result<()> {
+        self.tx.send(command)?;
+        Ok(())
+    }
+
+    /// Sync the handle with the handler
+    pub async fn sync(&self) -> Result<()> {
+        let (tx, rx) = oneshot::channel::<bool>();
+        self.send(Command::Sync(tx))?;
+        rx.await?;
+        Ok(())
+    }
+}
+
+/// event handler for updating a single data structure of all active filters
+#[derive(Debug)]
+pub struct FiltersHandler {
+    /// collection of FeroxFilters
+    data: Arc<FeroxFilters>,
+
+    /// Receiver half of mpsc from which `Command`s are processed
+    receiver: UnboundedReceiver<Command>,
+}
+
+/// implementation of event handler for filters
+impl FiltersHandler {
+    /// create new event handler
+    pub fn new(data: Arc<FeroxFilters>, receiver: UnboundedReceiver<Command>) -> Self {
+        Self { data, receiver }
+    }
+
+    /// Initialize new `FeroxFilters` and the sc side of an mpsc channel that is responsible for
+    /// updates to the aforementioned object.
+    pub fn initialize() -> (Joiner, FiltersHandle) {
+        log::trace!("enter: initialize");
+
+        let data = Arc::new(FeroxFilters::default());
+        let (tx, rx): FeroxChannel<Command> = mpsc::unbounded_channel();
+
+        let mut handler = Self::new(data.clone(), rx);
+
+        let task = tokio::spawn(async move { handler.start().await });
+
+        let event_handle = FiltersHandle::new(data, tx);
+
+        log::trace!("exit: initialize -> ({:?}, {:?})", task, event_handle);
+
+        (task, event_handle)
+    }
+
+    /// Start a single consumer task (sc side of mpsc)
+    ///
+    /// The consumer simply receives `Command` and acts accordingly
+    pub async fn start(&mut self) -> Result<()> {
+        log::trace!("enter: start({:?})", self);
+
+        while let Some(command) = self.receiver.recv().await {
+            match command {
+                Command::AddFilter(filter) => {
+                    self.data.push(filter)?;
+                }
+                Command::Sync(sender) => {
+                    log::debug!("filters: {:?}", self);
+                    sender.send(true).unwrap_or_default();
+                }
+                Command::Exit => break,
+                _ => {} // no other commands needed for FilterHandler
+            }
+        }
+
+        log::trace!("exit: start");
+        Ok(())
+    }
+}

src/event_handlers/inputs.rs

@@ -0,0 +1,152 @@
+use super::*;
+use crate::{
+    progress::PROGRESS_PRINTER,
+    scan_manager::{FeroxState, PAUSE_SCAN},
+    scanner::RESPONSES,
+    statistics::StatError,
+    utils::{open_file, write_to},
+    SLEEP_DURATION,
+};
+use anyhow::Result;
+use console::style;
+use crossterm::event::{self, Event, KeyCode};
+use std::{
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        Arc,
+    },
+    thread::sleep,
+    time::Duration,
+    time::{SystemTime, UNIX_EPOCH},
+};
+
+/// Atomic boolean flag, used to determine whether or not the terminal input handler should exit
+pub static SCAN_COMPLETE: AtomicBool = AtomicBool::new(false);
+
+/// Container for filters transmitter and FeroxFilters object
+pub struct TermInputHandler {
+    /// handles to other handlers
+    handles: Arc<Handles>,
+}
+
+/// implementation of event handler for terminal input
+///
+/// kicks off the following handlers related to terminal input:
+///     ctrl+c handler that saves scan state to disk
+///     enter handler that listens for enter during scans to drop into interactive scan cancel menu
+impl TermInputHandler {
+    /// Create new event handler
+    pub fn new(handles: Arc<Handles>) -> Self {
+        Self { handles }
+    }
+
+    /// Initialize the sigint and enter handlers that are responsible for handling initial user
+    /// interaction during scans
+    pub fn initialize(handles: Arc<Handles>) {
+        log::trace!("enter: initialize({:?})", handles);
+
+        let handler = Self::new(handles);
+        handler.start();
+
+        log::trace!("exit: initialize");
+    }
+
+    /// wrapper around sigint_handler and enter_handler
+    fn start(&self) {
+        tokio::task::spawn_blocking(Self::enter_handler);
+
+        if self.handles.config.save_state {
+            // start the ctrl+c handler
+            let cloned = self.handles.clone();
+
+            let result = ctrlc::set_handler(move || {
+                let _ = Self::sigint_handler(cloned.clone());
+            });
+
+            if result.is_err() {
+                log::warn!("Could not set Ctrl+c handler; scan state will not be saved");
+                self.handles
+                    .stats
+                    .send(Command::AddError(StatError::Other))
+                    .unwrap_or_default();
+            }
+        }
+    }
+
+    /// Writes the current state of the program to disk (if save_state is true) and then exits
+    pub fn sigint_handler(handles: Arc<Handles>) -> Result<()> {
+        log::trace!("enter: sigint_handler({:?})", handles);
+
+        let ts = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs();
+
+        let slug = if !handles.config.target_url.is_empty() {
+            // target url populated
+            handles
+                .config
+                .target_url
+                .replace("://", "_")
+                .replace("/", "_")
+                .replace(".", "_")
+        } else {
+            // stdin used
+            "stdin".to_string()
+        };
+
+        let filename = format!("ferox-{}-{}.state", slug, ts);
+        let warning = format!(
+            "🚨 Caught {} 🚨 saving scan state to {} ...",
+            style("ctrl+c").yellow(),
+            filename
+        );
+
+        PROGRESS_PRINTER.println(warning);
+
+        let state = FeroxState::new(
+            handles.ferox_scans()?,
+            handles.config.clone(),
+            &RESPONSES,
+            handles.stats.data.clone(),
+        );
+
+        let state_file = open_file(&filename);
+
+        let mut buffered_file = state_file?;
+        write_to(&state, &mut buffered_file, true)?;
+
+        log::trace!("exit: sigint_handler (end of program)");
+        std::process::exit(1);
+    }
+
+    /// Handles specific key events triggered by the user over stdin
+    fn enter_handler() {
+        // todo eventually move away from atomics, the blocking recv is the problem
+        log::trace!("enter: start_enter_handler");
+
+        loop {
+            if PAUSE_SCAN.load(Ordering::Relaxed) {
+                // if the scan is already paused, we don't want this event poller fighting the user
+                // over stdin
+                sleep(Duration::from_millis(SLEEP_DURATION));
+            } else if event::poll(Duration::from_millis(SLEEP_DURATION)).unwrap_or(false) {
+                // It's guaranteed that the `read()` won't block when the `poll()`
+                // function returns `true`
+
+                if let Ok(key_pressed) = event::read() {
+                    // ignore any other keys
+                    if key_pressed == Event::Key(KeyCode::Enter.into()) {
+                        // if the user presses Enter, set PAUSE_SCAN to true. The interactive menu
+                        // will be triggered and will handle setting PAUSE_SCAN to false
+                        PAUSE_SCAN.store(true, Ordering::Release);
+                    }
+                }
+            } else {
+                // Timeout expired and no `Event` is available; use the timeout to check SCAN_COMPLETE
+                if SCAN_COMPLETE.load(Ordering::Relaxed) {
+                    // scan has been marked complete by main, time to exit the loop
+                    break;
+                }
+            }
+        }
+        log::trace!("exit: start_enter_handler");
+    }
+}

src/event_handlers/mod.rs

@@ -0,0 +1,16 @@
+//! collection of event handlers (typically long-running tokio spawned tasks)
+mod statistics;
+mod filters;
+mod container;
+mod command;
+mod outputs;
+mod scans;
+mod inputs;
+
+pub use self::command::Command;
+pub use self::container::{Handles, Tasks};
+pub use self::filters::{FiltersHandle, FiltersHandler};
+pub use self::inputs::{TermInputHandler, SCAN_COMPLETE};
+pub use self::outputs::{TermOutHandle, TermOutHandler};
+pub use self::scans::{ScanHandle, ScanHandler};
+pub use self::statistics::{StatsHandle, StatsHandler};

src/event_handlers/outputs.rs

@@ -0,0 +1,286 @@
+use super::Command::AddToUsizeField;
+use super::*;
+
+use anyhow::{Context, Result};
+use tokio::sync::{mpsc, oneshot};
+
+use crate::{
+    config::Configuration,
+    progress::PROGRESS_PRINTER,
+    scanner::RESPONSES,
+    send_command, skip_fail,
+    statistics::StatField::ResourcesDiscovered,
+    traits::FeroxSerialize,
+    utils::{ferox_print, fmt_err, make_request, open_file, write_to},
+    CommandReceiver, CommandSender, Joiner,
+};
+use std::sync::Arc;
+
+#[derive(Debug)]
+/// Container for terminal output transmitter
+pub struct TermOutHandle {
+    /// Transmitter that sends to the TermOutHandler handler
+    pub tx: CommandSender,
+
+    /// Transmitter that sends to the FileOutHandler handler
+    pub tx_file: CommandSender,
+}
+
+/// implementation of OutputHandle
+impl TermOutHandle {
+    /// Given a CommandSender, create a new OutputHandle
+    pub fn new(tx: CommandSender, tx_file: CommandSender) -> Self {
+        Self { tx, tx_file }
+    }
+
+    /// Send the given Command over `tx`
+    pub fn send(&self, command: Command) -> Result<()> {
+        self.tx.send(command)?;
+        Ok(())
+    }
+
+    /// Sync the handle with the handler
+    pub async fn sync(&self, send_to_file: bool) -> Result<()> {
+        let (tx, rx) = oneshot::channel::<bool>();
+        self.send(Command::Sync(tx))?;
+
+        if send_to_file {
+            let (tx, rx) = oneshot::channel::<bool>();
+            self.tx_file.send(Command::Sync(tx))?;
+            rx.await?;
+        }
+
+        rx.await?;
+        Ok(())
+    }
+}
+
+#[derive(Debug)]
+/// Event handler for files
+pub struct FileOutHandler {
+    /// file output handler's receiver
+    receiver: CommandReceiver,
+
+    /// pointer to "global" configuration struct
+    config: Arc<Configuration>,
+}
+
+impl FileOutHandler {
+    /// Given a file tx/rx pair along with a filename and awaitable task, create
+    /// a FileOutHandler
+    fn new(rx: CommandReceiver, config: Arc<Configuration>) -> Self {
+        Self {
+            receiver: rx,
+            config,
+        }
+    }
+
+    /// Spawn a single consumer task (sc side of mpsc)
+    ///
+    /// The consumer simply receives responses from the terminal handler and writes them to disk
+    async fn start(&mut self, tx_stats: CommandSender) -> Result<()> {
+        log::trace!("enter: start_file_handler({:?})", tx_stats);
+
+        let mut file = open_file(&self.config.output)?;
+
+        log::info!("Writing scan results to {}", self.config.output);
+
+        while let Some(command) = self.receiver.recv().await {
+            match command {
+                Command::Report(response) => {
+                    skip_fail!(write_to(&*response, &mut file, self.config.json));
+                }
+                Command::Exit => {
+                    break;
+                }
+                Command::Sync(sender) => {
+                    skip_fail!(sender.send(true));
+                }
+                _ => {} // no more needed
+            }
+        }
+
+        // close the file before we tell statistics to save current data to the same file
+        drop(file);
+
+        send_command!(tx_stats, Command::Save);
+
+        log::trace!("exit: start_file_handler");
+        Ok(())
+    }
+}
+
+#[derive(Debug)]
+/// Event handler for terminal
+pub struct TermOutHandler {
+    /// terminal output handler's receiver
+    receiver: CommandReceiver,
+
+    /// file handler
+    tx_file: CommandSender,
+
+    /// optional file handler task
+    file_task: Option<Joiner>,
+
+    /// pointer to "global" configuration struct
+    config: Arc<Configuration>,
+}
+
+/// implementation of TermOutHandler
+impl TermOutHandler {
+    /// Given a terminal receiver along with a file transmitter and filename, create
+    /// an OutputHandler
+    fn new(
+        receiver: CommandReceiver,
+        tx_file: CommandSender,
+        file_task: Option<Joiner>,
+        config: Arc<Configuration>,
+    ) -> Self {
+        Self {
+            receiver,
+            tx_file,
+            config,
+            file_task,
+        }
+    }
+
+    /// Creates all required output handlers (terminal, file) and updates the given Handles/Tasks
+    pub fn initialize(
+        config: Arc<Configuration>,
+        tx_stats: CommandSender,
+    ) -> (Joiner, TermOutHandle) {
+        log::trace!("enter: initialize({:?}, {:?})", config, tx_stats);
+
+        let (tx_term, rx_term) = mpsc::unbounded_channel::<Command>();
+        let (tx_file, rx_file) = mpsc::unbounded_channel::<Command>();
+
+        let mut file_handler = FileOutHandler::new(rx_file, config.clone());
+
+        let tx_stats_clone = tx_stats.clone();
+
+        let file_task = if !config.output.is_empty() {
+            // -o used, need to spawn the thread for writing to disk
+            Some(tokio::spawn(async move {
+                file_handler.start(tx_stats_clone).await
+            }))
+        } else {
+            None
+        };
+
+        let mut term_handler = Self::new(rx_term, tx_file.clone(), file_task, config);
+        let term_task = tokio::spawn(async move { term_handler.start(tx_stats).await });
+
+        let event_handle = TermOutHandle::new(tx_term, tx_file);
+
+        log::trace!("exit: initialize -> ({:?}, {:?})", term_task, event_handle);
+
+        (term_task, event_handle)
+    }
+
+    /// Start a single consumer task (sc side of mpsc)
+    ///
+    /// The consumer simply receives `Command` and acts accordingly
+    async fn start(&mut self, tx_stats: CommandSender) -> Result<()> {
+        log::trace!("enter: start({:?})", tx_stats);
+
+        while let Some(command) = self.receiver.recv().await {
+            match command {
+                Command::Report(mut resp) => {
+                    let contains_sentry =
+                        self.config.status_codes.contains(&resp.status().as_u16());
+                    let unknown_sentry = !RESPONSES.contains(&resp); // !contains == unknown
+                    let should_process_response = contains_sentry && unknown_sentry;
+
+                    if should_process_response {
+                        // print to stdout
+                        ferox_print(&resp.as_str(), &PROGRESS_PRINTER);
+
+                        send_command!(tx_stats, AddToUsizeField(ResourcesDiscovered, 1));
+
+                        if self.file_task.is_some() {
+                            // -o used, need to send the report to be written out to disk
+                            self.tx_file
+                                .send(Command::Report(resp.clone()))
+                                .with_context(|| {
+                                    fmt_err(&format!("Could not send {} to file handler", resp))
+                                })?;
+                        }
+                    }
+                    log::trace!("report complete: {}", resp.url());
+
+                    if self.config.replay_client.is_some() && should_process_response {
+                        // replay proxy specified/client created and this response's status code is one that
+                        // should be replayed; not using logged_request due to replay proxy client
+                        make_request(
+                            self.config.replay_client.as_ref().unwrap(),
+                            &resp.url(),
+                            self.config.output_level,
+                            tx_stats.clone(),
+                        )
+                        .await
+                        .with_context(|| "Could not replay request through replay proxy")?;
+                    }
+
+                    if should_process_response {
+                        // add response to RESPONSES for serialization in case of ctrl+c
+                        // placed all by its lonesome like this so that RESPONSES can take ownership
+                        // of the FeroxResponse
+
+                        // before ownership is transferred, there's no real reason to keep the body anymore
+                        // so we can free that piece of data, reducing memory usage
+                        resp.drop_text();
+
+                        RESPONSES.insert(*resp);
+                    }
+                }
+                Command::Sync(sender) => {
+                    sender.send(true).unwrap_or_default();
+                }
+                Command::Exit => {
+                    if self.file_task.is_some() && self.tx_file.send(Command::Exit).is_ok() {
+                        self.file_task.as_mut().unwrap().await??; // wait for death
+                    }
+                    break;
+                }
+                _ => {} // no more commands needed
+            }
+        }
+        log::trace!("exit: start");
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    /// try to hit struct field coverage of FileOutHandler
+    fn struct_fields_of_file_out_handler() {
+        let (_, rx) = mpsc::unbounded_channel::<Command>();
+        let config = Arc::new(Configuration::new().unwrap());
+        let foh = FileOutHandler {
+            config,
+            receiver: rx,
+        };
+        println!("{:?}", foh);
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// try to hit struct field coverage of TermOutHandler
+    async fn struct_fields_of_term_out_handler() {
+        let (tx, rx) = mpsc::unbounded_channel::<Command>();
+        let (tx_file, _) = mpsc::unbounded_channel::<Command>();
+        let config = Arc::new(Configuration::new().unwrap());
+
+        let toh = TermOutHandler {
+            config,
+            file_task: None,
+            receiver: rx,
+            tx_file,
+        };
+
+        println!("{:?}", toh);
+        tx.send(Command::Exit).unwrap();
+    }
+}

src/event_handlers/scans.rs

@@ -0,0 +1,273 @@
+use std::collections::HashSet;
+use std::sync::Arc;
+
+use anyhow::{bail, Result};
+use tokio::sync::{mpsc, Semaphore};
+
+use crate::response::FeroxResponse;
+use crate::url::FeroxUrl;
+use crate::{
+    scan_manager::{FeroxScan, FeroxScans, ScanOrder},
+    scanner::FeroxScanner,
+    statistics::StatField::TotalScans,
+    CommandReceiver, CommandSender, FeroxChannel, Joiner, SLEEP_DURATION,
+};
+
+use super::command::Command::AddToUsizeField;
+use super::*;
+use tokio::time::Duration;
+
+#[derive(Debug)]
+/// Container for recursion transmitter and FeroxScans object
+pub struct ScanHandle {
+    /// FeroxScans object used across modules to track scans
+    pub data: Arc<FeroxScans>,
+
+    /// transmitter used to update `data`
+    pub tx: CommandSender,
+}
+
+/// implementation of RecursionHandle
+impl ScanHandle {
+    /// Given an Arc-wrapped FeroxScans and CommandSender, create a new RecursionHandle
+    pub fn new(data: Arc<FeroxScans>, tx: CommandSender) -> Self {
+        Self { data, tx }
+    }
+
+    /// Send the given Command over `tx`
+    pub fn send(&self, command: Command) -> Result<()> {
+        self.tx.send(command)?;
+        Ok(())
+    }
+}
+
+/// event handler for updating a single data structure of all FeroxScans
+#[derive(Debug)]
+pub struct ScanHandler {
+    /// collection of FeroxScans
+    data: Arc<FeroxScans>,
+
+    /// handles to other handlers needed to kick off a scan while already past main
+    handles: Arc<Handles>,
+
+    /// Receiver half of mpsc from which `Command`s are processed
+    receiver: CommandReceiver,
+
+    /// wordlist (re)used for each scan
+    wordlist: std::sync::Mutex<Option<Arc<HashSet<String>>>>,
+
+    /// group of scans that need to be joined
+    tasks: Vec<Arc<FeroxScan>>,
+
+    /// Maximum recursion depth, a depth of 0 is infinite recursion
+    max_depth: usize,
+
+    /// depths associated with the initial targets provided by the user
+    depths: Vec<(String, usize)>,
+
+    /// Bounded semaphore used as a barrier to limit concurrent scans
+    limiter: Arc<Semaphore>,
+}
+
+/// implementation of event handler for filters
+impl ScanHandler {
+    /// create new event handler
+    pub fn new(
+        data: Arc<FeroxScans>,
+        handles: Arc<Handles>,
+        max_depth: usize,
+        receiver: CommandReceiver,
+    ) -> Self {
+        let limit = handles.config.scan_limit;
+        let limiter = Semaphore::new(limit);
+
+        if limit == 0 {
+            // scan_limit == 0 means no limit should be imposed... however, scoping the Semaphore
+            // permit is tricky, so as a workaround, we'll add a ridiculous number of permits to
+            // the semaphore (1,152,921,504,606,846,975 to be exact) and call that 'unlimited'
+
+            // note to self: the docs say max is usize::MAX >> 3, however, threads will panic if
+            // that value is used (says adding (1) will overflow the semaphore, even though none
+            // are being added...)
+            limiter.add_permits(usize::MAX >> 4);
+        }
+
+        Self {
+            data,
+            handles,
+            receiver,
+            max_depth,
+            tasks: Vec::new(),
+            depths: Vec::new(),
+            limiter: Arc::new(limiter),
+            wordlist: std::sync::Mutex::new(None),
+        }
+    }
+
+    /// Set the wordlist
+    fn wordlist(&self, wordlist: Arc<HashSet<String>>) {
+        if let Ok(mut guard) = self.wordlist.lock() {
+            if guard.is_none() {
+                let _ = std::mem::replace(&mut *guard, Some(wordlist));
+            }
+        }
+    }
+
+    /// Initialize new `FeroxScans` and the sc side of an mpsc channel that is responsible for
+    /// updates to the aforementioned object.
+    pub fn initialize(handles: Arc<Handles>) -> (Joiner, ScanHandle) {
+        log::trace!("enter: initialize");
+
+        let data = Arc::new(FeroxScans::new(handles.config.output_level));
+        let (tx, rx): FeroxChannel<Command> = mpsc::unbounded_channel();
+
+        let max_depth = handles.config.depth;
+
+        let mut handler = Self::new(data.clone(), handles, max_depth, rx);
+
+        let task = tokio::spawn(async move { handler.start().await });
+
+        let event_handle = ScanHandle::new(data, tx);
+
+        log::trace!("exit: initialize -> ({:?}, {:?})", task, event_handle);
+
+        (task, event_handle)
+    }
+
+    /// Start a single consumer task (sc side of mpsc)
+    ///
+    /// The consumer simply receives `Command` and acts accordingly
+    pub async fn start(&mut self) -> Result<()> {
+        log::trace!("enter: start({:?})", self);
+
+        while let Some(command) = self.receiver.recv().await {
+            match command {
+                Command::ScanInitialUrls(targets) => {
+                    self.ordered_scan_url(targets, ScanOrder::Initial).await?;
+                }
+                Command::UpdateWordlist(wordlist) => {
+                    self.wordlist(wordlist);
+                }
+                Command::JoinTasks(sender) => {
+                    let ferox_scans = self.handles.ferox_scans().unwrap_or_default();
+                    let limiter_clone = self.limiter.clone();
+
+                    tokio::spawn(async move {
+                        while ferox_scans.has_active_scans() {
+                            tokio::time::sleep(Duration::from_millis(SLEEP_DURATION + 250)).await;
+                        }
+                        limiter_clone.close();
+                        sender.send(true).expect("oneshot channel failed");
+                    });
+                }
+                Command::TryRecursion(response) => {
+                    self.try_recursion(response).await?;
+                }
+                Command::Sync(sender) => {
+                    sender.send(true).unwrap_or_default();
+                }
+                _ => {} // no other commands needed for RecursionHandler
+            }
+        }
+
+        log::trace!("exit: start");
+        Ok(())
+    }
+
+    /// Helper to easily get the (locked) underlying wordlist
+    pub fn get_wordlist(&self) -> Result<Arc<HashSet<String>>> {
+        if let Ok(guard) = self.wordlist.lock().as_ref() {
+            if let Some(list) = guard.as_ref() {
+                return Ok(list.clone());
+            }
+        }
+
+        bail!("Could not get underlying wordlist")
+    }
+
+    /// wrapper around scanning a url to stay DRY
+    async fn ordered_scan_url(&mut self, targets: Vec<String>, order: ScanOrder) -> Result<()> {
+        log::trace!("enter: ordered_scan_url({:?}, {:?})", targets, order);
+
+        for target in targets {
+            if self.data.contains(&target) && matches!(order, ScanOrder::Latest) {
+                // FeroxScans knows about this url and scan isn't an Initial scan
+                // initial scans are skipped because when resuming from a .state file, the scans
+                // will already be populated in FeroxScans, so we need to not skip kicking off
+                // their scans
+                continue;
+            }
+
+            let scan = if let Some(ferox_scan) = self.data.get_scan_by_url(&target) {
+                ferox_scan // scan already known
+            } else {
+                self.data.add_directory_scan(&target, order).1 // add the new target; return FeroxScan
+            };
+
+            let list = self.get_wordlist()?;
+
+            log::info!("scan handler received {} - beginning scan", target);
+
+            if matches!(order, ScanOrder::Initial) {
+                // keeps track of the initial targets' scan depths in order to enforce the
+                // maximum recursion depth on any identified sub-directories
+                let url = FeroxUrl::from_string(&target, self.handles.clone());
+                let depth = url.depth().unwrap_or(0);
+                self.depths.push((target.clone(), depth));
+            }
+
+            let scanner = FeroxScanner::new(
+                &target,
+                order,
+                list,
+                self.limiter.clone(),
+                self.handles.clone(),
+            );
+
+            let task = tokio::spawn(async move {
+                if let Err(e) = scanner.scan_url().await {
+                    log::warn!("{}", e);
+                }
+            });
+
+            self.handles.stats.send(AddToUsizeField(TotalScans, 1))?;
+
+            scan.set_task(task).await?;
+
+            self.tasks.push(scan.clone());
+        }
+
+        log::trace!("exit: ordered_scan_url");
+        Ok(())
+    }
+
+    async fn try_recursion(&mut self, response: Box<FeroxResponse>) -> Result<()> {
+        log::trace!("enter: try_recursion({:?})", response,);
+
+        let mut base_depth = 1_usize;
+
+        for (base_url, base_url_depth) in &self.depths {
+            if response.url().as_str().starts_with(base_url) {
+                base_depth = *base_url_depth;
+            }
+        }
+
+        if response.reached_max_depth(base_depth, self.max_depth, self.handles.clone()) {
+            // at or past recursion depth
+            return Ok(());
+        }
+
+        if !response.is_directory() {
+            // not a directory
+            return Ok(());
+        }
+
+        let targets = vec![response.url().to_string()];
+        self.ordered_scan_url(targets, ScanOrder::Latest).await?;
+
+        log::info!("Added new directory to recursive scan: {}", response.url());
+
+        log::trace!("exit: try_recursion");
+        Ok(())
+    }
+}

src/event_handlers/statistics.rs

@@ -0,0 +1,171 @@
+use super::*;
+use crate::{
+    config::Configuration,
+    progress::{add_bar, BarType},
+    statistics::{StatField, Stats},
+    CommandSender, FeroxChannel, Joiner,
+};
+use anyhow::Result;
+use console::style;
+use indicatif::ProgressBar;
+use std::{sync::Arc, time::Instant};
+use tokio::sync::{
+    mpsc::{self, UnboundedReceiver},
+    oneshot,
+};
+
+#[derive(Debug)]
+/// Container for statistics transmitter and Stats object
+pub struct StatsHandle {
+    /// Stats object used across modules to track statistics
+    pub data: Arc<Stats>,
+
+    /// transmitter used to update `data`
+    pub tx: CommandSender,
+}
+
+/// implementation of StatsHandle
+impl StatsHandle {
+    /// Given an Arc-wrapped Stats and CommandSender, create a new StatsHandle
+    pub fn new(data: Arc<Stats>, tx: CommandSender) -> Self {
+        Self { data, tx }
+    }
+
+    /// Send the given Command over `tx`
+    pub fn send(&self, command: Command) -> Result<()> {
+        self.tx.send(command)?;
+        Ok(())
+    }
+
+    /// Sync the handle with the handler
+    pub async fn sync(&self) -> Result<()> {
+        let (tx, rx) = oneshot::channel::<bool>();
+        self.send(Command::Sync(tx))?;
+        rx.await?;
+        Ok(())
+    }
+}
+
+/// event handler struct for updating statistics
+#[derive(Debug)]
+pub struct StatsHandler {
+    /// overall scan's progress bar
+    bar: ProgressBar,
+
+    /// Receiver half of mpsc from which `StatCommand`s are processed
+    receiver: UnboundedReceiver<Command>,
+
+    /// data class that stores all statistics updates
+    stats: Arc<Stats>,
+}
+
+/// implementation of event handler for statistics
+impl StatsHandler {
+    /// create new event handler
+    fn new(stats: Arc<Stats>, rx_stats: UnboundedReceiver<Command>) -> Self {
+        // will be updated later via StatCommand; delay is for banner to print first
+        let bar = ProgressBar::hidden();
+
+        Self {
+            bar,
+            stats,
+            receiver: rx_stats,
+        }
+    }
+
+    /// Start a single consumer task (sc side of mpsc)
+    ///
+    /// The consumer simply receives `StatCommands` and updates the given `Stats` object as appropriate
+    async fn start(&mut self, output_file: &str) -> Result<()> {
+        log::trace!("enter: start({:?})", self);
+
+        let start = Instant::now();
+
+        while let Some(command) = self.receiver.recv().await {
+            match command as Command {
+                Command::AddError(err) => {
+                    self.stats.add_error(err);
+                    self.increment_bar();
+                }
+                Command::AddStatus(status) => {
+                    self.stats.add_status_code(status);
+
+                    self.increment_bar();
+                }
+                Command::AddRequest => {
+                    self.stats.add_request();
+                    self.increment_bar();
+                }
+                Command::Save => {
+                    self.stats
+                        .save(start.elapsed().as_secs_f64(), output_file)?;
+                }
+                Command::AddToUsizeField(field, value) => {
+                    self.stats.update_usize_field(field, value);
+
+                    if matches!(field, StatField::TotalScans) {
+                        self.bar.set_length(self.stats.total_expected() as u64);
+                    }
+                }
+                Command::SubtractFromUsizeField(field, value) => {
+                    self.stats.subtract_from_usize_field(field, value);
+
+                    if matches!(field, StatField::TotalExpected) {
+                        self.bar.set_length(self.stats.total_expected() as u64);
+                    }
+                }
+                Command::AddToF64Field(field, value) => self.stats.update_f64_field(field, value),
+                Command::CreateBar => {
+                    self.bar = add_bar("", self.stats.total_expected() as u64, BarType::Total);
+                }
+                Command::LoadStats(filename) => {
+                    self.stats.merge_from(&filename)?;
+                }
+                Command::Sync(sender) => {
+                    sender.send(true).unwrap_or_default();
+                }
+                Command::Exit => break,
+                _ => {} // no more commands needed
+            }
+        }
+
+        self.bar.finish();
+
+        log::debug!("{:#?}", *self.stats);
+        log::trace!("exit: start");
+        Ok(())
+    }
+
+    /// Wrapper around incrementing the overall scan's progress bar
+    fn increment_bar(&self) {
+        let msg = format!(
+            "{}:{:<7} {}:{:<7}",
+            style("found").green(),
+            self.stats.resources_discovered(),
+            style("errors").red(),
+            self.stats.errors(),
+        );
+
+        self.bar.set_message(&msg);
+        self.bar.inc(1);
+    }
+
+    /// Initialize new `Stats` object and the sc side of an mpsc channel that is responsible for
+    /// updates to the aforementioned object.
+    pub fn initialize(config: Arc<Configuration>) -> (Joiner, StatsHandle) {
+        log::trace!("enter: initialize");
+
+        let data = Arc::new(Stats::new(config.extensions.len(), config.json));
+        let (tx, rx): FeroxChannel<Command> = mpsc::unbounded_channel();
+
+        let mut handler = StatsHandler::new(data.clone(), rx);
+
+        let task = tokio::spawn(async move { handler.start(&config.output).await });
+
+        let event_handle = StatsHandle::new(data, tx);
+
+        log::trace!("exit: initialize -> ({:?}, {:?})", task, event_handle);
+
+        (task, event_handle)
+    }
+}

src/extractor.rs

@@ -1,504 +0,0 @@
-use crate::{
-    client,
-    config::{Configuration, CONFIGURATION},
-    scanner::SCANNED_URLS,
-    statistics::{
-        StatCommand::{self, UpdateUsizeField},
-        StatField::{LinksExtracted, TotalExpected},
-    },
-    utils::{format_url, make_request},
-    FeroxResponse,
-};
-use lazy_static::lazy_static;
-use regex::Regex;
-use reqwest::Url;
-use std::collections::HashSet;
-use tokio::sync::mpsc::UnboundedSender;
-
-/// Regular expression used in [LinkFinder](https://github.com/GerbenJavado/LinkFinder)
-///
-/// Incorporates change from this [Pull Request](https://github.com/GerbenJavado/LinkFinder/pull/66/files)
-const LINKFINDER_REGEX: &str = r#"(?:"|')(((?:[a-zA-Z]{1,10}://|//)[^"'/]{1,}\.[a-zA-Z]{2,}[^"']{0,})|((?:/|\.\./|\./)[^"'><,;| *()(%%$^/\\\[\]][^"'><,;|()]{1,})|([a-zA-Z0-9_\-/]{1,}/[a-zA-Z0-9_\-/]{1,}\.(?:[a-zA-Z]{1,4}|action)(?:[\?|#][^"|']{0,}|))|([a-zA-Z0-9_\-/]{1,}/[a-zA-Z0-9_\-/]{3,}(?:[\?|#][^"|']{0,}|))|([a-zA-Z0-9_\-.]{1,}\.(?:php|asp|aspx|jsp|json|action|html|js|txt|xml)(?:[\?|#][^"|']{0,}|)))(?:"|')"#;
-
-/// Regular expression to pull url paths from robots.txt
-///
-/// ref: https://developers.google.com/search/reference/robots_txt
-const ROBOTS_TXT_REGEX: &str =
-    r#"(?m)^ *(Allow|Disallow): *(?P<url_path>[a-zA-Z0-9._/?#@!&'()+,;%=-]+?)$"#; // multi-line (?m)
-
-lazy_static! {
-    /// `LINKFINDER_REGEX` as a regex::Regex type
-    static ref LINKS_REGEX: Regex = Regex::new(LINKFINDER_REGEX).unwrap();
-
-    /// `ROBOTS_TXT_REGEX` as a regex::Regex type
-    static ref ROBOTS_REGEX: Regex = Regex::new(ROBOTS_TXT_REGEX).unwrap();
-}
-
-/// Iterate over a given path, return a list of every sub-path found
-///
-/// example: `path` contains a link fragment `homepage/assets/img/icons/handshake.svg`
-/// the following fragments would be returned:
-///   - homepage/assets/img/icons/handshake.svg
-///   - homepage/assets/img/icons/
-///   - homepage/assets/img/
-///   - homepage/assets/
-///   - homepage/
-fn get_sub_paths_from_path(path: &str) -> Vec<String> {
-    log::trace!("enter: get_sub_paths_from_path({})", path);
-    let mut paths = vec![];
-
-    // filter out any empty strings caused by .split
-    let mut parts: Vec<&str> = path.split('/').filter(|s| !s.is_empty()).collect();
-
-    let length = parts.len();
-
-    for i in 0..length {
-        // iterate over all parts of the path
-        if parts.is_empty() {
-            // pop left us with an empty vector, we're done
-            break;
-        }
-
-        let mut possible_path = parts.join("/");
-
-        if possible_path.is_empty() {
-            // .join can result in an empty string, which we don't need, ignore
-            continue;
-        }
-
-        if i > 0 {
-            // this isn't the last index of the parts array
-            // ex: /buried/misc/stupidfile.php
-            // this block skips the file but sees all parent folders
-            possible_path = format!("{}/", possible_path);
-        }
-
-        paths.push(possible_path); // good sub-path found
-        parts.pop(); // use .pop() to remove the last part of the path and continue iteration
-    }
-
-    log::trace!("exit: get_sub_paths_from_path -> {:?}", paths);
-    paths
-}
-
-/// simple helper to stay DRY, trys to join a url + fragment and add it to the `links` HashSet
-fn add_link_to_set_of_links(link: &str, url: &Url, links: &mut HashSet<String>) {
-    log::trace!(
-        "enter: add_link_to_set_of_links({}, {}, {:?})",
-        link,
-        url.to_string(),
-        links
-    );
-    match url.join(&link) {
-        Ok(new_url) => {
-            links.insert(new_url.to_string());
-        }
-        Err(e) => {
-            log::error!("Could not join given url to the base url: {}", e);
-        }
-    }
-    log::trace!("exit: add_link_to_set_of_links");
-}
-
-/// Given a `reqwest::Response`, perform the following actions
-///   - parse the response's text for links using the linkfinder regex
-///   - for every link found take its url path and parse each sub-path
-///     - example: Response contains a link fragment `homepage/assets/img/icons/handshake.svg`
-///       with a base url of http://localhost, the following urls would be returned:
-///         - homepage/assets/img/icons/handshake.svg
-///         - homepage/assets/img/icons/
-///         - homepage/assets/img/
-///         - homepage/assets/
-///         - homepage/
-pub async fn get_links(
-    response: &FeroxResponse,
-    tx_stats: UnboundedSender<StatCommand>,
-) -> HashSet<String> {
-    log::trace!(
-        "enter: get_links({}, {:?})",
-        response.url().as_str(),
-        tx_stats
-    );
-
-    let mut links = HashSet::<String>::new();
-
-    let body = response.text();
-
-    for capture in LINKS_REGEX.captures_iter(&body) {
-        // remove single & double quotes from both ends of the capture
-        // capture[0] is the entire match, additional capture groups start at [1]
-        let link = capture[0].trim_matches(|c| c == '\'' || c == '"');
-
-        match Url::parse(link) {
-            Ok(absolute) => {
-                if absolute.domain() != response.url().domain()
-                    || absolute.host() != response.url().host()
-                {
-                    // domains/ips are not the same, don't scan things that aren't part of the original
-                    // target url
-                    continue;
-                }
-
-                add_all_sub_paths(absolute.path(), &response, &mut links);
-            }
-            Err(e) => {
-                // this is the expected error that happens when we try to parse a url fragment
-                //     ex: Url::parse("/login") -> Err("relative URL without a base")
-                // while this is technically an error, these are good results for us
-                if e.to_string().contains("relative URL without a base") {
-                    add_all_sub_paths(link, &response, &mut links);
-                } else {
-                    // unexpected error has occurred
-                    log::error!("Could not parse given url: {}", e);
-                }
-            }
-        }
-    }
-
-    let multiplier = CONFIGURATION.extensions.len().max(1);
-
-    update_stat!(tx_stats, UpdateUsizeField(LinksExtracted, links.len()));
-    update_stat!(
-        tx_stats,
-        UpdateUsizeField(TotalExpected, links.len() * multiplier)
-    );
-
-    log::trace!("exit: get_links -> {:?}", links);
-
-    links
-}
-
-/// take a url fragment like homepage/assets/img/icons/handshake.svg and
-/// incrementally add
-///     - homepage/assets/img/icons/
-///     - homepage/assets/img/
-///     - homepage/assets/
-///     - homepage/
-fn add_all_sub_paths(url_path: &str, response: &FeroxResponse, mut links: &mut HashSet<String>) {
-    log::trace!(
-        "enter: add_all_sub_paths({}, {}, {:?})",
-        url_path,
-        response,
-        links
-    );
-
-    for sub_path in get_sub_paths_from_path(url_path) {
-        log::debug!("Adding {} to {:?}", sub_path, links);
-        add_link_to_set_of_links(&sub_path, &response.url(), &mut links);
-    }
-
-    log::trace!("exit: add_all_sub_paths");
-}
-
-/// Wrapper around link extraction logic
-/// currently used in two places:
-///   - links from response bodys
-///   - links from robots.txt responses
-///
-/// general steps taken:
-///   - create a new Url object based on cli options/args
-///   - check if the new Url has already been seen/scanned -> None
-///   - make a request to the new Url ? -> Some(response) : None
-pub async fn request_feroxresponse_from_new_link(
-    url: &str,
-    tx_stats: UnboundedSender<StatCommand>,
-) -> Option<FeroxResponse> {
-    log::trace!(
-        "enter: request_feroxresponse_from_new_link({}, {:?})",
-        url,
-        tx_stats
-    );
-
-    // create a url based on the given command line options, return None on error
-    let new_url = match format_url(
-        &url,
-        &"",
-        CONFIGURATION.add_slash,
-        &CONFIGURATION.queries,
-        None,
-        tx_stats.clone(),
-    ) {
-        Ok(url) => url,
-        Err(_) => {
-            log::trace!("exit: request_feroxresponse_from_new_link -> None");
-            return None;
-        }
-    };
-
-    if SCANNED_URLS.get_scan_by_url(&new_url.to_string()).is_some() {
-        //we've seen the url before and don't need to scan again
-        log::trace!("exit: request_feroxresponse_from_new_link -> None");
-        return None;
-    }
-
-    // make the request and store the response
-    let new_response = match make_request(&CONFIGURATION.client, &new_url, tx_stats).await {
-        Ok(resp) => resp,
-        Err(_) => {
-            log::trace!("exit: request_feroxresponse_from_new_link -> None");
-            return None;
-        }
-    };
-
-    let new_ferox_response = FeroxResponse::from(new_response, true).await;
-
-    log::trace!(
-        "exit: request_feroxresponse_from_new_link -> {:?}",
-        new_ferox_response
-    );
-    Some(new_ferox_response)
-}
-
-/// helper function that simply requests /robots.txt on the given url's base url
-///
-/// example:
-///     http://localhost/api/users -> http://localhost/robots.txt
-///     
-/// The length of the given path has no effect on what's requested; it's always
-/// base url + /robots.txt
-pub async fn request_robots_txt(
-    base_url: &str,
-    config: &Configuration,
-    tx_stats: UnboundedSender<StatCommand>,
-) -> Option<FeroxResponse> {
-    log::trace!(
-        "enter: get_robots_file({}, CONFIGURATION, {:?})",
-        base_url,
-        tx_stats
-    );
-
-    // more often than not, domain/robots.txt will redirect to www.domain/robots.txt or something
-    // similar; to account for that, create a client that will follow redirects, regardless of
-    // what the user specified for the scanning client. Other than redirects, it will respect
-    // all other user specified settings
-    let follow_redirects = true;
-
-    let proxy = if config.proxy.is_empty() {
-        None
-    } else {
-        Some(config.proxy.as_str())
-    };
-
-    let client = client::initialize(
-        config.timeout,
-        &config.user_agent,
-        follow_redirects,
-        config.insecure,
-        &config.headers,
-        proxy,
-    );
-
-    if let Ok(mut url) = Url::parse(base_url) {
-        url.set_path("/robots.txt"); // overwrite existing path with /robots.txt
-
-        if let Ok(response) = make_request(&client, &url, tx_stats).await {
-            let ferox_response = FeroxResponse::from(response, true).await;
-
-            log::trace!("exit: get_robots_file -> {}", ferox_response);
-            return Some(ferox_response);
-        }
-    }
-
-    None
-}
-
-/// Entry point to perform link extraction from robots.txt
-///
-/// `base_url` can have paths and subpaths, however robots.txt will be requested from the
-/// root of the url
-/// given the url:
-///     http://localhost/stuff/things
-/// this function requests:
-///     http://localhost/robots.txt
-pub async fn extract_robots_txt(
-    base_url: &str,
-    config: &Configuration,
-    tx_stats: UnboundedSender<StatCommand>,
-) -> HashSet<String> {
-    log::trace!(
-        "enter: extract_robots_txt({}, CONFIGURATION, {:?})",
-        base_url,
-        tx_stats
-    );
-    let mut links = HashSet::new();
-
-    if let Some(response) = request_robots_txt(&base_url, &config, tx_stats.clone()).await {
-        for capture in ROBOTS_REGEX.captures_iter(response.text.as_str()) {
-            if let Some(new_path) = capture.name("url_path") {
-                if let Ok(mut new_url) = Url::parse(base_url) {
-                    new_url.set_path(new_path.as_str());
-                    add_all_sub_paths(new_url.path(), &response, &mut links);
-                }
-            }
-        }
-    }
-
-    let multiplier = CONFIGURATION.extensions.len().max(1);
-
-    update_stat!(tx_stats, UpdateUsizeField(LinksExtracted, links.len()));
-    update_stat!(
-        tx_stats,
-        UpdateUsizeField(TotalExpected, links.len() * multiplier)
-    );
-
-    log::trace!("exit: extract_robots_txt -> {:?}", links);
-    links
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::utils::make_request;
-    use crate::FeroxChannel;
-    use httpmock::Method::GET;
-    use httpmock::MockServer;
-    use reqwest::Client;
-    use tokio::sync::mpsc;
-
-    #[test]
-    /// extract sub paths from the given url fragment; expect 4 sub paths and that all are
-    /// in the expected array
-    fn extractor_get_sub_paths_from_path_with_multiple_paths() {
-        let path = "homepage/assets/img/icons/handshake.svg";
-        let paths = get_sub_paths_from_path(&path);
-        let expected = vec![
-            "homepage/",
-            "homepage/assets/",
-            "homepage/assets/img/",
-            "homepage/assets/img/icons/",
-            "homepage/assets/img/icons/handshake.svg",
-        ];
-
-        assert_eq!(paths.len(), expected.len());
-        for expected_path in expected {
-            assert_eq!(paths.contains(&expected_path.to_string()), true);
-        }
-    }
-
-    #[test]
-    /// extract sub paths from the given url fragment; expect 2 sub paths and that all are
-    /// in the expected array. the fragment is wrapped in slashes to ensure no empty strings are
-    /// returned
-    fn extractor_get_sub_paths_from_path_with_enclosing_slashes() {
-        let path = "/homepage/assets/";
-        let paths = get_sub_paths_from_path(&path);
-        let expected = vec!["homepage/", "homepage/assets"];
-
-        assert_eq!(paths.len(), expected.len());
-        for expected_path in expected {
-            assert_eq!(paths.contains(&expected_path.to_string()), true);
-        }
-    }
-
-    #[test]
-    /// extract sub paths from the given url fragment; expect 1 sub path, no forward slashes are
-    /// included
-    fn extractor_get_sub_paths_from_path_with_only_a_word() {
-        let path = "homepage";
-        let paths = get_sub_paths_from_path(&path);
-        let expected = vec!["homepage"];
-
-        assert_eq!(paths.len(), expected.len());
-        for expected_path in expected {
-            assert_eq!(paths.contains(&expected_path.to_string()), true);
-        }
-    }
-
-    #[test]
-    /// extract sub paths from the given url fragment; expect 1 sub path, forward slash removed
-    fn extractor_get_sub_paths_from_path_with_an_absolute_word() {
-        let path = "/homepage";
-        let paths = get_sub_paths_from_path(&path);
-        let expected = vec!["homepage"];
-
-        assert_eq!(paths.len(), expected.len());
-        for expected_path in expected {
-            assert_eq!(paths.contains(&expected_path.to_string()), true);
-        }
-    }
-
-    #[test]
-    /// test that a full url and fragment are joined correctly, then added to the given list
-    /// i.e. the happy path
-    fn extractor_add_link_to_set_of_links_happy_path() {
-        let url = Url::parse("https://localhost").unwrap();
-        let mut links = HashSet::<String>::new();
-        let link = "admin";
-
-        assert_eq!(links.len(), 0);
-        add_link_to_set_of_links(link, &url, &mut links);
-
-        assert_eq!(links.len(), 1);
-        assert!(links.contains("https://localhost/admin"));
-    }
-
-    #[test]
-    /// test that an invalid path fragment doesn't add anything to the set of links
-    fn extractor_add_link_to_set_of_links_with_non_base_url() {
-        let url = Url::parse("https://localhost").unwrap();
-        let mut links = HashSet::<String>::new();
-        let link = "\\\\\\\\";
-
-        assert_eq!(links.len(), 0);
-        add_link_to_set_of_links(link, &url, &mut links);
-
-        assert_eq!(links.len(), 0);
-        assert!(links.is_empty());
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
-    /// use make_request to generate a Response, and use the Response to test get_links;
-    /// the response will contain an absolute path to a domain that is not part of the scanned
-    /// domain; expect an empty set returned
-    async fn extractor_get_links_with_absolute_url_that_differs_from_target_domain(
-    ) -> Result<(), Box<dyn std::error::Error>> {
-        let srv = MockServer::start();
-
-        let mock = srv.mock(|when, then|{
-            when.method(GET)
-                .path("/some-path");
-            then.status(200)
-                .body("\"http://defintely.not.a.thing.probably.com/homepage/assets/img/icons/handshake.svg\"");
-        });
-
-        let client = Client::new();
-        let url = Url::parse(&srv.url("/some-path")).unwrap();
-        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
-
-        let response = make_request(&client, &url, tx.clone()).await.unwrap();
-
-        let ferox_response = FeroxResponse::from(response, true).await;
-
-        let links = get_links(&ferox_response, tx).await;
-
-        assert!(links.is_empty());
-
-        assert_eq!(mock.hits(), 1);
-        Ok(())
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
-    /// test that /robots.txt is correctly requested given a base url (happy path)
-    async fn request_robots_txt_with_and_without_proxy() {
-        let srv = MockServer::start();
-
-        let mock = srv.mock(|when, then| {
-            when.method(GET).path("/robots.txt");
-            then.status(200).body("this is a test");
-        });
-
-        let mut config = Configuration::default();
-
-        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
-
-        request_robots_txt(&srv.url("/api/users/stuff/things"), &config, tx.clone()).await;
-
-        // note: the proxy doesn't actually do anything other than hit a different code branch
-        // in this unit test; it would however have an effect on an integration test
-        config.proxy = srv.url("/ima-proxy");
-
-        request_robots_txt(&srv.url("/api/different/path"), &config, tx).await;
-
-        assert_eq!(mock.hits(), 2);
-    }
-}

src/extractor/builder.rs

@@ -0,0 +1,100 @@
+use super::*;
+use crate::event_handlers::Handles;
+use anyhow::{bail, Result};
+
+/// Regular expression used in [LinkFinder](https://github.com/GerbenJavado/LinkFinder)
+///
+/// Incorporates change from this [Pull Request](https://github.com/GerbenJavado/LinkFinder/pull/66/files)
+pub(super) const LINKFINDER_REGEX: &str = r#"(?:"|')(((?:[a-zA-Z]{1,10}://|//)[^"'/]{1,}\.[a-zA-Z]{2,}[^"']{0,})|((?:/|\.\./|\./)[^"'><,;| *()(%%$^/\\\[\]][^"'><,;|()]{1,})|([a-zA-Z0-9_\-/]{1,}/[a-zA-Z0-9_\-/]{1,}\.(?:[a-zA-Z]{1,4}|action)(?:[\?|#][^"|']{0,}|))|([a-zA-Z0-9_\-/]{1,}/[a-zA-Z0-9_\-/]{3,}(?:[\?|#][^"|']{0,}|))|([a-zA-Z0-9_\-.]{1,}\.(?:php|asp|aspx|jsp|json|action|html|js|txt|xml)(?:[\?|#][^"|']{0,}|)))(?:"|')"#;
+
+/// Regular expression to pull url paths from robots.txt
+///
+/// ref: https://developers.google.com/search/reference/robots_txt
+pub(super) const ROBOTS_TXT_REGEX: &str =
+    r#"(?m)^ *(Allow|Disallow): *(?P<url_path>[a-zA-Z0-9._/?#@!&'()+,;%=-]+?)$"#; // multi-line (?m)
+
+/// Which type of extraction should be performed
+#[derive(Debug, Copy, Clone)]
+pub enum ExtractionTarget {
+    /// Examine a response body and extract links
+    ResponseBody,
+
+    /// Examine robots.txt (specifically) and extract links
+    RobotsTxt,
+}
+
+/// responsible for building an `Extractor`
+pub struct ExtractorBuilder<'a> {
+    /// Response from which to extract links
+    response: Option<&'a FeroxResponse>,
+
+    /// Response from which to extract links
+    url: String,
+
+    /// Handles object to house the underlying mpsc transmitters
+    handles: Option<Arc<Handles>>,
+
+    /// type of extraction to be performed
+    target: ExtractionTarget,
+}
+
+/// ExtractorBuilder implementation
+impl<'a> Default for ExtractorBuilder<'a> {
+    fn default() -> Self {
+        Self {
+            response: None,
+            url: "".to_string(),
+            handles: None,
+            target: ExtractionTarget::ResponseBody,
+        }
+    }
+}
+
+/// ExtractorBuilder implementation
+impl<'a> ExtractorBuilder<'a> {
+    /// builder call to set `handles`
+    pub fn handles(&mut self, handles: Arc<Handles>) -> &mut Self {
+        self.handles = Some(handles);
+        self
+    }
+
+    /// builder call to set `url`
+    pub fn url(&mut self, url: &str) -> &mut Self {
+        self.url = url.to_string();
+        self
+    }
+
+    /// builder call to set `target`
+    pub fn target(&mut self, target: ExtractionTarget) -> &mut Self {
+        self.target = target;
+        self
+    }
+
+    /// builder call to set `response`
+    pub fn response(&mut self, response: &'a FeroxResponse) -> &mut Self {
+        self.response = Some(response);
+        self
+    }
+
+    /// finalize configuration of ExtratorBuilder and return an Extractor
+    ///
+    /// requires either with_url or with_response to have been used in the build process
+    pub fn build(&self) -> Result<Extractor<'a>> {
+        if (self.url.is_empty() && self.response.is_none()) || self.handles.is_none() {
+            bail!("Extractor requires a URL or a FeroxResponse be specified as well as a Handles object")
+        }
+
+        Ok(Extractor {
+            links_regex: Regex::new(LINKFINDER_REGEX).unwrap(),
+            robots_regex: Regex::new(ROBOTS_TXT_REGEX).unwrap(),
+            response: if self.response.is_some() {
+                Some(self.response.unwrap())
+            } else {
+                None
+            },
+            url: self.url.to_owned(),
+            handles: self.handles.as_ref().unwrap().clone(),
+            target: self.target,
+        })
+    }
+}

src/extractor/container.rs

@@ -0,0 +1,410 @@
+use super::*;
+use crate::{
+    client,
+    event_handlers::{
+        Command,
+        Command::{AddError, AddToUsizeField},
+        Handles,
+    },
+    scan_manager::ScanOrder,
+    statistics::{
+        StatError::Other,
+        StatField::{LinksExtracted, TotalExpected},
+    },
+    url::FeroxUrl,
+    utils::{logged_request, make_request},
+};
+use anyhow::{bail, Context, Result};
+use reqwest::{StatusCode, Url};
+use std::collections::HashSet;
+use tokio::sync::oneshot;
+
+/// Whether an active scan is recursive or not
+#[derive(Debug)]
+enum RecursionStatus {
+    /// Scan is recursive
+    Recursive,
+
+    /// Scan is not recursive
+    NotRecursive,
+}
+
+/// Handles all logic related to extracting links from requested source code
+#[derive(Debug)]
+pub struct Extractor<'a> {
+    /// `LINKFINDER_REGEX` as a regex::Regex type
+    pub(super) links_regex: Regex,
+
+    /// `ROBOTS_TXT_REGEX` as a regex::Regex type
+    pub(super) robots_regex: Regex,
+
+    /// Response from which to extract links
+    pub(super) response: Option<&'a FeroxResponse>,
+
+    /// Response from which to extract links
+    pub(super) url: String,
+
+    /// Handles object to house the underlying mpsc transmitters
+    pub(super) handles: Arc<Handles>,
+
+    /// type of extraction to be performed
+    pub(super) target: ExtractionTarget,
+}
+
+/// Extractor implementation
+impl<'a> Extractor<'a> {
+    /// business logic that handles getting links from a normal http body response
+    pub async fn extract(&self) -> Result<()> {
+        let links = match self.target {
+            ExtractionTarget::ResponseBody => self.extract_from_body().await?,
+            ExtractionTarget::RobotsTxt => self.extract_from_robots().await?,
+        };
+
+        let recursive = if self.handles.config.no_recursion {
+            RecursionStatus::NotRecursive
+        } else {
+            RecursionStatus::Recursive
+        };
+
+        let scanned_urls = self.handles.ferox_scans()?;
+
+        for link in links {
+            let mut resp = match self.request_link(&link).await {
+                Ok(resp) => resp,
+                Err(_) => continue,
+            };
+
+            // filter if necessary
+            if self
+                .handles
+                .filters
+                .data
+                .should_filter_response(&resp, self.handles.stats.tx.clone())
+            {
+                continue;
+            }
+
+            if resp.is_file() {
+                // very likely a file, simply request and report
+                log::debug!("Extracted file: {}", resp);
+
+                scanned_urls.add_file_scan(&resp.url().to_string(), ScanOrder::Latest);
+
+                if let Err(e) = resp.send_report(self.handles.output.tx.clone()) {
+                    log::warn!("Could not send FeroxResponse to output handler: {}", e);
+                }
+
+                continue;
+            }
+
+            if matches!(recursive, RecursionStatus::Recursive) {
+                log::debug!("Extracted Directory: {}", resp);
+
+                if !resp.url().as_str().ends_with('/')
+                    && (resp.status().is_success()
+                        || matches!(resp.status(), &StatusCode::FORBIDDEN))
+                {
+                    // if the url doesn't end with a /
+                    // and the response code is either a 2xx or 403
+
+                    // since all of these are 2xx or 403, recursion is only attempted if the
+                    // url ends in a /. I am actually ok with adding the slash and not
+                    // adding it, as both have merit.  Leaving it in for now to see how
+                    // things turn out (current as of: v1.1.0)
+                    resp.set_url(&format!("{}/", resp.url()));
+                }
+
+                self.handles
+                    .send_scan_command(Command::TryRecursion(Box::new(resp)))?;
+                let (tx, rx) = oneshot::channel::<bool>();
+                self.handles.send_scan_command(Command::Sync(tx))?;
+                rx.await?;
+            }
+        }
+        Ok(())
+    }
+
+    /// Given a `reqwest::Response`, perform the following actions
+    ///   - parse the response's text for links using the linkfinder regex
+    ///   - for every link found take its url path and parse each sub-path
+    ///     - example: Response contains a link fragment `homepage/assets/img/icons/handshake.svg`
+    ///       with a base url of http://localhost, the following urls would be returned:
+    ///         - homepage/assets/img/icons/handshake.svg
+    ///         - homepage/assets/img/icons/
+    ///         - homepage/assets/img/
+    ///         - homepage/assets/
+    ///         - homepage/
+    pub(super) async fn extract_from_body(&self) -> Result<HashSet<String>> {
+        log::trace!("enter: get_links");
+
+        let mut links = HashSet::<String>::new();
+
+        let body = self.response.unwrap().text();
+
+        for capture in self.links_regex.captures_iter(&body) {
+            // remove single & double quotes from both ends of the capture
+            // capture[0] is the entire match, additional capture groups start at [1]
+            let link = capture[0].trim_matches(|c| c == '\'' || c == '"');
+
+            match Url::parse(link) {
+                Ok(absolute) => {
+                    if absolute.domain() != self.response.unwrap().url().domain()
+                        || absolute.host() != self.response.unwrap().url().host()
+                    {
+                        // domains/ips are not the same, don't scan things that aren't part of the original
+                        // target url
+                        continue;
+                    }
+
+                    if self.add_all_sub_paths(absolute.path(), &mut links).is_err() {
+                        log::warn!("could not add sub-paths from {} to {:?}", absolute, links);
+                    }
+                }
+                Err(e) => {
+                    // this is the expected error that happens when we try to parse a url fragment
+                    //     ex: Url::parse("/login") -> Err("relative URL without a base")
+                    // while this is technically an error, these are good results for us
+                    if e.to_string().contains("relative URL without a base") {
+                        if self.add_all_sub_paths(link, &mut links).is_err() {
+                            log::warn!("could not add sub-paths from {} to {:?}", link, links);
+                        }
+                    } else {
+                        // unexpected error has occurred
+                        log::warn!("Could not parse given url: {}", e);
+                        self.handles.stats.send(AddError(Other)).unwrap_or_default();
+                    }
+                }
+            }
+        }
+
+        self.update_stats(links.len())?;
+
+        log::trace!("exit: get_links -> {:?}", links);
+
+        Ok(links)
+    }
+
+    /// take a url fragment like homepage/assets/img/icons/handshake.svg and
+    /// incrementally add
+    ///     - homepage/assets/img/icons/
+    ///     - homepage/assets/img/
+    ///     - homepage/assets/
+    ///     - homepage/
+    fn add_all_sub_paths(&self, url_path: &str, mut links: &mut HashSet<String>) -> Result<()> {
+        log::trace!("enter: add_all_sub_paths({}, {:?})", url_path, links);
+
+        for sub_path in self.get_sub_paths_from_path(url_path) {
+            self.add_link_to_set_of_links(&sub_path, &mut links)?;
+        }
+
+        log::trace!("exit: add_all_sub_paths");
+        Ok(())
+    }
+
+    /// Iterate over a given path, return a list of every sub-path found
+    ///
+    /// example: `path` contains a link fragment `homepage/assets/img/icons/handshake.svg`
+    /// the following fragments would be returned:
+    ///   - homepage/assets/img/icons/handshake.svg
+    ///   - homepage/assets/img/icons/
+    ///   - homepage/assets/img/
+    ///   - homepage/assets/
+    ///   - homepage/
+    pub(super) fn get_sub_paths_from_path(&self, path: &str) -> Vec<String> {
+        log::trace!("enter: get_sub_paths_from_path({})", path);
+        let mut paths = vec![];
+
+        // filter out any empty strings caused by .split
+        let mut parts: Vec<&str> = path.split('/').filter(|s| !s.is_empty()).collect();
+
+        let length = parts.len();
+
+        for i in 0..length {
+            // iterate over all parts of the path
+            if parts.is_empty() {
+                // pop left us with an empty vector, we're done
+                break;
+            }
+
+            let mut possible_path = parts.join("/");
+
+            if possible_path.is_empty() {
+                // .join can result in an empty string, which we don't need, ignore
+                continue;
+            }
+
+            if i > 0 {
+                // this isn't the last index of the parts array
+                // ex: /buried/misc/stupidfile.php
+                // this block skips the file but sees all parent folders
+                possible_path = format!("{}/", possible_path);
+            }
+
+            paths.push(possible_path); // good sub-path found
+            parts.pop(); // use .pop() to remove the last part of the path and continue iteration
+        }
+
+        log::trace!("exit: get_sub_paths_from_path -> {:?}", paths);
+        paths
+    }
+
+    /// simple helper to stay DRY, trys to join a url + fragment and add it to the `links` HashSet
+    pub(super) fn add_link_to_set_of_links(
+        &self,
+        link: &str,
+        links: &mut HashSet<String>,
+    ) -> Result<()> {
+        log::trace!("enter: add_link_to_set_of_links({}, {:?})", link, links);
+
+        let old_url = match self.target {
+            ExtractionTarget::ResponseBody => self.response.unwrap().url().clone(),
+            ExtractionTarget::RobotsTxt => match Url::parse(&self.url) {
+                Ok(u) => u,
+                Err(e) => {
+                    bail!("Could not parse {}: {}", self.url, e);
+                }
+            },
+        };
+
+        let new_url = old_url
+            .join(&link)
+            .with_context(|| format!("Could not join {} with {}", old_url, link))?;
+
+        links.insert(new_url.to_string());
+
+        log::trace!("exit: add_link_to_set_of_links");
+
+        Ok(())
+    }
+
+    /// Wrapper around link extraction logic
+    /// currently used in two places:
+    ///   - links from response bodies
+    ///   - links from robots.txt responses
+    ///
+    /// general steps taken:
+    ///   - create a new Url object based on cli options/args
+    ///   - check if the new Url has already been seen/scanned -> None
+    ///   - make a request to the new Url ? -> Some(response) : None
+    pub(super) async fn request_link(&self, url: &str) -> Result<FeroxResponse> {
+        log::trace!("enter: request_link({})", url);
+
+        let ferox_url = FeroxUrl::from_string(&url, self.handles.clone());
+
+        // create a url based on the given command line options
+        let new_url = ferox_url.format(&"", None)?;
+
+        let scanned_urls = self.handles.ferox_scans()?;
+
+        if scanned_urls.get_scan_by_url(&new_url.to_string()).is_some() {
+            //we've seen the url before and don't need to scan again
+            log::trace!("exit: request_link -> None");
+            bail!("previously seen url");
+        }
+
+        // make the request and store the response
+        let new_response = logged_request(&new_url, self.handles.clone()).await?;
+
+        let new_ferox_response =
+            FeroxResponse::from(new_response, true, self.handles.config.output_level).await;
+
+        log::trace!("exit: request_link -> {:?}", new_ferox_response);
+
+        Ok(new_ferox_response)
+    }
+
+    /// Entry point to perform link extraction from robots.txt
+    ///
+    /// `base_url` can have paths and subpaths, however robots.txt will be requested from the
+    /// root of the url
+    /// given the url:
+    ///     http://localhost/stuff/things
+    /// this function requests:
+    ///     http://localhost/robots.txt
+    pub(super) async fn extract_from_robots(&self) -> Result<HashSet<String>> {
+        log::trace!("enter: extract_robots_txt");
+
+        let mut links: HashSet<String> = HashSet::new();
+
+        let response = self.request_robots_txt().await?;
+
+        for capture in self.robots_regex.captures_iter(response.text()) {
+            if let Some(new_path) = capture.name("url_path") {
+                let mut new_url = Url::parse(&self.url)?;
+                new_url.set_path(new_path.as_str());
+                if self.add_all_sub_paths(&new_url.path(), &mut links).is_err() {
+                    log::warn!("could not add sub-paths from {} to {:?}", new_url, links);
+                }
+            }
+        }
+
+        self.update_stats(links.len())?;
+
+        log::trace!("exit: extract_robots_txt -> {:?}", links);
+        Ok(links)
+    }
+
+    /// helper function that simply requests /robots.txt on the given url's base url
+    ///
+    /// example:
+    ///     http://localhost/api/users -> http://localhost/robots.txt
+    ///     
+    /// The length of the given path has no effect on what's requested; it's always
+    /// base url + /robots.txt
+    pub(super) async fn request_robots_txt(&self) -> Result<FeroxResponse> {
+        log::trace!("enter: get_robots_file");
+
+        // more often than not, domain/robots.txt will redirect to www.domain/robots.txt or something
+        // similar; to account for that, create a client that will follow redirects, regardless of
+        // what the user specified for the scanning client. Other than redirects, it will respect
+        // all other user specified settings
+        let follow_redirects = true;
+
+        let proxy = if self.handles.config.proxy.is_empty() {
+            None
+        } else {
+            Some(self.handles.config.proxy.as_str())
+        };
+
+        let client = client::initialize(
+            self.handles.config.timeout,
+            &self.handles.config.user_agent,
+            follow_redirects,
+            self.handles.config.insecure,
+            &self.handles.config.headers,
+            proxy,
+        )?;
+
+        let mut url = Url::parse(&self.url)?;
+        url.set_path("/robots.txt"); // overwrite existing path with /robots.txt
+
+        // purposefully not using logged_request here due to using the special client
+        let response = make_request(
+            &client,
+            &url,
+            self.handles.config.output_level,
+            self.handles.stats.tx.clone(),
+        )
+        .await?;
+
+        let ferox_response =
+            FeroxResponse::from(response, true, self.handles.config.output_level).await;
+
+        log::trace!("exit: get_robots_file -> {}", ferox_response);
+        return Ok(ferox_response);
+    }
+
+    /// update total number of links extracted and expected responses
+    fn update_stats(&self, num_links: usize) -> Result<()> {
+        let multiplier = self.handles.config.extensions.len().max(1);
+
+        self.handles
+            .stats
+            .send(AddToUsizeField(LinksExtracted, num_links))?;
+        self.handles
+            .stats
+            .send(AddToUsizeField(TotalExpected, num_links * multiplier))?;
+
+        Ok(())
+    }
+}

src/extractor/mod.rs

@@ -0,0 +1,13 @@
+//! extract links from html source and robots.txt
+mod builder;
+mod container;
+#[cfg(test)]
+mod tests;
+
+pub use self::builder::ExtractionTarget;
+pub use self::builder::ExtractorBuilder;
+pub use self::container::Extractor;
+
+use crate::response::FeroxResponse;
+use regex::Regex;
+use std::sync::Arc;

src/extractor/tests.rs

@@ -0,0 +1,354 @@
+use super::builder::{LINKFINDER_REGEX, ROBOTS_TXT_REGEX};
+use super::*;
+use crate::config::{Configuration, OutputLevel};
+use crate::scan_manager::ScanOrder;
+use crate::{
+    event_handlers::Handles, scan_manager::FeroxScans, utils::make_request, Command, FeroxChannel,
+};
+use anyhow::Result;
+use httpmock::{Method::GET, MockServer};
+use lazy_static::lazy_static;
+use reqwest::{Client, StatusCode, Url};
+use std::collections::HashSet;
+use tokio::sync::mpsc;
+
+lazy_static! {
+    /// Extractor for testing robots.txt
+    static ref ROBOTS_EXT: Extractor<'static> = setup_extractor(ExtractionTarget::RobotsTxt, Arc::new(FeroxScans::default()));
+
+    /// Extractor for testing response bodies
+    static ref BODY_EXT: Extractor<'static> = setup_extractor(ExtractionTarget::ResponseBody, Arc::new(FeroxScans::default()));
+
+    /// FeroxResponse for Extractor
+    static ref RESPONSE: FeroxResponse = get_test_response();
+}
+
+/// constructor for the default FeroxResponse used during testing
+fn get_test_response() -> FeroxResponse {
+    let mut resp = FeroxResponse::default();
+    resp.set_text("nulla pharetra diam sit amet nisl suscipit adipiscing bibendum est");
+    resp
+}
+
+/// creates a single extractor that can be used to test standalone functions
+fn setup_extractor(target: ExtractionTarget, scanned_urls: Arc<FeroxScans>) -> Extractor<'static> {
+    let mut builder = ExtractorBuilder::default();
+
+    let builder = match target {
+        ExtractionTarget::ResponseBody => builder
+            .target(ExtractionTarget::ResponseBody)
+            .response(&RESPONSE),
+        ExtractionTarget::RobotsTxt => builder
+            .url("http://localhost")
+            .target(ExtractionTarget::RobotsTxt),
+    };
+
+    let config = Arc::new(Configuration::new().unwrap());
+    let handles = Arc::new(Handles::for_testing(Some(scanned_urls), Some(config)).0);
+
+    builder.handles(handles).build().unwrap()
+}
+
+#[test]
+/// extract sub paths from the given url fragment; expect 4 sub paths and that all are
+/// in the expected array
+fn extractor_get_sub_paths_from_path_with_multiple_paths() {
+    let path = "homepage/assets/img/icons/handshake.svg";
+    let r_paths = ROBOTS_EXT.get_sub_paths_from_path(&path);
+    let b_paths = BODY_EXT.get_sub_paths_from_path(&path);
+    let expected = vec![
+        "homepage/",
+        "homepage/assets/",
+        "homepage/assets/img/",
+        "homepage/assets/img/icons/",
+        "homepage/assets/img/icons/handshake.svg",
+    ];
+
+    assert_eq!(r_paths.len(), expected.len());
+    assert_eq!(b_paths.len(), expected.len());
+    for expected_path in expected {
+        assert_eq!(r_paths.contains(&expected_path.to_string()), true);
+        assert_eq!(b_paths.contains(&expected_path.to_string()), true);
+    }
+}
+
+#[test]
+/// extract sub paths from the given url fragment; expect 2 sub paths and that all are
+/// in the expected array. the fragment is wrapped in slashes to ensure no empty strings are
+/// returned
+fn extractor_get_sub_paths_from_path_with_enclosing_slashes() {
+    let path = "/homepage/assets/";
+    let r_paths = ROBOTS_EXT.get_sub_paths_from_path(&path);
+    let b_paths = BODY_EXT.get_sub_paths_from_path(&path);
+    let expected = vec!["homepage/", "homepage/assets"];
+
+    assert_eq!(r_paths.len(), expected.len());
+    assert_eq!(b_paths.len(), expected.len());
+    for expected_path in expected {
+        assert_eq!(r_paths.contains(&expected_path.to_string()), true);
+        assert_eq!(b_paths.contains(&expected_path.to_string()), true);
+    }
+}
+
+#[test]
+/// extract sub paths from the given url fragment; expect 1 sub path, no forward slashes are
+/// included
+fn extractor_get_sub_paths_from_path_with_only_a_word() {
+    let path = "homepage";
+    let r_paths = ROBOTS_EXT.get_sub_paths_from_path(&path);
+    let b_paths = BODY_EXT.get_sub_paths_from_path(&path);
+    let expected = vec!["homepage"];
+
+    assert_eq!(r_paths.len(), expected.len());
+    assert_eq!(b_paths.len(), expected.len());
+    for expected_path in expected {
+        assert_eq!(r_paths.contains(&expected_path.to_string()), true);
+        assert_eq!(b_paths.contains(&expected_path.to_string()), true);
+    }
+}
+
+#[test]
+/// extract sub paths from the given url fragment; expect 1 sub path, forward slash removed
+fn extractor_get_sub_paths_from_path_with_an_absolute_word() {
+    let path = "/homepage";
+    let r_paths = ROBOTS_EXT.get_sub_paths_from_path(&path);
+    let b_paths = BODY_EXT.get_sub_paths_from_path(&path);
+    let expected = vec!["homepage"];
+
+    assert_eq!(r_paths.len(), expected.len());
+    assert_eq!(b_paths.len(), expected.len());
+    for expected_path in expected {
+        assert_eq!(r_paths.contains(&expected_path.to_string()), true);
+        assert_eq!(b_paths.contains(&expected_path.to_string()), true);
+    }
+}
+
+#[test]
+/// test that an ExtractorBuilder without a FeroxResponse and without a URL bails
+fn extractor_builder_bails_when_neither_required_field_is_set() {
+    let handles = Arc::new(Handles::for_testing(None, None).0);
+
+    let extractor = ExtractorBuilder::default()
+        .url("")
+        .target(ExtractionTarget::RobotsTxt)
+        .handles(handles)
+        .build();
+
+    assert!(extractor.is_err());
+}
+
+#[test]
+/// Extractor with a non-base url bails
+fn extractor_with_non_base_url_bails() -> Result<()> {
+    let mut links = HashSet::<String>::new();
+    let link = "admin";
+    let handles = Arc::new(Handles::for_testing(None, None).0);
+
+    let extractor = ExtractorBuilder::default()
+        .url("\\\\\\")
+        .handles(handles)
+        .target(ExtractionTarget::RobotsTxt)
+        .build()?;
+
+    let result = extractor.add_link_to_set_of_links(link, &mut links);
+
+    assert!(result.is_err());
+    Ok(())
+}
+
+#[test]
+/// test that a full url and fragment are joined correctly, then added to the given list
+/// i.e. the happy path
+fn extractor_add_link_to_set_of_links_happy_path() {
+    let mut r_links = HashSet::<String>::new();
+    let r_link = "admin";
+    let mut b_links = HashSet::<String>::new();
+    let b_link = "shmadmin";
+
+    assert_eq!(r_links.len(), 0);
+    ROBOTS_EXT
+        .add_link_to_set_of_links(r_link, &mut r_links)
+        .unwrap();
+
+    assert_eq!(r_links.len(), 1);
+    assert!(r_links.contains("http://localhost/admin"));
+
+    assert_eq!(b_links.len(), 0);
+
+    BODY_EXT
+        .add_link_to_set_of_links(b_link, &mut b_links)
+        .unwrap();
+
+    assert_eq!(b_links.len(), 1);
+    assert!(b_links.contains("http://localhost/shmadmin"));
+}
+
+#[test]
+/// test that an invalid path fragment doesn't add anything to the set of links
+fn extractor_add_link_to_set_of_links_with_non_base_url() {
+    let mut links = HashSet::<String>::new();
+    let link = "\\\\\\\\";
+
+    assert_eq!(links.len(), 0);
+    assert!(ROBOTS_EXT
+        .add_link_to_set_of_links(link, &mut links)
+        .is_err());
+    assert!(BODY_EXT.add_link_to_set_of_links(link, &mut links).is_err());
+
+    assert_eq!(links.len(), 0);
+    assert!(links.is_empty());
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// use make_request to generate a Response, and use the Response to test get_links;
+/// the response will contain an absolute path to a domain that is not part of the scanned
+/// domain; expect an empty set returned
+async fn extractor_get_links_with_absolute_url_that_differs_from_target_domain() -> Result<()> {
+    let (tx_stats, _): FeroxChannel<Command> = mpsc::unbounded_channel();
+
+    let srv = MockServer::start();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/some-path");
+        then.status(200).body(
+            "\"http://defintely.not.a.thing.probably.com/homepage/assets/img/icons/handshake.svg\"",
+        );
+    });
+
+    let client = Client::new();
+    let url = Url::parse(&srv.url("/some-path")).unwrap();
+
+    let response = make_request(&client, &url, OutputLevel::Default, tx_stats.clone())
+        .await
+        .unwrap();
+    let (handles, _rx) = Handles::for_testing(None, None);
+
+    let handles = Arc::new(handles);
+    let ferox_response = FeroxResponse::from(response, true, OutputLevel::Default).await;
+
+    let extractor = Extractor {
+        links_regex: Regex::new(LINKFINDER_REGEX).unwrap(),
+        robots_regex: Regex::new(ROBOTS_TXT_REGEX).unwrap(),
+        response: Some(&ferox_response),
+        url: String::new(),
+        target: ExtractionTarget::ResponseBody,
+        handles: handles.clone(),
+    };
+
+    let links = extractor.extract_from_body().await?;
+
+    assert!(links.is_empty());
+    assert_eq!(mock.hits(), 1);
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test that /robots.txt is correctly requested given a base url (happy path)
+async fn request_robots_txt_without_proxy() -> Result<()> {
+    let handles = Arc::new(Handles::for_testing(None, None).0);
+
+    let srv = MockServer::start();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/robots.txt");
+        then.status(200).body("this is a test");
+    });
+
+    let extractor = Extractor {
+        links_regex: Regex::new(LINKFINDER_REGEX).unwrap(),
+        robots_regex: Regex::new(ROBOTS_TXT_REGEX).unwrap(),
+        response: None,
+        url: srv.url("/api/users/stuff/things"),
+        target: ExtractionTarget::RobotsTxt,
+        handles,
+    };
+
+    let resp = extractor.request_robots_txt().await?;
+
+    assert!(matches!(resp.status(), &StatusCode::OK));
+    println!("{}", resp);
+    assert_eq!(resp.content_length(), 14);
+    assert_eq!(mock.hits(), 1);
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test that /robots.txt is correctly requested given a base url (happy path) when a proxy is used
+async fn request_robots_txt_with_proxy() -> Result<()> {
+    let handles = Arc::new(Handles::for_testing(None, None).0);
+    let mut config = Configuration::new()?;
+
+    let srv = MockServer::start();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/robots.txt");
+        then.status(200).body("this is also a test");
+    });
+
+    // note: the proxy doesn't actually do anything other than hit a different code branch
+    // in this unit test; it would however have an effect on an integration test
+    config.proxy = srv.url("/ima-proxy");
+    config.no_recursion = true;
+
+    let extractor = ExtractorBuilder::default()
+        .url(&srv.url("/api/different/path"))
+        .target(ExtractionTarget::RobotsTxt)
+        .handles(handles)
+        .build()?;
+
+    let resp = extractor.request_robots_txt().await?;
+
+    assert!(matches!(resp.status(), &StatusCode::OK));
+    assert_eq!(resp.content_length(), 19);
+    assert_eq!(mock.hits(), 1);
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// request_link's happy path, expect back a FeroxResponse
+async fn request_link_happy_path() -> Result<()> {
+    let srv = MockServer::start();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/login.php");
+        then.status(200).body("this is a test");
+    });
+
+    let r_resp = ROBOTS_EXT.request_link(&srv.url("/login.php")).await?;
+    let b_resp = BODY_EXT.request_link(&srv.url("/login.php")).await?;
+
+    assert!(matches!(r_resp.status(), &StatusCode::OK));
+    assert!(matches!(b_resp.status(), &StatusCode::OK));
+    assert_eq!(r_resp.content_length(), 14);
+    assert_eq!(b_resp.content_length(), 14);
+    assert_eq!(mock.hits(), 2);
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// request_link should bail in the event that the url is already in scanned_urls
+async fn request_link_bails_on_seen_url() -> Result<()> {
+    let url = "/unique-for-this-test.php";
+    let srv = MockServer::start();
+    let served = srv.url(url);
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path(url);
+        then.status(200)
+            .body("this is a unique test, don't reuse the endpoint");
+    });
+
+    let scans = Arc::new(FeroxScans::default());
+    scans.add_file_scan(&served, ScanOrder::Latest);
+
+    let robots = setup_extractor(ExtractionTarget::RobotsTxt, scans.clone());
+    let body = setup_extractor(ExtractionTarget::ResponseBody, scans);
+
+    let r_resp = robots.request_link(&served).await;
+    let b_resp = body.request_link(&served).await;
+
+    assert!(r_resp.is_err());
+    assert!(b_resp.is_err());
+    assert_eq!(mock.hits(), 0); // function exits before requests can happen
+    Ok(())
+}

src/filters/container.rs

@@ -0,0 +1,56 @@
+use std::sync::Mutex;
+
+use anyhow::Result;
+
+use crate::response::FeroxResponse;
+use crate::{
+    event_handlers::Command::AddToUsizeField, statistics::StatField::WildcardsFiltered,
+    CommandSender,
+};
+
+use super::{FeroxFilter, WildcardFilter};
+
+/// Container around a collection of `FeroxFilters`s
+#[derive(Debug, Default)]
+pub struct FeroxFilters {
+    /// collection of `FeroxFilters`
+    pub filters: Mutex<Vec<Box<dyn FeroxFilter>>>,
+}
+
+/// implementation of FeroxFilter collection
+impl FeroxFilters {
+    /// add a single FeroxFilter to the collection
+    pub fn push(&self, filter: Box<dyn FeroxFilter>) -> Result<()> {
+        if let Ok(mut guard) = self.filters.lock() {
+            if guard.contains(&filter) {
+                return Ok(());
+            }
+
+            guard.push(filter)
+        }
+        Ok(())
+    }
+
+    /// Simple helper to stay DRY; determines whether or not a given `FeroxResponse` should be reported
+    /// to the user or not.
+    pub fn should_filter_response(
+        &self,
+        response: &FeroxResponse,
+        tx_stats: CommandSender,
+    ) -> bool {
+        if let Ok(filters) = self.filters.lock() {
+            for filter in filters.iter() {
+                // wildcard.should_filter goes here
+                if filter.should_filter_response(&response) {
+                    if filter.as_any().downcast_ref::<WildcardFilter>().is_some() {
+                        tx_stats
+                            .send(AddToUsizeField(WildcardsFiltered, 1))
+                            .unwrap_or_default();
+                    }
+                    return true;
+                }
+            }
+        }
+        false
+    }
+}

src/filters/init.rs

@@ -0,0 +1,94 @@
+use super::{
+    LinesFilter, RegexFilter, SimilarityFilter, SizeFilter, StatusCodeFilter, WordsFilter,
+};
+use crate::{
+    event_handlers::Handles,
+    response::FeroxResponse,
+    skip_fail,
+    utils::{fmt_err, logged_request},
+    Command::AddFilter,
+    SIMILARITY_THRESHOLD,
+};
+use anyhow::Result;
+use fuzzyhash::FuzzyHash;
+use regex::Regex;
+use reqwest::Url;
+use std::sync::Arc;
+
+/// add all user-supplied filters to the (already started) filters handler
+pub async fn initialize(handles: Arc<Handles>) -> Result<()> {
+    // add any status code filters to filters handler's FeroxFilters  (-C|--filter-status)
+    for code_filter in &handles.config.filter_status {
+        let filter = StatusCodeFilter {
+            filter_code: *code_filter,
+        };
+        let boxed_filter = Box::new(filter);
+        skip_fail!(handles.filters.send(AddFilter(boxed_filter)));
+    }
+
+    // add any line count filters to filters handler's FeroxFilters  (-N|--filter-lines)
+    for lines_filter in &handles.config.filter_line_count {
+        let filter = LinesFilter {
+            line_count: *lines_filter,
+        };
+        let boxed_filter = Box::new(filter);
+        skip_fail!(handles.filters.send(AddFilter(boxed_filter)));
+    }
+
+    // add any line count filters to filters handler's FeroxFilters  (-W|--filter-words)
+    for words_filter in &handles.config.filter_word_count {
+        let filter = WordsFilter {
+            word_count: *words_filter,
+        };
+        let boxed_filter = Box::new(filter);
+        skip_fail!(handles.filters.send(AddFilter(boxed_filter)));
+    }
+
+    // add any line count filters to filters handler's FeroxFilters  (-S|--filter-size)
+    for size_filter in &handles.config.filter_size {
+        let filter = SizeFilter {
+            content_length: *size_filter,
+        };
+        let boxed_filter = Box::new(filter);
+        skip_fail!(handles.filters.send(AddFilter(boxed_filter)));
+    }
+
+    // add any regex filters to filters handler's FeroxFilters  (-X|--filter-regex)
+    for regex_filter in &handles.config.filter_regex {
+        let raw = regex_filter;
+        let compiled = skip_fail!(Regex::new(&raw));
+
+        let filter = RegexFilter {
+            raw_string: raw.to_owned(),
+            compiled,
+        };
+        let boxed_filter = Box::new(filter);
+        skip_fail!(handles.filters.send(AddFilter(boxed_filter)));
+    }
+
+    // add any similarity filters to filters handler's FeroxFilters  (--filter-similar-to)
+    for similarity_filter in &handles.config.filter_similar {
+        // url as-is based on input, ignores user-specified url manipulation options (add-slash etc)
+        let url = skip_fail!(Url::parse(&similarity_filter));
+
+        // attempt to request the given url
+        let resp = skip_fail!(logged_request(&url, handles.clone()).await);
+
+        // if successful, create a filter based on the response's body
+        let fr = FeroxResponse::from(resp, true, handles.config.output_level).await;
+
+        // hash the response body and store the resulting hash in the filter object
+        let hash = FuzzyHash::new(&fr.text()).to_string();
+
+        let filter = SimilarityFilter {
+            text: hash,
+            threshold: SIMILARITY_THRESHOLD,
+        };
+
+        let boxed_filter = Box::new(filter);
+        skip_fail!(handles.filters.send(AddFilter(boxed_filter)));
+    }
+
+    handles.filters.sync().await?;
+    Ok(())
+}

src/filters/mod.rs

@@ -1,5 +1,20 @@
-//! module containing all of feroxbuster's filters
-mod traits;
+//! contains all of feroxbuster's filters
+use std::any::Any;
+use std::fmt::Debug;
+
+use crate::response::FeroxResponse;
+use crate::traits::{FeroxFilter, FeroxSerialize};
+
+pub use self::container::FeroxFilters;
+pub use self::init::initialize;
+pub use self::lines::LinesFilter;
+pub use self::regex::RegexFilter;
+pub use self::similarity::SimilarityFilter;
+pub use self::size::SizeFilter;
+pub use self::status_code::StatusCodeFilter;
+pub use self::wildcard::WildcardFilter;
+pub use self::words::WordsFilter;
+
 mod wildcard;
 mod status_code;
 mod words;
@@ -7,18 +22,7 @@ mod lines;
 mod size;
 mod regex;
 mod similarity;
+mod container;
 #[cfg(test)]
 mod tests;
-
-pub use self::lines::LinesFilter;
-pub use self::regex::RegexFilter;
-pub use self::similarity::SimilarityFilter;
-pub use self::size::SizeFilter;
-pub use self::status_code::StatusCodeFilter;
-pub use self::traits::FeroxFilter;
-pub use self::wildcard::WildcardFilter;
-pub use self::words::WordsFilter;
-
-use crate::{config::CONFIGURATION, utils::get_url_path_length, FeroxResponse, FeroxSerialize};
-use std::any::Any;
-use std::fmt::Debug;
+mod init;

src/filters/similarity.rs

@@ -17,7 +17,7 @@ impl FeroxFilter for SimilarityFilter {
     /// Check `FeroxResponse::text` against what was requested from the site passed in via
     /// --filter-similar-to
     fn should_filter_response(&self, response: &FeroxResponse) -> bool {
-        let other = FuzzyHash::new(&response.text);
+        let other = FuzzyHash::new(&response.text());
 
         if let Ok(result) = FuzzyHash::compare(&self.text, &other.to_string()) {
             return result >= self.threshold;

src/filters/tests.rs

@@ -1,12 +1,36 @@
 use super::*;
 use ::fuzzyhash::FuzzyHash;
 use ::regex::Regex;
-use reqwest::Url;
+
+#[test]
+/// simply test the default values for wildcardfilter, expect 0, 0
+fn wildcard_filter_default() {
+    let wcf = WildcardFilter::default();
+    assert_eq!(wcf.size, u64::MAX);
+    assert_eq!(wcf.dynamic, u64::MAX);
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn wildcard_filter_as_any() {
+    let filter = WildcardFilter::default();
+    let filter2 = WildcardFilter::default();
+
+    assert!(filter.box_eq(filter2.as_any()));
+
+    assert_eq!(
+        *filter.as_any().downcast_ref::<WildcardFilter>().unwrap(),
+        filter
+    );
+}
 
 #[test]
 /// just a simple test to increase code coverage by hitting as_any and the inner value
 fn lines_filter_as_any() {
     let filter = LinesFilter { line_count: 1 };
+    let filter2 = LinesFilter { line_count: 1 };
+
+    assert!(filter.box_eq(filter2.as_any()));
 
     assert_eq!(filter.line_count, 1);
     assert_eq!(
@@ -19,6 +43,9 @@ fn lines_filter_as_any() {
 /// just a simple test to increase code coverage by hitting as_any and the inner value
 fn words_filter_as_any() {
     let filter = WordsFilter { word_count: 1 };
+    let filter2 = WordsFilter { word_count: 1 };
+
+    assert!(filter.box_eq(filter2.as_any()));
 
     assert_eq!(filter.word_count, 1);
     assert_eq!(
@@ -31,6 +58,9 @@ fn words_filter_as_any() {
 /// just a simple test to increase code coverage by hitting as_any and the inner value
 fn size_filter_as_any() {
     let filter = SizeFilter { content_length: 1 };
+    let filter2 = SizeFilter { content_length: 1 };
+
+    assert!(filter.box_eq(filter2.as_any()));
 
     assert_eq!(filter.content_length, 1);
     assert_eq!(
@@ -43,6 +73,9 @@ fn size_filter_as_any() {
 /// just a simple test to increase code coverage by hitting as_any and the inner value
 fn status_code_filter_as_any() {
     let filter = StatusCodeFilter { filter_code: 200 };
+    let filter2 = StatusCodeFilter { filter_code: 200 };
+
+    assert!(filter.box_eq(filter2.as_any()));
 
     assert_eq!(filter.filter_code, 200);
     assert_eq!(
@@ -56,10 +89,17 @@ fn status_code_filter_as_any() {
 fn regex_filter_as_any() {
     let raw = r".*\.txt$";
     let compiled = Regex::new(raw).unwrap();
+    let compiled2 = Regex::new(raw).unwrap();
     let filter = RegexFilter {
         compiled,
         raw_string: raw.to_string(),
     };
+    let filter2 = RegexFilter {
+        compiled: compiled2,
+        raw_string: raw.to_string(),
+    };
+
+    assert!(filter.box_eq(filter2.as_any()));
 
     assert_eq!(filter.raw_string, r".*\.txt$");
     assert_eq!(
@@ -71,20 +111,17 @@ fn regex_filter_as_any() {
 #[test]
 /// test should_filter on WilcardFilter where static logic matches
 fn wildcard_should_filter_when_static_wildcard_found() {
-    let resp = FeroxResponse {
-        text: String::new(),
-        wildcard: true,
-        url: Url::parse("http://localhost").unwrap(),
-        content_length: 100,
-        word_count: 50,
-        line_count: 25,
-        headers: reqwest::header::HeaderMap::new(),
-        status: reqwest::StatusCode::OK,
-    };
+    let mut resp = FeroxResponse::default();
+    resp.set_wildcard(true);
+    resp.set_url("http://localhost");
+    resp.set_text(
+        "pellentesque diam volutpat commodo sed egestas egestas fringilla phasellus faucibus",
+    );
 
     let filter = WildcardFilter {
-        size: 100,
+        size: 83,
         dynamic: 0,
+        dont_filter: false,
     };
 
     assert!(filter.should_filter_response(&resp));
@@ -93,38 +130,28 @@ fn wildcard_should_filter_when_static_wildcard_found() {
 #[test]
 /// test should_filter on WilcardFilter where dynamic logic matches
 fn wildcard_should_filter_when_dynamic_wildcard_found() {
-    let resp = FeroxResponse {
-        text: String::new(),
-        wildcard: true,
-        url: Url::parse("http://localhost/stuff").unwrap(),
-        content_length: 100,
-        word_count: 50,
-        line_count: 25,
-        headers: reqwest::header::HeaderMap::new(),
-        status: reqwest::StatusCode::OK,
-    };
+    let mut resp = FeroxResponse::default();
+    resp.set_wildcard(true);
+    resp.set_url("http://localhost/stuff");
+    resp.set_text("pellentesque diam volutpat commodo sed egestas egestas fringilla");
 
     let filter = WildcardFilter {
         size: 0,
-        dynamic: 95,
+        dynamic: 59, // content-length - 5 (len('stuff'))
+        dont_filter: false,
     };
 
+    println!("resp: {:?}: filter: {:?}", resp, filter);
+
     assert!(filter.should_filter_response(&resp));
 }
 
 #[test]
 /// test should_filter on RegexFilter where regex matches body
 fn regexfilter_should_filter_when_regex_matches_on_response_body() {
-    let resp = FeroxResponse {
-        text: String::from("im a body response hurr durr!"),
-        wildcard: false,
-        url: Url::parse("http://localhost/stuff").unwrap(),
-        content_length: 100,
-        word_count: 50,
-        line_count: 25,
-        headers: reqwest::header::HeaderMap::new(),
-        status: reqwest::StatusCode::OK,
-    };
+    let mut resp = FeroxResponse::default();
+    resp.set_url("http://localhost/stuff");
+    resp.set_text("im a body response hurr durr!");
 
     let raw = r"response...rr";
 
@@ -139,16 +166,9 @@ fn regexfilter_should_filter_when_regex_matches_on_response_body() {
 #[test]
 /// a few simple tests for similarity filter
 fn similarity_filter_is_accurate() {
-    let mut resp = FeroxResponse {
-        text: String::from("sitting"),
-        wildcard: false,
-        url: Url::parse("http://localhost/stuff").unwrap(),
-        content_length: 100,
-        word_count: 50,
-        line_count: 25,
-        headers: reqwest::header::HeaderMap::new(),
-        status: reqwest::StatusCode::OK,
-    };
+    let mut resp = FeroxResponse::default();
+    resp.set_url("http://localhost/stuff");
+    resp.set_text("sitting");
 
     let mut filter = SimilarityFilter {
         text: FuzzyHash::new("kitten").to_string(),
@@ -158,14 +178,14 @@ fn similarity_filter_is_accurate() {
     // kitten/sitting is 57% similar, so a threshold of 95 should not be filtered
     assert!(!filter.should_filter_response(&resp));
 
-    resp.text = String::new();
+    resp.set_text("");
     filter.text = String::new();
     filter.threshold = 100;
 
     // two empty strings are the same, however ssdeep doesn't accept empty strings, expect false
     assert!(!filter.should_filter_response(&resp));
 
-    resp.text = String::from("some data to hash for the purposes of running a test");
+    resp.set_text("some data to hash for the purposes of running a test");
     filter.text = FuzzyHash::new("some data to hash for the purposes of running a te").to_string();
     filter.threshold = 17;
 
@@ -180,6 +200,13 @@ fn similarity_filter_as_any() {
         threshold: 95,
     };
 
+    let filter2 = SimilarityFilter {
+        text: String::from("stuff"),
+        threshold: 95,
+    };
+
+    assert!(filter.box_eq(filter2.as_any()));
+
     assert_eq!(filter.text, "stuff");
     assert_eq!(
         *filter.as_any().downcast_ref::<SimilarityFilter>().unwrap(),

src/filters/wildcard.rs

@@ -1,4 +1,5 @@
 use super::*;
+use crate::url::FeroxUrl;
 
 /// Data holder for two pieces of data needed when auto-filtering out wildcard responses
 ///
@@ -8,7 +9,7 @@ use super::*;
 ///
 /// `size` is size of the response that should be included with filters passed via runtime
 /// configuration and any static wildcard lengths.
-#[derive(Debug, Default, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq)]
 pub struct WildcardFilter {
     /// size of the response that will later be combined with the length of the path of the url
     /// requested
@@ -16,6 +17,32 @@ pub struct WildcardFilter {
 
     /// size of the response that should be included with filters passed via runtime configuration
     pub size: u64,
+
+    /// whether or not the user passed -D on the command line
+    pub(super) dont_filter: bool,
+}
+
+/// implementation of WildcardFilter
+impl WildcardFilter {
+    /// given a boolean representing whether -D was used or not, create a new WildcardFilter
+    pub fn new(dont_filter: bool) -> Self {
+        Self {
+            dont_filter,
+            ..Default::default()
+        }
+    }
+}
+
+/// implement default that populates both values with u64::MAX
+impl Default for WildcardFilter {
+    /// populate both values with u64::MAX
+    fn default() -> Self {
+        Self {
+            dont_filter: false,
+            size: u64::MAX,
+            dynamic: u64::MAX,
+        }
+    }
 }
 
 /// implementation of FeroxFilter for WildcardFilter
@@ -26,14 +53,14 @@ impl FeroxFilter for WildcardFilter {
         log::trace!("enter: should_filter_response({:?} {})", self, response);
 
         // quick return if dont_filter is set
-        if CONFIGURATION.dont_filter {
+        if self.dont_filter {
             // --dont-filter applies specifically to wildcard filters, it is not a 100% catch all
             // for not filtering anything.  As such, it should live in the implementation of
             // a wildcard filter
             return false;
         }
 
-        if self.size > 0 && self.size == response.content_length() {
+        if self.size != u64::MAX && self.size == response.content_length() {
             // static wildcard size found during testing
             // size isn't default, size equals response length, and auto-filter is on
             log::debug!("static wildcard: filtered out {}", response.url());
@@ -41,7 +68,7 @@ impl FeroxFilter for WildcardFilter {
             return true;
         }
 
-        if self.dynamic > 0 {
+        if self.dynamic != u64::MAX {
             // dynamic wildcard offset found during testing
 
             // I'm about to manually split this url path instead of using reqwest::Url's
@@ -49,7 +76,7 @@ impl FeroxFilter for WildcardFilter {
             // except that I don't want an empty string taking up the last index in the
             // event that the url ends with a forward slash.  It's ugly enough to be split
             // into its own function for readability.
-            let url_len = get_url_path_length(&response.url());
+            let url_len = FeroxUrl::path_length_of_url(&response.url());
 
             if url_len + self.dynamic == response.content_length() {
                 log::debug!("dynamic wildcard: filtered out {}", response.url());

src/heuristics.rs

@@ -1,274 +1,250 @@
-use crate::{
-    config::{CONFIGURATION, PROGRESS_PRINTER},
-    filters::WildcardFilter,
-    scanner::should_filter_response,
-    statistics::StatCommand,
-    utils::{ferox_print, format_url, get_url_path_length, make_request, status_colorizer},
-    FeroxResponse,
-};
+use std::sync::Arc;
+
+use anyhow::{bail, Result};
 use console::style;
-use indicatif::ProgressBar;
-use tokio::sync::mpsc::UnboundedSender;
 use uuid::Uuid;
 
+use crate::{
+    config::OutputLevel,
+    event_handlers::{Command, Handles},
+    filters::WildcardFilter,
+    progress::PROGRESS_PRINTER,
+    response::FeroxResponse,
+    skip_fail,
+    url::FeroxUrl,
+    utils::{ferox_print, fmt_err, logged_request, status_colorizer},
+};
+
 /// length of a standard UUID, used when determining wildcard responses
 const UUID_LENGTH: u64 = 32;
 
-/// Simple helper to return a uuid, formatted as lowercase without hyphens
-///
-/// `length` determines the number of uuids to string together. Each uuid
-/// is 32 characters long. So, a length of 1 return a 32 character string,
-/// a length of 2 returns a 64 character string, and so on...
-fn unique_string(length: usize) -> String {
-    log::trace!("enter: unique_string({})", length);
-    let mut ids = vec![];
-
-    for _ in 0..length {
-        ids.push(Uuid::new_v4().to_simple().to_string());
-    }
-
-    let unique_id = ids.join("");
-
-    log::trace!("exit: unique_string -> {}", unique_id);
-    unique_id
+/// wrapper around ugly string formatting
+macro_rules! format_template {
+    ($template:expr, $length:expr) => {
+        format!(
+            $template,
+            status_colorizer("WLD"),
+            "-",
+            "-",
+            "-",
+            style("auto-filtering").yellow(),
+            style($length).cyan(),
+            style("--dont-filter").yellow()
+        )
+    };
 }
 
-/// Tests the given url to see if it issues a wildcard response
-///
-/// In the event that url returns a wildcard response, a
-/// [WildcardFilter](struct.WildcardFilter.html) is created and returned to the caller.
-pub async fn wildcard_test(
-    target_url: &str,
-    bar: ProgressBar,
-    tx_term: UnboundedSender<FeroxResponse>,
-    tx_stats: UnboundedSender<StatCommand>,
-) -> Option<WildcardFilter> {
-    log::trace!(
-        "enter: wildcard_test({:?}, {:?}, {:?}, {:?})",
-        target_url,
-        bar,
-        tx_term,
-        tx_stats
-    );
+/// container for heuristics related info
+pub struct HeuristicTests {
+    /// Handles object for event handler interaction
+    handles: Arc<Handles>,
+}
 
-    if CONFIGURATION.dont_filter {
-        // early return, dont_filter scans don't need tested
-        log::trace!("exit: wildcard_test -> None");
-        return None;
+/// HeuristicTests implementation
+impl HeuristicTests {
+    /// create a new HeuristicTests struct
+    pub fn new(handles: Arc<Handles>) -> Self {
+        Self { handles }
     }
 
-    let tx_term_mwcr1 = tx_term.clone();
-    let tx_term_mwcr2 = tx_term.clone();
-    let tx_stats_mwcr1 = tx_stats.clone();
-    let tx_stats_mwcr2 = tx_stats.clone();
+    /// Simple helper to return a uuid, formatted as lowercase without hyphens
+    ///
+    /// `length` determines the number of uuids to string together. Each uuid
+    /// is 32 characters long. So, a length of 1 return a 32 character string,
+    /// a length of 2 returns a 64 character string, and so on...
+    fn unique_string(&self, length: usize) -> String {
+        log::trace!("enter: unique_string({})", length);
+        let mut ids = vec![];
 
-    if let Some(ferox_response) =
-        make_wildcard_request(&target_url, 1, tx_term_mwcr1, tx_stats_mwcr1).await
-    {
-        bar.inc(1);
+        for _ in 0..length {
+            ids.push(Uuid::new_v4().to_simple().to_string());
+        }
+
+        let unique_id = ids.join("");
+
+        log::trace!("exit: unique_string -> {}", unique_id);
+        unique_id
+    }
+
+    /// wrapper for sending a filter to the filters event handler
+    fn send_filter(&self, filter: WildcardFilter) -> Result<()> {
+        self.handles
+            .filters
+            .send(Command::AddFilter(Box::new(filter)))
+    }
+
+    /// Tests the given url to see if it issues a wildcard response
+    ///
+    /// In the event that url returns a wildcard response, a
+    /// [WildcardFilter](struct.WildcardFilter.html) is created and sent to the filters event
+    /// handler.
+    ///
+    /// Returns the number of times to increment the caller's progress bar
+    pub async fn wildcard(&self, target_url: &str) -> Result<u64> {
+        log::trace!("enter: wildcard_test({:?})", target_url);
+
+        if self.handles.config.dont_filter {
+            // early return, dont_filter scans don't need tested
+            log::trace!("exit: wildcard_test -> 0");
+            return Ok(0);
+        }
+
+        let ferox_url = FeroxUrl::from_string(target_url, self.handles.clone());
+
+        let ferox_response = self.make_wildcard_request(&ferox_url, 1).await?;
 
         // found a wildcard response
-        let mut wildcard = WildcardFilter::default();
+        let mut wildcard = WildcardFilter::new(self.handles.config.dont_filter);
 
         let wc_length = ferox_response.content_length();
 
         if wc_length == 0 {
-            log::trace!("exit: wildcard_test -> Some({:?})", wildcard);
-            return Some(wildcard);
+            log::trace!("exit: wildcard_test -> 1");
+            self.send_filter(wildcard)?;
+            return Ok(1);
         }
 
         // content length of wildcard is non-zero, perform additional tests:
         //   make a second request, with a known-sized (64) longer request
-        if let Some(resp_two) =
-            make_wildcard_request(&target_url, 3, tx_term_mwcr2, tx_stats_mwcr2).await
+        let resp_two = self.make_wildcard_request(&ferox_url, 3).await?;
+
+        let wc2_length = resp_two.content_length();
+
+        if wc2_length == wc_length + (UUID_LENGTH * 2) {
+            // second length is what we'd expect to see if the requested url is
+            // reflected in the response along with some static content; aka custom 404
+            let url_len = ferox_url.path_length()?;
+
+            wildcard.dynamic = wc_length - url_len;
+
+            if matches!(
+                self.handles.config.output_level,
+                OutputLevel::Default | OutputLevel::Quiet
+            ) {
+                let msg = format_template!("{} {:>9} {:>9} {:>9} Wildcard response is dynamic; {} ({} + url length) responses; toggle this behavior by using {}\n", wildcard.dynamic);
+                ferox_print(&msg, &PROGRESS_PRINTER);
+            }
+        } else if wc_length == wc2_length {
+            wildcard.size = wc_length;
+
+            if matches!(
+                self.handles.config.output_level,
+                OutputLevel::Default | OutputLevel::Quiet
+            ) {
+                let msg = format_template!("{} {:>9} {:>9} {:>9} Wildcard response is static; {} {} responses; toggle this behavior by using {}\n", wildcard.size);
+                ferox_print(&msg, &PROGRESS_PRINTER);
+            }
+        }
+
+        self.send_filter(wildcard)?;
+
+        log::trace!("exit: wildcard_test");
+        Ok(2)
+    }
+
+    /// Generates a uuid and appends it to the given target url. The reasoning is that the randomly
+    /// generated unique string should not exist on and be served by the target web server.
+    ///
+    /// Once the unique url is created, the request is sent to the server. If the server responds
+    /// back with a valid status code, the response is considered to be a wildcard response. If that
+    /// wildcard response has a 3xx status code, that redirection location is displayed to the user.
+    async fn make_wildcard_request(
+        &self,
+        target: &FeroxUrl,
+        length: usize,
+    ) -> Result<FeroxResponse> {
+        log::trace!("enter: make_wildcard_request({}, {})", target, length);
+
+        let unique_str = self.unique_string(length);
+        let nonexistent_url = target.format(&unique_str, None)?;
+
+        let response = logged_request(&nonexistent_url.to_owned(), self.handles.clone()).await?;
+
+        if self
+            .handles
+            .config
+            .status_codes
+            .contains(&response.status().as_u16())
         {
-            bar.inc(1);
+            // found a wildcard response
+            let mut ferox_response =
+                FeroxResponse::from(response, true, self.handles.config.output_level).await;
+            ferox_response.set_wildcard(true);
 
-            let wc2_length = resp_two.content_length();
-
-            if wc2_length == wc_length + (UUID_LENGTH * 2) {
-                // second length is what we'd expect to see if the requested url is
-                // reflected in the response along with some static content; aka custom 404
-                let url_len = get_url_path_length(&ferox_response.url());
-
-                wildcard.dynamic = wc_length - url_len;
-
-                if !CONFIGURATION.quiet {
-                    let msg = format!(
-                            "{} {:>9} {:>9} {:>9} Wildcard response is dynamic; {} ({} + url length) responses; toggle this behavior by using {}\n",
-                            status_colorizer("WLD"),
-                            "-",
-                            "-",
-                            "-",
-                            style("auto-filtering").yellow(),
-                            style(wc_length - url_len).cyan(),
-                            style("--dont-filter").yellow()
-                    );
-
-                    ferox_print(&msg, &PROGRESS_PRINTER);
-                }
-            } else if wc_length == wc2_length {
-                wildcard.size = wc_length;
-
-                if !CONFIGURATION.quiet {
-                    let msg = format!(
-                        "{} {:>9} {:>9} {:>9} Wildcard response is static; {} {} responses; toggle this behavior by using {}\n",
-                        status_colorizer("WLD"),
-                        "-",
-                        "-",
-                        "-",
-                        style("auto-filtering").yellow(),
-                        style(wc_length).cyan(),
-                        style("--dont-filter").yellow()
-                    );
-
-                    ferox_print(&msg, &PROGRESS_PRINTER);
-                }
-            }
-        } else {
-            bar.inc(2);
-        }
-
-        log::trace!("exit: wildcard_test -> Some({:?})", wildcard);
-        return Some(wildcard);
-    }
-
-    log::trace!("exit: wildcard_test -> None");
-    None
-}
-
-/// Generates a uuid and appends it to the given target url. The reasoning is that the randomly
-/// generated unique string should not exist on and be served by the target web server.
-///
-/// Once the unique url is created, the request is sent to the server. If the server responds
-/// back with a valid status code, the response is considered to be a wildcard response. If that
-/// wildcard response has a 3xx status code, that redirection location is displayed to the user.
-async fn make_wildcard_request(
-    target_url: &str,
-    length: usize,
-    tx_file: UnboundedSender<FeroxResponse>,
-    tx_stats: UnboundedSender<StatCommand>,
-) -> Option<FeroxResponse> {
-    log::trace!(
-        "enter: make_wildcard_request({}, {}, {:?}, {:?})",
-        target_url,
-        length,
-        tx_file,
-        tx_stats,
-    );
-
-    let unique_str = unique_string(length);
-
-    let nonexistent = match format_url(
-        target_url,
-        &unique_str,
-        CONFIGURATION.add_slash,
-        &CONFIGURATION.queries,
-        None,
-        tx_stats.clone(),
-    ) {
-        Ok(url) => url,
-        Err(e) => {
-            log::error!("{}", e);
-            log::trace!("exit: make_wildcard_request -> None");
-            return None;
-        }
-    };
-
-    match make_request(
-        &CONFIGURATION.client,
-        &nonexistent.to_owned(),
-        tx_stats.clone(),
-    )
-    .await
-    {
-        Ok(response) => {
-            if CONFIGURATION
-                .status_codes
-                .contains(&response.status().as_u16())
+            if self
+                .handles
+                .filters
+                .data
+                .should_filter_response(&ferox_response, self.handles.stats.tx.clone())
             {
-                // found a wildcard response
-                let mut ferox_response = FeroxResponse::from(response, true).await;
-                ferox_response.wildcard = true;
+                bail!("filtered response")
+            }
 
-                if !CONFIGURATION.quiet
-                    && !should_filter_response(&ferox_response, tx_stats.clone())
-                    && tx_file.send(ferox_response.clone()).is_err()
-                {
-                    return None;
+            if matches!(
+                self.handles.config.output_level,
+                OutputLevel::Default | OutputLevel::Quiet
+            ) {
+                let boxed = Box::new(ferox_response.clone());
+                self.handles.output.send(Command::Report(boxed))?;
+            }
+
+            log::trace!("exit: make_wildcard_request -> {}", ferox_response);
+            return Ok(ferox_response);
+        }
+
+        log::trace!("exit: make_wildcard_request -> Err");
+        bail!("uninteresting status code")
+    }
+
+    /// Simply tries to connect to all given sites before starting to scan
+    ///
+    /// In the event that no sites can be reached, the program will exit.
+    ///
+    /// Any urls that are found to be alive are returned to the caller.
+    pub async fn connectivity(&self, target_urls: &[String]) -> Result<Vec<String>> {
+        log::trace!("enter: connectivity_test({:?})", target_urls);
+
+        let mut good_urls = vec![];
+
+        for target_url in target_urls {
+            let url = FeroxUrl::from_string(&target_url, self.handles.clone());
+            let request = skip_fail!(url.format("", None));
+
+            let result = logged_request(&request, self.handles.clone()).await;
+
+            match result {
+                Ok(_) => {
+                    good_urls.push(target_url.to_owned());
                 }
-
-                log::trace!("exit: make_wildcard_request -> {}", ferox_response);
-                return Some(ferox_response);
-            }
-        }
-        Err(e) => {
-            log::warn!("{}", e);
-            log::trace!("exit: make_wildcard_request -> None");
-            return None;
-        }
-    }
-
-    log::trace!("exit: make_wildcard_request -> None");
-    None
-}
-
-/// Simply tries to connect to all given sites before starting to scan
-///
-/// In the event that no sites can be reached, the program will exit.
-///
-/// Any urls that are found to be alive are returned to the caller.
-pub async fn connectivity_test(
-    target_urls: &[String],
-    tx_stats: UnboundedSender<StatCommand>,
-) -> Vec<String> {
-    log::trace!(
-        "enter: connectivity_test({:?}, {:?})",
-        target_urls,
-        tx_stats
-    );
-
-    let mut good_urls = vec![];
-
-    for target_url in target_urls {
-        let request = match format_url(
-            target_url,
-            "",
-            CONFIGURATION.add_slash,
-            &CONFIGURATION.queries,
-            None,
-            tx_stats.clone(),
-        ) {
-            Ok(url) => url,
-            Err(e) => {
-                log::error!("{}", e);
-                continue;
-            }
-        };
-
-        match make_request(&CONFIGURATION.client, &request, tx_stats.clone()).await {
-            Ok(_) => {
-                good_urls.push(target_url.to_owned());
-            }
-            Err(e) => {
-                if !CONFIGURATION.quiet {
-                    ferox_print(
-                        &format!("Could not connect to {}, skipping...", target_url),
-                        &PROGRESS_PRINTER,
-                    );
+                Err(e) => {
+                    if matches!(
+                        self.handles.config.output_level,
+                        OutputLevel::Default | OutputLevel::Quiet
+                    ) {
+                        if e.to_string().contains(":SSL") {
+                            ferox_print(
+                                &format!("Could not connect to {} due to SSL errors (run with -k to ignore), skipping...", target_url),
+                                &PROGRESS_PRINTER,
+                            );
+                        } else {
+                            ferox_print(
+                                &format!("Could not connect to {}, skipping...", target_url),
+                                &PROGRESS_PRINTER,
+                            );
+                        }
+                    }
+                    log::warn!("{}", e);
                 }
-                log::error!("{}", e);
             }
         }
+
+        if good_urls.is_empty() {
+            bail!("Could not connect to any target provided");
+        }
+
+        log::trace!("exit: connectivity_test -> {:?}", good_urls);
+        Ok(good_urls)
     }
-
-    if good_urls.is_empty() {
-        log::error!("Could not connect to any target provided, exiting.");
-    }
-
-    log::trace!("exit: connectivity_test -> {:?}", good_urls);
-
-    good_urls
 }
 
 #[cfg(test)]
@@ -278,16 +254,10 @@ mod tests {
     #[test]
     /// request a unique string of 32bytes * a value returns correct result
     fn heuristics_unique_string_returns_correct_length() {
+        let (handles, _) = Handles::for_testing(None, None);
+        let tester = HeuristicTests::new(Arc::new(handles));
         for i in 0..10 {
-            assert_eq!(unique_string(i).len(), i * 32);
+            assert_eq!(tester.unique_string(i).len(), i * 32);
         }
     }
-
-    #[test]
-    /// simply test the default values for wildcardfilter, expect 0, 0
-    fn heuristics_wildcardfilter_dafaults() {
-        let wcf = WildcardFilter::default();
-        assert_eq!(wcf.size, 0);
-        assert_eq!(wcf.dynamic, 0);
-    }
 }

src/lib.rs

@@ -1,53 +1,46 @@
-pub mod utils;
+use anyhow::Result;
+use reqwest::StatusCode;
+use tokio::{
+    sync::mpsc::{UnboundedReceiver, UnboundedSender},
+    task::JoinHandle,
+};
+
+use crate::event_handlers::Command;
+
 pub mod banner;
-pub mod client;
 pub mod config;
-pub mod extractor;
+mod client;
+pub mod event_handlers;
 pub mod filters;
 pub mod heuristics;
 pub mod logger;
-pub mod parser;
+mod parser;
 pub mod progress;
-pub mod reporter;
 pub mod scan_manager;
 pub mod scanner;
 pub mod statistics;
+mod traits;
+pub mod utils;
+mod extractor;
+mod macros;
+mod url;
+mod response;
+mod message;
 
-use crate::utils::{get_url_path_length, status_colorizer};
-use console::{style, Color};
-use reqwest::header::{HeaderName, HeaderValue};
-use reqwest::{header::HeaderMap, Response, StatusCode, Url};
-use serde::{ser::SerializeStruct, Deserialize, Deserializer, Serialize, Serializer};
-use serde_json::Value;
-use std::collections::HashMap;
-use std::convert::{TryFrom, TryInto};
-use std::str::FromStr;
-use std::{error, fmt};
-use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender};
+/// Alias for tokio::sync::mpsc::UnboundedSender<Command>
+pub(crate) type CommandSender = UnboundedSender<Command>;
 
-/// Generic Result type to ease error handling in async contexts
-pub type FeroxResult<T> = std::result::Result<T, Box<dyn error::Error + Send + Sync + 'static>>;
+/// Alias for tokio::sync::mpsc::UnboundedSender<Command>
+pub(crate) type CommandReceiver = UnboundedReceiver<Command>;
 
-/// Simple Error implementation to allow for custom error returns
-#[derive(Debug, Default)]
-pub struct FeroxError {
-    /// fancy string that can be printed via Display
-    pub message: String,
-}
-
-impl error::Error for FeroxError {}
-
-impl fmt::Display for FeroxError {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "{}", &self.message)
-    }
-}
+/// Alias for tokio::task::JoinHandle<anyhow::Result<()>>
+pub(crate) type Joiner = JoinHandle<Result<()>>;
 
 /// Generic mpsc::unbounded_channel type to tidy up some code
-pub type FeroxChannel<T> = (UnboundedSender<T>, UnboundedReceiver<T>);
+pub(crate) type FeroxChannel<T> = (UnboundedSender<T>, UnboundedReceiver<T>);
 
 /// Version pulled from Cargo.toml at compile time
-pub const VERSION: &str = env!("CARGO_PKG_VERSION");
+pub(crate) const VERSION: &str = env!("CARGO_PKG_VERSION");
 
 /// Maximum number of file descriptors that can be opened during a scan
 pub const DEFAULT_OPEN_FILE_LIMIT: usize = 8192;
@@ -64,7 +57,10 @@ pub const DEFAULT_WORDLIST: &str =
     "/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt";
 
 /// Number of milliseconds to wait between polls of `PAUSE_SCAN` when user pauses a scan
-pub static SLEEP_DURATION: u64 = 500;
+pub(crate) const SLEEP_DURATION: u64 = 500;
+
+/// The percentage of requests as errors it takes to be deemed too high
+pub const HIGH_ERROR_RATIO: f64 = 0.90;
 
 /// Default list of status codes to report
 ///
@@ -94,443 +90,6 @@ pub const DEFAULT_STATUS_CODES: [StatusCode; 9] = [
 /// Expected location is in the same directory as the feroxbuster binary.
 pub const DEFAULT_CONFIG_NAME: &str = "ferox-config.toml";
 
-/// FeroxSerialize trait; represents different types that are Serialize and also implement
-/// as_str / as_json methods
-pub trait FeroxSerialize: Serialize {
-    /// Return a String representation of the object, generally the human readable version of the
-    /// implementor
-    fn as_str(&self) -> String;
-
-    /// Return an NDJSON representation of the object
-    fn as_json(&self) -> String;
-}
-
-/// A `FeroxResponse`, derived from a `Response` to a submitted `Request`
-#[derive(Debug, Clone)]
-pub struct FeroxResponse {
-    /// The final `Url` of this `FeroxResponse`
-    url: Url,
-
-    /// The `StatusCode` of this `FeroxResponse`
-    status: StatusCode,
-
-    /// The full response text
-    text: String,
-
-    /// The content-length of this response, if known
-    content_length: u64,
-
-    /// The number of lines contained in the body of this response, if known
-    line_count: usize,
-
-    /// The number of words contained in the body of this response, if known
-    word_count: usize,
-
-    /// The `Headers` of this `FeroxResponse`
-    headers: HeaderMap,
-
-    /// Wildcard response status
-    wildcard: bool,
-}
-
-/// Implement Display for FeroxResponse
-impl fmt::Display for FeroxResponse {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(
-            f,
-            "FeroxResponse {{ url: {}, status: {}, content-length: {} }}",
-            self.url(),
-            self.status(),
-            self.content_length()
-        )
-    }
-}
-
-/// `FeroxResponse` implementation
-impl FeroxResponse {
-    /// Get the `StatusCode` of this `FeroxResponse`
-    pub fn status(&self) -> &StatusCode {
-        &self.status
-    }
-
-    /// Get the final `Url` of this `FeroxResponse`.
-    pub fn url(&self) -> &Url {
-        &self.url
-    }
-
-    /// Get the full response text
-    pub fn text(&self) -> &str {
-        &self.text
-    }
-
-    /// Get the `Headers` of this `FeroxResponse`
-    pub fn headers(&self) -> &HeaderMap {
-        &self.headers
-    }
-
-    /// Get the content-length of this response, if known
-    pub fn content_length(&self) -> u64 {
-        self.content_length
-    }
-
-    /// Set `FeroxResponse`'s `url` attribute, has no affect if an error occurs
-    pub fn set_url(&mut self, url: &str) {
-        match Url::parse(&url) {
-            Ok(url) => {
-                self.url = url;
-            }
-            Err(e) => {
-                log::error!("Could not parse {} into a Url: {}", url, e);
-            }
-        };
-    }
-
-    /// Make a reasonable guess at whether the response is a file or not
-    ///
-    /// Examines the last part of a path to determine if it has an obvious extension
-    /// i.e. http://localhost/some/path/stuff.js where stuff.js indicates a file
-    ///
-    /// Additionally, inspects query parameters, as they're also often indicative of a file
-    pub fn is_file(&self) -> bool {
-        let has_extension = match self.url.path_segments() {
-            Some(path) => {
-                if let Some(last) = path.last() {
-                    last.contains('.') // last segment has some sort of extension, probably
-                } else {
-                    false
-                }
-            }
-            None => false,
-        };
-
-        self.url.query_pairs().count() > 0 || has_extension
-    }
-
-    /// Returns line count of the response text.
-    pub fn line_count(&self) -> usize {
-        self.line_count
-    }
-
-    /// Returns word count of the response text.
-    pub fn word_count(&self) -> usize {
-        self.word_count
-    }
-
-    /// Create a new `FeroxResponse` from the given `Response`
-    pub async fn from(response: Response, read_body: bool) -> Self {
-        let url = response.url().clone();
-        let status = response.status();
-        let headers = response.headers().clone();
-        let content_length = response.content_length().unwrap_or(0);
-
-        let text = if read_body {
-            // .text() consumes the response, must be called last
-            // additionally, --extract-links is currently the only place we use the body of the
-            // response, so we forego the processing if not performing extraction
-            match response.text().await {
-                // await the response's body
-                Ok(text) => text,
-                Err(e) => {
-                    log::error!("Could not parse body from response: {}", e);
-                    String::new()
-                }
-            }
-        } else {
-            String::new()
-        };
-
-        let line_count = text.lines().count();
-        let word_count = text.lines().map(|s| s.split_whitespace().count()).sum();
-
-        FeroxResponse {
-            url,
-            status,
-            content_length,
-            text,
-            headers,
-            line_count,
-            word_count,
-            wildcard: false,
-        }
-    }
-}
-
-/// Implement FeroxSerialusize::from(ize for FeroxRespons)e
-impl FeroxSerialize for FeroxResponse {
-    /// Simple wrapper around create_report_string
-    fn as_str(&self) -> String {
-        let lines = self.line_count().to_string();
-        let words = self.word_count().to_string();
-        let chars = self.content_length().to_string();
-        let status = self.status().as_str();
-        let wild_status = status_colorizer("WLD");
-
-        if self.wildcard {
-            // response is a wildcard, special messages abound when this is the case...
-
-            // create the base message
-            let mut message = format!(
-                "{} {:>8}l {:>8}w {:>8}c Got {} for {} (url length: {})\n",
-                wild_status,
-                lines,
-                words,
-                chars,
-                status_colorizer(&status),
-                self.url(),
-                get_url_path_length(&self.url())
-            );
-
-            if self.status().is_redirection() {
-                // when it's a redirect, show where it goes, if possible
-                if let Some(next_loc) = self.headers().get("Location") {
-                    let next_loc_str = next_loc.to_str().unwrap_or("Unknown");
-
-                    let redirect_msg = format!(
-                        "{} {:>9} {:>9} {:>9} {} redirects to => {}\n",
-                        wild_status,
-                        "-",
-                        "-",
-                        "-",
-                        self.url(),
-                        next_loc_str
-                    );
-
-                    message.push_str(&redirect_msg);
-                }
-            }
-
-            // base message + redirection message (if appropriate)
-            message
-        } else {
-            // not a wildcard, just create a normal entry
-            utils::create_report_string(
-                self.status.as_str(),
-                &lines,
-                &words,
-                &chars,
-                self.url().as_str(),
-            )
-        }
-    }
-
-    /// Create an NDJSON representation of the FeroxResponse
-    ///
-    /// (expanded for clarity)
-    /// ex:
-    /// {
-    ///    "type":"response",
-    ///    "url":"https://localhost.com/images",
-    ///    "path":"/images",
-    ///    "status":301,
-    ///    "content_length":179,
-    ///    "line_count":10,
-    ///    "word_count":16,
-    ///    "headers":{
-    ///       "x-content-type-options":"nosniff",
-    ///       "strict-transport-security":"max-age=31536000; includeSubDomains",
-    ///       "x-frame-options":"SAMEORIGIN",
-    ///       "connection":"keep-alive",
-    ///       "server":"nginx/1.16.1",
-    ///       "content-type":"text/html; charset=UTF-8",
-    ///       "referrer-policy":"origin-when-cross-origin",
-    ///       "content-security-policy":"default-src 'none'",
-    ///       "access-control-allow-headers":"X-Requested-With",
-    ///       "x-xss-protection":"1; mode=block",
-    ///       "content-length":"179",
-    ///       "date":"Mon, 23 Nov 2020 15:33:24 GMT",
-    ///       "location":"/images/",
-    ///       "access-control-allow-origin":"https://localhost.com"
-    ///    }
-    /// }\n
-    fn as_json(&self) -> String {
-        if let Ok(mut json) = serde_json::to_string(&self) {
-            json.push('\n');
-            json
-        } else {
-            format!("{{\"error\":\"could not convert {} to json\"}}", self.url())
-        }
-    }
-}
-
-/// Serialize implementation for FeroxResponse
-impl Serialize for FeroxResponse {
-    /// Function that handles serialization of a FeroxResponse to NDJSON
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        let mut headers = HashMap::new();
-        let mut state = serializer.serialize_struct("FeroxResponse", 7)?;
-
-        // need to convert the HeaderMap to a HashMap in order to pass it to the serializer
-        for (key, value) in &self.headers {
-            let k = key.as_str().to_owned();
-            let v = String::from_utf8_lossy(value.as_bytes());
-            headers.insert(k, v);
-        }
-
-        state.serialize_field("type", "response")?;
-        state.serialize_field("url", self.url.as_str())?;
-        state.serialize_field("path", self.url.path())?;
-        state.serialize_field("wildcard", &self.wildcard)?;
-        state.serialize_field("status", &self.status.as_u16())?;
-        state.serialize_field("content_length", &self.content_length)?;
-        state.serialize_field("line_count", &self.line_count)?;
-        state.serialize_field("word_count", &self.word_count)?;
-        state.serialize_field("headers", &headers)?;
-
-        state.end()
-    }
-}
-
-/// Deserialize implementation for FeroxResponse
-impl<'de> Deserialize<'de> for FeroxResponse {
-    /// Deserialize a FeroxResponse from a serde_json::Value
-    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
-    where
-        D: Deserializer<'de>,
-    {
-        let mut response = Self {
-            url: Url::parse("http://localhost").unwrap(),
-            status: StatusCode::OK,
-            text: String::new(),
-            content_length: 0,
-            headers: HeaderMap::new(),
-            wildcard: false,
-            line_count: 0,
-            word_count: 0,
-        };
-
-        let map: HashMap<String, Value> = HashMap::deserialize(deserializer)?;
-
-        for (key, value) in &map {
-            match key.as_str() {
-                "url" => {
-                    if let Some(url) = value.as_str() {
-                        if let Ok(parsed) = Url::parse(url) {
-                            response.url = parsed;
-                        }
-                    }
-                }
-                "status" => {
-                    if let Some(num) = value.as_u64() {
-                        if let Ok(smaller) = u16::try_from(num) {
-                            if let Ok(status) = StatusCode::from_u16(smaller) {
-                                response.status = status;
-                            }
-                        }
-                    }
-                }
-                "content_length" => {
-                    if let Some(num) = value.as_u64() {
-                        response.content_length = num;
-                    }
-                }
-                "line_count" => {
-                    if let Some(num) = value.as_u64() {
-                        response.line_count = num.try_into().unwrap_or_default();
-                    }
-                }
-                "word_count" => {
-                    if let Some(num) = value.as_u64() {
-                        response.word_count = num.try_into().unwrap_or_default();
-                    }
-                }
-                "headers" => {
-                    let mut headers = HeaderMap::<HeaderValue>::default();
-
-                    if let Some(map_headers) = value.as_object() {
-                        for (h_key, h_value) in map_headers {
-                            let h_value_str = h_value.as_str().unwrap_or("");
-                            let h_name = HeaderName::from_str(h_key)
-                                .unwrap_or_else(|_| HeaderName::from_str("Unknown").unwrap());
-                            let h_value_parsed = HeaderValue::from_str(h_value_str)
-                                .unwrap_or_else(|_| HeaderValue::from_str("Unknown").unwrap());
-                            headers.insert(h_name, h_value_parsed);
-                        }
-                    }
-
-                    response.headers = headers;
-                }
-                "wildcard" => {
-                    if let Some(result) = value.as_bool() {
-                        response.wildcard = result;
-                    }
-                }
-                _ => {}
-            }
-        }
-
-        Ok(response)
-    }
-}
-
-#[derive(Serialize, Deserialize, Default)]
-/// Representation of a log entry, can be represented as a human readable string or JSON
-pub struct FeroxMessage {
-    #[serde(rename = "type")]
-    /// Name of this type of struct, used for serialization, i.e. `{"type":"log"}`
-    kind: String,
-
-    /// The log message
-    pub message: String,
-
-    /// The log level
-    pub level: String,
-
-    /// The number of seconds elapsed since the scan started
-    pub time_offset: f32,
-
-    /// The module from which log::* was called
-    pub module: String,
-}
-
-/// Implementation of FeroxMessage
-impl FeroxSerialize for FeroxMessage {
-    /// Create an NDJSON representation of the log message
-    ///
-    /// (expanded for clarity)
-    /// ex:
-    /// {
-    ///   "type": "log",
-    ///   "message": "Sent https://localhost/api to file handler",
-    ///   "level": "DEBUG",
-    ///   "time_offset": 0.86333454,
-    ///   "module": "feroxbuster::reporter"
-    /// }\n
-    fn as_json(&self) -> String {
-        if let Ok(mut json) = serde_json::to_string(&self) {
-            json.push('\n');
-            json
-        } else {
-            String::from("{\"error\":\"could not convert to json\"}")
-        }
-    }
-
-    /// Create a string representation of the log message
-    ///
-    /// ex:  301       10l       16w      173c https://localhost/api
-    fn as_str(&self) -> String {
-        let (level_name, level_color) = match self.level.as_str() {
-            "ERROR" => ("ERR", Color::Red),
-            "WARN" => ("WRN", Color::Red),
-            "INFO" => ("INF", Color::Cyan),
-            "DEBUG" => ("DBG", Color::Yellow),
-            "TRACE" => ("TRC", Color::Magenta),
-            "WILDCARD" => ("WLD", Color::Cyan),
-            _ => ("UNK", Color::White),
-        };
-
-        format!(
-            "{} {:10.03} {} {}\n",
-            style(level_name).bg(level_color).black(),
-            style(self.time_offset).dim(),
-            self.module,
-            style(&self.message).dim(),
-        )
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -555,46 +114,4 @@ mod tests {
     fn default_version() {
         assert_eq!(VERSION, env!("CARGO_PKG_VERSION"));
     }
-
-    #[test]
-    /// test as_str method of FeroxMessage
-    fn ferox_message_as_str_returns_string_with_newline() {
-        let message = FeroxMessage {
-            message: "message".to_string(),
-            module: "utils".to_string(),
-            time_offset: 1.0,
-            level: "INFO".to_string(),
-            kind: "log".to_string(),
-        };
-        let message_str = message.as_str();
-
-        assert!(message_str.contains("INF"));
-        assert!(message_str.contains("1.000"));
-        assert!(message_str.contains("utils"));
-        assert!(message_str.contains("message"));
-        assert!(message_str.ends_with('\n'));
-    }
-
-    #[test]
-    /// test as_json method of FeroxMessage
-    fn ferox_message_as_json_returns_json_representation_of_ferox_message_with_newline() {
-        let message = FeroxMessage {
-            message: "message".to_string(),
-            module: "utils".to_string(),
-            time_offset: 1.0,
-            level: "INFO".to_string(),
-            kind: "log".to_string(),
-        };
-
-        let message_str = message.as_json();
-
-        let error_margin = f32::EPSILON;
-
-        let json: FeroxMessage = serde_json::from_str(&message_str).unwrap();
-        assert_eq!(json.module, message.module);
-        assert_eq!(json.message, message.message);
-        assert!((json.time_offset - message.time_offset).abs() < error_margin);
-        assert_eq!(json.level, message.level);
-        assert_eq!(json.kind, message.kind);
-    }
 }

src/logger.rs

@@ -1,24 +1,31 @@
-use crate::{
-    config::{CONFIGURATION, PROGRESS_PRINTER},
-    reporter::safe_file_write,
-    utils::open_file,
-    FeroxMessage, FeroxSerialize,
-};
-use env_logger::Builder;
 use std::env;
+use std::fs::OpenOptions;
+use std::io::BufWriter;
+use std::sync::{Arc, RwLock};
 use std::time::Instant;
 
+use anyhow::{Context, Result};
+use env_logger::Builder;
+
+use crate::{
+    config::Configuration,
+    message::FeroxMessage,
+    progress::PROGRESS_PRINTER,
+    traits::FeroxSerialize,
+    utils::{fmt_err, write_to},
+};
+
 /// Create a customized instance of
 /// [env_logger::Logger](https://docs.rs/env_logger/latest/env_logger/struct.Logger.html)
 /// with timer offset/color and set the log level based on `verbosity`
-pub fn initialize(verbosity: u8) {
+pub fn initialize(config: Arc<Configuration>) -> Result<()> {
     // use occurrences of -v on commandline to or verbosity = N in feroxconfig.toml to set
     // log level for the application; respects already specified RUST_LOG environment variable
     match env::var("RUST_LOG") {
         Ok(_) => {} // RUST_LOG found, don't override
         Err(_) => {
             // only set log level based on verbosity when RUST_LOG variable doesn't exist
-            match verbosity {
+            match config.verbosity {
                 0 => (),
                 1 => env::set_var("RUST_LOG", "warn"),
                 2 => env::set_var("RUST_LOG", "info"),
@@ -31,12 +38,22 @@ pub fn initialize(verbosity: u8) {
     let start = Instant::now();
     let mut builder = Builder::from_default_env();
 
-    let debug_file = open_file(&CONFIGURATION.debug_log);
+    let file = if !config.debug_log.is_empty() {
+        let f = OpenOptions::new() // std fs
+            .create(true)
+            .append(true)
+            .open(&config.debug_log)
+            .with_context(|| fmt_err(&format!("Could not open {}", &config.debug_log)))?;
+
+        let mut writer = BufWriter::new(f);
 
-    if let Some(buffered_file) = debug_file.clone() {
         // write out the configuration to the debug file if it exists
-        safe_file_write(&*CONFIGURATION, buffered_file, CONFIGURATION.json);
-    }
+        write_to(&*config, &mut writer, config.json)?;
+
+        Some(Arc::new(RwLock::new(writer)))
+    } else {
+        None
+    };
 
     builder
         .format(move |_, record| {
@@ -50,11 +67,15 @@ pub fn initialize(verbosity: u8) {
 
             PROGRESS_PRINTER.println(&log_entry.as_str());
 
-            if let Some(buffered_file) = debug_file.clone() {
-                safe_file_write(&log_entry, buffered_file, CONFIGURATION.json);
+            if let Some(buffered_file) = file.clone() {
+                if let Ok(mut unlocked) = buffered_file.write() {
+                    let _ = write_to(&log_entry, &mut unlocked, config.json);
+                }
             }
 
             Ok(())
         })
         .init();
+
+    Ok(())
 }

src/macros.rs

@@ -0,0 +1,23 @@
+#![macro_use]
+
+#[macro_export]
+/// wrapper to improve code readability
+macro_rules! send_command {
+    ($tx:expr, $value:expr) => {
+        $tx.send($value).unwrap_or_default();
+    };
+}
+
+#[macro_export]
+/// while looping, check for a Result, if Ok return the value, if Err, continue
+macro_rules! skip_fail {
+    ($res:expr) => {
+        match $res {
+            Ok(val) => val,
+            Err(e) => {
+                log::warn!("{}", fmt_err(&format!("{}; skipping...", e)));
+                continue;
+            }
+        }
+    };
+}

src/main.rs

@@ -1,96 +1,60 @@
-use crossterm::event::{self, Event, KeyCode};
+use std::{
+    collections::HashSet,
+    env::args,
+    fs::File,
+    io::{stderr, BufRead, BufReader},
+    ops::Index,
+    process::Command,
+    sync::{atomic::Ordering, Arc},
+};
+
+use anyhow::{bail, Context, Result};
+use futures::StreamExt;
+use tokio::{
+    io,
+    sync::{oneshot, Semaphore},
+};
+use tokio_util::codec::{FramedRead, LinesCodec};
+
 use feroxbuster::{
-    banner,
-    config::{CONFIGURATION, PROGRESS_BAR, PROGRESS_PRINTER},
-    heuristics, logger,
-    progress::{add_bar, BarType},
-    reporter,
-    scan_manager::{self, ScanStatus, PAUSE_SCAN},
-    scanner::{self, scan_url, SCANNED_URLS},
-    statistics::{
-        self,
-        StatCommand::{self, CreateBar, LoadStats, UpdateUsizeField},
-        StatField::InitialTargets,
-        Stats,
+    banner::{Banner, UPDATE_URL},
+    config::{Configuration, OutputLevel},
+    event_handlers::{
+        Command::{CreateBar, Exit, JoinTasks, LoadStats, ScanInitialUrls, UpdateWordlist},
+        FiltersHandler, Handles, ScanHandler, StatsHandler, Tasks, TermInputHandler,
+        TermOutHandler, SCAN_COMPLETE,
     },
-    update_stat,
-    utils::{ferox_print, get_current_depth, module_colorizer, status_colorizer},
-    FeroxError, FeroxResponse, FeroxResult, SLEEP_DURATION, VERSION,
+    filters, heuristics, logger,
+    progress::{PROGRESS_BAR, PROGRESS_PRINTER},
+    scan_manager::{self},
+    scanner,
+    utils::fmt_err,
 };
 #[cfg(not(target_os = "windows"))]
 use feroxbuster::{utils::set_open_file_limit, DEFAULT_OPEN_FILE_LIMIT};
-use futures::StreamExt;
-use std::{
-    collections::HashSet,
-    convert::TryInto,
-    fs::File,
-    io::{stderr, BufRead, BufReader},
-    process,
-    sync::{
-        atomic::{AtomicBool, Ordering},
-        Arc,
-    },
-    thread::sleep,
-    time::Duration,
-};
-use tokio::{io, sync::mpsc::UnboundedSender, task::JoinHandle};
-use tokio_util::codec::{FramedRead, LinesCodec};
+use lazy_static::lazy_static;
+use regex::Regex;
 
-/// Atomic boolean flag, used to determine whether or not the terminal input handler should exit
-pub static SCAN_COMPLETE: AtomicBool = AtomicBool::new(false);
-
-/// Handles specific key events triggered by the user over stdin
-fn terminal_input_handler() {
-    log::trace!("enter: terminal_input_handler");
-
-    loop {
-        if PAUSE_SCAN.load(Ordering::Relaxed) {
-            // if the scan is already paused, we don't want this event poller fighting the user
-            // over stdin
-            sleep(Duration::from_millis(SLEEP_DURATION));
-        } else if event::poll(Duration::from_millis(SLEEP_DURATION)).unwrap_or(false) {
-            // It's guaranteed that the `read()` won't block when the `poll()`
-            // function returns `true`
-
-            if let Ok(key_pressed) = event::read() {
-                // ignore any other keys
-                if key_pressed == Event::Key(KeyCode::Enter.into()) {
-                    // if the user presses Enter, set PAUSE_SCAN to true. The interactive menu
-                    // will be triggered and will handle setting PAUSE_SCAN to false
-                    PAUSE_SCAN.store(true, Ordering::Release);
-                }
-            }
-        } else {
-            // Timeout expired and no `Event` is available; use the timeout to check SCAN_COMPLETE
-            if SCAN_COMPLETE.load(Ordering::Relaxed) {
-                // scan has been marked complete by main, time to exit the loop
-                break;
-            }
-        }
-    }
-    log::trace!("exit: terminal_input_handler");
+lazy_static! {
+    /// Limits the number of parallel scans active at any given time when using --parallel
+    static ref PARALLEL_LIMITER: Semaphore = Semaphore::new(0);
 }
 
 /// Create a HashSet of Strings from the given wordlist then stores it inside an Arc
-fn get_unique_words_from_wordlist(path: &str) -> FeroxResult<Arc<HashSet<String>>> {
+fn get_unique_words_from_wordlist(path: &str) -> Result<Arc<HashSet<String>>> {
     log::trace!("enter: get_unique_words_from_wordlist({})", path);
 
-    let file = match File::open(&path) {
-        Ok(f) => f,
-        Err(e) => {
-            log::error!("Could not open wordlist: {}", e);
-            log::trace!("exit: get_unique_words_from_wordlist -> {}", e);
-
-            return Err(Box::new(e));
-        }
-    };
+    let file = File::open(&path).with_context(|| format!("Could not open {}", path))?;
 
     let reader = BufReader::new(file);
 
     let mut words = HashSet::new();
 
     for line in reader.lines() {
-        let result = line?;
+        let result = match line {
+            Ok(read_line) => read_line,
+            Err(_) => continue,
+        };
 
         if result.starts_with('#') || result.is_empty() {
             continue;
@@ -108,37 +72,27 @@ fn get_unique_words_from_wordlist(path: &str) -> FeroxResult<Arc<HashSet<String>
 }
 
 /// Determine whether it's a single url scan or urls are coming from stdin, then scan as needed
-async fn scan(
-    targets: Vec<String>,
-    stats: Arc<Stats>,
-    tx_term: UnboundedSender<FeroxResponse>,
-    tx_file: UnboundedSender<FeroxResponse>,
-    tx_stats: UnboundedSender<StatCommand>,
-) -> FeroxResult<()> {
-    log::trace!(
-        "enter: scan({:?}, {:?}, {:?}, {:?}, {:?})",
-        targets,
-        stats,
-        tx_term,
-        tx_file,
-        tx_stats
-    );
+async fn scan(targets: Vec<String>, handles: Arc<Handles>) -> Result<()> {
+    log::trace!("enter: scan({:?}, {:?})", targets, handles);
     // cloning an Arc is cheap (it's basically a pointer into the heap)
     // so that will allow for cheap/safe sharing of a single wordlist across multi-target scans
     // as well as additional directories found as part of recursion
-    let words =
-        tokio::spawn(async move { get_unique_words_from_wordlist(&CONFIGURATION.wordlist) })
-            .await??;
+
+    let words = {
+        let words_handles = handles.clone();
+        tokio::spawn(async move { get_unique_words_from_wordlist(&words_handles.config.wordlist) })
+            .await??
+    };
 
     if words.len() == 0 {
-        let err = FeroxError {
-            message: format!("Did not find any words in {}", CONFIGURATION.wordlist),
-        };
-
-        return Err(Box::new(err));
+        bail!("Did not find any words in {}", handles.config.wordlist);
     }
 
-    scanner::initialize(words.len(), &CONFIGURATION, tx_stats.clone()).await;
+    let scanned_urls = handles.ferox_scans()?;
+
+    handles.send_scan_command(UpdateWordlist(words.clone()))?;
+
+    scanner::initialize(words.len(), handles.clone()).await?;
 
     // at this point, the stat thread's progress bar can be created; things that needed to happen
     // first:
@@ -146,70 +100,35 @@ async fn scan(
     // - scanner initialized (this sent expected requests per directory to the stats thread, which
     //   having been set, makes it so the progress bar doesn't flash as full before anything has
     //   even happened
-    update_stat!(tx_stats, CreateBar);
+    if matches!(handles.config.output_level, OutputLevel::Default) {
+        // only create the bar if no --silent|--quiet
+        handles.stats.send(CreateBar)?;
 
-    if CONFIGURATION.resumed {
-        update_stat!(tx_stats, LoadStats(CONFIGURATION.resume_from.clone()));
-
-        SCANNED_URLS.print_known_responses();
-
-        if let Ok(scans) = SCANNED_URLS.scans.lock() {
-            for scan in scans.iter() {
-                if let Ok(locked_scan) = scan.lock() {
-                    if matches!(locked_scan.status, ScanStatus::Complete) {
-                        // these scans are complete, and just need to be shown to the user
-                        let pb = add_bar(
-                            &locked_scan.url,
-                            words.len().try_into().unwrap_or_default(),
-                            BarType::Message,
-                        );
-                        pb.finish();
-                    }
-                }
-            }
-        }
+        // blocks until the bar is created / avoids race condition in first two bars
+        handles.stats.sync().await?;
     }
 
-    let mut tasks = vec![];
-
-    for target in targets {
-        let word_clone = words.clone();
-        let term_clone = tx_term.clone();
-        let file_clone = tx_file.clone();
-        let tx_stats_clone = tx_stats.clone();
-        let stats_clone = stats.clone();
-
-        let task = tokio::spawn(async move {
-            let base_depth = get_current_depth(&target);
-            scan_url(
-                &target,
-                word_clone,
-                base_depth,
-                stats_clone,
-                term_clone,
-                file_clone,
-                tx_stats_clone,
-            )
-            .await;
-        });
-
-        tasks.push(task);
+    if handles.config.resumed {
+        // display what has already been completed
+        scanned_urls.print_known_responses();
+        scanned_urls.print_completed_bars(words.len())?;
     }
 
-    // drive execution of all accumulated futures
-    futures::future::join_all(tasks).await;
+    log::debug!("sending {:?} to be scanned as initial targets", targets);
+    handles.send_scan_command(ScanInitialUrls(targets))?;
+
     log::trace!("exit: scan");
 
     Ok(())
 }
 
 /// Get targets from either commandline or stdin, pass them back to the caller as a Result<Vec>
-async fn get_targets() -> FeroxResult<Vec<String>> {
-    log::trace!("enter: get_targets");
+async fn get_targets(handles: Arc<Handles>) -> Result<Vec<String>> {
+    log::trace!("enter: get_targets({:?})", handles);
 
     let mut targets = vec![];
 
-    if CONFIGURATION.stdin {
+    if handles.config.stdin {
         // got targets from stdin, i.e. cat sites | ./feroxbuster ...
         // just need to read the targets from stdin and spawn a future for each target found
         let stdin = io::stdin(); // tokio's stdin, not std
@@ -218,24 +137,25 @@ async fn get_targets() -> FeroxResult<Vec<String>> {
         while let Some(line) = reader.next().await {
             targets.push(line?);
         }
-    } else if CONFIGURATION.resumed {
+    } else if handles.config.resumed {
         // resume-from can't be used with --url, and --stdin is marked false for every resumed
         // scan, making it mutually exclusive from either of the other two options
-        if let Ok(scans) = SCANNED_URLS.scans.lock() {
+        let ferox_scans = handles.ferox_scans()?;
+
+        if let Ok(scans) = ferox_scans.scans.read() {
             for scan in scans.iter() {
-                // SCANNED_URLS gets deserialized scans added to it at program start if --resume-from
+                // ferox_scans gets deserialized scans added to it at program start if --resume-from
                 // is used, so scans that aren't marked complete still need to be scanned
-                if let Ok(locked_scan) = scan.lock() {
-                    if matches!(locked_scan.status, ScanStatus::Complete) {
-                        // this one's already done, ignore it
-                        continue;
-                    }
-                    targets.push(locked_scan.url.to_owned());
+                if scan.is_complete() {
+                    // this one's already done, ignore it
+                    continue;
                 }
+
+                targets.push(scan.url().to_owned());
             }
-        }
+        };
     } else {
-        targets.push(CONFIGURATION.target_url.clone());
+        targets.push(handles.config.target_url.clone());
     }
 
     log::trace!("exit: get_targets -> {:?}", targets);
@@ -245,7 +165,7 @@ async fn get_targets() -> FeroxResult<Vec<String>> {
 
 /// async main called from real main, broken out in this way to allow for some synchronous code
 /// to be executed before bringing the tokio runtime online
-async fn wrapped_main() {
+async fn wrapped_main(config: Arc<Configuration>) -> Result<()> {
     // join can only be called once, otherwise it causes the thread to panic
     tokio::task::spawn_blocking(move || {
         // ok, lazy_static! uses (unsurprisingly in retrospect) a lazy loading model where the
@@ -259,17 +179,34 @@ async fn wrapped_main() {
         PROGRESS_BAR.join().unwrap();
     });
 
-    let (stats, tx_stats, stats_handle) = statistics::initialize();
+    // spawn all event handlers, expect back a JoinHandle and a *Handle to the specific event
+    let (stats_task, stats_handle) = StatsHandler::initialize(config.clone());
+    let (filters_task, filters_handle) = FiltersHandler::initialize();
+    let (out_task, out_handle) =
+        TermOutHandler::initialize(config.clone(), stats_handle.tx.clone());
 
-    if !CONFIGURATION.time_limit.is_empty() {
+    // bundle up all the disparate handles and JoinHandles (tasks)
+    let handles = Arc::new(Handles::new(
+        stats_handle,
+        filters_handle,
+        out_handle,
+        config.clone(),
+    ));
+
+    let (scan_task, scan_handle) = ScanHandler::initialize(handles.clone());
+
+    handles.set_scan_handle(scan_handle); // must be done after Handles initialization
+
+    filters::initialize(handles.clone()).await?; // send user-supplied filters to the handler
+
+    // create new Tasks object, each of these handles is one that will be joined on later
+    let tasks = Tasks::new(out_task, stats_task, filters_task, scan_task);
+
+    if !config.time_limit.is_empty() {
         // --time-limit value not an empty string, need to kick off the thread that enforces
         // the limit
-
-        let max_time_stats = stats.clone();
-
-        tokio::spawn(async move {
-            scan_manager::start_max_time_thread(&CONFIGURATION.time_limit, max_time_stats).await
-        });
+        let time_handles = handles.clone();
+        tokio::spawn(async move { scan_manager::start_max_time_thread(time_handles).await });
     }
 
     // can't trace main until after logger is initialized and the above task is started
@@ -278,170 +215,179 @@ async fn wrapped_main() {
     // spawn a thread that listens for keyboard input on stdin, when a user presses enter
     // the input handler will toggle PAUSE_SCAN, which in turn is used to pause and resume
     // scans that are already running
-    tokio::task::spawn_blocking(terminal_input_handler);
+    // also starts ctrl+c handler
+    TermInputHandler::initialize(handles.clone());
 
-    let save_output = !CONFIGURATION.output.is_empty(); // was -o used?
+    if config.resumed {
+        let scanned_urls = handles.ferox_scans()?;
+        let from_here = config.resume_from.clone();
 
-    if CONFIGURATION.save_state {
-        // start the ctrl+c handler
-        scan_manager::initialize(stats.clone());
+        // populate FeroxScans object with previously seen scans
+        scanned_urls.add_serialized_scans(&from_here)?;
+
+        // populate Stats object with previously known statistics
+        handles.stats.send(LoadStats(from_here))?;
     }
 
-    let (tx_term, tx_file, term_handle, file_handle) =
-        reporter::initialize(&CONFIGURATION.output, save_output, tx_stats.clone());
-
     // get targets from command line or stdin
-    let targets = match get_targets().await {
+    let targets = match get_targets(handles.clone()).await {
         Ok(t) => t,
         Err(e) => {
             // should only happen in the event that there was an error reading from stdin
-            log::error!("{} {}", module_colorizer("main::get_targets"), e);
-            clean_up(
-                tx_term,
-                term_handle,
-                tx_file,
-                file_handle,
-                tx_stats,
-                stats_handle,
-                save_output,
-            )
-            .await;
-            return;
+            clean_up(handles, tasks).await?;
+            bail!("Could not get determine initial targets: {}", e);
         }
     };
 
-    update_stat!(tx_stats, UpdateUsizeField(InitialTargets, targets.len()));
+    // --parallel branch
+    if config.parallel > 0 {
+        log::trace!("enter: parallel branch");
 
-    if !CONFIGURATION.quiet {
-        // only print banner if -q isn't used
+        PARALLEL_LIMITER.add_permits(config.parallel);
+
+        let invocation = args();
+
+        let para_regex =
+            Regex::new("--stdin|-q|--quiet|--silent|--verbosity|-v|-vv|-vvv|-vvvv").unwrap();
+
+        // remove stdin since only the original process will process targets
+        // remove quiet and silent so we can force silent later to normalize output
+        let mut original = invocation
+            .filter(|s| !para_regex.is_match(s))
+            .collect::<Vec<String>>();
+
+        original.push("--silent".to_string()); // only output modifier allowed
+
+        // we need remove --parallel from command line so we don't hit this branch over and over
+        // but we must remove --parallel N manually; the filter above never sees --parallel and the
+        // value passed to it at the same time, so can't filter them out in one pass
+
+        // unwrap is fine, as it has to be in the args for us to be in this code branch
+        let parallel_index = original.iter().position(|s| *s == "--parallel").unwrap();
+
+        // remove --parallel
+        original.remove(parallel_index);
+
+        // remove N passed to --parallel (it's the same index again since everything shifts
+        // from removing --parallel)
+        original.remove(parallel_index);
+
+        // unvalidated targets fresh from stdin, just spawn children and let them do all checks
+        for target in targets {
+            // add the current target to the provided command
+            let mut cloned = original.clone();
+            cloned.push("-u".to_string());
+            cloned.push(target);
+
+            let bin = cloned.index(0).to_owned(); // user's path to feroxbuster
+            let args = cloned.index(1..).to_vec(); // and args
+
+            let permit = PARALLEL_LIMITER.acquire().await?;
+
+            log::debug!("parallel exec: {} {}", bin, args.join(" "));
+
+            tokio::task::spawn_blocking(move || {
+                let result = Command::new(bin)
+                    .args(&args)
+                    .spawn()
+                    .expect("failed to spawn a child process")
+                    .wait()
+                    .expect("child process errored during execution");
+
+                drop(permit);
+                result
+            });
+        }
+
+        clean_up(handles, tasks).await?;
+
+        log::trace!("exit: parallel branch && wrapped main");
+        return Ok(());
+    }
+
+    if matches!(config.output_level, OutputLevel::Default) {
+        // only print banner if output level is default (no banner on --quiet|--silent)
         let std_stderr = stderr(); // std::io::stderr
-        banner::initialize(
-            &targets,
-            &CONFIGURATION,
-            &VERSION,
-            std_stderr,
-            tx_stats.clone(),
-        )
-        .await;
+
+        let mut banner = Banner::new(&targets, &config);
+
+        // only interested in the side-effect that sets banner.update_status
+        let _ = banner.check_for_updates(UPDATE_URL, handles.clone()).await;
+
+        if banner.print_to(std_stderr, config.clone()).is_err() {
+            clean_up(handles, tasks).await?;
+            bail!(fmt_err("Could not print banner"));
+        }
+    }
+
+    {
+        let send_to_file = !config.output.is_empty();
+
+        // The TermOutHandler spawns a FileOutHandler, so errors in the FileOutHandler never bubble
+        // up due to the TermOutHandler never awaiting the result of FileOutHandler::start (that's
+        // done later here in main). sync checks that the tx/rx connection to the file handler works
+        if send_to_file && handles.output.sync(send_to_file).await.is_err() {
+            // output file specified and file handler could not initialize
+            clean_up(handles, tasks).await?;
+            let msg = format!("Couldn't start {} file handler", config.output);
+            bail!(fmt_err(&msg));
+        }
     }
 
     // discard non-responsive targets
-    let live_targets = heuristics::connectivity_test(&targets, tx_stats.clone()).await;
+    let live_targets = {
+        let test = heuristics::HeuristicTests::new(handles.clone());
+        let result = test.connectivity(&targets).await;
+        if result.is_err() {
+            clean_up(handles, tasks).await?;
+            bail!(fmt_err(&result.unwrap_err().to_string()));
+        }
+        result?
+    };
 
     if live_targets.is_empty() {
-        clean_up(
-            tx_term,
-            term_handle,
-            tx_file,
-            file_handle,
-            tx_stats,
-            stats_handle,
-            save_output,
-        )
-        .await;
-        return;
+        clean_up(handles, tasks).await?;
+        bail!(fmt_err("Could not find any live targets to scan"));
     }
 
     // kick off a scan against any targets determined to be responsive
-    match scan(
-        live_targets,
-        stats,
-        tx_term.clone(),
-        tx_file.clone(),
-        tx_stats.clone(),
-    )
-    .await
-    {
-        Ok(_) => {
-            log::info!("All scans complete!");
-        }
+    match scan(live_targets, handles.clone()).await {
+        Ok(_) => {}
         Err(e) => {
-            ferox_print(
-                &format!("{} while scanning: {}", status_colorizer("Error"), e),
-                &PROGRESS_PRINTER,
-            );
-            clean_up(
-                tx_term,
-                term_handle,
-                tx_file,
-                file_handle,
-                tx_stats,
-                stats_handle,
-                save_output,
-            )
-            .await;
-            process::exit(1);
+            clean_up(handles, tasks).await?;
+            bail!(fmt_err(&format!("Failed while scanning: {}", e)));
         }
-    };
+    }
 
-    clean_up(
-        tx_term,
-        term_handle,
-        tx_file,
-        file_handle,
-        tx_stats,
-        stats_handle,
-        save_output,
-    )
-    .await;
+    clean_up(handles, tasks).await?;
 
     log::trace!("exit: wrapped_main");
+    Ok(())
 }
 
 /// Single cleanup function that handles all the necessary drops/finishes etc required to gracefully
 /// shutdown the program
-async fn clean_up(
-    tx_term: UnboundedSender<FeroxResponse>,
-    term_handle: JoinHandle<()>,
-    tx_file: UnboundedSender<FeroxResponse>,
-    file_handle: Option<JoinHandle<()>>,
-    tx_stats: UnboundedSender<StatCommand>,
-    stats_handle: JoinHandle<()>,
-    save_output: bool,
-) {
-    log::trace!(
-        "enter: clean_up({:?}, {:?}, {:?}, {:?}, {:?}, {:?}, {})",
-        tx_term,
-        term_handle,
-        tx_file,
-        file_handle,
-        tx_stats,
-        stats_handle,
-        save_output
-    );
-    drop(tx_term);
-    log::trace!("dropped terminal output handler's transmitter");
+async fn clean_up(handles: Arc<Handles>, tasks: Tasks) -> Result<()> {
+    log::trace!("enter: clean_up({:?}, {:?})", handles, tasks);
 
-    log::trace!("awaiting terminal output handler's receiver");
-    // after dropping tx, we can await the future where rx lived
-    match term_handle.await {
-        Ok(_) => {}
-        Err(e) => {
-            log::error!("error awaiting terminal output handler's receiver: {}", e);
-        }
-    }
-    log::trace!("done awaiting terminal output handler's receiver");
+    let (tx, rx) = oneshot::channel::<bool>();
+    handles.send_scan_command(JoinTasks(tx))?;
+    rx.await?;
 
-    log::trace!("tx_file: {:?}", tx_file);
-    // the same drop/await process used on the terminal handler is repeated for the file handler
-    // we drop the file transmitter every time, because it's created no matter what
-    drop(tx_file);
+    log::info!("All scans complete!");
 
-    log::trace!("dropped file output handler's transmitter");
-    if save_output {
-        // but we only await if -o was specified
-        log::trace!("awaiting file output handler's receiver");
-        match file_handle.unwrap().await {
-            Ok(_) => {}
-            Err(e) => {
-                log::error!("error awaiting file output handler's receiver: {}", e);
-            }
-        }
-        log::trace!("done awaiting file output handler's receiver");
-    }
+    // terminal handler closes file handler if one is in use
+    handles.output.send(Exit)?;
+    tasks.terminal.await??;
+    log::trace!("terminal handler closed");
 
-    update_stat!(tx_stats, StatCommand::Exit); // send exit command and await the end of the future
-    stats_handle.await.unwrap_or_default();
+    handles.filters.send(Exit)?;
+    tasks.filters.await??;
+    log::trace!("filters handler closed");
+
+    handles.stats.send(Exit)?;
+    tasks.stats.await??;
+    log::trace!("stats handler closed");
 
     // mark all scans complete so the terminal input handler will exit cleanly
     SCAN_COMPLETE.store(true, Ordering::Relaxed);
@@ -450,14 +396,21 @@ async fn clean_up(
     // the final trace messages above
     PROGRESS_PRINTER.finish();
 
-    drop(tx_stats);
-
     log::trace!("exit: clean_up");
+    Ok(())
 }
 
-fn main() {
+fn main() -> Result<()> {
+    let config = Arc::new(Configuration::new().with_context(|| "Could not create Configuration")?);
+
     // setup logging based on the number of -v's used
-    logger::initialize(CONFIGURATION.verbosity);
+    if matches!(
+        config.output_level,
+        OutputLevel::Default | OutputLevel::Quiet
+    ) {
+        // don't log on --silent
+        logger::initialize(config.clone())?;
+    }
 
     // this function uses rlimit, which is not supported on windows
     #[cfg(not(target_os = "windows"))]
@@ -467,9 +420,13 @@ fn main() {
         .enable_all()
         .build()
     {
-        let future = wrapped_main();
-        runtime.block_on(future);
+        let future = wrapped_main(config);
+        if let Err(e) = runtime.block_on(future) {
+            eprintln!("{}", e);
+        };
     }
 
     log::trace!("exit: main");
+
+    Ok(())
 }

src/message.rs

@@ -0,0 +1,148 @@
+use anyhow::Context;
+use console::{style, Color};
+use serde::{Deserialize, Serialize};
+
+use crate::traits::FeroxSerialize;
+use crate::utils::fmt_err;
+
+#[derive(Serialize, Deserialize, Default)]
+/// Representation of a log entry, can be represented as a human readable string or JSON
+pub struct FeroxMessage {
+    #[serde(rename = "type")]
+    /// Name of this type of struct, used for serialization, i.e. `{"type":"log"}`
+    pub(crate) kind: String,
+
+    /// The log message
+    pub(crate) message: String,
+
+    /// The log level
+    pub(crate) level: String,
+
+    /// The number of seconds elapsed since the scan started
+    pub(crate) time_offset: f32,
+
+    /// The module from which log::* was called
+    pub(crate) module: String,
+}
+
+/// Implementation of FeroxMessage
+impl FeroxSerialize for FeroxMessage {
+    /// Create a string representation of the log message
+    ///
+    /// ex:  301       10l       16w      173c https://localhost/api
+    fn as_str(&self) -> String {
+        let (level_name, level_color) = match self.level.as_str() {
+            "ERROR" => ("ERR", Color::Red),
+            "WARN" => ("WRN", Color::Red),
+            "INFO" => ("INF", Color::Cyan),
+            "DEBUG" => ("DBG", Color::Yellow),
+            "TRACE" => ("TRC", Color::Magenta),
+            "WILDCARD" => ("WLD", Color::Cyan),
+            _ => ("UNK", Color::White),
+        };
+
+        format!(
+            "{} {:10.03} {} {}\n",
+            style(level_name).bg(level_color).black(),
+            style(self.time_offset).dim(),
+            self.module,
+            style(&self.message).dim(),
+        )
+    }
+
+    /// Create an NDJSON representation of the log message
+    ///
+    /// (expanded for clarity)
+    /// ex:
+    /// {
+    ///   "type": "log",
+    ///   "message": "Sent https://localhost/api to file handler",
+    ///   "level": "DEBUG",
+    ///   "time_offset": 0.86333454,
+    ///   "module": "feroxbuster::reporter"
+    /// }\n
+    fn as_json(&self) -> anyhow::Result<String> {
+        let mut json = serde_json::to_string(&self).with_context(|| {
+            fmt_err(&format!(
+                "Could not convert {}:{} to JSON",
+                self.level, self.message
+            ))
+        })?;
+        json.push('\n');
+        Ok(json)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    /// test as_str method of FeroxMessage
+    fn ferox_message_as_str_returns_string_with_newline() {
+        let message = FeroxMessage {
+            message: "message".to_string(),
+            module: "utils".to_string(),
+            time_offset: 1.0,
+            level: "INFO".to_string(),
+            kind: "log".to_string(),
+        };
+        let message_str = message.as_str();
+
+        assert!(message_str.contains("INF"));
+        assert!(message_str.contains("1.000"));
+        assert!(message_str.contains("utils"));
+        assert!(message_str.contains("message"));
+        assert!(message_str.ends_with('\n'));
+    }
+
+    #[test]
+    /// test as_json method of FeroxMessage
+    fn ferox_message_as_json_returns_json_representation_of_ferox_message_with_newline() {
+        let message = FeroxMessage {
+            message: "message".to_string(),
+            module: "utils".to_string(),
+            time_offset: 1.0,
+            level: "INFO".to_string(),
+            kind: "log".to_string(),
+        };
+
+        let message_str = message.as_json().unwrap();
+
+        let error_margin = f32::EPSILON;
+
+        let json: FeroxMessage = serde_json::from_str(&message_str).unwrap();
+        assert_eq!(json.module, message.module);
+        assert_eq!(json.message, message.message);
+        assert!((json.time_offset - message.time_offset).abs() < error_margin);
+        assert_eq!(json.level, message.level);
+        assert_eq!(json.kind, message.kind);
+    }
+
+    #[test]
+    /// test defaults for coverage
+    fn message_defaults() {
+        let msg = FeroxMessage::default();
+        assert_eq!(msg.level, String::new());
+        assert_eq!(msg.kind, String::new());
+        assert_eq!(msg.message, String::new());
+        assert_eq!(msg.module, String::new());
+        assert!(msg.time_offset < 0.1);
+    }
+
+    #[test]
+    /// ensure WILDCARD messages serialize to WLD and anything not known to UNK
+    fn message_as_str_edges() {
+        let mut msg = FeroxMessage {
+            message: "message".to_string(),
+            module: "utils".to_string(),
+            time_offset: 1.0,
+            level: "WILDCARD".to_string(),
+            kind: "log".to_string(),
+        };
+        assert!(console::strip_ansi_codes(&msg.as_str()).starts_with("WLD"));
+
+        msg.level = "UNKNOWN".to_string();
+        assert!(console::strip_ansi_codes(&msg.as_str()).starts_with("UNK"));
+    }
+}

src/parser.rs

@@ -68,6 +68,7 @@ pub fn initialize() -> App<'static, 'static> {
                 .long("verbosity")
                 .takes_value(false)
                 .multiple(true)
+                .conflicts_with("silent")
                 .help("Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v's is probably too much)"),
         )
         .arg(
@@ -115,12 +116,32 @@ pub fn initialize() -> App<'static, 'static> {
                     "Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)",
                 ),
         )
+        .arg(
+            Arg::with_name("silent")
+                .long("silent")
+                .takes_value(false)
+                .conflicts_with("quiet")
+                .help("Only print URLs + turn off logging (good for piping a list of urls to other commands)")
+        )
         .arg(
             Arg::with_name("quiet")
                 .short("q")
                 .long("quiet")
                 .takes_value(false)
-                .help("Only print URLs; Don't print status codes, response size, running config, etc...")
+                .help("Hide progress bars and banner (good for tmux windows w/ notifications)")
+        )
+        .arg(
+            Arg::with_name("auto_tune")
+                .long("auto-tune")
+                .takes_value(false)
+                .conflicts_with("auto_bail")
+                .help("Automatically lower scan rate when an excessive amount of errors are encountered")
+        )
+        .arg(
+            Arg::with_name("auto_bail")
+                .long("auto-bail")
+                .takes_value(false)
+                .help("Automatically stop scanning when an excessive amount of errors are encountered")
         )
         .arg(
             Arg::with_name("json")
@@ -327,6 +348,22 @@ pub fn initialize() -> App<'static, 'static> {
                 .takes_value(true)
                 .help("Limit total number of concurrent scans (default: 0, i.e. no limit)")
         )
+        .arg(
+            Arg::with_name("parallel")
+                .long("parallel")
+                .value_name("PARALLEL_SCANS")
+                .takes_value(true)
+                .requires("stdin")
+                .help("Run parallel feroxbuster instances (one child process per url passed via stdin)")
+        )
+        .arg(
+            Arg::with_name("rate_limit")
+                .long("rate-limit")
+                .value_name("RATE_LIMIT")
+                .takes_value(true)
+                .conflicts_with("auto_tune")
+                .help("Limit number of requests per second (per directory) (default: 0, i.e. no limit)")
+        )
         .arg(
             Arg::with_name("time_limit")
                 .long("time-limit")
@@ -357,7 +394,7 @@ EXAMPLES:
         ./feroxbuster -u http://[::1] --no-recursion -vv
 
     Read urls from STDIN; pipe only resulting urls out to another tool
-        cat targets | ./feroxbuster --stdin --quiet -s 200 301 302 --redirects -x js | fff -s 200 -o js-files
+        cat targets | ./feroxbuster --stdin --silent -s 200 301 302 --redirects -x js | fff -s 200 -o js-files
 
     Proxy traffic through Burp
         ./feroxbuster -u http://127.1 --insecure --proxy http://127.0.0.1:8080

src/progress.rs

@@ -1,7 +1,16 @@
-use crate::config::{CONFIGURATION, PROGRESS_BAR};
-use indicatif::{ProgressBar, ProgressStyle};
+use indicatif::{MultiProgress, ProgressBar, ProgressDrawTarget, ProgressStyle};
+use lazy_static::lazy_static;
+
+lazy_static! {
+    /// Global progress bar that houses other progress bars
+    pub static ref PROGRESS_BAR: MultiProgress = MultiProgress::with_draw_target(ProgressDrawTarget::stdout());
+
+    /// Global progress bar that is only used for printing messages that don't jack up other bars
+    pub static ref PROGRESS_PRINTER: ProgressBar = add_bar("", 0, BarType::Hidden);
+}
 
 /// Types of ProgressBars that can be added to `PROGRESS_BAR`
+#[derive(Copy, Clone)]
 pub enum BarType {
     /// no template used / not visible
     Hidden,
@@ -14,6 +23,9 @@ pub enum BarType {
 
     /// bar used to show overall scan metrics
     Total,
+
+    /// simpler output bar that shows only the directory being scanned (no updating info)
+    Quiet,
 }
 
 /// Add an [indicatif::ProgressBar](https://docs.rs/indicatif/latest/indicatif/struct.ProgressBar.html)
@@ -21,22 +33,18 @@ pub enum BarType {
 pub fn add_bar(prefix: &str, length: u64, bar_type: BarType) -> ProgressBar {
     let mut style = ProgressStyle::default_bar().progress_chars("#>-");
 
-    style = if CONFIGURATION.quiet {
-        style.template("")
-    } else {
-        match bar_type {
-            BarType::Hidden => style.template(""),
-            BarType::Default => style.template(
-                "[{bar:.cyan/blue}] - {elapsed:<4} {pos:>7}/{len:7} {per_sec:7} {prefix}",
-            ),
-            BarType::Message => style.template(&format!(
-                "[{{bar:.cyan/blue}}] - {{elapsed:<4}} {{pos:>7}}/{{len:7}} {:7} {{prefix}}",
-                "-"
-            )),
-            BarType::Total => {
-                style.template("[{bar:.yellow/blue}] - {elapsed:<4} {pos:>7}/{len:7} {eta:7} {msg}")
-            }
+    style = match bar_type {
+        BarType::Hidden => style.template(""),
+        BarType::Default => style
+            .template("[{bar:.cyan/blue}] - {elapsed:<4} {pos:>7}/{len:7} {per_sec:7} {prefix}"),
+        BarType::Message => style.template(&format!(
+            "[{{bar:.cyan/blue}}] - {{elapsed:<4}} {{pos:>7}}/{{len:7}} {:7} {{prefix}}",
+            "-"
+        )),
+        BarType::Total => {
+            style.template("[{bar:.yellow/blue}] - {elapsed:<4} {pos:>7}/{len:7} {eta:7} {msg}")
         }
+        BarType::Quiet => style.template("Scanning: {prefix}"),
     };
 
     let progress_bar = PROGRESS_BAR.add(ProgressBar::new(length));

src/reporter.rs

@@ -1,261 +0,0 @@
-use crate::{
-    config::{CONFIGURATION, PROGRESS_PRINTER},
-    scanner::RESPONSES,
-    statistics::{
-        StatCommand::{self, UpdateUsizeField},
-        StatField::ResourcesDiscovered,
-    },
-    utils::{ferox_print, make_request, open_file},
-    FeroxChannel, FeroxResponse, FeroxSerialize,
-};
-use console::strip_ansi_codes;
-use std::{
-    fs, io,
-    io::Write,
-    sync::{Arc, Once, RwLock},
-};
-use tokio::{
-    sync::mpsc::{self, UnboundedReceiver, UnboundedSender},
-    task::JoinHandle,
-};
-
-/// Singleton buffered file behind an Arc/RwLock; used for file writes from two locations:
-///     - [logger::initialize](../logger/fn.initialize.html) (specifically a closure on the global logger instance)
-///     - `reporter::spawn_file_handler`
-pub static mut LOCKED_FILE: Option<Arc<RwLock<io::BufWriter<fs::File>>>> = None;
-
-/// An initializer Once variable used to create `LOCKED_FILE`
-static INIT: Once = Once::new();
-
-// Accessing a `static mut` is unsafe much of the time, but if we do so
-// in a synchronized fashion (e.g., write once or read all) then we're
-// good to go!
-//
-// This function will only call `open_file` once, and will
-// otherwise always return the value returned from the first invocation.
-pub fn get_cached_file_handle(filename: &str) -> Option<Arc<RwLock<io::BufWriter<fs::File>>>> {
-    unsafe {
-        INIT.call_once(|| {
-            LOCKED_FILE = open_file(&filename);
-        });
-        LOCKED_FILE.clone()
-    }
-}
-
-/// Creates all required output handlers (terminal, file) and returns
-/// the transmitter sides of each mpsc along with each receiver's future's JoinHandle to be awaited
-///
-/// Any other module that needs to write a Response to stdout or output results to a file should
-/// be passed a clone of the appropriate returned transmitter
-pub fn initialize(
-    output_file: &str,
-    save_output: bool,
-    tx_stats: UnboundedSender<StatCommand>,
-) -> (
-    UnboundedSender<FeroxResponse>,
-    UnboundedSender<FeroxResponse>,
-    JoinHandle<()>,
-    Option<JoinHandle<()>>,
-) {
-    log::trace!(
-        "enter: initialize({}, {}, {:?})",
-        output_file,
-        save_output,
-        tx_stats
-    );
-
-    let (tx_rpt, rx_rpt): FeroxChannel<FeroxResponse> = mpsc::unbounded_channel();
-    let (tx_file, rx_file): FeroxChannel<FeroxResponse> = mpsc::unbounded_channel();
-
-    let file_clone = tx_file.clone();
-    let stats_clone = tx_stats.clone();
-
-    let term_reporter = tokio::spawn(async move {
-        spawn_terminal_reporter(rx_rpt, file_clone, stats_clone, save_output).await
-    });
-
-    let file_reporter = if save_output {
-        // -o used, need to spawn the thread for writing to disk
-        let file_clone = output_file.to_string();
-        Some(tokio::spawn(async move {
-            spawn_file_reporter(rx_file, tx_stats, &file_clone).await
-        }))
-    } else {
-        None
-    };
-
-    log::trace!(
-        "exit: initialize -> ({:?}, {:?}, {:?}, {:?})",
-        tx_rpt,
-        tx_file,
-        term_reporter,
-        file_reporter
-    );
-    (tx_rpt, tx_file, term_reporter, file_reporter)
-}
-
-/// Spawn a single consumer task (sc side of mpsc)
-///
-/// The consumer simply receives responses and prints them if they meet the given
-/// reporting criteria
-async fn spawn_terminal_reporter(
-    mut resp_chan: UnboundedReceiver<FeroxResponse>,
-    file_chan: UnboundedSender<FeroxResponse>,
-    tx_stats: UnboundedSender<StatCommand>,
-    save_output: bool,
-) {
-    log::trace!(
-        "enter: spawn_terminal_reporter({:?}, {:?}, {:?}, {})",
-        resp_chan,
-        file_chan,
-        tx_stats,
-        save_output
-    );
-
-    while let Some(mut resp) = resp_chan.recv().await {
-        log::trace!("received {} on reporting channel", resp.url());
-
-        let contains_sentry = CONFIGURATION.status_codes.contains(&resp.status().as_u16());
-        let unknown_sentry = !RESPONSES.contains(&resp); // !contains == unknown
-        let should_process_response = contains_sentry && unknown_sentry;
-
-        if should_process_response {
-            // print to stdout
-            ferox_print(&resp.as_str(), &PROGRESS_PRINTER);
-
-            update_stat!(tx_stats, UpdateUsizeField(ResourcesDiscovered, 1));
-
-            if save_output {
-                // -o used, need to send the report to be written out to disk
-                match file_chan.send(resp.clone()) {
-                    Ok(_) => {
-                        log::debug!("Sent {} to file handler", resp.url());
-                    }
-                    Err(e) => {
-                        log::error!("Could not send {} to file handler: {}", resp.url(), e);
-                    }
-                }
-            }
-        }
-        log::trace!("report complete: {}", resp.url());
-
-        if CONFIGURATION.replay_client.is_some() && should_process_response {
-            // replay proxy specified/client created and this response's status code is one that
-            // should be replayed
-            match make_request(
-                CONFIGURATION.replay_client.as_ref().unwrap(),
-                &resp.url(),
-                tx_stats.clone(),
-            )
-            .await
-            {
-                Ok(_) => {}
-                Err(e) => {
-                    log::error!("{}", e);
-                }
-            }
-        }
-
-        if should_process_response {
-            // add response to RESPONSES for serialization in case of ctrl+c
-            // placed all by its lonesome like this so that RESPONSES can take ownership
-            // of the FeroxResponse
-
-            // before ownership is transferred, there's no real reason to keep the body anymore
-            // so we can free that piece of data, reducing memory usage
-            resp.text = String::new();
-
-            RESPONSES.insert(resp);
-        }
-    }
-    log::trace!("exit: spawn_terminal_reporter");
-}
-
-/// Spawn a single consumer task (sc side of mpsc)
-///
-/// The consumer simply receives responses and writes them to the given output file if they meet
-/// the given reporting criteria
-async fn spawn_file_reporter(
-    mut report_channel: UnboundedReceiver<FeroxResponse>,
-    tx_stats: UnboundedSender<StatCommand>,
-    output_file: &str,
-) {
-    let buffered_file = match get_cached_file_handle(&CONFIGURATION.output) {
-        Some(file) => file,
-        None => {
-            log::trace!("exit: spawn_file_reporter");
-            return;
-        }
-    };
-
-    log::trace!(
-        "enter: spawn_file_reporter({:?}, {})",
-        report_channel,
-        output_file
-    );
-
-    log::info!("Writing scan results to {}", output_file);
-
-    while let Some(response) = report_channel.recv().await {
-        safe_file_write(&response, buffered_file.clone(), CONFIGURATION.json);
-    }
-
-    update_stat!(tx_stats, StatCommand::Save);
-
-    log::trace!("exit: spawn_file_reporter");
-}
-
-/// Given a string and a reference to a locked buffered file, write the contents and flush
-/// the buffer to disk.
-pub fn safe_file_write<T>(
-    value: &T,
-    locked_file: Arc<RwLock<io::BufWriter<fs::File>>>,
-    convert_to_json: bool,
-) where
-    T: FeroxSerialize,
-{
-    // note to future self: adding logging of anything other than error to this function
-    // is a bad idea. we call this function while processing records generated by the logger.
-    // If we then call log::... while already processing some logging output, it results in
-    // the second log entry being injected into the first.
-
-    let contents = if convert_to_json {
-        value.as_json()
-    } else {
-        value.as_str()
-    };
-
-    let contents = strip_ansi_codes(&contents);
-
-    if let Ok(mut handle) = locked_file.write() {
-        // write lock acquired
-        match handle.write(contents.as_bytes()) {
-            Ok(_) => {}
-            Err(e) => {
-                log::error!("could not write report to disk: {}", e);
-            }
-        }
-
-        match handle.flush() {
-            // this function is used within async functions/loops, so i'm flushing so that in
-            // the event of a ctrl+c or w/e results seen so far are saved instead of left lying
-            // around in the buffer
-            Ok(_) => {}
-            Err(e) => {
-                log::error!("error writing to file: {}", e);
-            }
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    #[should_panic]
-    /// asserts that an empty string for a filename returns None
-    fn reporter_get_cached_file_handle_without_filename_returns_none() {
-        let _used = get_cached_file_handle(&"").unwrap();
-    }
-}

src/response.rs

@@ -0,0 +1,640 @@
+use std::{
+    collections::HashMap,
+    convert::{TryFrom, TryInto},
+    fmt,
+    str::FromStr,
+    sync::Arc,
+};
+
+use anyhow::{Context, Result};
+use reqwest::{
+    header::{HeaderMap, HeaderName, HeaderValue},
+    Response, StatusCode, Url,
+};
+use serde::ser::SerializeStruct;
+use serde::{Deserialize, Deserializer, Serialize, Serializer};
+use serde_json::Value;
+
+use crate::{
+    config::OutputLevel,
+    event_handlers::{Command, Handles},
+    traits::FeroxSerialize,
+    url::FeroxUrl,
+    utils::{self, fmt_err, status_colorizer},
+    CommandSender,
+};
+
+/// A `FeroxResponse`, derived from a `Response` to a submitted `Request`
+#[derive(Debug, Clone)]
+pub struct FeroxResponse {
+    /// The final `Url` of this `FeroxResponse`
+    url: Url,
+
+    /// The `StatusCode` of this `FeroxResponse`
+    status: StatusCode,
+
+    /// The full response text
+    text: String,
+
+    /// The content-length of this response, if known
+    content_length: u64,
+
+    /// The number of lines contained in the body of this response, if known
+    line_count: usize,
+
+    /// The number of words contained in the body of this response, if known
+    word_count: usize,
+
+    /// The `Headers` of this `FeroxResponse`
+    headers: HeaderMap,
+
+    /// Wildcard response status
+    wildcard: bool,
+
+    /// whether the user passed --quiet|--silent on the command line
+    pub(crate) output_level: OutputLevel,
+}
+
+/// implement Default trait for FeroxResponse
+impl Default for FeroxResponse {
+    /// return a default reqwest::Url and then normal defaults after that
+    fn default() -> Self {
+        Self {
+            url: Url::parse("http://localhost").unwrap(),
+            status: Default::default(),
+            text: "".to_string(),
+            content_length: 0,
+            line_count: 0,
+            word_count: 0,
+            headers: Default::default(),
+            wildcard: false,
+            output_level: Default::default(),
+        }
+    }
+}
+
+/// Implement Display for FeroxResponse
+impl fmt::Display for FeroxResponse {
+    /// formatter for Display
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(
+            f,
+            "FeroxResponse {{ url: {}, status: {}, content-length: {} }}",
+            self.url(),
+            self.status(),
+            self.content_length()
+        )
+    }
+}
+
+/// `FeroxResponse` implementation
+impl FeroxResponse {
+    /// Get the `StatusCode` of this `FeroxResponse`
+    pub fn status(&self) -> &StatusCode {
+        &self.status
+    }
+
+    /// Get the `wildcard` of this `FeroxResponse`
+    pub fn wildcard(&self) -> bool {
+        self.wildcard
+    }
+
+    /// Get the final `Url` of this `FeroxResponse`.
+    pub fn url(&self) -> &Url {
+        &self.url
+    }
+
+    /// Get the full response text
+    pub fn text(&self) -> &str {
+        &self.text
+    }
+
+    /// Get the `Headers` of this `FeroxResponse`
+    pub fn headers(&self) -> &HeaderMap {
+        &self.headers
+    }
+
+    /// Get the content-length of this response, if known
+    pub fn content_length(&self) -> u64 {
+        self.content_length
+    }
+
+    /// Set `FeroxResponse`'s `url` attribute, has no affect if an error occurs
+    pub fn set_url(&mut self, url: &str) {
+        match Url::parse(&url) {
+            Ok(url) => {
+                self.url = url;
+            }
+            Err(e) => {
+                log::warn!("Could not parse {} into a Url: {}", url, e);
+            }
+        };
+    }
+
+    /// set `wildcard` attribute
+    pub fn set_wildcard(&mut self, is_wildcard: bool) {
+        self.wildcard = is_wildcard;
+    }
+
+    /// set `text` attribute; update words/lines/content_length
+    #[cfg(test)]
+    pub fn set_text(&mut self, text: &str) {
+        self.text = String::from(text);
+        self.content_length = self.text.len() as u64;
+        self.line_count = self.text.lines().count();
+        self.word_count = self
+            .text
+            .lines()
+            .map(|s| s.split_whitespace().count())
+            .sum();
+    }
+
+    /// free the `text` data, reducing memory usage
+    pub fn drop_text(&mut self) {
+        self.text = String::new();
+    }
+
+    /// Make a reasonable guess at whether the response is a file or not
+    ///
+    /// Examines the last part of a path to determine if it has an obvious extension
+    /// i.e. http://localhost/some/path/stuff.js where stuff.js indicates a file
+    ///
+    /// Additionally, inspects query parameters, as they're also often indicative of a file
+    pub fn is_file(&self) -> bool {
+        let has_extension = match self.url.path_segments() {
+            Some(path) => {
+                if let Some(last) = path.last() {
+                    last.contains('.') // last segment has some sort of extension, probably
+                } else {
+                    false
+                }
+            }
+            None => false,
+        };
+
+        self.url.query_pairs().count() > 0 || has_extension
+    }
+
+    /// Returns line count of the response text.
+    pub fn line_count(&self) -> usize {
+        self.line_count
+    }
+
+    /// Returns word count of the response text.
+    pub fn word_count(&self) -> usize {
+        self.word_count
+    }
+
+    /// Create a new `FeroxResponse` from the given `Response`
+    pub async fn from(response: Response, read_body: bool, output_level: OutputLevel) -> Self {
+        let url = response.url().clone();
+        let status = response.status();
+        let headers = response.headers().clone();
+        let content_length = response.content_length().unwrap_or(0);
+
+        let text = if read_body {
+            // .text() consumes the response, must be called last
+            // additionally, --extract-links is currently the only place we use the body of the
+            // response, so we forego the processing if not performing extraction
+            match response.text().await {
+                // await the response's body
+                Ok(text) => text,
+                Err(e) => {
+                    log::warn!("Could not parse body from response: {}", e);
+                    String::new()
+                }
+            }
+        } else {
+            String::new()
+        };
+
+        let line_count = text.lines().count();
+        let word_count = text.lines().map(|s| s.split_whitespace().count()).sum();
+
+        FeroxResponse {
+            url,
+            status,
+            content_length,
+            text,
+            headers,
+            line_count,
+            word_count,
+            output_level,
+            wildcard: false,
+        }
+    }
+
+    /// Helper function that determines if the configured maximum recursion depth has been reached
+    ///
+    /// Essentially looks at the Url path and determines how many directories are present in the
+    /// given Url
+    pub(crate) fn reached_max_depth(
+        &self,
+        base_depth: usize,
+        max_depth: usize,
+        handles: Arc<Handles>,
+    ) -> bool {
+        log::trace!(
+            "enter: reached_max_depth({}, {}, {:?})",
+            base_depth,
+            max_depth,
+            handles
+        );
+
+        if max_depth == 0 {
+            // early return, as 0 means recurse forever; no additional processing needed
+            log::trace!("exit: reached_max_depth -> false");
+            return false;
+        }
+        let url = FeroxUrl::from_url(&self.url, handles);
+        let depth = url.depth().unwrap_or_default(); // 0 on error
+
+        if depth - base_depth >= max_depth {
+            return true;
+        }
+
+        log::trace!("exit: reached_max_depth -> false");
+        false
+    }
+
+    /// Helper function to determine suitability for recursion
+    ///
+    /// handles 2xx and 3xx responses by either checking if the url ends with a / (2xx)
+    /// or if the Location header is present and matches the base url + / (3xx)
+    pub fn is_directory(&self) -> bool {
+        log::trace!("enter: is_directory({})", self);
+
+        if self.status().is_redirection() {
+            // status code is 3xx
+            match self.headers().get("Location") {
+                // and has a Location header
+                Some(loc) => {
+                    // get absolute redirect Url based on the already known base url
+                    log::debug!("Location header: {:?}", loc);
+
+                    if let Ok(loc_str) = loc.to_str() {
+                        if let Ok(abs_url) = self.url().join(loc_str) {
+                            if format!("{}/", self.url()) == abs_url.as_str() {
+                                // if current response's Url + / == the absolute redirection
+                                // location, we've found a directory suitable for recursion
+                                log::debug!(
+                                    "found directory suitable for recursion: {}",
+                                    self.url()
+                                );
+                                log::trace!("exit: is_directory -> true");
+                                return true;
+                            }
+                        }
+                    }
+                }
+                None => {
+                    log::debug!("expected Location header, but none was found: {}", self);
+                    log::trace!("exit: is_directory -> false");
+                    return false;
+                }
+            }
+        } else if self.status().is_success() || matches!(self.status(), &StatusCode::FORBIDDEN) {
+            // status code is 2xx or 403, need to check if it ends in /
+
+            if self.url().as_str().ends_with('/') {
+                log::debug!("{} is directory suitable for recursion", self.url());
+                log::trace!("exit: is_directory -> true");
+                return true;
+            }
+        }
+
+        log::trace!("exit: is_directory -> false");
+        false
+    }
+
+    /// Simple helper to send a `FeroxResponse` over the tx side of an `mpsc::unbounded_channel`
+    pub fn send_report(self, report_sender: CommandSender) -> Result<()> {
+        log::trace!("enter: send_report({:?}", report_sender);
+
+        report_sender.send(Command::Report(Box::new(self)))?;
+
+        log::trace!("exit: send_report");
+        Ok(())
+    }
+}
+
+/// Implement FeroxSerialize for FeroxResponse
+impl FeroxSerialize for FeroxResponse {
+    /// Simple wrapper around create_report_string
+    fn as_str(&self) -> String {
+        let lines = self.line_count().to_string();
+        let words = self.word_count().to_string();
+        let chars = self.content_length().to_string();
+        let status = self.status().as_str();
+        let wild_status = status_colorizer("WLD");
+
+        if self.wildcard && matches!(self.output_level, OutputLevel::Default | OutputLevel::Quiet) {
+            // --silent was not used and response is a wildcard, special messages abound when
+            // this is the case...
+
+            // create the base message
+            let mut message = format!(
+                "{} {:>8}l {:>8}w {:>8}c Got {} for {} (url length: {})\n",
+                wild_status,
+                lines,
+                words,
+                chars,
+                status_colorizer(&status),
+                self.url(),
+                FeroxUrl::path_length_of_url(&self.url)
+            );
+
+            if self.status().is_redirection() {
+                // when it's a redirect, show where it goes, if possible
+                if let Some(next_loc) = self.headers().get("Location") {
+                    let next_loc_str = next_loc.to_str().unwrap_or("Unknown");
+
+                    let redirect_msg = format!(
+                        "{} {:>9} {:>9} {:>9} {} redirects to => {}\n",
+                        wild_status,
+                        "-",
+                        "-",
+                        "-",
+                        self.url(),
+                        next_loc_str
+                    );
+
+                    message.push_str(&redirect_msg);
+                }
+            }
+
+            // base message + redirection message (if appropriate)
+            message
+        } else {
+            // not a wildcard, just create a normal entry
+            utils::create_report_string(
+                self.status.as_str(),
+                &lines,
+                &words,
+                &chars,
+                self.url().as_str(),
+                self.output_level,
+            )
+        }
+    }
+
+    /// Create an NDJSON representation of the FeroxResponse
+    ///
+    /// (expanded for clarity)
+    /// ex:
+    /// {
+    ///    "type":"response",
+    ///    "url":"https://localhost.com/images",
+    ///    "path":"/images",
+    ///    "status":301,
+    ///    "content_length":179,
+    ///    "line_count":10,
+    ///    "word_count":16,
+    ///    "headers":{
+    ///       "x-content-type-options":"nosniff",
+    ///       "strict-transport-security":"max-age=31536000; includeSubDomains",
+    ///       "x-frame-options":"SAMEORIGIN",
+    ///       "connection":"keep-alive",
+    ///       "server":"nginx/1.16.1",
+    ///       "content-type":"text/html; charset=UTF-8",
+    ///       "referrer-policy":"origin-when-cross-origin",
+    ///       "content-security-policy":"default-src 'none'",
+    ///       "access-control-allow-headers":"X-Requested-With",
+    ///       "x-xss-protection":"1; mode=block",
+    ///       "content-length":"179",
+    ///       "date":"Mon, 23 Nov 2020 15:33:24 GMT",
+    ///       "location":"/images/",
+    ///       "access-control-allow-origin":"https://localhost.com"
+    ///    }
+    /// }\n
+    fn as_json(&self) -> anyhow::Result<String> {
+        let mut json = serde_json::to_string(&self)
+            .with_context(|| fmt_err(&format!("Could not convert {} to JSON", self.url())))?;
+        json.push('\n');
+        Ok(json)
+    }
+}
+
+/// Serialize implementation for FeroxResponse
+impl Serialize for FeroxResponse {
+    /// Function that handles serialization of a FeroxResponse to NDJSON
+    fn serialize<S>(&self, serializer: S) -> anyhow::Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let mut headers = HashMap::new();
+        let mut state = serializer.serialize_struct("FeroxResponse", 7)?;
+
+        // need to convert the HeaderMap to a HashMap in order to pass it to the serializer
+        for (key, value) in &self.headers {
+            let k = key.as_str().to_owned();
+            let v = String::from_utf8_lossy(value.as_bytes());
+            headers.insert(k, v);
+        }
+
+        state.serialize_field("type", "response")?;
+        state.serialize_field("url", self.url.as_str())?;
+        state.serialize_field("path", self.url.path())?;
+        state.serialize_field("wildcard", &self.wildcard)?;
+        state.serialize_field("status", &self.status.as_u16())?;
+        state.serialize_field("content_length", &self.content_length)?;
+        state.serialize_field("line_count", &self.line_count)?;
+        state.serialize_field("word_count", &self.word_count)?;
+        state.serialize_field("headers", &headers)?;
+
+        state.end()
+    }
+}
+
+/// Deserialize implementation for FeroxResponse
+impl<'de> Deserialize<'de> for FeroxResponse {
+    /// Deserialize a FeroxResponse from a serde_json::Value
+    fn deserialize<D>(deserializer: D) -> anyhow::Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let mut response = Self {
+            url: Url::parse("http://localhost").unwrap(),
+            status: StatusCode::OK,
+            text: String::new(),
+            content_length: 0,
+            headers: HeaderMap::new(),
+            wildcard: false,
+            output_level: Default::default(),
+            line_count: 0,
+            word_count: 0,
+        };
+
+        let map: HashMap<String, Value> = HashMap::deserialize(deserializer)?;
+
+        for (key, value) in &map {
+            match key.as_str() {
+                "url" => {
+                    if let Some(url) = value.as_str() {
+                        if let Ok(parsed) = Url::parse(url) {
+                            response.url = parsed;
+                        }
+                    }
+                }
+                "status" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(smaller) = u16::try_from(num) {
+                            if let Ok(status) = StatusCode::from_u16(smaller) {
+                                response.status = status;
+                            }
+                        }
+                    }
+                }
+                "content_length" => {
+                    if let Some(num) = value.as_u64() {
+                        response.content_length = num;
+                    }
+                }
+                "line_count" => {
+                    if let Some(num) = value.as_u64() {
+                        response.line_count = num.try_into().unwrap_or_default();
+                    }
+                }
+                "word_count" => {
+                    if let Some(num) = value.as_u64() {
+                        response.word_count = num.try_into().unwrap_or_default();
+                    }
+                }
+                "headers" => {
+                    let mut headers = HeaderMap::<HeaderValue>::default();
+
+                    if let Some(map_headers) = value.as_object() {
+                        for (h_key, h_value) in map_headers {
+                            let h_value_str = h_value.as_str().unwrap_or("");
+                            let h_name = HeaderName::from_str(h_key)
+                                .unwrap_or_else(|_| HeaderName::from_str("Unknown").unwrap());
+                            let h_value_parsed = HeaderValue::from_str(h_value_str)
+                                .unwrap_or_else(|_| HeaderValue::from_str("Unknown").unwrap());
+                            headers.insert(h_name, h_value_parsed);
+                        }
+                    }
+
+                    response.headers = headers;
+                }
+                "wildcard" => {
+                    if let Some(result) = value.as_bool() {
+                        response.wildcard = result;
+                    }
+                }
+                _ => {}
+            }
+        }
+
+        Ok(response)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    /// call reached_max_depth with max depth of zero, which is infinite recursion, expect false
+    fn reached_max_depth_returns_early_on_zero() {
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+        let url = Url::parse("http://localhost").unwrap();
+        let response = FeroxResponse {
+            url,
+            status: Default::default(),
+            text: "".to_string(),
+            content_length: 0,
+            line_count: 0,
+            word_count: 0,
+            headers: Default::default(),
+            wildcard: false,
+            output_level: Default::default(),
+        };
+        let result = response.reached_max_depth(0, 0, handles);
+        assert!(!result);
+    }
+
+    #[test]
+    /// call reached_max_depth with url depth equal to max depth, expect true
+    fn reached_max_depth_current_depth_equals_max() {
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+
+        let url = Url::parse("http://localhost/one/two").unwrap();
+        let response = FeroxResponse {
+            url,
+            status: Default::default(),
+            text: "".to_string(),
+            content_length: 0,
+            line_count: 0,
+            word_count: 0,
+            headers: Default::default(),
+            wildcard: false,
+            output_level: Default::default(),
+        };
+
+        let result = response.reached_max_depth(0, 2, handles);
+        assert!(result);
+    }
+
+    #[test]
+    /// call reached_max_depth with url dpeth less than max depth, expect false
+    fn reached_max_depth_current_depth_less_than_max() {
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+        let url = Url::parse("http://localhost").unwrap();
+        let response = FeroxResponse {
+            url,
+            status: Default::default(),
+            text: "".to_string(),
+            content_length: 0,
+            line_count: 0,
+            word_count: 0,
+            headers: Default::default(),
+            wildcard: false,
+            output_level: Default::default(),
+        };
+
+        let result = response.reached_max_depth(0, 2, handles);
+        assert!(!result);
+    }
+
+    #[test]
+    /// call reached_max_depth with url of 2, base depth of 2, and max depth of 2, expect false
+    fn reached_max_depth_base_depth_equals_max_depth() {
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+        let url = Url::parse("http://localhost/one/two").unwrap();
+        let response = FeroxResponse {
+            url,
+            status: Default::default(),
+            text: "".to_string(),
+            content_length: 0,
+            line_count: 0,
+            word_count: 0,
+            headers: Default::default(),
+            wildcard: false,
+            output_level: Default::default(),
+        };
+
+        let result = response.reached_max_depth(2, 2, handles);
+        assert!(!result);
+    }
+
+    #[test]
+    /// call reached_max_depth with url depth greater than max depth, expect true
+    fn reached_max_depth_current_greater_than_max() {
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+        let url = Url::parse("http://localhost/one/two/three").unwrap();
+        let response = FeroxResponse {
+            url,
+            status: Default::default(),
+            text: "".to_string(),
+            content_length: 0,
+            line_count: 0,
+            word_count: 0,
+            headers: Default::default(),
+            wildcard: false,
+            output_level: Default::default(),
+        };
+
+        let result = response.reached_max_depth(0, 2, handles);
+        assert!(result);
+    }
+}

src/scan_manager/menu.rs

@@ -0,0 +1,135 @@
+use crate::progress::PROGRESS_BAR;
+use console::{measure_text_width, pad_str, style, Alignment, Term};
+use indicatif::ProgressDrawTarget;
+
+/// Interactive scan cancellation menu
+#[derive(Debug)]
+pub(super) struct Menu {
+    /// character to use as visual separator of lines
+    separator: String,
+
+    /// name of menu
+    name: String,
+
+    /// header: name surrounded by separators
+    header: String,
+
+    /// instructions
+    instructions: String,
+
+    /// footer: instructions surrounded by separators
+    footer: String,
+
+    /// target for output
+    term: Term,
+}
+
+/// Implementation of Menu
+impl Menu {
+    /// Creates new Menu
+    pub(super) fn new() -> Self {
+        let separator = "─".to_string();
+
+        let instructions = format!(
+            "Enter a {} list of indexes to {} (ex: 2,3)",
+            style("comma-separated").yellow(),
+            style("cancel").red(),
+        );
+
+        let name = format!(
+            "{} {} {}",
+            "💀",
+            style("Scan Cancel Menu").bright().yellow(),
+            "💀"
+        );
+
+        let longest = measure_text_width(&instructions).max(measure_text_width(&name));
+
+        let border = separator.repeat(longest);
+
+        let padded_name = pad_str(&name, longest, Alignment::Center, None);
+
+        let header = format!("{}\n{}\n{}", border, padded_name, border);
+        let footer = format!("{}\n{}\n{}", border, instructions, border);
+
+        Self {
+            separator,
+            name,
+            header,
+            instructions,
+            footer,
+            term: Term::stderr(),
+        }
+    }
+
+    /// print menu header
+    pub(super) fn print_header(&self) {
+        self.println(&self.header);
+    }
+
+    /// print menu footer
+    pub(super) fn print_footer(&self) {
+        self.println(&self.footer);
+    }
+
+    /// set PROGRESS_BAR bar target to hidden
+    pub(super) fn hide_progress_bars(&self) {
+        PROGRESS_BAR.set_draw_target(ProgressDrawTarget::hidden());
+    }
+
+    /// set PROGRESS_BAR bar target to hidden
+    pub(super) fn show_progress_bars(&self) {
+        PROGRESS_BAR.set_draw_target(ProgressDrawTarget::stdout());
+    }
+
+    /// Wrapper around console's Term::clear_screen and flush
+    pub(super) fn clear_screen(&self) {
+        self.term.clear_screen().unwrap_or_default();
+        self.term.flush().unwrap_or_default();
+    }
+
+    /// Wrapper around console's Term::write_line
+    pub(super) fn println(&self, msg: &str) {
+        self.term.write_line(msg).unwrap_or_default();
+    }
+
+    /// split a string into vec of usizes
+    pub(super) fn split_to_nums(&self, line: &str) -> Vec<usize> {
+        line.split(',')
+            .map(|s| {
+                s.trim().to_string().parse::<usize>().unwrap_or_else(|e| {
+                    self.println(&format!("Found non-numeric input: {}", e));
+                    0
+                })
+            })
+            .filter(|m| *m != 0)
+            .collect()
+    }
+
+    /// get comma-separated list of scan indexes from the user
+    pub(super) fn get_scans_from_user(&self) -> Option<Vec<usize>> {
+        if let Ok(line) = self.term.read_line() {
+            Some(self.split_to_nums(&line))
+        } else {
+            None
+        }
+    }
+
+    /// Given a url, confirm with user that we should cancel
+    pub(super) fn confirm_cancellation(&self, url: &str) -> char {
+        self.println(&format!(
+            "You sure you wanna cancel this scan: {}? [Y/n]",
+            url
+        ));
+
+        self.term.read_char().unwrap_or('n')
+    }
+}
+
+/// Default implementation for Menu
+impl Default for Menu {
+    /// return Menu::new as default
+    fn default() -> Menu {
+        Menu::new()
+    }
+}

src/scan_manager/mod.rs

@@ -0,0 +1,17 @@
+mod scan_container;
+mod response_container;
+mod scan;
+mod menu;
+mod utils;
+mod order;
+mod state;
+#[cfg(test)]
+mod tests;
+
+pub(self) use menu::Menu;
+pub use order::ScanOrder;
+pub use response_container::FeroxResponses;
+pub use scan::{FeroxScan, ScanStatus, ScanType};
+pub use scan_container::{FeroxScans, PAUSE_SCAN};
+pub use state::FeroxState;
+pub use utils::{resume_scan, start_max_time_thread};

src/scan_manager/order.rs

@@ -0,0 +1,10 @@
+#[derive(Debug, Copy, Clone)]
+/// Simple enum to designate whether a URL was passed in by the user (Initial) or found during
+/// scanning (Latest)
+pub enum ScanOrder {
+    /// Url was passed in by the user
+    Initial,
+
+    /// Url was found during scanning
+    Latest,
+}

src/scan_manager/response_container.rs

@@ -0,0 +1,55 @@
+use crate::response::FeroxResponse;
+use serde::{ser::SerializeSeq, Serialize, Serializer};
+use std::sync::{Arc, RwLock};
+
+/// Container around a locked vector of `FeroxResponse`s, adds wrappers for insertion and search
+#[derive(Debug, Default)]
+pub struct FeroxResponses {
+    /// Internal structure: locked hashset of `FeroxScan`s
+    pub responses: Arc<RwLock<Vec<FeroxResponse>>>,
+}
+
+/// Serialize implementation for FeroxResponses
+impl Serialize for FeroxResponses {
+    /// Function that handles serialization of FeroxResponses
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        if let Ok(responses) = self.responses.read() {
+            let mut seq = serializer.serialize_seq(Some(responses.len()))?;
+
+            for response in responses.iter() {
+                seq.serialize_element(response)?;
+            }
+
+            seq.end()
+        } else {
+            // if for some reason we can't unlock the mutex, just write an empty list
+            let seq = serializer.serialize_seq(Some(0))?;
+            seq.end()
+        }
+    }
+}
+
+/// Implementation of `FeroxResponses`
+impl FeroxResponses {
+    /// Add a `FeroxResponse` to the internal container
+    pub fn insert(&self, response: FeroxResponse) {
+        if let Ok(mut responses) = self.responses.write() {
+            responses.push(response);
+        }
+    }
+
+    /// Simple check for whether or not a FeroxResponse is contained within the inner container
+    pub fn contains(&self, other: &FeroxResponse) -> bool {
+        if let Ok(responses) = self.responses.read() {
+            for response in responses.iter() {
+                if response.url() == other.url() {
+                    return true;
+                }
+            }
+        }
+        false
+    }
+}

src/scan_manager/scan.rs

@@ -0,0 +1,508 @@
+use super::*;
+use crate::{
+    config::OutputLevel,
+    progress::{add_bar, BarType},
+    scanner::PolicyTrigger,
+};
+use anyhow::Result;
+use console::style;
+use indicatif::ProgressBar;
+use serde::{ser::SerializeStruct, Deserialize, Deserializer, Serialize, Serializer};
+use serde_json::Value;
+use std::{
+    collections::HashMap,
+    fmt,
+    sync::{Arc, Mutex},
+    time::Instant,
+};
+
+use std::sync::atomic::{AtomicUsize, Ordering};
+use tokio::{sync, task::JoinHandle};
+use uuid::Uuid;
+
+/// Struct to hold scan-related state
+///
+/// The purpose of this container is to open up the pathway to aborting currently running tasks and
+/// serialization of all scan state into a state file in order to resume scans that were cut short
+#[derive(Debug)]
+pub struct FeroxScan {
+    /// UUID that uniquely ID's the scan
+    pub(super) id: String,
+
+    /// The URL that to be scanned
+    pub(super) url: String,
+
+    /// The type of scan
+    pub(super) scan_type: ScanType,
+
+    /// The order in which the scan was received
+    pub(super) scan_order: ScanOrder,
+
+    /// Number of requests to populate the progress bar with
+    pub(super) num_requests: u64,
+
+    /// Status of this scan
+    pub(super) status: Mutex<ScanStatus>,
+
+    /// The spawned tokio task performing this scan (uses tokio::sync::Mutex)
+    pub(super) task: sync::Mutex<Option<JoinHandle<()>>>,
+
+    /// The progress bar associated with this scan
+    pub(super) progress_bar: Mutex<Option<ProgressBar>>,
+
+    /// whether or not the user passed --silent|--quiet on the command line
+    pub(super) output_level: OutputLevel,
+
+    /// tracker for overall number of 403s seen by the FeroxScan instance
+    pub(super) status_403s: AtomicUsize,
+
+    /// tracker for overall number of 429s seen by the FeroxScan instance
+    pub(super) status_429s: AtomicUsize,
+
+    /// tracker for total number of errors encountered by the FeroxScan instance
+    pub(super) errors: AtomicUsize,
+
+    /// tracker for the time at which this scan was started
+    pub(super) start_time: Instant,
+}
+
+/// Default implementation for FeroxScan
+impl Default for FeroxScan {
+    /// Create a default FeroxScan, populates ID with a new UUID
+    fn default() -> Self {
+        let new_id = Uuid::new_v4().to_simple().to_string();
+
+        FeroxScan {
+            id: new_id,
+            task: sync::Mutex::new(None), // tokio mutex
+            status: Mutex::new(ScanStatus::default()),
+            num_requests: 0,
+            scan_order: ScanOrder::Latest,
+            url: String::new(),
+            progress_bar: Mutex::new(None),
+            scan_type: ScanType::File,
+            output_level: Default::default(),
+            errors: Default::default(),
+            status_429s: Default::default(),
+            status_403s: Default::default(),
+            start_time: Instant::now(),
+        }
+    }
+}
+
+/// Implementation of FeroxScan
+impl FeroxScan {
+    /// Stop a currently running scan
+    pub async fn abort(&self) -> Result<()> {
+        log::trace!("enter: abort");
+
+        match self.task.try_lock() {
+            Ok(mut guard) => {
+                if let Some(task) = std::mem::replace(&mut *guard, None) {
+                    log::trace!("aborting {:?}", self);
+                    task.abort();
+                    self.set_status(ScanStatus::Cancelled)?;
+                    self.stop_progress_bar();
+                }
+            }
+            Err(e) => {
+                log::warn!("Could not acquire lock to abort scan (we're already waiting for its results): {:?} {}", self, e);
+            }
+        }
+        log::trace!("exit: abort");
+        Ok(())
+    }
+
+    /// getter for url
+    pub fn url(&self) -> &str {
+        &self.url
+    }
+
+    /// small wrapper to set the JoinHandle
+    pub async fn set_task(&self, task: JoinHandle<()>) -> Result<()> {
+        let mut guard = self.task.lock().await;
+        let _ = std::mem::replace(&mut *guard, Some(task));
+        Ok(())
+    }
+
+    /// small wrapper to set ScanStatus
+    pub fn set_status(&self, status: ScanStatus) -> Result<()> {
+        if let Ok(mut guard) = self.status.lock() {
+            let _ = std::mem::replace(&mut *guard, status);
+        }
+        Ok(())
+    }
+
+    /// Simple helper to call .finish on the scan's progress bar
+    pub(super) fn stop_progress_bar(&self) {
+        if let Ok(guard) = self.progress_bar.lock() {
+            if guard.is_some() {
+                (*guard).as_ref().unwrap().finish_at_current_pos()
+            }
+        }
+    }
+
+    /// Simple helper get a progress bar
+    pub fn progress_bar(&self) -> ProgressBar {
+        match self.progress_bar.lock() {
+            Ok(mut guard) => {
+                if guard.is_some() {
+                    (*guard).as_ref().unwrap().clone()
+                } else {
+                    let bar_type = match self.output_level {
+                        OutputLevel::Default => BarType::Default,
+                        OutputLevel::Quiet => BarType::Quiet,
+                        OutputLevel::Silent => BarType::Hidden,
+                    };
+
+                    let pb = add_bar(&self.url, self.num_requests, bar_type);
+                    pb.reset_elapsed();
+
+                    let _ = std::mem::replace(&mut *guard, Some(pb.clone()));
+
+                    pb
+                }
+            }
+            Err(_) => {
+                log::warn!("Could not unlock progress bar on {:?}", self);
+
+                let bar_type = match self.output_level {
+                    OutputLevel::Default => BarType::Default,
+                    OutputLevel::Quiet => BarType::Quiet,
+                    OutputLevel::Silent => BarType::Hidden,
+                };
+
+                let pb = add_bar(&self.url, self.num_requests, bar_type);
+                pb.reset_elapsed();
+
+                pb
+            }
+        }
+    }
+
+    /// Given a URL and ProgressBar, create a new FeroxScan, wrap it in an Arc and return it
+    pub fn new(
+        url: &str,
+        scan_type: ScanType,
+        scan_order: ScanOrder,
+        num_requests: u64,
+        output_level: OutputLevel,
+        pb: Option<ProgressBar>,
+    ) -> Arc<Self> {
+        Arc::new(Self {
+            url: url.to_string(),
+            scan_type,
+            scan_order,
+            num_requests,
+            output_level,
+            progress_bar: Mutex::new(pb),
+            ..Default::default()
+        })
+    }
+
+    /// Mark the scan as complete and stop the scan's progress bar
+    pub fn finish(&self) -> Result<()> {
+        self.set_status(ScanStatus::Complete)?;
+        self.stop_progress_bar();
+        Ok(())
+    }
+
+    /// small wrapper to inspect ScanType and ScanStatus to see if a Directory scan is running or
+    /// in the queue to be run
+    pub fn is_active(&self) -> bool {
+        if let Ok(guard) = self.status.lock() {
+            return matches!(
+                (self.scan_type, *guard),
+                (ScanType::Directory, ScanStatus::Running)
+                    | (ScanType::Directory, ScanStatus::NotStarted)
+            );
+        }
+        false
+    }
+
+    /// small wrapper to inspect ScanStatus and see if it's Complete
+    pub fn is_complete(&self) -> bool {
+        if let Ok(guard) = self.status.lock() {
+            return matches!(*guard, ScanStatus::Complete);
+        }
+        false
+    }
+
+    /// await a task's completion, similar to a thread's join; perform necessary bookkeeping
+    pub async fn join(&self) {
+        log::trace!("enter join({:?})", self);
+        let mut guard = self.task.lock().await;
+
+        if guard.is_some() {
+            if let Some(task) = std::mem::replace(&mut *guard, None) {
+                task.await.unwrap();
+                self.set_status(ScanStatus::Complete)
+                    .unwrap_or_else(|e| log::warn!("Could not mark scan complete: {}", e))
+            }
+        }
+
+        log::trace!("exit join({:?})", self);
+    }
+    /// increment the value in question by 1
+    pub(crate) fn add_403(&self) {
+        self.status_403s.fetch_add(1, Ordering::Relaxed);
+    }
+
+    /// increment the value in question by 1
+    pub(crate) fn add_429(&self) {
+        self.status_429s.fetch_add(1, Ordering::Relaxed);
+    }
+
+    /// increment the value in question by 1
+    pub(crate) fn add_error(&self) {
+        self.errors.fetch_add(1, Ordering::Relaxed);
+    }
+
+    /// simple wrapper to call the appropriate getter based on the given PolicyTrigger
+    pub fn num_errors(&self, trigger: PolicyTrigger) -> usize {
+        match trigger {
+            PolicyTrigger::Status403 => self.status_403s(),
+            PolicyTrigger::Status429 => self.status_429s(),
+            PolicyTrigger::Errors => self.errors(),
+        }
+    }
+
+    /// return the number of errors seen by this scan
+    fn errors(&self) -> usize {
+        self.errors.load(Ordering::Relaxed)
+    }
+
+    /// return the number of 403s seen by this scan
+    fn status_403s(&self) -> usize {
+        self.status_403s.load(Ordering::Relaxed)
+    }
+
+    /// return the number of 429s seen by this scan
+    fn status_429s(&self) -> usize {
+        self.status_429s.load(Ordering::Relaxed)
+    }
+
+    /// return the number of requests per second performed by this scan's scanner
+    pub fn requests_per_second(&self) -> u64 {
+        if !self.is_active() {
+            return 0;
+        }
+
+        let reqs = self.requests();
+        let seconds = self.start_time.elapsed().as_secs();
+
+        reqs.checked_div(seconds).unwrap_or(0)
+    }
+
+    /// return the number of requests performed by this scan's scanner
+    pub fn requests(&self) -> u64 {
+        self.progress_bar().position()
+    }
+}
+
+/// Display implementation
+impl fmt::Display for FeroxScan {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let status = if let Ok(guard) = self.status.lock() {
+            match *guard {
+                ScanStatus::NotStarted => style("not started").bright().blue(),
+                ScanStatus::Complete => style("complete").green(),
+                ScanStatus::Cancelled => style("cancelled").red(),
+                ScanStatus::Running => style("running").bright().yellow(),
+            }
+        } else {
+            style("unknown").red()
+        };
+
+        write!(f, "{:12} {}", status, self.url)
+    }
+}
+
+/// PartialEq implementation; uses FeroxScan.id for comparison
+impl PartialEq for FeroxScan {
+    fn eq(&self, other: &Self) -> bool {
+        self.id == other.id
+    }
+}
+
+/// Serialize implementation for FeroxScan
+impl Serialize for FeroxScan {
+    /// Function that handles serialization of a FeroxScan
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let mut state = serializer.serialize_struct("FeroxScan", 4)?;
+
+        state.serialize_field("id", &self.id)?;
+        state.serialize_field("url", &self.url)?;
+        state.serialize_field("scan_type", &self.scan_type)?;
+        state.serialize_field("status", &self.status)?;
+        state.serialize_field("num_requests", &self.num_requests)?;
+
+        state.end()
+    }
+}
+
+/// Deserialize implementation for FeroxScan
+impl<'de> Deserialize<'de> for FeroxScan {
+    /// Deserialize a FeroxScan from a serde_json::Value
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let mut scan = Self::default();
+
+        let map: HashMap<String, Value> = HashMap::deserialize(deserializer)?;
+
+        for (key, value) in &map {
+            match key.as_str() {
+                "id" => {
+                    if let Some(id) = value.as_str() {
+                        scan.id = id.to_string();
+                    }
+                }
+                "scan_type" => {
+                    if let Some(scan_type) = value.as_str() {
+                        scan.scan_type = match scan_type {
+                            "File" => ScanType::File,
+                            "Directory" => ScanType::Directory,
+                            _ => ScanType::File,
+                        }
+                    }
+                }
+                "status" => {
+                    if let Some(status) = value.as_str() {
+                        scan.status = Mutex::new(match status {
+                            "NotStarted" => ScanStatus::NotStarted,
+                            "Running" => ScanStatus::Running,
+                            "Complete" => ScanStatus::Complete,
+                            "Cancelled" => ScanStatus::Cancelled,
+                            _ => ScanStatus::default(),
+                        })
+                    }
+                }
+                "url" => {
+                    if let Some(url) = value.as_str() {
+                        scan.url = url.to_string();
+                    }
+                }
+                "num_requests" => {
+                    if let Some(num_requests) = value.as_u64() {
+                        scan.num_requests = num_requests;
+                    }
+                }
+                _ => {}
+            }
+        }
+
+        Ok(scan)
+    }
+}
+
+/// Simple enum used to flag a `FeroxScan` as likely a directory or file
+#[derive(Debug, Serialize, Deserialize, Copy, Clone)]
+pub enum ScanType {
+    /// Just a file being requested
+    File,
+
+    /// A an entire directory that might be scanned
+    Directory,
+}
+
+/// Default implementation for ScanType
+impl Default for ScanType {
+    /// Return ScanType::File as default
+    fn default() -> Self {
+        Self::File
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, Copy, Clone)]
+/// Simple enum to represent a scan's current status ([in]complete, cancelled)
+pub enum ScanStatus {
+    /// Scan hasn't started yet
+    NotStarted,
+
+    /// Scan finished normally
+    Complete,
+
+    /// Scan was cancelled by the user
+    Cancelled,
+
+    /// Scan has started, but hasn't finished, nor been cancelled
+    Running,
+}
+
+/// Default implementation for ScanStatus
+impl Default for ScanStatus {
+    /// Default variant for ScanStatus is NotStarted
+    fn default() -> Self {
+        Self::NotStarted
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::thread::sleep;
+    use tokio::time::Duration;
+
+    #[test]
+    /// ensure that num_errors returns the correct values for the given PolicyTrigger
+    ///
+    /// covers tests for add_[403,429,error] and the related getters in addition to num_errors
+    fn num_errors_returns_correct_values() {
+        let scan = FeroxScan::new(
+            "http://localhost",
+            ScanType::Directory,
+            ScanOrder::Latest,
+            1000,
+            OutputLevel::Default,
+            None,
+        );
+
+        scan.add_error();
+        scan.add_403();
+        scan.add_403();
+        scan.add_429();
+        scan.add_429();
+        scan.add_429();
+
+        assert_eq!(scan.num_errors(PolicyTrigger::Errors), 1);
+        assert_eq!(scan.num_errors(PolicyTrigger::Status403), 2);
+        assert_eq!(scan.num_errors(PolicyTrigger::Status429), 3);
+    }
+
+    #[test]
+    /// ensure that requests_per_second returns the correct values
+    fn requests_per_second_returns_correct_values() {
+        let scan = FeroxScan {
+            id: "".to_string(),
+            url: "".to_string(),
+            scan_type: ScanType::Directory,
+            scan_order: ScanOrder::Initial,
+            num_requests: 0,
+            status: Mutex::new(ScanStatus::Running),
+            task: Default::default(),
+            progress_bar: Mutex::new(None),
+            output_level: Default::default(),
+            status_403s: Default::default(),
+            status_429s: Default::default(),
+            errors: Default::default(),
+            start_time: Instant::now(),
+        };
+
+        let pb = scan.progress_bar();
+        pb.set_position(100);
+
+        sleep(Duration::new(1, 0));
+
+        let req_sec = scan.requests_per_second();
+
+        assert_eq!(req_sec, 100);
+
+        scan.finish().unwrap();
+        assert_eq!(scan.requests_per_second(), 0);
+    }
+}

src/scan_manager/scan_container.rs

@@ -0,0 +1,499 @@
+use super::scan::ScanType;
+use super::*;
+use crate::{
+    config::OutputLevel,
+    progress::PROGRESS_PRINTER,
+    progress::{add_bar, BarType},
+    scanner::RESPONSES,
+    traits::FeroxSerialize,
+    SLEEP_DURATION,
+};
+use anyhow::Result;
+use reqwest::StatusCode;
+use serde::{ser::SerializeSeq, Serialize, Serializer};
+use std::{
+    convert::TryInto,
+    fs::File,
+    io::BufReader,
+    ops::Index,
+    sync::{
+        atomic::{AtomicBool, AtomicUsize, Ordering},
+        Arc, Mutex, RwLock,
+    },
+    thread::sleep,
+};
+use tokio::time::{self, Duration};
+
+/// Single atomic number that gets incremented once, used to track first thread to interact with
+/// when pausing a scan
+static INTERACTIVE_BARRIER: AtomicUsize = AtomicUsize::new(0);
+
+/// Atomic boolean flag, used to determine whether or not a scan should pause or resume
+pub static PAUSE_SCAN: AtomicBool = AtomicBool::new(false);
+
+/// Container around a locked hashset of `FeroxScan`s, adds wrappers for insertion and searching
+#[derive(Debug, Default)]
+pub struct FeroxScans {
+    /// Internal structure: locked hashset of `FeroxScan`s
+    pub scans: RwLock<Vec<Arc<FeroxScan>>>,
+
+    /// menu used for providing a way for users to cancel a scan
+    menu: Menu,
+
+    /// number of requests expected per scan (mirrors the same on Stats); used for initializing
+    /// progress bars and feroxscans
+    bar_length: Mutex<u64>,
+
+    /// whether or not the user passed --silent|--quiet on the command line
+    output_level: OutputLevel,
+}
+
+/// Serialize implementation for FeroxScans
+///
+/// purposefully skips menu attribute
+impl Serialize for FeroxScans {
+    /// Function that handles serialization of FeroxScans
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        if let Ok(scans) = self.scans.read() {
+            let mut seq = serializer.serialize_seq(Some(scans.len()))?;
+            for scan in scans.iter() {
+                seq.serialize_element(&*scan).unwrap_or_default();
+            }
+
+            seq.end()
+        } else {
+            // if for some reason we can't unlock the RwLock, just write an empty list
+            let seq = serializer.serialize_seq(Some(0))?;
+            seq.end()
+        }
+    }
+}
+
+/// Implementation of `FeroxScans`
+impl FeroxScans {
+    /// given an OutputLevel, create a new FeroxScans object
+    pub fn new(output_level: OutputLevel) -> Self {
+        Self {
+            output_level,
+            ..Default::default()
+        }
+    }
+
+    /// Add a `FeroxScan` to the internal container
+    ///
+    /// If the internal container did NOT contain the scan, true is returned; else false
+    pub fn insert(&self, scan: Arc<FeroxScan>) -> bool {
+        // If the container did contain the scan, set sentry to false
+        // If the container did not contain the scan, set sentry to true
+        let sentry = !self.contains(&scan.url);
+
+        if sentry {
+            // can't update the internal container while the scan itself is locked, so first
+            // lock the scan and check the container for the scan's presence, then add if
+            // not found
+            match self.scans.write() {
+                Ok(mut scans) => {
+                    scans.push(scan);
+                }
+                Err(e) => {
+                    log::warn!("FeroxScans' container's mutex is poisoned: {}", e);
+                    return false;
+                }
+            }
+        }
+
+        sentry
+    }
+
+    /// load serialized FeroxScan(s) into this FeroxScans  
+    pub fn add_serialized_scans(&self, filename: &str) -> Result<()> {
+        log::trace!("enter: add_serialized_scans({})", filename);
+        let file = File::open(filename)?;
+
+        let reader = BufReader::new(file);
+        let state: serde_json::Value = serde_json::from_reader(reader)?;
+
+        if let Some(scans) = state.get("scans") {
+            if let Some(arr_scans) = scans.as_array() {
+                for scan in arr_scans {
+                    let mut deser_scan: FeroxScan =
+                        serde_json::from_value(scan.clone()).unwrap_or_default();
+                    // FeroxScans gets -q value from config as usual; the FeroxScans themselves
+                    // rely on that value being passed in. If the user starts a scan without -q
+                    // and resumes the scan but adds -q, FeroxScan will not have the proper value
+                    // without the line below
+                    deser_scan.output_level = self.output_level;
+
+                    log::debug!("added: {}", deser_scan);
+                    self.insert(Arc::new(deser_scan));
+                }
+            }
+        }
+
+        log::trace!("exit: add_serialized_scans");
+        Ok(())
+    }
+
+    /// Simple check for whether or not a FeroxScan is contained within the inner container based
+    /// on the given URL
+    pub fn contains(&self, url: &str) -> bool {
+        if let Ok(scans) = self.scans.read() {
+            for scan in scans.iter() {
+                if scan.url == url {
+                    return true;
+                }
+            }
+        }
+        false
+    }
+
+    /// Find and return a `FeroxScan` based on the given URL
+    pub fn get_scan_by_url(&self, url: &str) -> Option<Arc<FeroxScan>> {
+        if let Ok(guard) = self.scans.read() {
+            for scan in guard.iter() {
+                if scan.url == url {
+                    return Some(scan.clone());
+                }
+            }
+        }
+        None
+    }
+
+    pub(super) fn get_base_scan_by_url(&self, url: &str) -> Option<Arc<FeroxScan>> {
+        log::trace!("enter: get_sub_paths_from_path({})", url);
+
+        // rmatch_indices returns tuples in index, match form, i.e. (10, "/")
+        // with the furthest-right match in the first position in the vector
+        let matches: Vec<_> = url.rmatch_indices('/').collect();
+
+        // iterate from the furthest right matching index and check the given url from the
+        // start to the furthest-right '/' character. compare that slice to the urls associated
+        // with directory scans and return the first match, since it should be the 'deepest'
+        // match.
+        // Example:
+        //   url: http://shmocalhost/src/release/examples/stuff.php
+        //   scans:
+        //      http://shmocalhost/src/statistics
+        //      http://shmocalhost/src/banner
+        //      http://shmocalhost/src/release
+        //      http://shmocalhost/src/release/examples
+        //
+        //  returns: http://shmocalhost/src/release/examples
+        if let Ok(guard) = self.scans.read() {
+            for (idx, _) in &matches {
+                for scan in guard.iter() {
+                    let slice = url.index(0..*idx);
+                    if slice == scan.url || format!("{}/", slice).as_str() == scan.url {
+                        log::trace!("enter: get_sub_paths_from_path -> {}", scan);
+                        return Some(scan.clone());
+                    }
+                }
+            }
+        }
+
+        log::trace!("enter: get_sub_paths_from_path -> None");
+        None
+    }
+    /// add one to either 403 or 429 tracker in the scan related to the given url
+    pub fn increment_status_code(&self, url: &str, code: StatusCode) {
+        if let Some(scan) = self.get_base_scan_by_url(url) {
+            match code {
+                StatusCode::TOO_MANY_REQUESTS => {
+                    scan.add_429();
+                }
+                StatusCode::FORBIDDEN => {
+                    scan.add_403();
+                }
+                _ => {}
+            }
+        }
+    }
+
+    /// add one to either 403 or 429 tracker in the scan related to the given url
+    pub fn increment_error(&self, url: &str) {
+        if let Some(scan) = self.get_base_scan_by_url(url) {
+            scan.add_error();
+        }
+    }
+
+    /// Print all FeroxScans of type Directory
+    ///
+    /// Example:
+    ///   0: complete   https://10.129.45.20
+    ///   9: complete   https://10.129.45.20/images
+    ///  10: complete   https://10.129.45.20/assets
+    pub async fn display_scans(&self) {
+        let scans = {
+            // written this way in order to grab the vector and drop the lock immediately
+            // otherwise the spawned task that this is a part of is no longer Send due to
+            // the scan.task.lock().await below while the lock is held (RwLock is not Send)
+            self.scans
+                .read()
+                .expect("Could not acquire lock in display_scans")
+                .clone()
+        };
+
+        for (i, scan) in scans.iter().enumerate() {
+            if matches!(scan.scan_order, ScanOrder::Initial) || scan.task.try_lock().is_err() {
+                // original target passed in via either -u or --stdin
+                continue;
+            }
+
+            if matches!(scan.scan_type, ScanType::Directory) {
+                // we're only interested in displaying directory scans, as those are
+                // the only ones that make sense to be stopped
+                let scan_msg = format!("{:3}: {}", i, scan);
+                self.menu.println(&scan_msg);
+            }
+        }
+    }
+
+    /// Given a list of indexes, cancel their associated FeroxScans
+    async fn cancel_scans(&self, indexes: Vec<usize>) -> usize {
+        let menu_pause_duration = Duration::from_millis(SLEEP_DURATION);
+
+        let mut num_cancelled = 0_usize;
+
+        for num in indexes {
+            let selected = match self.scans.read() {
+                Ok(u_scans) => {
+                    // check if number provided is out of range
+                    if num >= u_scans.len() {
+                        // usize can't be negative, just need to handle exceeding bounds
+                        self.menu
+                            .println(&format!("The number {} is not a valid choice.", num));
+                        sleep(menu_pause_duration);
+                        continue;
+                    }
+                    u_scans.index(num).clone()
+                }
+                Err(..) => continue,
+            };
+
+            let input = self.menu.confirm_cancellation(&selected.url);
+
+            if input == 'y' || input == '\n' {
+                self.menu.println(&format!("Stopping {}...", selected.url));
+
+                selected
+                    .abort()
+                    .await
+                    .unwrap_or_else(|e| log::warn!("Could not cancel task: {}", e));
+
+                let pb = selected.progress_bar();
+                num_cancelled += pb.length() as usize - pb.position() as usize
+            } else {
+                self.menu.println("Ok, doing nothing...");
+            }
+
+            sleep(menu_pause_duration);
+        }
+
+        num_cancelled
+    }
+
+    /// CLI menu that allows for interactive cancellation of recursed-into directories
+    async fn interactive_menu(&self) -> usize {
+        self.menu.hide_progress_bars();
+        self.menu.clear_screen();
+        self.menu.print_header();
+        self.display_scans().await;
+        self.menu.print_footer();
+
+        let mut num_cancelled = 0_usize;
+
+        if let Some(input) = self.menu.get_scans_from_user() {
+            num_cancelled += self.cancel_scans(input).await;
+        };
+
+        self.menu.clear_screen();
+        self.menu.show_progress_bars();
+
+        num_cancelled
+    }
+
+    /// prints all known responses that the scanner has already seen
+    pub fn print_known_responses(&self) {
+        if let Ok(mut responses) = RESPONSES.responses.write() {
+            for response in responses.iter_mut() {
+                if self.output_level != response.output_level {
+                    // set the output_level prior to printing the response to ensure that the
+                    // response's setting aligns with the overall configuration (since we're
+                    // calling this from a resumed state)
+                    response.output_level = self.output_level;
+                }
+                PROGRESS_PRINTER.println(response.as_str());
+            }
+        }
+    }
+
+    /// if a resumed scan is already complete, display a completed progress bar to the user
+    pub fn print_completed_bars(&self, bar_length: usize) -> Result<()> {
+        let bar_type = match self.output_level {
+            OutputLevel::Default => BarType::Message,
+            OutputLevel::Quiet => BarType::Quiet,
+            OutputLevel::Silent => return Ok(()), // fast exit when --silent was used
+        };
+
+        if let Ok(scans) = self.scans.read() {
+            for scan in scans.iter() {
+                if scan.is_complete() {
+                    // these scans are complete, and just need to be shown to the user
+                    let pb = add_bar(
+                        &scan.url,
+                        bar_length.try_into().unwrap_or_default(),
+                        bar_type,
+                    );
+                    pb.finish();
+                }
+            }
+        }
+        Ok(())
+    }
+
+    /// Forced the calling thread into a busy loop
+    ///
+    /// Every `SLEEP_DURATION` milliseconds, the function examines the result stored in `PAUSE_SCAN`
+    ///
+    /// When the value stored in `PAUSE_SCAN` becomes `false`, the function returns, exiting the busy
+    /// loop
+    pub async fn pause(&self, get_user_input: bool) -> usize {
+        // function uses tokio::time, not std
+
+        // local testing showed a pretty slow increase (less than linear) in CPU usage as # of
+        // concurrent scans rose when SLEEP_DURATION was set to 500, using that as the default for now
+        let mut interval = time::interval(time::Duration::from_millis(SLEEP_DURATION));
+        let mut num_cancelled = 0_usize;
+
+        if INTERACTIVE_BARRIER.load(Ordering::Relaxed) == 0 {
+            INTERACTIVE_BARRIER.fetch_add(1, Ordering::Relaxed);
+
+            if get_user_input {
+                num_cancelled += self.interactive_menu().await;
+                PAUSE_SCAN.store(false, Ordering::Relaxed);
+                self.print_known_responses();
+            }
+        }
+
+        loop {
+            // first tick happens immediately, all others wait the specified duration
+            interval.tick().await;
+
+            if !PAUSE_SCAN.load(Ordering::Acquire) {
+                // PAUSE_SCAN is false, so we can exit the busy loop
+
+                if INTERACTIVE_BARRIER.load(Ordering::Relaxed) == 1 {
+                    INTERACTIVE_BARRIER.fetch_sub(1, Ordering::Relaxed);
+                }
+
+                log::trace!("exit: pause_scan -> {}", num_cancelled);
+                return num_cancelled;
+            }
+        }
+    }
+
+    /// set the bar length of FeroxScans
+    pub fn set_bar_length(&self, bar_length: u64) {
+        if let Ok(mut guard) = self.bar_length.lock() {
+            *guard = bar_length;
+        }
+    }
+
+    /// Given a url, create a new `FeroxScan` and add it to `FeroxScans`
+    ///
+    /// If `FeroxScans` did not already contain the scan, return true; otherwise return false
+    ///
+    /// Also return a reference to the new `FeroxScan`
+    pub(super) fn add_scan(
+        &self,
+        url: &str,
+        scan_type: ScanType,
+        scan_order: ScanOrder,
+    ) -> (bool, Arc<FeroxScan>) {
+        let bar_length = if let Ok(guard) = self.bar_length.lock() {
+            *guard
+        } else {
+            0
+        };
+
+        let bar = match scan_type {
+            ScanType::Directory => {
+                let bar_type = match self.output_level {
+                    OutputLevel::Default => BarType::Default,
+                    OutputLevel::Quiet => BarType::Quiet,
+                    OutputLevel::Silent => BarType::Hidden,
+                };
+
+                let progress_bar = add_bar(&url, bar_length, bar_type);
+
+                progress_bar.reset_elapsed();
+
+                Some(progress_bar)
+            }
+            ScanType::File => None,
+        };
+
+        let ferox_scan = FeroxScan::new(
+            &url,
+            scan_type,
+            scan_order,
+            bar_length,
+            self.output_level,
+            bar,
+        );
+
+        // If the set did not contain the scan, true is returned.
+        // If the set did contain the scan, false is returned.
+        let response = self.insert(ferox_scan.clone());
+
+        (response, ferox_scan)
+    }
+
+    /// Given a url, create a new `FeroxScan` and add it to `FeroxScans` as a Directory Scan
+    ///
+    /// If `FeroxScans` did not already contain the scan, return true; otherwise return false
+    ///
+    /// Also return a reference to the new `FeroxScan`
+    pub fn add_directory_scan(&self, url: &str, scan_order: ScanOrder) -> (bool, Arc<FeroxScan>) {
+        self.add_scan(&url, ScanType::Directory, scan_order)
+    }
+
+    /// Given a url, create a new `FeroxScan` and add it to `FeroxScans` as a File Scan
+    ///
+    /// If `FeroxScans` did not already contain the scan, return true; otherwise return false
+    ///
+    /// Also return a reference to the new `FeroxScan`
+    pub fn add_file_scan(&self, url: &str, scan_order: ScanOrder) -> (bool, Arc<FeroxScan>) {
+        self.add_scan(&url, ScanType::File, scan_order)
+    }
+
+    /// small helper to determine whether any scans are active or not
+    pub fn has_active_scans(&self) -> bool {
+        if let Ok(guard) = self.scans.read() {
+            for scan in guard.iter() {
+                if scan.is_active() {
+                    return true;
+                }
+            }
+        }
+        false
+    }
+
+    /// Retrieve all active scans
+    pub fn get_active_scans(&self) -> Vec<Arc<FeroxScan>> {
+        let mut scans = vec![];
+
+        if let Ok(guard) = self.scans.read() {
+            for scan in guard.iter() {
+                if !scan.is_active() {
+                    continue;
+                }
+                scans.push(scan.clone());
+            }
+        }
+        scans
+    }
+}

src/scan_manager/state.rs

@@ -0,0 +1,53 @@
+use super::*;
+use crate::{config::Configuration, statistics::Stats, traits::FeroxSerialize, utils::fmt_err};
+use anyhow::{Context, Result};
+use serde::Serialize;
+use std::sync::Arc;
+
+/// Data container for (de)?serialization of multiple items
+#[derive(Serialize, Debug)]
+pub struct FeroxState {
+    /// Known scans
+    scans: Arc<FeroxScans>,
+
+    /// Current running config
+    config: Arc<Configuration>,
+
+    /// Known responses
+    responses: &'static FeroxResponses,
+
+    /// Gathered statistics
+    statistics: Arc<Stats>,
+}
+
+/// implementation of FeroxState
+impl FeroxState {
+    /// create new FeroxState object
+    pub fn new(
+        scans: Arc<FeroxScans>,
+        config: Arc<Configuration>,
+        responses: &'static FeroxResponses,
+        statistics: Arc<Stats>,
+    ) -> Self {
+        Self {
+            scans,
+            config,
+            responses,
+            statistics,
+        }
+    }
+}
+
+/// FeroxSerialize implementation for FeroxState
+impl FeroxSerialize for FeroxState {
+    /// Simply return debug format of FeroxState to satisfy as_str
+    fn as_str(&self) -> String {
+        format!("{:?}", self)
+    }
+
+    /// Simple call to produce a JSON string using the given FeroxState
+    fn as_json(&self) -> Result<String> {
+        Ok(serde_json::to_string(&self)
+            .with_context(|| fmt_err("Could not convert scan's running state to JSON"))?)
+    }
+}

src/scan_manager/tests.rs

@@ -0,0 +1,594 @@
+use super::*;
+use crate::{
+    config::{Configuration, OutputLevel},
+    event_handlers::Handles,
+    response::FeroxResponse,
+    scanner::RESPONSES,
+    statistics::Stats,
+    traits::FeroxSerialize,
+    SLEEP_DURATION, VERSION,
+};
+use indicatif::ProgressBar;
+use predicates::prelude::*;
+use std::sync::{atomic::Ordering, Arc};
+use std::thread::sleep;
+use std::time::Instant;
+use tokio::time::{self, Duration};
+
+#[test]
+/// test that ScanType's default is File
+fn default_scantype_is_file() {
+    match ScanType::default() {
+        ScanType::File => {}
+        ScanType::Directory => panic!(),
+    }
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// tests that pause_scan pauses execution and releases execution when PAUSE_SCAN is toggled
+/// the spinner used during the test has had .finish_and_clear called on it, meaning that
+/// a new one will be created, taking the if branch within the function
+async fn scanner_pause_scan_with_finished_spinner() {
+    let now = time::Instant::now();
+    let urls = FeroxScans::default();
+
+    PAUSE_SCAN.store(true, Ordering::Relaxed);
+
+    let expected = time::Duration::from_secs(2);
+
+    tokio::spawn(async move {
+        time::sleep(expected).await;
+        PAUSE_SCAN.store(false, Ordering::Relaxed);
+    });
+
+    urls.pause(false).await;
+
+    assert!(now.elapsed() > expected);
+}
+
+#[test]
+/// add an unknown url to the hashset, expect true
+fn add_url_to_list_of_scanned_urls_with_unknown_url() {
+    let urls = FeroxScans::default();
+    let url = "http://unknown_url";
+    let (result, _scan) = urls.add_scan(url, ScanType::Directory, ScanOrder::Latest);
+    assert_eq!(result, true);
+}
+
+#[test]
+/// add a known url to the hashset, with a trailing slash, expect false
+fn add_url_to_list_of_scanned_urls_with_known_url() {
+    let urls = FeroxScans::default();
+    let pb = ProgressBar::new(1);
+    let url = "http://unknown_url/";
+
+    let scan = FeroxScan::new(
+        url,
+        ScanType::Directory,
+        ScanOrder::Latest,
+        pb.length(),
+        OutputLevel::Default,
+        Some(pb),
+    );
+
+    assert_eq!(urls.insert(scan), true);
+
+    let (result, _scan) = urls.add_scan(url, ScanType::Directory, ScanOrder::Latest);
+
+    assert_eq!(result, false);
+}
+
+#[test]
+/// stop_progress_bar should stop the progress bar
+fn stop_progress_bar_stops_bar() {
+    let pb = ProgressBar::new(1);
+    let url = "http://unknown_url/";
+
+    let scan = FeroxScan::new(
+        url,
+        ScanType::Directory,
+        ScanOrder::Latest,
+        pb.length(),
+        OutputLevel::Default,
+        Some(pb),
+    );
+
+    assert_eq!(
+        scan.progress_bar
+            .lock()
+            .unwrap()
+            .as_ref()
+            .unwrap()
+            .is_finished(),
+        false
+    );
+
+    scan.stop_progress_bar();
+
+    assert_eq!(
+        scan.progress_bar
+            .lock()
+            .unwrap()
+            .as_ref()
+            .unwrap()
+            .is_finished(),
+        true
+    );
+}
+
+#[test]
+/// add a known url to the hashset, without a trailing slash, expect false
+fn add_url_to_list_of_scanned_urls_with_known_url_without_slash() {
+    let urls = FeroxScans::default();
+    let url = "http://unknown_url";
+
+    let scan = FeroxScan::new(
+        url,
+        ScanType::File,
+        ScanOrder::Latest,
+        0,
+        OutputLevel::Default,
+        None,
+    );
+
+    assert_eq!(urls.insert(scan), true);
+
+    let (result, _scan) = urls.add_scan(url, ScanType::File, ScanOrder::Latest);
+
+    assert_eq!(result, false);
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// just increasing coverage, no real expectations
+async fn call_display_scans() {
+    let urls = FeroxScans::default();
+    let pb = ProgressBar::new(1);
+    let pb_two = ProgressBar::new(2);
+    let url = "http://unknown_url/";
+    let url_two = "http://unknown_url/fa";
+    let scan = FeroxScan::new(
+        url,
+        ScanType::Directory,
+        ScanOrder::Latest,
+        pb.length(),
+        OutputLevel::Default,
+        Some(pb),
+    );
+    let scan_two = FeroxScan::new(
+        url_two,
+        ScanType::Directory,
+        ScanOrder::Latest,
+        pb_two.length(),
+        OutputLevel::Default,
+        Some(pb_two),
+    );
+
+    scan_two.finish().unwrap(); // one complete, one incomplete
+    scan_two
+        .set_task(tokio::spawn(async move {
+            sleep(Duration::from_millis(SLEEP_DURATION));
+        }))
+        .await
+        .unwrap();
+
+    assert_eq!(urls.insert(scan), true);
+    assert_eq!(urls.insert(scan_two), true);
+
+    urls.display_scans().await;
+}
+
+#[test]
+/// ensure that PartialEq compares FeroxScan.id fields
+fn partial_eq_compares_the_id_field() {
+    let url = "http://unknown_url/";
+    let scan = FeroxScan::new(
+        url,
+        ScanType::Directory,
+        ScanOrder::Latest,
+        0,
+        OutputLevel::Default,
+        None,
+    );
+    let scan_two = FeroxScan::new(
+        url,
+        ScanType::Directory,
+        ScanOrder::Latest,
+        0,
+        OutputLevel::Default,
+        None,
+    );
+
+    assert!(!scan.eq(&scan_two));
+
+    let scan_two = scan.clone();
+
+    assert!(scan.eq(&scan_two));
+}
+
+#[test]
+/// show that a new progress bar is created if one doesn't exist
+fn ferox_scan_get_progress_bar_when_none_is_set() {
+    let scan = FeroxScan::default();
+
+    assert!(scan.progress_bar.lock().unwrap().is_none()); // no pb exists
+
+    let pb = scan.progress_bar();
+
+    assert!(scan.progress_bar.lock().unwrap().is_some()); // new pb created
+    assert!(!pb.is_finished()) // not finished
+}
+
+#[test]
+/// given a JSON entry representing a FeroxScan, test that it deserializes into the proper type
+/// with the right attributes
+fn ferox_scan_deserialize() {
+    let fs_json = r#"{"id":"057016a14769414aac9a7a62707598cb","url":"https://spiritanimal.com","scan_type":"Directory","status":"Complete"}"#;
+    let fs_json_two = r#"{"id":"057016a14769414aac9a7a62707598cb","url":"https://spiritanimal.com","scan_type":"Not Correct","status":"Cancelled"}"#;
+    let fs_json_three = r#"{"id":"057016a14769414aac9a7a62707598cb","url":"https://spiritanimal.com","scan_type":"Not Correct","status":"","num_requests":42}"#;
+
+    let fs: FeroxScan = serde_json::from_str(fs_json).unwrap();
+    let fs_two: FeroxScan = serde_json::from_str(fs_json_two).unwrap();
+    let fs_three: FeroxScan = serde_json::from_str(fs_json_three).unwrap();
+    assert_eq!(fs.url, "https://spiritanimal.com");
+
+    match fs.scan_type {
+        ScanType::Directory => {}
+        ScanType::File => {
+            panic!();
+        }
+    }
+    match fs_two.scan_type {
+        ScanType::Directory => {
+            panic!();
+        }
+        ScanType::File => {}
+    }
+
+    match *fs.progress_bar.lock().unwrap() {
+        None => {}
+        Some(_) => {
+            panic!();
+        }
+    }
+    assert!(matches!(*fs.status.lock().unwrap(), ScanStatus::Complete));
+    assert!(matches!(
+        *fs_two.status.lock().unwrap(),
+        ScanStatus::Cancelled
+    ));
+    assert!(matches!(
+        *fs_three.status.lock().unwrap(),
+        ScanStatus::NotStarted
+    ));
+    assert_eq!(fs_three.num_requests, 42);
+    assert_eq!(fs.id, "057016a14769414aac9a7a62707598cb");
+}
+
+#[test]
+/// given a FeroxScan, test that it serializes into the proper JSON entry
+fn ferox_scan_serialize() {
+    let fs = FeroxScan::new(
+        "https://spiritanimal.com",
+        ScanType::Directory,
+        ScanOrder::Latest,
+        0,
+        OutputLevel::Default,
+        None,
+    );
+    let fs_json = format!(
+        r#"{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","status":"NotStarted","num_requests":0}}"#,
+        fs.id
+    );
+    assert_eq!(fs_json, serde_json::to_string(&*fs).unwrap());
+}
+
+#[test]
+/// given a FeroxScans, test that it serializes into the proper JSON entry
+fn ferox_scans_serialize() {
+    let ferox_scan = FeroxScan::new(
+        "https://spiritanimal.com",
+        ScanType::Directory,
+        ScanOrder::Latest,
+        0,
+        OutputLevel::Default,
+        None,
+    );
+    let ferox_scans = FeroxScans::default();
+    let ferox_scans_json = format!(
+        r#"[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","status":"NotStarted","num_requests":0}}]"#,
+        ferox_scan.id
+    );
+    ferox_scans.scans.write().unwrap().push(ferox_scan);
+    assert_eq!(
+        ferox_scans_json,
+        serde_json::to_string(&ferox_scans).unwrap()
+    );
+}
+
+#[test]
+/// given a FeroxResponses, test that it serializes into the proper JSON entry
+fn ferox_responses_serialize() {
+    let json_response = r#"{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{"server":"nginx/1.16.1"}}"#;
+    let response: FeroxResponse = serde_json::from_str(json_response).unwrap();
+
+    let responses = FeroxResponses::default();
+    responses.insert(response);
+    // responses has a response now
+
+    // serialized should be a list of responses
+    let expected = format!("[{}]", json_response);
+
+    let serialized = serde_json::to_string(&responses).unwrap();
+    assert_eq!(expected, serialized);
+}
+
+#[test]
+/// given a FeroxResponse, test that it serializes into the proper JSON entry
+fn ferox_response_serialize_and_deserialize() {
+    // deserialize
+    let json_response = r#"{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{"server":"nginx/1.16.1"}}"#;
+    let response: FeroxResponse = serde_json::from_str(json_response).unwrap();
+
+    assert_eq!(response.url().as_str(), "https://nerdcore.com/css");
+    assert_eq!(response.url().path(), "/css");
+    assert_eq!(response.wildcard(), true);
+    assert_eq!(response.status().as_u16(), 301);
+    assert_eq!(response.content_length(), 173);
+    assert_eq!(response.line_count(), 10);
+    assert_eq!(response.word_count(), 16);
+    assert_eq!(response.headers().get("server").unwrap(), "nginx/1.16.1");
+
+    // serialize, however, this can fail when headers are out of order
+    let new_json = serde_json::to_string(&response).unwrap();
+    assert_eq!(json_response, new_json);
+}
+
+#[test]
+/// test FeroxSerialize implementation of FeroxState
+fn feroxstates_feroxserialize_implementation() {
+    let ferox_scan = FeroxScan::new(
+        "https://spiritanimal.com",
+        ScanType::Directory,
+        ScanOrder::Latest,
+        0,
+        OutputLevel::Default,
+        None,
+    );
+    let ferox_scans = FeroxScans::default();
+    let saved_id = ferox_scan.id.clone();
+    ferox_scans.insert(ferox_scan);
+
+    let config = Configuration::new().unwrap();
+    let stats = Arc::new(Stats::new(config.extensions.len(), config.json));
+
+    let json_response = r#"{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{"server":"nginx/1.16.1"}}"#;
+    let response: FeroxResponse = serde_json::from_str(json_response).unwrap();
+    RESPONSES.insert(response);
+
+    let ferox_state = FeroxState::new(
+        Arc::new(ferox_scans),
+        Arc::new(Configuration::new().unwrap()),
+        &RESPONSES,
+        stats,
+    );
+
+    let expected_strs = predicates::str::contains("scans: FeroxScans").and(
+        predicate::str::contains("config: Configuration")
+            .and(predicate::str::contains("responses: FeroxResponses"))
+            .and(predicate::str::contains("nerdcore.com"))
+            .and(predicate::str::contains("/css"))
+            .and(predicate::str::contains("https://spiritanimal.com")),
+    );
+
+    assert!(expected_strs.eval(&ferox_state.as_str()));
+
+    let json_state = ferox_state.as_json().unwrap();
+    let expected = format!(
+        r#"{{"scans":[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","status":"NotStarted","num_requests":0}}],"config":{{"type":"configuration","wordlist":"/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt","config":"","proxy":"","replay_proxy":"","target_url":"","status_codes":[200,204,301,302,307,308,401,403,405],"replay_codes":[200,204,301,302,307,308,401,403,405],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"silent":false,"quiet":false,"auto_bail":false,"auto_tune":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/{}","redirects":false,"insecure":false,"extensions":[],"headers":{{}},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":false,"depth":4,"scan_limit":0,"parallel":0,"rate_limit":0,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false,"resumed":false,"resume_from":"","save_state":false,"time_limit":"","filter_similar":[]}},"responses":[{{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{{"server":"nginx/1.16.1"}}}}]"#,
+        saved_id, VERSION
+    );
+    println!("{}\n{}", expected, json_state);
+    assert!(predicates::str::contains(expected).eval(&json_state));
+}
+
+#[should_panic]
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// call start_max_time_thread with a valid timespec, expect a panic, but only after a certain
+/// number of seconds
+async fn start_max_time_thread_panics_after_delay() {
+    let now = time::Instant::now();
+    let delay = time::Duration::new(3, 0);
+
+    let config = Configuration {
+        time_limit: String::from("3s"),
+        ..Default::default()
+    };
+
+    let handles = Arc::new(Handles::for_testing(None, Some(Arc::new(config))).0);
+
+    start_max_time_thread(handles).await;
+
+    assert!(now.elapsed() > delay);
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// call start_max_time_thread with a timespec that's too large to be parsed correctly, expect
+/// immediate return and no panic, as the sigint handler is never called
+async fn start_max_time_thread_returns_immediately_with_too_large_input() {
+    let now = time::Instant::now();
+    let delay = time::Duration::new(1, 0);
+    let config = Configuration {
+        time_limit: String::from("18446744073709551616m"),
+        ..Default::default()
+    };
+
+    let handles = Arc::new(Handles::for_testing(None, Some(Arc::new(config))).0);
+
+    // pub const MAX: usize = usize::MAX; // 18_446_744_073_709_551_615usize
+    start_max_time_thread(handles).await; // can't fit in dest u64
+
+    assert!(now.elapsed() < delay); // assuming function call will take less than 1second
+}
+
+#[test]
+/// coverage for FeroxScan's Display implementation
+fn feroxscan_display() {
+    let scan = FeroxScan {
+        id: "".to_string(),
+        url: String::from("http://localhost"),
+        scan_order: ScanOrder::Latest,
+        scan_type: Default::default(),
+        num_requests: 0,
+        start_time: Instant::now(),
+        output_level: OutputLevel::Default,
+        status_403s: Default::default(),
+        status_429s: Default::default(),
+        status: Default::default(),
+        task: tokio::sync::Mutex::new(None),
+        progress_bar: std::sync::Mutex::new(None),
+        errors: Default::default(),
+    };
+
+    let not_started = format!("{}", scan);
+
+    assert!(predicate::str::contains("not started")
+        .and(predicate::str::contains("localhost"))
+        .eval(&not_started));
+
+    scan.set_status(ScanStatus::Complete).unwrap();
+    let complete = format!("{}", scan);
+    assert!(predicate::str::contains("complete")
+        .and(predicate::str::contains("localhost"))
+        .eval(&complete));
+
+    scan.set_status(ScanStatus::Cancelled).unwrap();
+    let cancelled = format!("{}", scan);
+    assert!(predicate::str::contains("cancelled")
+        .and(predicate::str::contains("localhost"))
+        .eval(&cancelled));
+
+    scan.set_status(ScanStatus::Running).unwrap();
+    let running = format!("{}", scan);
+    assert!(predicate::str::contains("running")
+        .and(predicate::str::contains("localhost"))
+        .eval(&running));
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// call FeroxScan::abort, ensure status becomes cancelled
+async fn ferox_scan_abort() {
+    let scan = FeroxScan {
+        id: "".to_string(),
+        url: String::from("http://localhost"),
+        scan_order: ScanOrder::Latest,
+        scan_type: Default::default(),
+        num_requests: 0,
+        start_time: Instant::now(),
+        output_level: OutputLevel::Default,
+        status_403s: Default::default(),
+        status_429s: Default::default(),
+        status: std::sync::Mutex::new(ScanStatus::Running),
+        task: tokio::sync::Mutex::new(Some(tokio::spawn(async move {
+            sleep(Duration::from_millis(SLEEP_DURATION * 2));
+        }))),
+        progress_bar: std::sync::Mutex::new(None),
+        errors: Default::default(),
+    };
+
+    scan.abort().await.unwrap();
+
+    assert!(matches!(
+        *scan.status.lock().unwrap(),
+        ScanStatus::Cancelled
+    ));
+}
+
+#[test]
+/// call a few menu functions for coverage's sake
+///
+/// there's not a trivial way to test these programmatically (at least i'm too lazy rn to do it)
+/// and their correctness can be verified easily manually; just calling for now
+fn menu_print_header_and_footer() {
+    let menu = Menu::new();
+    menu.clear_screen();
+    menu.print_header();
+    menu.print_footer();
+    menu.hide_progress_bars();
+    menu.show_progress_bars();
+}
+
+#[test]
+/// ensure spaces are trimmed and numbers are returned from split_to_nums
+fn split_to_nums_is_correct() {
+    let menu = Menu::new();
+
+    let nums = menu.split_to_nums("1, 3,      4");
+
+    assert_eq!(nums, vec![1, 3, 4]);
+}
+
+#[test]
+/// given a deep url, find the correct scan
+fn get_base_scan_by_url_finds_correct_scan() {
+    let urls = FeroxScans::default();
+    let url = "http://localhost";
+    let url1 = "http://localhost/stuff";
+    let url2 = "http://shlocalhost/stuff/things";
+    let url3 = "http://shlocalhost/stuff/things/mostuff";
+    let (_, scan) = urls.add_scan(url, ScanType::Directory, ScanOrder::Latest);
+    let (_, scan1) = urls.add_scan(url1, ScanType::Directory, ScanOrder::Latest);
+    let (_, scan2) = urls.add_scan(url2, ScanType::Directory, ScanOrder::Latest);
+    let (_, scan3) = urls.add_scan(url3, ScanType::Directory, ScanOrder::Latest);
+
+    assert_eq!(
+        urls.get_base_scan_by_url("http://localhost/things.php")
+            .unwrap()
+            .id,
+        scan.id
+    );
+    assert_eq!(
+        urls.get_base_scan_by_url("http://localhost/stuff/things.php")
+            .unwrap()
+            .id,
+        scan1.id
+    );
+    assert_eq!(
+        urls.get_base_scan_by_url("http://shlocalhost/stuff/things/mostuff.php")
+            .unwrap()
+            .id,
+        scan2.id
+    );
+    assert_eq!(
+        urls.get_base_scan_by_url("http://shlocalhost/stuff/things/mostuff/mothings.php")
+            .unwrap()
+            .id,
+        scan3.id
+    );
+}
+
+#[test]
+/// given a shallow url without a trailing slash, find the correct scan
+fn get_base_scan_by_url_finds_correct_scan_without_trailing_slash() {
+    let urls = FeroxScans::default();
+    let url = "http://localhost";
+    let (_, scan) = urls.add_scan(url, ScanType::Directory, ScanOrder::Latest);
+    assert_eq!(
+        urls.get_base_scan_by_url("http://localhost/BKPMiherrortBPKcw")
+            .unwrap()
+            .id,
+        scan.id
+    );
+}
+
+#[test]
+/// given a shallow url with a trailing slash, find the correct scan
+fn get_base_scan_by_url_finds_correct_scan_with_trailing_slash() {
+    let urls = FeroxScans::default();
+    let url = "http://127.0.0.1:41971/";
+    let (_, scan) = urls.add_scan(url, ScanType::Directory, ScanOrder::Latest);
+    assert_eq!(
+        urls.get_base_scan_by_url("http://127.0.0.1:41971/BKPMiherrortBPKcw")
+            .unwrap()
+            .id,
+        scan.id
+    );
+}

src/scan_manager/utils.rs

@@ -0,0 +1,92 @@
+#[cfg(not(test))]
+use crate::event_handlers::TermInputHandler;
+use crate::{
+    config::Configuration, event_handlers::Handles, parser::TIMESPEC_REGEX, scanner::RESPONSES,
+};
+
+use std::{fs::File, io::BufReader, sync::Arc};
+use tokio::time;
+
+/// Given a string representing some number of seconds, minutes, hours, or days, convert
+/// that representation to seconds and then wait for those seconds to elapse.  Once that period
+/// of time has elapsed, kill all currently running scans and dump a state file to disk that can
+/// be used to resume any unfinished scan.
+pub async fn start_max_time_thread(handles: Arc<Handles>) {
+    log::trace!("enter: start_max_time_thread({:?})", handles);
+
+    // as this function has already made it through the parser, which calls is_match on
+    // the value passed to --time-limit using TIMESPEC_REGEX; we can safely assume that
+    // the capture groups are populated; can expect something like 10m, 30s, 1h, etc...
+    let captures = TIMESPEC_REGEX.captures(&handles.config.time_limit).unwrap();
+    let length_match = captures.get(1).unwrap();
+    let measurement_match = captures.get(2).unwrap();
+
+    if let Ok(length) = length_match.as_str().parse::<u64>() {
+        let length_in_secs = match measurement_match.as_str().to_ascii_lowercase().as_str() {
+            "s" => length,
+            "m" => length * 60,           // minutes
+            "h" => length * 60 * 60,      // hours
+            "d" => length * 60 * 60 * 24, // days
+            _ => length,
+        };
+
+        log::debug!(
+            "max time limit as string: {} and as seconds: {}",
+            handles.config.time_limit,
+            length_in_secs
+        );
+
+        time::sleep(time::Duration::new(length_in_secs, 0)).await;
+
+        log::trace!("exit: start_max_time_thread");
+
+        #[cfg(test)]
+        panic!(handles);
+        #[cfg(not(test))]
+        let _ = TermInputHandler::sigint_handler(handles.clone());
+    }
+
+    log::warn!(
+        "Could not parse the value provided ({}), can't enforce time limit",
+        handles.config.time_limit
+    );
+}
+
+/// Primary logic used to load a Configuration from disk and populate the appropriate data
+/// structures
+pub fn resume_scan(filename: &str) -> Configuration {
+    log::trace!("enter: resume_scan({})", filename);
+
+    let file = File::open(filename).unwrap_or_else(|e| {
+        log::error!("{}", e);
+        log::error!("Could not open state file, exiting");
+        std::process::exit(1);
+    });
+
+    let reader = BufReader::new(file);
+    let state: serde_json::Value = serde_json::from_reader(reader).unwrap();
+
+    let conf = state.get("config").unwrap_or_else(|| {
+        log::error!("Could not load configuration from state file, exiting");
+        std::process::exit(1);
+    });
+
+    let config = serde_json::from_value(conf.clone()).unwrap_or_else(|e| {
+        log::error!("{}", e);
+        log::error!("Could not deserialize configuration found in state file, exiting");
+        std::process::exit(1);
+    });
+
+    if let Some(responses) = state.get("responses") {
+        if let Some(arr_responses) = responses.as_array() {
+            for response in arr_responses {
+                if let Ok(deser_resp) = serde_json::from_value(response.clone()) {
+                    RESPONSES.insert(deser_resp);
+                }
+            }
+        }
+    }
+
+    log::trace!("exit: resume_scan -> {:?}", config);
+    config
+}

src/scanner.rs

@@ -1,986 +0,0 @@
-use crate::{
-    config::{Configuration, CONFIGURATION},
-    extractor::{extract_robots_txt, get_links, request_feroxresponse_from_new_link},
-    filters::{
-        FeroxFilter, LinesFilter, RegexFilter, SimilarityFilter, SizeFilter, StatusCodeFilter,
-        WildcardFilter, WordsFilter,
-    },
-    heuristics,
-    scan_manager::{FeroxResponses, FeroxScans, ScanStatus, PAUSE_SCAN},
-    statistics::{
-        StatCommand::{self, UpdateF64Field, UpdateUsizeField},
-        StatField::{DirScanTimes, ExpectedPerScan, TotalScans, WildcardsFiltered},
-        Stats,
-    },
-    utils::{format_url, get_current_depth, make_request},
-    FeroxChannel, FeroxResponse, SIMILARITY_THRESHOLD,
-};
-use futures::{
-    future::{BoxFuture, FutureExt},
-    stream, StreamExt,
-};
-use fuzzyhash::FuzzyHash;
-use lazy_static::lazy_static;
-use regex::Regex;
-use reqwest::{StatusCode, Url};
-#[cfg(not(test))]
-use std::process::exit;
-use std::{
-    collections::HashSet,
-    convert::TryInto,
-    ops::Deref,
-    sync::atomic::{AtomicUsize, Ordering},
-    sync::{Arc, RwLock},
-    time::Instant,
-};
-use tokio::{
-    sync::{
-        mpsc::{self, UnboundedReceiver, UnboundedSender},
-        Semaphore,
-    },
-    task::JoinHandle,
-};
-
-/// Single atomic number that gets incremented at least once, used to track first scan(s) vs. all
-/// others found during recursion
-///
-/// -u means this will be incremented once
-/// --stdin means this will be incremented by the number of targets passed via STDIN
-static CALL_COUNT: AtomicUsize = AtomicUsize::new(0);
-
-lazy_static! {
-    /// Set of urls that have been sent to [scan_url](fn.scan_url.html), used for deduplication
-    pub static ref SCANNED_URLS: FeroxScans = FeroxScans::default();
-
-    /// Vector of implementors of the FeroxFilter trait
-    static ref FILTERS: Arc<RwLock<Vec<Box<dyn FeroxFilter>>>> = Arc::new(RwLock::new(Vec::<Box<dyn FeroxFilter>>::new()));
-
-    /// Vector of FeroxResponse objects
-    pub static ref RESPONSES: FeroxResponses = FeroxResponses::default();
-
-    /// Bounded semaphore used as a barrier to limit concurrent scans
-    static ref SCAN_LIMITER: Semaphore = Semaphore::new(CONFIGURATION.scan_limit);
-
-
-}
-
-/// Adds the given FeroxFilter to the given list of FeroxFilter implementors
-///
-/// If the given list did not already contain the filter, return true; otherwise return false
-fn add_filter_to_list_of_ferox_filters(
-    filter: Box<dyn FeroxFilter>,
-    ferox_filters: Arc<RwLock<Vec<Box<dyn FeroxFilter>>>>,
-) -> bool {
-    log::trace!(
-        "enter: add_filter_to_list_of_ferox_filters({:?}, {:?})",
-        filter,
-        ferox_filters
-    );
-
-    match ferox_filters.write() {
-        Ok(mut filters) => {
-            // If the set did not contain the assigned filter, true is returned.
-            // If the set did contain the assigned filter, false is returned.
-            if filters.contains(&filter) {
-                log::trace!("exit: add_filter_to_list_of_ferox_filters -> false");
-                return false;
-            }
-
-            filters.push(filter);
-
-            log::trace!("exit: add_filter_to_list_of_ferox_filters -> true");
-            true
-        }
-        Err(e) => {
-            // poisoned lock
-            log::error!("Set of wildcard filters poisoned: {}", e);
-            log::trace!("exit: add_filter_to_list_of_ferox_filters -> false");
-            false
-        }
-    }
-}
-
-/// Spawn a single consumer task (sc side of mpsc)
-///
-/// The consumer simply receives Urls and scans them
-fn spawn_recursion_handler(
-    mut recursion_channel: UnboundedReceiver<String>,
-    wordlist: Arc<HashSet<String>>,
-    base_depth: usize,
-    stats: Arc<Stats>,
-    tx_term: UnboundedSender<FeroxResponse>,
-    tx_file: UnboundedSender<FeroxResponse>,
-    tx_stats: UnboundedSender<StatCommand>,
-) -> BoxFuture<'static, Vec<Arc<JoinHandle<()>>>> {
-    log::trace!(
-        "enter: spawn_recursion_handler({:?}, wordlist[{} words...], {}, {:?}, {:?}, {:?}, {:?})",
-        recursion_channel,
-        wordlist.len(),
-        base_depth,
-        stats,
-        tx_term,
-        tx_file,
-        tx_stats
-    );
-
-    let boxed_future = async move {
-        let mut scans = vec![];
-
-        while let Some(resp) = recursion_channel.recv().await {
-            let (unknown, scan) = SCANNED_URLS.add_directory_scan(&resp, stats.clone());
-
-            if !unknown {
-                // not unknown, i.e. we've seen the url before and don't need to scan again
-                continue;
-            }
-
-            update_stat!(tx_stats, UpdateUsizeField(TotalScans, 1));
-
-            log::info!("received {} on recursion channel", resp);
-
-            let term_clone = tx_term.clone();
-            let file_clone = tx_file.clone();
-            let tx_stats_clone = tx_stats.clone();
-            let stats_clone = stats.clone();
-            let resp_clone = resp.clone();
-            let list_clone = wordlist.clone();
-
-            let future = tokio::spawn(async move {
-                scan_url(
-                    resp_clone.to_owned().as_str(),
-                    list_clone,
-                    base_depth,
-                    stats_clone,
-                    term_clone,
-                    file_clone,
-                    tx_stats_clone,
-                )
-                .await
-            });
-
-            let shared_task = Arc::new(future);
-
-            if let Ok(mut u_scan) = scan.lock() {
-                u_scan.task = Some(shared_task.clone());
-            }
-
-            scans.push(shared_task);
-        }
-        scans
-    }
-    .boxed();
-
-    log::trace!("exit: spawn_recursion_handler -> BoxFuture<'static, Vec<JoinHandle<()>>>");
-    boxed_future
-}
-
-/// Creates a vector of formatted Urls
-///
-/// At least one value will be returned (base_url + word)
-///
-/// If any extensions were passed to the program, each extension will add a
-/// (base_url + word + ext) Url to the vector
-fn create_urls(
-    target_url: &str,
-    word: &str,
-    extensions: &[String],
-    tx_stats: UnboundedSender<StatCommand>,
-) -> Vec<Url> {
-    log::trace!(
-        "enter: create_urls({}, {}, {:?}, {:?})",
-        target_url,
-        word,
-        extensions,
-        tx_stats
-    );
-
-    let mut urls = vec![];
-
-    if let Ok(url) = format_url(
-        &target_url,
-        &word,
-        CONFIGURATION.add_slash,
-        &CONFIGURATION.queries,
-        None,
-        tx_stats.clone(),
-    ) {
-        urls.push(url); // default request, i.e. no extension
-    }
-
-    for ext in extensions.iter() {
-        if let Ok(url) = format_url(
-            &target_url,
-            &word,
-            CONFIGURATION.add_slash,
-            &CONFIGURATION.queries,
-            Some(ext),
-            tx_stats.clone(),
-        ) {
-            urls.push(url); // any extensions passed in
-        }
-    }
-
-    log::trace!("exit: create_urls -> {:?}", urls);
-    urls
-}
-
-/// Helper function to determine suitability for recursion
-///
-/// handles 2xx and 3xx responses by either checking if the url ends with a / (2xx)
-/// or if the Location header is present and matches the base url + / (3xx)
-fn response_is_directory(response: &FeroxResponse) -> bool {
-    log::trace!("enter: is_directory({})", response);
-
-    if response.status().is_redirection() {
-        // status code is 3xx
-        match response.headers().get("Location") {
-            // and has a Location header
-            Some(loc) => {
-                // get absolute redirect Url based on the already known base url
-                log::debug!("Location header: {:?}", loc);
-
-                if let Ok(loc_str) = loc.to_str() {
-                    if let Ok(abs_url) = response.url().join(loc_str) {
-                        if format!("{}/", response.url()) == abs_url.as_str() {
-                            // if current response's Url + / == the absolute redirection
-                            // location, we've found a directory suitable for recursion
-                            log::debug!(
-                                "found directory suitable for recursion: {}",
-                                response.url()
-                            );
-                            log::trace!("exit: is_directory -> true");
-                            return true;
-                        }
-                    }
-                }
-            }
-            None => {
-                log::debug!("expected Location header, but none was found: {}", response);
-                log::trace!("exit: is_directory -> false");
-                return false;
-            }
-        }
-    } else if response.status().is_success() || matches!(response.status(), &StatusCode::FORBIDDEN)
-    {
-        // status code is 2xx or 403, need to check if it ends in /
-
-        if response.url().as_str().ends_with('/') {
-            log::debug!("{} is directory suitable for recursion", response.url());
-            log::trace!("exit: is_directory -> true");
-            return true;
-        }
-    }
-
-    log::trace!("exit: is_directory -> false");
-    false
-}
-
-/// Helper function that determines if the configured maximum recursion depth has been reached
-///
-/// Essentially looks at the Url path and determines how many directories are present in the
-/// given Url
-fn reached_max_depth(url: &Url, base_depth: usize, max_depth: usize) -> bool {
-    log::trace!(
-        "enter: reached_max_depth({}, {}, {})",
-        url,
-        base_depth,
-        max_depth
-    );
-
-    if max_depth == 0 {
-        // early return, as 0 means recurse forever; no additional processing needed
-        log::trace!("exit: reached_max_depth -> false");
-        return false;
-    }
-
-    let depth = get_current_depth(url.as_str());
-
-    if depth - base_depth >= max_depth {
-        return true;
-    }
-
-    log::trace!("exit: reached_max_depth -> false");
-    false
-}
-
-/// Helper function that wraps logic to check for recursion opportunities
-///
-/// When a recursion opportunity is found, the new url is sent across the recursion channel
-async fn try_recursion(
-    response: &FeroxResponse,
-    base_depth: usize,
-    transmitter: UnboundedSender<String>,
-) {
-    log::trace!(
-        "enter: try_recursion({}, {}, {:?})",
-        response,
-        base_depth,
-        transmitter,
-    );
-
-    if !reached_max_depth(response.url(), base_depth, CONFIGURATION.depth)
-        && response_is_directory(&response)
-    {
-        if CONFIGURATION.redirects {
-            // response is 2xx can simply send it because we're following redirects
-            log::info!("Added new directory to recursive scan: {}", response.url());
-
-            match transmitter.send(String::from(response.url().as_str())) {
-                Ok(_) => {
-                    log::debug!("sent {} across channel to begin a new scan", response.url());
-                }
-                Err(e) => {
-                    log::error!(
-                        "Could not send {} to recursion handler: {}",
-                        response.url(),
-                        e
-                    );
-                }
-            }
-        } else {
-            let new_url = String::from(response.url().as_str());
-
-            log::info!("Added new directory to recursive scan: {}", new_url);
-
-            match transmitter.send(new_url) {
-                Ok(_) => {}
-                Err(e) => {
-                    log::error!(
-                        "Could not send {}/ to recursion handler: {}",
-                        response.url(),
-                        e
-                    );
-                }
-            }
-        }
-    }
-    log::trace!("exit: try_recursion");
-}
-
-/// Simple helper to stay DRY; determines whether or not a given `FeroxResponse` should be reported
-/// to the user or not.
-pub fn should_filter_response(
-    response: &FeroxResponse,
-    tx_stats: UnboundedSender<StatCommand>,
-) -> bool {
-    match FILTERS.read() {
-        Ok(filters) => {
-            for filter in filters.iter() {
-                // wildcard.should_filter goes here
-                if filter.should_filter_response(&response) {
-                    if filter.as_any().downcast_ref::<WildcardFilter>().is_some() {
-                        update_stat!(tx_stats, UpdateUsizeField(WildcardsFiltered, 1))
-                    }
-                    return true;
-                }
-            }
-        }
-        Err(e) => {
-            log::error!("{}", e);
-        }
-    }
-    false
-}
-
-/// Wrapper for [make_request](fn.make_request.html)
-///
-/// Handles making multiple requests based on the presence of extensions
-///
-/// Attempts recursion when appropriate and sends Responses to the report handler for processing
-async fn make_requests(
-    target_url: &str,
-    word: &str,
-    base_depth: usize,
-    stats: Arc<Stats>,
-    dir_chan: UnboundedSender<String>,
-    report_chan: UnboundedSender<FeroxResponse>,
-    tx_stats: UnboundedSender<StatCommand>,
-) {
-    log::trace!(
-        "enter: make_requests({}, {}, {}, {:?}, {:?}, {:?}, {:?})",
-        target_url,
-        word,
-        base_depth,
-        stats,
-        dir_chan,
-        report_chan,
-        tx_stats
-    );
-
-    let urls = create_urls(
-        &target_url,
-        &word,
-        &CONFIGURATION.extensions,
-        tx_stats.clone(),
-    );
-
-    for url in urls {
-        if let Ok(response) = make_request(&CONFIGURATION.client, &url, tx_stats.clone()).await {
-            // response came back without error, convert it to FeroxResponse
-            let ferox_response = FeroxResponse::from(response, true).await;
-
-            // do recursion if appropriate
-            if !CONFIGURATION.no_recursion {
-                try_recursion(&ferox_response, base_depth, dir_chan.clone()).await;
-            }
-
-            // purposefully doing recursion before filtering. the thought process is that
-            // even though this particular url is filtered, subsequent urls may not
-
-            if should_filter_response(&ferox_response, tx_stats.clone()) {
-                continue;
-            }
-
-            if CONFIGURATION.extract_links && !ferox_response.status().is_redirection() {
-                let new_links = get_links(&ferox_response, tx_stats.clone()).await;
-
-                for new_link in new_links {
-                    let mut new_ferox_response = match request_feroxresponse_from_new_link(
-                        &new_link,
-                        tx_stats.clone(),
-                    )
-                    .await
-                    {
-                        Some(resp) => resp,
-                        None => continue,
-                    };
-
-                    // filter if necessary
-                    if should_filter_response(&new_ferox_response, tx_stats.clone()) {
-                        continue;
-                    }
-
-                    if new_ferox_response.is_file() {
-                        // very likely a file, simply request and report
-                        log::debug!("Singular extraction: {}", new_ferox_response);
-
-                        SCANNED_URLS
-                            .add_file_scan(&new_ferox_response.url().to_string(), stats.clone());
-
-                        send_report(report_chan.clone(), new_ferox_response);
-
-                        continue;
-                    }
-
-                    if !CONFIGURATION.no_recursion {
-                        log::debug!("Recursive extraction: {}", new_ferox_response);
-
-                        if !new_ferox_response.url().as_str().ends_with('/')
-                            && (new_ferox_response.status().is_success()
-                                || matches!(new_ferox_response.status(), &StatusCode::FORBIDDEN))
-                        {
-                            // if the url doesn't end with a /
-                            // and the response code is either a 2xx or 403
-
-                            // since all of these are 2xx or 403, recursion is only attempted if the
-                            // url ends in a /. I am actually ok with adding the slash and not
-                            // adding it, as both have merit.  Leaving it in for now to see how
-                            // things turn out (current as of: v1.1.0)
-                            new_ferox_response.set_url(&format!("{}/", new_ferox_response.url()));
-                        }
-
-                        try_recursion(&new_ferox_response, base_depth, dir_chan.clone()).await;
-                    }
-                }
-            }
-
-            // everything else should be reported
-            send_report(report_chan.clone(), ferox_response);
-        }
-    }
-    log::trace!("exit: make_requests");
-}
-
-/// Simple helper to send a `FeroxResponse` over the tx side of an `mpsc::unbounded_channel`
-pub fn send_report(report_sender: UnboundedSender<FeroxResponse>, response: FeroxResponse) {
-    log::trace!("enter: send_report({:?}, {}", report_sender, response);
-
-    match report_sender.send(response) {
-        Ok(_) => {}
-        Err(e) => {
-            log::error!("{}", e);
-        }
-    }
-
-    log::trace!("exit: send_report");
-}
-
-/// Request /robots.txt from given url
-async fn scan_robots_txt(
-    target_url: &str,
-    base_depth: usize,
-    stats: Arc<Stats>,
-    tx_term: UnboundedSender<FeroxResponse>,
-    tx_dir: UnboundedSender<String>,
-    tx_stats: UnboundedSender<StatCommand>,
-) {
-    log::trace!(
-        "enter: scan_robots_txt({}, {}, {:?}, {:?}, {:?}, {:?})",
-        target_url,
-        base_depth,
-        stats,
-        tx_term,
-        tx_dir,
-        tx_stats
-    );
-
-    let robots_links = extract_robots_txt(&target_url, &CONFIGURATION, tx_stats.clone()).await;
-
-    for robot_link in robots_links {
-        // create a url based on the given command line options, continue on error
-        let mut ferox_response =
-            match request_feroxresponse_from_new_link(&robot_link, tx_stats.clone()).await {
-                Some(resp) => resp,
-                None => continue,
-            };
-
-        if should_filter_response(&ferox_response, tx_stats.clone()) {
-            continue;
-        }
-
-        if ferox_response.is_file() {
-            log::debug!("File extracted from robots.txt: {}", ferox_response);
-            SCANNED_URLS.add_file_scan(&robot_link, stats.clone());
-            send_report(tx_term.clone(), ferox_response);
-        } else if !CONFIGURATION.no_recursion {
-            log::debug!("Directory extracted from robots.txt: {}", ferox_response);
-            // todo this code is essentially the same as another piece around ~467 of this file
-            if !ferox_response.url().as_str().ends_with('/')
-                && (ferox_response.status().is_success()
-                    || matches!(ferox_response.status(), &StatusCode::FORBIDDEN))
-            {
-                // if the url doesn't end with a /
-                // and the response code is either a 2xx or 403
-                ferox_response.set_url(&format!("{}/", ferox_response.url()));
-            }
-
-            try_recursion(&ferox_response, base_depth, tx_dir.clone()).await;
-        }
-    }
-    log::trace!("exit: scan_robots_txt");
-}
-
-/// Scan a given url using a given wordlist
-///
-/// This is the primary entrypoint for the scanner
-pub async fn scan_url(
-    target_url: &str,
-    wordlist: Arc<HashSet<String>>,
-    base_depth: usize,
-    stats: Arc<Stats>,
-    tx_term: UnboundedSender<FeroxResponse>,
-    tx_file: UnboundedSender<FeroxResponse>,
-    tx_stats: UnboundedSender<StatCommand>,
-) {
-    log::trace!(
-        "enter: scan_url({:?}, wordlist[{} words...], {}, {:?}, {:?}, {:?}, {:?})",
-        target_url,
-        wordlist.len(),
-        base_depth,
-        stats,
-        tx_term,
-        tx_file,
-        tx_stats
-    );
-
-    log::info!("Starting scan against: {}", target_url);
-
-    let scan_timer = Instant::now();
-
-    let (tx_dir, rx_dir): FeroxChannel<String> = mpsc::unbounded_channel();
-
-    if CALL_COUNT.load(Ordering::Relaxed) < stats.initial_targets.load(Ordering::Relaxed) {
-        CALL_COUNT.fetch_add(1, Ordering::Relaxed);
-
-        if CONFIGURATION.extract_links {
-            // only grab robots.txt on the initial scan_url calls. all fresh dirs will be passed
-            // to try_recursion
-            scan_robots_txt(
-                target_url,
-                base_depth,
-                stats.clone(),
-                tx_term.clone(),
-                tx_dir.clone(),
-                tx_stats.clone(),
-            )
-            .await;
-        }
-
-        update_stat!(tx_stats, UpdateUsizeField(TotalScans, 1));
-
-        // this protection allows us to add the first scanned url to SCANNED_URLS
-        // from within the scan_url function instead of the recursion handler
-        SCANNED_URLS.add_directory_scan(&target_url, stats.clone());
-    }
-
-    let ferox_scan = match SCANNED_URLS.get_scan_by_url(&target_url) {
-        Some(scan) => {
-            if let Ok(mut u_scan) = scan.lock() {
-                u_scan.status = ScanStatus::Running;
-            }
-            scan
-        }
-        None => {
-            log::error!(
-                "Could not find FeroxScan associated with {}; this shouldn't happen... exiting",
-                target_url
-            );
-            return;
-        }
-    };
-
-    let progress_bar = match ferox_scan.lock() {
-        Ok(mut scan) => scan.progress_bar(),
-        Err(e) => {
-            log::error!("FeroxScan's ({:?}) mutex is poisoned: {}", ferox_scan, e);
-            return;
-        }
-    };
-
-    // When acquire is called and the semaphore has remaining permits, the function immediately
-    // returns a permit. However, if no remaining permits are available, acquire (asynchronously)
-    // waits until an outstanding permit is dropped. At this point, the freed permit is assigned
-    // to the caller.
-    let permit = SCAN_LIMITER.acquire().await;
-
-    // Arc clones to be passed around to the various scans
-    let wildcard_bar = progress_bar.clone();
-    let heuristics_term_clone = tx_term.clone();
-    let heuristics_stats_clone = tx_stats.clone();
-    let recurser_term_clone = tx_term.clone();
-    let recurser_file_clone = tx_file.clone();
-    let recurser_stats_clone = tx_stats.clone();
-    let recurser_words = wordlist.clone();
-    let looping_words = wordlist.clone();
-    let looping_stats = stats.clone();
-
-    let recurser = tokio::spawn(async move {
-        spawn_recursion_handler(
-            rx_dir,
-            recurser_words,
-            base_depth,
-            stats.clone(),
-            recurser_term_clone,
-            recurser_file_clone,
-            recurser_stats_clone,
-        )
-        .await
-    });
-
-    // add any wildcard filters to `FILTERS`
-    let filter = match heuristics::wildcard_test(
-        &target_url,
-        wildcard_bar,
-        heuristics_term_clone,
-        heuristics_stats_clone,
-    )
-    .await
-    {
-        Some(f) => Box::new(f),
-        None => Box::new(WildcardFilter::default()),
-    };
-
-    add_filter_to_list_of_ferox_filters(filter, FILTERS.clone());
-
-    // producer tasks (mp of mpsc); responsible for making requests
-    let producers = stream::iter(looping_words.deref().to_owned())
-        .map(|word| {
-            let txd = tx_dir.clone();
-            let txr = tx_term.clone();
-            let txs = tx_stats.clone();
-            let pb = progress_bar.clone(); // progress bar is an Arc around internal state
-            let tgt = target_url.to_string(); // done to satisfy 'static lifetime below
-            let lst = looping_stats.clone();
-            (
-                tokio::spawn(async move {
-                    if PAUSE_SCAN.load(Ordering::Acquire) {
-                        // for every word in the wordlist, check to see if PAUSE_SCAN is set to true
-                        // when true; enter a busy loop that only exits by setting PAUSE_SCAN back
-                        // to false
-                        SCANNED_URLS.pause(true).await;
-                    }
-                    make_requests(&tgt, &word, base_depth, lst, txd, txr, txs).await
-                }),
-                pb,
-            )
-        })
-        .for_each_concurrent(CONFIGURATION.threads, |(resp, bar)| async move {
-            match resp.await {
-                Ok(_) => {
-                    bar.inc((CONFIGURATION.extensions.len() + 1) as u64);
-                }
-                Err(e) => {
-                    log::error!("error awaiting a response: {}", e);
-                }
-            }
-        });
-
-    // await tx tasks
-    log::trace!("awaiting scan producers");
-    producers.await;
-    log::trace!("done awaiting scan producers");
-
-    update_stat!(
-        tx_stats,
-        UpdateF64Field(DirScanTimes, scan_timer.elapsed().as_secs_f64())
-    );
-
-    // drop the current permit so the semaphore will allow another scan to proceed
-    drop(permit);
-
-    if let Ok(mut scan) = ferox_scan.lock() {
-        scan.finish();
-    }
-
-    // manually drop tx in order for the rx task's while loops to eval to false
-    log::trace!("dropped recursion handler's transmitter");
-    drop(tx_dir);
-
-    // note: in v1.11.2 i removed the join_all call that used to handle the recurser handles.
-    // nothing appears to change by having them removed, however, if ever a revert is needed
-    // this is the place and anything prior to 1.11.2 will have the code to do so
-    let _ = recurser.await.unwrap_or_default();
-
-    log::trace!("exit: scan_url");
-}
-
-/// Perform steps necessary to run scans that only need to be performed once (warming up the
-/// engine, as it were)
-pub async fn initialize(
-    num_words: usize,
-    config: &Configuration,
-    tx_stats: UnboundedSender<StatCommand>,
-) {
-    log::trace!(
-        "enter: initialize({}, {:?}, {:?})",
-        num_words,
-        config,
-        tx_stats
-    );
-
-    // number of requests only needs to be calculated once, and then can be reused
-    let num_reqs_expected: u64 = if config.extensions.is_empty() {
-        num_words.try_into().unwrap()
-    } else {
-        let total = num_words * (config.extensions.len() + 1);
-        total.try_into().unwrap()
-    };
-
-    // tell Stats object about the number of expected requests
-    update_stat!(
-        tx_stats,
-        UpdateUsizeField(ExpectedPerScan, num_reqs_expected as usize)
-    );
-
-    // add any status code filters to `FILTERS` (-C|--filter-status)
-    for code_filter in &config.filter_status {
-        let filter = StatusCodeFilter {
-            filter_code: *code_filter,
-        };
-        let boxed_filter = Box::new(filter);
-        add_filter_to_list_of_ferox_filters(boxed_filter, FILTERS.clone());
-    }
-
-    // add any line count filters to `FILTERS` (-N|--filter-lines)
-    for lines_filter in &config.filter_line_count {
-        let filter = LinesFilter {
-            line_count: *lines_filter,
-        };
-        let boxed_filter = Box::new(filter);
-        add_filter_to_list_of_ferox_filters(boxed_filter, FILTERS.clone());
-    }
-
-    // add any line count filters to `FILTERS` (-W|--filter-words)
-    for words_filter in &config.filter_word_count {
-        let filter = WordsFilter {
-            word_count: *words_filter,
-        };
-        let boxed_filter = Box::new(filter);
-        add_filter_to_list_of_ferox_filters(boxed_filter, FILTERS.clone());
-    }
-
-    // add any line count filters to `FILTERS` (-S|--filter-size)
-    for size_filter in &config.filter_size {
-        let filter = SizeFilter {
-            content_length: *size_filter,
-        };
-        let boxed_filter = Box::new(filter);
-        add_filter_to_list_of_ferox_filters(boxed_filter, FILTERS.clone());
-    }
-
-    // add any regex filters to `FILTERS` (-X|--filter-regex)
-    for regex_filter in &config.filter_regex {
-        let raw = regex_filter;
-        let compiled = match Regex::new(&raw) {
-            Ok(regex) => regex,
-            Err(e) => {
-                log::error!("Invalid regular expression: {}", e);
-                #[cfg(test)]
-                panic!();
-                #[cfg(not(test))]
-                exit(1);
-            }
-        };
-
-        let filter = RegexFilter {
-            raw_string: raw.to_owned(),
-            compiled,
-        };
-        let boxed_filter = Box::new(filter);
-        add_filter_to_list_of_ferox_filters(boxed_filter, FILTERS.clone());
-    }
-
-    // add any similarity filters to `FILTERS` (--filter-similar-to)
-    for similarity_filter in &config.filter_similar {
-        // url as-is based on input, ignores user-specified url manipulation options (add-slash etc)
-        if let Ok(url) = format_url(
-            &similarity_filter,
-            &"",
-            false,
-            &Vec::new(),
-            None,
-            tx_stats.clone(),
-        ) {
-            // attempt to request the given url
-            if let Ok(resp) = make_request(&CONFIGURATION.client, &url, tx_stats.clone()).await {
-                // if successful, create a filter based on the response's body
-                let fr = FeroxResponse::from(resp, true).await;
-
-                // hash the response body and store the resulting hash in the filter object
-                let hash = FuzzyHash::new(&fr.text()).to_string();
-
-                let filter = SimilarityFilter {
-                    text: hash,
-                    threshold: SIMILARITY_THRESHOLD,
-                };
-
-                let boxed_filter = Box::new(filter);
-                add_filter_to_list_of_ferox_filters(boxed_filter, FILTERS.clone());
-            }
-        }
-    }
-
-    if config.scan_limit == 0 {
-        // scan_limit == 0 means no limit should be imposed... however, scoping the Semaphore
-        // permit is tricky, so as a workaround, we'll add a ridiculous number of permits to
-        // the semaphore (1,152,921,504,606,846,975 to be exact) and call that 'unlimited'
-        SCAN_LIMITER.add_permits(usize::MAX >> 4);
-    }
-
-    log::trace!("exit: initialize");
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    /// sending url + word without any extensions should get back one url with the joined word
-    fn create_urls_no_extension_returns_base_url_with_word() {
-        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
-        let urls = create_urls("http://localhost", "turbo", &[], tx);
-        assert_eq!(urls, [Url::parse("http://localhost/turbo").unwrap()])
-    }
-
-    #[test]
-    /// sending url + word + 1 extension should get back two urls, one base and one with extension
-    fn create_urls_one_extension_returns_two_urls() {
-        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
-        let urls = create_urls("http://localhost", "turbo", &[String::from("js")], tx);
-        assert_eq!(
-            urls,
-            [
-                Url::parse("http://localhost/turbo").unwrap(),
-                Url::parse("http://localhost/turbo.js").unwrap()
-            ]
-        )
-    }
-
-    #[test]
-    /// sending url + word + multiple extensions should get back n+1 urls
-    fn create_urls_multiple_extensions_returns_n_plus_one_urls() {
-        let ext_vec = vec![
-            vec![String::from("js")],
-            vec![String::from("js"), String::from("php")],
-            vec![String::from("js"), String::from("php"), String::from("pdf")],
-            vec![
-                String::from("js"),
-                String::from("php"),
-                String::from("pdf"),
-                String::from("tar.gz"),
-            ],
-        ];
-
-        let base = Url::parse("http://localhost/turbo").unwrap();
-        let js = Url::parse("http://localhost/turbo.js").unwrap();
-        let php = Url::parse("http://localhost/turbo.php").unwrap();
-        let pdf = Url::parse("http://localhost/turbo.pdf").unwrap();
-        let tar = Url::parse("http://localhost/turbo.tar.gz").unwrap();
-
-        let expected = vec![
-            vec![base.clone(), js.clone()],
-            vec![base.clone(), js.clone(), php.clone()],
-            vec![base.clone(), js.clone(), php.clone(), pdf.clone()],
-            vec![base, js, php, pdf, tar],
-        ];
-
-        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
-
-        for (i, ext_set) in ext_vec.into_iter().enumerate() {
-            let urls = create_urls("http://localhost", "turbo", &ext_set, tx.clone());
-            assert_eq!(urls, expected[i]);
-        }
-    }
-
-    #[test]
-    /// call reached_max_depth with max depth of zero, which is infinite recursion, expect false
-    fn reached_max_depth_returns_early_on_zero() {
-        let url = Url::parse("http://localhost").unwrap();
-        let result = reached_max_depth(&url, 0, 0);
-        assert!(!result);
-    }
-
-    #[test]
-    /// call reached_max_depth with url depth equal to max depth, expect true
-    fn reached_max_depth_current_depth_equals_max() {
-        let url = Url::parse("http://localhost/one/two").unwrap();
-        let result = reached_max_depth(&url, 0, 2);
-        assert!(result);
-    }
-
-    #[test]
-    /// call reached_max_depth with url dpeth less than max depth, expect false
-    fn reached_max_depth_current_depth_less_than_max() {
-        let url = Url::parse("http://localhost").unwrap();
-        let result = reached_max_depth(&url, 0, 2);
-        assert!(!result);
-    }
-
-    #[test]
-    /// call reached_max_depth with url of 2, base depth of 2, and max depth of 2, expect false
-    fn reached_max_depth_base_depth_equals_max_depth() {
-        let url = Url::parse("http://localhost/one/two").unwrap();
-        let result = reached_max_depth(&url, 2, 2);
-        assert!(!result);
-    }
-
-    #[test]
-    /// call reached_max_depth with url depth greater than max depth, expect true
-    fn reached_max_depth_current_greater_than_max() {
-        let url = Url::parse("http://localhost/one/two/three").unwrap();
-        let result = reached_max_depth(&url, 0, 2);
-        assert!(result);
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
-    #[should_panic]
-    /// call initialize with a bad regex, triggering a panic
-    async fn initialize_panics_on_bad_regex() {
-        let config = Configuration {
-            filter_regex: vec![r"(".to_string()],
-            ..Default::default()
-        };
-        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
-        initialize(1, &config, tx).await;
-    }
-}

src/scanner/ferox_scanner.rs

@@ -0,0 +1,185 @@
+use std::{collections::HashSet, ops::Deref, sync::atomic::Ordering, sync::Arc, time::Instant};
+
+use anyhow::{bail, Result};
+use futures::{stream, StreamExt};
+use lazy_static::lazy_static;
+use tokio::sync::Semaphore;
+
+use crate::{
+    event_handlers::{
+        Command::{AddError, AddToF64Field, SubtractFromUsizeField},
+        Handles,
+    },
+    extractor::{ExtractionTarget::RobotsTxt, ExtractorBuilder},
+    heuristics,
+    scan_manager::{FeroxResponses, ScanOrder, ScanStatus, PAUSE_SCAN},
+    statistics::{
+        StatError::Other,
+        StatField::{DirScanTimes, TotalExpected},
+    },
+    utils::fmt_err,
+};
+
+use super::requester::Requester;
+
+lazy_static! {
+    /// Vector of FeroxResponse objects
+    pub static ref RESPONSES: FeroxResponses = FeroxResponses::default();
+    // todo consider removing this
+}
+/// handles the main muscle movement of scanning a url
+pub struct FeroxScanner {
+    /// handles to handlers and config
+    pub(super) handles: Arc<Handles>,
+
+    /// url that will be scanned
+    pub(super) target_url: String,
+
+    /// whether or not this scanner is targeting an initial target specified by the user or one
+    /// found via recursion
+    order: ScanOrder,
+
+    /// wordlist that's already been read from disk
+    wordlist: Arc<HashSet<String>>,
+
+    /// limiter that restricts the number of active FeroxScanners  
+    scan_limiter: Arc<Semaphore>,
+}
+
+/// FeroxScanner implementation
+impl FeroxScanner {
+    /// create a new FeroxScanner
+    pub fn new(
+        target_url: &str,
+        order: ScanOrder,
+        wordlist: Arc<HashSet<String>>,
+        scan_limiter: Arc<Semaphore>,
+        handles: Arc<Handles>,
+    ) -> Self {
+        Self {
+            order,
+            handles,
+            wordlist,
+            scan_limiter,
+            target_url: target_url.to_string(),
+        }
+    }
+
+    /// Scan a given url using a given wordlist
+    ///
+    /// This is the primary entrypoint for the scanner
+    pub async fn scan_url(&self) -> Result<()> {
+        log::trace!("enter: scan_url");
+        log::info!("Starting scan against: {}", self.target_url);
+
+        let scan_timer = Instant::now();
+
+        if matches!(self.order, ScanOrder::Initial) && self.handles.config.extract_links {
+            // only grab robots.txt on the initial scan_url calls. all fresh dirs will be passed
+            // to try_recursion
+            let extractor = ExtractorBuilder::default()
+                .url(&self.target_url)
+                .handles(self.handles.clone())
+                .target(RobotsTxt)
+                .build()?;
+
+            let _ = extractor.extract().await;
+        }
+
+        let scanned_urls = self.handles.ferox_scans()?;
+
+        let ferox_scan = match scanned_urls.get_scan_by_url(&self.target_url) {
+            Some(scan) => {
+                scan.set_status(ScanStatus::Running)?;
+                scan
+            }
+            None => {
+                let msg = format!(
+                    "Could not find FeroxScan associated with {}; this shouldn't happen... exiting",
+                    self.target_url
+                );
+                bail!(fmt_err(&msg))
+            }
+        };
+
+        let progress_bar = ferox_scan.progress_bar();
+
+        // When acquire is called and the semaphore has remaining permits, the function immediately
+        // returns a permit. However, if no remaining permits are available, acquire (asynchronously)
+        // waits until an outstanding permit is dropped, at which point, the freed permit is assigned
+        // to the caller.
+        let _permit = self.scan_limiter.acquire().await;
+
+        // Arc clones to be passed around to the various scans
+        let looping_words = self.wordlist.clone();
+
+        {
+            let test = heuristics::HeuristicTests::new(self.handles.clone());
+            if let Ok(num_reqs) = test.wildcard(&self.target_url).await {
+                progress_bar.inc(num_reqs);
+            }
+        }
+
+        let requester = Arc::new(Requester::from(self, ferox_scan.clone())?);
+        let increment_len = (self.handles.config.extensions.len() + 1) as u64;
+
+        // producer tasks (mp of mpsc); responsible for making requests
+        let producers = stream::iter(looping_words.deref().to_owned())
+            .map(|word| {
+                let pb = progress_bar.clone(); // progress bar is an Arc around internal state
+                let scanned_urls_clone = scanned_urls.clone();
+                let requester_clone = requester.clone();
+                let handles_clone = self.handles.clone();
+                (
+                    tokio::spawn(async move {
+                        if PAUSE_SCAN.load(Ordering::Acquire) {
+                            // for every word in the wordlist, check to see if PAUSE_SCAN is set to true
+                            // when true; enter a busy loop that only exits by setting PAUSE_SCAN back
+                            // to false
+                            let num_cancelled = scanned_urls_clone.pause(true).await;
+                            if num_cancelled > 0 {
+                                handles_clone
+                                    .stats
+                                    .send(SubtractFromUsizeField(TotalExpected, num_cancelled))
+                                    .unwrap_or_else(|e| {
+                                        log::warn!("Could not update overall scan bar: {}", e)
+                                    });
+                            }
+                        }
+                        requester_clone
+                            .request(&word)
+                            .await
+                            .unwrap_or_else(|e| log::warn!("Requester encountered an error: {}", e))
+                    }),
+                    pb,
+                )
+            })
+            .for_each_concurrent(self.handles.config.threads, |(resp, bar)| async move {
+                match resp.await {
+                    Ok(_) => {
+                        bar.inc(increment_len);
+                    }
+                    Err(e) => {
+                        log::warn!("error awaiting a response: {}", e);
+                        self.handles.stats.send(AddError(Other)).unwrap_or_default();
+                    }
+                }
+            });
+
+        // await tx tasks
+        log::trace!("awaiting scan producers");
+        producers.await;
+        log::trace!("done awaiting scan producers");
+
+        self.handles.stats.send(AddToF64Field(
+            DirScanTimes,
+            scan_timer.elapsed().as_secs_f64(),
+        ))?;
+
+        ferox_scan.finish()?;
+
+        log::trace!("exit: scan_url");
+
+        Ok(())
+    }
+}

src/scanner/init.rs

@@ -0,0 +1,34 @@
+use crate::{
+    event_handlers::{Command::AddToUsizeField, Handles},
+    statistics::StatField::ExpectedPerScan,
+};
+use anyhow::Result;
+use std::{convert::TryInto, sync::Arc};
+
+/// Perform steps necessary to run scans that only need to be performed once (warming up the
+/// engine, as it were)
+pub async fn initialize(num_words: usize, handles: Arc<Handles>) -> Result<()> {
+    log::trace!("enter: initialize({}, {:?})", num_words, handles);
+
+    // number of requests only needs to be calculated once, and then can be reused
+    let num_reqs_expected: u64 = if handles.config.extensions.is_empty() {
+        num_words.try_into()?
+    } else {
+        let total = num_words * (handles.config.extensions.len() + 1);
+        total.try_into()?
+    };
+
+    {
+        // no real reason to keep the arc around beyond this call
+        let scans = handles.ferox_scans()?;
+        scans.set_bar_length(num_reqs_expected);
+    }
+
+    // tell Stats object about the number of expected requests
+    handles
+        .stats
+        .send(AddToUsizeField(ExpectedPerScan, num_reqs_expected as usize))?;
+
+    log::trace!("exit: initialize");
+    Ok(())
+}

src/scanner/limit_heap.rs

@@ -0,0 +1,171 @@
+use std::fmt::{Debug, Formatter, Result};
+
+/// bespoke variation on an array-backed max-heap
+///
+/// 255 possible values generated from the initial requests/second
+///
+/// when no additional errors are encountered, the left child is taken (increasing req/sec)
+/// if errors have increased since the last interval, the right child is taken (decreasing req/sec)
+///
+/// formula for each child:
+/// - left: (|parent - current|) / 2 + current
+/// - right: current - ((|parent - current|) / 2)
+pub(super) struct LimitHeap {
+    /// backing array, 255 nodes == height of 7 ( 2^(h+1) -1 nodes )
+    pub(super) inner: [i32; 255],
+
+    /// original # of requests / second
+    pub(super) original: i32,
+
+    /// current position w/in the backing array
+    pub(super) current: usize,
+}
+
+/// default implementation of a LimitHeap
+impl Default for LimitHeap {
+    /// zero-initialize the backing array
+    fn default() -> Self {
+        Self {
+            inner: [0; 255],
+            original: 0,
+            current: 0,
+        }
+    }
+}
+
+/// Debug implementation of a LimitHeap
+impl Debug for LimitHeap {
+    /// return debug representation that conforms to <32 elements in array
+    fn fmt(&self, f: &mut Formatter<'_>) -> Result {
+        let msg = format!(
+            "LimitHeap {{ original: {}, current: {}, inner: [{}...] }}",
+            self.original, self.current, self.inner[0]
+        );
+        write!(f, "{}", msg)
+    }
+}
+
+/// implementation of a LimitHeap
+impl LimitHeap {
+    /// move to right child, return node's index from which the move was requested
+    pub(super) fn move_right(&mut self) -> usize {
+        if self.has_children() {
+            let tmp = self.current;
+            self.current = self.current * 2 + 2;
+            return tmp;
+        }
+        self.current
+    }
+
+    /// move to left child, return node's index from which the move was requested
+    pub(super) fn move_left(&mut self) -> usize {
+        if self.has_children() {
+            let tmp = self.current;
+            self.current = self.current * 2 + 1;
+            return tmp;
+        }
+        self.current
+    }
+
+    /// move to parent, return node's index from which the move was requested
+    pub(super) fn move_up(&mut self) -> usize {
+        if self.has_parent() {
+            let tmp = self.current;
+            self.current = (self.current - 1) / 2;
+            return tmp;
+        }
+        self.current
+    }
+
+    /// move directly to the given index
+    pub(super) fn move_to(&mut self, index: usize) {
+        self.current = index;
+    }
+
+    /// get the current node's value
+    pub(super) fn value(&self) -> i32 {
+        self.inner[self.current]
+    }
+
+    /// set the current node's value
+    pub(super) fn set_value(&mut self, value: i32) {
+        self.inner[self.current] = value;
+    }
+
+    /// check that this node has a parent (true for all except root)
+    pub(super) fn has_parent(&self) -> bool {
+        self.current > 0
+    }
+
+    /// get node's parent's value or self.original if at the root
+    pub(super) fn parent_value(&mut self) -> i32 {
+        if self.has_parent() {
+            let current = self.move_up();
+            let val = self.value();
+            self.move_to(current);
+            return val;
+        }
+        self.original
+    }
+
+    /// check if the current node has children
+    pub(super) fn has_children(&self) -> bool {
+        // inner structure is a complete tree, just check for the right child
+        self.current * 2 + 2 <= self.inner.len()
+    }
+
+    /// get current node's right child's value
+    fn right_child_value(&mut self) -> i32 {
+        let tmp = self.move_right();
+        let val = self.value();
+        self.move_to(tmp);
+        val
+    }
+
+    /// set current node's left child's value
+    fn set_left_child(&mut self) {
+        let parent = self.parent_value();
+        let current = self.value();
+        let value = ((parent - current).abs() / 2) + current;
+
+        self.move_left();
+        self.set_value(value);
+        self.move_up();
+    }
+
+    /// set current node's right child's value
+    fn set_right_child(&mut self) {
+        let parent = self.parent_value();
+        let current = self.value();
+        let value = current - ((parent - current).abs() / 2);
+
+        self.move_right();
+        self.set_value(value);
+        self.move_up();
+    }
+
+    /// iterate over the backing array, filling in each child's value based on the original value
+    pub(super) fn build(&mut self) {
+        // ex: original is 400
+        // arr[0] == 200
+        // arr[1] (left child) == 300
+        // arr[2] (right child) == 100
+        let root = self.original / 2;
+
+        self.inner[0] = root; // set root node to half of the original value
+        self.inner[1] = ((self.original - root).abs() / 2) + root;
+        self.inner[2] = root - ((self.original - root).abs() / 2);
+
+        // start with index 1 and fill in each child below that node
+        for i in 1..self.inner.len() {
+            self.move_to(i);
+
+            if self.has_children() && self.right_child_value() == 0 {
+                // this node has an unset child since the rchild is 0
+                self.set_left_child();
+                self.set_right_child();
+            }
+        }
+        self.move_to(0); // reset current index to the root of the tree
+    }
+}

src/scanner/mod.rs

@@ -0,0 +1,12 @@
+mod ferox_scanner;
+mod utils;
+mod init;
+#[cfg(test)]
+mod tests;
+mod limit_heap;
+mod policy_data;
+mod requester;
+
+pub use self::ferox_scanner::{FeroxScanner, RESPONSES};
+pub use self::init::initialize;
+pub use self::utils::PolicyTrigger;

src/scanner/policy_data.rs

@@ -0,0 +1,309 @@
+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+
+use crate::{atomic_load, atomic_store, config::RequesterPolicy};
+
+use super::limit_heap::LimitHeap;
+
+/// data regarding policy and metadata about last enforced trigger etc...
+#[derive(Default, Debug)]
+pub struct PolicyData {
+    /// how to handle exceptional cases such as too many errors / 403s / 429s etc
+    pub(super) policy: RequesterPolicy,
+
+    /// whether or not we're in the middle of a cooldown period
+    pub(super) cooling_down: AtomicBool,
+
+    /// length of time to pause tuning after making an adjustment
+    pub(super) wait_time: u64,
+
+    /// rate limit (at last interval)
+    limit: AtomicUsize,
+
+    /// number of errors (at last interval)
+    pub(super) errors: AtomicUsize,
+
+    /// whether or not the owning Requester should remove the rate_limiter, happens when a scan
+    /// has been limited and moves back up to the point of its original scan speed
+    pub(super) remove_limit: AtomicBool,
+
+    /// heap of values used for adjusting # of requests/second
+    pub(super) heap: std::sync::RwLock<LimitHeap>,
+}
+
+/// implementation of PolicyData
+impl PolicyData {
+    /// given a RequesterPolicy, create a new PolicyData
+    pub fn new(policy: RequesterPolicy, timeout: u64) -> Self {
+        // can use this as a tweak for how aggressively adjustments should be made when tuning
+        let wait_time = ((timeout as f64 / 2.0) * 1000.0) as u64;
+
+        Self {
+            policy,
+            wait_time,
+            ..Default::default()
+        }
+    }
+
+    /// setter for requests / second; populates the underlying heap with values from req/sec seed
+    pub(super) fn set_reqs_sec(&self, reqs_sec: usize) {
+        if let Ok(mut guard) = self.heap.write() {
+            guard.original = reqs_sec as i32;
+            guard.build();
+            self.set_limit(guard.inner[0] as usize); // set limit to 1/2 of current request rate
+        }
+    }
+
+    /// setter for errors
+    pub(super) fn set_errors(&self, errors: usize) {
+        atomic_store!(self.errors, errors);
+    }
+
+    /// setter for limit
+    fn set_limit(&self, limit: usize) {
+        atomic_store!(self.limit, limit);
+    }
+
+    /// getter for limit
+    pub(super) fn get_limit(&self) -> usize {
+        atomic_load!(self.limit)
+    }
+
+    /// adjust the rate of requests per second up (increase rate)
+    pub(super) fn adjust_up(&self, streak_counter: &usize) {
+        if let Ok(mut heap) = self.heap.try_write() {
+            if *streak_counter > 2 {
+                // streak of 3 upward moves in a row, traverse the tree upward instead of to a
+                // higher-valued branch lower in the tree
+                let current = heap.value();
+                heap.move_up();
+                heap.move_up();
+                if current > heap.value() {
+                    // the tree's structure makes it so that sometimes 2 moves up results in a
+                    // value greater than the current node's and other times we need to move 3 up
+                    // to arrive at a greater value
+                    if heap.has_parent() && heap.parent_value() > current {
+                        // all nodes except 0th node (root)
+                        heap.move_up();
+                    } else if !heap.has_parent() {
+                        // been here enough that we can try resuming the scan to its original
+                        // speed (no limiting at all)
+                        atomic_store!(self.remove_limit, true);
+                    }
+                }
+                self.set_limit(heap.value() as usize);
+            } else if heap.has_children() {
+                // streak not at 3, just check that we can move down, and do so
+                heap.move_left();
+                self.set_limit(heap.value() as usize);
+            } else {
+                // tree bottomed out, need to move back up the tree a bit
+                let current = heap.value();
+                heap.move_up();
+                heap.move_up();
+
+                if current > heap.value() {
+                    heap.move_up();
+                }
+
+                self.set_limit(heap.value() as usize);
+            }
+        }
+    }
+
+    /// adjust the rate of requests per second down (decrease rate)
+    pub(super) fn adjust_down(&self) {
+        if let Ok(mut heap) = self.heap.try_write() {
+            if heap.has_children() {
+                heap.move_right();
+                self.set_limit(heap.value() as usize);
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    /// PolicyData builds and sets correct values for the inner heap when set_reqs_sec is called
+    fn set_reqs_sec_builds_heap_and_sets_initial_value() {
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        assert_eq!(pd.wait_time, 3500);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+        assert_eq!(pd.heap.read().unwrap().original, 400);
+        assert_eq!(pd.heap.read().unwrap().current, 0);
+        assert_eq!(pd.heap.read().unwrap().inner[0], 200);
+        assert_eq!(pd.heap.read().unwrap().inner[1], 300);
+        assert_eq!(pd.heap.read().unwrap().inner[2], 100);
+    }
+
+    #[test]
+    /// PolicyData setters/getters tests for code coverage / sanity
+    fn policy_data_getters_and_setters() {
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_errors(20);
+        assert_eq!(pd.errors.load(Ordering::Relaxed), 20);
+        pd.set_limit(200);
+        assert_eq!(pd.get_limit(), 200);
+    }
+
+    #[test]
+    /// PolicyData adjust_down sets the limit to the correct value
+    fn policy_data_adjust_down_simple() {
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+        pd.adjust_down();
+        assert_eq!(pd.get_limit(), 100);
+    }
+
+    #[test]
+    /// PolicyData adjust_down sets the limit to the correct value when no child nodes are present
+    fn policy_data_adjust_down_no_children() {
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+        let mut guard = pd.heap.write().unwrap();
+        guard.move_to(250);
+        guard.set_value(27);
+        pd.set_limit(guard.value() as usize);
+        drop(guard);
+
+        pd.adjust_down();
+        assert_eq!(pd.get_limit(), 27);
+    }
+
+    #[test]
+    /// PolicyData adjust_up sets the limit to the correct value
+    fn policy_data_adjust_up_simple() {
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+        pd.adjust_up(&0);
+        assert_eq!(pd.get_limit(), 300);
+    }
+
+    #[test]
+    /// PolicyData adjust_up sets the limit to the correct value
+    fn policy_data_adjust_up_with_streak_and_2_moves() {
+        // original: 400
+        // [200, 300, 100, 350, 250, 150, 50, 375, 325, 275, 225, 175, 125, 75, 25, ...]
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+
+        // 2 moves
+        pd.heap.write().unwrap().move_to(9);
+        assert_eq!(pd.heap.read().unwrap().value(), 275);
+        pd.adjust_up(&3);
+        assert_eq!(pd.heap.read().unwrap().value(), 300);
+        assert_eq!(pd.limit.load(Ordering::Relaxed), 300);
+        assert_eq!(pd.remove_limit.load(Ordering::Relaxed), false);
+    }
+
+    #[test]
+    /// PolicyData adjust_up sets the limit to the correct value
+    fn policy_data_adjust_up_with_streak_and_2_moves_to_arrive_at_root() {
+        // original: 400
+        // [200, 300, 100, 350, 250, 150, 50, 375, 325, 275, 225, 175, 125, 75, 25, ...]
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+
+        pd.heap.write().unwrap().move_to(4);
+        assert_eq!(pd.heap.read().unwrap().value(), 250);
+        pd.adjust_up(&3);
+        assert_eq!(pd.heap.read().unwrap().value(), 200);
+        assert_eq!(pd.limit.load(Ordering::Relaxed), 200);
+        assert_eq!(pd.remove_limit.load(Ordering::Relaxed), true);
+    }
+
+    #[test]
+    /// PolicyData adjust_up sets the limit to the correct value
+    fn policy_data_adjust_up_with_streak_and_2_moves_to_find_less_than_current() {
+        // original: 400
+        // [200, 300, 100, 350, 250, 150, 50, 375, 325, 275, 225, 175, 125, 75, 25, ...]
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+
+        pd.heap.write().unwrap().move_to(15);
+        assert_eq!(pd.heap.read().unwrap().value(), 387);
+        pd.adjust_up(&3);
+        assert_eq!(pd.heap.read().unwrap().value(), 350);
+        assert_eq!(pd.limit.load(Ordering::Relaxed), 350);
+        assert_eq!(pd.remove_limit.load(Ordering::Relaxed), false);
+    }
+
+    #[test]
+    /// PolicyData adjust_up sets the limit to the correct value
+    fn policy_data_adjust_up_with_streak_and_3_moves() {
+        // original: 400
+        // [200, 300, 100, 350, 250, 150, 50, 375, 325, 275, 225, 175, 125, 75, 25, ...]
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+
+        pd.heap.write().unwrap().move_to(19);
+        assert_eq!(pd.heap.read().unwrap().value(), 287);
+        pd.adjust_up(&3);
+        assert_eq!(pd.heap.read().unwrap().value(), 300);
+        assert_eq!(pd.limit.load(Ordering::Relaxed), 300);
+        assert_eq!(pd.remove_limit.load(Ordering::Relaxed), false);
+    }
+
+    #[test]
+    /// PolicyData adjust_up sets the limit to the correct value
+    fn policy_data_adjust_up_with_no_children_2_moves() {
+        // original: 400
+        // [200, 300, 100, 350, 250, 150, 50, 375, 325, 275, 225, 175, 125, 75, 25, ...]
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+
+        pd.heap.write().unwrap().move_to(241);
+
+        assert_eq!(pd.heap.read().unwrap().value(), 41);
+        pd.adjust_up(&0);
+        assert_eq!(pd.heap.read().unwrap().value(), 43);
+        assert_eq!(pd.limit.load(Ordering::Relaxed), 43);
+        assert_eq!(pd.remove_limit.load(Ordering::Relaxed), false);
+    }
+
+    #[test]
+    /// PolicyData adjust_up sets the limit to the correct value
+    fn policy_data_adjust_up_with_no_children_3_moves() {
+        // original: 400
+        // [200, 300, 100, 350, 250, 150, 50, 375, 325, 275, 225, 175, 125, 75, 25, ...]
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+
+        pd.heap.write().unwrap().move_to(240);
+
+        assert_eq!(pd.heap.read().unwrap().value(), 45);
+        pd.adjust_up(&0);
+        assert_eq!(pd.heap.read().unwrap().value(), 37);
+        assert_eq!(pd.limit.load(Ordering::Relaxed), 37);
+        assert_eq!(pd.remove_limit.load(Ordering::Relaxed), false);
+    }
+
+    #[test]
+    /// hit some of the out of the way corners of limitheap for coverage
+    fn increase_limit_heap_coverage_by_hitting_edge_cases() {
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+
+        println!("{:?}", pd.heap.read().unwrap()); // debug derivation
+
+        pd.heap.write().unwrap().move_to(240);
+        assert_eq!(pd.heap.write().unwrap().move_right(), 240);
+        assert_eq!(pd.heap.write().unwrap().move_left(), 240);
+
+        pd.heap.write().unwrap().move_to(0);
+        assert_eq!(pd.heap.write().unwrap().move_up(), 0);
+        assert_eq!(pd.heap.write().unwrap().parent_value(), 400);
+    }
+}

src/scanner/tests.rs

@@ -0,0 +1,28 @@
+use std::sync::Arc;
+
+use tokio::sync::Semaphore;
+
+use crate::{
+    config::OutputLevel,
+    event_handlers::Handles,
+    scan_manager::{FeroxScans, ScanOrder},
+};
+
+use super::*;
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+#[should_panic]
+/// try to hit struct field coverage of FileOutHandler
+async fn get_scan_by_url_bails_on_unfound_url() {
+    let sem = Semaphore::new(10);
+    let urls = FeroxScans::new(OutputLevel::Default);
+
+    let scanner = FeroxScanner::new(
+        "http://localhost",
+        ScanOrder::Initial,
+        Arc::new(Default::default()),
+        Arc::new(sem),
+        Arc::new(Handles::for_testing(Some(Arc::new(urls)), None).0),
+    );
+    scanner.scan_url().await.unwrap();
+}

src/scanner/utils.rs

@@ -0,0 +1,12 @@
+#[derive(Copy, Clone, PartialEq, Debug)]
+/// represents different situations where different criteria can trigger auto-tune/bail behavior
+pub enum PolicyTrigger {
+    /// excessive 403 trigger
+    Status403,
+
+    /// excessive 429 trigger
+    Status429,
+
+    /// excessive general errors
+    Errors,
+}

src/statistics.rs

@@ -1,833 +0,0 @@
-use crate::{
-    config::CONFIGURATION,
-    progress::{add_bar, BarType},
-    reporter::{get_cached_file_handle, safe_file_write},
-    FeroxChannel, FeroxSerialize,
-};
-use console::style;
-use indicatif::ProgressBar;
-use reqwest::StatusCode;
-use serde::{Deserialize, Serialize};
-use std::{
-    fs::File,
-    io::BufReader,
-    sync::{
-        atomic::{AtomicUsize, Ordering},
-        Arc, Mutex,
-    },
-    time::Instant,
-};
-use tokio::{
-    sync::mpsc::{self, UnboundedReceiver, UnboundedSender},
-    task::JoinHandle,
-};
-
-/// Wrapper `Atomic*.fetch_add` to save me from writing Ordering::Relaxed a bajillion times
-///
-/// default is to increment by 1, second arg can be used to increment by a different value
-macro_rules! atomic_increment {
-    ($metric:expr) => {
-        $metric.fetch_add(1, Ordering::Relaxed);
-    };
-
-    ($metric:expr, $value:expr) => {
-        $metric.fetch_add($value, Ordering::Relaxed);
-    };
-}
-
-/// Wrapper around `Atomic*.load` to save me from writing Ordering::Relaxed a bajillion times
-macro_rules! atomic_load {
-    ($metric:expr) => {
-        $metric.load(Ordering::Relaxed);
-    };
-}
-
-/// Data collection of statistics related to a scan
-#[derive(Default, Deserialize, Debug, Serialize)]
-pub struct Stats {
-    #[serde(rename = "type")]
-    /// Name of this type of struct, used for serialization, i.e. `{"type":"statistics"}`
-    kind: String,
-
-    /// tracker for number of timeouts seen by the client
-    timeouts: AtomicUsize,
-
-    /// tracker for total number of requests sent by the client
-    requests: AtomicUsize,
-
-    /// tracker for total number of requests expected to send if the scan runs to completion
-    ///
-    /// Note: this is a per-scan expectation; `expected_requests * current # of scans` would be
-    /// indicative of the current expectation at any given time, but is a moving target.  
-    pub expected_per_scan: AtomicUsize,
-
-    /// tracker for accumulating total number of requests expected (i.e. as a new scan is started
-    /// this value should increase by `expected_requests`
-    total_expected: AtomicUsize,
-
-    /// tracker for total number of errors encountered by the client
-    errors: AtomicUsize,
-
-    /// tracker for overall number of 2xx status codes seen by the client
-    successes: AtomicUsize,
-
-    /// tracker for overall number of 3xx status codes seen by the client
-    redirects: AtomicUsize,
-
-    /// tracker for overall number of 4xx status codes seen by the client
-    client_errors: AtomicUsize,
-
-    /// tracker for overall number of 5xx status codes seen by the client
-    server_errors: AtomicUsize,
-
-    /// tracker for number of scans performed, this directly equates to number of directories
-    /// recursed into and affects the total number of expected requests
-    total_scans: AtomicUsize,
-
-    /// tracker for initial number of requested targets
-    pub initial_targets: AtomicUsize,
-
-    /// tracker for number of links extracted when `--extract-links` is used; sources are
-    /// response bodies and robots.txt as of v1.11.0
-    links_extracted: AtomicUsize,
-
-    /// tracker for overall number of 200s seen by the client
-    status_200s: AtomicUsize,
-
-    /// tracker for overall number of 301s seen by the client
-    status_301s: AtomicUsize,
-
-    /// tracker for overall number of 302s seen by the client
-    status_302s: AtomicUsize,
-
-    /// tracker for overall number of 401s seen by the client
-    status_401s: AtomicUsize,
-
-    /// tracker for overall number of 403s seen by the client
-    status_403s: AtomicUsize,
-
-    /// tracker for overall number of 429s seen by the client
-    status_429s: AtomicUsize,
-
-    /// tracker for overall number of 500s seen by the client
-    status_500s: AtomicUsize,
-
-    /// tracker for overall number of 503s seen by the client
-    status_503s: AtomicUsize,
-
-    /// tracker for overall number of 504s seen by the client
-    status_504s: AtomicUsize,
-
-    /// tracker for overall number of 508s seen by the client
-    status_508s: AtomicUsize,
-
-    /// tracker for overall number of wildcard urls filtered out by the client
-    wildcards_filtered: AtomicUsize,
-
-    /// tracker for overall number of all filtered responses
-    responses_filtered: AtomicUsize,
-
-    /// tracker for number of files found
-    resources_discovered: AtomicUsize,
-
-    /// tracker for number of errors triggered during URL formatting
-    url_format_errors: AtomicUsize,
-
-    /// tracker for number of errors triggered by the `reqwest::RedirectPolicy`
-    redirection_errors: AtomicUsize,
-
-    /// tracker for number of errors related to the connecting
-    connection_errors: AtomicUsize,
-
-    /// tracker for number of errors related to the request used
-    request_errors: AtomicUsize,
-
-    /// tracker for each directory's total scan time in seconds as a float
-    directory_scan_times: Mutex<Vec<f64>>,
-
-    /// tracker for total runtime
-    total_runtime: Mutex<Vec<f64>>,
-}
-
-/// FeroxSerialize implementation for Stats
-impl FeroxSerialize for Stats {
-    /// Simply return empty string here to disable serializing this to the output file as a string
-    /// due to it looking like garbage
-    fn as_str(&self) -> String {
-        String::new()
-    }
-
-    /// Simple call to produce a JSON string using the given Stats object
-    fn as_json(&self) -> String {
-        serde_json::to_string(&self).unwrap_or_default()
-    }
-}
-
-/// implementation of statistics data collection struct
-impl Stats {
-    /// Small wrapper for default to set `kind` to "statistics" and `total_runtime` to have at least
-    /// one value
-    pub fn new() -> Self {
-        Self {
-            kind: String::from("statistics"),
-            total_runtime: Mutex::new(vec![0.0]),
-            ..Default::default()
-        }
-    }
-
-    /// increment `requests` field by one
-    fn add_request(&self) {
-        atomic_increment!(self.requests);
-    }
-
-    /// given an `Instant` update total runtime
-    fn update_runtime(&self, seconds: f64) {
-        if let Ok(mut runtime) = self.total_runtime.lock() {
-            runtime[0] = seconds;
-        }
-    }
-
-    /// save an instance of `Stats` to disk after updating the total runtime for the scan
-    fn save(&self, seconds: f64, location: &str) {
-        let buffered_file = match get_cached_file_handle(location) {
-            Some(file) => file,
-            None => {
-                return;
-            }
-        };
-
-        self.update_runtime(seconds);
-
-        safe_file_write(self, buffered_file, CONFIGURATION.json);
-    }
-
-    /// Inspect the given `StatError` and increment the appropriate fields
-    ///
-    /// Implies incrementing:
-    ///     - requests
-    ///     - errors
-    pub fn add_error(&self, error: StatError) {
-        self.add_request();
-        atomic_increment!(self.errors);
-
-        match error {
-            StatError::Timeout => {
-                atomic_increment!(self.timeouts);
-            }
-            StatError::Status403 => {
-                atomic_increment!(self.status_403s);
-                atomic_increment!(self.client_errors);
-            }
-            StatError::UrlFormat => {
-                atomic_increment!(self.url_format_errors);
-            }
-            StatError::Redirection => {
-                atomic_increment!(self.redirection_errors);
-            }
-            StatError::Connection => {
-                atomic_increment!(self.connection_errors);
-            }
-            StatError::Request => {
-                atomic_increment!(self.request_errors);
-            }
-            StatError::Other => {
-                atomic_increment!(self.errors);
-            }
-        }
-    }
-
-    /// Inspect the given `StatusCode` and increment the appropriate fields
-    ///
-    /// Implies incrementing:
-    ///     - requests
-    ///     - status_403s (when code is 403)
-    ///     - errors (when code is [45]xx)
-    fn add_status_code(&self, status: StatusCode) {
-        self.add_request();
-
-        if status.is_success() {
-            atomic_increment!(self.successes);
-        } else if status.is_redirection() {
-            atomic_increment!(self.redirects);
-        } else if status.is_client_error() {
-            atomic_increment!(self.client_errors);
-        } else if status.is_server_error() {
-            atomic_increment!(self.server_errors);
-        }
-
-        match status {
-            StatusCode::FORBIDDEN => {
-                atomic_increment!(self.status_403s);
-            }
-            StatusCode::OK => {
-                atomic_increment!(self.status_200s);
-            }
-            StatusCode::MOVED_PERMANENTLY => {
-                atomic_increment!(self.status_301s);
-            }
-            StatusCode::FOUND => {
-                atomic_increment!(self.status_302s);
-            }
-            StatusCode::UNAUTHORIZED => {
-                atomic_increment!(self.status_401s);
-            }
-            StatusCode::TOO_MANY_REQUESTS => {
-                atomic_increment!(self.status_429s);
-            }
-            StatusCode::INTERNAL_SERVER_ERROR => {
-                atomic_increment!(self.status_500s);
-            }
-            StatusCode::SERVICE_UNAVAILABLE => {
-                atomic_increment!(self.status_503s);
-            }
-            StatusCode::GATEWAY_TIMEOUT => {
-                atomic_increment!(self.status_504s);
-            }
-            StatusCode::LOOP_DETECTED => {
-                atomic_increment!(self.status_508s);
-            }
-            _ => {} // other status codes ignored for stat gathering
-        }
-    }
-
-    /// Update a `Stats` field of type f64
-    fn update_f64_field(&self, field: StatField, value: f64) {
-        if let StatField::DirScanTimes = field {
-            if let Ok(mut locked_times) = self.directory_scan_times.lock() {
-                locked_times.push(value);
-            }
-        }
-    }
-
-    /// Update a `Stats` field of type usize
-    fn update_usize_field(&self, field: StatField, value: usize) {
-        match field {
-            StatField::ExpectedPerScan => {
-                atomic_increment!(self.expected_per_scan, value);
-            }
-            StatField::TotalScans => {
-                let multiplier = CONFIGURATION.extensions.len().max(1);
-
-                atomic_increment!(self.total_scans, value);
-                atomic_increment!(
-                    self.total_expected,
-                    value * self.expected_per_scan.load(Ordering::Relaxed) * multiplier
-                );
-            }
-            StatField::TotalExpected => {
-                atomic_increment!(self.total_expected, value);
-            }
-            StatField::LinksExtracted => {
-                atomic_increment!(self.links_extracted, value);
-            }
-            StatField::WildcardsFiltered => {
-                atomic_increment!(self.wildcards_filtered, value);
-                atomic_increment!(self.responses_filtered, value);
-            }
-            StatField::ResponsesFiltered => {
-                atomic_increment!(self.responses_filtered, value);
-            }
-            StatField::ResourcesDiscovered => {
-                atomic_increment!(self.resources_discovered, value);
-            }
-            StatField::InitialTargets => {
-                atomic_increment!(self.initial_targets, value);
-            }
-            _ => {} // f64 fields
-        }
-    }
-
-    /// Merge a given `Stats` object from a json entry written to disk when handling a Ctrl+c
-    ///
-    /// This is only ever called when resuming a scan from disk
-    pub fn merge_from(&self, filename: &str) {
-        if let Ok(file) = File::open(filename) {
-            let reader = BufReader::new(file);
-            let state: serde_json::Value = serde_json::from_reader(reader).unwrap();
-
-            if let Some(state_stats) = state.get("statistics") {
-                if let Ok(d_stats) = serde_json::from_value::<Stats>(state_stats.clone()) {
-                    atomic_increment!(self.successes, atomic_load!(d_stats.successes));
-                    atomic_increment!(self.timeouts, atomic_load!(d_stats.timeouts));
-                    atomic_increment!(self.requests, atomic_load!(d_stats.requests));
-                    atomic_increment!(self.errors, atomic_load!(d_stats.errors));
-                    atomic_increment!(self.redirects, atomic_load!(d_stats.redirects));
-                    atomic_increment!(self.client_errors, atomic_load!(d_stats.client_errors));
-                    atomic_increment!(self.server_errors, atomic_load!(d_stats.server_errors));
-                    atomic_increment!(self.links_extracted, atomic_load!(d_stats.links_extracted));
-                    atomic_increment!(self.status_200s, atomic_load!(d_stats.status_200s));
-                    atomic_increment!(self.status_301s, atomic_load!(d_stats.status_301s));
-                    atomic_increment!(self.status_302s, atomic_load!(d_stats.status_302s));
-                    atomic_increment!(self.status_401s, atomic_load!(d_stats.status_401s));
-                    atomic_increment!(self.status_403s, atomic_load!(d_stats.status_403s));
-                    atomic_increment!(self.status_429s, atomic_load!(d_stats.status_429s));
-                    atomic_increment!(self.status_500s, atomic_load!(d_stats.status_500s));
-                    atomic_increment!(self.status_503s, atomic_load!(d_stats.status_503s));
-                    atomic_increment!(self.status_504s, atomic_load!(d_stats.status_504s));
-                    atomic_increment!(self.status_508s, atomic_load!(d_stats.status_508s));
-                    atomic_increment!(
-                        self.wildcards_filtered,
-                        atomic_load!(d_stats.wildcards_filtered)
-                    );
-                    atomic_increment!(
-                        self.responses_filtered,
-                        atomic_load!(d_stats.responses_filtered)
-                    );
-                    atomic_increment!(
-                        self.resources_discovered,
-                        atomic_load!(d_stats.resources_discovered)
-                    );
-                    atomic_increment!(
-                        self.url_format_errors,
-                        atomic_load!(d_stats.url_format_errors)
-                    );
-                    atomic_increment!(
-                        self.connection_errors,
-                        atomic_load!(d_stats.connection_errors)
-                    );
-                    atomic_increment!(
-                        self.redirection_errors,
-                        atomic_load!(d_stats.redirection_errors)
-                    );
-                    atomic_increment!(self.request_errors, atomic_load!(d_stats.request_errors));
-
-                    if let Ok(scan_times) = d_stats.directory_scan_times.lock() {
-                        for scan_time in scan_times.iter() {
-                            self.update_f64_field(StatField::DirScanTimes, *scan_time);
-                        }
-                    }
-                }
-            }
-        }
-    }
-}
-
-#[derive(Debug)]
-/// Enum variants used to inform the `StatCommand` protocol what `Stats` fields should be updated
-pub enum StatError {
-    /// Represents a 403 response code
-    Status403,
-
-    /// Represents a timeout error
-    Timeout,
-
-    /// Represents a URL formatting error
-    UrlFormat,
-
-    /// Represents an error encountered during redirection
-    Redirection,
-
-    /// Represents an error encountered during connection
-    Connection,
-
-    /// Represents an error resulting from the client's request
-    Request,
-
-    /// Represents any other error not explicitly defined above
-    Other,
-}
-
-/// Protocol definition for updating a Stats object via mpsc
-#[derive(Debug)]
-pub enum StatCommand {
-    /// Add one to the total number of requests
-    AddRequest,
-
-    /// Add one to the proper field(s) based on the given `StatError`
-    AddError(StatError),
-
-    /// Add one to the proper field(s) based on the given `StatusCode`
-    AddStatus(StatusCode),
-
-    /// Create the progress bar (`BarType::Total`) that is updated from the stats thread
-    CreateBar,
-
-    /// Update a `Stats` field that corresponds to the given `StatField` by the given `usize` value
-    UpdateUsizeField(StatField, usize),
-
-    /// Update a `Stats` field that corresponds to the given `StatField` by the given `f64` value
-    UpdateF64Field(StatField, f64),
-
-    /// Save a `Stats` object to disk using `reporter::get_cached_file_handle`
-    Save,
-
-    /// Load a `Stats` object from disk
-    LoadStats(String),
-
-    /// Break out of the (infinite) mpsc receive loop
-    Exit,
-}
-
-/// Enum representing fields whose updates need to be performed in batches instead of one at
-/// a time
-#[derive(Debug)]
-pub enum StatField {
-    /// Due to the necessary order of events, the number of requests expected to be sent isn't
-    /// known until after `statistics::initialize` is called. This command allows for updating
-    /// the `expected_per_scan` field after initialization
-    ExpectedPerScan,
-
-    /// Translates to `total_scans`
-    TotalScans,
-
-    /// Translates to `links_extracted`
-    LinksExtracted,
-
-    /// Translates to `total_expected`
-    TotalExpected,
-
-    /// Translates to `wildcards_filtered`
-    WildcardsFiltered,
-
-    /// Translates to `responses_filtered`
-    ResponsesFiltered,
-
-    /// Translates to `resources_discovered`
-    ResourcesDiscovered,
-
-    /// Translates to `initial_targets`
-    InitialTargets,
-
-    /// Translates to `directory_scan_times`; assumes a single append to the vector
-    DirScanTimes,
-}
-
-/// Spawn a single consumer task (sc side of mpsc)
-///
-/// The consumer simply receives `StatCommands` and updates the given `Stats` object as appropriate
-pub async fn spawn_statistics_handler(
-    mut rx_stats: UnboundedReceiver<StatCommand>,
-    stats: Arc<Stats>,
-    tx_stats: UnboundedSender<StatCommand>,
-) {
-    log::trace!(
-        "enter: spawn_statistics_handler({:?}, {:?}, {:?})",
-        rx_stats,
-        stats,
-        tx_stats
-    );
-
-    // will be updated later via StatCommand; delay is for banner to print first
-    let mut bar = ProgressBar::hidden();
-
-    let start = Instant::now();
-
-    while let Some(command) = rx_stats.recv().await {
-        match command as StatCommand {
-            StatCommand::AddError(err) => {
-                stats.add_error(err);
-                increment_bar(&bar, stats.clone());
-            }
-            StatCommand::AddStatus(status) => {
-                stats.add_status_code(status);
-                increment_bar(&bar, stats.clone());
-            }
-            StatCommand::AddRequest => {
-                stats.add_request();
-                increment_bar(&bar, stats.clone());
-            }
-            StatCommand::Save => stats.save(start.elapsed().as_secs_f64(), &CONFIGURATION.output),
-            StatCommand::UpdateUsizeField(field, value) => {
-                let update_len = matches!(field, StatField::TotalScans);
-                stats.update_usize_field(field, value);
-
-                if update_len {
-                    bar.set_length(atomic_load!(stats.total_expected) as u64)
-                }
-            }
-            StatCommand::UpdateF64Field(field, value) => stats.update_f64_field(field, value),
-            StatCommand::CreateBar => {
-                bar = add_bar(
-                    "",
-                    atomic_load!(stats.total_expected) as u64,
-                    BarType::Total,
-                );
-            }
-            StatCommand::LoadStats(filename) => {
-                stats.merge_from(&filename);
-            }
-            StatCommand::Exit => break,
-        }
-    }
-
-    bar.finish();
-
-    log::debug!("{:#?}", *stats);
-    log::trace!("exit: spawn_statistics_handler")
-}
-
-/// Wrapper around incrementing the overall scan's progress bar
-fn increment_bar(bar: &ProgressBar, stats: Arc<Stats>) {
-    let msg = format!(
-        "{}:{:<7} {}:{:<7}",
-        style("found").green(),
-        atomic_load!(stats.resources_discovered),
-        style("errors").red(),
-        atomic_load!(stats.errors),
-    );
-
-    bar.set_message(&msg);
-    bar.inc(1);
-}
-
-/// Initialize new `Stats` object and the sc side of an mpsc channel that is responsible for
-/// updates to the aforementioned object.
-pub fn initialize() -> (Arc<Stats>, UnboundedSender<StatCommand>, JoinHandle<()>) {
-    log::trace!("enter: initialize");
-
-    let stats_tracker = Arc::new(Stats::new());
-    let stats_cloned = stats_tracker.clone();
-    let (tx_stats, rx_stats): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
-    let tx_stats_cloned = tx_stats.clone();
-    let stats_thread = tokio::spawn(async move {
-        spawn_statistics_handler(rx_stats, stats_cloned, tx_stats_cloned).await
-    });
-
-    log::trace!(
-        "exit: initialize -> ({:?}, {:?}, {:?})",
-        stats_tracker,
-        tx_stats,
-        stats_thread
-    );
-
-    (stats_tracker, tx_stats, stats_thread)
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use std::fs::write;
-    use tempfile::NamedTempFile;
-
-    /// simple helper to reduce code reuse
-    fn setup_stats_test() -> (Arc<Stats>, UnboundedSender<StatCommand>, JoinHandle<()>) {
-        initialize()
-    }
-
-    /// another helper to stay DRY; must be called after any sent commands and before any checks
-    /// performed against the Stats object
-    async fn teardown_stats_test(sender: UnboundedSender<StatCommand>, handle: JoinHandle<()>) {
-        // send exit and await, once the await completes, stats should be updated
-        sender.send(StatCommand::Exit).unwrap_or_default();
-        handle.await.unwrap();
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
-    /// when sent StatCommand::Exit, function should exit its while loop (runs forever otherwise)
-    async fn statistics_handler_exits() {
-        let (_, sender, handle) = setup_stats_test();
-
-        sender.send(StatCommand::Exit).unwrap_or_default();
-
-        handle.await.unwrap(); // blocks on the handler's while loop
-
-        // if we've made it here, the test has succeeded
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
-    /// when sent StatCommand::AddRequest, stats object should reflect the change
-    async fn statistics_handler_increments_requests() {
-        let (stats, tx, handle) = setup_stats_test();
-
-        tx.send(StatCommand::AddRequest).unwrap_or_default();
-        tx.send(StatCommand::AddRequest).unwrap_or_default();
-        tx.send(StatCommand::AddRequest).unwrap_or_default();
-
-        teardown_stats_test(tx, handle).await;
-
-        assert_eq!(stats.requests.load(Ordering::Relaxed), 3);
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
-    /// when sent StatCommand::AddRequest, stats object should reflect the change
-    ///
-    /// incrementing a 403 (tracked in status_403s) should also increment:
-    ///     - errors
-    ///     - requests
-    ///     - client_errors
-    async fn statistics_handler_increments_403() {
-        let (stats, tx, handle) = setup_stats_test();
-
-        let err = StatCommand::AddError(StatError::Status403);
-        let err2 = StatCommand::AddError(StatError::Status403);
-
-        tx.send(err).unwrap_or_default();
-        tx.send(err2).unwrap_or_default();
-
-        teardown_stats_test(tx, handle).await;
-
-        assert_eq!(stats.errors.load(Ordering::Relaxed), 2);
-        assert_eq!(stats.requests.load(Ordering::Relaxed), 2);
-        assert_eq!(stats.status_403s.load(Ordering::Relaxed), 2);
-        assert_eq!(stats.client_errors.load(Ordering::Relaxed), 2);
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
-    /// when sent StatCommand::AddRequest, stats object should reflect the change
-    ///
-    /// incrementing a 403 (tracked in status_403s) should also increment:
-    ///     - requests
-    ///     - client_errors
-    async fn statistics_handler_increments_403_via_status_code() {
-        let (stats, tx, handle) = setup_stats_test();
-
-        let err = StatCommand::AddStatus(reqwest::StatusCode::FORBIDDEN);
-        let err2 = StatCommand::AddStatus(reqwest::StatusCode::FORBIDDEN);
-
-        tx.send(err).unwrap_or_default();
-        tx.send(err2).unwrap_or_default();
-
-        teardown_stats_test(tx, handle).await;
-
-        assert_eq!(stats.requests.load(Ordering::Relaxed), 2);
-        assert_eq!(stats.status_403s.load(Ordering::Relaxed), 2);
-        assert_eq!(stats.client_errors.load(Ordering::Relaxed), 2);
-    }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
-    /// when sent StatCommand::AddStatus, stats object should reflect the change
-    ///
-    /// incrementing a 500 (tracked in server_errors) should also increment:
-    ///     - requests
-    async fn statistics_handler_increments_500_via_status_code() {
-        let (stats, tx, handle) = setup_stats_test();
-
-        let err = StatCommand::AddStatus(reqwest::StatusCode::INTERNAL_SERVER_ERROR);
-        let err2 = StatCommand::AddStatus(reqwest::StatusCode::INTERNAL_SERVER_ERROR);
-
-        tx.send(err).unwrap_or_default();
-        tx.send(err2).unwrap_or_default();
-
-        teardown_stats_test(tx, handle).await;
-
-        assert_eq!(stats.requests.load(Ordering::Relaxed), 2);
-        assert_eq!(stats.server_errors.load(Ordering::Relaxed), 2);
-    }
-
-    #[test]
-    /// when Stats::add_error receives StatError::Timeout, it should increment the following:
-    ///     - timeouts
-    ///     - requests
-    ///     - errors
-    fn stats_increments_timeouts() {
-        let stats = Stats::new();
-        stats.add_error(StatError::Timeout);
-        stats.add_error(StatError::Timeout);
-        stats.add_error(StatError::Timeout);
-        stats.add_error(StatError::Timeout);
-
-        assert_eq!(stats.errors.load(Ordering::Relaxed), 4);
-        assert_eq!(stats.requests.load(Ordering::Relaxed), 4);
-        assert_eq!(stats.timeouts.load(Ordering::Relaxed), 4);
-    }
-
-    #[test]
-    /// when Stats::update_usize_field receives StatField::WildcardsFiltered, it should increment
-    /// the following:
-    ///     - responses_filtered
-    fn stats_increments_wildcards() {
-        let stats = Stats::new();
-        assert_eq!(stats.responses_filtered.load(Ordering::Relaxed), 0);
-        assert_eq!(stats.wildcards_filtered.load(Ordering::Relaxed), 0);
-
-        stats.update_usize_field(StatField::WildcardsFiltered, 1);
-        stats.update_usize_field(StatField::WildcardsFiltered, 1);
-
-        assert_eq!(stats.responses_filtered.load(Ordering::Relaxed), 2);
-        assert_eq!(stats.wildcards_filtered.load(Ordering::Relaxed), 2);
-    }
-
-    #[test]
-    /// when Stats::update_usize_field receives StatField::ResponsesFiltered, it should increment
-    fn stats_increments_responses_filtered() {
-        let stats = Stats::new();
-        assert_eq!(stats.responses_filtered.load(Ordering::Relaxed), 0);
-
-        stats.update_usize_field(StatField::ResponsesFiltered, 1);
-        stats.update_usize_field(StatField::ResponsesFiltered, 1);
-        stats.update_usize_field(StatField::ResponsesFiltered, 1);
-
-        assert_eq!(stats.responses_filtered.load(Ordering::Relaxed), 3);
-    }
-
-    #[test]
-    /// Stats::merge_from should properly incrememnt expected fields and ignore others
-    fn stats_merge_from_alters_correct_fields() {
-        let contents = r#"{"statistics":{"type":"statistics","timeouts":1,"requests":9207,"expected_per_scan":707,"total_expected":9191,"errors":3,"successes":720,"redirects":13,"client_errors":8474,"server_errors":2,"total_scans":13,"initial_targets":1,"links_extracted":51,"status_403s":3,"status_200s":720,"status_301s":12,"status_302s":1,"status_401s":4,"status_429s":2,"status_500s":5,"status_503s":9,"status_504s":6,"status_508s":7,"wildcards_filtered":707,"responses_filtered":707,"resources_discovered":27,"directory_scan_times":[2.211973078,1.989015505,1.898675839,3.9714468910000003,4.938152838,5.256073528,6.021986595,6.065740734,6.42633762,7.095142125,7.336982137,5.319785619,4.843649778],"total_runtime":[11.556575456000001],"url_format_errors":17,"redirection_errors":12,"connection_errors":21,"request_errors":4}}"#;
-        let stats = Stats::new();
-        let tfile = NamedTempFile::new().unwrap();
-        write(&tfile, contents).unwrap();
-
-        stats.merge_from(tfile.path().to_str().unwrap());
-
-        // as of 1.11.1; all Stats fields are accounted for whether they're updated in merge_from
-        // or not
-        assert_eq!(atomic_load!(stats.timeouts), 1);
-        assert_eq!(atomic_load!(stats.requests), 9207);
-        assert_eq!(atomic_load!(stats.expected_per_scan), 0); // not updated in merge_from
-        assert_eq!(atomic_load!(stats.total_expected), 0); // not updated in merge_from
-        assert_eq!(atomic_load!(stats.errors), 3);
-        assert_eq!(atomic_load!(stats.successes), 720);
-        assert_eq!(atomic_load!(stats.redirects), 13);
-        assert_eq!(atomic_load!(stats.client_errors), 8474);
-        assert_eq!(atomic_load!(stats.server_errors), 2);
-        assert_eq!(atomic_load!(stats.total_scans), 0); // not updated in merge_from
-        assert_eq!(atomic_load!(stats.initial_targets), 0); // not updated in merge_from
-        assert_eq!(atomic_load!(stats.links_extracted), 51);
-        assert_eq!(atomic_load!(stats.status_200s), 720);
-        assert_eq!(atomic_load!(stats.status_301s), 12);
-        assert_eq!(atomic_load!(stats.status_302s), 1);
-        assert_eq!(atomic_load!(stats.status_401s), 4);
-        assert_eq!(atomic_load!(stats.status_403s), 3);
-        assert_eq!(atomic_load!(stats.status_429s), 2);
-        assert_eq!(atomic_load!(stats.status_500s), 5);
-        assert_eq!(atomic_load!(stats.status_503s), 9);
-        assert_eq!(atomic_load!(stats.status_504s), 6);
-        assert_eq!(atomic_load!(stats.status_508s), 7);
-        assert_eq!(atomic_load!(stats.wildcards_filtered), 707);
-        assert_eq!(atomic_load!(stats.responses_filtered), 707);
-        assert_eq!(atomic_load!(stats.resources_discovered), 27);
-        assert_eq!(atomic_load!(stats.url_format_errors), 17);
-        assert_eq!(atomic_load!(stats.redirection_errors), 12);
-        assert_eq!(atomic_load!(stats.connection_errors), 21);
-        assert_eq!(atomic_load!(stats.request_errors), 4);
-        assert_eq!(stats.directory_scan_times.lock().unwrap().len(), 13);
-        for scan in stats.directory_scan_times.lock().unwrap().iter() {
-            assert!(scan.max(0.0) > 0.0); // all scans are non-zero
-        }
-        // total_runtime not updated in merge_from
-        assert_eq!(stats.total_runtime.lock().unwrap().len(), 1);
-        assert!((stats.total_runtime.lock().unwrap()[0] - 0.0).abs() < f64::EPSILON);
-    }
-
-    #[test]
-    /// ensure update runtime overwrites the default 0th entry
-    fn update_runtime_works() {
-        let stats = Stats::new();
-        assert!((stats.total_runtime.lock().unwrap()[0] - 0.0).abs() < f64::EPSILON);
-        stats.update_runtime(20.2);
-        assert!((stats.total_runtime.lock().unwrap()[0] - 20.2).abs() < f64::EPSILON);
-    }
-
-    #[test]
-    /// Stats::save should write contents of Stats to disk
-    fn save_writes_stats_object_to_disk() {
-        let stats = Stats::new();
-        stats.add_request();
-        stats.add_request();
-        stats.add_request();
-        stats.add_request();
-        stats.add_error(StatError::Timeout);
-        stats.add_error(StatError::Timeout);
-        stats.add_error(StatError::Timeout);
-        stats.add_error(StatError::Timeout);
-        stats.add_status_code(StatusCode::OK);
-        stats.add_status_code(StatusCode::OK);
-        stats.add_status_code(StatusCode::OK);
-        let outfile = "/tmp/stuff";
-        stats.save(174.33, outfile);
-        assert!(stats.as_json().contains("statistics"));
-        assert!(stats.as_json().contains("11")); // requests made
-        assert!(stats.as_str().is_empty());
-    }
-}

src/statistics/container.rs

@@ -0,0 +1,625 @@
+use std::{
+    fs::File,
+    io::BufReader,
+    sync::{
+        atomic::{AtomicUsize, Ordering},
+        Mutex,
+    },
+};
+
+use anyhow::{Context, Result};
+use reqwest::StatusCode;
+use serde::{Deserialize, Serialize};
+
+use crate::{
+    traits::FeroxSerialize,
+    utils::{fmt_err, open_file, write_to},
+};
+
+use super::{error::StatError, field::StatField};
+
+/// Data collection of statistics related to a scan
+#[derive(Default, Deserialize, Debug, Serialize)]
+pub struct Stats {
+    #[serde(rename = "type")]
+    /// Name of this type of struct, used for serialization, i.e. `{"type":"statistics"}`
+    kind: String,
+
+    /// tracker for number of timeouts seen by the client
+    timeouts: AtomicUsize,
+
+    /// tracker for total number of requests sent by the client
+    pub(crate) requests: AtomicUsize,
+
+    /// tracker for total number of requests expected to send if the scan runs to completion
+    ///
+    /// Note: this is a per-scan expectation; `expected_requests * current # of scans` would be
+    /// indicative of the current expectation at any given time, but is a moving target.  
+    expected_per_scan: AtomicUsize,
+
+    /// tracker for accumulating total number of requests expected (i.e. as a new scan is started
+    /// this value should increase by `expected_requests`
+    total_expected: AtomicUsize,
+
+    /// tracker for total number of errors encountered by the client
+    pub(crate) errors: AtomicUsize,
+
+    /// tracker for overall number of 2xx status codes seen by the client
+    successes: AtomicUsize,
+
+    /// tracker for overall number of 3xx status codes seen by the client
+    redirects: AtomicUsize,
+
+    /// tracker for overall number of 4xx status codes seen by the client
+    client_errors: AtomicUsize,
+
+    /// tracker for overall number of 5xx status codes seen by the client
+    server_errors: AtomicUsize,
+
+    /// tracker for number of scans performed, this directly equates to number of directories
+    /// recursed into and affects the total number of expected requests
+    pub(crate) total_scans: AtomicUsize,
+
+    /// tracker for initial number of requested targets
+    initial_targets: AtomicUsize,
+
+    /// tracker for number of links extracted when `--extract-links` is used; sources are
+    /// response bodies and robots.txt as of v1.11.0
+    links_extracted: AtomicUsize,
+
+    /// tracker for overall number of 200s seen by the client
+    status_200s: AtomicUsize,
+
+    /// tracker for overall number of 301s seen by the client
+    status_301s: AtomicUsize,
+
+    /// tracker for overall number of 302s seen by the client
+    status_302s: AtomicUsize,
+
+    /// tracker for overall number of 401s seen by the client
+    status_401s: AtomicUsize,
+
+    /// tracker for overall number of 403s seen by the client
+    pub(crate) status_403s: AtomicUsize,
+
+    /// tracker for overall number of 429s seen by the client
+    pub(crate) status_429s: AtomicUsize,
+
+    /// tracker for overall number of 500s seen by the client
+    status_500s: AtomicUsize,
+
+    /// tracker for overall number of 503s seen by the client
+    status_503s: AtomicUsize,
+
+    /// tracker for overall number of 504s seen by the client
+    status_504s: AtomicUsize,
+
+    /// tracker for overall number of 508s seen by the client
+    status_508s: AtomicUsize,
+
+    /// tracker for overall number of wildcard urls filtered out by the client
+    wildcards_filtered: AtomicUsize,
+
+    /// tracker for overall number of all filtered responses
+    responses_filtered: AtomicUsize,
+
+    /// tracker for number of files found
+    resources_discovered: AtomicUsize,
+
+    /// tracker for number of errors triggered during URL formatting
+    url_format_errors: AtomicUsize,
+
+    /// tracker for number of errors triggered by the `reqwest::RedirectPolicy`
+    redirection_errors: AtomicUsize,
+
+    /// tracker for number of errors related to the connecting
+    connection_errors: AtomicUsize,
+
+    /// tracker for number of errors related to the request used
+    request_errors: AtomicUsize,
+
+    /// tracker for each directory's total scan time in seconds as a float
+    directory_scan_times: Mutex<Vec<f64>>,
+
+    /// tracker for total runtime
+    total_runtime: Mutex<Vec<f64>>,
+
+    /// tracker for the number of extensions the user specified
+    #[serde(skip)]
+    num_extensions: usize,
+
+    /// tracker for whether to use json during serialization or not
+    #[serde(skip)]
+    json: bool,
+}
+
+/// FeroxSerialize implementation for Stats
+impl FeroxSerialize for Stats {
+    /// Simply return empty string here to disable serializing this to the output file as a string
+    /// due to it looking like garbage
+    fn as_str(&self) -> String {
+        String::new()
+    }
+
+    /// Simple call to produce a JSON string using the given Stats object
+    fn as_json(&self) -> Result<String> {
+        Ok(serde_json::to_string(&self)?)
+    }
+}
+
+/// implementation of statistics data collection struct
+impl Stats {
+    /// Small wrapper for default to set `kind` to "statistics" and `total_runtime` to have at least
+    /// one value
+    pub fn new(num_extensions: usize, is_json: bool) -> Self {
+        Self {
+            num_extensions,
+            json: is_json,
+            kind: String::from("statistics"),
+            total_runtime: Mutex::new(vec![0.0]),
+            ..Default::default()
+        }
+    }
+
+    /// public getter for expected_per_scan
+    pub fn expected_per_scan(&self) -> usize {
+        atomic_load!(self.expected_per_scan)
+    }
+
+    /// public getter for resources_discovered
+    pub fn resources_discovered(&self) -> usize {
+        atomic_load!(self.resources_discovered)
+    }
+
+    /// public getter for errors
+    pub fn errors(&self) -> usize {
+        atomic_load!(self.errors)
+    }
+
+    /// public getter for status_403s
+    pub fn status_403s(&self) -> usize {
+        atomic_load!(self.status_403s)
+    }
+
+    /// public getter for status_429s
+    pub fn status_429s(&self) -> usize {
+        atomic_load!(self.status_429s)
+    }
+
+    /// public getter for total_expected
+    pub fn total_expected(&self) -> usize {
+        atomic_load!(self.total_expected)
+    }
+
+    /// public getter for initial_targets
+    pub fn initial_targets(&self) -> usize {
+        atomic_load!(self.initial_targets)
+    }
+
+    /// increment `requests` field by one
+    pub fn add_request(&self) {
+        atomic_increment!(self.requests);
+    }
+
+    /// given an `Instant` update total runtime
+    fn update_runtime(&self, seconds: f64) {
+        if let Ok(mut runtime) = self.total_runtime.lock() {
+            runtime[0] = seconds;
+        }
+    }
+
+    /// save an instance of `Stats` to disk after updating the total runtime for the scan
+    pub fn save(&self, seconds: f64, location: &str) -> Result<()> {
+        let mut file = open_file(location)?;
+
+        self.update_runtime(seconds);
+
+        write_to(self, &mut file, self.json)?;
+
+        Ok(())
+    }
+
+    /// Inspect the given `StatError` and increment the appropriate fields
+    ///
+    /// Implies incrementing:
+    ///     - requests
+    ///     - errors
+    pub fn add_error(&self, error: StatError) {
+        self.add_request();
+        atomic_increment!(self.errors);
+
+        match error {
+            StatError::Timeout => {
+                atomic_increment!(self.timeouts);
+            }
+            StatError::UrlFormat => {
+                atomic_increment!(self.url_format_errors);
+            }
+            StatError::Redirection => {
+                atomic_increment!(self.redirection_errors);
+            }
+            StatError::Connection => {
+                atomic_increment!(self.connection_errors);
+            }
+            StatError::Request => {
+                atomic_increment!(self.request_errors);
+            }
+            _ => {} // no need to hit Other as we always increment self.errors anyway
+        }
+    }
+
+    /// Inspect the given `StatusCode` and increment the appropriate fields
+    ///
+    /// Implies incrementing:
+    ///     - requests
+    ///     - appropriate status_* codes
+    ///     - errors (when code is [45]xx)
+    pub fn add_status_code(&self, status: StatusCode) {
+        self.add_request();
+
+        if status.is_success() {
+            atomic_increment!(self.successes);
+        } else if status.is_redirection() {
+            atomic_increment!(self.redirects);
+        } else if status.is_client_error() {
+            atomic_increment!(self.client_errors);
+        } else if status.is_server_error() {
+            atomic_increment!(self.server_errors);
+        }
+
+        match status {
+            StatusCode::OK => {
+                atomic_increment!(self.status_200s);
+            }
+            StatusCode::MOVED_PERMANENTLY => {
+                atomic_increment!(self.status_301s);
+            }
+            StatusCode::FOUND => {
+                atomic_increment!(self.status_302s);
+            }
+            StatusCode::UNAUTHORIZED => {
+                atomic_increment!(self.status_401s);
+            }
+            StatusCode::FORBIDDEN => {
+                atomic_increment!(self.status_403s);
+            }
+            StatusCode::TOO_MANY_REQUESTS => {
+                atomic_increment!(self.status_429s);
+            }
+            StatusCode::INTERNAL_SERVER_ERROR => {
+                atomic_increment!(self.status_500s);
+            }
+            StatusCode::SERVICE_UNAVAILABLE => {
+                atomic_increment!(self.status_503s);
+            }
+            StatusCode::GATEWAY_TIMEOUT => {
+                atomic_increment!(self.status_504s);
+            }
+            StatusCode::LOOP_DETECTED => {
+                atomic_increment!(self.status_508s);
+            }
+            _ => {} // other status codes ignored for stat gathering
+        }
+    }
+
+    /// Update a `Stats` field of type f64
+    pub fn update_f64_field(&self, field: StatField, value: f64) {
+        if let StatField::DirScanTimes = field {
+            if let Ok(mut locked_times) = self.directory_scan_times.lock() {
+                locked_times.push(value);
+            }
+        }
+    }
+
+    /// subtract a value from the given field
+    pub fn subtract_from_usize_field(&self, field: StatField, value: usize) {
+        if let StatField::TotalExpected = field {
+            self.total_expected.fetch_sub(value, Ordering::Relaxed);
+        }
+    }
+
+    /// Update a `Stats` field of type usize
+    pub fn update_usize_field(&self, field: StatField, value: usize) {
+        match field {
+            StatField::ExpectedPerScan => {
+                atomic_increment!(self.expected_per_scan, value);
+            }
+            StatField::TotalScans => {
+                let multiplier = self.num_extensions.max(1);
+
+                atomic_increment!(self.total_scans, value);
+                atomic_increment!(
+                    self.total_expected,
+                    value * self.expected_per_scan.load(Ordering::Relaxed) * multiplier
+                );
+            }
+            StatField::TotalExpected => {
+                atomic_increment!(self.total_expected, value);
+            }
+            StatField::LinksExtracted => {
+                atomic_increment!(self.links_extracted, value);
+            }
+            StatField::WildcardsFiltered => {
+                atomic_increment!(self.wildcards_filtered, value);
+                atomic_increment!(self.responses_filtered, value);
+            }
+            StatField::ResponsesFiltered => {
+                atomic_increment!(self.responses_filtered, value);
+            }
+            StatField::ResourcesDiscovered => {
+                atomic_increment!(self.resources_discovered, value);
+            }
+            StatField::InitialTargets => {
+                atomic_increment!(self.initial_targets, value);
+            }
+            _ => {} // f64 fields
+        }
+    }
+
+    /// Merge a given `Stats` object from a json entry written to disk when handling a Ctrl+c
+    ///
+    /// This is only ever called when resuming a scan from disk
+    pub fn merge_from(&self, filename: &str) -> Result<()> {
+        let file = File::open(filename)
+            .with_context(|| fmt_err(&format!("Could not open {}", filename)))?;
+        let reader = BufReader::new(file);
+        let state: serde_json::Value = serde_json::from_reader(reader)?;
+
+        if let Some(state_stats) = state.get("statistics") {
+            let d_stats = serde_json::from_value::<Stats>(state_stats.clone())?;
+            atomic_increment!(self.successes, atomic_load!(d_stats.successes));
+            atomic_increment!(self.timeouts, atomic_load!(d_stats.timeouts));
+            atomic_increment!(self.requests, atomic_load!(d_stats.requests));
+            atomic_increment!(self.errors, atomic_load!(d_stats.errors));
+            atomic_increment!(self.redirects, atomic_load!(d_stats.redirects));
+            atomic_increment!(self.client_errors, atomic_load!(d_stats.client_errors));
+            atomic_increment!(self.server_errors, atomic_load!(d_stats.server_errors));
+            atomic_increment!(self.links_extracted, atomic_load!(d_stats.links_extracted));
+            atomic_increment!(self.status_200s, atomic_load!(d_stats.status_200s));
+            atomic_increment!(self.status_301s, atomic_load!(d_stats.status_301s));
+            atomic_increment!(self.status_302s, atomic_load!(d_stats.status_302s));
+            atomic_increment!(self.status_401s, atomic_load!(d_stats.status_401s));
+            atomic_increment!(self.status_403s, atomic_load!(d_stats.status_403s));
+            atomic_increment!(self.status_429s, atomic_load!(d_stats.status_429s));
+            atomic_increment!(self.status_500s, atomic_load!(d_stats.status_500s));
+            atomic_increment!(self.status_503s, atomic_load!(d_stats.status_503s));
+            atomic_increment!(self.status_504s, atomic_load!(d_stats.status_504s));
+            atomic_increment!(self.status_508s, atomic_load!(d_stats.status_508s));
+            atomic_increment!(
+                self.wildcards_filtered,
+                atomic_load!(d_stats.wildcards_filtered)
+            );
+            atomic_increment!(
+                self.responses_filtered,
+                atomic_load!(d_stats.responses_filtered)
+            );
+            atomic_increment!(
+                self.resources_discovered,
+                atomic_load!(d_stats.resources_discovered)
+            );
+            atomic_increment!(
+                self.url_format_errors,
+                atomic_load!(d_stats.url_format_errors)
+            );
+            atomic_increment!(
+                self.connection_errors,
+                atomic_load!(d_stats.connection_errors)
+            );
+            atomic_increment!(
+                self.redirection_errors,
+                atomic_load!(d_stats.redirection_errors)
+            );
+            atomic_increment!(self.request_errors, atomic_load!(d_stats.request_errors));
+
+            if let Ok(scan_times) = d_stats.directory_scan_times.lock() {
+                for scan_time in scan_times.iter() {
+                    self.update_f64_field(StatField::DirScanTimes, *scan_time);
+                }
+            };
+        }
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::{config::Configuration, Command};
+    use std::fs::write;
+    use tempfile::NamedTempFile;
+
+    use super::super::*;
+    use super::*;
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// when sent StatCommand::AddRequest, stats object should reflect the change
+    async fn statistics_handler_increments_requests() -> Result<()> {
+        let (task, handle) = setup_stats_test();
+
+        handle.tx.send(Command::AddRequest)?;
+        handle.tx.send(Command::AddRequest)?;
+        handle.tx.send(Command::AddRequest)?;
+
+        teardown_stats_test(handle.tx.clone(), task).await;
+
+        assert_eq!(handle.data.requests.load(Ordering::Relaxed), 3);
+
+        Ok(())
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// when sent StatCommand::AddRequest, stats object should reflect the change
+    ///
+    /// incrementing a 403 (tracked in status_403s) should also increment:
+    ///     - requests
+    ///     - client_errors
+    async fn statistics_handler_increments_403_via_status_code() {
+        let (task, handle) = setup_stats_test();
+
+        let err = Command::AddStatus(reqwest::StatusCode::FORBIDDEN);
+        let err2 = Command::AddStatus(reqwest::StatusCode::FORBIDDEN);
+
+        handle.tx.send(err).unwrap_or_default();
+        handle.tx.send(err2).unwrap_or_default();
+
+        teardown_stats_test(handle.tx.clone(), task).await;
+
+        assert_eq!(handle.data.requests.load(Ordering::Relaxed), 2);
+        assert_eq!(handle.data.status_403s.load(Ordering::Relaxed), 2);
+        assert_eq!(handle.data.client_errors.load(Ordering::Relaxed), 2);
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// when sent StatCommand::AddStatus, stats object should reflect the change
+    ///
+    /// incrementing a 500 (tracked in server_errors) should also increment:
+    ///     - requests
+    async fn statistics_handler_increments_500_via_status_code() -> Result<()> {
+        let (task, handle) = setup_stats_test();
+
+        let err = Command::AddStatus(reqwest::StatusCode::INTERNAL_SERVER_ERROR);
+        let err2 = Command::AddStatus(reqwest::StatusCode::INTERNAL_SERVER_ERROR);
+
+        handle.tx.send(err)?;
+        handle.tx.send(err2)?;
+
+        teardown_stats_test(handle.tx.clone(), task).await;
+
+        assert_eq!(handle.data.requests.load(Ordering::Relaxed), 2);
+        assert_eq!(handle.data.server_errors.load(Ordering::Relaxed), 2);
+
+        Ok(())
+    }
+
+    #[test]
+    /// when Stats::add_error receives StatError::Timeout, it should increment the following:
+    ///     - timeouts
+    ///     - requests
+    ///     - errors
+    fn stats_increments_timeouts() {
+        let config = Configuration::new().unwrap();
+        let stats = Stats::new(config.extensions.len(), config.json);
+
+        stats.add_error(StatError::Timeout);
+        stats.add_error(StatError::Timeout);
+        stats.add_error(StatError::Timeout);
+        stats.add_error(StatError::Timeout);
+
+        assert_eq!(stats.errors.load(Ordering::Relaxed), 4);
+        assert_eq!(stats.requests.load(Ordering::Relaxed), 4);
+        assert_eq!(stats.timeouts.load(Ordering::Relaxed), 4);
+    }
+
+    #[test]
+    /// when Stats::update_usize_field receives StatField::WildcardsFiltered, it should increment
+    /// the following:
+    ///     - responses_filtered
+    fn stats_increments_wildcards() {
+        let config = Configuration::new().unwrap();
+        let stats = Stats::new(config.extensions.len(), config.json);
+
+        assert_eq!(stats.responses_filtered.load(Ordering::Relaxed), 0);
+        assert_eq!(stats.wildcards_filtered.load(Ordering::Relaxed), 0);
+
+        stats.update_usize_field(StatField::WildcardsFiltered, 1);
+        stats.update_usize_field(StatField::WildcardsFiltered, 1);
+
+        assert_eq!(stats.responses_filtered.load(Ordering::Relaxed), 2);
+        assert_eq!(stats.wildcards_filtered.load(Ordering::Relaxed), 2);
+    }
+
+    #[test]
+    /// when Stats::update_usize_field receives StatField::ResponsesFiltered, it should increment
+    fn stats_increments_responses_filtered() {
+        let config = Configuration::new().unwrap();
+        let stats = Stats::new(config.extensions.len(), config.json);
+
+        assert_eq!(stats.responses_filtered.load(Ordering::Relaxed), 0);
+
+        stats.update_usize_field(StatField::ResponsesFiltered, 1);
+        stats.update_usize_field(StatField::ResponsesFiltered, 1);
+        stats.update_usize_field(StatField::ResponsesFiltered, 1);
+
+        assert_eq!(stats.responses_filtered.load(Ordering::Relaxed), 3);
+    }
+
+    #[test]
+    /// Stats::merge_from should properly increment expected fields and ignore others
+    fn stats_merge_from_alters_correct_fields() {
+        let contents = r#"{"statistics":{"type":"statistics","timeouts":1,"requests":9207,"expected_per_scan":707,"total_expected":9191,"errors":3,"successes":720,"redirects":13,"client_errors":8474,"server_errors":2,"total_scans":13,"initial_targets":1,"links_extracted":51,"status_403s":3,"status_200s":720,"status_301s":12,"status_302s":1,"status_401s":4,"status_429s":2,"status_500s":5,"status_503s":9,"status_504s":6,"status_508s":7,"wildcards_filtered":707,"responses_filtered":707,"resources_discovered":27,"directory_scan_times":[2.211973078,1.989015505,1.898675839,3.9714468910000003,4.938152838,5.256073528,6.021986595,6.065740734,6.42633762,7.095142125,7.336982137,5.319785619,4.843649778],"total_runtime":[11.556575456000001],"url_format_errors":17,"redirection_errors":12,"connection_errors":21,"request_errors":4}}"#;
+        let config = Configuration::new().unwrap();
+        let stats = Stats::new(config.extensions.len(), config.json);
+
+        let tfile = NamedTempFile::new().unwrap();
+        write(&tfile, contents).unwrap();
+
+        stats.merge_from(tfile.path().to_str().unwrap()).unwrap();
+
+        // as of 2.1.0; all Stats fields are accounted for whether they're updated in merge_from
+        // or not
+        assert_eq!(atomic_load!(stats.timeouts), 1);
+        assert_eq!(atomic_load!(stats.requests), 9207);
+        assert_eq!(atomic_load!(stats.expected_per_scan), 0); // not updated in merge_from
+        assert_eq!(atomic_load!(stats.total_expected), 0); // not updated in merge_from
+        assert_eq!(atomic_load!(stats.errors), 3);
+        assert_eq!(atomic_load!(stats.successes), 720);
+        assert_eq!(atomic_load!(stats.redirects), 13);
+        assert_eq!(atomic_load!(stats.client_errors), 8474);
+        assert_eq!(atomic_load!(stats.server_errors), 2);
+        assert_eq!(atomic_load!(stats.total_scans), 0); // not updated in merge_from
+        assert_eq!(atomic_load!(stats.initial_targets), 0); // not updated in merge_from
+        assert_eq!(atomic_load!(stats.links_extracted), 51);
+        assert_eq!(atomic_load!(stats.status_200s), 720);
+        assert_eq!(atomic_load!(stats.status_301s), 12);
+        assert_eq!(atomic_load!(stats.status_302s), 1);
+        assert_eq!(atomic_load!(stats.status_401s), 4);
+        assert_eq!(atomic_load!(stats.status_403s), 3);
+        assert_eq!(atomic_load!(stats.status_429s), 2);
+        assert_eq!(atomic_load!(stats.status_500s), 5);
+        assert_eq!(atomic_load!(stats.status_503s), 9);
+        assert_eq!(atomic_load!(stats.status_504s), 6);
+        assert_eq!(atomic_load!(stats.status_508s), 7);
+        assert_eq!(atomic_load!(stats.wildcards_filtered), 707);
+        assert_eq!(atomic_load!(stats.responses_filtered), 707);
+        assert_eq!(atomic_load!(stats.resources_discovered), 27);
+        assert_eq!(atomic_load!(stats.url_format_errors), 17);
+        assert_eq!(atomic_load!(stats.redirection_errors), 12);
+        assert_eq!(atomic_load!(stats.connection_errors), 21);
+        assert_eq!(atomic_load!(stats.request_errors), 4);
+        assert_eq!(stats.directory_scan_times.lock().unwrap().len(), 13);
+        for scan in stats.directory_scan_times.lock().unwrap().iter() {
+            assert!(scan.max(0.0) > 0.0); // all scans are non-zero
+        }
+        // total_runtime not updated in merge_from
+        assert_eq!(stats.total_runtime.lock().unwrap().len(), 1);
+        assert!((stats.total_runtime.lock().unwrap()[0] - 0.0).abs() < f64::EPSILON);
+    }
+
+    #[test]
+    /// ensure update runtime overwrites the default 0th entry
+    fn update_runtime_works() {
+        let config = Configuration::new().unwrap();
+        let stats = Stats::new(config.extensions.len(), config.json);
+
+        assert!((stats.total_runtime.lock().unwrap()[0] - 0.0).abs() < f64::EPSILON);
+        stats.update_runtime(20.2);
+        assert!((stats.total_runtime.lock().unwrap()[0] - 20.2).abs() < f64::EPSILON);
+    }
+
+    #[test]
+    /// ensure status_403s returns the correct value
+    fn status_403s_returns_correct_value() {
+        let config = Configuration::new().unwrap();
+        let stats = Stats::new(config.extensions.len(), config.json);
+        stats.status_403s.store(12, Ordering::Relaxed);
+        assert_eq!(stats.status_403s(), 12);
+    }
+
+    #[test]
+    /// ensure status_403s returns the correct value
+    fn status_429s_returns_correct_value() {
+        let config = Configuration::new().unwrap();
+        let stats = Stats::new(config.extensions.len(), config.json);
+        stats.status_429s.store(141, Ordering::Relaxed);
+        assert_eq!(stats.status_429s(), 141);
+    }
+}

src/statistics/error.rs

@@ -0,0 +1,21 @@
+#[derive(Debug, Copy, Clone)]
+/// Enum variants used to inform the `StatCommand` protocol what `Stats` fields should be updated
+pub enum StatError {
+    /// Represents a timeout error
+    Timeout,
+
+    /// Represents a URL formatting error
+    UrlFormat,
+
+    /// Represents an error encountered during redirection
+    Redirection,
+
+    /// Represents an error encountered during connection
+    Connection,
+
+    /// Represents an error resulting from the client's request
+    Request,
+
+    /// Represents any other error not explicitly defined above
+    Other,
+}

src/statistics/field.rs

@@ -0,0 +1,33 @@
+/// Enum representing fields whose updates need to be performed in batches instead of one at
+/// a time
+#[derive(Debug, Copy, Clone)]
+pub enum StatField {
+    /// Due to the necessary order of events, the number of requests expected to be sent isn't
+    /// known until after `statistics::initialize` is called. This command allows for updating
+    /// the `expected_per_scan` field after initialization
+    ExpectedPerScan,
+
+    /// Translates to `total_scans`
+    TotalScans,
+
+    /// Translates to `links_extracted`
+    LinksExtracted,
+
+    /// Translates to `total_expected`
+    TotalExpected,
+
+    /// Translates to `wildcards_filtered`
+    WildcardsFiltered,
+
+    /// Translates to `responses_filtered`
+    ResponsesFiltered,
+
+    /// Translates to `resources_discovered`
+    ResourcesDiscovered,
+
+    /// Translates to `initial_targets`
+    InitialTargets,
+
+    /// Translates to `directory_scan_times`; assumes a single append to the vector
+    DirScanTimes,
+}

src/statistics/init.rs

@@ -0,0 +1 @@
+

src/statistics/macros.rs

@@ -0,0 +1,37 @@
+#![macro_use]
+
+/// Wrapper `Atomic*.fetch_add` to save me from writing Ordering::Relaxed a bajillion times
+///
+/// default is to increment by 1, second arg can be used to increment by a different value
+#[macro_export]
+macro_rules! atomic_increment {
+    ($metric:expr) => {
+        $metric.fetch_add(1, Ordering::Relaxed);
+    };
+
+    ($metric:expr, $value:expr) => {
+        $metric.fetch_add($value, Ordering::Relaxed);
+    };
+}
+
+/// Wrapper around `Atomic*.load` to save me from writing Ordering::Relaxed a bajillion times
+#[macro_export]
+macro_rules! atomic_load {
+    ($metric:expr) => {
+        $metric.load(Ordering::Relaxed);
+    };
+    ($metric:expr, $ordering:expr) => {
+        $metric.load($ordering);
+    };
+}
+
+/// Wrapper around `Atomic*.store` to save me from writing Ordering::Relaxed a bajillion times
+#[macro_export]
+macro_rules! atomic_store {
+    ($metric:expr, $value:expr) => {
+        $metric.store($value, Ordering::Relaxed);
+    };
+    ($metric:expr, $value:expr, $ordering:expr) => {
+        $metric.store($value, $ordering);
+    };
+}

src/statistics/mod.rs

@@ -0,0 +1,13 @@
+mod error;
+mod macros;
+mod container;
+mod field;
+#[cfg(test)]
+mod tests;
+
+pub use self::container::Stats;
+pub use self::error::StatError;
+pub use self::field::StatField;
+
+#[cfg(test)]
+use self::tests::{setup_stats_test, teardown_stats_test};

src/statistics/tests.rs

@@ -0,0 +1,66 @@
+use super::*;
+use crate::{
+    config::Configuration,
+    event_handlers::{Command, StatsHandle, StatsHandler},
+    traits::FeroxSerialize,
+    CommandSender, Joiner,
+};
+use anyhow::Result;
+use reqwest::StatusCode;
+use std::sync::Arc;
+use tempfile::NamedTempFile;
+
+/// simple helper to reduce code reuse
+pub fn setup_stats_test() -> (Joiner, StatsHandle) {
+    let config = Arc::new(Configuration::new().unwrap());
+    StatsHandler::initialize(config)
+}
+
+/// another helper to stay DRY; must be called after any sent commands and before any checks
+/// performed against the Stats object
+pub async fn teardown_stats_test(sender: CommandSender, task: Joiner) {
+    // send exit and await, once the await completes, stats should be updated
+    sender.send(Command::Exit).unwrap_or_default();
+    task.await.unwrap().unwrap();
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// when sent StatCommand::Exit, function should exit its while loop (runs forever otherwise)
+async fn statistics_handler_exits() -> Result<()> {
+    let (task, handle) = setup_stats_test();
+
+    handle.tx.send(Command::Exit)?;
+
+    task.await??; // blocks on the handler's while loop
+
+    // if we've made it here, the test has succeeded
+    Ok(())
+}
+
+#[test]
+/// Stats::save should write contents of Stats to disk
+fn save_writes_stats_object_to_disk() {
+    let config = Configuration::new().unwrap();
+    let stats = Stats::new(config.extensions.len(), config.json);
+
+    stats.add_request();
+    stats.add_request();
+    stats.add_request();
+    stats.add_request();
+    stats.add_error(StatError::Timeout);
+    stats.add_error(StatError::Timeout);
+    stats.add_error(StatError::Timeout);
+    stats.add_error(StatError::Timeout);
+    stats.add_status_code(StatusCode::OK);
+    stats.add_status_code(StatusCode::OK);
+    stats.add_status_code(StatusCode::OK);
+    let outfile = NamedTempFile::new().unwrap();
+    if stats
+        .save(174.33, &outfile.path().to_str().unwrap())
+        .is_ok()
+    {}
+
+    assert!(stats.as_json().unwrap().contains("statistics"));
+    assert!(stats.as_json().unwrap().contains("11")); // requests made
+    assert!(stats.as_str().is_empty());
+}

src/traits.rs

@@ -1,4 +1,9 @@
-use super::*;
+//! collection of all traits used
+use crate::response::FeroxResponse;
+use anyhow::Result;
+use serde::Serialize;
+use std::any::Any;
+use std::fmt::Debug;
 
 // references:
 //   https://dev.to/magnusstrale/rust-trait-objects-in-a-vector-non-trivial-4co5
@@ -25,3 +30,14 @@ impl PartialEq for Box<dyn FeroxFilter> {
         self.box_eq(other.as_any())
     }
 }
+
+/// FeroxSerialize trait; represents different types that are Serialize and also implement
+/// as_str / as_json methods
+pub trait FeroxSerialize: Serialize {
+    /// Return a String representation of the object, generally the human readable version of the
+    /// implementor
+    fn as_str(&self) -> String;
+
+    /// Return an NDJSON representation of the object
+    fn as_json(&self) -> Result<String>;
+}

src/url.rs

@@ -0,0 +1,463 @@
+use crate::{event_handlers::Handles, statistics::StatError::UrlFormat, Command::AddError};
+use anyhow::{anyhow, bail, Result};
+use reqwest::Url;
+use std::{convert::TryInto, fmt, sync::Arc};
+
+/// abstraction around target urls; collects all Url related shenanigans in one place
+#[derive(Debug)]
+pub struct FeroxUrl {
+    /// string representation of the target url
+    target: String,
+
+    /// Handles object for grabbing config values
+    handles: Arc<Handles>,
+}
+
+/// implementation of FeroxUrl
+impl FeroxUrl {
+    /// Create new FeroxUrl given a target url as a string
+    pub fn from_string(target: &str, handles: Arc<Handles>) -> Self {
+        Self {
+            handles,
+            target: String::from(target),
+        }
+    }
+
+    /// Create new FeroxUrl given a target url as a reqwest::Url
+    pub fn from_url(target: &Url, handles: Arc<Handles>) -> Self {
+        Self {
+            handles,
+            target: target.as_str().to_string(),
+        }
+    }
+
+    /// Creates a vector of formatted Urls
+    ///
+    /// At least one value will be returned (base_url + word)
+    ///
+    /// If any extensions were passed to the program, each extension will add a
+    /// (base_url + word + ext) Url to the vector
+    pub fn formatted_urls(&self, word: &str) -> Result<Vec<Url>> {
+        log::trace!("enter: formatted_urls({})", word);
+
+        let mut urls = vec![];
+
+        match self.format(word, None) {
+            // default request, i.e. no extension
+            Ok(url) => urls.push(url),
+            Err(_) => self.handles.stats.send(AddError(UrlFormat))?,
+        }
+
+        for ext in self.handles.config.extensions.iter() {
+            match self.format(word, Some(ext)) {
+                // any extensions passed in
+                Ok(url) => urls.push(url),
+                Err(_) => self.handles.stats.send(AddError(UrlFormat))?,
+            }
+        }
+
+        log::trace!("exit: formatted_urls -> {:?}", urls);
+        Ok(urls)
+    }
+
+    /// Simple helper to generate a `Url`
+    ///
+    /// Errors during parsing `url` or joining `word` are propagated up the call stack
+    pub fn format(&self, word: &str, extension: Option<&str>) -> Result<Url> {
+        log::trace!("enter: format({}, {:?})", word, extension);
+
+        if Url::parse(&word).is_ok() {
+            // when a full url is passed in as a word to be joined to a base url using
+            // reqwest::Url::join, the result is that the word (url) completely overwrites the base
+            // url, potentially resulting in requests to places that aren't actually the target
+            // specified.
+            //
+            // in order to resolve the issue, we check if the word from the wordlist is a parsable URL
+            // and if so, don't do any further processing
+            let message = format!("word ({}) from wordlist is a URL, skipping...", word);
+            log::warn!("{}", message);
+            log::trace!("exit: format -> Err({})", message);
+            bail!(message);
+        }
+
+        // from reqwest::Url::join
+        //   Note: a trailing slash is significant. Without it, the last path component
+        //   is considered to be a “file” name to be removed to get at the “directory”
+        //   that is used as the base
+        //
+        // the transforms that occur here will need to keep this in mind, i.e. add a slash to preserve
+        // the current directory sent as part of the url
+        let url = if word.is_empty() {
+            // v1.0.6: added during --extract-links feature implementation to support creating urls
+            // that were extracted from response bodies, i.e. http://localhost/some/path/js/main.js
+            self.target.to_string()
+        } else if !self.target.ends_with('/') {
+            format!("{}/", self.target)
+        } else {
+            self.target.to_string()
+        };
+
+        // extensions and slashes are mutually exclusive cases
+        let word = if extension.is_some() {
+            format!("{}.{}", word, extension.unwrap())
+        } else if self.handles.config.add_slash && !word.ends_with('/') {
+            // -f used, and word doesn't already end with a /
+            format!("{}/", word)
+        } else if word.starts_with("//") {
+            // bug ID'd by @Sicks3c, when a wordlist contains words that begin with 2 forward slashes
+            // i.e. //1_40_0/static/js, it gets joined onto the base url in a surprising way
+            // ex: https://localhost/ + //1_40_0/static/js -> https://1_40_0/static/js
+            // this is due to the fact that //... is a valid url. The fix is introduced here in 1.12.2
+            // and simply removes prefixed forward slashes if there are two of them. Additionally,
+            // trim_start_matches will trim the pattern until it's gone, so even if there are more than
+            // 2 /'s, they'll still be trimmed
+            word.trim_start_matches('/').to_string()
+        } else {
+            String::from(word)
+        };
+
+        let base_url = Url::parse(&url)?;
+        let joined = base_url.join(&word)?;
+
+        if self.handles.config.queries.is_empty() {
+            // no query params to process
+            log::trace!("exit: format -> {}", joined);
+            Ok(joined)
+        } else {
+            let with_params =
+                Url::parse_with_params(joined.as_str(), &self.handles.config.queries)?;
+            log::trace!("exit: format_url -> {}", with_params);
+            Ok(with_params) // request with params attached
+        }
+    }
+
+    /// Gets the length of a url's path
+    pub fn path_length(&self) -> Result<u64> {
+        let parsed = Url::parse(&self.target)?;
+        Ok(FeroxUrl::path_length_of_url(&parsed))
+    }
+
+    /// Gets the length of a url's path
+    ///
+    /// example: http://localhost/stuff -> 5
+    pub fn path_length_of_url(url: &Url) -> u64 {
+        log::trace!("enter: get_path_length({})", url);
+
+        let path = url.path();
+
+        let segments = if let Some(split) = path.strip_prefix('/') {
+            split.split_terminator('/')
+        } else {
+            log::trace!("exit: get_path_length -> 0");
+            return 0;
+        };
+
+        if let Some(last) = segments.last() {
+            // failure on conversion should be very unlikely. While a usize can absolutely overflow a
+            // u64, the generally accepted maximum for the length of a url is ~2000.  so the value we're
+            // putting into the u64 should never realistically be anywhere close to producing an
+            // overflow.
+            // usize max: 18,446,744,073,709,551,615
+            // u64 max:   9,223,372,036,854,775,807
+            let url_len: u64 = last
+                .len()
+                .try_into()
+                .expect("Failed usize -> u64 conversion");
+
+            log::trace!("exit: get_path_length -> {}", url_len);
+            return url_len;
+        }
+
+        log::trace!("exit: get_path_length -> 0");
+        0
+    }
+
+    /// Simple helper to abstract away adding a forward-slash to a url if not present
+    ///
+    /// used mostly for deduplication purposes and url state tracking
+    pub fn normalize(&self) -> String {
+        log::trace!("enter: normalize");
+
+        let normalized = if self.target.ends_with('/') {
+            self.target.to_string()
+        } else {
+            format!("{}/", self.target)
+        };
+
+        log::trace!("exit: normalize -> {}", normalized);
+        normalized
+    }
+
+    /// Helper function that determines the current depth of a given url
+    ///
+    /// Essentially looks at the Url path and determines how many directories are present in the
+    /// given Url
+    ///
+    /// http://localhost -> 1
+    /// http://localhost/ -> 1
+    /// http://localhost/stuff -> 2
+    /// ...
+    ///
+    /// returns 0 on error and relative urls
+    pub fn depth(&self) -> Result<usize> {
+        log::trace!("enter: get_depth");
+
+        let target = self.normalize();
+
+        let parsed = Url::parse(&target)?;
+        let parts = parsed
+            .path_segments()
+            .ok_or_else(|| anyhow!("No path segments found"))?;
+
+        // at least an empty string returned by the Split, meaning top-level urls
+        let mut depth = 0;
+
+        for _ in parts {
+            depth += 1;
+        }
+
+        log::trace!("exit: get_depth -> {}", depth);
+        Ok(depth)
+    }
+}
+
+/// Display implementation for a FeroxUrl
+impl fmt::Display for FeroxUrl {
+    /// formatter for FeroxUrl
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}", &self.target)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::config::Configuration;
+
+    #[test]
+    /// sending url + word without any extensions should get back one url with the joined word
+    fn formatted_urls_no_extension_returns_base_url_with_word() {
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+        let url = FeroxUrl::from_string("http://localhost", handles);
+        let urls = url.formatted_urls("turbo").unwrap();
+        assert_eq!(urls, [Url::parse("http://localhost/turbo").unwrap()])
+    }
+
+    #[test]
+    /// sending url + word + 1 extension should get back two urls, one base and one with extension
+    fn formatted_urls_one_extension_returns_two_urls() {
+        let config = Configuration {
+            extensions: vec![String::from("js")],
+            ..Default::default()
+        };
+
+        let handles = Arc::new(Handles::for_testing(None, Some(Arc::new(config))).0);
+        let url = FeroxUrl::from_string("http://localhost", handles);
+        let urls = url.formatted_urls("turbo").unwrap();
+
+        assert_eq!(
+            urls,
+            [
+                Url::parse("http://localhost/turbo").unwrap(),
+                Url::parse("http://localhost/turbo.js").unwrap()
+            ]
+        )
+    }
+
+    #[test]
+    /// sending url + word + multiple extensions should get back n+1 urls
+    fn formatted_urls_multiple_extensions_returns_n_plus_one_urls() {
+        let ext_vec = vec![
+            vec![String::from("js")],
+            vec![String::from("js"), String::from("php")],
+            vec![String::from("js"), String::from("php"), String::from("pdf")],
+            vec![
+                String::from("js"),
+                String::from("php"),
+                String::from("pdf"),
+                String::from("tar.gz"),
+            ],
+        ];
+        let base = Url::parse("http://localhost/turbo").unwrap();
+        let js = Url::parse("http://localhost/turbo.js").unwrap();
+        let php = Url::parse("http://localhost/turbo.php").unwrap();
+        let pdf = Url::parse("http://localhost/turbo.pdf").unwrap();
+        let tar = Url::parse("http://localhost/turbo.tar.gz").unwrap();
+
+        let expected = vec![
+            vec![base.clone(), js.clone()],
+            vec![base.clone(), js.clone(), php.clone()],
+            vec![base.clone(), js.clone(), php.clone(), pdf.clone()],
+            vec![base, js, php, pdf, tar],
+        ];
+
+        for (i, ext_set) in ext_vec.into_iter().enumerate() {
+            let config = Configuration {
+                extensions: ext_set,
+                ..Default::default()
+            };
+
+            let handles = Arc::new(Handles::for_testing(None, Some(Arc::new(config))).0);
+            let url = FeroxUrl::from_string("http://localhost", handles);
+
+            let urls = url.formatted_urls("turbo").unwrap();
+            assert_eq!(urls, expected[i]);
+        }
+    }
+
+    #[test]
+    /// base url returns 1
+    fn depth_base_url_returns_1() {
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+        let url = FeroxUrl::from_string("http://localhost", handles);
+
+        let depth = url.depth().unwrap();
+        assert_eq!(depth, 1);
+    }
+
+    #[test]
+    /// base url with slash returns 1
+    fn depth_base_url_with_slash_returns_1() {
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+        let url = FeroxUrl::from_string("http://localhost/", handles);
+
+        let depth = url.depth().unwrap();
+        assert_eq!(depth, 1);
+    }
+
+    #[test]
+    /// base url + 1 dir returns 2
+    fn depth_one_dir_returns_2() {
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+        let url = FeroxUrl::from_string("http://localhost/src", handles);
+
+        let depth = url.depth().unwrap();
+        assert_eq!(depth, 2);
+    }
+
+    #[test]
+    /// base url + 1 dir and slash returns 2
+    fn depth_one_dir_with_slash_returns_2() {
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+        let url = FeroxUrl::from_string("http://localhost/src/", handles);
+
+        let depth = url.depth().unwrap();
+        assert_eq!(depth, 2);
+    }
+
+    #[test]
+    /// base url + 1 word + no slash + no extension
+    fn format_url_normal() {
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+        let url = FeroxUrl::from_string("http://localhost", handles);
+        let formatted = url.format("stuff", None).unwrap();
+
+        assert_eq!(
+            formatted,
+            reqwest::Url::parse("http://localhost/stuff").unwrap()
+        );
+    }
+
+    #[test]
+    /// base url + no word + no slash + no extension
+    fn format_url_no_word() {
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+        let url = FeroxUrl::from_string("http://localhost", handles);
+        let formatted = url.format("", None).unwrap();
+        assert_eq!(formatted, reqwest::Url::parse("http://localhost").unwrap());
+    }
+
+    #[test]
+    /// base url + word + no slash + no extension + queries
+    fn format_url_joins_queries() {
+        let config = Configuration {
+            queries: vec![(String::from("stuff"), String::from("things"))],
+            ..Default::default()
+        };
+
+        let handles = Arc::new(Handles::for_testing(None, Some(Arc::new(config))).0);
+        let url = FeroxUrl::from_string("http://localhost", handles);
+        let formatted = url.format("lazer", None).unwrap();
+
+        assert_eq!(
+            formatted,
+            reqwest::Url::parse("http://localhost/lazer?stuff=things").unwrap()
+        );
+    }
+
+    #[test]
+    /// base url + no word + no slash + no extension + queries
+    fn format_url_without_word_joins_queries() {
+        let config = Configuration {
+            queries: vec![(String::from("stuff"), String::from("things"))],
+            ..Default::default()
+        };
+
+        let handles = Arc::new(Handles::for_testing(None, Some(Arc::new(config))).0);
+        let url = FeroxUrl::from_string("http://localhost", handles);
+        let formatted = url.format("", None).unwrap();
+
+        assert_eq!(
+            formatted,
+            reqwest::Url::parse("http://localhost/?stuff=things").unwrap()
+        );
+    }
+
+    #[test]
+    #[should_panic]
+    /// no base url is an error
+    fn format_url_no_url() {
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+        let url = FeroxUrl::from_string("", handles);
+        url.format("stuff", None).unwrap();
+    }
+
+    #[test]
+    /// word prepended with slash is adjusted for correctness
+    fn format_url_word_with_preslash() {
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+        let url = FeroxUrl::from_string("http://localhost", handles);
+        let formatted = url.format("/stuff", None).unwrap();
+
+        assert_eq!(
+            formatted,
+            reqwest::Url::parse("http://localhost/stuff").unwrap()
+        );
+    }
+
+    #[test]
+    /// word with appended slash allows the slash to persist
+    fn format_url_word_with_postslash() {
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+        let url = FeroxUrl::from_string("http://localhost", handles);
+        let formatted = url.format("stuff/", None).unwrap();
+
+        assert_eq!(
+            formatted,
+            reqwest::Url::parse("http://localhost/stuff/").unwrap()
+        );
+    }
+
+    #[test]
+    /// word with two prepended slashes doesn't discard the entire domain
+    fn format_url_word_with_two_prepended_slashes() {
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+        let url = FeroxUrl::from_string("http://localhost", handles);
+        let formatted = url.format("//upload/img", None).unwrap();
+
+        assert_eq!(
+            formatted,
+            reqwest::Url::parse("http://localhost/upload/img").unwrap()
+        );
+    }
+
+    #[test]
+    /// word that is a fully formed url, should return an error
+    fn format_url_word_that_is_a_url() {
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+        let url = FeroxUrl::from_string("http://localhost", handles);
+        let formatted = url.format("http://schmocalhost", None);
+
+        assert!(formatted.is_err());
+    }
+}

src/utils.rs

@@ -1,94 +1,43 @@
-#![macro_use]
-use crate::{
-    config::{CONFIGURATION, PROGRESS_PRINTER},
-    statistics::{
-        StatCommand::{self, AddError, AddStatus},
-        StatError::{Connection, Other, Redirection, Request, Timeout, UrlFormat},
-    },
-    FeroxError, FeroxResult,
-};
+use anyhow::{bail, Context, Result};
 use console::{strip_ansi_codes, style, user_attended};
 use indicatif::ProgressBar;
-use reqwest::{Client, Response, Url};
+use reqwest::{Client, Response, StatusCode, Url};
 #[cfg(not(target_os = "windows"))]
 use rlimit::{getrlimit, setrlimit, Resource, Rlim};
-use std::convert::TryInto;
-use std::sync::{Arc, RwLock};
-use std::{fs, io};
+use std::{
+    fs,
+    io::{self, BufWriter, Write},
+    sync::Arc,
+};
 use tokio::sync::mpsc::UnboundedSender;
 
+use crate::{
+    config::OutputLevel,
+    event_handlers::{
+        Command::{self, AddError, AddStatus},
+        Handles,
+    },
+    progress::PROGRESS_PRINTER,
+    send_command,
+    statistics::StatError::{Connection, Other, Redirection, Request, Timeout},
+    traits::FeroxSerialize,
+};
+
 /// Given the path to a file, open the file in append mode (create it if it doesn't exist) and
-/// return a reference to the file that is buffered and locked
-pub fn open_file(filename: &str) -> Option<Arc<RwLock<io::BufWriter<fs::File>>>> {
+/// return a reference to the buffered file
+pub fn open_file(filename: &str) -> Result<BufWriter<fs::File>> {
     log::trace!("enter: open_file({})", filename);
 
-    match fs::OpenOptions::new() // std fs
+    let file = fs::OpenOptions::new() // std fs
         .create(true)
         .append(true)
         .open(filename)
-    {
-        Ok(file) => {
-            let writer = io::BufWriter::new(file); // std io
+        .with_context(|| fmt_err(&format!("Could not open {}", filename)))?;
 
-            let locked_file = Some(Arc::new(RwLock::new(writer)));
+    let writer = BufWriter::new(file); // std io
 
-            log::trace!("exit: open_file -> {:?}", locked_file);
-            locked_file
-        }
-        Err(e) => {
-            log::error!("{}", e);
-            log::trace!("exit: open_file -> None");
-            None
-        }
-    }
-}
-
-/// Helper function that determines the current depth of a given url
-///
-/// Essentially looks at the Url path and determines how many directories are present in the
-/// given Url
-///
-/// http://localhost -> 1
-/// http://localhost/ -> 1
-/// http://localhost/stuff -> 2
-/// ...
-///
-/// returns 0 on error and relative urls
-pub fn get_current_depth(target: &str) -> usize {
-    log::trace!("enter: get_current_depth({})", target);
-
-    let target = normalize_url(target);
-
-    match Url::parse(&target) {
-        Ok(url) => {
-            if let Some(parts) = url.path_segments() {
-                // at least an empty string returned by the Split, meaning top-level urls
-                let mut depth = 0;
-
-                for _ in parts {
-                    depth += 1;
-                }
-
-                let return_val = depth;
-
-                log::trace!("exit: get_current_depth -> {}", return_val);
-                return return_val;
-            };
-
-            log::debug!(
-                "get_current_depth called on a Url that cannot be a base: {}",
-                url
-            );
-            log::trace!("exit: get_current_depth -> 0");
-
-            0
-        }
-        Err(e) => {
-            log::error!("could not parse to url: {}", e);
-            log::trace!("exit: get_current_depth -> 0");
-            0
-        }
-    }
+    log::trace!("exit: open_file -> {:?}", writer);
+    Ok(writer)
 }
 
 /// Takes in a string and examines the first character to return a color version of the same string
@@ -105,6 +54,11 @@ pub fn status_colorizer(status: &str) -> String {
     }
 }
 
+/// simple wrapper to stay DRY
+pub fn fmt_err(msg: &str) -> String {
+    format!("{}: {}", status_colorizer("ERROR"), msg)
+}
+
 /// Takes in a string and colors it using console::style
 ///
 /// mainly putting this here in case i want to change the color later, making any changes easy
@@ -112,41 +66,6 @@ pub fn module_colorizer(modname: &str) -> String {
     style(modname).cyan().to_string()
 }
 
-/// Gets the length of a url's path
-///
-/// example: http://localhost/stuff -> 5
-pub fn get_url_path_length(url: &Url) -> u64 {
-    log::trace!("enter: get_url_path_length({})", url);
-
-    let path = url.path();
-
-    let segments = if let Some(split) = path.strip_prefix('/') {
-        split.split_terminator('/')
-    } else {
-        log::trace!("exit: get_url_path_length -> 0");
-        return 0;
-    };
-
-    if let Some(last) = segments.last() {
-        // failure on conversion should be very unlikely. While a usize can absolutely overflow a
-        // u64, the generally accepted maximum for the length of a url is ~2000.  so the value we're
-        // putting into the u64 should never realistically be anywhere close to producing an
-        // overflow.
-        // usize max: 18,446,744,073,709,551,615
-        // u64 max:   9,223,372,036,854,775,807
-        let url_len: u64 = last
-            .len()
-            .try_into()
-            .expect("Failed usize -> u64 conversion");
-
-        log::trace!("exit: get_url_path_length -> {}", url_len);
-        return url_len;
-    }
-
-    log::trace!("exit: get_url_path_length -> 0");
-    0
-}
-
 /// Simple helper to abstract away the check for an attached terminal.
 ///
 /// If a terminal is attached, progress bars are visible and the progress bar is used to print
@@ -166,125 +85,31 @@ pub fn ferox_print(msg: &str, bar: &ProgressBar) {
     }
 }
 
-#[macro_export]
-/// wrapper to improve code readability
-macro_rules! update_stat {
-    ($tx:expr, $value:expr) => {
-        $tx.send($value).unwrap_or_default();
-    };
-}
+/// wrapper for make_request used to pass error/response codes to FeroxScans for per-scan stats
+/// tracking of information related to auto-tune/bail
+pub async fn logged_request(url: &Url, handles: Arc<Handles>) -> Result<Response> {
+    let client = &handles.config.client;
+    let level = handles.config.output_level;
+    let tx_stats = handles.stats.tx.clone();
 
-/// Simple helper to generate a `Url`
-///
-/// Errors during parsing `url` or joining `word` are propagated up the call stack
-pub fn format_url(
-    url: &str,
-    word: &str,
-    add_slash: bool,
-    queries: &[(String, String)],
-    extension: Option<&str>,
-    tx_stats: UnboundedSender<StatCommand>,
-) -> FeroxResult<Url> {
-    log::trace!(
-        "enter: format_url({}, {}, {}, {:?} {:?}, {:?})",
-        url,
-        word,
-        add_slash,
-        queries,
-        extension,
-        tx_stats
-    );
+    let response = make_request(client, url, level, tx_stats).await;
 
-    if Url::parse(&word).is_ok() {
-        // when a full url is passed in as a word to be joined to a base url using
-        // reqwest::Url::join, the result is that the word (url) completely overwrites the base
-        // url, potentially resulting in requests to places that aren't actually the target
-        // specified.
-        //
-        // in order to resolve the issue, we check if the word from the wordlist is a parsable URL
-        // and if so, don't do any further processing
-        let message = format!(
-            "word ({}) from the wordlist is actually a URL, skipping...",
-            word
-        );
-        log::warn!("{}", message);
+    let scans = handles.ferox_scans()?;
 
-        let err = FeroxError { message };
-
-        update_stat!(tx_stats, AddError(UrlFormat));
-
-        log::trace!("exit: format_url -> {}", err);
-        return Err(Box::new(err));
-    }
-
-    // from reqwest::Url::join
-    //   Note: a trailing slash is significant. Without it, the last path component
-    //   is considered to be a “file” name to be removed to get at the “directory”
-    //   that is used as the base
-    //
-    // the transforms that occur here will need to keep this in mind, i.e. add a slash to preserve
-    // the current directory sent as part of the url
-    let url = if word.is_empty() {
-        // v1.0.6: added during --extract-links feature implementation to support creating urls
-        // that were extracted from response bodies, i.e. http://localhost/some/path/js/main.js
-        url.to_string()
-    } else if !url.ends_with('/') {
-        format!("{}/", url)
-    } else {
-        url.to_string()
-    };
-
-    let base_url = reqwest::Url::parse(&url)?;
-
-    // extensions and slashes are mutually exclusive cases
-    let word = if extension.is_some() {
-        format!("{}.{}", word, extension.unwrap())
-    } else if add_slash && !word.ends_with('/') {
-        // -f used, and word doesn't already end with a /
-        format!("{}/", word)
-    } else if word.starts_with("//") {
-        // bug ID'd by @Sicks3c, when a wordlist contains words that begin with 2 forward slashes
-        // i.e. //1_40_0/static/js, it gets joined onto the base url in a surprising way
-        // ex: https://localhost/ + //1_40_0/static/js -> https://1_40_0/static/js
-        // this is due to the fact that //... is a valid url. The fix is introduced here in 1.12.2
-        // and simply removes prefixed forward slashes if there are two of them. Additionally,
-        // trim_start_matches will trim the pattern until it's gone, so even if there are more than
-        // 2 /'s, they'll still be trimmed
-        word.trim_start_matches('/').to_string()
-    } else {
-        String::from(word)
-    };
-
-    match base_url.join(&word) {
-        Ok(request) => {
-            if queries.is_empty() {
-                // no query params to process
-                log::trace!("exit: format_url -> {}", request);
-                Ok(request)
-            } else {
-                match reqwest::Url::parse_with_params(request.as_str(), queries) {
-                    Ok(req_w_params) => {
-                        log::trace!("exit: format_url -> {}", req_w_params);
-                        Ok(req_w_params) // request with params attached
-                    }
-                    Err(e) => {
-                        log::error!(
-                            "Could not add query params {:?} to {}: {}",
-                            queries,
-                            request,
-                            e
-                        );
-                        log::trace!("exit: format_url -> {}", request);
-                        Ok(request) // couldn't process params, return initially ok url
-                    }
+    match response {
+        Ok(resp) => {
+            match resp.status() {
+                StatusCode::TOO_MANY_REQUESTS | StatusCode::FORBIDDEN => {
+                    scans.increment_status_code(url.as_str(), resp.status());
                 }
+                _ => {}
             }
+            Ok(resp)
         }
         Err(e) => {
-            update_stat!(tx_stats, AddError(UrlFormat));
-            log::trace!("exit: format_url -> {}", e);
-            log::error!("Could not join {} with {}", word, base_url);
-            Err(Box::new(e))
+            log::warn!("err: {:?}", e);
+            scans.increment_error(url.as_str());
+            bail!(e)
         }
     }
 }
@@ -293,58 +118,59 @@ pub fn format_url(
 pub async fn make_request(
     client: &Client,
     url: &Url,
-    tx_stats: UnboundedSender<StatCommand>,
-) -> FeroxResult<Response> {
+    output_level: OutputLevel,
+    tx_stats: UnboundedSender<Command>,
+) -> Result<Response> {
     log::trace!(
-        "enter: make_request(CONFIGURATION.Client, {}, {:?})",
+        "enter: make_request(Configuration::Client, {}, {:?}, {:?})",
         url,
+        output_level,
         tx_stats
     );
 
     match client.get(url.to_owned()).send().await {
         Err(e) => {
-            let mut log_level = log::Level::Error;
-
             log::trace!("exit: make_request -> {}", e);
+
             if e.is_timeout() {
-                // only warn for timeouts, while actual errors are still left as errors
-                log_level = log::Level::Warn;
-                update_stat!(tx_stats, AddError(Timeout));
+                send_command!(tx_stats, AddError(Timeout));
             } else if e.is_redirect() {
                 if let Some(last_redirect) = e.url() {
                     // get where we were headed (last_redirect) and where we came from (url)
                     let fancy_message = format!("{} !=> {}", url, last_redirect);
 
                     let report = if let Some(msg_status) = e.status() {
-                        update_stat!(tx_stats, AddStatus(msg_status));
-                        create_report_string(msg_status.as_str(), "-1", "-1", "-1", &fancy_message)
+                        send_command!(tx_stats, AddStatus(msg_status));
+                        create_report_string(
+                            msg_status.as_str(),
+                            "-1",
+                            "-1",
+                            "-1",
+                            &fancy_message,
+                            output_level,
+                        )
                     } else {
-                        create_report_string("UNK", "-1", "-1", "-1", &fancy_message)
+                        create_report_string("UNK", "-1", "-1", "-1", &fancy_message, output_level)
                     };
 
-                    update_stat!(tx_stats, AddError(Redirection));
+                    send_command!(tx_stats, AddError(Redirection));
 
                     ferox_print(&report, &PROGRESS_PRINTER)
                 };
             } else if e.is_connect() {
-                update_stat!(tx_stats, AddError(Connection));
+                send_command!(tx_stats, AddError(Connection));
             } else if e.is_request() {
-                update_stat!(tx_stats, AddError(Request));
+                send_command!(tx_stats, AddError(Request));
             } else {
-                update_stat!(tx_stats, AddError(Other));
+                send_command!(tx_stats, AddError(Other));
             }
 
-            if matches!(log_level, log::Level::Error) {
-                log::error!("Error while making request: {}", e);
-            } else {
-                log::warn!("Error while making request: {}", e);
-            }
-
-            Err(Box::new(e))
+            log::warn!("Error while making request: {}", e);
+            bail!("{}", e)
         }
         Ok(resp) => {
             log::trace!("exit: make_request -> {:?}", resp);
-            update_stat!(tx_stats, AddStatus(resp.status()));
+            send_command!(tx_stats, AddStatus(resp.status()));
             Ok(resp)
         }
     }
@@ -360,9 +186,10 @@ pub fn create_report_string(
     word_count: &str,
     content_length: &str,
     url: &str,
+    output_level: OutputLevel,
 ) -> String {
-    if CONFIGURATION.quiet {
-        // -q used, just need the url
+    if matches!(output_level, OutputLevel::Silent) {
+        // --silent used, just need the url
         format!("{}\n", url)
     } else {
         // normal printing with status and sizes
@@ -425,27 +252,44 @@ pub fn set_open_file_limit(limit: usize) -> bool {
     false
 }
 
-/// Simple helper to abstract away adding a forward-slash to a url if not present
-///
-/// used mostly for deduplication purposes and url state tracking
-pub fn normalize_url(url: &str) -> String {
-    log::trace!("enter: normalize_url({})", url);
+/// Given a string and a reference to a locked buffered file, write the contents and flush
+/// the buffer to disk.
+pub fn write_to<T>(
+    value: &T,
+    file: &mut io::BufWriter<fs::File>,
+    convert_to_json: bool,
+) -> Result<()>
+where
+    T: FeroxSerialize,
+{
+    // note to future self: adding logging of anything other than error to this function
+    // is a bad idea. we call this function while processing records generated by the logger.
+    // If we then call log::... while already processing some logging output, it results in
+    // the second log entry being injected into the first.
 
-    let normalized = if url.ends_with('/') {
-        url.to_string()
+    let contents = if convert_to_json {
+        value.as_json()?
     } else {
-        format!("{}/", url)
+        value.as_str()
     };
 
-    log::trace!("exit: normalize_url -> {}", normalized);
-    normalized
+    let contents = strip_ansi_codes(&contents);
+
+    let written = file.write(contents.as_bytes())?;
+
+    if written > 0 {
+        // this function is used within async functions/loops, so i'm flushing so that in
+        // the event of a ctrl+c or w/e results seen so far are saved instead of left lying
+        // around in the buffer
+        file.flush()?;
+    }
+
+    Ok(())
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::FeroxChannel;
-    use tokio::sync::mpsc;
 
     #[test]
     /// set_open_file_limit with a low requested limit succeeds
@@ -475,161 +319,6 @@ mod tests {
         assert!(!set_open_file_limit(hard.as_usize())); // returns false
     }
 
-    #[test]
-    /// base url returns 1
-    fn get_current_depth_base_url_returns_1() {
-        let depth = get_current_depth("http://localhost");
-        assert_eq!(depth, 1);
-    }
-
-    #[test]
-    /// base url with slash returns 1
-    fn get_current_depth_base_url_with_slash_returns_1() {
-        let depth = get_current_depth("http://localhost/");
-        assert_eq!(depth, 1);
-    }
-
-    #[test]
-    /// base url + 1 dir returns 2
-    fn get_current_depth_one_dir_returns_2() {
-        let depth = get_current_depth("http://localhost/src");
-        assert_eq!(depth, 2);
-    }
-
-    #[test]
-    /// base url + 1 dir and slash returns 2
-    fn get_current_depth_one_dir_with_slash_returns_2() {
-        let depth = get_current_depth("http://localhost/src/");
-        assert_eq!(depth, 2);
-    }
-
-    #[test]
-    /// base url + 1 dir and slash returns 2
-    fn get_current_depth_single_forward_slash_is_zero() {
-        let depth = get_current_depth("");
-        assert_eq!(depth, 0);
-    }
-
-    #[test]
-    /// base url + 1 word + no slash + no extension
-    fn format_url_normal() {
-        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
-        assert_eq!(
-            format_url("http://localhost", "stuff", false, &Vec::new(), None, tx).unwrap(),
-            reqwest::Url::parse("http://localhost/stuff").unwrap()
-        );
-    }
-
-    #[test]
-    /// base url + no word + no slash + no extension
-    fn format_url_no_word() {
-        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
-        assert_eq!(
-            format_url("http://localhost", "", false, &Vec::new(), None, tx).unwrap(),
-            reqwest::Url::parse("http://localhost").unwrap()
-        );
-    }
-
-    #[test]
-    /// base url + word + no slash + no extension + queries
-    fn format_url_joins_queries() {
-        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
-        assert_eq!(
-            format_url(
-                "http://localhost",
-                "lazer",
-                false,
-                &[(String::from("stuff"), String::from("things"))],
-                None,
-                tx
-            )
-            .unwrap(),
-            reqwest::Url::parse("http://localhost/lazer?stuff=things").unwrap()
-        );
-    }
-
-    #[test]
-    /// base url + no word + no slash + no extension + queries
-    fn format_url_without_word_joins_queries() {
-        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
-        assert_eq!(
-            format_url(
-                "http://localhost",
-                "",
-                false,
-                &[(String::from("stuff"), String::from("things"))],
-                None,
-                tx
-            )
-            .unwrap(),
-            reqwest::Url::parse("http://localhost/?stuff=things").unwrap()
-        );
-    }
-
-    #[test]
-    #[should_panic]
-    /// no base url is an error
-    fn format_url_no_url() {
-        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
-        format_url("", "stuff", false, &Vec::new(), None, tx).unwrap();
-    }
-
-    #[test]
-    /// word prepended with slash is adjusted for correctness
-    fn format_url_word_with_preslash() {
-        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
-        assert_eq!(
-            format_url("http://localhost", "/stuff", false, &Vec::new(), None, tx).unwrap(),
-            reqwest::Url::parse("http://localhost/stuff").unwrap()
-        );
-    }
-
-    #[test]
-    /// word with appended slash allows the slash to persist
-    fn format_url_word_with_postslash() {
-        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
-        assert_eq!(
-            format_url("http://localhost", "stuff/", false, &Vec::new(), None, tx).unwrap(),
-            reqwest::Url::parse("http://localhost/stuff/").unwrap()
-        );
-    }
-
-    #[test]
-    /// word with two prepended slashes doesn't discard the entire domain
-    fn format_url_word_with_two_prepended_slashes() {
-        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
-
-        let result = format_url(
-            "http://localhost",
-            "//upload/img",
-            false,
-            &Vec::new(),
-            None,
-            tx,
-        )
-        .unwrap();
-
-        assert_eq!(
-            result,
-            reqwest::Url::parse("http://localhost/upload/img").unwrap()
-        );
-    }
-
-    #[test]
-    /// word that is a fully formed url, should return an error
-    fn format_url_word_that_is_a_url() {
-        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
-        let url = format_url(
-            "http://localhost",
-            "http://schmocalhost",
-            false,
-            &Vec::new(),
-            None,
-            tx,
-        );
-        assert!(url.is_err());
-    }
-
     #[test]
     /// status colorizer uses red for 500s
     fn status_colorizer_uses_red_for_500s() {

tests/test_banner.rs

@@ -86,7 +86,7 @@ fn banner_prints_replay_proxy() -> Result<(), Box<dyn std::error::Error>> {
 #[test]
 /// test allows non-existent wordlist to trigger the banner printing to stderr
 /// expect to see all mandatory prints + multiple headers
-fn banner_prints_headers() -> Result<(), Box<dyn std::error::Error>> {
+fn banner_prints_headers() {
     Command::cargo_bin("feroxbuster")
         .unwrap()
         .arg("--url")
@@ -111,7 +111,6 @@ fn banner_prints_headers() -> Result<(), Box<dyn std::error::Error>> {
                 .and(predicate::str::contains("mostuff: mothings"))
                 .and(predicate::str::contains("─┴─")),
         );
-    Ok(())
 }
 
 #[test]
@@ -161,7 +160,7 @@ fn banner_prints_filter_sizes() {
 #[test]
 /// test allows non-existent wordlist to trigger the banner printing to stderr
 /// expect to see all mandatory prints + queries
-fn banner_prints_queries() -> Result<(), Box<dyn std::error::Error>> {
+fn banner_prints_queries() {
     Command::cargo_bin("feroxbuster")
         .unwrap()
         .arg("--url")
@@ -186,13 +185,12 @@ fn banner_prints_queries() -> Result<(), Box<dyn std::error::Error>> {
                 .and(predicate::str::contains("stuff=things"))
                 .and(predicate::str::contains("─┴─")),
         );
-    Ok(())
 }
 
 #[test]
 /// test allows non-existent wordlist to trigger the banner printing to stderr
 /// expect to see all mandatory prints + status codes
-fn banner_prints_status_codes() -> Result<(), Box<dyn std::error::Error>> {
+fn banner_prints_status_codes() {
     Command::cargo_bin("feroxbuster")
         .unwrap()
         .arg("--url")
@@ -213,13 +211,12 @@ fn banner_prints_status_codes() -> Result<(), Box<dyn std::error::Error>> {
                 .and(predicate::str::contains("[201, 301, 401]"))
                 .and(predicate::str::contains("─┴─")),
         );
-    Ok(())
 }
 
 #[test]
 /// test allows non-existent wordlist to trigger the banner printing to stderr
 /// expect to see all mandatory prints + replay codes
-fn banner_prints_replay_codes() -> Result<(), Box<dyn std::error::Error>> {
+fn banner_prints_replay_codes() {
     Command::cargo_bin("feroxbuster")
         .unwrap()
         .arg("--url")
@@ -244,13 +241,12 @@ fn banner_prints_replay_codes() -> Result<(), Box<dyn std::error::Error>> {
                 .and(predicate::str::contains("[200, 302]"))
                 .and(predicate::str::contains("─┴─")),
         );
-    Ok(())
 }
 
 #[test]
 /// test allows non-existent wordlist to trigger the banner printing to stderr
 /// expect to see all mandatory prints + output file
-fn banner_prints_output_file() -> Result<(), Box<dyn std::error::Error>> {
+fn banner_prints_output_file() {
     Command::cargo_bin("feroxbuster")
         .unwrap()
         .arg("--url")
@@ -270,15 +266,17 @@ fn banner_prints_output_file() -> Result<(), Box<dyn std::error::Error>> {
                 .and(predicate::str::contains("User-Agent"))
                 .and(predicate::str::contains("Output File"))
                 .and(predicate::str::contains("/super/cool/path"))
+                .and(predicate::str::contains(
+                    "ERROR: Couldn't start /super/cool/path file handler",
+                ))
                 .and(predicate::str::contains("─┴─")),
         );
-    Ok(())
 }
 
 #[test]
 /// test allows non-existent wordlist to trigger the banner printing to stderr
 /// expect to see all mandatory prints + insecure
-fn banner_prints_insecure() -> Result<(), Box<dyn std::error::Error>> {
+fn banner_prints_insecure() {
     Command::cargo_bin("feroxbuster")
         .unwrap()
         .arg("--url")
@@ -299,13 +297,12 @@ fn banner_prints_insecure() -> Result<(), Box<dyn std::error::Error>> {
                 .and(predicate::str::contains("true"))
                 .and(predicate::str::contains("─┴─")),
         );
-    Ok(())
 }
 
 #[test]
 /// test allows non-existent wordlist to trigger the banner printing to stderr
 /// expect to see all mandatory prints + follow redirects
-fn banner_prints_redirects() -> Result<(), Box<dyn std::error::Error>> {
+fn banner_prints_redirects() {
     Command::cargo_bin("feroxbuster")
         .unwrap()
         .arg("--url")
@@ -326,13 +323,12 @@ fn banner_prints_redirects() -> Result<(), Box<dyn std::error::Error>> {
                 .and(predicate::str::contains("true"))
                 .and(predicate::str::contains("─┴─")),
         );
-    Ok(())
 }
 
 #[test]
 /// test allows non-existent wordlist to trigger the banner printing to stderr
 /// expect to see all mandatory prints + extensions
-fn banner_prints_extensions() -> Result<(), Box<dyn std::error::Error>> {
+fn banner_prints_extensions() {
     Command::cargo_bin("feroxbuster")
         .unwrap()
         .arg("--url")
@@ -356,13 +352,12 @@ fn banner_prints_extensions() -> Result<(), Box<dyn std::error::Error>> {
                 .and(predicate::str::contains("[js, pdf]"))
                 .and(predicate::str::contains("─┴─")),
         );
-    Ok(())
 }
 
 #[test]
 /// test allows non-existent wordlist to trigger the banner printing to stderr
 /// expect to see all mandatory prints + dont_filter
-fn banner_prints_dont_filter() -> Result<(), Box<dyn std::error::Error>> {
+fn banner_prints_dont_filter() {
     Command::cargo_bin("feroxbuster")
         .unwrap()
         .arg("--url")
@@ -383,13 +378,12 @@ fn banner_prints_dont_filter() -> Result<(), Box<dyn std::error::Error>> {
                 .and(predicate::str::contains("false"))
                 .and(predicate::str::contains("─┴─")),
         );
-    Ok(())
 }
 
 #[test]
 /// test allows non-existent wordlist to trigger the banner printing to stderr
 /// expect to see all mandatory prints + verbosity=1
-fn banner_prints_verbosity_one() -> Result<(), Box<dyn std::error::Error>> {
+fn banner_prints_verbosity_one() {
     Command::cargo_bin("feroxbuster")
         .unwrap()
         .arg("--url")
@@ -410,13 +404,12 @@ fn banner_prints_verbosity_one() -> Result<(), Box<dyn std::error::Error>> {
                 .and(predicate::str::contains("│ 1"))
                 .and(predicate::str::contains("─┴─")),
         );
-    Ok(())
 }
 
 #[test]
 /// test allows non-existent wordlist to trigger the banner printing to stderr
 /// expect to see all mandatory prints + verbosity=2
-fn banner_prints_verbosity_two() -> Result<(), Box<dyn std::error::Error>> {
+fn banner_prints_verbosity_two() {
     Command::cargo_bin("feroxbuster")
         .unwrap()
         .arg("--url")
@@ -437,13 +430,12 @@ fn banner_prints_verbosity_two() -> Result<(), Box<dyn std::error::Error>> {
                 .and(predicate::str::contains("│ 2"))
                 .and(predicate::str::contains("─┴─")),
         );
-    Ok(())
 }
 
 #[test]
 /// test allows non-existent wordlist to trigger the banner printing to stderr
 /// expect to see all mandatory prints + verbosity=3
-fn banner_prints_verbosity_three() -> Result<(), Box<dyn std::error::Error>> {
+fn banner_prints_verbosity_three() {
     Command::cargo_bin("feroxbuster")
         .unwrap()
         .arg("--url")
@@ -464,13 +456,12 @@ fn banner_prints_verbosity_three() -> Result<(), Box<dyn std::error::Error>> {
                 .and(predicate::str::contains("│ 3"))
                 .and(predicate::str::contains("─┴─")),
         );
-    Ok(())
 }
 
 #[test]
 /// test allows non-existent wordlist to trigger the banner printing to stderr
 /// expect to see all mandatory prints + verbosity=4
-fn banner_prints_verbosity_four() -> Result<(), Box<dyn std::error::Error>> {
+fn banner_prints_verbosity_four() {
     Command::cargo_bin("feroxbuster")
         .unwrap()
         .arg("--url")
@@ -491,13 +482,12 @@ fn banner_prints_verbosity_four() -> Result<(), Box<dyn std::error::Error>> {
                 .and(predicate::str::contains("│ 4"))
                 .and(predicate::str::contains("─┴─")),
         );
-    Ok(())
 }
 
 #[test]
 /// test allows non-existent wordlist to trigger the banner printing to stderr
 /// expect to see all mandatory prints + add slash
-fn banner_prints_add_slash() -> Result<(), Box<dyn std::error::Error>> {
+fn banner_prints_add_slash() {
     Command::cargo_bin("feroxbuster")
         .unwrap()
         .arg("--url")
@@ -518,13 +508,12 @@ fn banner_prints_add_slash() -> Result<(), Box<dyn std::error::Error>> {
                 .and(predicate::str::contains("true"))
                 .and(predicate::str::contains("─┴─")),
         );
-    Ok(())
 }
 
 #[test]
 /// test allows non-existent wordlist to trigger the banner printing to stderr
 /// expect to see all mandatory prints + INFINITE recursion
-fn banner_prints_infinite_depth() -> Result<(), Box<dyn std::error::Error>> {
+fn banner_prints_infinite_depth() {
     Command::cargo_bin("feroxbuster")
         .unwrap()
         .arg("--url")
@@ -546,13 +535,12 @@ fn banner_prints_infinite_depth() -> Result<(), Box<dyn std::error::Error>> {
                 .and(predicate::str::contains("INFINITE"))
                 .and(predicate::str::contains("─┴─")),
         );
-    Ok(())
 }
 
 #[test]
 /// test allows non-existent wordlist to trigger the banner printing to stderr
 /// expect to see all mandatory prints + recursion depth
-fn banner_prints_recursion_depth() -> Result<(), Box<dyn std::error::Error>> {
+fn banner_prints_recursion_depth() {
     Command::cargo_bin("feroxbuster")
         .unwrap()
         .arg("--url")
@@ -574,13 +562,12 @@ fn banner_prints_recursion_depth() -> Result<(), Box<dyn std::error::Error>> {
                 .and(predicate::str::contains("343214"))
                 .and(predicate::str::contains("─┴─")),
         );
-    Ok(())
 }
 
 #[test]
 /// test allows non-existent wordlist to trigger the banner printing to stderr
 /// expect to see all mandatory prints + no recursion
-fn banner_prints_no_recursion() -> Result<(), Box<dyn std::error::Error>> {
+fn banner_prints_no_recursion() {
     Command::cargo_bin("feroxbuster")
         .unwrap()
         .arg("--url")
@@ -601,13 +588,12 @@ fn banner_prints_no_recursion() -> Result<(), Box<dyn std::error::Error>> {
                 .and(predicate::str::contains("true"))
                 .and(predicate::str::contains("─┴─")),
         );
-    Ok(())
 }
 
 #[test]
 /// test allows non-existent wordlist to trigger the banner printing to stderr
 /// expect to see nothing
-fn banner_doesnt_print() -> Result<(), Box<dyn std::error::Error>> {
+fn banner_doesnt_print() {
     Command::cargo_bin("feroxbuster")
         .unwrap()
         .arg("--url")
@@ -615,14 +601,15 @@ fn banner_doesnt_print() -> Result<(), Box<dyn std::error::Error>> {
         .arg("-q")
         .assert()
         .success()
-        .stderr(predicate::str::is_empty());
-    Ok(())
+        .stderr(predicate::str::contains(
+            "Could not connect to any target provided",
+        ));
 }
 
 #[test]
 /// test allows non-existent wordlist to trigger the banner printing to stderr
 /// expect to see all mandatory prints + extract-links
-fn banner_prints_extract_links() -> Result<(), Box<dyn std::error::Error>> {
+fn banner_prints_extract_links() {
     Command::cargo_bin("feroxbuster")
         .unwrap()
         .arg("--url")
@@ -643,13 +630,12 @@ fn banner_prints_extract_links() -> Result<(), Box<dyn std::error::Error>> {
                 .and(predicate::str::contains("true"))
                 .and(predicate::str::contains("─┴─")),
         );
-    Ok(())
 }
 
 #[test]
 /// test allows non-existent wordlist to trigger the banner printing to stderr
 /// expect to see all mandatory prints + scan-limit
-fn banner_prints_scan_limit() -> Result<(), Box<dyn std::error::Error>> {
+fn banner_prints_scan_limit() {
     Command::cargo_bin("feroxbuster")
         .unwrap()
         .arg("--url")
@@ -671,13 +657,12 @@ fn banner_prints_scan_limit() -> Result<(), Box<dyn std::error::Error>> {
                 .and(predicate::str::contains("│ 4"))
                 .and(predicate::str::contains("─┴─")),
         );
-    Ok(())
 }
 
 #[test]
 /// test allows non-existent wordlist to trigger the banner printing to stderr
 /// expect to see all mandatory prints + filter-status
-fn banner_prints_filter_status() -> Result<(), Box<dyn std::error::Error>> {
+fn banner_prints_filter_status() {
     Command::cargo_bin("feroxbuster")
         .unwrap()
         .arg("--url")
@@ -699,7 +684,6 @@ fn banner_prints_filter_status() -> Result<(), Box<dyn std::error::Error>> {
                 .and(predicate::str::contains("│ [200]"))
                 .and(predicate::str::contains("─┴─")),
         );
-    Ok(())
 }
 
 #[test]
@@ -837,3 +821,154 @@ fn banner_prints_similarity_filter() {
                 .and(predicate::str::contains("─┴─")),
         );
 }
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + rate limit
+fn banner_prints_rate_limit() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--rate-limit")
+        .arg("6735")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Requests per Second"))
+                .and(predicate::str::contains("│ 6735"))
+                .and(predicate::str::contains("─┴─")),
+        );
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + auto tune
+fn banner_prints_auto_tune() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--auto-tune")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Auto Tune"))
+                .and(predicate::str::contains("│ true"))
+                .and(predicate::str::contains("─┴─")),
+        );
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + auto bail
+fn banner_prints_auto_bail() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--auto-bail")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Auto Bail"))
+                .and(predicate::str::contains("│ true"))
+                .and(predicate::str::contains("─┴─")),
+        );
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see no banner output
+fn banner_doesnt_print_when_silent() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--silent")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .not()
+                .and(predicate::str::contains("Target Url").not())
+                .and(predicate::str::contains("http://localhost").not())
+                .and(predicate::str::contains("Threads").not())
+                .and(predicate::str::contains("Wordlist").not())
+                .and(predicate::str::contains("Status Codes").not())
+                .and(predicate::str::contains("Timeout (secs)").not())
+                .and(predicate::str::contains("User-Agent").not()),
+        );
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see no banner output
+fn banner_doesnt_print_when_quiet() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--quiet")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .not()
+                .and(predicate::str::contains("Target Url").not())
+                .and(predicate::str::contains("http://localhost").not())
+                .and(predicate::str::contains("Threads").not())
+                .and(predicate::str::contains("Wordlist").not())
+                .and(predicate::str::contains("Status Codes").not())
+                .and(predicate::str::contains("Timeout (secs)").not())
+                .and(predicate::str::contains("User-Agent").not()),
+        );
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see nothing as --parallel forces --silent to be true
+fn banner_prints_parallel() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--stdin")
+        .arg("--parallel")
+        .arg("4316")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .not()
+                .and(predicate::str::contains("Target Url").not())
+                .and(predicate::str::contains("Parallel Scans").not())
+                .and(predicate::str::contains("Threads").not())
+                .and(predicate::str::contains("Wordlist").not())
+                .and(predicate::str::contains("Status Codes").not())
+                .and(predicate::str::contains("Timeout (secs)").not())
+                .and(predicate::str::contains("User-Agent").not()),
+        );
+}

tests/test_extractor.rs

@@ -263,7 +263,7 @@ fn extractor_finds_robots_txt_links_and_displays_files_or_scans_directories() {
 
     let mock_disallowed = srv.mock(|when, then| {
         when.method(GET).path("/disallowed-subdir");
-        then.status(404);
+        then.status(403);
     });
 
     let cmd = Command::cargo_bin("feroxbuster")
@@ -296,6 +296,80 @@ fn extractor_finds_robots_txt_links_and_displays_files_or_scans_directories() {
     teardown_tmp_directory(tmp_dir);
 }
 
+#[test]
+/// serve a robots.txt with a file and and a folder link contained within it. ferox should
+/// find both links and request each one. This is the non-recursive version of the test above
+fn extractor_finds_robots_txt_links_and_displays_files_non_recursive() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("im a little teapot"); // 18
+    });
+
+    let mock_two = srv.mock(|when, then| {
+        when.method(GET).path("/robots.txt");
+        then.status(200).body(
+            r#"
+            User-agent: *
+            Crawl-delay: 10
+            # CSS, JS, Images
+            Allow: /misc/*.css$
+            Disallow: /misc/stupidfile.php
+               Disallow: /disallowed-subdir/
+            "#,
+        );
+    });
+
+    let mock_file = srv.mock(|when, then| {
+        when.method(GET).path("/misc/stupidfile.php");
+        then.status(200).body("im a little teapot too"); // 22
+    });
+
+    let mock_scanned_file = srv.mock(|when, then| {
+        when.method(GET).path("/misc/LICENSE");
+        then.status(200).body("i too, am a container for tea"); // 29
+    });
+
+    let mock_dir = srv.mock(|when, _| {
+        when.method(GET).path("/misc/");
+    });
+
+    let mock_disallowed = srv.mock(|when, then| {
+        when.method(GET).path("/disallowed-subdir");
+        then.status(404);
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--extract-links")
+        .arg("--no-recursion")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .and(predicate::str::contains("18c"))
+            .and(predicate::str::contains("/misc/stupidfile.php"))
+            .and(predicate::str::contains("22c"))
+            .and(predicate::str::contains("/misc/LICENSE").not())
+            .and(predicate::str::contains("29c").not())
+            .and(predicate::str::contains("200").count(2)),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_dir.hits(), 1);
+    assert_eq!(mock_two.hits(), 1);
+    assert_eq!(mock_file.hits(), 1);
+    assert_eq!(mock_disallowed.hits(), 1);
+    assert_eq!(mock_scanned_file.hits(), 0);
+    teardown_tmp_directory(tmp_dir);
+}
+
 #[test]
 /// send a request to a page that contains a link that contains a directory that returns a 403
 /// --extract-links should find the link and make recurse into the 403 directory, finding LICENSE

tests/test_heuristics.rs

@@ -90,6 +90,28 @@ fn test_one_good_and_one_bad_target_scan_succeeds() -> Result<(), Box<dyn std::e
     Ok(())
 }
 
+#[test]
+/// test passes one target with SSL issues via -u to the scanner, expected result is that the
+/// scanner dies and prints an SSL specific error message
+fn test_single_target_cannot_connect_due_to_ssl_errors() -> Result<(), Box<dyn std::error::Error>> {
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("https://expired.badssl.com")
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .assert()
+        .success()
+        .stdout(
+            predicate::str::contains("Could not connect to https://expired.badssl.com due to SSL errors (run with -k to ignore), skipping...", )
+        );
+
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
 #[test]
 /// test pipes two good targets to the scanner, expected result is that both targets
 /// are scanned successfully and no error is reported (result of issue #169)
@@ -295,7 +317,7 @@ fn heuristics_wildcard_test_with_two_static_wildcards() {
 
 #[test]
 /// test finds a static wildcard and reports nothing to stdout
-fn heuristics_wildcard_test_with_two_static_wildcards_with_quiet_enabled(
+fn heuristics_wildcard_test_with_two_static_wildcards_with_silent_enabled(
 ) -> Result<(), Box<dyn std::error::Error>> {
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
@@ -321,7 +343,7 @@ fn heuristics_wildcard_test_with_two_static_wildcards_with_quiet_enabled(
         .arg("--wordlist")
         .arg(file.as_os_str())
         .arg("--add-slash")
-        .arg("-q")
+        .arg("--silent")
         .unwrap();
 
     teardown_tmp_directory(tmp_dir);

tests/test_main.rs

@@ -1,13 +1,14 @@
 mod utils;
 use assert_cmd::Command;
 use httpmock::Method::GET;
-use httpmock::MockServer;
+use httpmock::{MockServer, Regex};
 use predicates::prelude::*;
+use std::fs::read_to_string;
 use utils::{setup_tmp_directory, teardown_tmp_directory};
 
 #[test]
 /// send the function a file to which we dont have permission in order to execute error branch
-fn main_use_root_owned_file_as_wordlist() -> Result<(), Box<dyn std::error::Error>> {
+fn main_use_root_owned_file_as_wordlist() {
     let srv = MockServer::start();
 
     let mock = srv.mock(|when, then| {
@@ -23,12 +24,13 @@ fn main_use_root_owned_file_as_wordlist() -> Result<(), Box<dyn std::error::Erro
         .arg("/etc/shadow")
         .arg("-vvvv")
         .assert()
-        .failure()
-        .stdout(predicate::str::contains("Permission denied (os error 13)"));
+        .success()
+        .stderr(predicate::str::contains(
+            "Failed while scanning: Could not open /etc/shadow",
+        ));
 
     // connectivity test hits it once
     assert_eq!(mock.hits(), 1);
-    Ok(())
 }
 
 #[test]
@@ -50,8 +52,10 @@ fn main_use_empty_wordlist() -> Result<(), Box<dyn std::error::Error>> {
         .arg(file.as_os_str())
         .arg("-vvvv")
         .assert()
-        .failure()
-        .stdout(predicate::str::contains("Did not find any words in"));
+        .success()
+        .stderr(predicate::str::contains(
+            "Failed while scanning: Did not find any words in",
+        ));
 
     assert_eq!(mock.hits(), 1);
 
@@ -86,3 +90,66 @@ fn main_use_empty_stdin_targets() -> Result<(), Box<dyn std::error::Error>> {
 
     Ok(())
 }
+
+#[test]
+/// send three targets over stdin, expect parallel to spawn children and each child config to show
+/// up in the output file
+fn main_parallel_spawns_children() -> Result<(), Box<dyn std::error::Error>> {
+    let t1 = MockServer::start();
+    let t2 = MockServer::start();
+    let t3 = MockServer::start();
+
+    let words = [
+        String::from("LICENSE"),
+        String::from("stuff"),
+        String::from("things"),
+        String::from("mostuff"),
+        String::from("mothings"),
+    ];
+    let (word_tmp_dir, wordlist) = setup_tmp_directory(&words, "wordlist")?;
+    let (output_dir, outfile) = setup_tmp_directory(&[], "output-file")?;
+    let (tgt_tmp_dir, targets) =
+        setup_tmp_directory(&[t1.url("/"), t2.url("/"), t3.url("/")], "targets")?;
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--stdin")
+        .arg("--parallel")
+        .arg("2")
+        .arg("-vvvv")
+        .arg("--debug-log")
+        .arg(outfile.as_os_str())
+        .arg("--wordlist")
+        .arg(wordlist.as_os_str())
+        .pipe_stdin(targets)
+        .unwrap()
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("Could not connect to any target provided")
+                .and(predicate::str::contains("Target Url"))
+                .not(), // no target url found
+        );
+
+    let contents = read_to_string(outfile).unwrap();
+    println!("contents: {}", contents);
+
+    assert!(contents.contains("parallel branch && wrapped main")); // exits parallel branch
+
+    // DBG      0.007 feroxbuster parallel exec: target/debug/feroxbuster
+    //   --debug-log /tmp/.tmpAjRts6/output-file --wordlist /tmp/.tmpS4CKKq/wordlist
+    //   --silent -u http://127.0.0.1:41979/
+    let r1 = Regex::new(&format!("parallel exec:.*-u {}", t1.url("/"))).unwrap();
+    let r2 = Regex::new(&format!("parallel exec:.*-u {}", t2.url("/"))).unwrap();
+    let r3 = Regex::new(&format!("parallel exec:.*-u {}", t3.url("/"))).unwrap();
+
+    assert!(r1.is_match(&contents)); // all 3 were spawned
+    assert!(r2.is_match(&contents));
+    assert!(r3.is_match(&contents));
+
+    teardown_tmp_directory(word_tmp_dir);
+    teardown_tmp_directory(tgt_tmp_dir);
+    teardown_tmp_directory(output_dir);
+
+    Ok(())
+}

tests/test_policies.rs

@@ -0,0 +1,432 @@
+mod utils;
+use assert_cmd::prelude::*;
+use httpmock::Method::GET;
+use httpmock::MockServer;
+use regex::Regex;
+use std::fs::{read_to_string, write};
+use std::path::Path;
+use std::process::Command;
+use std::time::Instant;
+use tokio::time::Duration;
+use utils::{setup_tmp_directory, teardown_tmp_directory};
+
+// tests/policy-test-error-words is a wordlist with the following attributes:
+// - 60 errors per error category (error, 403, 429)
+// - 1000 words tagged as normal for noise/padding
+// - each error string is 6_RANDOM_ASCII{error,status403,status429,normal}6_RANDOM_ASCII
+// examples:
+// - BKPMiherrortBPKcw
+// - lTjbLpstatus403fZQaFD
+// - ZhGBHGstatus429SIUZvI
+// - ufzEXWnormalOLhbLM
+// these words will be used along with pattern matching to trigger different policies
+
+#[test]
+/// --auto-bail should cancel a scan with spurious errors  
+fn auto_bail_cancels_scan_with_timeouts() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["ignored".to_string()], "wordlist").unwrap();
+    let (log_dir, logfile) = setup_tmp_directory(&[], "debug-log").unwrap();
+
+    let policy_words = read_to_string(Path::new("tests/policy-test-words.shuffled")).unwrap();
+
+    write(&file, policy_words).unwrap();
+
+    assert_eq!(file.metadata().unwrap().len(), 117720); // sanity check on wordlist size
+
+    let error_mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z]{6}error[a-zA-Z]{6}").unwrap());
+        then.delay(Duration::new(3, 0))
+            .status(200)
+            .body("verboten, nerd");
+    });
+
+    let other_errors_mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z]{6}(status429|status403)[a-zA-Z]{6}").unwrap());
+        then.status(200).body("other errors are a 200");
+    });
+
+    let normal_reqs_mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z]{6}normal[a-zA-Z]{6}").unwrap());
+        then.status(200).body("any normal request is a 200");
+    });
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--auto-bail")
+        .arg("--dont-filter")
+        .arg("--timeout")
+        .arg("2")
+        .arg("--threads")
+        .arg("4")
+        .arg("--debug-log")
+        .arg(logfile.as_os_str())
+        .arg("-vvvv")
+        .arg("--json")
+        .assert()
+        .success();
+
+    let debug_log = read_to_string(logfile).unwrap();
+
+    // read debug log to get the number of errors enforced
+    for line in debug_log.lines() {
+        let log: serde_json::Value = serde_json::from_str(&line).unwrap_or_default();
+        if let Some(message) = log.get("message") {
+            let str_msg = message.as_str().unwrap_or_default().to_string();
+
+            if str_msg.starts_with("Stats") {
+                let re = Regex::new("total_expected: ([0-9]+),").unwrap();
+                assert!(re.is_match(&str_msg));
+                let total_expected = re
+                    .captures(&str_msg)
+                    .unwrap()
+                    .get(1)
+                    .map_or("", |m| m.as_str())
+                    .parse::<usize>()
+                    .unwrap();
+
+                println!("expected: {}", total_expected);
+                // without bailing, should be 6180; after bail decreases significantly
+                assert!(total_expected < 5000);
+            }
+        }
+    }
+
+    teardown_tmp_directory(tmp_dir);
+    teardown_tmp_directory(log_dir);
+
+    assert!(normal_reqs_mock.hits() < 6000); // not all requests should make it
+    assert!(error_mock.hits() >= 25); // need at least 25 to trigger the policy
+    assert!(other_errors_mock.hits() <= 120); // may or may not see all other error requests
+}
+
+#[test]
+/// --auto-bail should cancel a scan with spurious 403s
+fn auto_bail_cancels_scan_with_403s() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["ignored".to_string()], "wordlist").unwrap();
+    let (log_dir, logfile) = setup_tmp_directory(&[], "debug-log").unwrap();
+
+    let policy_words = read_to_string(Path::new("tests/policy-test-words.shuffled")).unwrap();
+
+    write(&file, policy_words).unwrap();
+
+    assert_eq!(file.metadata().unwrap().len(), 117720); // sanity check on wordlist size
+
+    let error_mock = srv.mock(|when, then| {
+        when.method(GET).path_matches(
+            Regex::new("/[a-zA-Z]{6}(error|status429|status403)[a-zA-Z]{6}").unwrap(),
+        );
+        then.status(200).body("other errors are still a 200");
+    });
+
+    let normal_reqs_mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z]{6}normal[a-zA-Z]{6}").unwrap());
+        then.status(403)
+            .body("these guys need to be 403 in order to trigger 90% threshold");
+    });
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--auto-bail")
+        .arg("--dont-filter")
+        .arg("--threads")
+        .arg("4")
+        .arg("--debug-log")
+        .arg(logfile.as_os_str())
+        .arg("-vvvv")
+        .arg("--json")
+        .assert()
+        .success();
+
+    println!("log filesize: {}", logfile.metadata().unwrap().len());
+    let debug_log = read_to_string(logfile).unwrap();
+
+    // read debug log to get the number of errors enforced
+    for line in debug_log.lines() {
+        let log: serde_json::Value = serde_json::from_str(&line).unwrap_or_default();
+        if let Some(message) = log.get("message") {
+            let str_msg = message.as_str().unwrap_or_default().to_string();
+
+            if str_msg.starts_with("Stats") {
+                println!("{}", str_msg);
+                let re = Regex::new("total_expected: ([0-9]+),").unwrap();
+                assert!(re.is_match(&str_msg));
+                let total_expected = re
+                    .captures(&str_msg)
+                    .unwrap()
+                    .get(1)
+                    .map_or("", |m| m.as_str())
+                    .parse::<usize>()
+                    .unwrap();
+                println!("total_expected: {}", total_expected);
+                assert!(total_expected < 5000);
+            }
+        }
+    }
+
+    teardown_tmp_directory(tmp_dir);
+    teardown_tmp_directory(log_dir);
+
+    assert!(normal_reqs_mock.hits() + error_mock.hits() > 25); // must have at least 50 reqs fly
+
+    // expect much less in the way of requests for this one, 90% is measured against requests made,
+    // not requests expected, so 90% can be reached very quickly. for the same reason, the
+    // num_enforced can be less than 50
+    assert!(normal_reqs_mock.hits() < 500);
+    assert!(error_mock.hits() <= 180); // may or may not see all other error requests
+}
+
+#[test]
+/// --auto-bail should cancel a scan with spurious 429s
+fn auto_bail_cancels_scan_with_429s() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["ignored".to_string()], "wordlist").unwrap();
+    let (log_dir, logfile) = setup_tmp_directory(&[], "debug-log").unwrap();
+
+    let policy_words = read_to_string(Path::new("tests/policy-test-words.shuffled")).unwrap();
+
+    write(&file, policy_words).unwrap();
+
+    assert_eq!(file.metadata().unwrap().len(), 117720); // sanity check on wordlist size
+
+    let error_mock = srv.mock(|when, then| {
+        when.method(GET).path_matches(
+            Regex::new("/[a-zA-Z]{6}(error|status429|status403)[a-zA-Z]{6}").unwrap(),
+        );
+        then.status(200).body("other errors are still a 200");
+    });
+
+    let normal_reqs_mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z]{6}normal[a-zA-Z]{6}").unwrap());
+        then.status(429)
+            .body("these guys need to be 403 in order to trigger 90% threshold");
+    });
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--auto-bail")
+        .arg("--dont-filter")
+        .arg("--threads")
+        .arg("4")
+        .arg("--debug-log")
+        .arg(logfile.as_os_str())
+        .arg("-vvvv")
+        .arg("--json")
+        .assert()
+        .success();
+
+    println!("log filesize: {}", logfile.metadata().unwrap().len());
+    let debug_log = read_to_string(logfile).unwrap();
+
+    // read debug log to get the number of errors enforced
+    for line in debug_log.lines() {
+        let log: serde_json::Value = serde_json::from_str(&line).unwrap_or_default();
+        if let Some(message) = log.get("message") {
+            let str_msg = message.as_str().unwrap_or_default().to_string();
+
+            if str_msg.starts_with("Stats") {
+                println!("{}", str_msg);
+                let re = Regex::new("total_expected: ([0-9]+),").unwrap();
+                assert!(re.is_match(&str_msg));
+                let total_expected = re
+                    .captures(&str_msg)
+                    .unwrap()
+                    .get(1)
+                    .map_or("", |m| m.as_str())
+                    .parse::<usize>()
+                    .unwrap();
+                println!("total_expected: {}", total_expected);
+                assert!(total_expected < 5000);
+            }
+        }
+    }
+
+    teardown_tmp_directory(tmp_dir);
+    teardown_tmp_directory(log_dir);
+
+    assert!(normal_reqs_mock.hits() + error_mock.hits() > 25); // must have at least 50 reqs fly
+
+    // expect much less in the way of requests for this one, 90% is measured against requests made,
+    // not requests expected, so 90% can be reached very quickly. for the same reason, the
+    // num_enforced can be less than 50
+    assert!(normal_reqs_mock.hits() < 500);
+    assert!(error_mock.hits() <= 180); // may or may not see all other error requests
+}
+
+#[test]
+/// --auto-tune should slow a scan with spurious 429s
+fn auto_tune_slows_scan_with_429s() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["ignored".to_string()], "wordlist").unwrap();
+
+    let policy_words = read_to_string(Path::new("tests/policy-test-words.shuffled")).unwrap();
+
+    write(&file, policy_words).unwrap();
+
+    assert_eq!(file.metadata().unwrap().len(), 117720); // sanity check on wordlist size
+
+    let error_mock = srv.mock(|when, then| {
+        when.method(GET).path_matches(
+            Regex::new("/[a-zA-Z]{6}(error|status429|status403)[a-zA-Z]{6}").unwrap(),
+        );
+        then.status(200).body("other errors are still a 200");
+    });
+
+    let normal_reqs_mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z]{6}normal[a-zA-Z]{6}").unwrap());
+        then.status(429)
+            .body("these guys need to be 429 in order to trigger 30% threshold");
+    });
+
+    let start = Instant::now();
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--auto-tune")
+        .arg("--dont-filter")
+        .arg("--time-limit")
+        .arg("7s")
+        .arg("--threads")
+        .arg("4")
+        .assert()
+        .failure();
+
+    teardown_tmp_directory(tmp_dir);
+
+    assert!(normal_reqs_mock.hits() + error_mock.hits() > 25); // must have at least 50 reqs fly
+
+    println!("elapsed: {}", start.elapsed().as_millis()); // 3523ms without tuning
+    assert!(normal_reqs_mock.hits() < 500);
+    assert!(error_mock.hits() <= 180); // may or may not see all other error requests
+    assert!(start.elapsed().as_millis() >= 7000); // scan should hit time limit due to limiting
+}
+
+#[test]
+/// --auto-tune should slow a scan with spurious 403s
+fn auto_tune_slows_scan_with_403s() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["ignored".to_string()], "wordlist").unwrap();
+
+    let policy_words = read_to_string(Path::new("tests/policy-test-words.shuffled")).unwrap();
+
+    write(&file, policy_words).unwrap();
+
+    assert_eq!(file.metadata().unwrap().len(), 117720); // sanity check on wordlist size
+
+    let error_mock = srv.mock(|when, then| {
+        when.method(GET).path_matches(
+            Regex::new("/[a-zA-Z]{6}(error|status429|status403)[a-zA-Z]{6}").unwrap(),
+        );
+        then.status(200).body("other errors are still a 200");
+    });
+
+    let normal_reqs_mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z]{6}normal[a-zA-Z]{6}").unwrap());
+        then.status(403)
+            .body("these guys need to be 403 in order to trigger 90% threshold");
+    });
+
+    let start = Instant::now();
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--auto-tune")
+        .arg("--dont-filter")
+        .arg("--time-limit")
+        .arg("7s")
+        .arg("--threads")
+        .arg("4")
+        .assert()
+        .failure();
+
+    teardown_tmp_directory(tmp_dir);
+
+    assert!(normal_reqs_mock.hits() + error_mock.hits() > 25); // must have at least 50 reqs fly
+
+    println!("elapsed: {}", start.elapsed().as_millis()); // 3523ms without tuning
+    assert!(normal_reqs_mock.hits() < 500);
+    assert!(error_mock.hits() <= 180); // may or may not see all other error requests
+    assert!(start.elapsed().as_millis() >= 7000); // scan should hit time limit due to limiting
+}
+
+#[test]
+/// --auto-tune should slow a scan with spurious errors
+fn auto_tune_slows_scan_with_general_errors() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["ignored".to_string()], "wordlist").unwrap();
+
+    let policy_words = read_to_string(Path::new("tests/policy-test-words.shuffled")).unwrap();
+
+    write(&file, policy_words).unwrap();
+
+    assert_eq!(file.metadata().unwrap().len(), 117720); // sanity check on wordlist size
+
+    let error_mock = srv.mock(|when, then| {
+        when.method(GET).path_matches(
+            Regex::new("/[a-zA-Z]{6}(error|status429|status403)[a-zA-Z]{6}").unwrap(),
+        );
+        then.status(200).body("other errors are still a 200");
+    });
+
+    let normal_reqs_mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z]{6}normal[a-zA-Z]{6}").unwrap());
+        then.status(200)
+            .body("these guys need to be 429 in order to trigger 30% threshold")
+            .delay(Duration::new(3, 0));
+    });
+
+    let start = Instant::now();
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--auto-tune")
+        .arg("--dont-filter")
+        .arg("--time-limit")
+        .arg("7s")
+        .arg("--threads")
+        .arg("4")
+        .arg("--timeout")
+        .arg("2")
+        .assert()
+        .failure();
+
+    teardown_tmp_directory(tmp_dir);
+
+    println!("elapsed: {}", start.elapsed().as_millis()); // 3523ms without tuning
+    assert!(normal_reqs_mock.hits() < 500);
+    assert!(error_mock.hits() <= 180); // may or may not see all other error requests
+    assert!(start.elapsed().as_millis() >= 7000); // scan should hit time limit due to limiting
+}

tests/test_scan_manager.rs

@@ -30,7 +30,7 @@ fn resume_scan_works() {
     let scans = format!(r#""scans":[{},{}]"#, complete_scan, incomplete_scan);
 
     let config = format!(
-        r#""config": {{"type":"configuration","wordlist":"{}","config":"","proxy":"","replay_proxy":"","target_url":"{}","status_codes":[200,204,301,302,307,308,401,403,405],"replay_codes":[200,204,301,302,307,308,401,403,405],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"quiet":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/1.9.0","redirects":false,"insecure":false,"extensions":[],"headers":{{}},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":false,"depth":2,"scan_limit":1,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false}}"#,
+        r#""config": {{"type":"configuration","wordlist":"{}","config":"","proxy":"","replay_proxy":"","target_url":"{}","status_codes":[200,204,301,302,307,308,401,403,405],"replay_codes":[200,204,301,302,307,308,401,403,405],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"silent":false,"quiet":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/1.9.0","redirects":false,"insecure":false,"extensions":[],"headers":{{}},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":false,"depth":2,"scan_limit":1,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false}}"#,
         file.to_string_lossy(),
         srv.url("/")
     );

tests/test_scanner.rs

@@ -3,7 +3,7 @@ use assert_cmd::prelude::*;
 use httpmock::Method::GET;
 use httpmock::MockServer;
 use predicates::prelude::*;
-use std::process::Command;
+use std::{process::Command, time};
 use utils::{setup_tmp_directory, teardown_tmp_directory};
 
 #[test]
@@ -235,7 +235,7 @@ fn scanner_single_request_scan_with_file_output_and_tack_q(
 }
 
 #[test]
-/// send an invalid output file, expect nothing to be written to disk
+/// send an invalid output file, expect scan to fail
 fn scanner_single_request_scan_with_invalid_file_output() -> Result<(), Box<dyn std::error::Error>>
 {
     let srv = MockServer::start();
@@ -263,7 +263,7 @@ fn scanner_single_request_scan_with_invalid_file_output() -> Result<(), Box<dyn
     let contents = std::fs::read_to_string(outfile);
     assert!(contents.is_err());
 
-    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock.hits(), 0);
     teardown_tmp_directory(tmp_dir);
     Ok(())
 }
@@ -458,7 +458,7 @@ fn scanner_single_request_scan_with_debug_logging() {
     assert!(contents.contains("DBG"));
     assert!(contents.contains("INF"));
     assert!(contents.contains("feroxbuster All scans complete!"));
-    assert!(contents.contains("feroxbuster exit: terminal_input_handler"));
+    assert!(contents.contains("feroxbuster::event_handlers::inputs exit: start_enter_handler"));
 
     assert_eq!(mock.hits(), 1);
     teardown_tmp_directory(tmp_dir);
@@ -496,9 +496,13 @@ fn scanner_single_request_scan_with_debug_logging_as_json() {
     assert!(contents.contains("\"level\":\"DEBUG\""));
     assert!(contents.contains("\"level\":\"INFO\""));
     assert!(contents.contains("time_offset"));
-    assert!(contents.contains("\"module\":\"feroxbuster::scanner\""));
+    assert!(contents.contains("exit: main"));
+    assert!(contents.contains(&srv.url("/LICENSE")));
+    assert!(contents.contains("\"module\":\"feroxbuster::response\""));
+    assert!(contents.contains("\"module\":\"feroxbuster::url\""));
+    assert!(contents.contains("\"module\":\"feroxbuster::event_handlers::inputs\""));
+    assert!(contents.contains("exit: start_enter_handler"));
     assert!(contents.contains("All scans complete!"));
-    assert!(contents.contains("exit: terminal_input_handler"));
 
     assert_eq!(mock.hits(), 1);
     teardown_tmp_directory(tmp_dir);
@@ -596,3 +600,41 @@ fn scanner_recursion_works_with_403_directories() {
 
     teardown_tmp_directory(tmp_dir);
 }
+
+#[test]
+/// kick off scan with a time limit;  
+fn rate_limit_enforced_when_specified() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(
+        &[
+            "css".to_string(),
+            "stuff".to_string(),
+            "css1".to_string(),
+            "css2".to_string(),
+            "css3".to_string(),
+            "css4".to_string(),
+        ],
+        "wordlist",
+    )
+    .unwrap();
+
+    let now = time::Instant::now();
+    let lower_bound = time::Duration::new(5, 0);
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--rate-limit")
+        .arg("1")
+        .assert()
+        .success();
+
+    // --rate-limit is 1, so the test should take roughly 5 seconds, so elapsed should be at least
+    // 5 seconds. If not rate-limited, this test takes about 500ms without rate limiting
+    assert!(now.elapsed() > lower_bound);
+
+    teardown_tmp_directory(tmp_dir);
+}