Merge pull request #336 from epi052/335-fix-wildcard-filter-when-response-is-zero

335 fix wildcard filter when response is zero

.cargo/config

@@ -0,0 +1,5 @@
+[target.armv7-unknown-linux-gnueabihf]
+linker = "arm-linux-gnueabihf-gcc"
+
+[target.aarch64-unknown-linux-gnu]
+linker = "aarch64-linux-gnu-gcc"

.github/FUNDING.yml

@@ -0,0 +1,4 @@
+# These are supported funding model platforms
+
+github: [epi052]
+ko_fi: epi052

.github/dependabot.yml

@@ -0,0 +1,7 @@
+version: 2
+updates:
+- package-ecosystem: cargo
+  directory: "/"
+  schedule:
+    interval: daily
+  open-pull-requests-limit: 10

.github/pull_request_template.md

@@ -4,14 +4,14 @@ Long form explanations of most of the items below can be found in the [CONTRIBUT
 
 ## Branching checklist
 - [ ] There is an issue associated with your PR (bug, feature, etc.. if not, create one)
-- [ ] Your PR description references the associated issue (i.e. fixes #123)
+- [ ] Your PR description references the associated issue (i.e. fixes #123456)
 - [ ] Code is in its own branch
 - [ ] Branch name is related to the PR contents
-- [ ] PR targets master
+- [ ] PR targets main
 
 ## Static analysis checks
 - [ ] All rust files are formatted using `cargo fmt`
-- [ ] All `clippy` checks pass when running `cargo clippy --all-targets --all-features -- -D warnings -A clippy::deref_addrof`
+- [ ] All `clippy` checks pass when running `cargo clippy --all-targets --all-features -- -D warnings -A clippy::mutex-atomic`
 - [ ] All existing tests pass
 
 ## Documentation

.github/stale.yml

@@ -0,0 +1,17 @@
+# Number of days of inactivity before an issue becomes stale
+daysUntilStale: 14
+# Number of days of inactivity before a stale issue is closed
+daysUntilClose: 7
+# Issues with these labels will never be considered stale
+exemptLabels:
+  - pinned
+  - security
+# Label to use when marking an issue as stale
+staleLabel: stale
+# Comment to post when marking an issue as stale. Set to `false` to disable
+markComment: >
+  This issue has been automatically marked as stale because it has not had
+  recent activity. It will be closed if no further activity occurs. Thank you
+  for your contributions.
+# Comment to post when closing a stale issue. Set to `false` to disable
+closeComment: false

.github/workflows/build.yml

@@ -4,11 +4,13 @@ on: [push]
 
 jobs:
   build-nix:
+    env:
+      IN_PIPELINE: true
     runs-on: ${{ matrix.os }}
-    if: github.ref == 'refs/heads/master'
+    if: github.ref == 'refs/heads/main'
     strategy:
       matrix:
-        type: [ubuntu-x64, ubuntu-x86]
+        type: [ubuntu-x64, ubuntu-x86, armv7, aarch64]
         include:
           - type: ubuntu-x64
             os: ubuntu-latest
@@ -22,12 +24,25 @@ jobs:
             name: x86-linux-feroxbuster
             path: target/i686-unknown-linux-musl/release/feroxbuster
             pkg_config_path: /usr/lib/i686-linux-gnu/pkgconfig
+          - type: armv7
+            os: ubuntu-latest
+            target: armv7-unknown-linux-gnueabihf
+            name: armv7-feroxbuster
+            path: target/armv7-unknown-linux-gnueabihf/release/feroxbuster
+            pkg_config_path: /usr/lib/x86_64-linux-gnu/pkgconfig
+          - type: aarch64
+            os: ubuntu-latest
+            target: aarch64-unknown-linux-gnu
+            name: aarch64-feroxbuster
+            path: target/aarch64-unknown-linux-gnu/release/feroxbuster
+            pkg_config_path: /usr/lib/x86_64-linux-gnu/pkgconfig
     steps:
       - uses: actions/checkout@v2
       - name: Install System Dependencies
         run: |
+          env
           sudo apt-get update
-          sudo apt-get install -y --no-install-recommends libssl-dev pkg-config
+          sudo apt-get install -y --no-install-recommends libssl-dev pkg-config gcc-arm-linux-gnueabihf gcc-aarch64-linux-gnu
       - uses: actions-rs/toolchain@v1
         with:
           toolchain: stable
@@ -43,7 +58,7 @@ jobs:
           args: --release --target=${{ matrix.target }}
       - name: Strip symbols from binary
         run: |
-          strip -s ${{ matrix.path }}
+          strip -s ${{ matrix.path }} || arm-linux-gnueabihf-strip -s ${{ matrix.path }} || aarch64-linux-gnu-strip -s ${{ matrix.path }}
       - name: Build tar.gz for homebrew installs
         if: matrix.type == 'ubuntu-x64'
         run: |
@@ -72,8 +87,10 @@ jobs:
           path: ./target/x86_64-unknown-linux-musl/debian/*
 
   build-macos:
+    env:
+      IN_PIPELINE: true
     runs-on: macos-latest
-    if: github.ref == 'refs/heads/master'
+    if: github.ref == 'refs/heads/main'
     steps:
       - uses: actions/checkout@v2
       - uses: actions-rs/toolchain@v1
@@ -102,8 +119,10 @@ jobs:
           path: x86_64-macos-feroxbuster.tar.gz
 
   build-windows:
+    env:
+      IN_PIPELINE: true
     runs-on: ${{ matrix.os }}
-    if: github.ref == 'refs/heads/master'
+    if: github.ref == 'refs/heads/main'
     strategy:
       matrix:
         type: [windows-x64, windows-x86]
@@ -134,4 +153,3 @@ jobs:
         with:
           name: ${{ matrix.name }}
           path: ${{ matrix.path }}
-

.github/workflows/check.yml

@@ -61,4 +61,4 @@ jobs:
       - uses: actions-rs/cargo@v1
         with:
           command: clippy
-          args: --all-targets --all-features -- -D warnings -A clippy::deref_addrof
+          args: --all-targets --all-features -- -D warnings -A clippy::deref_addrof -A clippy::mutex-atomic

.github/workflows/coverage.yml

@@ -23,9 +23,13 @@ jobs:
           RUSTFLAGS: '-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort'
           RUSTDOCFLAGS: '-Cpanic=abort'
       - uses: actions-rs/grcov@v0.1
+      - uses: actions/upload-artifact@v2
+        with:
+          name: lcov.info
+          path: lcov.info
       - name: Convert lcov to xml
         run: |
-          curl -O https://raw.githubusercontent.com/eriwen/lcov-to-cobertura-xml/master/lcov_cobertura/lcov_cobertura.py
+          curl -O https://raw.githubusercontent.com/epi052/lcov-to-cobertura-xml/master/lcov_cobertura/lcov_cobertura.py
           chmod +x lcov_cobertura.py
           ./lcov_cobertura.py ./lcov.info
       - uses: codecov/codecov-action@v1
@@ -34,3 +38,7 @@ jobs:
           file: ./coverage.xml
           name: codecov-umbrella
           fail_ci_if_error: true
+      - uses: actions/upload-artifact@v2
+        with:
+          name: coverage.xml
+          path: ./coverage.xml

.gitignore

@@ -3,10 +3,6 @@
 debug/
 target/
 
-# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
-# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
-Cargo.lock
-
 # These are backup files generated by rustfmt
 **/*.rs.bk
 
@@ -22,3 +18,12 @@ img/**
 # scripts to check code coverage using nightly compiler
 check-coverage.sh
 lcov_cobertura.py
+
+# dockerignore file that makes it so i can work on the docker config without copying a 4GB manifest or w/e it is
+.dockerignore
+
+# state file created during tests
+ferox-http*
+
+# python stuff cuz reasons
+Pipfile*

.rustfmt.toml

@@ -0,0 +1 @@
+reorder_modules = false

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "feroxbuster"
-version = "1.9.0"
+version = "2.3.3"
 authors = ["Ben 'epi' Risher <epibar052@gmail.com>"]
 license = "MIT"
 edition = "2018"
@@ -10,37 +10,47 @@ description = "A fast, simple, recursive content discovery tool."
 categories = ["command-line-utilities"]
 keywords = ["pentest", "enumeration", "url-bruteforce", "content-discovery", "web"]
 exclude = [".github/*", "img/*", "check-coverage.sh"]
+build = "build.rs"
 
 [badges]
 maintenance = { status = "actively-developed" }
 
+[build-dependencies]
+clap = "2.33"
+regex = "1"
+lazy_static = "1.4"
+dirs = "3.0"
+
 [dependencies]
 futures = { version = "0.3"}
-tokio = { version = "0.2", features = ["full"] }
-tokio-util = {version = "0.3", features = ["codec"]}
+tokio = { version = "1.10", features = ["full"] }
+tokio-util = {version = "0.6.6", features = ["codec"]}
 log = "0.4"
-env_logger = "0.8"
-reqwest = { version = "0.10", features = ["socks"] }
-clap = "2"
+env_logger = "0.9"
+reqwest = { version = "0.11", features = ["socks"] }
+clap = "2.33"
 lazy_static = "1.4"
 toml = "0.5"
-serde = { version = "1.0", features = ["derive"] }
+serde = { version = "1.0", features = ["derive", "rc"] }
 serde_json = "1.0"
 uuid = { version = "0.8", features = ["v4"] }
 indicatif = "0.15"
-console = "0.12"
+console = "0.14"
 openssl = { version = "0.10", features = ["vendored"] }
 dirs = "3.0"
 regex = "1"
-crossterm = "0.18"
-rlimit = "0.5"
-ctrlc = "3.1"
+crossterm = "0.20"
+rlimit = "0.6"
+ctrlc = "3.2"
+fuzzyhash = "0.2.1"
+anyhow = "1.0"
+leaky-bucket = "0.10.0"
 
 [dev-dependencies]
 tempfile = "3.1"
-httpmock = "0.4.5"
-assert_cmd = "1.0.1"
-predicates = "1.0.5"
+httpmock = "0.6.2"
+assert_cmd = "2.0"
+predicates = "2.0"
 
 [profile.release]
 lto = true
@@ -54,4 +64,7 @@ conf-files = ["/etc/feroxbuster/ferox-config.toml"]
 assets = [
     ["target/release/feroxbuster", "/usr/bin/", "755"],
     ["ferox-config.toml.example", "/etc/feroxbuster/ferox-config.toml", "644"],
+    ["shell_completions/feroxbuster.bash", "/usr/share/bash-completion/completions/feroxbuster.bash", "644"],
+    ["shell_completions/feroxbuster.fish", "/usr/share/fish/completions/feroxbuster.fish", "644"],
+    ["shell_completions/_feroxbuster", "/usr/share/zsh/vendor-completions/_feroxbuster", "644"],
 ]

Dockerfile

@@ -1,8 +1,10 @@
 FROM alpine:latest
 LABEL maintainer="wfnintr@null.net"
 
+RUN sed -i -e 's/v[[:digit:]]\..*\//edge\//g' /etc/apk/repositories && apk upgrade --update-cache --available
+
 # download default wordlists 
-RUN apk add --no-cache --virtual .depends subversion && \
+RUN apk add --no-cache --virtual .depends subversion font-noto-emoji && \
 	svn export https://github.com/danielmiessler/SecLists/trunk/Discovery/Web-Content /usr/share/seclists/Discovery/Web-Content && \
 	apk del .depends
 

Makefile

@@ -0,0 +1,79 @@
+default_prefix = /usr/local
+prefix ?= $(default_prefix)
+exec_prefix = $(prefix)
+bindir = $(exec_prefix)/bin
+datarootdir = $(prefix)/share
+datadir = $(datarootdir)
+example_config = ferox-config.toml.example
+config_file = ferox-config.toml
+completion_dir = shell_completions
+completion_prefix = $(completion_dir)/$(BIN)
+
+BIN=feroxbuster
+SHR_SOURCES = $(shell find src -type f -wholename '*src/*.rs') Cargo.toml Cargo.lock
+
+RELEASE = debug
+DEBUG ?= 0
+
+ifeq (0, $(DEBUG))
+	ARGS = --release
+	RELEASE = release
+endif
+
+VENDORED ?= 0
+ifeq (1,$(VENDORED))
+    ARGS += --frozen
+endif
+
+TARGET = target/$(RELEASE)
+
+.PHONY: all clean install uninstall test update
+
+all: cli
+cli: $(TARGET)/$(BIN) $(TARGET)/$(BIN).1.gz $(SHR_SOURCES)
+install: all install-cli
+
+verify:
+	cargo fmt
+	cargo clippy --all-targets --all-features -- -D warnings -A clippy::mutex-atomic
+	cargo test
+
+clean:
+	cargo clean
+
+vendor: vendor.tar
+
+vendor.tar:
+	cargo vendor
+	tar pcf vendor.tar vendor
+	rm -rf vendor
+
+install-cli: cli
+	install -Dm 0644 "$(completion_prefix).bash" "$(DESTDIR)/usr/share/bash-completion/completions/$(BIN).bash"
+	install -Dm 0644 "$(completion_prefix).fish" "$(DESTDIR)/usr/share/fish/completions/$(BIN).fish"
+	install -Dm 0644 "$(completion_dir)/_$(BIN)" "$(DESTDIR)/usr/share/zsh/vendor-completions/_$(BIN)"
+	install -sDm 0755 "$(TARGET)/$(BIN)" "$(DESTDIR)$(bindir)/$(BIN)"
+	install -Dm 0644 "$(TARGET)/$(BIN).1.gz" "$(DESTDIR)$(datadir)/man/man1/$(BIN).1.gz"
+	install -Dm 0644 "$(example_config)" "$(DESTDIR)/etc/$(BIN)/$(config_file)"
+
+uninstall:
+	rm -f "$(DESTDIR)$(bindir)/$(BIN)"
+	rm -f "$(DESTDIR)$(datadir)/man/man1/$(BIN).1.gz"
+	rm -rf "$(DESTDIR)/etc/$(BIN)/"
+	rm -f "$(DESTDIR)/usr/share/bash-completion/completions/$(BIN).bash"
+	rm -f "$(DESTDIR)/usr/share/zsh/vendor-completions/_$(BIN)"
+	rm -f "$(DESTDIR)/usr/share/fish/completions/$(BIN).fish"
+
+extract:
+ifeq (1, $(VENDORED))
+	tar pxf vendor.tar
+endif
+
+$(TARGET)/$(BIN): extract
+	mkdir -p .cargo
+	cp debian/cargo.config .cargo/config.toml
+	cargo build $(ARGS)
+
+$(TARGET)/$(BIN).1.gz: $(TARGET)/$(BIN)
+	help2man --no-info $< | gzip -c > $@.partial
+	mv $@.partial $@

build.rs

@@ -0,0 +1,86 @@
+use std::fs::{copy, create_dir_all, OpenOptions};
+use std::io::{Read, Seek, SeekFrom, Write};
+extern crate clap;
+extern crate dirs;
+
+use clap::Shell;
+
+include!("src/parser.rs");
+
+fn main() {
+    println!("cargo:rerun-if-env-changed=src/parser.rs");
+
+    if std::env::var("DOCS_RS").is_ok() {
+        return; // only build when we're not generating docs
+    }
+
+    let outdir = "shell_completions";
+
+    let mut app = initialize();
+
+    let shells: [Shell; 4] = [Shell::Bash, Shell::Fish, Shell::Zsh, Shell::PowerShell];
+
+    for shell in &shells {
+        app.gen_completions("feroxbuster", *shell, outdir);
+    }
+
+    // 0xdf pointed out an oddity when tab-completing options that expect file paths, the fix we
+    // landed on was to add -o plusdirs to the bash completion script. The following code aims to
+    // automate that fix and have it present in all future builds
+    let mut contents = String::new();
+
+    let mut bash_file = OpenOptions::new()
+        .read(true)
+        .write(true)
+        .open(format!("{}/feroxbuster.bash", outdir))
+        .expect("Couldn't open bash completion script");
+
+    bash_file
+        .read_to_string(&mut contents)
+        .expect("Couldn't read bash completion script");
+
+    contents = contents.replace("default feroxbuster", "default -o plusdirs feroxbuster");
+
+    bash_file
+        .seek(SeekFrom::Start(0))
+        .expect("Couldn't seek to position 0 in bash completion script");
+
+    bash_file
+        .write_all(contents.as_bytes())
+        .expect("Couldn't write updated bash completion script to disk");
+
+    // hunter0x8 let me know that when installing via cargo, it would be nice if we dropped a
+    // config file during the build process. The following code will place an example config in
+    // the user's configuration directory
+    //   - linux: $XDG_CONFIG_HOME or $HOME/.config
+    //   - macOS: $HOME/Library/Application Support
+    //   - windows: {FOLDERID_RoamingAppData}
+    let mut config_dir = dirs::config_dir().expect("Couldn't resolve user's config directory");
+    config_dir = config_dir.join("feroxbuster"); // $HOME/.config/feroxbuster
+
+    if !config_dir.exists() {
+        // recursively create the feroxbuster directory and all of its parent components if
+        // they are missing
+        if !config_dir.exists() {
+            // recursively create the feroxbuster directory and all of its parent components if
+            // they are missing
+            if create_dir_all(&config_dir).is_err() {
+                // only copy the config file when we're not running in the CI/CD pipeline
+                // which fails with permission denied
+                eprintln!("Couldn't create one or more directories needed to copy the config file");
+                return;
+            }
+        }
+    }
+
+    // hard-coding config name here to not rely on the crate we're building, if DEFAULT_CONFIG_NAME
+    // ever changes, this will need to be updated
+    let config_file = config_dir.join("ferox-config.toml");
+
+    if !config_file.exists() {
+        // config file doesn't exist, add it to the config directory
+        if copy("ferox-config.toml.example", config_file).is_err() {
+            eprintln!("Couldn't copy example config into config directory");
+        }
+    }
+}

ferox-config.toml.example

@@ -16,8 +16,13 @@
 # replay_proxy = "http://127.0.0.1:8081"
 # replay_codes = [200, 302]
 # verbosity = 1
+# parallel = 8
 # scan_limit = 6
+# rate_limit = 250
 # quiet = true
+# silent = true
+# auto_tune = true
+# auto_bail = true
 # json = true
 # output = "/targets/ellingson_mineral_company/gibson.txt"
 # debug_log = "/var/log/find-the-derp.log"
@@ -25,6 +30,7 @@
 # redirects = true
 # insecure = true
 # extensions = ["php", "html"]
+# url_denylist = ["http://dont-scan.me", "https://also-not.me"]
 # no_recursion = true
 # add_slash = true
 # stdin = true
@@ -33,10 +39,12 @@
 # depth = 1
 # filter_size = [5174]
 # filter_regex = ["^ignore me$"]
+# filter_similar = ["https://somesite.com/soft404"]
 # filter_word_count = [993]
 # filter_line_count = [35, 36]
 # queries = [["name","value"], ["rick", "astley"]]
 # save_state = false
+# time_limit = "10m"
 
 # headers can be specified on multiple lines or as an inline table
 #

install-nix.sh

@@ -3,54 +3,63 @@
 BASE_URL=https://github.com/epi052/feroxbuster/releases/latest/download
 
 MAC_ZIP=x86_64-macos-feroxbuster.zip
-MAC_URL="${BASE_URL}/${MAC_ZIP}"
+MAC_URL="$BASE_URL/$MAC_ZIP"
 
 LIN32_ZIP=x86-linux-feroxbuster.zip
-LIN32_URL="${BASE_URL}/${LIN32_ZIP}"
+LIN32_URL="$BASE_URL/$LIN32_ZIP"
 
 LIN64_ZIP=x86_64-linux-feroxbuster.zip
-LIN64_URL="${BASE_URL}/${LIN64_ZIP}"
+LIN64_URL="$BASE_URL/$LIN64_ZIP"
 
 EMOJI_URL=https://gist.github.com/epi052/8196b550ea51d0907ad4b93751b1b57d/raw/6112c9f32ae07922983fdc549c54fd3fb9a38e4c/NotoColorEmoji.ttf
 
 echo "[+] Installing feroxbuster!"
 
+which unzip &>/dev/null
+if [ "$?" = "0" ]; then
+  echo "[+] unzip found"
+else
+  echo "[ ] unzip not found, exiting. "
+  exit -1
+fi
+
 if [[ "$(uname)" == "Darwin" ]]; then
-    echo "[=] Found MacOS, downloading from ${MAC_URL}"
+  echo "[=] Found MacOS, downloading from $MAC_URL"
 
-    curl -sLO "${MAC_URL}"
-    unzip -o "${MAC_ZIP}" > /dev/null
-    rm "${MAC_ZIP}"
+  curl -sLO "$MAC_URL"
+  unzip -o "$MAC_ZIP" >/dev/null
+  rm "$MAC_ZIP"
 elif [[ "$(expr substr $(uname -s) 1 5)" == "Linux" ]]; then
-    if [[ $(getconf LONG_BIT) == 32 ]]; then
-        echo "[=] Found 32-bit Linux, downloading from ${LIN32_URL}"
+  if [[ $(getconf LONG_BIT) == 32 ]]; then
+    echo "[=] Found 32-bit Linux, downloading from $LIN32_URL"
 
-        curl -sLO "${LIN32_URL}"
-        unzip -o "${LIN32_ZIP}" > /dev/null
-        rm "${LIN32_ZIP}"
-    else
-        echo "[=] Found 64-bit Linux, downloading from ${LIN64_URL}"
+    curl -sLO "$LIN32_URL"
+    unzip -o "$LIN32_ZIP" >/dev/null
+    rm "$LIN32_ZIP"
+  else
+    echo "[=] Found 64-bit Linux, downloading from $LIN64_URL"
 
-        curl -sLO "${LIN64_URL}"
-        unzip -o "${LIN64_ZIP}" > /dev/null
-        rm "${LIN64_ZIP}"
-    fi
+    curl -sLO "$LIN64_URL"
+    unzip -o "$LIN64_ZIP" >/dev/null
+    rm "$LIN64_ZIP"
+  fi
 
+  if [[ -e ~/.fonts/NotoColorEmoji.ttf ]]; then
+    echo "[=] Found Noto Emoji Font, skipping install"
+  else
     echo "[=] Installing Noto Emoji Font"
     mkdir -p ~/.fonts
     pushd ~/.fonts 2>&1 >/dev/null
 
-    curl -sLO "${EMOJI_URL}"
+    curl -sLO "$EMOJI_URL"
 
     fc-cache -f -v >/dev/null
 
     popd 2>&1 >/dev/null
     echo "[+] Noto Emoji Font installed"
+  fi
 fi
 
 chmod +x ./feroxbuster
 
 echo "[+] Installed feroxbuster version $(./feroxbuster -V)"
-
-
-

shell_completions/_feroxbuster

@@ -0,0 +1,102 @@
+#compdef feroxbuster
+
+autoload -U is-at-least
+
+_feroxbuster() {
+    typeset -A opt_args
+    typeset -a _arguments_options
+    local ret=1
+
+    if is-at-least 5.2; then
+        _arguments_options=(-s -S -C)
+    else
+        _arguments_options=(-s -C)
+    fi
+
+    local context curcontext="$curcontext" state line
+    _arguments "${_arguments_options[@]}" \
+'-w+[Path to the wordlist]' \
+'--wordlist=[Path to the wordlist]' \
+'*-u+[The target URL(s) (required, unless --stdin used)]' \
+'*--url=[The target URL(s) (required, unless --stdin used)]' \
+'-t+[Number of concurrent threads (default: 50)]' \
+'--threads=[Number of concurrent threads (default: 50)]' \
+'-d+[Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)]' \
+'--depth=[Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)]' \
+'-T+[Number of seconds before a request times out (default: 7)]' \
+'--timeout=[Number of seconds before a request times out (default: 7)]' \
+'-p+[Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)]' \
+'--proxy=[Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)]' \
+'-P+[Send only unfiltered requests through a Replay Proxy, instead of all requests]' \
+'--replay-proxy=[Send only unfiltered requests through a Replay Proxy, instead of all requests]' \
+'*-R+[Status Codes to send through a Replay Proxy when found (default: --status-codes value)]' \
+'*--replay-codes=[Status Codes to send through a Replay Proxy when found (default: --status-codes value)]' \
+'*-s+[Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)]' \
+'*--status-codes=[Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)]' \
+'-o+[Output file to write results to (use w/ --json for JSON entries)]' \
+'--output=[Output file to write results to (use w/ --json for JSON entries)]' \
+'(-u --url)--resume-from=[State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)]' \
+'--debug-log=[Output file to write log entries (use w/ --json for JSON entries)]' \
+'-a+[Sets the User-Agent (default: feroxbuster/VERSION)]' \
+'--user-agent=[Sets the User-Agent (default: feroxbuster/VERSION)]' \
+'*-x+[File extension(s) to search for (ex: -x php -x pdf js)]' \
+'*--extensions=[File extension(s) to search for (ex: -x php -x pdf js)]' \
+'*--dont-scan=[URL(s) to exclude from recursion/scans]' \
+'*-H+[Specify HTTP headers (ex: -H Header:val '\''stuff: things'\'')]' \
+'*--headers=[Specify HTTP headers (ex: -H Header:val '\''stuff: things'\'')]' \
+'*-Q+[Specify URL query parameters (ex: -Q token=stuff -Q secret=key)]' \
+'*--query=[Specify URL query parameters (ex: -Q token=stuff -Q secret=key)]' \
+'*-S+[Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)]' \
+'*--filter-size=[Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)]' \
+'*-X+[Filter out messages via regular expression matching on the response'\''s body (ex: -X '\''^ignore me$'\'')]' \
+'*--filter-regex=[Filter out messages via regular expression matching on the response'\''s body (ex: -X '\''^ignore me$'\'')]' \
+'*-W+[Filter out messages of a particular word count (ex: -W 312 -W 91,82)]' \
+'*--filter-words=[Filter out messages of a particular word count (ex: -W 312 -W 91,82)]' \
+'*-N+[Filter out messages of a particular line count (ex: -N 20 -N 31,30)]' \
+'*--filter-lines=[Filter out messages of a particular line count (ex: -N 20 -N 31,30)]' \
+'*-C+[Filter out status codes (deny list) (ex: -C 200 -C 401)]' \
+'*--filter-status=[Filter out status codes (deny list) (ex: -C 200 -C 401)]' \
+'*--filter-similar-to=[Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)]' \
+'-L+[Limit total number of concurrent scans (default: 0, i.e. no limit)]' \
+'--scan-limit=[Limit total number of concurrent scans (default: 0, i.e. no limit)]' \
+'--parallel=[Run parallel feroxbuster instances (one child process per url passed via stdin)]' \
+'(--auto-tune)--rate-limit=[Limit number of requests per second (per directory) (default: 0, i.e. no limit)]' \
+'--time-limit=[Limit total run time of all scans (ex: --time-limit 10m)]' \
+'(--silent)*-v[Increase verbosity level (use -vv or more for greater effect. \[CAUTION\] 4 -v'\''s is probably too much)]' \
+'(--silent)*--verbosity[Increase verbosity level (use -vv or more for greater effect. \[CAUTION\] 4 -v'\''s is probably too much)]' \
+'(-q --quiet)--silent[Only print URLs + turn off logging (good for piping a list of urls to other commands)]' \
+'-q[Hide progress bars and banner (good for tmux windows w/ notifications)]' \
+'--quiet[Hide progress bars and banner (good for tmux windows w/ notifications)]' \
+'(--auto-bail)--auto-tune[Automatically lower scan rate when an excessive amount of errors are encountered]' \
+'--auto-bail[Automatically stop scanning when an excessive amount of errors are encountered]' \
+'--json[Emit JSON logs to --output and --debug-log instead of normal text]' \
+'-D[Don'\''t auto-filter wildcard responses]' \
+'--dont-filter[Don'\''t auto-filter wildcard responses]' \
+'-r[Follow redirects]' \
+'--redirects[Follow redirects]' \
+'-k[Disables TLS certificate validation]' \
+'--insecure[Disables TLS certificate validation]' \
+'-n[Do not scan recursively]' \
+'--no-recursion[Do not scan recursively]' \
+'(-x --extensions)-f[Append / to each request]' \
+'(-x --extensions)--add-slash[Append / to each request]' \
+'(-u --url)--stdin[Read url(s) from STDIN]' \
+'-e[Extract links from response body (html, javascript, etc...); make new requests based on findings (default: false)]' \
+'--extract-links[Extract links from response body (html, javascript, etc...); make new requests based on findings (default: false)]' \
+'-h[Prints help information]' \
+'--help[Prints help information]' \
+'-V[Prints version information]' \
+'--version[Prints version information]' \
+&& ret=0
+    
+}
+
+(( $+functions[_feroxbuster_commands] )) ||
+_feroxbuster_commands() {
+    local commands; commands=(
+        
+    )
+    _describe -t commands 'feroxbuster commands' commands "$@"
+}
+
+_feroxbuster "$@"

shell_completions/_feroxbuster.ps1

@@ -0,0 +1,101 @@
+
+using namespace System.Management.Automation
+using namespace System.Management.Automation.Language
+
+Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
+    param($wordToComplete, $commandAst, $cursorPosition)
+
+    $commandElements = $commandAst.CommandElements
+    $command = @(
+        'feroxbuster'
+        for ($i = 1; $i -lt $commandElements.Count; $i++) {
+            $element = $commandElements[$i]
+            if ($element -isnot [StringConstantExpressionAst] -or
+                $element.StringConstantType -ne [StringConstantType]::BareWord -or
+                $element.Value.StartsWith('-')) {
+                break
+        }
+        $element.Value
+    }) -join ';'
+
+    $completions = @(switch ($command) {
+        'feroxbuster' {
+            [CompletionResult]::new('-w', 'w', [CompletionResultType]::ParameterName, 'Path to the wordlist')
+            [CompletionResult]::new('--wordlist', 'wordlist', [CompletionResultType]::ParameterName, 'Path to the wordlist')
+            [CompletionResult]::new('-u', 'u', [CompletionResultType]::ParameterName, 'The target URL(s) (required, unless --stdin used)')
+            [CompletionResult]::new('--url', 'url', [CompletionResultType]::ParameterName, 'The target URL(s) (required, unless --stdin used)')
+            [CompletionResult]::new('-t', 't', [CompletionResultType]::ParameterName, 'Number of concurrent threads (default: 50)')
+            [CompletionResult]::new('--threads', 'threads', [CompletionResultType]::ParameterName, 'Number of concurrent threads (default: 50)')
+            [CompletionResult]::new('-d', 'd', [CompletionResultType]::ParameterName, 'Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)')
+            [CompletionResult]::new('--depth', 'depth', [CompletionResultType]::ParameterName, 'Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)')
+            [CompletionResult]::new('-T', 'T', [CompletionResultType]::ParameterName, 'Number of seconds before a request times out (default: 7)')
+            [CompletionResult]::new('--timeout', 'timeout', [CompletionResultType]::ParameterName, 'Number of seconds before a request times out (default: 7)')
+            [CompletionResult]::new('-p', 'p', [CompletionResultType]::ParameterName, 'Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)')
+            [CompletionResult]::new('--proxy', 'proxy', [CompletionResultType]::ParameterName, 'Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)')
+            [CompletionResult]::new('-P', 'P', [CompletionResultType]::ParameterName, 'Send only unfiltered requests through a Replay Proxy, instead of all requests')
+            [CompletionResult]::new('--replay-proxy', 'replay-proxy', [CompletionResultType]::ParameterName, 'Send only unfiltered requests through a Replay Proxy, instead of all requests')
+            [CompletionResult]::new('-R', 'R', [CompletionResultType]::ParameterName, 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)')
+            [CompletionResult]::new('--replay-codes', 'replay-codes', [CompletionResultType]::ParameterName, 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)')
+            [CompletionResult]::new('-s', 's', [CompletionResultType]::ParameterName, 'Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)')
+            [CompletionResult]::new('--status-codes', 'status-codes', [CompletionResultType]::ParameterName, 'Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)')
+            [CompletionResult]::new('-o', 'o', [CompletionResultType]::ParameterName, 'Output file to write results to (use w/ --json for JSON entries)')
+            [CompletionResult]::new('--output', 'output', [CompletionResultType]::ParameterName, 'Output file to write results to (use w/ --json for JSON entries)')
+            [CompletionResult]::new('--resume-from', 'resume-from', [CompletionResultType]::ParameterName, 'State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)')
+            [CompletionResult]::new('--debug-log', 'debug-log', [CompletionResultType]::ParameterName, 'Output file to write log entries (use w/ --json for JSON entries)')
+            [CompletionResult]::new('-a', 'a', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/VERSION)')
+            [CompletionResult]::new('--user-agent', 'user-agent', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/VERSION)')
+            [CompletionResult]::new('-x', 'x', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js)')
+            [CompletionResult]::new('--extensions', 'extensions', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js)')
+            [CompletionResult]::new('--dont-scan', 'dont-scan', [CompletionResultType]::ParameterName, 'URL(s) to exclude from recursion/scans')
+            [CompletionResult]::new('-H', 'H', [CompletionResultType]::ParameterName, 'Specify HTTP headers (ex: -H Header:val ''stuff: things'')')
+            [CompletionResult]::new('--headers', 'headers', [CompletionResultType]::ParameterName, 'Specify HTTP headers (ex: -H Header:val ''stuff: things'')')
+            [CompletionResult]::new('-Q', 'Q', [CompletionResultType]::ParameterName, 'Specify URL query parameters (ex: -Q token=stuff -Q secret=key)')
+            [CompletionResult]::new('--query', 'query', [CompletionResultType]::ParameterName, 'Specify URL query parameters (ex: -Q token=stuff -Q secret=key)')
+            [CompletionResult]::new('-S', 'S', [CompletionResultType]::ParameterName, 'Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)')
+            [CompletionResult]::new('--filter-size', 'filter-size', [CompletionResultType]::ParameterName, 'Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)')
+            [CompletionResult]::new('-X', 'X', [CompletionResultType]::ParameterName, 'Filter out messages via regular expression matching on the response''s body (ex: -X ''^ignore me$'')')
+            [CompletionResult]::new('--filter-regex', 'filter-regex', [CompletionResultType]::ParameterName, 'Filter out messages via regular expression matching on the response''s body (ex: -X ''^ignore me$'')')
+            [CompletionResult]::new('-W', 'W', [CompletionResultType]::ParameterName, 'Filter out messages of a particular word count (ex: -W 312 -W 91,82)')
+            [CompletionResult]::new('--filter-words', 'filter-words', [CompletionResultType]::ParameterName, 'Filter out messages of a particular word count (ex: -W 312 -W 91,82)')
+            [CompletionResult]::new('-N', 'N', [CompletionResultType]::ParameterName, 'Filter out messages of a particular line count (ex: -N 20 -N 31,30)')
+            [CompletionResult]::new('--filter-lines', 'filter-lines', [CompletionResultType]::ParameterName, 'Filter out messages of a particular line count (ex: -N 20 -N 31,30)')
+            [CompletionResult]::new('-C', 'C', [CompletionResultType]::ParameterName, 'Filter out status codes (deny list) (ex: -C 200 -C 401)')
+            [CompletionResult]::new('--filter-status', 'filter-status', [CompletionResultType]::ParameterName, 'Filter out status codes (deny list) (ex: -C 200 -C 401)')
+            [CompletionResult]::new('--filter-similar-to', 'filter-similar-to', [CompletionResultType]::ParameterName, 'Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)')
+            [CompletionResult]::new('-L', 'L', [CompletionResultType]::ParameterName, 'Limit total number of concurrent scans (default: 0, i.e. no limit)')
+            [CompletionResult]::new('--scan-limit', 'scan-limit', [CompletionResultType]::ParameterName, 'Limit total number of concurrent scans (default: 0, i.e. no limit)')
+            [CompletionResult]::new('--parallel', 'parallel', [CompletionResultType]::ParameterName, 'Run parallel feroxbuster instances (one child process per url passed via stdin)')
+            [CompletionResult]::new('--rate-limit', 'rate-limit', [CompletionResultType]::ParameterName, 'Limit number of requests per second (per directory) (default: 0, i.e. no limit)')
+            [CompletionResult]::new('--time-limit', 'time-limit', [CompletionResultType]::ParameterName, 'Limit total run time of all scans (ex: --time-limit 10m)')
+            [CompletionResult]::new('-v', 'v', [CompletionResultType]::ParameterName, 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v''s is probably too much)')
+            [CompletionResult]::new('--verbosity', 'verbosity', [CompletionResultType]::ParameterName, 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v''s is probably too much)')
+            [CompletionResult]::new('--silent', 'silent', [CompletionResultType]::ParameterName, 'Only print URLs + turn off logging (good for piping a list of urls to other commands)')
+            [CompletionResult]::new('-q', 'q', [CompletionResultType]::ParameterName, 'Hide progress bars and banner (good for tmux windows w/ notifications)')
+            [CompletionResult]::new('--quiet', 'quiet', [CompletionResultType]::ParameterName, 'Hide progress bars and banner (good for tmux windows w/ notifications)')
+            [CompletionResult]::new('--auto-tune', 'auto-tune', [CompletionResultType]::ParameterName, 'Automatically lower scan rate when an excessive amount of errors are encountered')
+            [CompletionResult]::new('--auto-bail', 'auto-bail', [CompletionResultType]::ParameterName, 'Automatically stop scanning when an excessive amount of errors are encountered')
+            [CompletionResult]::new('--json', 'json', [CompletionResultType]::ParameterName, 'Emit JSON logs to --output and --debug-log instead of normal text')
+            [CompletionResult]::new('-D', 'D', [CompletionResultType]::ParameterName, 'Don''t auto-filter wildcard responses')
+            [CompletionResult]::new('--dont-filter', 'dont-filter', [CompletionResultType]::ParameterName, 'Don''t auto-filter wildcard responses')
+            [CompletionResult]::new('-r', 'r', [CompletionResultType]::ParameterName, 'Follow redirects')
+            [CompletionResult]::new('--redirects', 'redirects', [CompletionResultType]::ParameterName, 'Follow redirects')
+            [CompletionResult]::new('-k', 'k', [CompletionResultType]::ParameterName, 'Disables TLS certificate validation')
+            [CompletionResult]::new('--insecure', 'insecure', [CompletionResultType]::ParameterName, 'Disables TLS certificate validation')
+            [CompletionResult]::new('-n', 'n', [CompletionResultType]::ParameterName, 'Do not scan recursively')
+            [CompletionResult]::new('--no-recursion', 'no-recursion', [CompletionResultType]::ParameterName, 'Do not scan recursively')
+            [CompletionResult]::new('-f', 'f', [CompletionResultType]::ParameterName, 'Append / to each request')
+            [CompletionResult]::new('--add-slash', 'add-slash', [CompletionResultType]::ParameterName, 'Append / to each request')
+            [CompletionResult]::new('--stdin', 'stdin', [CompletionResultType]::ParameterName, 'Read url(s) from STDIN')
+            [CompletionResult]::new('-e', 'e', [CompletionResultType]::ParameterName, 'Extract links from response body (html, javascript, etc...); make new requests based on findings (default: false)')
+            [CompletionResult]::new('--extract-links', 'extract-links', [CompletionResultType]::ParameterName, 'Extract links from response body (html, javascript, etc...); make new requests based on findings (default: false)')
+            [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Prints help information')
+            [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Prints help information')
+            [CompletionResult]::new('-V', 'V', [CompletionResultType]::ParameterName, 'Prints version information')
+            [CompletionResult]::new('--version', 'version', [CompletionResultType]::ParameterName, 'Prints version information')
+            break
+        }
+    })
+
+    $completions.Where{ $_.CompletionText -like "$wordToComplete*" } |
+        Sort-Object -Property ListItemText
+}

shell_completions/feroxbuster.bash

@@ -0,0 +1,229 @@
+_feroxbuster() {
+    local i cur prev opts cmds
+    COMPREPLY=()
+    cur="${COMP_WORDS[COMP_CWORD]}"
+    prev="${COMP_WORDS[COMP_CWORD-1]}"
+    cmd=""
+    opts=""
+
+    for i in ${COMP_WORDS[@]}
+    do
+        case "${i}" in
+            feroxbuster)
+                cmd="feroxbuster"
+                ;;
+            
+            *)
+                ;;
+        esac
+    done
+
+    case "${cmd}" in
+        feroxbuster)
+            opts=" -v -q -D -r -k -n -f -e -h -V -w -u -t -d -T -p -P -R -s -o -a -x -H -Q -S -X -W -N -C -L  --verbosity --silent --quiet --auto-tune --auto-bail --json --dont-filter --redirects --insecure --no-recursion --add-slash --stdin --extract-links --help --version --wordlist --url --threads --depth --timeout --proxy --replay-proxy --replay-codes --status-codes --output --resume-from --debug-log --user-agent --extensions --dont-scan --headers --query --filter-size --filter-regex --filter-words --filter-lines --filter-status --filter-similar-to --scan-limit --parallel --rate-limit --time-limit  "
+            if [[ ${cur} == -* || ${COMP_CWORD} -eq 1 ]] ; then
+                COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
+                return 0
+            fi
+            case "${prev}" in
+                
+                --wordlist)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -w)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --url)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -u)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --threads)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -t)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --depth)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -d)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --timeout)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -T)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --proxy)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -p)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --replay-proxy)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -P)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --replay-codes)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -R)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --status-codes)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -s)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --output)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -o)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --resume-from)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --debug-log)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --user-agent)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -a)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --extensions)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -x)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --dont-scan)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --headers)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -H)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --query)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -Q)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --filter-size)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -S)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --filter-regex)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -X)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --filter-words)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -W)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --filter-lines)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -N)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --filter-status)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -C)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --filter-similar-to)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --scan-limit)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -L)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --parallel)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --rate-limit)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --time-limit)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                *)
+                    COMPREPLY=()
+                    ;;
+            esac
+            COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
+            return 0
+            ;;
+        
+    esac
+}
+
+complete -F _feroxbuster -o bashdefault -o default -o plusdirs feroxbuster

shell_completions/feroxbuster.fish

@@ -0,0 +1,42 @@
+complete -c feroxbuster -n "__fish_use_subcommand" -s w -l wordlist -d 'Path to the wordlist'
+complete -c feroxbuster -n "__fish_use_subcommand" -s u -l url -d 'The target URL(s) (required, unless --stdin used)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s t -l threads -d 'Number of concurrent threads (default: 50)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s d -l depth -d 'Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s T -l timeout -d 'Number of seconds before a request times out (default: 7)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s p -l proxy -d 'Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s P -l replay-proxy -d 'Send only unfiltered requests through a Replay Proxy, instead of all requests'
+complete -c feroxbuster -n "__fish_use_subcommand" -s R -l replay-codes -d 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s s -l status-codes -d 'Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s o -l output -d 'Output file to write results to (use w/ --json for JSON entries)'
+complete -c feroxbuster -n "__fish_use_subcommand" -l resume-from -d 'State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)'
+complete -c feroxbuster -n "__fish_use_subcommand" -l debug-log -d 'Output file to write log entries (use w/ --json for JSON entries)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s a -l user-agent -d 'Sets the User-Agent (default: feroxbuster/VERSION)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s x -l extensions -d 'File extension(s) to search for (ex: -x php -x pdf js)'
+complete -c feroxbuster -n "__fish_use_subcommand" -l dont-scan -d 'URL(s) to exclude from recursion/scans'
+complete -c feroxbuster -n "__fish_use_subcommand" -s H -l headers -d 'Specify HTTP headers (ex: -H Header:val \'stuff: things\')'
+complete -c feroxbuster -n "__fish_use_subcommand" -s Q -l query -d 'Specify URL query parameters (ex: -Q token=stuff -Q secret=key)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s S -l filter-size -d 'Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s X -l filter-regex -d 'Filter out messages via regular expression matching on the response\'s body (ex: -X \'^ignore me$\')'
+complete -c feroxbuster -n "__fish_use_subcommand" -s W -l filter-words -d 'Filter out messages of a particular word count (ex: -W 312 -W 91,82)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s N -l filter-lines -d 'Filter out messages of a particular line count (ex: -N 20 -N 31,30)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s C -l filter-status -d 'Filter out status codes (deny list) (ex: -C 200 -C 401)'
+complete -c feroxbuster -n "__fish_use_subcommand" -l filter-similar-to -d 'Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s L -l scan-limit -d 'Limit total number of concurrent scans (default: 0, i.e. no limit)'
+complete -c feroxbuster -n "__fish_use_subcommand" -l parallel -d 'Run parallel feroxbuster instances (one child process per url passed via stdin)'
+complete -c feroxbuster -n "__fish_use_subcommand" -l rate-limit -d 'Limit number of requests per second (per directory) (default: 0, i.e. no limit)'
+complete -c feroxbuster -n "__fish_use_subcommand" -l time-limit -d 'Limit total run time of all scans (ex: --time-limit 10m)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s v -l verbosity -d 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v\'s is probably too much)'
+complete -c feroxbuster -n "__fish_use_subcommand" -l silent -d 'Only print URLs + turn off logging (good for piping a list of urls to other commands)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s q -l quiet -d 'Hide progress bars and banner (good for tmux windows w/ notifications)'
+complete -c feroxbuster -n "__fish_use_subcommand" -l auto-tune -d 'Automatically lower scan rate when an excessive amount of errors are encountered'
+complete -c feroxbuster -n "__fish_use_subcommand" -l auto-bail -d 'Automatically stop scanning when an excessive amount of errors are encountered'
+complete -c feroxbuster -n "__fish_use_subcommand" -l json -d 'Emit JSON logs to --output and --debug-log instead of normal text'
+complete -c feroxbuster -n "__fish_use_subcommand" -s D -l dont-filter -d 'Don\'t auto-filter wildcard responses'
+complete -c feroxbuster -n "__fish_use_subcommand" -s r -l redirects -d 'Follow redirects'
+complete -c feroxbuster -n "__fish_use_subcommand" -s k -l insecure -d 'Disables TLS certificate validation'
+complete -c feroxbuster -n "__fish_use_subcommand" -s n -l no-recursion -d 'Do not scan recursively'
+complete -c feroxbuster -n "__fish_use_subcommand" -s f -l add-slash -d 'Append / to each request'
+complete -c feroxbuster -n "__fish_use_subcommand" -l stdin -d 'Read url(s) from STDIN'
+complete -c feroxbuster -n "__fish_use_subcommand" -s e -l extract-links -d 'Extract links from response body (html, javascript, etc...); make new requests based on findings (default: false)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s h -l help -d 'Prints help information'
+complete -c feroxbuster -n "__fish_use_subcommand" -s V -l version -d 'Prints version information'

src/banner.rs

@@ -1,699 +0,0 @@
-use crate::config::{Configuration, CONFIGURATION};
-use crate::utils::{make_request, status_colorizer};
-use console::style;
-use reqwest::{Client, Url};
-use serde_json::Value;
-use std::io::Write;
-
-/// macro helper to abstract away repetitive string formatting
-macro_rules! format_banner_entry_helper {
-    // \u{0020} -> unicode space
-    // \u{2502} -> vertical box drawing character, i.e. │
-    ($rune:expr, $name:expr, $value:expr, $indent:expr, $col_width:expr) => {
-        format!(
-            "\u{0020}{:\u{0020}<indent$}{:\u{0020}<col_w$}\u{2502}\u{0020}{}",
-            $rune,
-            $name,
-            $value,
-            indent = $indent,
-            col_w = $col_width
-        )
-    };
-    ($rune:expr, $name:expr, $value:expr, $value2:expr, $indent:expr, $col_width:expr) => {
-        format!(
-            "\u{0020}{:\u{0020}<indent$}{:\u{0020}<col_w$}\u{2502}\u{0020}{}:\u{0020}{}",
-            $rune,
-            $name,
-            $value,
-            $value2,
-            indent = $indent,
-            col_w = $col_width
-        )
-    };
-}
-
-/// macro that wraps another macro helper to abstract away repetitive string formatting
-macro_rules! format_banner_entry {
-    // 4 -> unicode emoji padding width
-    // 22 -> column width (when unicode rune is 4 bytes wide, 23 when it's 3)
-    // hardcoded since macros don't allow let statements
-    ($rune:expr, $name:expr, $value:expr) => {
-        format_banner_entry_helper!($rune, $name, $value, 3, 22)
-    };
-    ($rune:expr, $name:expr, $value1:expr, $value2:expr) => {
-        format_banner_entry_helper!($rune, $name, $value1, $value2, 3, 22)
-    };
-}
-
-/// Url used to query github's api; specifically used to look for the latest tagged release name
-const UPDATE_URL: &str = "https://api.github.com/repos/epi052/feroxbuster/releases/latest";
-
-/// Simple enum to hold three different update states
-#[derive(Debug)]
-enum UpdateStatus {
-    /// this version and latest release are the same
-    UpToDate,
-
-    /// this version and latest release are not the same
-    OutOfDate,
-
-    /// some error occurred during version check
-    Unknown,
-}
-
-/// Makes a request to the given url, expecting to receive a JSON response that contains a field
-/// named `tag_name` that holds a value representing the latest tagged release of this tool.
-///
-/// ex: v1.1.0
-///
-/// Returns `UpdateStatus`
-async fn needs_update(client: &Client, url: &str, bin_version: &str) -> UpdateStatus {
-    log::trace!("enter: needs_update({:?}, {})", client, url);
-
-    let unknown = UpdateStatus::Unknown;
-
-    let api_url = match Url::parse(url) {
-        Ok(url) => url,
-        Err(e) => {
-            log::error!("{}", e);
-            log::trace!("exit: needs_update -> {:?}", unknown);
-            return unknown;
-        }
-    };
-
-    if let Ok(response) = make_request(&client, &api_url).await {
-        let body = response.text().await.unwrap_or_default();
-
-        let json_response: Value = serde_json::from_str(&body).unwrap_or_default();
-
-        if json_response.is_null() {
-            // unwrap_or_default above should result in a null value for the json_response variable
-            log::error!("Could not parse JSON from response body");
-            log::trace!("exit: needs_update -> {:?}", unknown);
-            return unknown;
-        }
-
-        let latest_version = match json_response["tag_name"].as_str() {
-            Some(tag) => tag.trim_start_matches('v'),
-            None => {
-                log::error!("Could not get version field from JSON response");
-                log::debug!("{}", json_response);
-                log::trace!("exit: needs_update -> {:?}", unknown);
-                return unknown;
-            }
-        };
-
-        // if we've gotten this far, we have a string in the form of X.X.X where X is a number
-        // all that's left is to compare the current version with the version found above
-
-        return if latest_version == bin_version {
-            // there's really only two possible outcomes if we accept that the tag conforms to
-            // the X.X.X pattern:
-            //   1. the version strings match, meaning we're up to date
-            //   2. the version strings do not match, meaning we're out of date
-            //
-            // except for developers working on this code, nobody should ever be in a situation
-            // where they have a version greater than the latest tagged release
-            log::trace!("exit: needs_update -> UpdateStatus::UpToDate");
-            UpdateStatus::UpToDate
-        } else {
-            log::trace!("exit: needs_update -> UpdateStatus::OutOfDate");
-            UpdateStatus::OutOfDate
-        };
-    }
-
-    log::trace!("exit: needs_update -> {:?}", unknown);
-    unknown
-}
-
-/// Prints the banner to stdout.
-///
-/// Only prints those settings which are either always present, or passed in by the user.
-pub async fn initialize<W>(targets: &[String], config: &Configuration, version: &str, mut writer: W)
-where
-    W: Write,
-{
-    let artwork = format!(
-        r#"
- ___  ___  __   __     __      __         __   ___
-|__  |__  |__) |__) | /  `    /  \ \_/ | |  \ |__
-|    |___ |  \ |  \ | \__,    \__/ / \ | |__/ |___
-by Ben "epi" Risher {}                  ver: {}"#,
-        '\u{1F913}', version
-    );
-
-    let status = needs_update(&CONFIGURATION.client, UPDATE_URL, version).await;
-
-    let top = "───────────────────────────┬──────────────────────";
-    let addl_section = "──────────────────────────────────────────────────";
-    let bottom = "───────────────────────────┴──────────────────────";
-
-    writeln!(&mut writer, "{}", artwork).unwrap_or_default();
-    writeln!(&mut writer, "{}", top).unwrap_or_default();
-
-    // begin with always printed items
-    for target in targets {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1F3af}", "Target Url", target)
-        )
-        .unwrap_or_default(); // 🎯
-    }
-
-    let mut codes = vec![];
-
-    for code in &config.status_codes {
-        codes.push(status_colorizer(&code.to_string()))
-    }
-
-    writeln!(
-        &mut writer,
-        "{}",
-        format_banner_entry!("\u{1F680}", "Threads", config.threads)
-    )
-    .unwrap_or_default(); // 🚀
-
-    writeln!(
-        &mut writer,
-        "{}",
-        format_banner_entry!("\u{1f4d6}", "Wordlist", config.wordlist)
-    )
-    .unwrap_or_default(); // 📖
-
-    writeln!(
-        &mut writer,
-        "{}",
-        format_banner_entry!(
-            "\u{1F197}",
-            "Status Codes",
-            format!("[{}]", codes.join(", "))
-        )
-    )
-    .unwrap_or_default(); // 🆗
-
-    if !config.filter_status.is_empty() {
-        // exception here for optional print due to me wanting the allows and denys to be printed
-        // one after the other
-        let mut code_filters = vec![];
-
-        for code in &config.filter_status {
-            code_filters.push(status_colorizer(&code.to_string()))
-        }
-
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!(
-                "\u{1f5d1}",
-                "Status Code Filters",
-                format!("[{}]", code_filters.join(", "))
-            )
-        )
-        .unwrap_or_default(); // 🗑
-    }
-
-    writeln!(
-        &mut writer,
-        "{}",
-        format_banner_entry!("\u{1f4a5}", "Timeout (secs)", config.timeout)
-    )
-    .unwrap_or_default(); // 💥
-
-    writeln!(
-        &mut writer,
-        "{}",
-        format_banner_entry!("\u{1F9a1}", "User-Agent", config.user_agent)
-    )
-    .unwrap_or_default(); // 🦡
-
-    // followed by the maybe printed or variably displayed values
-    if !config.config.is_empty() {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1f489}", "Config File", config.config)
-        )
-        .unwrap_or_default(); // 💉
-    }
-
-    if !config.proxy.is_empty() {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1f48e}", "Proxy", config.proxy)
-        )
-        .unwrap_or_default(); // 💎
-    }
-
-    if !config.replay_proxy.is_empty() {
-        // i include replay codes logic here because in config.rs, replay codes are set to the
-        // value in status codes, meaning it's never empty
-
-        let mut replay_codes = vec![];
-
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1f3a5}", "Replay Proxy", config.replay_proxy)
-        )
-        .unwrap_or_default(); // 🎥
-
-        for code in &config.replay_codes {
-            replay_codes.push(status_colorizer(&code.to_string()))
-        }
-
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!(
-                "\u{1f4fc}",
-                "Replay Proxy Codes",
-                format!("[{}]", replay_codes.join(", "))
-            )
-        )
-        .unwrap_or_default(); // 📼
-    }
-
-    if !config.headers.is_empty() {
-        for (name, value) in &config.headers {
-            writeln!(
-                &mut writer,
-                "{}",
-                format_banner_entry!("\u{1f92f}", "Header", name, value)
-            )
-            .unwrap_or_default(); // 🤯
-        }
-    }
-
-    if !config.filter_size.is_empty() {
-        for filter in &config.filter_size {
-            writeln!(
-                &mut writer,
-                "{}",
-                format_banner_entry!("\u{1f4a2}", "Size Filter", filter)
-            )
-            .unwrap_or_default(); // 💢
-        }
-    }
-
-    for filter in &config.filter_word_count {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1f4a2}", "Word Count Filter", filter)
-        )
-        .unwrap_or_default(); // 💢
-    }
-
-    for filter in &config.filter_line_count {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1f4a2}", "Line Count Filter", filter)
-        )
-        .unwrap_or_default(); // 💢
-    }
-
-    for filter in &config.filter_regex {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1f4a2}", "Regex Filter", filter)
-        )
-        .unwrap_or_default(); // 💢
-    }
-
-    if config.extract_links {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1F50E}", "Extract Links", config.extract_links)
-        )
-        .unwrap_or_default(); // 🔎
-    }
-
-    if config.json {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1F9d4}", "JSON Output", config.json)
-        )
-        .unwrap_or_default(); // 🧔
-    }
-
-    if !config.queries.is_empty() {
-        for query in &config.queries {
-            writeln!(
-                &mut writer,
-                "{}",
-                format_banner_entry!(
-                    "\u{1f914}",
-                    "Query Parameter",
-                    format!("{}={}", query.0, query.1)
-                )
-            )
-            .unwrap_or_default(); // 🤔
-        }
-    }
-
-    if !config.output.is_empty() {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1f4be}", "Output File", config.output)
-        )
-        .unwrap_or_default(); // 💾
-    }
-
-    if !config.debug_log.is_empty() {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1fab2}", "Debugging Log", config.debug_log)
-        )
-        .unwrap_or_default(); // 🪲
-    }
-
-    if !config.extensions.is_empty() {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!(
-                "\u{1f4b2}",
-                "Extensions",
-                format!("[{}]", config.extensions.join(", "))
-            )
-        )
-        .unwrap_or_default(); // 💲
-    }
-
-    if config.insecure {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1f513}", "Insecure", config.insecure)
-        )
-        .unwrap_or_default(); // 🔓
-    }
-
-    if config.redirects {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1f4cd}", "Follow Redirects", config.redirects)
-        )
-        .unwrap_or_default(); // 📍
-    }
-
-    if config.dont_filter {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1f92a}", "Filter Wildcards", !config.dont_filter)
-        )
-        .unwrap_or_default(); // 🤪
-    }
-
-    match config.verbosity {
-        //speaker medium volume (increasing with verbosity to loudspeaker)
-        1 => {
-            writeln!(
-                &mut writer,
-                "{}",
-                format_banner_entry!("\u{1f508}", "Verbosity", config.verbosity)
-            )
-            .unwrap_or_default(); // 🔈
-        }
-        2 => {
-            writeln!(
-                &mut writer,
-                "{}",
-                format_banner_entry!("\u{1f509}", "Verbosity", config.verbosity)
-            )
-            .unwrap_or_default(); // 🔉
-        }
-        3 => {
-            writeln!(
-                &mut writer,
-                "{}",
-                format_banner_entry!("\u{1f50a}", "Verbosity", config.verbosity)
-            )
-            .unwrap_or_default(); // 🔊
-        }
-        4 => {
-            writeln!(
-                &mut writer,
-                "{}",
-                format_banner_entry!("\u{1f4e2}", "Verbosity", config.verbosity)
-            )
-            .unwrap_or_default(); // 📢
-        }
-        _ => {}
-    }
-
-    if config.add_slash {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1fa93}", "Add Slash", config.add_slash)
-        )
-        .unwrap_or_default(); // 🪓
-    }
-
-    if !config.no_recursion {
-        if config.depth == 0 {
-            writeln!(
-                &mut writer,
-                "{}",
-                format_banner_entry!("\u{1f503}", "Recursion Depth", "INFINITE")
-            )
-            .unwrap_or_default(); // 🔃
-        } else {
-            writeln!(
-                &mut writer,
-                "{}",
-                format_banner_entry!("\u{1f503}", "Recursion Depth", config.depth)
-            )
-            .unwrap_or_default(); // 🔃
-        }
-    } else {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1f6ab}", "Do Not Recurse", config.no_recursion)
-        )
-        .unwrap_or_default(); // 🚫
-    }
-
-    if CONFIGURATION.scan_limit > 0 {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1f9a5}", "Concurrent Scan Limit", config.scan_limit)
-        )
-        .unwrap_or_default(); // 🦥
-    }
-
-    if matches!(status, UpdateStatus::OutOfDate) {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!(
-                "\u{1f389}",
-                "New Version Available",
-                "https://github.com/epi052/feroxbuster/releases/latest"
-            )
-        )
-        .unwrap_or_default(); // 🎉
-    }
-
-    writeln!(&mut writer, "{}", bottom).unwrap_or_default();
-    // ⏯
-    writeln!(
-        &mut writer,
-        " \u{23ef}   Press [{}] to {}|{} your scan",
-        style("ENTER").yellow(),
-        style("pause").red(),
-        style("resume").green()
-    )
-    .unwrap_or_default();
-    writeln!(&mut writer, "{}", addl_section).unwrap_or_default();
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::VERSION;
-    use httpmock::Method::GET;
-    use httpmock::{Mock, MockServer};
-    use std::fs::read_to_string;
-    use std::io::stderr;
-    use std::time::Duration;
-    use tempfile::NamedTempFile;
-
-    #[tokio::test(core_threads = 1)]
-    /// test to hit no execution of targets for loop in banner
-    async fn banner_intialize_without_targets() {
-        let config = Configuration::default();
-        initialize(&[], &config, VERSION, stderr()).await;
-    }
-
-    #[tokio::test(core_threads = 1)]
-    /// test to hit no execution of statuscode for loop in banner
-    async fn banner_intialize_without_status_codes() {
-        let mut config = Configuration::default();
-        config.status_codes = vec![];
-        initialize(
-            &[String::from("http://localhost")],
-            &config,
-            VERSION,
-            stderr(),
-        )
-        .await;
-    }
-
-    #[tokio::test(core_threads = 1)]
-    /// test to hit an empty config file
-    async fn banner_intialize_without_config_file() {
-        let mut config = Configuration::default();
-        config.config = String::new();
-        initialize(
-            &[String::from("http://localhost")],
-            &config,
-            VERSION,
-            stderr(),
-        )
-        .await;
-    }
-
-    #[tokio::test(core_threads = 1)]
-    /// test to hit an empty config file
-    async fn banner_intialize_without_queries() {
-        let mut config = Configuration::default();
-        config.queries = vec![(String::new(), String::new())];
-        initialize(
-            &[String::from("http://localhost")],
-            &config,
-            VERSION,
-            stderr(),
-        )
-        .await;
-    }
-
-    #[tokio::test(core_threads = 1)]
-    /// test to show that a new version is available for download
-    async fn banner_intialize_with_mismatched_version() {
-        let config = Configuration::default();
-        let file = NamedTempFile::new().unwrap();
-        initialize(
-            &[String::from("http://localhost")],
-            &config,
-            "mismatched-version",
-            &file,
-        )
-        .await;
-        let contents = read_to_string(file.path()).unwrap();
-        println!("contents: {}", contents);
-        assert!(contents.contains("New Version Available"));
-        assert!(contents.contains("https://github.com/epi052/feroxbuster/releases/latest"));
-    }
-
-    #[tokio::test(core_threads = 1)]
-    /// test that
-    async fn banner_needs_update_returns_unknown_with_bad_url() {
-        let result = needs_update(&CONFIGURATION.client, &"", VERSION).await;
-        assert!(matches!(result, UpdateStatus::Unknown));
-    }
-
-    #[tokio::test(core_threads = 1)]
-    /// test return value of good url to needs_update
-    async fn banner_needs_update_returns_up_to_date() {
-        let srv = MockServer::start();
-
-        let mock = Mock::new()
-            .expect_method(GET)
-            .expect_path("/latest")
-            .return_status(200)
-            .return_body("{\"tag_name\":\"v1.1.0\"}")
-            .create_on(&srv);
-
-        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.1.0").await;
-
-        assert_eq!(mock.times_called(), 1);
-        assert!(matches!(result, UpdateStatus::UpToDate));
-    }
-
-    #[tokio::test(core_threads = 1)]
-    /// test return value of good url to needs_update that returns a newer version than current
-    async fn banner_needs_update_returns_out_of_date() {
-        let srv = MockServer::start();
-
-        let mock = Mock::new()
-            .expect_method(GET)
-            .expect_path("/latest")
-            .return_status(200)
-            .return_body("{\"tag_name\":\"v1.1.0\"}")
-            .create_on(&srv);
-
-        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1").await;
-
-        assert_eq!(mock.times_called(), 1);
-        assert!(matches!(result, UpdateStatus::OutOfDate));
-    }
-
-    #[tokio::test(core_threads = 1)]
-    /// test return value of good url that times out
-    async fn banner_needs_update_returns_unknown_on_timeout() {
-        let srv = MockServer::start();
-
-        let mock = Mock::new()
-            .expect_method(GET)
-            .expect_path("/latest")
-            .return_status(200)
-            .return_body("{\"tag_name\":\"v1.1.0\"}")
-            .return_with_delay(Duration::from_secs(8))
-            .create_on(&srv);
-
-        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1").await;
-
-        assert_eq!(mock.times_called(), 1);
-        assert!(matches!(result, UpdateStatus::Unknown));
-    }
-
-    #[tokio::test(core_threads = 1)]
-    /// test return value of good url with bad json response
-    async fn banner_needs_update_returns_unknown_on_bad_json_response() {
-        let srv = MockServer::start();
-
-        let mock = Mock::new()
-            .expect_method(GET)
-            .expect_path("/latest")
-            .return_status(200)
-            .return_body("not json")
-            .create_on(&srv);
-
-        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1").await;
-
-        assert_eq!(mock.times_called(), 1);
-        assert!(matches!(result, UpdateStatus::Unknown));
-    }
-
-    #[tokio::test(core_threads = 1)]
-    /// test return value of good url with json response that lacks the tag_name field
-    async fn banner_needs_update_returns_unknown_on_json_without_correct_tag() {
-        let srv = MockServer::start();
-
-        let mock = Mock::new()
-            .expect_method(GET)
-            .expect_path("/latest")
-            .return_status(200)
-            .return_body("{\"no tag_name\": \"doesn't exist\"}")
-            .create_on(&srv);
-
-        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1").await;
-
-        assert_eq!(mock.times_called(), 1);
-        assert!(matches!(result, UpdateStatus::Unknown));
-    }
-}

src/banner/container.rs

@@ -0,0 +1,564 @@
+use super::entry::BannerEntry;
+use crate::{
+    config::Configuration,
+    event_handlers::Handles,
+    utils::{logged_request, status_colorizer},
+    VERSION,
+};
+use anyhow::{bail, Result};
+use console::{style, Emoji};
+use reqwest::Url;
+use serde_json::Value;
+use std::{io::Write, sync::Arc};
+
+/// Url used to query github's api; specifically used to look for the latest tagged release name
+pub const UPDATE_URL: &str = "https://api.github.com/repos/epi052/feroxbuster/releases/latest";
+
+/// Simple enum to hold three different update states
+#[derive(Debug)]
+pub(super) enum UpdateStatus {
+    /// this version and latest release are the same
+    UpToDate,
+
+    /// this version and latest release are not the same
+    OutOfDate,
+
+    /// some error occurred during version check
+    Unknown,
+}
+
+/// Banner object, contains multiple BannerEntry's and knows how to display itself
+pub struct Banner {
+    /// all live targets
+    targets: Vec<BannerEntry>,
+
+    /// represents Configuration.status_codes
+    status_codes: BannerEntry,
+
+    /// represents Configuration.filter_status
+    filter_status: BannerEntry,
+
+    /// represents Configuration.threads
+    threads: BannerEntry,
+
+    /// represents Configuration.wordlist
+    wordlist: BannerEntry,
+
+    /// represents Configuration.timeout
+    timeout: BannerEntry,
+
+    /// represents Configuration.user_agent
+    user_agent: BannerEntry,
+
+    /// represents Configuration.config
+    config: BannerEntry,
+
+    /// represents Configuration.proxy
+    proxy: BannerEntry,
+
+    /// represents Configuration.replay_proxy
+    replay_proxy: BannerEntry,
+
+    /// represents Configuration.replay_codes
+    replay_codes: BannerEntry,
+
+    /// represents Configuration.headers
+    headers: Vec<BannerEntry>,
+
+    /// represents Configuration.filter_size
+    filter_size: Vec<BannerEntry>,
+
+    /// represents Configuration.filter_similar
+    filter_similar: Vec<BannerEntry>,
+
+    /// represents Configuration.filter_word_count
+    filter_word_count: Vec<BannerEntry>,
+
+    /// represents Configuration.filter_line_count
+    filter_line_count: Vec<BannerEntry>,
+
+    /// represents Configuration.filter_regex
+    filter_regex: Vec<BannerEntry>,
+
+    /// represents Configuration.extract_links
+    extract_links: BannerEntry,
+
+    /// represents Configuration.json
+    json: BannerEntry,
+
+    /// represents Configuration.output
+    output: BannerEntry,
+
+    /// represents Configuration.debug_log
+    debug_log: BannerEntry,
+
+    /// represents Configuration.extensions
+    extensions: BannerEntry,
+
+    /// represents Configuration.insecure
+    insecure: BannerEntry,
+
+    /// represents Configuration.redirects
+    redirects: BannerEntry,
+
+    /// represents Configuration.dont_filter
+    dont_filter: BannerEntry,
+
+    /// represents Configuration.queries
+    queries: Vec<BannerEntry>,
+
+    /// represents Configuration.verbosity
+    verbosity: BannerEntry,
+
+    /// represents Configuration.add_slash
+    add_slash: BannerEntry,
+
+    /// represents Configuration.no_recursion
+    no_recursion: BannerEntry,
+
+    /// represents Configuration.scan_limit
+    scan_limit: BannerEntry,
+
+    /// represents Configuration.time_limit
+    time_limit: BannerEntry,
+
+    /// represents Configuration.rate_limit
+    rate_limit: BannerEntry,
+
+    /// represents Configuration.parallel
+    parallel: BannerEntry,
+
+    /// represents Configuration.auto_tune
+    auto_tune: BannerEntry,
+
+    /// represents Configuration.auto_bail
+    auto_bail: BannerEntry,
+
+    /// represents Configuration.url_denylist
+    url_denylist: Vec<BannerEntry>,
+
+    /// current version of feroxbuster
+    pub(super) version: String,
+
+    /// whether or not there is a known new version
+    pub(super) update_status: UpdateStatus,
+}
+
+/// implementation of Banner
+impl Banner {
+    /// Create a new Banner from a Configuration and live targets
+    pub fn new(tgts: &[String], config: &Configuration) -> Self {
+        let mut targets = Vec::new();
+        let mut url_denylist = Vec::new();
+        let mut code_filters = Vec::new();
+        let mut replay_codes = Vec::new();
+        let mut headers = Vec::new();
+        let mut filter_size = Vec::new();
+        let mut filter_similar = Vec::new();
+        let mut filter_word_count = Vec::new();
+        let mut filter_line_count = Vec::new();
+        let mut filter_regex = Vec::new();
+        let mut queries = Vec::new();
+
+        for target in tgts {
+            targets.push(BannerEntry::new("🎯", "Target Url", target));
+        }
+
+        for denied_url in &config.url_denylist {
+            url_denylist.push(BannerEntry::new("🚫", "Don't Scan", denied_url));
+        }
+
+        let mut codes = vec![];
+        for code in &config.status_codes {
+            codes.push(status_colorizer(&code.to_string()))
+        }
+        let status_codes =
+            BannerEntry::new("👌", "Status Codes", &format!("[{}]", codes.join(", ")));
+
+        for code in &config.filter_status {
+            code_filters.push(status_colorizer(&code.to_string()))
+        }
+        let filter_status = BannerEntry::new(
+            "💢",
+            "Status Code Filters",
+            &format!("[{}]", code_filters.join(", ")),
+        );
+
+        for code in &config.replay_codes {
+            replay_codes.push(status_colorizer(&code.to_string()))
+        }
+        let replay_codes = BannerEntry::new(
+            "📼",
+            "Replay Proxy Codes",
+            &format!("[{}]", replay_codes.join(", ")),
+        );
+
+        for (name, value) in &config.headers {
+            headers.push(BannerEntry::new(
+                "🤯",
+                "Header",
+                &format!("{}: {}", name, value),
+            ));
+        }
+
+        for filter in &config.filter_size {
+            filter_size.push(BannerEntry::new("💢", "Size Filter", &filter.to_string()));
+        }
+
+        for filter in &config.filter_similar {
+            filter_similar.push(BannerEntry::new("💢", "Similarity Filter", filter));
+        }
+
+        for filter in &config.filter_word_count {
+            filter_word_count.push(BannerEntry::new(
+                "💢",
+                "Word Count Filter",
+                &filter.to_string(),
+            ));
+        }
+
+        for filter in &config.filter_line_count {
+            filter_line_count.push(BannerEntry::new(
+                "💢",
+                "Line Count Filter",
+                &filter.to_string(),
+            ));
+        }
+
+        for filter in &config.filter_regex {
+            filter_regex.push(BannerEntry::new("💢", "Regex Filter", filter));
+        }
+
+        for query in &config.queries {
+            queries.push(BannerEntry::new(
+                "🤔",
+                "Query Parameter",
+                &format!("{}={}", query.0, query.1),
+            ));
+        }
+
+        let volume = ["🔈", "🔉", "🔊", "📢"];
+        let verbosity = if let 1..=4 = config.verbosity {
+            //speaker medium volume (increasing with verbosity to loudspeaker)
+            BannerEntry::new(
+                volume[config.verbosity as usize - 1],
+                "Verbosity",
+                &config.verbosity.to_string(),
+            )
+        } else {
+            BannerEntry::default()
+        };
+
+        let no_recursion = if !config.no_recursion {
+            let depth = if config.depth == 0 {
+                "INFINITE".to_string()
+            } else {
+                config.depth.to_string()
+            };
+
+            BannerEntry::new("🔃", "Recursion Depth", &depth)
+        } else {
+            BannerEntry::new("🚫", "Do Not Recurse", &config.no_recursion.to_string())
+        };
+
+        let scan_limit = BannerEntry::new(
+            "🦥",
+            "Concurrent Scan Limit",
+            &config.scan_limit.to_string(),
+        );
+
+        let replay_proxy = BannerEntry::new("🎥", "Replay Proxy", &config.replay_proxy);
+        let auto_tune = BannerEntry::new("🎶", "Auto Tune", &config.auto_tune.to_string());
+        let auto_bail = BannerEntry::new("🪣", "Auto Bail", &config.auto_bail.to_string());
+        let cfg = BannerEntry::new("💉", "Config File", &config.config);
+        let proxy = BannerEntry::new("💎", "Proxy", &config.proxy);
+        let threads = BannerEntry::new("🚀", "Threads", &config.threads.to_string());
+        let wordlist = BannerEntry::new("📖", "Wordlist", &config.wordlist);
+        let timeout = BannerEntry::new("💥", "Timeout (secs)", &config.timeout.to_string());
+        let user_agent = BannerEntry::new("🦡", "User-Agent", &config.user_agent);
+        let extract_links =
+            BannerEntry::new("🔎", "Extract Links", &config.extract_links.to_string());
+        let json = BannerEntry::new("🧔", "JSON Output", &config.json.to_string());
+        let output = BannerEntry::new("💾", "Output File", &config.output);
+        let debug_log = BannerEntry::new("🪲", "Debugging Log", &config.debug_log);
+        let extensions = BannerEntry::new(
+            "💲",
+            "Extensions",
+            &format!("[{}]", config.extensions.join(", ")),
+        );
+        let insecure = BannerEntry::new("🔓", "Insecure", &config.insecure.to_string());
+        let redirects = BannerEntry::new("📍", "Follow Redirects", &config.redirects.to_string());
+        let dont_filter =
+            BannerEntry::new("🤪", "Filter Wildcards", &(!config.dont_filter).to_string());
+        let add_slash = BannerEntry::new("🪓", "Add Slash", &config.add_slash.to_string());
+        let time_limit = BannerEntry::new("🕖", "Time Limit", &config.time_limit);
+        let parallel = BannerEntry::new("🛤", "Parallel Scans", &config.parallel.to_string());
+        let rate_limit =
+            BannerEntry::new("🚧", "Requests per Second", &config.rate_limit.to_string());
+
+        Self {
+            targets,
+            status_codes,
+            threads,
+            wordlist,
+            filter_status,
+            timeout,
+            user_agent,
+            auto_bail,
+            auto_tune,
+            proxy,
+            replay_codes,
+            replay_proxy,
+            headers,
+            filter_size,
+            filter_similar,
+            filter_word_count,
+            filter_line_count,
+            filter_regex,
+            extract_links,
+            parallel,
+            json,
+            queries,
+            output,
+            debug_log,
+            extensions,
+            insecure,
+            dont_filter,
+            redirects,
+            verbosity,
+            add_slash,
+            no_recursion,
+            rate_limit,
+            scan_limit,
+            time_limit,
+            url_denylist,
+            config: cfg,
+            version: VERSION.to_string(),
+            update_status: UpdateStatus::Unknown,
+        }
+    }
+
+    /// get a fancy header for the banner
+    fn header(&self) -> String {
+        let artwork = format!(
+            r#"
+ ___  ___  __   __     __      __         __   ___
+|__  |__  |__) |__) | /  `    /  \ \_/ | |  \ |__
+|    |___ |  \ |  \ | \__,    \__/ / \ | |__/ |___
+by Ben "epi" Risher {}                 ver: {}"#,
+            Emoji("🤓", &format!("{:<2}", "\u{0020}")),
+            self.version
+        );
+
+        let top = "───────────────────────────┬──────────────────────";
+
+        format!("{}\n{}", artwork, top)
+    }
+
+    /// get a fancy footer for the banner
+    fn footer(&self) -> String {
+        let addl_section = "──────────────────────────────────────────────────";
+        let bottom = "───────────────────────────┴──────────────────────";
+
+        let instructions = format!(
+            " 🏁  Press [{}] to use the {}™",
+            style("ENTER").yellow(),
+            style("Scan Cancel Menu").bright().yellow(),
+        );
+
+        format!("{}\n{}\n{}", bottom, instructions, addl_section)
+    }
+
+    /// Makes a request to the given url, expecting to receive a JSON response that contains a field
+    /// named `tag_name` that holds a value representing the latest tagged release of this tool.
+    ///
+    /// ex: v1.1.0
+    pub async fn check_for_updates(&mut self, url: &str, handles: Arc<Handles>) -> Result<()> {
+        log::trace!("enter: needs_update({}, {:?})", url, handles);
+
+        let api_url = Url::parse(url)?;
+
+        let result = logged_request(&api_url, handles.clone()).await?;
+        let body = result.text().await?;
+
+        let json_response: Value = serde_json::from_str(&body)?;
+
+        let latest_version = match json_response["tag_name"].as_str() {
+            Some(tag) => tag.trim_start_matches('v'),
+            None => {
+                bail!("JSON has no tag_name: {}", json_response);
+            }
+        };
+
+        // if we've gotten this far, we have a string in the form of X.X.X where X is a number
+        // all that's left is to compare the current version with the version found above
+
+        if latest_version == self.version {
+            // there's really only two possible outcomes if we accept that the tag conforms to
+            // the X.X.X pattern:
+            //   1. the version strings match, meaning we're up to date
+            //   2. the version strings do not match, meaning we're out of date
+            //
+            // except for developers working on this code, nobody should ever be in a situation
+            // where they have a version greater than the latest tagged release
+            self.update_status = UpdateStatus::UpToDate;
+        } else {
+            self.update_status = UpdateStatus::OutOfDate;
+        }
+
+        log::trace!("exit: check_for_updates -> {:?}", self.update_status);
+        Ok(())
+    }
+
+    /// display the banner on Write writer
+    pub fn print_to<W>(&self, mut writer: W, config: Arc<Configuration>) -> Result<()>
+    where
+        W: Write,
+    {
+        writeln!(&mut writer, "{}", self.header())?;
+
+        // begin with always printed items
+        for target in &self.targets {
+            writeln!(&mut writer, "{}", target)?;
+        }
+
+        for denied_url in &self.url_denylist {
+            writeln!(&mut writer, "{}", denied_url)?;
+        }
+
+        writeln!(&mut writer, "{}", self.threads)?;
+        writeln!(&mut writer, "{}", self.wordlist)?;
+        writeln!(&mut writer, "{}", self.status_codes)?;
+
+        if !config.filter_status.is_empty() {
+            // exception here for an optional print in the middle of always printed values is due
+            // to me wanting the allows and denys to be printed one after the other
+            writeln!(&mut writer, "{}", self.filter_status)?;
+        }
+
+        writeln!(&mut writer, "{}", self.timeout)?;
+        writeln!(&mut writer, "{}", self.user_agent)?;
+
+        // followed by the maybe printed or variably displayed values
+        if !config.config.is_empty() {
+            writeln!(&mut writer, "{}", self.config)?;
+        }
+
+        if !config.proxy.is_empty() {
+            writeln!(&mut writer, "{}", self.proxy)?;
+        }
+
+        if !config.replay_proxy.is_empty() {
+            // i include replay codes logic here because in config.rs, replay codes are set to the
+            // value in status codes, meaning it's never empty
+            writeln!(&mut writer, "{}", self.replay_proxy)?;
+            writeln!(&mut writer, "{}", self.replay_codes)?;
+        }
+
+        for header in &self.headers {
+            writeln!(&mut writer, "{}", header)?;
+        }
+
+        for filter in &self.filter_size {
+            writeln!(&mut writer, "{}", filter)?;
+        }
+
+        for filter in &self.filter_similar {
+            writeln!(&mut writer, "{}", filter)?;
+        }
+
+        for filter in &self.filter_word_count {
+            writeln!(&mut writer, "{}", filter)?;
+        }
+
+        for filter in &self.filter_line_count {
+            writeln!(&mut writer, "{}", filter)?;
+        }
+
+        for filter in &self.filter_regex {
+            writeln!(&mut writer, "{}", filter)?;
+        }
+
+        if config.extract_links {
+            writeln!(&mut writer, "{}", self.extract_links)?;
+        }
+
+        if config.json {
+            writeln!(&mut writer, "{}", self.json)?;
+        }
+
+        for query in &self.queries {
+            writeln!(&mut writer, "{}", query)?;
+        }
+
+        if !config.output.is_empty() {
+            writeln!(&mut writer, "{}", self.output)?;
+        }
+
+        if !config.debug_log.is_empty() {
+            writeln!(&mut writer, "{}", self.debug_log)?;
+        }
+
+        if !config.extensions.is_empty() {
+            writeln!(&mut writer, "{}", self.extensions)?;
+        }
+
+        if config.insecure {
+            writeln!(&mut writer, "{}", self.insecure)?;
+        }
+
+        if config.auto_bail {
+            writeln!(&mut writer, "{}", self.auto_bail)?;
+        }
+        if config.auto_tune {
+            writeln!(&mut writer, "{}", self.auto_tune)?;
+        }
+
+        if config.redirects {
+            writeln!(&mut writer, "{}", self.redirects)?;
+        }
+
+        if config.dont_filter {
+            writeln!(&mut writer, "{}", self.dont_filter)?;
+        }
+
+        if let 1..=4 = config.verbosity {
+            writeln!(&mut writer, "{}", self.verbosity)?;
+        }
+
+        if config.add_slash {
+            writeln!(&mut writer, "{}", self.add_slash)?;
+        }
+
+        writeln!(&mut writer, "{}", self.no_recursion)?;
+
+        if config.scan_limit > 0 {
+            writeln!(&mut writer, "{}", self.scan_limit)?;
+        }
+
+        if config.parallel > 0 {
+            writeln!(&mut writer, "{}", self.parallel)?;
+        }
+
+        if config.rate_limit > 0 {
+            writeln!(&mut writer, "{}", self.rate_limit)?;
+        }
+
+        if !config.time_limit.is_empty() {
+            writeln!(&mut writer, "{}", self.time_limit)?;
+        }
+
+        if matches!(self.update_status, UpdateStatus::OutOfDate) {
+            let update = BannerEntry::new(
+                "🎉",
+                "New Version Available",
+                "https://github.com/epi052/feroxbuster/releases/latest",
+            );
+            writeln!(&mut writer, "{}", update)?;
+        }
+
+        writeln!(&mut writer, "{}", self.footer())?;
+
+        Ok(())
+    }
+}

src/banner/entry.rs

@@ -0,0 +1,57 @@
+use console::{measure_text_width, Emoji};
+use std::fmt;
+
+/// Initial visual indentation size used in formatting banner entries
+const INDENT: usize = 3;
+
+/// Column width used in formatting banner entries
+const COL_WIDTH: usize = 22;
+
+/// Represents a single line on the banner
+#[derive(Default)]
+pub(super) struct BannerEntry {
+    /// emoji used in the banner entry
+    emoji: String,
+
+    /// title used in the banner entry
+    title: String,
+
+    /// value passed in via config/cli/defaults
+    value: String,
+}
+
+/// implementation of a banner entry
+impl BannerEntry {
+    /// Create a new banner entry from given fields
+    pub fn new(emoji: &str, title: &str, value: &str) -> Self {
+        BannerEntry {
+            emoji: emoji.to_string(),
+            title: title.to_string(),
+            value: value.to_string(),
+        }
+    }
+
+    /// Simple wrapper for emoji or fallback when terminal doesn't support emoji
+    fn format_emoji(&self) -> String {
+        let width = measure_text_width(&self.emoji);
+        let pad_len = width * width;
+        let pad = format!("{:<pad_len$}", "\u{0020}", pad_len = pad_len);
+        Emoji(&self.emoji, &pad).to_string()
+    }
+}
+
+/// Display implementation for a banner entry
+impl fmt::Display for BannerEntry {
+    /// Display formatter for the given banner entry
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "\u{0020}{:\u{0020}<indent$}{:\u{0020}<width$}\u{2502}\u{0020}{}",
+            self.format_emoji(),
+            self.title,
+            self.value,
+            indent = INDENT,
+            width = COL_WIDTH
+        )
+    }
+}

src/banner/mod.rs

@@ -0,0 +1,8 @@
+//! all logic related to building/printing the banner seen when scans start
+mod container;
+mod entry;
+
+#[cfg(test)]
+mod tests;
+
+pub use self::container::{Banner, UPDATE_URL};

src/banner/tests.rs

@@ -0,0 +1,174 @@
+use super::container::UpdateStatus;
+use super::*;
+use crate::{config::Configuration, event_handlers::Handles, scan_manager::FeroxScans};
+use httpmock::Method::GET;
+use httpmock::MockServer;
+use std::{io::stderr, sync::Arc, time::Duration};
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test to hit no execution of targets for loop in banner
+async fn banner_intialize_without_targets() {
+    let config = Configuration::new().unwrap();
+    let banner = Banner::new(&[], &config);
+    banner.print_to(stderr(), Arc::new(config)).unwrap();
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test to hit no execution of statuscode for loop in banner
+async fn banner_intialize_without_status_codes() {
+    let config = Configuration {
+        status_codes: vec![],
+        ..Default::default()
+    };
+
+    let banner = Banner::new(&[String::from("http://localhost")], &config);
+    banner.print_to(stderr(), Arc::new(config)).unwrap();
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test to hit an empty config file
+async fn banner_intialize_without_config_file() {
+    let config = Configuration {
+        config: String::new(),
+        ..Default::default()
+    };
+
+    let banner = Banner::new(&[String::from("http://localhost")], &config);
+    banner.print_to(stderr(), Arc::new(config)).unwrap();
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test to hit an empty queries
+async fn banner_intialize_without_queries() {
+    let config = Configuration {
+        queries: vec![(String::new(), String::new())],
+        ..Default::default()
+    };
+
+    let banner = Banner::new(&[String::from("http://localhost")], &config);
+    banner.print_to(stderr(), Arc::new(config)).unwrap();
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test that
+async fn banner_needs_update_returns_unknown_with_bad_url() {
+    let handles = Arc::new(Handles::for_testing(None, None).0);
+
+    let mut banner = Banner::new(
+        &[String::from("http://localhost")],
+        &Configuration::new().unwrap(),
+    );
+
+    let _ = banner.check_for_updates("", handles).await;
+
+    assert!(matches!(banner.update_status, UpdateStatus::Unknown));
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test return value of good url to needs_update
+async fn banner_needs_update_returns_up_to_date() {
+    let srv = MockServer::start();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/latest");
+        then.status(200).body("{\"tag_name\":\"v1.1.0\"}");
+    });
+    let scans = Arc::new(FeroxScans::default());
+
+    let handles = Arc::new(Handles::for_testing(Some(scans), None).0);
+
+    let mut banner = Banner::new(&[srv.url("")], &Configuration::new().unwrap());
+    banner.version = String::from("1.1.0");
+
+    let _ = banner.check_for_updates(&srv.url("/latest"), handles).await;
+
+    assert_eq!(mock.hits(), 1);
+    assert!(matches!(banner.update_status, UpdateStatus::UpToDate));
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test return value of good url to needs_update that returns a newer version than current
+async fn banner_needs_update_returns_out_of_date() {
+    let srv = MockServer::start();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/latest");
+        then.status(200).body("{\"tag_name\":\"v1.1.0\"}");
+    });
+
+    let scans = Arc::new(FeroxScans::default());
+
+    let handles = Arc::new(Handles::for_testing(Some(scans), None).0);
+
+    let mut banner = Banner::new(&[srv.url("")], &Configuration::new().unwrap());
+    banner.version = String::from("1.0.1");
+
+    let _ = banner.check_for_updates(&srv.url("/latest"), handles).await;
+
+    assert_eq!(mock.hits(), 1);
+    assert!(matches!(banner.update_status, UpdateStatus::OutOfDate));
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test return value of good url that times out
+async fn banner_needs_update_returns_unknown_on_timeout() {
+    let srv = MockServer::start();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/latest");
+        then.status(200)
+            .body("{\"tag_name\":\"v1.1.0\"}")
+            .delay(Duration::from_secs(8));
+    });
+
+    let handles = Arc::new(Handles::for_testing(None, None).0);
+
+    let mut banner = Banner::new(&[srv.url("")], &Configuration::new().unwrap());
+
+    let _ = banner.check_for_updates(&srv.url("/latest"), handles).await;
+
+    assert_eq!(mock.hits(), 1);
+    assert!(matches!(banner.update_status, UpdateStatus::Unknown));
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test return value of good url with bad json response
+async fn banner_needs_update_returns_unknown_on_bad_json_response() {
+    let srv = MockServer::start();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/latest");
+        then.status(200).body("not json");
+    });
+
+    let handles = Arc::new(Handles::for_testing(None, None).0);
+
+    let mut banner = Banner::new(&[srv.url("")], &Configuration::new().unwrap());
+
+    let _ = banner.check_for_updates(&srv.url("/latest"), handles).await;
+
+    assert_eq!(mock.hits(), 1);
+    assert!(matches!(banner.update_status, UpdateStatus::Unknown));
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test return value of good url with json response that lacks the tag_name field
+async fn banner_needs_update_returns_unknown_on_json_without_correct_tag() {
+    let srv = MockServer::start();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/latest");
+        then.status(200)
+            .body("{\"no tag_name\": \"doesn't exist\"}");
+    });
+
+    let handles = Arc::new(Handles::for_testing(None, None).0);
+
+    let mut banner = Banner::new(&[srv.url("")], &Configuration::new().unwrap());
+    banner.version = String::from("1.0.1");
+
+    let _ = banner.check_for_updates(&srv.url("/latest"), handles).await;
+
+    assert_eq!(mock.hits(), 1);
+    assert!(matches!(banner.update_status, UpdateStatus::Unknown));
+}

src/client.rs

@@ -1,10 +1,8 @@
-use crate::utils::{module_colorizer, status_colorizer};
+use anyhow::Result;
 use reqwest::header::HeaderMap;
 use reqwest::{redirect::Policy, Client, Proxy};
 use std::collections::HashMap;
 use std::convert::TryInto;
-#[cfg(not(test))]
-use std::process::exit;
 use std::time::Duration;
 
 /// Create and return an instance of [reqwest::Client](https://docs.rs/reqwest/latest/reqwest/struct.Client.html)
@@ -15,15 +13,14 @@ pub fn initialize(
     insecure: bool,
     headers: &HashMap<String, String>,
     proxy: Option<&str>,
-) -> Client {
+) -> Result<Client> {
     let policy = if redirects {
         Policy::limited(10)
     } else {
         Policy::none()
     };
 
-    // try_into returns infallible as its error, unwrap is safe here
-    let header_map: HeaderMap = headers.try_into().unwrap();
+    let header_map: HeaderMap = headers.try_into()?;
 
     let client = Client::builder()
         .timeout(Duration::new(timeout, 0))
@@ -32,51 +29,15 @@ pub fn initialize(
         .default_headers(header_map)
         .redirect(policy);
 
-    let client = match proxy {
-        // a proxy is specified, need to add it to the client
-        Some(some_proxy) => {
-            if !some_proxy.is_empty() {
-                // it's not an empty string
-                match Proxy::all(some_proxy) {
-                    Ok(proxy_obj) => client.proxy(proxy_obj),
-                    Err(e) => {
-                        eprintln!(
-                            "{} {} {}",
-                            status_colorizer("ERROR"),
-                            module_colorizer("Client::initialize"),
-                            e
-                        );
-
-                        #[cfg(test)]
-                        panic!();
-                        #[cfg(not(test))]
-                        exit(1);
-                    }
-                }
-            } else {
-                client // Some("") was used?
-            }
-        }
-        // no proxy specified
-        None => client,
-    };
-
-    match client.build() {
-        Ok(client) => client,
-        Err(e) => {
-            eprintln!(
-                "{} {} {}",
-                status_colorizer("ERROR"),
-                module_colorizer("Client::build"),
-                e
-            );
-
-            #[cfg(test)]
-            panic!();
-            #[cfg(not(test))]
-            exit(1);
+    if let Some(some_proxy) = proxy {
+        if !some_proxy.is_empty() {
+            // it's not an empty string; set the proxy
+            let proxy_obj = Proxy::all(some_proxy)?;
+            return Ok(client.proxy(proxy_obj).build()?);
         }
     }
+
+    Ok(client.build()?)
 }
 
 #[cfg(test)]
@@ -88,7 +49,7 @@ mod tests {
     /// create client with a bad proxy, expect panic
     fn client_with_bad_proxy() {
         let headers = HashMap::new();
-        initialize(0, "stuff", true, false, &headers, Some("not a valid proxy"));
+        initialize(0, "stuff", true, false, &headers, Some("not a valid proxy")).unwrap();
     }
 
     #[test]
@@ -96,6 +57,6 @@ mod tests {
     fn client_with_good_proxy() {
         let headers = HashMap::new();
         let proxy = "http://127.0.0.1:8080";
-        initialize(0, "stuff", true, true, &headers, Some(proxy));
+        initialize(0, "stuff", true, true, &headers, Some(proxy)).unwrap();
     }
 }

src/config/mod.rs

@@ -0,0 +1,9 @@
+//! all logic related to instantiating a running configuration
+
+mod container;
+mod utils;
+#[cfg(test)]
+mod tests;
+
+pub use self::container::Configuration;
+pub use self::utils::{determine_output_level, OutputLevel, RequesterPolicy};

src/config/tests.rs

@@ -0,0 +1,419 @@
+use super::utils::*;
+use super::*;
+use crate::{traits::FeroxSerialize, DEFAULT_CONFIG_NAME};
+use std::{collections::HashMap, fs::write};
+use tempfile::TempDir;
+
+/// creates a dummy configuration file for testing
+fn setup_config_test() -> Configuration {
+    let data = r#"
+            wordlist = "/some/path"
+            status_codes = [201, 301, 401]
+            replay_codes = [201, 301]
+            threads = 40
+            timeout = 5
+            proxy = "http://127.0.0.1:8080"
+            replay_proxy = "http://127.0.0.1:8081"
+            quiet = true
+            silent = true
+            auto_tune = true
+            auto_bail = true
+            verbosity = 1
+            scan_limit = 6
+            parallel = 14
+            rate_limit = 250
+            time_limit = "10m"
+            output = "/some/otherpath"
+            debug_log = "/yet/anotherpath"
+            resume_from = "/some/state/file"
+            redirects = true
+            insecure = true
+            extensions = ["html", "php", "js"]
+            url_denylist = ["http://dont-scan.me", "https://also-not.me"]
+            headers = {stuff = "things", mostuff = "mothings"}
+            queries = [["name","value"], ["rick", "astley"]]
+            no_recursion = true
+            add_slash = true
+            stdin = true
+            dont_filter = true
+            extract_links = true
+            json = true
+            save_state = false
+            depth = 1
+            filter_size = [4120]
+            filter_regex = ["^ignore me$"]
+            filter_similar = ["https://somesite.com/soft404"]
+            filter_word_count = [994, 992]
+            filter_line_count = [34]
+            filter_status = [201]
+        "#;
+    let tmp_dir = TempDir::new().unwrap();
+    let file = tmp_dir.path().join(DEFAULT_CONFIG_NAME);
+    write(&file, data).unwrap();
+    Configuration::parse_config(file).unwrap()
+}
+
+#[test]
+/// test that all default config values meet expectations
+fn default_configuration() {
+    let config = Configuration::default();
+    assert_eq!(config.wordlist, wordlist());
+    assert_eq!(config.proxy, String::new());
+    assert_eq!(config.target_url, String::new());
+    assert_eq!(config.time_limit, String::new());
+    assert_eq!(config.resume_from, String::new());
+    assert_eq!(config.debug_log, String::new());
+    assert_eq!(config.config, String::new());
+    assert_eq!(config.replay_proxy, String::new());
+    assert_eq!(config.status_codes, status_codes());
+    assert_eq!(config.replay_codes, config.status_codes);
+    assert!(config.replay_client.is_none());
+    assert_eq!(config.threads, threads());
+    assert_eq!(config.depth, depth());
+    assert_eq!(config.timeout, timeout());
+    assert_eq!(config.verbosity, 0);
+    assert_eq!(config.scan_limit, 0);
+    assert!(!config.silent);
+    assert!(!config.quiet);
+    assert_eq!(config.output_level, OutputLevel::Default);
+    assert!(!config.dont_filter);
+    assert!(!config.auto_tune);
+    assert!(!config.auto_bail);
+    assert_eq!(config.requester_policy, RequesterPolicy::Default);
+    assert!(!config.no_recursion);
+    assert!(!config.json);
+    assert!(config.save_state);
+    assert!(!config.stdin);
+    assert!(!config.add_slash);
+    assert!(!config.redirects);
+    assert!(!config.extract_links);
+    assert!(!config.insecure);
+    assert_eq!(config.queries, Vec::new());
+    assert_eq!(config.filter_size, Vec::<u64>::new());
+    assert_eq!(config.extensions, Vec::<String>::new());
+    assert_eq!(config.url_denylist, Vec::<String>::new());
+    assert_eq!(config.filter_regex, Vec::<String>::new());
+    assert_eq!(config.filter_similar, Vec::<String>::new());
+    assert_eq!(config.filter_word_count, Vec::<usize>::new());
+    assert_eq!(config.filter_line_count, Vec::<usize>::new());
+    assert_eq!(config.filter_status, Vec::<u16>::new());
+    assert_eq!(config.headers, HashMap::new());
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_wordlist() {
+    let config = setup_config_test();
+    assert_eq!(config.wordlist, "/some/path");
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_debug_log() {
+    let config = setup_config_test();
+    assert_eq!(config.debug_log, "/yet/anotherpath");
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_status_codes() {
+    let config = setup_config_test();
+    assert_eq!(config.status_codes, vec![201, 301, 401]);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_replay_codes() {
+    let config = setup_config_test();
+    assert_eq!(config.replay_codes, vec![201, 301]);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_threads() {
+    let config = setup_config_test();
+    assert_eq!(config.threads, 40);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_depth() {
+    let config = setup_config_test();
+    assert_eq!(config.depth, 1);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_scan_limit() {
+    let config = setup_config_test();
+    assert_eq!(config.scan_limit, 6);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_parallel() {
+    let config = setup_config_test();
+    assert_eq!(config.parallel, 14);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_rate_limit() {
+    let config = setup_config_test();
+    assert_eq!(config.rate_limit, 250);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_timeout() {
+    let config = setup_config_test();
+    assert_eq!(config.timeout, 5);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_proxy() {
+    let config = setup_config_test();
+    assert_eq!(config.proxy, "http://127.0.0.1:8080");
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_replay_proxy() {
+    let config = setup_config_test();
+    assert_eq!(config.replay_proxy, "http://127.0.0.1:8081");
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_silent() {
+    let config = setup_config_test();
+    assert!(config.silent);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_quiet() {
+    let config = setup_config_test();
+    assert!(config.quiet);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_json() {
+    let config = setup_config_test();
+    assert!(config.json);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_auto_bail() {
+    let config = setup_config_test();
+    assert!(config.auto_bail);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_auto_tune() {
+    let config = setup_config_test();
+    assert!(config.auto_tune);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_verbosity() {
+    let config = setup_config_test();
+    assert_eq!(config.verbosity, 1);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_output() {
+    let config = setup_config_test();
+    assert_eq!(config.output, "/some/otherpath");
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_redirects() {
+    let config = setup_config_test();
+    assert!(config.redirects);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_insecure() {
+    let config = setup_config_test();
+    assert!(config.insecure);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_no_recursion() {
+    let config = setup_config_test();
+    assert!(config.no_recursion);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_stdin() {
+    let config = setup_config_test();
+    assert!(config.stdin);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_dont_filter() {
+    let config = setup_config_test();
+    assert!(config.dont_filter);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_add_slash() {
+    let config = setup_config_test();
+    assert!(config.add_slash);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_extract_links() {
+    let config = setup_config_test();
+    assert!(config.extract_links);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_extensions() {
+    let config = setup_config_test();
+    assert_eq!(config.extensions, vec!["html", "php", "js"]);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_url_denylist() {
+    let config = setup_config_test();
+    assert_eq!(
+        config.url_denylist,
+        vec!["http://dont-scan.me", "https://also-not.me"]
+    );
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_filter_regex() {
+    let config = setup_config_test();
+    assert_eq!(config.filter_regex, vec!["^ignore me$"]);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_filter_similar() {
+    let config = setup_config_test();
+    assert_eq!(config.filter_similar, vec!["https://somesite.com/soft404"]);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_filter_size() {
+    let config = setup_config_test();
+    assert_eq!(config.filter_size, vec![4120]);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_filter_word_count() {
+    let config = setup_config_test();
+    assert_eq!(config.filter_word_count, vec![994, 992]);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_filter_line_count() {
+    let config = setup_config_test();
+    assert_eq!(config.filter_line_count, vec![34]);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_filter_status() {
+    let config = setup_config_test();
+    assert_eq!(config.filter_status, vec![201]);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_save_state() {
+    let config = setup_config_test();
+    assert!(!config.save_state);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_time_limit() {
+    let config = setup_config_test();
+    assert_eq!(config.time_limit, "10m");
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_resume_from() {
+    let config = setup_config_test();
+    assert_eq!(config.resume_from, "/some/state/file");
+}
+
+#[test]
+/// parse the test config and see that the values parsed are correct
+fn config_reads_headers() {
+    let config = setup_config_test();
+    let mut headers = HashMap::new();
+    headers.insert("stuff".to_string(), "things".to_string());
+    headers.insert("mostuff".to_string(), "mothings".to_string());
+    assert_eq!(config.headers, headers);
+}
+
+#[test]
+/// parse the test config and see that the values parsed are correct
+fn config_reads_queries() {
+    let config = setup_config_test();
+    let queries = vec![
+        ("name".to_string(), "value".to_string()),
+        ("rick".to_string(), "astley".to_string()),
+    ];
+    assert_eq!(config.queries, queries);
+}
+
+#[test]
+#[should_panic]
+/// test that an error message is printed and panic is called when report_and_exit is called
+fn config_report_and_exit_works() {
+    report_and_exit("some message");
+}
+
+#[test]
+/// test as_str method of Configuration
+fn as_str_returns_string_with_newline() {
+    let config = Configuration::new().unwrap();
+    let config_str = config.as_str();
+    println!("{}", config_str);
+    assert!(config_str.starts_with("Configuration {"));
+    assert!(config_str.ends_with("}\n"));
+    assert!(config_str.contains("replay_codes:"));
+    assert!(config_str.contains("client: Client {"));
+    assert!(config_str.contains("user_agent: \"feroxbuster"));
+}
+
+#[test]
+/// test as_json method of Configuration
+fn as_json_returns_json_representation_of_configuration_with_newline() {
+    let mut config = Configuration::new().unwrap();
+    config.timeout = 12;
+    config.depth = 2;
+    let config_str = config.as_json().unwrap();
+    let json: Configuration = serde_json::from_str(&config_str).unwrap();
+    assert_eq!(json.config, config.config);
+    assert_eq!(json.wordlist, config.wordlist);
+    assert_eq!(json.replay_codes, config.replay_codes);
+    assert_eq!(json.timeout, config.timeout);
+    assert_eq!(json.depth, config.depth);
+}

src/config/utils.rs

@@ -0,0 +1,182 @@
+use crate::{
+    utils::{module_colorizer, status_colorizer},
+    DEFAULT_STATUS_CODES, DEFAULT_WORDLIST, VERSION,
+};
+#[cfg(not(test))]
+use std::process::exit;
+
+/// simple helper to clean up some code reuse below; panics under test / exits in prod
+pub(super) fn report_and_exit(err: &str) -> ! {
+    eprintln!(
+        "{} {}: {}",
+        status_colorizer("ERROR"),
+        module_colorizer("Configuration::new"),
+        err
+    );
+
+    #[cfg(test)]
+    panic!();
+    #[cfg(not(test))]
+    exit(1);
+}
+
+// functions timeout, threads, status_codes, user_agent, wordlist, save_state, and depth are used to provide
+// defaults in the event that a ferox-config.toml is found but one or more of the values below
+// aren't listed in the config.  This way, we get the correct defaults upon Deserialization
+
+/// default Configuration type for use in json output
+pub(super) fn serialized_type() -> String {
+    String::from("configuration")
+}
+
+/// default timeout value
+pub(super) fn timeout() -> u64 {
+    7
+}
+
+/// default save_state value
+pub(super) fn save_state() -> bool {
+    true
+}
+
+/// default threads value
+pub(super) fn threads() -> usize {
+    50
+}
+
+/// default status codes
+pub(super) fn status_codes() -> Vec<u16> {
+    DEFAULT_STATUS_CODES
+        .iter()
+        .map(|code| code.as_u16())
+        .collect()
+}
+
+/// default wordlist
+pub(super) fn wordlist() -> String {
+    String::from(DEFAULT_WORDLIST)
+}
+
+/// default user-agent
+pub(super) fn user_agent() -> String {
+    format!("feroxbuster/{}", VERSION)
+}
+
+/// default recursion depth
+pub(super) fn depth() -> usize {
+    4
+}
+
+/// enum representing the three possible states for informational output (not logging verbosity)
+#[derive(Debug, Copy, Clone, PartialEq)]
+pub enum OutputLevel {
+    /// normal scan, no --quiet|--silent
+    Default,
+
+    /// quiet scan, print some information, but not all (new in versions >= 2.0.0)
+    Quiet,
+
+    /// silent scan, only print urls (used to be --quiet in versions 1.x.x)
+    Silent,
+}
+
+/// implement a default for OutputLevel
+impl Default for OutputLevel {
+    /// return Default
+    fn default() -> Self {
+        Self::Default
+    }
+}
+
+/// given the current settings for quiet and silent, determine output_level (DRY helper)
+pub fn determine_output_level(quiet: bool, silent: bool) -> OutputLevel {
+    if quiet && silent {
+        // user COULD have both as true in config file, take the more quiet of the two
+        OutputLevel::Silent
+    } else if quiet {
+        OutputLevel::Quiet
+    } else if silent {
+        OutputLevel::Silent
+    } else {
+        OutputLevel::Default
+    }
+}
+
+/// represents actions the Requester should take in certain situations
+#[derive(Debug, PartialEq, Copy, Clone)]
+pub enum RequesterPolicy {
+    /// automatically try to lower request rate in order to reduce errors
+    AutoTune,
+
+    /// automatically bail at certain error thresholds
+    AutoBail,
+
+    /// just let that junk run super natural
+    Default,
+}
+
+/// default implementation for RequesterPolicy
+impl Default for RequesterPolicy {
+    /// Default as default
+    fn default() -> Self {
+        Self::Default
+    }
+}
+
+/// given the current settings for quiet and silent, determine output_level (DRY helper)
+pub fn determine_requester_policy(auto_tune: bool, auto_bail: bool) -> RequesterPolicy {
+    if auto_tune && auto_bail {
+        // user COULD have both as true in config file, take the more aggressive of the two
+        RequesterPolicy::AutoBail
+    } else if auto_tune {
+        RequesterPolicy::AutoTune
+    } else if auto_bail {
+        RequesterPolicy::AutoBail
+    } else {
+        RequesterPolicy::Default
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    /// test determine_output_level returns higher of the two levels if both given values are true
+    fn determine_output_level_returns_correct_results() {
+        let mut level = determine_output_level(true, true);
+        assert_eq!(level, OutputLevel::Silent);
+
+        level = determine_output_level(false, true);
+        assert_eq!(level, OutputLevel::Silent);
+
+        level = determine_output_level(false, false);
+        assert_eq!(level, OutputLevel::Default);
+
+        level = determine_output_level(true, false);
+        assert_eq!(level, OutputLevel::Quiet);
+    }
+
+    #[test]
+    /// test determine_requester_policy returns higher of the two levels if both given values are true
+    fn determine_requester_policy_returns_correct_results() {
+        let mut level = determine_requester_policy(true, true);
+        assert_eq!(level, RequesterPolicy::AutoBail);
+
+        level = determine_requester_policy(false, true);
+        assert_eq!(level, RequesterPolicy::AutoBail);
+
+        level = determine_requester_policy(false, false);
+        assert_eq!(level, RequesterPolicy::Default);
+
+        level = determine_requester_policy(true, false);
+        assert_eq!(level, RequesterPolicy::AutoTune);
+    }
+
+    #[test]
+    #[should_panic]
+    /// report_and_exit should panic/exit when called
+    fn report_and_exit_panics_under_test() {
+        report_and_exit("test");
+    }
+}

src/event_handlers/command.rs

@@ -0,0 +1,68 @@
+use std::sync::Arc;
+
+use reqwest::StatusCode;
+use tokio::sync::oneshot::Sender;
+
+use crate::response::FeroxResponse;
+use crate::{
+    statistics::{StatError, StatField},
+    traits::FeroxFilter,
+};
+
+/// Protocol definition for updating an event handler via mpsc
+#[derive(Debug)]
+pub enum Command {
+    /// Add one to the total number of requests
+    AddRequest,
+
+    /// Add one to the proper field(s) based on the given `StatError`
+    AddError(StatError),
+
+    /// Add one to the proper field(s) based on the given `StatusCode`
+    AddStatus(StatusCode),
+
+    /// Create the progress bar (`BarType::Total`) that is updated from the stats thread
+    CreateBar,
+
+    /// Add to a `Stats` field that corresponds to the given `StatField` by the given `usize` value
+    AddToUsizeField(StatField, usize),
+
+    /// Subtract from a `Stats` field that corresponds to the given `StatField` by the given `usize` value
+    SubtractFromUsizeField(StatField, usize),
+
+    /// Update a `Stats` field that corresponds to the given `StatField` by the given `f64` value
+    AddToF64Field(StatField, f64),
+
+    /// Save a `Stats` object to disk using `reporter::get_cached_file_handle`
+    Save,
+
+    /// Load a `Stats` object from disk
+    LoadStats(String),
+
+    /// Add a `FeroxFilter` implementor to `FilterHandler`'s instance of `FeroxFilters`
+    AddFilter(Box<dyn FeroxFilter>),
+
+    /// Send a `FeroxResponse` to the output handler for reporting
+    Report(Box<FeroxResponse>),
+
+    /// Send a group of urls to be scanned (only used for the urls passed in explicitly by the user)
+    ScanInitialUrls(Vec<String>),
+
+    /// Determine whether or not recursion is appropriate, given a FeroxResponse, if so start a scan
+    TryRecursion(Box<FeroxResponse>),
+
+    /// Send a pointer to the wordlist to the recursion handler
+    UpdateWordlist(Arc<Vec<String>>),
+
+    /// Instruct the ScanHandler to join on all known scans, use sender to notify main when done
+    JoinTasks(Sender<bool>),
+
+    /// Command used to test that a spawned task succeeded in initialization
+    Ping,
+
+    /// Just receive a sender and reply, used for slowing down the main thread
+    Sync(Sender<bool>),
+
+    /// Break out of the (infinite) mpsc receive loop
+    Exit,
+}

src/event_handlers/container.rs

@@ -0,0 +1,135 @@
+use super::*;
+use crate::config::Configuration;
+use crate::event_handlers::scans::ScanHandle;
+use crate::scan_manager::FeroxScans;
+use crate::Joiner;
+#[cfg(test)]
+use crate::{filters::FeroxFilters, statistics::Stats, Command};
+use anyhow::{bail, Result};
+use std::sync::{Arc, RwLock};
+#[cfg(test)]
+use tokio::sync::mpsc::{self, UnboundedReceiver};
+
+#[derive(Debug)]
+/// Simple container for multiple JoinHandles
+pub struct Tasks {
+    /// JoinHandle for terminal handler
+    pub terminal: Joiner,
+
+    /// JoinHandle for statistics handler
+    pub stats: Joiner,
+
+    /// JoinHandle for filters handler
+    pub filters: Joiner,
+
+    /// JoinHandle for scans handler
+    pub scans: Joiner,
+}
+
+/// Tasks implementation
+impl Tasks {
+    /// Given JoinHandles for terminal, statistics, and filters create a new Tasks object
+    pub fn new(terminal: Joiner, stats: Joiner, filters: Joiner, scans: Joiner) -> Self {
+        Self {
+            terminal,
+            stats,
+            filters,
+            scans,
+        }
+    }
+}
+
+#[derive(Debug)]
+/// Container for the different *Handles that will be shared across modules
+pub struct Handles {
+    /// Handle for statistics
+    pub stats: StatsHandle,
+
+    /// Handle for filters
+    pub filters: FiltersHandle,
+
+    /// Handle for output (terminal/file)
+    pub output: TermOutHandle,
+
+    /// Handle for Configuration
+    pub config: Arc<Configuration>,
+
+    /// Handle for recursion
+    pub scans: RwLock<Option<ScanHandle>>,
+}
+
+/// implementation of Handles
+impl Handles {
+    /// Given a StatsHandle, FiltersHandle, and OutputHandle, create a Handles object
+    pub fn new(
+        stats: StatsHandle,
+        filters: FiltersHandle,
+        output: TermOutHandle,
+        config: Arc<Configuration>,
+    ) -> Self {
+        Self {
+            stats,
+            filters,
+            output,
+            config,
+            scans: RwLock::new(None),
+        }
+    }
+
+    /// create a Handles object suitable for unit testing (non-functional)
+    #[cfg(test)]
+    pub fn for_testing(
+        scanned_urls: Option<Arc<FeroxScans>>,
+        config: Option<Arc<Configuration>>,
+    ) -> (Self, UnboundedReceiver<Command>) {
+        let configuration = config.unwrap_or_else(|| Arc::new(Configuration::new().unwrap()));
+        let (tx, rx) = mpsc::unbounded_channel::<Command>();
+        let terminal_handle = TermOutHandle::new(tx.clone(), tx.clone());
+        let stats_handle = StatsHandle::new(
+            Arc::new(Stats::new(
+                configuration.extensions.len(),
+                configuration.json,
+            )),
+            tx.clone(),
+        );
+        let filters_handle = FiltersHandle::new(Arc::new(FeroxFilters::default()), tx.clone());
+        let handles = Self::new(stats_handle, filters_handle, terminal_handle, configuration);
+        if let Some(sh) = scanned_urls {
+            let scan_handle = ScanHandle::new(sh, tx);
+            handles.set_scan_handle(scan_handle);
+        }
+        (handles, rx)
+    }
+
+    /// Set the ScanHandle object
+    pub fn set_scan_handle(&self, handle: ScanHandle) {
+        if let Ok(mut guard) = self.scans.write() {
+            if guard.is_none() {
+                let _ = std::mem::replace(&mut *guard, Some(handle));
+            }
+        }
+    }
+
+    /// Helper to easily send a Command over the (locked) underlying CommandSender object
+    pub fn send_scan_command(&self, command: Command) -> Result<()> {
+        if let Ok(guard) = self.scans.read().as_ref() {
+            if let Some(handle) = guard.as_ref() {
+                handle.send(command)?;
+                return Ok(());
+            }
+        }
+
+        bail!("Could not get underlying CommandSender object")
+    }
+
+    /// Helper to easily get the (locked) underlying FeroxScans object
+    pub fn ferox_scans(&self) -> Result<Arc<FeroxScans>> {
+        if let Ok(guard) = self.scans.read().as_ref() {
+            if let Some(handle) = guard.as_ref() {
+                return Ok(handle.data.clone());
+            }
+        }
+
+        bail!("Could not get underlying FeroxScans")
+    }
+}

src/event_handlers/filters.rs

@@ -0,0 +1,101 @@
+use super::*;
+use crate::{filters::FeroxFilters, CommandSender, FeroxChannel, Joiner};
+use anyhow::Result;
+use std::sync::Arc;
+use tokio::sync::{
+    mpsc::{self, UnboundedReceiver},
+    oneshot,
+};
+
+#[derive(Debug)]
+/// Container for filters transmitter and FeroxFilters object
+pub struct FiltersHandle {
+    /// FeroxFilters object used across modules to track active filters
+    pub data: Arc<FeroxFilters>,
+
+    /// transmitter used to update `data`
+    pub tx: CommandSender,
+}
+
+/// implementation of FiltersHandle
+impl FiltersHandle {
+    /// Given an Arc-wrapped FeroxFilters and CommandSender, create a new FiltersHandle
+    pub fn new(data: Arc<FeroxFilters>, tx: CommandSender) -> Self {
+        Self { data, tx }
+    }
+
+    /// Send the given Command over `tx`
+    pub fn send(&self, command: Command) -> Result<()> {
+        self.tx.send(command)?;
+        Ok(())
+    }
+
+    /// Sync the handle with the handler
+    pub async fn sync(&self) -> Result<()> {
+        let (tx, rx) = oneshot::channel::<bool>();
+        self.send(Command::Sync(tx))?;
+        rx.await?;
+        Ok(())
+    }
+}
+
+/// event handler for updating a single data structure of all active filters
+#[derive(Debug)]
+pub struct FiltersHandler {
+    /// collection of FeroxFilters
+    data: Arc<FeroxFilters>,
+
+    /// Receiver half of mpsc from which `Command`s are processed
+    receiver: UnboundedReceiver<Command>,
+}
+
+/// implementation of event handler for filters
+impl FiltersHandler {
+    /// create new event handler
+    pub fn new(data: Arc<FeroxFilters>, receiver: UnboundedReceiver<Command>) -> Self {
+        Self { data, receiver }
+    }
+
+    /// Initialize new `FeroxFilters` and the sc side of an mpsc channel that is responsible for
+    /// updates to the aforementioned object.
+    pub fn initialize() -> (Joiner, FiltersHandle) {
+        log::trace!("enter: initialize");
+
+        let data = Arc::new(FeroxFilters::default());
+        let (tx, rx): FeroxChannel<Command> = mpsc::unbounded_channel();
+
+        let mut handler = Self::new(data.clone(), rx);
+
+        let task = tokio::spawn(async move { handler.start().await });
+
+        let event_handle = FiltersHandle::new(data, tx);
+
+        log::trace!("exit: initialize -> ({:?}, {:?})", task, event_handle);
+
+        (task, event_handle)
+    }
+
+    /// Start a single consumer task (sc side of mpsc)
+    ///
+    /// The consumer simply receives `Command` and acts accordingly
+    pub async fn start(&mut self) -> Result<()> {
+        log::trace!("enter: start({:?})", self);
+
+        while let Some(command) = self.receiver.recv().await {
+            match command {
+                Command::AddFilter(filter) => {
+                    self.data.push(filter)?;
+                }
+                Command::Sync(sender) => {
+                    log::debug!("filters: {:?}", self);
+                    sender.send(true).unwrap_or_default();
+                }
+                Command::Exit => break,
+                _ => {} // no other commands needed for FilterHandler
+            }
+        }
+
+        log::trace!("exit: start");
+        Ok(())
+    }
+}

src/event_handlers/inputs.rs

@@ -0,0 +1,144 @@
+use super::*;
+use crate::{
+    progress::PROGRESS_PRINTER,
+    scan_manager::{FeroxState, PAUSE_SCAN},
+    scanner::RESPONSES,
+    statistics::StatError,
+    utils::slugify_filename,
+    utils::{open_file, write_to},
+    SLEEP_DURATION,
+};
+use anyhow::Result;
+use console::style;
+use crossterm::event::{self, Event, KeyCode};
+use std::{
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        Arc,
+    },
+    thread::sleep,
+    time::Duration,
+};
+
+/// Atomic boolean flag, used to determine whether or not the terminal input handler should exit
+pub static SCAN_COMPLETE: AtomicBool = AtomicBool::new(false);
+
+/// Container for filters transmitter and FeroxFilters object
+pub struct TermInputHandler {
+    /// handles to other handlers
+    handles: Arc<Handles>,
+}
+
+/// implementation of event handler for terminal input
+///
+/// kicks off the following handlers related to terminal input:
+///     ctrl+c handler that saves scan state to disk
+///     enter handler that listens for enter during scans to drop into interactive scan cancel menu
+impl TermInputHandler {
+    /// Create new event handler
+    pub fn new(handles: Arc<Handles>) -> Self {
+        Self { handles }
+    }
+
+    /// Initialize the sigint and enter handlers that are responsible for handling initial user
+    /// interaction during scans
+    pub fn initialize(handles: Arc<Handles>) {
+        log::trace!("enter: initialize({:?})", handles);
+
+        let handler = Self::new(handles);
+        handler.start();
+
+        log::trace!("exit: initialize");
+    }
+
+    /// wrapper around sigint_handler and enter_handler
+    fn start(&self) {
+        tokio::task::spawn_blocking(Self::enter_handler);
+
+        if self.handles.config.save_state {
+            // start the ctrl+c handler
+            let cloned = self.handles.clone();
+
+            let result = ctrlc::set_handler(move || {
+                let _ = Self::sigint_handler(cloned.clone());
+            });
+
+            if result.is_err() {
+                log::warn!("Could not set Ctrl+c handler; scan state will not be saved");
+                self.handles
+                    .stats
+                    .send(Command::AddError(StatError::Other))
+                    .unwrap_or_default();
+            }
+        }
+    }
+
+    /// Writes the current state of the program to disk (if save_state is true) and then exits
+    pub fn sigint_handler(handles: Arc<Handles>) -> Result<()> {
+        log::trace!("enter: sigint_handler({:?})", handles);
+
+        let filename = if !handles.config.target_url.is_empty() {
+            // target url populated
+            slugify_filename(&handles.config.target_url, "ferox", "state")
+        } else {
+            // stdin used
+            slugify_filename("stdin", "ferox", "state")
+        };
+
+        let warning = format!(
+            "🚨 Caught {} 🚨 saving scan state to {} ...",
+            style("ctrl+c").yellow(),
+            filename
+        );
+
+        PROGRESS_PRINTER.println(warning);
+
+        let state = FeroxState::new(
+            handles.ferox_scans()?,
+            handles.config.clone(),
+            &RESPONSES,
+            handles.stats.data.clone(),
+        );
+
+        let state_file = open_file(&filename);
+
+        let mut buffered_file = state_file?;
+        write_to(&state, &mut buffered_file, true)?;
+
+        log::trace!("exit: sigint_handler (end of program)");
+        std::process::exit(1);
+    }
+
+    /// Handles specific key events triggered by the user over stdin
+    fn enter_handler() {
+        // todo eventually move away from atomics, the blocking recv is the problem
+        log::trace!("enter: start_enter_handler");
+
+        loop {
+            if PAUSE_SCAN.load(Ordering::Relaxed) {
+                // if the scan is already paused, we don't want this event poller fighting the user
+                // over stdin
+                sleep(Duration::from_millis(SLEEP_DURATION));
+            } else if event::poll(Duration::from_millis(SLEEP_DURATION)).unwrap_or(false) {
+                // It's guaranteed that the `read()` won't block when the `poll()`
+                // function returns `true`
+
+                if let Ok(key_pressed) = event::read() {
+                    // ignore any other keys
+                    if key_pressed == Event::Key(KeyCode::Enter.into()) {
+                        // if the user presses Enter, set PAUSE_SCAN to true. The interactive menu
+                        // will be triggered and will handle setting PAUSE_SCAN to false
+                        PAUSE_SCAN.store(true, Ordering::Release);
+                    }
+                }
+            } else {
+                // Timeout expired and no `Event` is available; use the timeout to check SCAN_COMPLETE
+                if SCAN_COMPLETE.load(Ordering::Relaxed) {
+                    // scan has been marked complete by main, time to exit the loop
+                    break;
+                }
+            }
+        }
+        log::trace!("exit: start_enter_handler");
+    }
+}

src/event_handlers/mod.rs

@@ -0,0 +1,16 @@
+//! collection of event handlers (typically long-running tokio spawned tasks)
+mod statistics;
+mod filters;
+mod container;
+mod command;
+mod outputs;
+mod scans;
+mod inputs;
+
+pub use self::command::Command;
+pub use self::container::{Handles, Tasks};
+pub use self::filters::{FiltersHandle, FiltersHandler};
+pub use self::inputs::{TermInputHandler, SCAN_COMPLETE};
+pub use self::outputs::{TermOutHandle, TermOutHandler};
+pub use self::scans::{ScanHandle, ScanHandler};
+pub use self::statistics::{StatsHandle, StatsHandler};

src/event_handlers/outputs.rs

@@ -0,0 +1,286 @@
+use super::Command::AddToUsizeField;
+use super::*;
+
+use anyhow::{Context, Result};
+use tokio::sync::{mpsc, oneshot};
+
+use crate::{
+    config::Configuration,
+    progress::PROGRESS_PRINTER,
+    scanner::RESPONSES,
+    send_command, skip_fail,
+    statistics::StatField::ResourcesDiscovered,
+    traits::FeroxSerialize,
+    utils::{ferox_print, fmt_err, make_request, open_file, write_to},
+    CommandReceiver, CommandSender, Joiner,
+};
+use std::sync::Arc;
+
+#[derive(Debug)]
+/// Container for terminal output transmitter
+pub struct TermOutHandle {
+    /// Transmitter that sends to the TermOutHandler handler
+    pub tx: CommandSender,
+
+    /// Transmitter that sends to the FileOutHandler handler
+    pub tx_file: CommandSender,
+}
+
+/// implementation of OutputHandle
+impl TermOutHandle {
+    /// Given a CommandSender, create a new OutputHandle
+    pub fn new(tx: CommandSender, tx_file: CommandSender) -> Self {
+        Self { tx, tx_file }
+    }
+
+    /// Send the given Command over `tx`
+    pub fn send(&self, command: Command) -> Result<()> {
+        self.tx.send(command)?;
+        Ok(())
+    }
+
+    /// Sync the handle with the handler
+    pub async fn sync(&self, send_to_file: bool) -> Result<()> {
+        let (tx, rx) = oneshot::channel::<bool>();
+        self.send(Command::Sync(tx))?;
+
+        if send_to_file {
+            let (tx, rx) = oneshot::channel::<bool>();
+            self.tx_file.send(Command::Sync(tx))?;
+            rx.await?;
+        }
+
+        rx.await?;
+        Ok(())
+    }
+}
+
+#[derive(Debug)]
+/// Event handler for files
+pub struct FileOutHandler {
+    /// file output handler's receiver
+    receiver: CommandReceiver,
+
+    /// pointer to "global" configuration struct
+    config: Arc<Configuration>,
+}
+
+impl FileOutHandler {
+    /// Given a file tx/rx pair along with a filename and awaitable task, create
+    /// a FileOutHandler
+    fn new(rx: CommandReceiver, config: Arc<Configuration>) -> Self {
+        Self {
+            receiver: rx,
+            config,
+        }
+    }
+
+    /// Spawn a single consumer task (sc side of mpsc)
+    ///
+    /// The consumer simply receives responses from the terminal handler and writes them to disk
+    async fn start(&mut self, tx_stats: CommandSender) -> Result<()> {
+        log::trace!("enter: start_file_handler({:?})", tx_stats);
+
+        let mut file = open_file(&self.config.output)?;
+
+        log::info!("Writing scan results to {}", self.config.output);
+
+        while let Some(command) = self.receiver.recv().await {
+            match command {
+                Command::Report(response) => {
+                    skip_fail!(write_to(&*response, &mut file, self.config.json));
+                }
+                Command::Exit => {
+                    break;
+                }
+                Command::Sync(sender) => {
+                    skip_fail!(sender.send(true));
+                }
+                _ => {} // no more needed
+            }
+        }
+
+        // close the file before we tell statistics to save current data to the same file
+        drop(file);
+
+        send_command!(tx_stats, Command::Save);
+
+        log::trace!("exit: start_file_handler");
+        Ok(())
+    }
+}
+
+#[derive(Debug)]
+/// Event handler for terminal
+pub struct TermOutHandler {
+    /// terminal output handler's receiver
+    receiver: CommandReceiver,
+
+    /// file handler
+    tx_file: CommandSender,
+
+    /// optional file handler task
+    file_task: Option<Joiner>,
+
+    /// pointer to "global" configuration struct
+    config: Arc<Configuration>,
+}
+
+/// implementation of TermOutHandler
+impl TermOutHandler {
+    /// Given a terminal receiver along with a file transmitter and filename, create
+    /// an OutputHandler
+    fn new(
+        receiver: CommandReceiver,
+        tx_file: CommandSender,
+        file_task: Option<Joiner>,
+        config: Arc<Configuration>,
+    ) -> Self {
+        Self {
+            receiver,
+            tx_file,
+            file_task,
+            config,
+        }
+    }
+
+    /// Creates all required output handlers (terminal, file) and updates the given Handles/Tasks
+    pub fn initialize(
+        config: Arc<Configuration>,
+        tx_stats: CommandSender,
+    ) -> (Joiner, TermOutHandle) {
+        log::trace!("enter: initialize({:?}, {:?})", config, tx_stats);
+
+        let (tx_term, rx_term) = mpsc::unbounded_channel::<Command>();
+        let (tx_file, rx_file) = mpsc::unbounded_channel::<Command>();
+
+        let mut file_handler = FileOutHandler::new(rx_file, config.clone());
+
+        let tx_stats_clone = tx_stats.clone();
+
+        let file_task = if !config.output.is_empty() {
+            // -o used, need to spawn the thread for writing to disk
+            Some(tokio::spawn(async move {
+                file_handler.start(tx_stats_clone).await
+            }))
+        } else {
+            None
+        };
+
+        let mut term_handler = Self::new(rx_term, tx_file.clone(), file_task, config);
+        let term_task = tokio::spawn(async move { term_handler.start(tx_stats).await });
+
+        let event_handle = TermOutHandle::new(tx_term, tx_file);
+
+        log::trace!("exit: initialize -> ({:?}, {:?})", term_task, event_handle);
+
+        (term_task, event_handle)
+    }
+
+    /// Start a single consumer task (sc side of mpsc)
+    ///
+    /// The consumer simply receives `Command` and acts accordingly
+    async fn start(&mut self, tx_stats: CommandSender) -> Result<()> {
+        log::trace!("enter: start({:?})", tx_stats);
+
+        while let Some(command) = self.receiver.recv().await {
+            match command {
+                Command::Report(mut resp) => {
+                    let contains_sentry =
+                        self.config.status_codes.contains(&resp.status().as_u16());
+                    let unknown_sentry = !RESPONSES.contains(&resp); // !contains == unknown
+                    let should_process_response = contains_sentry && unknown_sentry;
+
+                    if should_process_response {
+                        // print to stdout
+                        ferox_print(&resp.as_str(), &PROGRESS_PRINTER);
+
+                        send_command!(tx_stats, AddToUsizeField(ResourcesDiscovered, 1));
+
+                        if self.file_task.is_some() {
+                            // -o used, need to send the report to be written out to disk
+                            self.tx_file
+                                .send(Command::Report(resp.clone()))
+                                .with_context(|| {
+                                    fmt_err(&format!("Could not send {} to file handler", resp))
+                                })?;
+                        }
+                    }
+                    log::trace!("report complete: {}", resp.url());
+
+                    if self.config.replay_client.is_some() && should_process_response {
+                        // replay proxy specified/client created and this response's status code is one that
+                        // should be replayed; not using logged_request due to replay proxy client
+                        make_request(
+                            self.config.replay_client.as_ref().unwrap(),
+                            resp.url(),
+                            self.config.output_level,
+                            tx_stats.clone(),
+                        )
+                        .await
+                        .with_context(|| "Could not replay request through replay proxy")?;
+                    }
+
+                    if should_process_response {
+                        // add response to RESPONSES for serialization in case of ctrl+c
+                        // placed all by its lonesome like this so that RESPONSES can take ownership
+                        // of the FeroxResponse
+
+                        // before ownership is transferred, there's no real reason to keep the body anymore
+                        // so we can free that piece of data, reducing memory usage
+                        resp.drop_text();
+
+                        RESPONSES.insert(*resp);
+                    }
+                }
+                Command::Sync(sender) => {
+                    sender.send(true).unwrap_or_default();
+                }
+                Command::Exit => {
+                    if self.file_task.is_some() && self.tx_file.send(Command::Exit).is_ok() {
+                        self.file_task.as_mut().unwrap().await??; // wait for death
+                    }
+                    break;
+                }
+                _ => {} // no more commands needed
+            }
+        }
+        log::trace!("exit: start");
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    /// try to hit struct field coverage of FileOutHandler
+    fn struct_fields_of_file_out_handler() {
+        let (_, rx) = mpsc::unbounded_channel::<Command>();
+        let config = Arc::new(Configuration::new().unwrap());
+        let foh = FileOutHandler {
+            config,
+            receiver: rx,
+        };
+        println!("{:?}", foh);
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// try to hit struct field coverage of TermOutHandler
+    async fn struct_fields_of_term_out_handler() {
+        let (tx, rx) = mpsc::unbounded_channel::<Command>();
+        let (tx_file, _) = mpsc::unbounded_channel::<Command>();
+        let config = Arc::new(Configuration::new().unwrap());
+
+        let toh = TermOutHandler {
+            config,
+            file_task: None,
+            receiver: rx,
+            tx_file,
+        };
+
+        println!("{:?}", toh);
+        tx.send(Command::Exit).unwrap();
+    }
+}

src/event_handlers/scans.rs

@@ -0,0 +1,282 @@
+use std::sync::Arc;
+
+use anyhow::{bail, Result};
+use tokio::sync::{mpsc, Semaphore};
+
+use crate::{
+    response::FeroxResponse,
+    scan_manager::{FeroxScan, FeroxScans, ScanOrder},
+    scanner::FeroxScanner,
+    statistics::StatField::TotalScans,
+    url::FeroxUrl,
+    utils::should_deny_url,
+    CommandReceiver, CommandSender, FeroxChannel, Joiner, SLEEP_DURATION,
+};
+
+use super::command::Command::AddToUsizeField;
+use super::*;
+use reqwest::Url;
+use tokio::time::Duration;
+
+#[derive(Debug)]
+/// Container for recursion transmitter and FeroxScans object
+pub struct ScanHandle {
+    /// FeroxScans object used across modules to track scans
+    pub data: Arc<FeroxScans>,
+
+    /// transmitter used to update `data`
+    pub tx: CommandSender,
+}
+
+/// implementation of RecursionHandle
+impl ScanHandle {
+    /// Given an Arc-wrapped FeroxScans and CommandSender, create a new RecursionHandle
+    pub fn new(data: Arc<FeroxScans>, tx: CommandSender) -> Self {
+        Self { data, tx }
+    }
+
+    /// Send the given Command over `tx`
+    pub fn send(&self, command: Command) -> Result<()> {
+        self.tx.send(command)?;
+        Ok(())
+    }
+}
+
+/// event handler for updating a single data structure of all FeroxScans
+#[derive(Debug)]
+pub struct ScanHandler {
+    /// collection of FeroxScans
+    data: Arc<FeroxScans>,
+
+    /// handles to other handlers needed to kick off a scan while already past main
+    handles: Arc<Handles>,
+
+    /// Receiver half of mpsc from which `Command`s are processed
+    receiver: CommandReceiver,
+
+    /// wordlist (re)used for each scan
+    wordlist: std::sync::Mutex<Option<Arc<Vec<String>>>>,
+
+    /// group of scans that need to be joined
+    tasks: Vec<Arc<FeroxScan>>,
+
+    /// Maximum recursion depth, a depth of 0 is infinite recursion
+    max_depth: usize,
+
+    /// depths associated with the initial targets provided by the user
+    depths: Vec<(String, usize)>,
+
+    /// Bounded semaphore used as a barrier to limit concurrent scans
+    limiter: Arc<Semaphore>,
+}
+
+/// implementation of event handler for filters
+impl ScanHandler {
+    /// create new event handler
+    pub fn new(
+        data: Arc<FeroxScans>,
+        handles: Arc<Handles>,
+        max_depth: usize,
+        receiver: CommandReceiver,
+    ) -> Self {
+        let limit = handles.config.scan_limit;
+        let limiter = Semaphore::new(limit);
+
+        if limit == 0 {
+            // scan_limit == 0 means no limit should be imposed... however, scoping the Semaphore
+            // permit is tricky, so as a workaround, we'll add a ridiculous number of permits to
+            // the semaphore (1,152,921,504,606,846,975 to be exact) and call that 'unlimited'
+
+            // note to self: the docs say max is usize::MAX >> 3, however, threads will panic if
+            // that value is used (says adding (1) will overflow the semaphore, even though none
+            // are being added...)
+            limiter.add_permits(usize::MAX >> 4);
+        }
+
+        Self {
+            data,
+            handles,
+            receiver,
+            max_depth,
+            tasks: Vec::new(),
+            depths: Vec::new(),
+            limiter: Arc::new(limiter),
+            wordlist: std::sync::Mutex::new(None),
+        }
+    }
+
+    /// Set the wordlist
+    fn wordlist(&self, wordlist: Arc<Vec<String>>) {
+        if let Ok(mut guard) = self.wordlist.lock() {
+            if guard.is_none() {
+                let _ = std::mem::replace(&mut *guard, Some(wordlist));
+            }
+        }
+    }
+
+    /// Initialize new `FeroxScans` and the sc side of an mpsc channel that is responsible for
+    /// updates to the aforementioned object.
+    pub fn initialize(handles: Arc<Handles>) -> (Joiner, ScanHandle) {
+        log::trace!("enter: initialize");
+
+        let data = Arc::new(FeroxScans::new(handles.config.output_level));
+        let (tx, rx): FeroxChannel<Command> = mpsc::unbounded_channel();
+
+        let max_depth = handles.config.depth;
+
+        let mut handler = Self::new(data.clone(), handles, max_depth, rx);
+
+        let task = tokio::spawn(async move { handler.start().await });
+
+        let event_handle = ScanHandle::new(data, tx);
+
+        log::trace!("exit: initialize -> ({:?}, {:?})", task, event_handle);
+
+        (task, event_handle)
+    }
+
+    /// Start a single consumer task (sc side of mpsc)
+    ///
+    /// The consumer simply receives `Command` and acts accordingly
+    pub async fn start(&mut self) -> Result<()> {
+        log::trace!("enter: start({:?})", self);
+
+        while let Some(command) = self.receiver.recv().await {
+            match command {
+                Command::ScanInitialUrls(targets) => {
+                    self.ordered_scan_url(targets, ScanOrder::Initial).await?;
+                }
+                Command::UpdateWordlist(wordlist) => {
+                    self.wordlist(wordlist);
+                }
+                Command::JoinTasks(sender) => {
+                    let ferox_scans = self.handles.ferox_scans().unwrap_or_default();
+                    let limiter_clone = self.limiter.clone();
+
+                    tokio::spawn(async move {
+                        while ferox_scans.has_active_scans() {
+                            tokio::time::sleep(Duration::from_millis(SLEEP_DURATION + 250)).await;
+                        }
+                        limiter_clone.close();
+                        sender.send(true).expect("oneshot channel failed");
+                    });
+                }
+                Command::TryRecursion(response) => {
+                    self.try_recursion(response).await?;
+                }
+                Command::Sync(sender) => {
+                    sender.send(true).unwrap_or_default();
+                }
+                _ => {} // no other commands needed for RecursionHandler
+            }
+        }
+
+        log::trace!("exit: start");
+        Ok(())
+    }
+
+    /// Helper to easily get the (locked) underlying wordlist
+    pub fn get_wordlist(&self) -> Result<Arc<Vec<String>>> {
+        if let Ok(guard) = self.wordlist.lock().as_ref() {
+            if let Some(list) = guard.as_ref() {
+                return Ok(list.clone());
+            }
+        }
+
+        bail!("Could not get underlying wordlist")
+    }
+
+    /// wrapper around scanning a url to stay DRY
+    async fn ordered_scan_url(&mut self, targets: Vec<String>, order: ScanOrder) -> Result<()> {
+        log::trace!("enter: ordered_scan_url({:?}, {:?})", targets, order);
+        let should_test_deny = !self.handles.config.url_denylist.is_empty();
+
+        for target in targets {
+            if self.data.contains(&target) && matches!(order, ScanOrder::Latest) {
+                // FeroxScans knows about this url and scan isn't an Initial scan
+                // initial scans are skipped because when resuming from a .state file, the scans
+                // will already be populated in FeroxScans, so we need to not skip kicking off
+                // their scans
+                continue;
+            }
+
+            let scan = if let Some(ferox_scan) = self.data.get_scan_by_url(&target) {
+                ferox_scan // scan already known
+            } else {
+                self.data.add_directory_scan(&target, order).1 // add the new target; return FeroxScan
+            };
+
+            if should_test_deny && should_deny_url(&Url::parse(&target)?, self.handles.clone())? {
+                // response was caught by a user-provided deny list
+                // checking this last, since it's most susceptible to longer runtimes due to what
+                // input is received
+                continue;
+            }
+
+            let list = self.get_wordlist()?;
+
+            log::info!("scan handler received {} - beginning scan", target);
+
+            if matches!(order, ScanOrder::Initial) {
+                // keeps track of the initial targets' scan depths in order to enforce the
+                // maximum recursion depth on any identified sub-directories
+                let url = FeroxUrl::from_string(&target, self.handles.clone());
+                let depth = url.depth().unwrap_or(0);
+                self.depths.push((target.clone(), depth));
+            }
+
+            let scanner = FeroxScanner::new(
+                &target,
+                order,
+                list,
+                self.limiter.clone(),
+                self.handles.clone(),
+            );
+
+            let task = tokio::spawn(async move {
+                if let Err(e) = scanner.scan_url().await {
+                    log::warn!("{}", e);
+                }
+            });
+
+            self.handles.stats.send(AddToUsizeField(TotalScans, 1))?;
+
+            scan.set_task(task).await?;
+
+            self.tasks.push(scan.clone());
+        }
+
+        log::trace!("exit: ordered_scan_url");
+        Ok(())
+    }
+
+    async fn try_recursion(&mut self, response: Box<FeroxResponse>) -> Result<()> {
+        log::trace!("enter: try_recursion({:?})", response,);
+
+        if !response.is_directory() {
+            // not a directory, quick exit
+            return Ok(());
+        }
+
+        let mut base_depth = 1_usize;
+
+        for (base_url, base_url_depth) in &self.depths {
+            if response.url().as_str().starts_with(base_url) {
+                base_depth = *base_url_depth;
+            }
+        }
+
+        if response.reached_max_depth(base_depth, self.max_depth, self.handles.clone()) {
+            // at or past recursion depth
+            return Ok(());
+        }
+
+        let targets = vec![response.url().to_string()];
+        self.ordered_scan_url(targets, ScanOrder::Latest).await?;
+
+        log::info!("Added new directory to recursive scan: {}", response.url());
+
+        log::trace!("exit: try_recursion");
+        Ok(())
+    }
+}

src/event_handlers/statistics.rs

@@ -0,0 +1,171 @@
+use super::*;
+use crate::{
+    config::Configuration,
+    progress::{add_bar, BarType},
+    statistics::{StatField, Stats},
+    CommandSender, FeroxChannel, Joiner,
+};
+use anyhow::Result;
+use console::style;
+use indicatif::ProgressBar;
+use std::{sync::Arc, time::Instant};
+use tokio::sync::{
+    mpsc::{self, UnboundedReceiver},
+    oneshot,
+};
+
+#[derive(Debug)]
+/// Container for statistics transmitter and Stats object
+pub struct StatsHandle {
+    /// Stats object used across modules to track statistics
+    pub data: Arc<Stats>,
+
+    /// transmitter used to update `data`
+    pub tx: CommandSender,
+}
+
+/// implementation of StatsHandle
+impl StatsHandle {
+    /// Given an Arc-wrapped Stats and CommandSender, create a new StatsHandle
+    pub fn new(data: Arc<Stats>, tx: CommandSender) -> Self {
+        Self { data, tx }
+    }
+
+    /// Send the given Command over `tx`
+    pub fn send(&self, command: Command) -> Result<()> {
+        self.tx.send(command)?;
+        Ok(())
+    }
+
+    /// Sync the handle with the handler
+    pub async fn sync(&self) -> Result<()> {
+        let (tx, rx) = oneshot::channel::<bool>();
+        self.send(Command::Sync(tx))?;
+        rx.await?;
+        Ok(())
+    }
+}
+
+/// event handler struct for updating statistics
+#[derive(Debug)]
+pub struct StatsHandler {
+    /// overall scan's progress bar
+    bar: ProgressBar,
+
+    /// Receiver half of mpsc from which `StatCommand`s are processed
+    receiver: UnboundedReceiver<Command>,
+
+    /// data class that stores all statistics updates
+    stats: Arc<Stats>,
+}
+
+/// implementation of event handler for statistics
+impl StatsHandler {
+    /// create new event handler
+    fn new(stats: Arc<Stats>, rx_stats: UnboundedReceiver<Command>) -> Self {
+        // will be updated later via StatCommand; delay is for banner to print first
+        let bar = ProgressBar::hidden();
+
+        Self {
+            bar,
+            stats,
+            receiver: rx_stats,
+        }
+    }
+
+    /// Start a single consumer task (sc side of mpsc)
+    ///
+    /// The consumer simply receives `StatCommands` and updates the given `Stats` object as appropriate
+    async fn start(&mut self, output_file: &str) -> Result<()> {
+        log::trace!("enter: start({:?})", self);
+
+        let start = Instant::now();
+
+        while let Some(command) = self.receiver.recv().await {
+            match command as Command {
+                Command::AddError(err) => {
+                    self.stats.add_error(err);
+                    self.increment_bar();
+                }
+                Command::AddStatus(status) => {
+                    self.stats.add_status_code(status);
+
+                    self.increment_bar();
+                }
+                Command::AddRequest => {
+                    self.stats.add_request();
+                    self.increment_bar();
+                }
+                Command::Save => {
+                    self.stats
+                        .save(start.elapsed().as_secs_f64(), output_file)?;
+                }
+                Command::AddToUsizeField(field, value) => {
+                    self.stats.update_usize_field(field, value);
+
+                    if matches!(field, StatField::TotalScans) {
+                        self.bar.set_length(self.stats.total_expected() as u64);
+                    }
+                }
+                Command::SubtractFromUsizeField(field, value) => {
+                    self.stats.subtract_from_usize_field(field, value);
+
+                    if matches!(field, StatField::TotalExpected) {
+                        self.bar.set_length(self.stats.total_expected() as u64);
+                    }
+                }
+                Command::AddToF64Field(field, value) => self.stats.update_f64_field(field, value),
+                Command::CreateBar => {
+                    self.bar = add_bar("", self.stats.total_expected() as u64, BarType::Total);
+                }
+                Command::LoadStats(filename) => {
+                    self.stats.merge_from(&filename)?;
+                }
+                Command::Sync(sender) => {
+                    sender.send(true).unwrap_or_default();
+                }
+                Command::Exit => break,
+                _ => {} // no more commands needed
+            }
+        }
+
+        self.bar.finish();
+
+        log::debug!("{:#?}", *self.stats);
+        log::trace!("exit: start");
+        Ok(())
+    }
+
+    /// Wrapper around incrementing the overall scan's progress bar
+    fn increment_bar(&self) {
+        let msg = format!(
+            "{}:{:<7} {}:{:<7}",
+            style("found").green(),
+            self.stats.resources_discovered(),
+            style("errors").red(),
+            self.stats.errors(),
+        );
+
+        self.bar.set_message(&msg);
+        self.bar.inc(1);
+    }
+
+    /// Initialize new `Stats` object and the sc side of an mpsc channel that is responsible for
+    /// updates to the aforementioned object.
+    pub fn initialize(config: Arc<Configuration>) -> (Joiner, StatsHandle) {
+        log::trace!("enter: initialize");
+
+        let data = Arc::new(Stats::new(config.extensions.len(), config.json));
+        let (tx, rx): FeroxChannel<Command> = mpsc::unbounded_channel();
+
+        let mut handler = StatsHandler::new(data.clone(), rx);
+
+        let task = tokio::spawn(async move { handler.start(&config.output).await });
+
+        let event_handle = StatsHandle::new(data, tx);
+
+        log::trace!("exit: initialize -> ({:?}, {:?})", task, event_handle);
+
+        (task, event_handle)
+    }
+}

src/extractor.rs

@@ -1,269 +0,0 @@
-use crate::FeroxResponse;
-use lazy_static::lazy_static;
-use regex::Regex;
-use reqwest::Url;
-use std::collections::HashSet;
-
-/// Regular expression used in [LinkFinder](https://github.com/GerbenJavado/LinkFinder)
-///
-/// Incorporates change from this [Pull Request](https://github.com/GerbenJavado/LinkFinder/pull/66/files)
-const LINKFINDER_REGEX: &str = r#"(?:"|')(((?:[a-zA-Z]{1,10}://|//)[^"'/]{1,}\.[a-zA-Z]{2,}[^"']{0,})|((?:/|\.\./|\./)[^"'><,;| *()(%%$^/\\\[\]][^"'><,;|()]{1,})|([a-zA-Z0-9_\-/]{1,}/[a-zA-Z0-9_\-/]{1,}\.(?:[a-zA-Z]{1,4}|action)(?:[\?|#][^"|']{0,}|))|([a-zA-Z0-9_\-/]{1,}/[a-zA-Z0-9_\-/]{3,}(?:[\?|#][^"|']{0,}|))|([a-zA-Z0-9_\-.]{1,}\.(?:php|asp|aspx|jsp|json|action|html|js|txt|xml)(?:[\?|#][^"|']{0,}|)))(?:"|')"#;
-
-lazy_static! {
-    /// `LINKFINDER_REGEX` as a regex::Regex type
-    static ref REGEX: Regex = Regex::new(LINKFINDER_REGEX).unwrap();
-}
-
-/// Iterate over a given path, return a list of every sub-path found
-///
-/// example: `path` contains a link fragment `homepage/assets/img/icons/handshake.svg`
-/// the following fragments would be returned:
-///   - homepage/assets/img/icons/handshake.svg
-///   - homepage/assets/img/icons/
-///   - homepage/assets/img/
-///   - homepage/assets/
-///   - homepage/
-fn get_sub_paths_from_path(path: &str) -> Vec<String> {
-    log::trace!("enter: get_sub_paths_from_path({})", path);
-    let mut paths = vec![];
-
-    // filter out any empty strings caused by .split
-    let mut parts: Vec<&str> = path.split('/').filter(|s| !s.is_empty()).collect();
-
-    let length = parts.len();
-
-    for _ in 0..length {
-        // iterate over all parts of the path
-        if parts.is_empty() {
-            // pop left us with an empty vector, we're done
-            break;
-        }
-
-        let possible_path = parts.join("/");
-
-        if possible_path.is_empty() {
-            // .join can result in an empty string, which we don't need, ignore
-            continue;
-        }
-
-        paths.push(possible_path); // good sub-path found
-        parts.pop(); // use .pop() to remove the last part of the path and continue iteration
-    }
-
-    log::trace!("exit: get_sub_paths_from_path -> {:?}", paths);
-    paths
-}
-
-/// simple helper to stay DRY, trys to join a url + fragment and add it to the `links` HashSet
-fn add_link_to_set_of_links(link: &str, url: &Url, links: &mut HashSet<String>) {
-    log::trace!(
-        "enter: add_link_to_set_of_links({}, {}, {:?})",
-        link,
-        url.to_string(),
-        links
-    );
-    match url.join(&link) {
-        Ok(new_url) => {
-            links.insert(new_url.to_string());
-        }
-        Err(e) => {
-            log::error!("Could not join given url to the base url: {}", e);
-        }
-    }
-    log::trace!("exit: add_link_to_set_of_links");
-}
-
-/// Given a `reqwest::Response`, perform the following actions
-///   - parse the response's text for links using the linkfinder regex
-///   - for every link found take its url path and parse each sub-path
-///     - example: Response contains a link fragment `homepage/assets/img/icons/handshake.svg`
-///       with a base url of http://localhost, the following urls would be returned:
-///         - homepage/assets/img/icons/handshake.svg
-///         - homepage/assets/img/icons/
-///         - homepage/assets/img/
-///         - homepage/assets/
-///         - homepage/
-pub async fn get_links(response: &FeroxResponse) -> HashSet<String> {
-    log::trace!("enter: get_links({})", response.url().as_str());
-
-    let mut links = HashSet::<String>::new();
-
-    let body = response.text();
-
-    for capture in REGEX.captures_iter(&body) {
-        // remove single & double quotes from both ends of the capture
-        // capture[0] is the entire match, additional capture groups start at [1]
-        let link = capture[0].trim_matches(|c| c == '\'' || c == '"');
-
-        match Url::parse(link) {
-            Ok(absolute) => {
-                if absolute.domain() != response.url().domain()
-                    || absolute.host() != response.url().host()
-                {
-                    // domains/ips are not the same, don't scan things that aren't part of the original
-                    // target url
-                    continue;
-                }
-
-                for sub_path in get_sub_paths_from_path(absolute.path()) {
-                    // take a url fragment like homepage/assets/img/icons/handshake.svg and
-                    // incrementally add
-                    //     - homepage/assets/img/icons/
-                    //     - homepage/assets/img/
-                    //     - homepage/assets/
-                    //     - homepage/
-                    log::debug!("Adding {} to {:?}", sub_path, links);
-                    add_link_to_set_of_links(&sub_path, &response.url(), &mut links);
-                }
-            }
-            Err(e) => {
-                // this is the expected error that happens when we try to parse a url fragment
-                //     ex: Url::parse("/login") -> Err("relative URL without a base")
-                // while this is technically an error, these are good results for us
-                if e.to_string().contains("relative URL without a base") {
-                    for sub_path in get_sub_paths_from_path(link) {
-                        // incrementally save all sub-paths that led to the relative url's resource
-                        log::debug!("Adding {} to {:?}", sub_path, links);
-                        add_link_to_set_of_links(&sub_path, &response.url(), &mut links);
-                    }
-                } else {
-                    // unexpected error has occurred
-                    log::error!("Could not parse given url: {}", e);
-                }
-            }
-        }
-    }
-
-    log::trace!("exit: get_links -> {:?}", links);
-    links
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::utils::make_request;
-    use httpmock::Method::GET;
-    use httpmock::{Mock, MockServer};
-    use reqwest::Client;
-
-    #[test]
-    /// extract sub paths from the given url fragment; expect 4 sub paths and that all are
-    /// in the expected array
-    fn extractor_get_sub_paths_from_path_with_multiple_paths() {
-        let path = "homepage/assets/img/icons/handshake.svg";
-        let paths = get_sub_paths_from_path(&path);
-        let expected = vec![
-            "homepage",
-            "homepage/assets",
-            "homepage/assets/img",
-            "homepage/assets/img/icons",
-            "homepage/assets/img/icons/handshake.svg",
-        ];
-
-        assert_eq!(paths.len(), expected.len());
-        for expected_path in expected {
-            assert_eq!(paths.contains(&expected_path.to_string()), true);
-        }
-    }
-
-    #[test]
-    /// extract sub paths from the given url fragment; expect 2 sub paths and that all are
-    /// in the expected array. the fragment is wrapped in slashes to ensure no empty strings are
-    /// returned
-    fn extractor_get_sub_paths_from_path_with_enclosing_slashes() {
-        let path = "/homepage/assets/";
-        let paths = get_sub_paths_from_path(&path);
-        let expected = vec!["homepage", "homepage/assets"];
-
-        assert_eq!(paths.len(), expected.len());
-        for expected_path in expected {
-            assert_eq!(paths.contains(&expected_path.to_string()), true);
-        }
-    }
-
-    #[test]
-    /// extract sub paths from the given url fragment; expect 1 sub path, no forward slashes are
-    /// included
-    fn extractor_get_sub_paths_from_path_with_only_a_word() {
-        let path = "homepage";
-        let paths = get_sub_paths_from_path(&path);
-        let expected = vec!["homepage"];
-
-        assert_eq!(paths.len(), expected.len());
-        for expected_path in expected {
-            assert_eq!(paths.contains(&expected_path.to_string()), true);
-        }
-    }
-
-    #[test]
-    /// extract sub paths from the given url fragment; expect 1 sub path, forward slash removed
-    fn extractor_get_sub_paths_from_path_with_an_absolute_word() {
-        let path = "/homepage";
-        let paths = get_sub_paths_from_path(&path);
-        let expected = vec!["homepage"];
-
-        assert_eq!(paths.len(), expected.len());
-        for expected_path in expected {
-            assert_eq!(paths.contains(&expected_path.to_string()), true);
-        }
-    }
-
-    #[test]
-    /// test that a full url and fragment are joined correctly, then added to the given list
-    /// i.e. the happy path
-    fn extractor_add_link_to_set_of_links_happy_path() {
-        let url = Url::parse("https://localhost").unwrap();
-        let mut links = HashSet::<String>::new();
-        let link = "admin";
-
-        assert_eq!(links.len(), 0);
-        add_link_to_set_of_links(link, &url, &mut links);
-
-        assert_eq!(links.len(), 1);
-        assert!(links.contains("https://localhost/admin"));
-    }
-
-    #[test]
-    /// test that an invalid path fragment doesn't add anything to the set of links
-    fn extractor_add_link_to_set_of_links_with_non_base_url() {
-        let url = Url::parse("https://localhost").unwrap();
-        let mut links = HashSet::<String>::new();
-        let link = "\\\\\\\\";
-
-        assert_eq!(links.len(), 0);
-        add_link_to_set_of_links(link, &url, &mut links);
-
-        assert_eq!(links.len(), 0);
-        assert!(links.is_empty());
-    }
-
-    #[tokio::test(core_threads = 1)]
-    /// use make_request to generate a Response, and use the Response to test get_links;
-    /// the response will contain an absolute path to a domain that is not part of the scanned
-    /// domain; expect an empty set returned
-    async fn extractor_get_links_with_absolute_url_that_differs_from_target_domain(
-    ) -> Result<(), Box<dyn std::error::Error>> {
-        let srv = MockServer::start();
-
-        let mock = Mock::new()
-            .expect_method(GET)
-            .expect_path("/some-path")
-            .return_status(200)
-            .return_body("\"http://defintely.not.a.thing.probably.com/homepage/assets/img/icons/handshake.svg\"")
-            .create_on(&srv);
-
-        let client = Client::new();
-        let url = Url::parse(&srv.url("/some-path")).unwrap();
-
-        let response = make_request(&client, &url).await.unwrap();
-
-        let ferox_response = FeroxResponse::from(response, true).await;
-
-        let links = get_links(&ferox_response).await;
-
-        assert!(links.is_empty());
-
-        assert_eq!(mock.times_called(), 1);
-        Ok(())
-    }
-}

src/extractor/builder.rs

@@ -0,0 +1,100 @@
+use super::*;
+use crate::event_handlers::Handles;
+use anyhow::{bail, Result};
+
+/// Regular expression used in [LinkFinder](https://github.com/GerbenJavado/LinkFinder)
+///
+/// Incorporates change from this [Pull Request](https://github.com/GerbenJavado/LinkFinder/pull/66/files)
+pub(super) const LINKFINDER_REGEX: &str = r#"(?:"|')(((?:[a-zA-Z]{1,10}://|//)[^"'/]{1,}\.[a-zA-Z]{2,}[^"']{0,})|((?:/|\.\./|\./)[^"'><,;| *()(%%$^/\\\[\]][^"'><,;|()]{1,})|([a-zA-Z0-9_\-/]{1,}/[a-zA-Z0-9_\-/]{1,}\.(?:[a-zA-Z]{1,4}|action)(?:[\?|#][^"|']{0,}|))|([a-zA-Z0-9_\-/]{1,}/[a-zA-Z0-9_\-/]{3,}(?:[\?|#][^"|']{0,}|))|([a-zA-Z0-9_\-.]{1,}\.(?:php|asp|aspx|jsp|json|action|html|js|txt|xml)(?:[\?|#][^"|']{0,}|)))(?:"|')"#;
+
+/// Regular expression to pull url paths from robots.txt
+///
+/// ref: https://developers.google.com/search/reference/robots_txt
+pub(super) const ROBOTS_TXT_REGEX: &str =
+    r#"(?m)^ *(Allow|Disallow): *(?P<url_path>[a-zA-Z0-9._/?#@!&'()+,;%=-]+?)$"#; // multi-line (?m)
+
+/// Which type of extraction should be performed
+#[derive(Debug, Copy, Clone)]
+pub enum ExtractionTarget {
+    /// Examine a response body and extract links
+    ResponseBody,
+
+    /// Examine robots.txt (specifically) and extract links
+    RobotsTxt,
+}
+
+/// responsible for building an `Extractor`
+pub struct ExtractorBuilder<'a> {
+    /// Response from which to extract links
+    response: Option<&'a FeroxResponse>,
+
+    /// Response from which to extract links
+    url: String,
+
+    /// Handles object to house the underlying mpsc transmitters
+    handles: Option<Arc<Handles>>,
+
+    /// type of extraction to be performed
+    target: ExtractionTarget,
+}
+
+/// ExtractorBuilder implementation
+impl<'a> Default for ExtractorBuilder<'a> {
+    fn default() -> Self {
+        Self {
+            response: None,
+            url: "".to_string(),
+            handles: None,
+            target: ExtractionTarget::ResponseBody,
+        }
+    }
+}
+
+/// ExtractorBuilder implementation
+impl<'a> ExtractorBuilder<'a> {
+    /// builder call to set `handles`
+    pub fn handles(&mut self, handles: Arc<Handles>) -> &mut Self {
+        self.handles = Some(handles);
+        self
+    }
+
+    /// builder call to set `url`
+    pub fn url(&mut self, url: &str) -> &mut Self {
+        self.url = url.to_string();
+        self
+    }
+
+    /// builder call to set `target`
+    pub fn target(&mut self, target: ExtractionTarget) -> &mut Self {
+        self.target = target;
+        self
+    }
+
+    /// builder call to set `response`
+    pub fn response(&mut self, response: &'a FeroxResponse) -> &mut Self {
+        self.response = Some(response);
+        self
+    }
+
+    /// finalize configuration of ExtratorBuilder and return an Extractor
+    ///
+    /// requires either with_url or with_response to have been used in the build process
+    pub fn build(&self) -> Result<Extractor<'a>> {
+        if (self.url.is_empty() && self.response.is_none()) || self.handles.is_none() {
+            bail!("Extractor requires a URL or a FeroxResponse be specified as well as a Handles object")
+        }
+
+        Ok(Extractor {
+            links_regex: Regex::new(LINKFINDER_REGEX).unwrap(),
+            robots_regex: Regex::new(ROBOTS_TXT_REGEX).unwrap(),
+            response: if self.response.is_some() {
+                Some(self.response.unwrap())
+            } else {
+                None
+            },
+            url: self.url.to_owned(),
+            handles: self.handles.as_ref().unwrap().clone(),
+            target: self.target,
+        })
+    }
+}

src/extractor/container.rs

@@ -0,0 +1,429 @@
+use super::*;
+use crate::utils::should_deny_url;
+use crate::{
+    client,
+    event_handlers::{
+        Command,
+        Command::{AddError, AddToUsizeField},
+        Handles,
+    },
+    scan_manager::ScanOrder,
+    statistics::{
+        StatError::Other,
+        StatField::{LinksExtracted, TotalExpected},
+    },
+    url::FeroxUrl,
+    utils::{logged_request, make_request},
+};
+use anyhow::{bail, Context, Result};
+use reqwest::{StatusCode, Url};
+use std::collections::HashSet;
+use tokio::sync::oneshot;
+
+/// Whether an active scan is recursive or not
+#[derive(Debug)]
+enum RecursionStatus {
+    /// Scan is recursive
+    Recursive,
+
+    /// Scan is not recursive
+    NotRecursive,
+}
+
+/// Handles all logic related to extracting links from requested source code
+#[derive(Debug)]
+pub struct Extractor<'a> {
+    /// `LINKFINDER_REGEX` as a regex::Regex type
+    pub(super) links_regex: Regex,
+
+    /// `ROBOTS_TXT_REGEX` as a regex::Regex type
+    pub(super) robots_regex: Regex,
+
+    /// Response from which to extract links
+    pub(super) response: Option<&'a FeroxResponse>,
+
+    /// Response from which to extract links
+    pub(super) url: String,
+
+    /// Handles object to house the underlying mpsc transmitters
+    pub(super) handles: Arc<Handles>,
+
+    /// type of extraction to be performed
+    pub(super) target: ExtractionTarget,
+}
+
+/// Extractor implementation
+impl<'a> Extractor<'a> {
+    /// perform extraction from the given target and return any links found
+    pub async fn extract(&self) -> Result<HashSet<String>> {
+        log::trace!("enter: extract (this fn has associated trace exit msg)");
+        match self.target {
+            ExtractionTarget::ResponseBody => Ok(self.extract_from_body().await?),
+            ExtractionTarget::RobotsTxt => Ok(self.extract_from_robots().await?),
+        }
+    }
+
+    /// given a set of links from a normal http body response, task the request handler to make
+    /// the requests
+    pub async fn request_links(&self, links: HashSet<String>) -> Result<()> {
+        log::trace!("enter: request_links({:?})", links);
+        let recursive = if self.handles.config.no_recursion {
+            RecursionStatus::NotRecursive
+        } else {
+            RecursionStatus::Recursive
+        };
+
+        let scanned_urls = self.handles.ferox_scans()?;
+
+        for link in links {
+            let mut resp = match self.request_link(&link).await {
+                Ok(resp) => resp,
+                Err(_) => continue,
+            };
+
+            // filter if necessary
+            if self
+                .handles
+                .filters
+                .data
+                .should_filter_response(&resp, self.handles.stats.tx.clone())
+            {
+                continue;
+            }
+
+            if resp.is_file() {
+                // very likely a file, simply request and report
+                log::debug!("Extracted file: {}", resp);
+
+                scanned_urls.add_file_scan(&resp.url().to_string(), ScanOrder::Latest);
+
+                if let Err(e) = resp.send_report(self.handles.output.tx.clone()) {
+                    log::warn!("Could not send FeroxResponse to output handler: {}", e);
+                }
+
+                continue;
+            }
+
+            if matches!(recursive, RecursionStatus::Recursive) {
+                log::debug!("Extracted Directory: {}", resp);
+
+                if !resp.url().as_str().ends_with('/')
+                    && (resp.status().is_success()
+                        || matches!(resp.status(), &StatusCode::FORBIDDEN))
+                {
+                    // if the url doesn't end with a /
+                    // and the response code is either a 2xx or 403
+
+                    // since all of these are 2xx or 403, recursion is only attempted if the
+                    // url ends in a /. I am actually ok with adding the slash and not
+                    // adding it, as both have merit.  Leaving it in for now to see how
+                    // things turn out (current as of: v1.1.0)
+                    resp.set_url(&format!("{}/", resp.url()));
+                }
+
+                self.handles
+                    .send_scan_command(Command::TryRecursion(Box::new(resp)))?;
+                let (tx, rx) = oneshot::channel::<bool>();
+                self.handles.send_scan_command(Command::Sync(tx))?;
+                rx.await?;
+            }
+        }
+        log::trace!("exit: request_links");
+        Ok(())
+    }
+
+    /// Given a `reqwest::Response`, perform the following actions
+    ///   - parse the response's text for links using the linkfinder regex
+    ///   - for every link found take its url path and parse each sub-path
+    ///     - example: Response contains a link fragment `homepage/assets/img/icons/handshake.svg`
+    ///       with a base url of http://localhost, the following urls would be returned:
+    ///         - homepage/assets/img/icons/handshake.svg
+    ///         - homepage/assets/img/icons/
+    ///         - homepage/assets/img/
+    ///         - homepage/assets/
+    ///         - homepage/
+    pub(super) async fn extract_from_body(&self) -> Result<HashSet<String>> {
+        log::trace!("enter: get_links");
+
+        let mut links = HashSet::<String>::new();
+
+        let body = self.response.unwrap().text();
+
+        for capture in self.links_regex.captures_iter(body) {
+            // remove single & double quotes from both ends of the capture
+            // capture[0] is the entire match, additional capture groups start at [1]
+            let link = capture[0].trim_matches(|c| c == '\'' || c == '"');
+
+            match Url::parse(link) {
+                Ok(absolute) => {
+                    if absolute.domain() != self.response.unwrap().url().domain()
+                        || absolute.host() != self.response.unwrap().url().host()
+                    {
+                        // domains/ips are not the same, don't scan things that aren't part of the original
+                        // target url
+                        continue;
+                    }
+
+                    if self.add_all_sub_paths(absolute.path(), &mut links).is_err() {
+                        log::warn!("could not add sub-paths from {} to {:?}", absolute, links);
+                    }
+                }
+                Err(e) => {
+                    // this is the expected error that happens when we try to parse a url fragment
+                    //     ex: Url::parse("/login") -> Err("relative URL without a base")
+                    // while this is technically an error, these are good results for us
+                    if e.to_string().contains("relative URL without a base") {
+                        if self.add_all_sub_paths(link, &mut links).is_err() {
+                            log::warn!("could not add sub-paths from {} to {:?}", link, links);
+                        }
+                    } else {
+                        // unexpected error has occurred
+                        log::warn!("Could not parse given url: {}", e);
+                        self.handles.stats.send(AddError(Other)).unwrap_or_default();
+                    }
+                }
+            }
+        }
+
+        self.update_stats(links.len())?;
+
+        log::trace!("exit: get_links -> {:?}", links);
+
+        Ok(links)
+    }
+
+    /// take a url fragment like homepage/assets/img/icons/handshake.svg and
+    /// incrementally add
+    ///     - homepage/assets/img/icons/
+    ///     - homepage/assets/img/
+    ///     - homepage/assets/
+    ///     - homepage/
+    fn add_all_sub_paths(&self, url_path: &str, mut links: &mut HashSet<String>) -> Result<()> {
+        log::trace!("enter: add_all_sub_paths({}, {:?})", url_path, links);
+
+        for sub_path in self.get_sub_paths_from_path(url_path) {
+            self.add_link_to_set_of_links(&sub_path, &mut links)?;
+        }
+
+        log::trace!("exit: add_all_sub_paths");
+        Ok(())
+    }
+
+    /// Iterate over a given path, return a list of every sub-path found
+    ///
+    /// example: `path` contains a link fragment `homepage/assets/img/icons/handshake.svg`
+    /// the following fragments would be returned:
+    ///   - homepage/assets/img/icons/handshake.svg
+    ///   - homepage/assets/img/icons/
+    ///   - homepage/assets/img/
+    ///   - homepage/assets/
+    ///   - homepage/
+    pub(super) fn get_sub_paths_from_path(&self, path: &str) -> Vec<String> {
+        log::trace!("enter: get_sub_paths_from_path({})", path);
+        let mut paths = vec![];
+
+        // filter out any empty strings caused by .split
+        let mut parts: Vec<&str> = path.split('/').filter(|s| !s.is_empty()).collect();
+
+        let length = parts.len();
+
+        for i in 0..length {
+            // iterate over all parts of the path
+            if parts.is_empty() {
+                // pop left us with an empty vector, we're done
+                break;
+            }
+
+            let mut possible_path = parts.join("/");
+
+            if possible_path.is_empty() {
+                // .join can result in an empty string, which we don't need, ignore
+                continue;
+            }
+
+            if i > 0 {
+                // this isn't the last index of the parts array
+                // ex: /buried/misc/stupidfile.php
+                // this block skips the file but sees all parent folders
+                possible_path = format!("{}/", possible_path);
+            }
+
+            paths.push(possible_path); // good sub-path found
+            parts.pop(); // use .pop() to remove the last part of the path and continue iteration
+        }
+
+        log::trace!("exit: get_sub_paths_from_path -> {:?}", paths);
+        paths
+    }
+
+    /// simple helper to stay DRY, trys to join a url + fragment and add it to the `links` HashSet
+    pub(super) fn add_link_to_set_of_links(
+        &self,
+        link: &str,
+        links: &mut HashSet<String>,
+    ) -> Result<()> {
+        log::trace!("enter: add_link_to_set_of_links({}, {:?})", link, links);
+
+        let old_url = match self.target {
+            ExtractionTarget::ResponseBody => self.response.unwrap().url().clone(),
+            ExtractionTarget::RobotsTxt => match Url::parse(&self.url) {
+                Ok(u) => u,
+                Err(e) => {
+                    bail!("Could not parse {}: {}", self.url, e);
+                }
+            },
+        };
+
+        let new_url = old_url
+            .join(link)
+            .with_context(|| format!("Could not join {} with {}", old_url, link))?;
+
+        links.insert(new_url.to_string());
+
+        log::trace!("exit: add_link_to_set_of_links");
+
+        Ok(())
+    }
+
+    /// Wrapper around link extraction logic
+    /// currently used in two places:
+    ///   - links from response bodies
+    ///   - links from robots.txt responses
+    ///
+    /// general steps taken:
+    ///   - create a new Url object based on cli options/args
+    ///   - check if the new Url has already been seen/scanned -> None
+    ///   - make a request to the new Url ? -> Some(response) : None
+    pub(super) async fn request_link(&self, url: &str) -> Result<FeroxResponse> {
+        log::trace!("enter: request_link({})", url);
+
+        let ferox_url = FeroxUrl::from_string(url, self.handles.clone());
+
+        // create a url based on the given command line options
+        let new_url = ferox_url.format("", None)?;
+
+        let scanned_urls = self.handles.ferox_scans()?;
+
+        if scanned_urls.get_scan_by_url(&new_url.to_string()).is_some() {
+            //we've seen the url before and don't need to scan again
+            log::trace!("exit: request_link -> None");
+            bail!("previously seen url");
+        }
+
+        if !self.handles.config.url_denylist.is_empty()
+            && should_deny_url(&new_url, self.handles.clone())?
+        {
+            // can't allow a denied url to be requested
+            bail!(
+                "prevented request to {} due to {:?}",
+                url,
+                self.handles.config.url_denylist
+            );
+        }
+
+        // make the request and store the response
+        let new_response = logged_request(&new_url, self.handles.clone()).await?;
+
+        let new_ferox_response =
+            FeroxResponse::from(new_response, true, self.handles.config.output_level).await;
+
+        log::trace!("exit: request_link -> {:?}", new_ferox_response);
+
+        Ok(new_ferox_response)
+    }
+
+    /// Entry point to perform link extraction from robots.txt
+    ///
+    /// `base_url` can have paths and subpaths, however robots.txt will be requested from the
+    /// root of the url
+    /// given the url:
+    ///     http://localhost/stuff/things
+    /// this function requests:
+    ///     http://localhost/robots.txt
+    pub(super) async fn extract_from_robots(&self) -> Result<HashSet<String>> {
+        log::trace!("enter: extract_robots_txt");
+
+        let mut links: HashSet<String> = HashSet::new();
+
+        let response = self.request_robots_txt().await?;
+
+        for capture in self.robots_regex.captures_iter(response.text()) {
+            if let Some(new_path) = capture.name("url_path") {
+                let mut new_url = Url::parse(&self.url)?;
+                new_url.set_path(new_path.as_str());
+                if self.add_all_sub_paths(new_url.path(), &mut links).is_err() {
+                    log::warn!("could not add sub-paths from {} to {:?}", new_url, links);
+                }
+            }
+        }
+
+        self.update_stats(links.len())?;
+
+        log::trace!("exit: extract_robots_txt -> {:?}", links);
+        Ok(links)
+    }
+
+    /// helper function that simply requests /robots.txt on the given url's base url
+    ///
+    /// example:
+    ///     http://localhost/api/users -> http://localhost/robots.txt
+    ///     
+    /// The length of the given path has no effect on what's requested; it's always
+    /// base url + /robots.txt
+    pub(super) async fn request_robots_txt(&self) -> Result<FeroxResponse> {
+        log::trace!("enter: get_robots_file");
+
+        // more often than not, domain/robots.txt will redirect to www.domain/robots.txt or something
+        // similar; to account for that, create a client that will follow redirects, regardless of
+        // what the user specified for the scanning client. Other than redirects, it will respect
+        // all other user specified settings
+        let follow_redirects = true;
+
+        let proxy = if self.handles.config.proxy.is_empty() {
+            None
+        } else {
+            Some(self.handles.config.proxy.as_str())
+        };
+
+        let client = client::initialize(
+            self.handles.config.timeout,
+            &self.handles.config.user_agent,
+            follow_redirects,
+            self.handles.config.insecure,
+            &self.handles.config.headers,
+            proxy,
+        )?;
+
+        let mut url = Url::parse(&self.url)?;
+        url.set_path("/robots.txt"); // overwrite existing path with /robots.txt
+
+        // purposefully not using logged_request here due to using the special client
+        let response = make_request(
+            &client,
+            &url,
+            self.handles.config.output_level,
+            self.handles.stats.tx.clone(),
+        )
+        .await?;
+
+        let ferox_response =
+            FeroxResponse::from(response, true, self.handles.config.output_level).await;
+
+        log::trace!("exit: get_robots_file -> {}", ferox_response);
+        Ok(ferox_response)
+    }
+
+    /// update total number of links extracted and expected responses
+    fn update_stats(&self, num_links: usize) -> Result<()> {
+        let multiplier = self.handles.config.extensions.len().max(1);
+
+        self.handles
+            .stats
+            .send(AddToUsizeField(LinksExtracted, num_links))?;
+        self.handles
+            .stats
+            .send(AddToUsizeField(TotalExpected, num_links * multiplier))?;
+
+        Ok(())
+    }
+}

src/extractor/mod.rs

@@ -0,0 +1,13 @@
+//! extract links from html source and robots.txt
+mod builder;
+mod container;
+#[cfg(test)]
+mod tests;
+
+pub use self::builder::ExtractionTarget;
+pub use self::builder::ExtractorBuilder;
+pub use self::container::Extractor;
+
+use crate::response::FeroxResponse;
+use regex::Regex;
+use std::sync::Arc;

src/extractor/tests.rs

@@ -0,0 +1,354 @@
+use super::builder::{LINKFINDER_REGEX, ROBOTS_TXT_REGEX};
+use super::*;
+use crate::config::{Configuration, OutputLevel};
+use crate::scan_manager::ScanOrder;
+use crate::{
+    event_handlers::Handles, scan_manager::FeroxScans, utils::make_request, Command, FeroxChannel,
+};
+use anyhow::Result;
+use httpmock::{Method::GET, MockServer};
+use lazy_static::lazy_static;
+use reqwest::{Client, StatusCode, Url};
+use std::collections::HashSet;
+use tokio::sync::mpsc;
+
+lazy_static! {
+    /// Extractor for testing robots.txt
+    static ref ROBOTS_EXT: Extractor<'static> = setup_extractor(ExtractionTarget::RobotsTxt, Arc::new(FeroxScans::default()));
+
+    /// Extractor for testing response bodies
+    static ref BODY_EXT: Extractor<'static> = setup_extractor(ExtractionTarget::ResponseBody, Arc::new(FeroxScans::default()));
+
+    /// FeroxResponse for Extractor
+    static ref RESPONSE: FeroxResponse = get_test_response();
+}
+
+/// constructor for the default FeroxResponse used during testing
+fn get_test_response() -> FeroxResponse {
+    let mut resp = FeroxResponse::default();
+    resp.set_text("nulla pharetra diam sit amet nisl suscipit adipiscing bibendum est");
+    resp
+}
+
+/// creates a single extractor that can be used to test standalone functions
+fn setup_extractor(target: ExtractionTarget, scanned_urls: Arc<FeroxScans>) -> Extractor<'static> {
+    let mut builder = ExtractorBuilder::default();
+
+    let builder = match target {
+        ExtractionTarget::ResponseBody => builder
+            .target(ExtractionTarget::ResponseBody)
+            .response(&RESPONSE),
+        ExtractionTarget::RobotsTxt => builder
+            .url("http://localhost")
+            .target(ExtractionTarget::RobotsTxt),
+    };
+
+    let config = Arc::new(Configuration::new().unwrap());
+    let handles = Arc::new(Handles::for_testing(Some(scanned_urls), Some(config)).0);
+
+    builder.handles(handles).build().unwrap()
+}
+
+#[test]
+/// extract sub paths from the given url fragment; expect 4 sub paths and that all are
+/// in the expected array
+fn extractor_get_sub_paths_from_path_with_multiple_paths() {
+    let path = "homepage/assets/img/icons/handshake.svg";
+    let r_paths = ROBOTS_EXT.get_sub_paths_from_path(path);
+    let b_paths = BODY_EXT.get_sub_paths_from_path(path);
+    let expected = vec![
+        "homepage/",
+        "homepage/assets/",
+        "homepage/assets/img/",
+        "homepage/assets/img/icons/",
+        "homepage/assets/img/icons/handshake.svg",
+    ];
+
+    assert_eq!(r_paths.len(), expected.len());
+    assert_eq!(b_paths.len(), expected.len());
+    for expected_path in expected {
+        assert!(r_paths.contains(&expected_path.to_string()));
+        assert!(b_paths.contains(&expected_path.to_string()));
+    }
+}
+
+#[test]
+/// extract sub paths from the given url fragment; expect 2 sub paths and that all are
+/// in the expected array. the fragment is wrapped in slashes to ensure no empty strings are
+/// returned
+fn extractor_get_sub_paths_from_path_with_enclosing_slashes() {
+    let path = "/homepage/assets/";
+    let r_paths = ROBOTS_EXT.get_sub_paths_from_path(path);
+    let b_paths = BODY_EXT.get_sub_paths_from_path(path);
+    let expected = vec!["homepage/", "homepage/assets"];
+
+    assert_eq!(r_paths.len(), expected.len());
+    assert_eq!(b_paths.len(), expected.len());
+    for expected_path in expected {
+        assert!(r_paths.contains(&expected_path.to_string()));
+        assert!(b_paths.contains(&expected_path.to_string()));
+    }
+}
+
+#[test]
+/// extract sub paths from the given url fragment; expect 1 sub path, no forward slashes are
+/// included
+fn extractor_get_sub_paths_from_path_with_only_a_word() {
+    let path = "homepage";
+    let r_paths = ROBOTS_EXT.get_sub_paths_from_path(path);
+    let b_paths = BODY_EXT.get_sub_paths_from_path(path);
+    let expected = vec!["homepage"];
+
+    assert_eq!(r_paths.len(), expected.len());
+    assert_eq!(b_paths.len(), expected.len());
+    for expected_path in expected {
+        assert!(r_paths.contains(&expected_path.to_string()));
+        assert!(b_paths.contains(&expected_path.to_string()));
+    }
+}
+
+#[test]
+/// extract sub paths from the given url fragment; expect 1 sub path, forward slash removed
+fn extractor_get_sub_paths_from_path_with_an_absolute_word() {
+    let path = "/homepage";
+    let r_paths = ROBOTS_EXT.get_sub_paths_from_path(path);
+    let b_paths = BODY_EXT.get_sub_paths_from_path(path);
+    let expected = vec!["homepage"];
+
+    assert_eq!(r_paths.len(), expected.len());
+    assert_eq!(b_paths.len(), expected.len());
+    for expected_path in expected {
+        assert!(r_paths.contains(&expected_path.to_string()));
+        assert!(b_paths.contains(&expected_path.to_string()));
+    }
+}
+
+#[test]
+/// test that an ExtractorBuilder without a FeroxResponse and without a URL bails
+fn extractor_builder_bails_when_neither_required_field_is_set() {
+    let handles = Arc::new(Handles::for_testing(None, None).0);
+
+    let extractor = ExtractorBuilder::default()
+        .url("")
+        .target(ExtractionTarget::RobotsTxt)
+        .handles(handles)
+        .build();
+
+    assert!(extractor.is_err());
+}
+
+#[test]
+/// Extractor with a non-base url bails
+fn extractor_with_non_base_url_bails() -> Result<()> {
+    let mut links = HashSet::<String>::new();
+    let link = "admin";
+    let handles = Arc::new(Handles::for_testing(None, None).0);
+
+    let extractor = ExtractorBuilder::default()
+        .url("\\\\\\")
+        .handles(handles)
+        .target(ExtractionTarget::RobotsTxt)
+        .build()?;
+
+    let result = extractor.add_link_to_set_of_links(link, &mut links);
+
+    assert!(result.is_err());
+    Ok(())
+}
+
+#[test]
+/// test that a full url and fragment are joined correctly, then added to the given list
+/// i.e. the happy path
+fn extractor_add_link_to_set_of_links_happy_path() {
+    let mut r_links = HashSet::<String>::new();
+    let r_link = "admin";
+    let mut b_links = HashSet::<String>::new();
+    let b_link = "shmadmin";
+
+    assert_eq!(r_links.len(), 0);
+    ROBOTS_EXT
+        .add_link_to_set_of_links(r_link, &mut r_links)
+        .unwrap();
+
+    assert_eq!(r_links.len(), 1);
+    assert!(r_links.contains("http://localhost/admin"));
+
+    assert_eq!(b_links.len(), 0);
+
+    BODY_EXT
+        .add_link_to_set_of_links(b_link, &mut b_links)
+        .unwrap();
+
+    assert_eq!(b_links.len(), 1);
+    assert!(b_links.contains("http://localhost/shmadmin"));
+}
+
+#[test]
+/// test that an invalid path fragment doesn't add anything to the set of links
+fn extractor_add_link_to_set_of_links_with_non_base_url() {
+    let mut links = HashSet::<String>::new();
+    let link = "\\\\\\\\";
+
+    assert_eq!(links.len(), 0);
+    assert!(ROBOTS_EXT
+        .add_link_to_set_of_links(link, &mut links)
+        .is_err());
+    assert!(BODY_EXT.add_link_to_set_of_links(link, &mut links).is_err());
+
+    assert_eq!(links.len(), 0);
+    assert!(links.is_empty());
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// use make_request to generate a Response, and use the Response to test get_links;
+/// the response will contain an absolute path to a domain that is not part of the scanned
+/// domain; expect an empty set returned
+async fn extractor_get_links_with_absolute_url_that_differs_from_target_domain() -> Result<()> {
+    let (tx_stats, _): FeroxChannel<Command> = mpsc::unbounded_channel();
+
+    let srv = MockServer::start();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/some-path");
+        then.status(200).body(
+            "\"http://defintely.not.a.thing.probably.com/homepage/assets/img/icons/handshake.svg\"",
+        );
+    });
+
+    let client = Client::new();
+    let url = Url::parse(&srv.url("/some-path")).unwrap();
+
+    let response = make_request(&client, &url, OutputLevel::Default, tx_stats.clone())
+        .await
+        .unwrap();
+    let (handles, _rx) = Handles::for_testing(None, None);
+
+    let handles = Arc::new(handles);
+    let ferox_response = FeroxResponse::from(response, true, OutputLevel::Default).await;
+
+    let extractor = Extractor {
+        links_regex: Regex::new(LINKFINDER_REGEX).unwrap(),
+        robots_regex: Regex::new(ROBOTS_TXT_REGEX).unwrap(),
+        response: Some(&ferox_response),
+        url: String::new(),
+        target: ExtractionTarget::ResponseBody,
+        handles: handles.clone(),
+    };
+
+    let links = extractor.extract_from_body().await?;
+
+    assert!(links.is_empty());
+    assert_eq!(mock.hits(), 1);
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test that /robots.txt is correctly requested given a base url (happy path)
+async fn request_robots_txt_without_proxy() -> Result<()> {
+    let handles = Arc::new(Handles::for_testing(None, None).0);
+
+    let srv = MockServer::start();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/robots.txt");
+        then.status(200).body("this is a test");
+    });
+
+    let extractor = Extractor {
+        links_regex: Regex::new(LINKFINDER_REGEX).unwrap(),
+        robots_regex: Regex::new(ROBOTS_TXT_REGEX).unwrap(),
+        response: None,
+        url: srv.url("/api/users/stuff/things"),
+        target: ExtractionTarget::RobotsTxt,
+        handles,
+    };
+
+    let resp = extractor.request_robots_txt().await?;
+
+    assert!(matches!(resp.status(), &StatusCode::OK));
+    println!("{}", resp);
+    assert_eq!(resp.content_length(), 14);
+    assert_eq!(mock.hits(), 1);
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test that /robots.txt is correctly requested given a base url (happy path) when a proxy is used
+async fn request_robots_txt_with_proxy() -> Result<()> {
+    let handles = Arc::new(Handles::for_testing(None, None).0);
+    let mut config = Configuration::new()?;
+
+    let srv = MockServer::start();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/robots.txt");
+        then.status(200).body("this is also a test");
+    });
+
+    // note: the proxy doesn't actually do anything other than hit a different code branch
+    // in this unit test; it would however have an effect on an integration test
+    config.proxy = srv.url("/ima-proxy");
+    config.no_recursion = true;
+
+    let extractor = ExtractorBuilder::default()
+        .url(&srv.url("/api/different/path"))
+        .target(ExtractionTarget::RobotsTxt)
+        .handles(handles)
+        .build()?;
+
+    let resp = extractor.request_robots_txt().await?;
+
+    assert!(matches!(resp.status(), &StatusCode::OK));
+    assert_eq!(resp.content_length(), 19);
+    assert_eq!(mock.hits(), 1);
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// request_link's happy path, expect back a FeroxResponse
+async fn request_link_happy_path() -> Result<()> {
+    let srv = MockServer::start();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/login.php");
+        then.status(200).body("this is a test");
+    });
+
+    let r_resp = ROBOTS_EXT.request_link(&srv.url("/login.php")).await?;
+    let b_resp = BODY_EXT.request_link(&srv.url("/login.php")).await?;
+
+    assert!(matches!(r_resp.status(), &StatusCode::OK));
+    assert!(matches!(b_resp.status(), &StatusCode::OK));
+    assert_eq!(r_resp.content_length(), 14);
+    assert_eq!(b_resp.content_length(), 14);
+    assert_eq!(mock.hits(), 2);
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// request_link should bail in the event that the url is already in scanned_urls
+async fn request_link_bails_on_seen_url() -> Result<()> {
+    let url = "/unique-for-this-test.php";
+    let srv = MockServer::start();
+    let served = srv.url(url);
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path(url);
+        then.status(200)
+            .body("this is a unique test, don't reuse the endpoint");
+    });
+
+    let scans = Arc::new(FeroxScans::default());
+    scans.add_file_scan(&served, ScanOrder::Latest);
+
+    let robots = setup_extractor(ExtractionTarget::RobotsTxt, scans.clone());
+    let body = setup_extractor(ExtractionTarget::ResponseBody, scans);
+
+    let r_resp = robots.request_link(&served).await;
+    let b_resp = body.request_link(&served).await;
+
+    assert!(r_resp.is_err());
+    assert!(b_resp.is_err());
+    assert_eq!(mock.hits(), 0); // function exits before requests can happen
+    Ok(())
+}

src/filters.rs

@@ -1,422 +0,0 @@
-use crate::config::CONFIGURATION;
-use crate::utils::get_url_path_length;
-use crate::FeroxResponse;
-use regex::Regex;
-use std::any::Any;
-use std::fmt::Debug;
-
-// references:
-//   https://dev.to/magnusstrale/rust-trait-objects-in-a-vector-non-trivial-4co5
-//   https://stackoverflow.com/questions/25339603/how-to-test-for-equality-between-trait-objects
-
-/// FeroxFilter trait; represents different types of possible filters that can be applied to
-/// responses
-pub trait FeroxFilter: Debug + Send + Sync {
-    /// Determine whether or not this particular filter should be applied or not
-    fn should_filter_response(&self, response: &FeroxResponse) -> bool;
-
-    /// delegates to the FeroxFilter-implementing type which gives us the actual type of self
-    fn box_eq(&self, other: &dyn Any) -> bool;
-
-    /// gives us `other` as Any in box_eq
-    fn as_any(&self) -> &dyn Any;
-}
-
-/// implementation of PartialEq, necessary long-form due to "trait cannot be made into an object"
-/// error when attempting to derive PartialEq on the trait itself
-impl PartialEq for Box<dyn FeroxFilter> {
-    /// Perform a comparison of two implementors of the FeroxFilter trait
-    fn eq(&self, other: &Box<dyn FeroxFilter>) -> bool {
-        self.box_eq(other.as_any())
-    }
-}
-
-/// Data holder for two pieces of data needed when auto-filtering out wildcard responses
-///
-/// `dynamic` is the size of the response that will later be combined with the length
-/// of the path of the url requested and used to determine interesting pages from custom
-/// 404s where the requested url is reflected back in the response
-///
-/// `size` is size of the response that should be included with filters passed via runtime
-/// configuration and any static wildcard lengths.
-#[derive(Debug, Default, Clone, PartialEq)]
-pub struct WildcardFilter {
-    /// size of the response that will later be combined with the length of the path of the url
-    /// requested
-    pub dynamic: u64,
-
-    /// size of the response that should be included with filters passed via runtime configuration
-    pub size: u64,
-}
-
-/// implementation of FeroxFilter for WildcardFilter
-impl FeroxFilter for WildcardFilter {
-    /// Examine size, dynamic, and content_len to determine whether or not the response received
-    /// is a wildcard response and therefore should be filtered out
-    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
-        log::trace!("enter: should_filter_response({:?} {})", self, response);
-
-        // quick return if dont_filter is set
-        if CONFIGURATION.dont_filter {
-            // --dont-filter applies specifically to wildcard filters, it is not a 100% catch all
-            // for not filtering anything.  As such, it should live in the implementation of
-            // a wildcard filter
-            return false;
-        }
-
-        if self.size > 0 && self.size == response.content_length() {
-            // static wildcard size found during testing
-            // size isn't default, size equals response length, and auto-filter is on
-            log::debug!("static wildcard: filtered out {}", response.url());
-            log::trace!("exit: should_filter_response -> true");
-            return true;
-        }
-
-        if self.dynamic > 0 {
-            // dynamic wildcard offset found during testing
-
-            // I'm about to manually split this url path instead of using reqwest::Url's
-            // builtin parsing. The reason is that they call .split() on the url path
-            // except that I don't want an empty string taking up the last index in the
-            // event that the url ends with a forward slash.  It's ugly enough to be split
-            // into its own function for readability.
-            let url_len = get_url_path_length(&response.url());
-
-            if url_len + self.dynamic == response.content_length() {
-                log::debug!("dynamic wildcard: filtered out {}", response.url());
-                log::trace!("exit: should_filter_response -> true");
-                return true;
-            }
-        }
-        log::trace!("exit: should_filter_response -> false");
-        false
-    }
-
-    /// Compare one WildcardFilter to another
-    fn box_eq(&self, other: &dyn Any) -> bool {
-        other.downcast_ref::<Self>().map_or(false, |a| self == a)
-    }
-
-    /// Return self as Any for dynamic dispatch purposes
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-}
-
-/// Simple implementor of FeroxFilter; used to filter out status codes specified using
-/// -C|--filter-status
-#[derive(Default, Debug, PartialEq)]
-pub struct StatusCodeFilter {
-    /// Status code that should not be displayed to the user
-    pub filter_code: u16,
-}
-
-/// implementation of FeroxFilter for StatusCodeFilter
-impl FeroxFilter for StatusCodeFilter {
-    /// Check `filter_code` against what was passed in via -C|--filter-status
-    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
-        log::trace!("enter: should_filter_response({:?} {})", self, response);
-
-        if response.status().as_u16() == self.filter_code {
-            log::debug!(
-                "filtered out {} based on --filter-status of {}",
-                response.url(),
-                self.filter_code
-            );
-            log::trace!("exit: should_filter_response -> true");
-            return true;
-        }
-
-        log::trace!("exit: should_filter_response -> false");
-        false
-    }
-
-    /// Compare one StatusCodeFilter to another
-    fn box_eq(&self, other: &dyn Any) -> bool {
-        other.downcast_ref::<Self>().map_or(false, |a| self == a)
-    }
-
-    /// Return self as Any for dynamic dispatch purposes
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-}
-
-/// Simple implementor of FeroxFilter; used to filter out responses based on the number of lines
-/// in a Response body; specified using -N|--filter-lines
-#[derive(Default, Debug, PartialEq)]
-pub struct LinesFilter {
-    /// Number of lines in a Response's body that should be filtered
-    pub line_count: usize,
-}
-
-/// implementation of FeroxFilter for LinesFilter
-impl FeroxFilter for LinesFilter {
-    /// Check `line_count` against what was passed in via -N|--filter-lines
-    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
-        log::trace!("enter: should_filter_response({:?} {})", self, response);
-
-        let result = response.line_count() == self.line_count;
-
-        log::trace!("exit: should_filter_response -> {}", result);
-
-        result
-    }
-
-    /// Compare one LinesFilter to another
-    fn box_eq(&self, other: &dyn Any) -> bool {
-        other.downcast_ref::<Self>().map_or(false, |a| self == a)
-    }
-
-    /// Return self as Any for dynamic dispatch purposes
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-}
-
-/// Simple implementor of FeroxFilter; used to filter out responses based on the number of words
-/// in a Response body; specified using -W|--filter-words
-#[derive(Default, Debug, PartialEq)]
-pub struct WordsFilter {
-    /// Number of words in a Response's body that should be filtered
-    pub word_count: usize,
-}
-
-/// implementation of FeroxFilter for WordsFilter
-impl FeroxFilter for WordsFilter {
-    /// Check `word_count` against what was passed in via -W|--filter-words
-    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
-        log::trace!("enter: should_filter_response({:?} {})", self, response);
-
-        let result = response.word_count() == self.word_count;
-
-        log::trace!("exit: should_filter_response -> {}", result);
-
-        result
-    }
-
-    /// Compare one WordsFilter to another
-    fn box_eq(&self, other: &dyn Any) -> bool {
-        other.downcast_ref::<Self>().map_or(false, |a| self == a)
-    }
-
-    /// Return self as Any for dynamic dispatch purposes
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-}
-
-/// Simple implementor of FeroxFilter; used to filter out responses based on the length of a
-/// Response body; specified using -S|--filter-size
-#[derive(Default, Debug, PartialEq)]
-pub struct SizeFilter {
-    /// Overall length of a Response's body that should be filtered
-    pub content_length: u64,
-}
-
-/// implementation of FeroxFilter for SizeFilter
-impl FeroxFilter for SizeFilter {
-    /// Check `content_length` against what was passed in via -S|--filter-size
-    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
-        log::trace!("enter: should_filter_response({:?} {})", self, response);
-
-        let result = response.content_length() == self.content_length;
-
-        log::trace!("exit: should_filter_response -> {}", result);
-
-        result
-    }
-
-    /// Compare one SizeFilter to another
-    fn box_eq(&self, other: &dyn Any) -> bool {
-        other.downcast_ref::<Self>().map_or(false, |a| self == a)
-    }
-
-    /// Return self as Any for dynamic dispatch purposes
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-}
-
-/// Simple implementor of FeroxFilter; used to filter out responses based on a given regular
-/// expression; specified using -X|--filter-regex
-#[derive(Debug)]
-pub struct RegexFilter {
-    /// Regular expression to be applied to the response body for filtering, compiled
-    pub compiled: Regex,
-
-    /// Regular expression as passed in on the command line, not compiled
-    pub raw_string: String,
-}
-
-/// implementation of FeroxFilter for RegexFilter
-impl FeroxFilter for RegexFilter {
-    /// Check `expression` against the response body, if the expression matches, the response
-    /// should be filtered out
-    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
-        log::trace!("enter: should_filter_response({:?} {})", self, response);
-
-        let result = self.compiled.is_match(response.text());
-
-        log::trace!("exit: should_filter_response -> {}", result);
-
-        result
-    }
-
-    /// Compare one SizeFilter to another
-    fn box_eq(&self, other: &dyn Any) -> bool {
-        other.downcast_ref::<Self>().map_or(false, |a| self == a)
-    }
-
-    /// Return self as Any for dynamic dispatch purposes
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-}
-
-/// PartialEq implementation for RegexFilter
-impl PartialEq for RegexFilter {
-    /// Simple comparison of the raw string passed in via the command line
-    fn eq(&self, other: &RegexFilter) -> bool {
-        self.raw_string == other.raw_string
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use reqwest::Url;
-
-    #[test]
-    /// just a simple test to increase code coverage by hitting as_any and the inner value
-    fn lines_filter_as_any() {
-        let filter = LinesFilter { line_count: 1 };
-
-        assert_eq!(filter.line_count, 1);
-        assert_eq!(
-            *filter.as_any().downcast_ref::<LinesFilter>().unwrap(),
-            filter
-        );
-    }
-
-    #[test]
-    /// just a simple test to increase code coverage by hitting as_any and the inner value
-    fn words_filter_as_any() {
-        let filter = WordsFilter { word_count: 1 };
-
-        assert_eq!(filter.word_count, 1);
-        assert_eq!(
-            *filter.as_any().downcast_ref::<WordsFilter>().unwrap(),
-            filter
-        );
-    }
-
-    #[test]
-    /// just a simple test to increase code coverage by hitting as_any and the inner value
-    fn size_filter_as_any() {
-        let filter = SizeFilter { content_length: 1 };
-
-        assert_eq!(filter.content_length, 1);
-        assert_eq!(
-            *filter.as_any().downcast_ref::<SizeFilter>().unwrap(),
-            filter
-        );
-    }
-
-    #[test]
-    /// just a simple test to increase code coverage by hitting as_any and the inner value
-    fn status_code_filter_as_any() {
-        let filter = StatusCodeFilter { filter_code: 200 };
-
-        assert_eq!(filter.filter_code, 200);
-        assert_eq!(
-            *filter.as_any().downcast_ref::<StatusCodeFilter>().unwrap(),
-            filter
-        );
-    }
-
-    #[test]
-    /// just a simple test to increase code coverage by hitting as_any and the inner value
-    fn regex_filter_as_any() {
-        let raw = r".*\.txt$";
-        let compiled = Regex::new(raw).unwrap();
-        let filter = RegexFilter {
-            compiled,
-            raw_string: raw.to_string(),
-        };
-
-        assert_eq!(filter.raw_string, r".*\.txt$");
-        assert_eq!(
-            *filter.as_any().downcast_ref::<RegexFilter>().unwrap(),
-            filter
-        );
-    }
-
-    #[test]
-    /// test should_filter on WilcardFilter where static logic matches
-    fn wildcard_should_filter_when_static_wildcard_found() {
-        let resp = FeroxResponse {
-            text: String::new(),
-            wildcard: true,
-            url: Url::parse("http://localhost").unwrap(),
-            content_length: 100,
-            word_count: 50,
-            line_count: 25,
-            headers: reqwest::header::HeaderMap::new(),
-            status: reqwest::StatusCode::OK,
-        };
-
-        let filter = WildcardFilter {
-            size: 100,
-            dynamic: 0,
-        };
-
-        assert!(filter.should_filter_response(&resp));
-    }
-
-    #[test]
-    /// test should_filter on WilcardFilter where dynamic logic matches
-    fn wildcard_should_filter_when_dynamic_wildcard_found() {
-        let resp = FeroxResponse {
-            text: String::new(),
-            wildcard: true,
-            url: Url::parse("http://localhost/stuff").unwrap(),
-            content_length: 100,
-            word_count: 50,
-            line_count: 25,
-            headers: reqwest::header::HeaderMap::new(),
-            status: reqwest::StatusCode::OK,
-        };
-
-        let filter = WildcardFilter {
-            size: 0,
-            dynamic: 95,
-        };
-
-        assert!(filter.should_filter_response(&resp));
-    }
-
-    #[test]
-    /// test should_filter on RegexFilter where regex matches body
-    fn regexfilter_should_filter_when_regex_matches_on_response_body() {
-        let resp = FeroxResponse {
-            text: String::from("im a body response hurr durr!"),
-            wildcard: false,
-            url: Url::parse("http://localhost/stuff").unwrap(),
-            content_length: 100,
-            word_count: 50,
-            line_count: 25,
-            headers: reqwest::header::HeaderMap::new(),
-            status: reqwest::StatusCode::OK,
-        };
-
-        let raw = r"response...rr";
-
-        let filter = RegexFilter {
-            raw_string: raw.to_string(),
-            compiled: Regex::new(raw).unwrap(),
-        };
-
-        assert!(filter.should_filter_response(&resp));
-    }
-}

src/filters/container.rs

@@ -0,0 +1,56 @@
+use std::sync::Mutex;
+
+use anyhow::Result;
+
+use crate::response::FeroxResponse;
+use crate::{
+    event_handlers::Command::AddToUsizeField, statistics::StatField::WildcardsFiltered,
+    CommandSender,
+};
+
+use super::{FeroxFilter, WildcardFilter};
+
+/// Container around a collection of `FeroxFilters`s
+#[derive(Debug, Default)]
+pub struct FeroxFilters {
+    /// collection of `FeroxFilters`
+    pub filters: Mutex<Vec<Box<dyn FeroxFilter>>>,
+}
+
+/// implementation of FeroxFilter collection
+impl FeroxFilters {
+    /// add a single FeroxFilter to the collection
+    pub fn push(&self, filter: Box<dyn FeroxFilter>) -> Result<()> {
+        if let Ok(mut guard) = self.filters.lock() {
+            if guard.contains(&filter) {
+                return Ok(());
+            }
+
+            guard.push(filter)
+        }
+        Ok(())
+    }
+
+    /// Simple helper to stay DRY; determines whether or not a given `FeroxResponse` should be reported
+    /// to the user or not.
+    pub fn should_filter_response(
+        &self,
+        response: &FeroxResponse,
+        tx_stats: CommandSender,
+    ) -> bool {
+        if let Ok(filters) = self.filters.lock() {
+            for filter in filters.iter() {
+                // wildcard.should_filter goes here
+                if filter.should_filter_response(response) {
+                    if filter.as_any().downcast_ref::<WildcardFilter>().is_some() {
+                        tx_stats
+                            .send(AddToUsizeField(WildcardsFiltered, 1))
+                            .unwrap_or_default();
+                    }
+                    return true;
+                }
+            }
+        }
+        false
+    }
+}

src/filters/init.rs

@@ -0,0 +1,94 @@
+use super::{
+    LinesFilter, RegexFilter, SimilarityFilter, SizeFilter, StatusCodeFilter, WordsFilter,
+};
+use crate::{
+    event_handlers::Handles,
+    response::FeroxResponse,
+    skip_fail,
+    utils::{fmt_err, logged_request},
+    Command::AddFilter,
+    SIMILARITY_THRESHOLD,
+};
+use anyhow::Result;
+use fuzzyhash::FuzzyHash;
+use regex::Regex;
+use reqwest::Url;
+use std::sync::Arc;
+
+/// add all user-supplied filters to the (already started) filters handler
+pub async fn initialize(handles: Arc<Handles>) -> Result<()> {
+    // add any status code filters to filters handler's FeroxFilters  (-C|--filter-status)
+    for code_filter in &handles.config.filter_status {
+        let filter = StatusCodeFilter {
+            filter_code: *code_filter,
+        };
+        let boxed_filter = Box::new(filter);
+        skip_fail!(handles.filters.send(AddFilter(boxed_filter)));
+    }
+
+    // add any line count filters to filters handler's FeroxFilters  (-N|--filter-lines)
+    for lines_filter in &handles.config.filter_line_count {
+        let filter = LinesFilter {
+            line_count: *lines_filter,
+        };
+        let boxed_filter = Box::new(filter);
+        skip_fail!(handles.filters.send(AddFilter(boxed_filter)));
+    }
+
+    // add any line count filters to filters handler's FeroxFilters  (-W|--filter-words)
+    for words_filter in &handles.config.filter_word_count {
+        let filter = WordsFilter {
+            word_count: *words_filter,
+        };
+        let boxed_filter = Box::new(filter);
+        skip_fail!(handles.filters.send(AddFilter(boxed_filter)));
+    }
+
+    // add any line count filters to filters handler's FeroxFilters  (-S|--filter-size)
+    for size_filter in &handles.config.filter_size {
+        let filter = SizeFilter {
+            content_length: *size_filter,
+        };
+        let boxed_filter = Box::new(filter);
+        skip_fail!(handles.filters.send(AddFilter(boxed_filter)));
+    }
+
+    // add any regex filters to filters handler's FeroxFilters  (-X|--filter-regex)
+    for regex_filter in &handles.config.filter_regex {
+        let raw = regex_filter;
+        let compiled = skip_fail!(Regex::new(raw));
+
+        let filter = RegexFilter {
+            raw_string: raw.to_owned(),
+            compiled,
+        };
+        let boxed_filter = Box::new(filter);
+        skip_fail!(handles.filters.send(AddFilter(boxed_filter)));
+    }
+
+    // add any similarity filters to filters handler's FeroxFilters  (--filter-similar-to)
+    for similarity_filter in &handles.config.filter_similar {
+        // url as-is based on input, ignores user-specified url manipulation options (add-slash etc)
+        let url = skip_fail!(Url::parse(similarity_filter));
+
+        // attempt to request the given url
+        let resp = skip_fail!(logged_request(&url, handles.clone()).await);
+
+        // if successful, create a filter based on the response's body
+        let fr = FeroxResponse::from(resp, true, handles.config.output_level).await;
+
+        // hash the response body and store the resulting hash in the filter object
+        let hash = FuzzyHash::new(&fr.text()).to_string();
+
+        let filter = SimilarityFilter {
+            text: hash,
+            threshold: SIMILARITY_THRESHOLD,
+        };
+
+        let boxed_filter = Box::new(filter);
+        skip_fail!(handles.filters.send(AddFilter(boxed_filter)));
+    }
+
+    handles.filters.sync().await?;
+    Ok(())
+}

src/filters/lines.rs

@@ -0,0 +1,33 @@
+use super::*;
+
+/// Simple implementor of FeroxFilter; used to filter out responses based on the number of lines
+/// in a Response body; specified using -N|--filter-lines
+#[derive(Default, Debug, PartialEq)]
+pub struct LinesFilter {
+    /// Number of lines in a Response's body that should be filtered
+    pub line_count: usize,
+}
+
+/// implementation of FeroxFilter for LinesFilter
+impl FeroxFilter for LinesFilter {
+    /// Check `line_count` against what was passed in via -N|--filter-lines
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        let result = response.line_count() == self.line_count;
+
+        log::trace!("exit: should_filter_response -> {}", result);
+
+        result
+    }
+
+    /// Compare one LinesFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/filters/mod.rs

@@ -0,0 +1,28 @@
+//! contains all of feroxbuster's filters
+use std::any::Any;
+use std::fmt::Debug;
+
+use crate::response::FeroxResponse;
+use crate::traits::{FeroxFilter, FeroxSerialize};
+
+pub use self::container::FeroxFilters;
+pub use self::init::initialize;
+pub use self::lines::LinesFilter;
+pub use self::regex::RegexFilter;
+pub use self::similarity::SimilarityFilter;
+pub use self::size::SizeFilter;
+pub use self::status_code::StatusCodeFilter;
+pub use self::wildcard::WildcardFilter;
+pub use self::words::WordsFilter;
+
+mod wildcard;
+mod status_code;
+mod words;
+mod lines;
+mod size;
+mod regex;
+mod similarity;
+mod container;
+#[cfg(test)]
+mod tests;
+mod init;

src/filters/regex.rs

@@ -0,0 +1,46 @@
+use super::*;
+use ::regex::Regex;
+
+/// Simple implementor of FeroxFilter; used to filter out responses based on a given regular
+/// expression; specified using -X|--filter-regex
+#[derive(Debug)]
+pub struct RegexFilter {
+    /// Regular expression to be applied to the response body for filtering, compiled
+    pub compiled: Regex,
+
+    /// Regular expression as passed in on the command line, not compiled
+    pub raw_string: String,
+}
+
+/// implementation of FeroxFilter for RegexFilter
+impl FeroxFilter for RegexFilter {
+    /// Check `expression` against the response body, if the expression matches, the response
+    /// should be filtered out
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        let result = self.compiled.is_match(response.text());
+
+        log::trace!("exit: should_filter_response -> {}", result);
+
+        result
+    }
+
+    /// Compare one SizeFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
+
+/// PartialEq implementation for RegexFilter
+impl PartialEq for RegexFilter {
+    /// Simple comparison of the raw string passed in via the command line
+    fn eq(&self, other: &RegexFilter) -> bool {
+        self.raw_string == other.raw_string
+    }
+}

src/filters/similarity.rs

@@ -0,0 +1,40 @@
+use super::*;
+use fuzzyhash::FuzzyHash;
+
+/// Simple implementor of FeroxFilter; used to filter out responses based on the similarity of a
+/// Response body with a known response; specified using --filter-similar-to
+#[derive(Default, Debug, PartialEq)]
+pub struct SimilarityFilter {
+    /// Response's body to be used for comparison for similarity
+    pub text: String,
+
+    /// Percentage of similarity at which a page is determined to be a near-duplicate of another
+    pub threshold: u32,
+}
+
+/// implementation of FeroxFilter for SimilarityFilter
+impl FeroxFilter for SimilarityFilter {
+    /// Check `FeroxResponse::text` against what was requested from the site passed in via
+    /// --filter-similar-to
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        let other = FuzzyHash::new(&response.text());
+
+        if let Ok(result) = FuzzyHash::compare(&self.text, &other.to_string()) {
+            return result >= self.threshold;
+        }
+
+        // couldn't hash the response, don't filter
+        log::warn!("Could not hash body from {}", response.as_str());
+        false
+    }
+
+    /// Compare one SimilarityFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/filters/size.rs

@@ -0,0 +1,33 @@
+use super::*;
+
+/// Simple implementor of FeroxFilter; used to filter out responses based on the length of a
+/// Response body; specified using -S|--filter-size
+#[derive(Default, Debug, PartialEq)]
+pub struct SizeFilter {
+    /// Overall length of a Response's body that should be filtered
+    pub content_length: u64,
+}
+
+/// implementation of FeroxFilter for SizeFilter
+impl FeroxFilter for SizeFilter {
+    /// Check `content_length` against what was passed in via -S|--filter-size
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        let result = response.content_length() == self.content_length;
+
+        log::trace!("exit: should_filter_response -> {}", result);
+
+        result
+    }
+
+    /// Compare one SizeFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/filters/status_code.rs

@@ -0,0 +1,40 @@
+use super::*;
+
+/// Simple implementor of FeroxFilter; used to filter out status codes specified using
+/// -C|--filter-status
+#[derive(Default, Debug, PartialEq)]
+pub struct StatusCodeFilter {
+    /// Status code that should not be displayed to the user
+    pub filter_code: u16,
+}
+
+/// implementation of FeroxFilter for StatusCodeFilter
+impl FeroxFilter for StatusCodeFilter {
+    /// Check `filter_code` against what was passed in via -C|--filter-status
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        if response.status().as_u16() == self.filter_code {
+            log::debug!(
+                "filtered out {} based on --filter-status of {}",
+                response.url(),
+                self.filter_code
+            );
+            log::trace!("exit: should_filter_response -> true");
+            return true;
+        }
+
+        log::trace!("exit: should_filter_response -> false");
+        false
+    }
+
+    /// Compare one StatusCodeFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/filters/tests.rs

@@ -0,0 +1,228 @@
+use super::*;
+use ::fuzzyhash::FuzzyHash;
+use ::regex::Regex;
+
+#[test]
+/// simply test the default values for wildcardfilter, expect 0, 0
+fn wildcard_filter_default() {
+    let wcf = WildcardFilter::default();
+    assert_eq!(wcf.size, u64::MAX);
+    assert_eq!(wcf.dynamic, u64::MAX);
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn wildcard_filter_as_any() {
+    let filter = WildcardFilter::default();
+    let filter2 = WildcardFilter::default();
+
+    assert!(filter.box_eq(filter2.as_any()));
+
+    assert_eq!(
+        *filter.as_any().downcast_ref::<WildcardFilter>().unwrap(),
+        filter
+    );
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn lines_filter_as_any() {
+    let filter = LinesFilter { line_count: 1 };
+    let filter2 = LinesFilter { line_count: 1 };
+
+    assert!(filter.box_eq(filter2.as_any()));
+
+    assert_eq!(filter.line_count, 1);
+    assert_eq!(
+        *filter.as_any().downcast_ref::<LinesFilter>().unwrap(),
+        filter
+    );
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn words_filter_as_any() {
+    let filter = WordsFilter { word_count: 1 };
+    let filter2 = WordsFilter { word_count: 1 };
+
+    assert!(filter.box_eq(filter2.as_any()));
+
+    assert_eq!(filter.word_count, 1);
+    assert_eq!(
+        *filter.as_any().downcast_ref::<WordsFilter>().unwrap(),
+        filter
+    );
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn size_filter_as_any() {
+    let filter = SizeFilter { content_length: 1 };
+    let filter2 = SizeFilter { content_length: 1 };
+
+    assert!(filter.box_eq(filter2.as_any()));
+
+    assert_eq!(filter.content_length, 1);
+    assert_eq!(
+        *filter.as_any().downcast_ref::<SizeFilter>().unwrap(),
+        filter
+    );
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn status_code_filter_as_any() {
+    let filter = StatusCodeFilter { filter_code: 200 };
+    let filter2 = StatusCodeFilter { filter_code: 200 };
+
+    assert!(filter.box_eq(filter2.as_any()));
+
+    assert_eq!(filter.filter_code, 200);
+    assert_eq!(
+        *filter.as_any().downcast_ref::<StatusCodeFilter>().unwrap(),
+        filter
+    );
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn regex_filter_as_any() {
+    let raw = r".*\.txt$";
+    let compiled = Regex::new(raw).unwrap();
+    let compiled2 = Regex::new(raw).unwrap();
+    let filter = RegexFilter {
+        compiled,
+        raw_string: raw.to_string(),
+    };
+    let filter2 = RegexFilter {
+        compiled: compiled2,
+        raw_string: raw.to_string(),
+    };
+
+    assert!(filter.box_eq(filter2.as_any()));
+
+    assert_eq!(filter.raw_string, r".*\.txt$");
+    assert_eq!(
+        *filter.as_any().downcast_ref::<RegexFilter>().unwrap(),
+        filter
+    );
+}
+
+#[test]
+/// test should_filter on WilcardFilter where static logic matches
+fn wildcard_should_filter_when_static_wildcard_found() {
+    let mut resp = FeroxResponse::default();
+    resp.set_wildcard(true);
+    resp.set_url("http://localhost");
+    resp.set_text(
+        "pellentesque diam volutpat commodo sed egestas egestas fringilla phasellus faucibus",
+    );
+
+    let filter = WildcardFilter {
+        size: 83,
+        dynamic: 0,
+        dont_filter: false,
+    };
+
+    assert!(filter.should_filter_response(&resp));
+}
+
+#[test]
+/// test should_filter on WilcardFilter where static logic matches but response length is 0
+fn wildcard_should_filter_when_static_wildcard_len_is_zero() {
+    let mut resp = FeroxResponse::default();
+    resp.set_wildcard(true);
+    resp.set_url("http://localhost");
+
+    // default WildcardFilter is used in the code that executes when response.content_length() == 0
+    let filter = WildcardFilter::new(false);
+
+    assert!(filter.should_filter_response(&resp));
+}
+
+#[test]
+/// test should_filter on WilcardFilter where dynamic logic matches
+fn wildcard_should_filter_when_dynamic_wildcard_found() {
+    let mut resp = FeroxResponse::default();
+    resp.set_wildcard(true);
+    resp.set_url("http://localhost/stuff");
+    resp.set_text("pellentesque diam volutpat commodo sed egestas egestas fringilla");
+
+    let filter = WildcardFilter {
+        size: 0,
+        dynamic: 59, // content-length - 5 (len('stuff'))
+        dont_filter: false,
+    };
+
+    println!("resp: {:?}: filter: {:?}", resp, filter);
+
+    assert!(filter.should_filter_response(&resp));
+}
+
+#[test]
+/// test should_filter on RegexFilter where regex matches body
+fn regexfilter_should_filter_when_regex_matches_on_response_body() {
+    let mut resp = FeroxResponse::default();
+    resp.set_url("http://localhost/stuff");
+    resp.set_text("im a body response hurr durr!");
+
+    let raw = r"response...rr";
+
+    let filter = RegexFilter {
+        raw_string: raw.to_string(),
+        compiled: Regex::new(raw).unwrap(),
+    };
+
+    assert!(filter.should_filter_response(&resp));
+}
+
+#[test]
+/// a few simple tests for similarity filter
+fn similarity_filter_is_accurate() {
+    let mut resp = FeroxResponse::default();
+    resp.set_url("http://localhost/stuff");
+    resp.set_text("sitting");
+
+    let mut filter = SimilarityFilter {
+        text: FuzzyHash::new("kitten").to_string(),
+        threshold: 95,
+    };
+
+    // kitten/sitting is 57% similar, so a threshold of 95 should not be filtered
+    assert!(!filter.should_filter_response(&resp));
+
+    resp.set_text("");
+    filter.text = String::new();
+    filter.threshold = 100;
+
+    // two empty strings are the same, however ssdeep doesn't accept empty strings, expect false
+    assert!(!filter.should_filter_response(&resp));
+
+    resp.set_text("some data to hash for the purposes of running a test");
+    filter.text = FuzzyHash::new("some data to hash for the purposes of running a te").to_string();
+    filter.threshold = 17;
+
+    assert!(filter.should_filter_response(&resp));
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn similarity_filter_as_any() {
+    let filter = SimilarityFilter {
+        text: String::from("stuff"),
+        threshold: 95,
+    };
+
+    let filter2 = SimilarityFilter {
+        text: String::from("stuff"),
+        threshold: 95,
+    };
+
+    assert!(filter.box_eq(filter2.as_any()));
+
+    assert_eq!(filter.text, "stuff");
+    assert_eq!(
+        *filter.as_any().downcast_ref::<SimilarityFilter>().unwrap(),
+        filter
+    );
+}

src/filters/wildcard.rs

@@ -0,0 +1,108 @@
+use super::*;
+use crate::url::FeroxUrl;
+
+/// Data holder for two pieces of data needed when auto-filtering out wildcard responses
+///
+/// `dynamic` is the size of the response that will later be combined with the length
+/// of the path of the url requested and used to determine interesting pages from custom
+/// 404s where the requested url is reflected back in the response
+///
+/// `size` is size of the response that should be included with filters passed via runtime
+/// configuration and any static wildcard lengths.
+#[derive(Debug, Clone, PartialEq)]
+pub struct WildcardFilter {
+    /// size of the response that will later be combined with the length of the path of the url
+    /// requested
+    pub dynamic: u64,
+
+    /// size of the response that should be included with filters passed via runtime configuration
+    pub size: u64,
+
+    /// whether or not the user passed -D on the command line
+    pub(super) dont_filter: bool,
+}
+
+/// implementation of WildcardFilter
+impl WildcardFilter {
+    /// given a boolean representing whether -D was used or not, create a new WildcardFilter
+    pub fn new(dont_filter: bool) -> Self {
+        Self {
+            dont_filter,
+            ..Default::default()
+        }
+    }
+}
+
+/// implement default that populates both values with u64::MAX
+impl Default for WildcardFilter {
+    /// populate both values with u64::MAX
+    fn default() -> Self {
+        Self {
+            dont_filter: false,
+            size: u64::MAX,
+            dynamic: u64::MAX,
+        }
+    }
+}
+
+/// implementation of FeroxFilter for WildcardFilter
+impl FeroxFilter for WildcardFilter {
+    /// Examine size, dynamic, and content_len to determine whether or not the response received
+    /// is a wildcard response and therefore should be filtered out
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        // quick return if dont_filter is set
+        if self.dont_filter {
+            // --dont-filter applies specifically to wildcard filters, it is not a 100% catch all
+            // for not filtering anything.  As such, it should live in the implementation of
+            // a wildcard filter
+            return false;
+        }
+
+        if self.size != u64::MAX && self.size == response.content_length() {
+            // static wildcard size found during testing
+            // size isn't default, size equals response length, and auto-filter is on
+            log::debug!("static wildcard: filtered out {}", response.url());
+            log::trace!("exit: should_filter_response -> true");
+            return true;
+        }
+
+        if self.size == u64::MAX && response.content_length() == 0 {
+            // static wildcard size found during testing
+            // but response length was zero; pointed out by @Tib3rius
+            log::debug!("static wildcard: filtered out {}", response.url());
+            log::trace!("exit: should_filter_response -> true");
+            return true;
+        }
+
+        if self.dynamic != u64::MAX {
+            // dynamic wildcard offset found during testing
+
+            // I'm about to manually split this url path instead of using reqwest::Url's
+            // builtin parsing. The reason is that they call .split() on the url path
+            // except that I don't want an empty string taking up the last index in the
+            // event that the url ends with a forward slash.  It's ugly enough to be split
+            // into its own function for readability.
+            let url_len = FeroxUrl::path_length_of_url(response.url());
+
+            if url_len + self.dynamic == response.content_length() {
+                log::debug!("dynamic wildcard: filtered out {}", response.url());
+                log::trace!("exit: should_filter_response -> true");
+                return true;
+            }
+        }
+        log::trace!("exit: should_filter_response -> false");
+        false
+    }
+
+    /// Compare one WildcardFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/filters/words.rs

@@ -0,0 +1,33 @@
+use super::*;
+
+/// Simple implementor of FeroxFilter; used to filter out responses based on the number of words
+/// in a Response body; specified using -W|--filter-words
+#[derive(Default, Debug, PartialEq)]
+pub struct WordsFilter {
+    /// Number of words in a Response's body that should be filtered
+    pub word_count: usize,
+}
+
+/// implementation of FeroxFilter for WordsFilter
+impl FeroxFilter for WordsFilter {
+    /// Check `word_count` against what was passed in via -W|--filter-words
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        let result = response.word_count() == self.word_count;
+
+        log::trace!("exit: should_filter_response -> {}", result);
+
+        result
+    }
+
+    /// Compare one WordsFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/heuristics.rs

@@ -1,247 +1,250 @@
-use crate::{
-    config::{CONFIGURATION, PROGRESS_PRINTER},
-    filters::WildcardFilter,
-    scanner::should_filter_response,
-    utils::{ferox_print, format_url, get_url_path_length, make_request, status_colorizer},
-    FeroxResponse,
-};
+use std::sync::Arc;
+
+use anyhow::{bail, Result};
 use console::style;
-use indicatif::ProgressBar;
-use tokio::sync::mpsc::UnboundedSender;
 use uuid::Uuid;
 
+use crate::{
+    config::OutputLevel,
+    event_handlers::{Command, Handles},
+    filters::WildcardFilter,
+    progress::PROGRESS_PRINTER,
+    response::FeroxResponse,
+    skip_fail,
+    url::FeroxUrl,
+    utils::{ferox_print, fmt_err, logged_request, status_colorizer},
+};
+
 /// length of a standard UUID, used when determining wildcard responses
 const UUID_LENGTH: u64 = 32;
 
-/// Simple helper to return a uuid, formatted as lowercase without hyphens
-///
-/// `length` determines the number of uuids to string together. Each uuid
-/// is 32 characters long. So, a length of 1 return a 32 character string,
-/// a length of 2 returns a 64 character string, and so on...
-fn unique_string(length: usize) -> String {
-    log::trace!("enter: unique_string({})", length);
-    let mut ids = vec![];
-
-    for _ in 0..length {
-        ids.push(Uuid::new_v4().to_simple().to_string());
-    }
-
-    let unique_id = ids.join("");
-
-    log::trace!("exit: unique_string -> {}", unique_id);
-    unique_id
+/// wrapper around ugly string formatting
+macro_rules! format_template {
+    ($template:expr, $length:expr) => {
+        format!(
+            $template,
+            status_colorizer("WLD"),
+            "-",
+            "-",
+            "-",
+            style("auto-filtering").yellow(),
+            style($length).cyan(),
+            style("--dont-filter").yellow()
+        )
+    };
 }
 
-/// Tests the given url to see if it issues a wildcard response
-///
-/// In the event that url returns a wildcard response, a
-/// [WildcardFilter](struct.WildcardFilter.html) is created and returned to the caller.
-pub async fn wildcard_test(
-    target_url: &str,
-    bar: ProgressBar,
-    tx_term: UnboundedSender<FeroxResponse>,
-) -> Option<WildcardFilter> {
-    log::trace!(
-        "enter: wildcard_test({:?}, {:?}, {:?})",
-        target_url,
-        bar,
-        tx_term
-    );
+/// container for heuristics related info
+pub struct HeuristicTests {
+    /// Handles object for event handler interaction
+    handles: Arc<Handles>,
+}
 
-    if CONFIGURATION.dont_filter {
-        // early return, dont_filter scans don't need tested
-        log::trace!("exit: wildcard_test -> None");
-        return None;
+/// HeuristicTests implementation
+impl HeuristicTests {
+    /// create a new HeuristicTests struct
+    pub fn new(handles: Arc<Handles>) -> Self {
+        Self { handles }
     }
 
-    let tx_clone_one = tx_term.clone();
-    let tx_clone_two = tx_term.clone();
+    /// Simple helper to return a uuid, formatted as lowercase without hyphens
+    ///
+    /// `length` determines the number of uuids to string together. Each uuid
+    /// is 32 characters long. So, a length of 1 return a 32 character string,
+    /// a length of 2 returns a 64 character string, and so on...
+    fn unique_string(&self, length: usize) -> String {
+        log::trace!("enter: unique_string({})", length);
+        let mut ids = vec![];
 
-    if let Some(ferox_response) = make_wildcard_request(&target_url, 1, tx_clone_one).await {
-        bar.inc(1);
+        for _ in 0..length {
+            ids.push(Uuid::new_v4().to_simple().to_string());
+        }
+
+        let unique_id = ids.join("");
+
+        log::trace!("exit: unique_string -> {}", unique_id);
+        unique_id
+    }
+
+    /// wrapper for sending a filter to the filters event handler
+    fn send_filter(&self, filter: WildcardFilter) -> Result<()> {
+        self.handles
+            .filters
+            .send(Command::AddFilter(Box::new(filter)))
+    }
+
+    /// Tests the given url to see if it issues a wildcard response
+    ///
+    /// In the event that url returns a wildcard response, a
+    /// [WildcardFilter](struct.WildcardFilter.html) is created and sent to the filters event
+    /// handler.
+    ///
+    /// Returns the number of times to increment the caller's progress bar
+    pub async fn wildcard(&self, target_url: &str) -> Result<u64> {
+        log::trace!("enter: wildcard_test({:?})", target_url);
+
+        if self.handles.config.dont_filter {
+            // early return, dont_filter scans don't need tested
+            log::trace!("exit: wildcard_test -> 0");
+            return Ok(0);
+        }
+
+        let ferox_url = FeroxUrl::from_string(target_url, self.handles.clone());
+
+        let ferox_response = self.make_wildcard_request(&ferox_url, 1).await?;
 
         // found a wildcard response
-        let mut wildcard = WildcardFilter::default();
+        let mut wildcard = WildcardFilter::new(self.handles.config.dont_filter);
 
         let wc_length = ferox_response.content_length();
 
         if wc_length == 0 {
-            log::trace!("exit: wildcard_test -> Some({:?})", wildcard);
-            return Some(wildcard);
+            log::trace!("exit: wildcard_test -> 1");
+            self.send_filter(wildcard)?;
+            return Ok(1);
         }
 
         // content length of wildcard is non-zero, perform additional tests:
         //   make a second request, with a known-sized (64) longer request
-        if let Some(resp_two) = make_wildcard_request(&target_url, 3, tx_clone_two).await {
-            bar.inc(1);
+        let resp_two = self.make_wildcard_request(&ferox_url, 3).await?;
 
-            let wc2_length = resp_two.content_length();
+        let wc2_length = resp_two.content_length();
 
-            if wc2_length == wc_length + (UUID_LENGTH * 2) {
-                // second length is what we'd expect to see if the requested url is
-                // reflected in the response along with some static content; aka custom 404
-                let url_len = get_url_path_length(&ferox_response.url());
+        if wc2_length == wc_length + (UUID_LENGTH * 2) {
+            // second length is what we'd expect to see if the requested url is
+            // reflected in the response along with some static content; aka custom 404
+            let url_len = ferox_url.path_length()?;
 
-                wildcard.dynamic = wc_length - url_len;
+            wildcard.dynamic = wc_length - url_len;
 
-                if !CONFIGURATION.quiet {
-                    let msg = format!(
-                            "{} {:>9} {:>9} {:>9} Wildcard response is dynamic; {} ({} + url length) responses; toggle this behavior by using {}\n",
-                            status_colorizer("WLD"),
-                            "-",
-                            "-",
-                            "-",
-                            style("auto-filtering").yellow(),
-                            style(wc_length - url_len).cyan(),
-                            style("--dont-filter").yellow()
-                    );
-
-                    ferox_print(&msg, &PROGRESS_PRINTER);
-                }
-            } else if wc_length == wc2_length {
-                wildcard.size = wc_length;
-
-                if !CONFIGURATION.quiet {
-                    let msg = format!(
-                        "{} {:>9} {:>9} {:>9} Wildcard response is static; {} {} responses; toggle this behavior by using {}\n",
-                        status_colorizer("WLD"),
-                        "-",
-                        "-",
-                        "-",
-                        style("auto-filtering").yellow(),
-                        style(wc_length).cyan(),
-                        style("--dont-filter").yellow()
-                    );
-
-                    ferox_print(&msg, &PROGRESS_PRINTER);
-                }
+            if matches!(
+                self.handles.config.output_level,
+                OutputLevel::Default | OutputLevel::Quiet
+            ) {
+                let msg = format_template!("{} {:>9} {:>9} {:>9} Wildcard response is dynamic; {} ({} + url length) responses; toggle this behavior by using {}\n", wildcard.dynamic);
+                ferox_print(&msg, &PROGRESS_PRINTER);
+            }
+        } else if wc_length == wc2_length {
+            wildcard.size = wc_length;
+
+            if matches!(
+                self.handles.config.output_level,
+                OutputLevel::Default | OutputLevel::Quiet
+            ) {
+                let msg = format_template!("{} {:>9} {:>9} {:>9} Wildcard response is static; {} {} responses; toggle this behavior by using {}\n", wildcard.size);
+                ferox_print(&msg, &PROGRESS_PRINTER);
             }
-        } else {
-            bar.inc(2);
         }
 
-        log::trace!("exit: wildcard_test -> Some({:?})", wildcard);
-        return Some(wildcard);
+        self.send_filter(wildcard)?;
+
+        log::trace!("exit: wildcard_test");
+        Ok(2)
     }
 
-    log::trace!("exit: wildcard_test -> None");
-    None
-}
+    /// Generates a uuid and appends it to the given target url. The reasoning is that the randomly
+    /// generated unique string should not exist on and be served by the target web server.
+    ///
+    /// Once the unique url is created, the request is sent to the server. If the server responds
+    /// back with a valid status code, the response is considered to be a wildcard response. If that
+    /// wildcard response has a 3xx status code, that redirection location is displayed to the user.
+    async fn make_wildcard_request(
+        &self,
+        target: &FeroxUrl,
+        length: usize,
+    ) -> Result<FeroxResponse> {
+        log::trace!("enter: make_wildcard_request({}, {})", target, length);
 
-/// Generates a uuid and appends it to the given target url. The reasoning is that the randomly
-/// generated unique string should not exist on and be served by the target web server.
-///
-/// Once the unique url is created, the request is sent to the server. If the server responds
-/// back with a valid status code, the response is considered to be a wildcard response. If that
-/// wildcard response has a 3xx status code, that redirection location is displayed to the user.
-async fn make_wildcard_request(
-    target_url: &str,
-    length: usize,
-    tx_file: UnboundedSender<FeroxResponse>,
-) -> Option<FeroxResponse> {
-    log::trace!(
-        "enter: make_wildcard_request({}, {}, {:?})",
-        target_url,
-        length,
-        tx_file
-    );
+        let unique_str = self.unique_string(length);
+        let nonexistent_url = target.format(&unique_str, None)?;
 
-    let unique_str = unique_string(length);
+        let response = logged_request(&nonexistent_url.to_owned(), self.handles.clone()).await?;
 
-    let nonexistent = match format_url(
-        target_url,
-        &unique_str,
-        CONFIGURATION.add_slash,
-        &CONFIGURATION.queries,
-        None,
-    ) {
-        Ok(url) => url,
-        Err(e) => {
-            log::error!("{}", e);
-            log::trace!("exit: make_wildcard_request -> None");
-            return None;
-        }
-    };
+        if self
+            .handles
+            .config
+            .status_codes
+            .contains(&response.status().as_u16())
+        {
+            // found a wildcard response
+            let mut ferox_response =
+                FeroxResponse::from(response, true, self.handles.config.output_level).await;
+            ferox_response.set_wildcard(true);
 
-    match make_request(&CONFIGURATION.client, &nonexistent.to_owned()).await {
-        Ok(response) => {
-            if CONFIGURATION
-                .status_codes
-                .contains(&response.status().as_u16())
+            if self
+                .handles
+                .filters
+                .data
+                .should_filter_response(&ferox_response, self.handles.stats.tx.clone())
             {
-                // found a wildcard response
-                let mut ferox_response = FeroxResponse::from(response, true).await;
-                ferox_response.wildcard = true;
+                bail!("filtered response")
+            }
 
-                if !CONFIGURATION.quiet
-                    && !should_filter_response(&ferox_response)
-                    && tx_file.send(ferox_response.clone()).is_err()
-                {
-                    return None;
+            if matches!(
+                self.handles.config.output_level,
+                OutputLevel::Default | OutputLevel::Quiet
+            ) {
+                let boxed = Box::new(ferox_response.clone());
+                self.handles.output.send(Command::Report(boxed))?;
+            }
+
+            log::trace!("exit: make_wildcard_request -> {}", ferox_response);
+            return Ok(ferox_response);
+        }
+
+        log::trace!("exit: make_wildcard_request -> Err");
+        bail!("uninteresting status code")
+    }
+
+    /// Simply tries to connect to all given sites before starting to scan
+    ///
+    /// In the event that no sites can be reached, the program will exit.
+    ///
+    /// Any urls that are found to be alive are returned to the caller.
+    pub async fn connectivity(&self, target_urls: &[String]) -> Result<Vec<String>> {
+        log::trace!("enter: connectivity_test({:?})", target_urls);
+
+        let mut good_urls = vec![];
+
+        for target_url in target_urls {
+            let url = FeroxUrl::from_string(target_url, self.handles.clone());
+            let request = skip_fail!(url.format("", None));
+
+            let result = logged_request(&request, self.handles.clone()).await;
+
+            match result {
+                Ok(_) => {
+                    good_urls.push(target_url.to_owned());
                 }
-
-                log::trace!("exit: make_wildcard_request -> {}", ferox_response);
-                return Some(ferox_response);
-            }
-        }
-        Err(e) => {
-            log::warn!("{}", e);
-            log::trace!("exit: make_wildcard_request -> None");
-            return None;
-        }
-    }
-    log::trace!("exit: make_wildcard_request -> None");
-    None
-}
-
-/// Simply tries to connect to all given sites before starting to scan
-///
-/// In the event that no sites can be reached, the program will exit.
-///
-/// Any urls that are found to be alive are returned to the caller.
-pub async fn connectivity_test(target_urls: &[String]) -> Vec<String> {
-    log::trace!("enter: connectivity_test({:?})", target_urls);
-
-    let mut good_urls = vec![];
-
-    for target_url in target_urls {
-        let request = match format_url(
-            target_url,
-            "",
-            CONFIGURATION.add_slash,
-            &CONFIGURATION.queries,
-            None,
-        ) {
-            Ok(url) => url,
-            Err(e) => {
-                log::error!("{}", e);
-                continue;
-            }
-        };
-
-        match make_request(&CONFIGURATION.client, &request).await {
-            Ok(_) => {
-                good_urls.push(target_url.to_owned());
-            }
-            Err(e) => {
-                if !CONFIGURATION.quiet {
-                    ferox_print(
-                        &format!("Could not connect to {}, skipping...", target_url),
-                        &PROGRESS_PRINTER,
-                    );
+                Err(e) => {
+                    if matches!(
+                        self.handles.config.output_level,
+                        OutputLevel::Default | OutputLevel::Quiet
+                    ) {
+                        if e.to_string().contains(":SSL") {
+                            ferox_print(
+                                &format!("Could not connect to {} due to SSL errors (run with -k to ignore), skipping...", target_url),
+                                &PROGRESS_PRINTER,
+                            );
+                        } else {
+                            ferox_print(
+                                &format!("Could not connect to {}, skipping...", target_url),
+                                &PROGRESS_PRINTER,
+                            );
+                        }
+                    }
+                    log::warn!("{}", e);
                 }
-                log::error!("{}", e);
             }
         }
+
+        if good_urls.is_empty() {
+            bail!("Could not connect to any target provided");
+        }
+
+        log::trace!("exit: connectivity_test -> {:?}", good_urls);
+        Ok(good_urls)
     }
-
-    if good_urls.is_empty() {
-        log::error!("Could not connect to any target provided, exiting.");
-    }
-
-    log::trace!("exit: connectivity_test -> {:?}", good_urls);
-
-    good_urls
 }
 
 #[cfg(test)]
@@ -251,16 +254,10 @@ mod tests {
     #[test]
     /// request a unique string of 32bytes * a value returns correct result
     fn heuristics_unique_string_returns_correct_length() {
+        let (handles, _) = Handles::for_testing(None, None);
+        let tester = HeuristicTests::new(Arc::new(handles));
         for i in 0..10 {
-            assert_eq!(unique_string(i).len(), i * 32);
+            assert_eq!(tester.unique_string(i).len(), i * 32);
         }
     }
-
-    #[test]
-    /// simply test the default values for wildcardfilter, expect 0, 0
-    fn heuristics_wildcardfilter_dafaults() {
-        let wcf = WildcardFilter::default();
-        assert_eq!(wcf.size, 0);
-        assert_eq!(wcf.dynamic, 0);
-    }
 }

src/lib.rs

@@ -1,55 +1,52 @@
+use anyhow::Result;
+use reqwest::StatusCode;
+use tokio::{
+    sync::mpsc::{UnboundedReceiver, UnboundedSender},
+    task::JoinHandle,
+};
+
+use crate::event_handlers::Command;
+
 pub mod banner;
-pub mod client;
 pub mod config;
-pub mod extractor;
+mod client;
+pub mod event_handlers;
 pub mod filters;
 pub mod heuristics;
 pub mod logger;
-pub mod parser;
+mod parser;
 pub mod progress;
-pub mod reporter;
 pub mod scan_manager;
 pub mod scanner;
+pub mod statistics;
+mod traits;
 pub mod utils;
+mod extractor;
+mod macros;
+mod url;
+mod response;
+mod message;
 
-use crate::utils::{get_url_path_length, status_colorizer};
-use console::{style, Color};
-use reqwest::header::{HeaderName, HeaderValue};
-use reqwest::{header::HeaderMap, Response, StatusCode, Url};
-use serde::{ser::SerializeStruct, Deserialize, Deserializer, Serialize, Serializer};
-use serde_json::Value;
-use std::collections::HashMap;
-use std::convert::{TryFrom, TryInto};
-use std::str::FromStr;
-use std::{error, fmt};
-use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender};
+/// Alias for tokio::sync::mpsc::UnboundedSender<Command>
+pub(crate) type CommandSender = UnboundedSender<Command>;
 
-/// Generic Result type to ease error handling in async contexts
-pub type FeroxResult<T> = std::result::Result<T, Box<dyn error::Error + Send + Sync + 'static>>;
+/// Alias for tokio::sync::mpsc::UnboundedSender<Command>
+pub(crate) type CommandReceiver = UnboundedReceiver<Command>;
 
-/// Simple Error implementation to allow for custom error returns
-#[derive(Debug, Default)]
-pub struct FeroxError {
-    /// fancy string that can be printed via Display
-    pub message: String,
-}
-
-impl error::Error for FeroxError {}
-
-impl fmt::Display for FeroxError {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "{}", &self.message)
-    }
-}
+/// Alias for tokio::task::JoinHandle<anyhow::Result<()>>
+pub(crate) type Joiner = JoinHandle<Result<()>>;
 
 /// Generic mpsc::unbounded_channel type to tidy up some code
-pub type FeroxChannel<T> = (UnboundedSender<T>, UnboundedReceiver<T>);
+pub(crate) type FeroxChannel<T> = (UnboundedSender<T>, UnboundedReceiver<T>);
 
 /// Version pulled from Cargo.toml at compile time
-pub const VERSION: &str = env!("CARGO_PKG_VERSION");
+pub(crate) const VERSION: &str = env!("CARGO_PKG_VERSION");
 
 /// Maximum number of file descriptors that can be opened during a scan
-pub const DEFAULT_OPEN_FILE_LIMIT: usize = 8192;
+pub const DEFAULT_OPEN_FILE_LIMIT: u64 = 8192;
+
+/// Default value used to determine near-duplicate web pages (equivalent to 95%)
+pub const SIMILARITY_THRESHOLD: u32 = 95;
 
 /// Default wordlist to use when `-w|--wordlist` isn't specified and not `wordlist` isn't set
 /// in a [ferox-config.toml](constant.DEFAULT_CONFIG_NAME.html) config file.
@@ -60,7 +57,10 @@ pub const DEFAULT_WORDLIST: &str =
     "/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt";
 
 /// Number of milliseconds to wait between polls of `PAUSE_SCAN` when user pauses a scan
-pub static SLEEP_DURATION: u64 = 500;
+pub(crate) const SLEEP_DURATION: u64 = 500;
+
+/// The percentage of requests as errors it takes to be deemed too high
+pub const HIGH_ERROR_RATIO: f64 = 0.90;
 
 /// Default list of status codes to report
 ///
@@ -73,7 +73,8 @@ pub static SLEEP_DURATION: u64 = 500;
 /// * 401 Unauthorized
 /// * 403 Forbidden
 /// * 405 Method Not Allowed
-pub const DEFAULT_STATUS_CODES: [StatusCode; 9] = [
+/// * 500 Internal Server Error
+pub const DEFAULT_STATUS_CODES: [StatusCode; 10] = [
     StatusCode::OK,
     StatusCode::NO_CONTENT,
     StatusCode::MOVED_PERMANENTLY,
@@ -83,6 +84,7 @@ pub const DEFAULT_STATUS_CODES: [StatusCode; 9] = [
     StatusCode::UNAUTHORIZED,
     StatusCode::FORBIDDEN,
     StatusCode::METHOD_NOT_ALLOWED,
+    StatusCode::INTERNAL_SERVER_ERROR,
 ];
 
 /// Default filename for config file settings
@@ -90,443 +92,6 @@ pub const DEFAULT_STATUS_CODES: [StatusCode; 9] = [
 /// Expected location is in the same directory as the feroxbuster binary.
 pub const DEFAULT_CONFIG_NAME: &str = "ferox-config.toml";
 
-/// FeroxSerialize trait; represents different types that are Serialize and also implement
-/// as_str / as_json methods
-pub trait FeroxSerialize: Serialize {
-    /// Return a String representation of the object, generally the human readable version of the
-    /// implementor
-    fn as_str(&self) -> String;
-
-    /// Return an NDJSON representation of the object
-    fn as_json(&self) -> String;
-}
-
-/// A `FeroxResponse`, derived from a `Response` to a submitted `Request`
-#[derive(Debug, Clone)]
-pub struct FeroxResponse {
-    /// The final `Url` of this `FeroxResponse`
-    url: Url,
-
-    /// The `StatusCode` of this `FeroxResponse`
-    status: StatusCode,
-
-    /// The full response text
-    text: String,
-
-    /// The content-length of this response, if known
-    content_length: u64,
-
-    /// The number of lines contained in the body of this response, if known
-    line_count: usize,
-
-    /// The number of words contained in the body of this response, if known
-    word_count: usize,
-
-    /// The `Headers` of this `FeroxResponse`
-    headers: HeaderMap,
-
-    /// Wildcard response status
-    wildcard: bool,
-}
-
-/// Implement Display for FeroxResponse
-impl fmt::Display for FeroxResponse {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(
-            f,
-            "FeroxResponse {{ url: {}, status: {}, content-length: {} }}",
-            self.url(),
-            self.status(),
-            self.content_length()
-        )
-    }
-}
-
-/// `FeroxResponse` implementation
-impl FeroxResponse {
-    /// Get the `StatusCode` of this `FeroxResponse`
-    pub fn status(&self) -> &StatusCode {
-        &self.status
-    }
-
-    /// Get the final `Url` of this `FeroxResponse`.
-    pub fn url(&self) -> &Url {
-        &self.url
-    }
-
-    /// Get the full response text
-    pub fn text(&self) -> &str {
-        &self.text
-    }
-
-    /// Get the `Headers` of this `FeroxResponse`
-    pub fn headers(&self) -> &HeaderMap {
-        &self.headers
-    }
-
-    /// Get the content-length of this response, if known
-    pub fn content_length(&self) -> u64 {
-        self.content_length
-    }
-
-    /// Set `FeroxResponse`'s `url` attribute, has no affect if an error occurs
-    pub fn set_url(&mut self, url: &str) {
-        match Url::parse(&url) {
-            Ok(url) => {
-                self.url = url;
-            }
-            Err(e) => {
-                log::error!("Could not parse {} into a Url: {}", url, e);
-            }
-        };
-    }
-
-    /// Make a reasonable guess at whether the response is a file or not
-    ///
-    /// Examines the last part of a path to determine if it has an obvious extension
-    /// i.e. http://localhost/some/path/stuff.js where stuff.js indicates a file
-    ///
-    /// Additionally, inspects query parameters, as they're also often indicative of a file
-    pub fn is_file(&self) -> bool {
-        let has_extension = match self.url.path_segments() {
-            Some(path) => {
-                if let Some(last) = path.last() {
-                    last.contains('.') // last segment has some sort of extension, probably
-                } else {
-                    false
-                }
-            }
-            None => false,
-        };
-
-        self.url.query_pairs().count() > 0 || has_extension
-    }
-
-    /// Returns line count of the response text.
-    pub fn line_count(&self) -> usize {
-        self.line_count
-    }
-
-    /// Returns word count of the response text.
-    pub fn word_count(&self) -> usize {
-        self.word_count
-    }
-
-    /// Create a new `FeroxResponse` from the given `Response`
-    pub async fn from(response: Response, read_body: bool) -> Self {
-        let url = response.url().clone();
-        let status = response.status();
-        let headers = response.headers().clone();
-        let content_length = response.content_length().unwrap_or(0);
-
-        let text = if read_body {
-            // .text() consumes the response, must be called last
-            // additionally, --extract-links is currently the only place we use the body of the
-            // response, so we forego the processing if not performing extraction
-            match response.text().await {
-                // await the response's body
-                Ok(text) => text,
-                Err(e) => {
-                    log::error!("Could not parse body from response: {}", e);
-                    String::new()
-                }
-            }
-        } else {
-            String::new()
-        };
-
-        let line_count = text.lines().count();
-        let word_count = text.lines().map(|s| s.split_whitespace().count()).sum();
-
-        FeroxResponse {
-            url,
-            status,
-            content_length,
-            text,
-            headers,
-            line_count,
-            word_count,
-            wildcard: false,
-        }
-    }
-}
-
-/// Implement FeroxSerialusize::from(ize for FeroxRespons)e
-impl FeroxSerialize for FeroxResponse {
-    /// Simple wrapper around create_report_string
-    fn as_str(&self) -> String {
-        let lines = self.line_count().to_string();
-        let words = self.word_count().to_string();
-        let chars = self.content_length().to_string();
-        let status = self.status().as_str();
-        let wild_status = status_colorizer("WLD");
-
-        if self.wildcard {
-            // response is a wildcard, special messages abound when this is the case...
-
-            // create the base message
-            let mut message = format!(
-                "{} {:>8}l {:>8}w {:>8}c Got {} for {} (url length: {})\n",
-                wild_status,
-                lines,
-                words,
-                chars,
-                status_colorizer(&status),
-                self.url(),
-                get_url_path_length(&self.url())
-            );
-
-            if self.status().is_redirection() {
-                // when it's a redirect, show where it goes, if possible
-                if let Some(next_loc) = self.headers().get("Location") {
-                    let next_loc_str = next_loc.to_str().unwrap_or("Unknown");
-
-                    let redirect_msg = format!(
-                        "{} {:>9} {:>9} {:>9} {} redirects to => {}\n",
-                        wild_status,
-                        "-",
-                        "-",
-                        "-",
-                        self.url(),
-                        next_loc_str
-                    );
-
-                    message.push_str(&redirect_msg);
-                }
-            }
-
-            // base message + redirection message (if appropriate)
-            message
-        } else {
-            // not a wildcard, just create a normal entry
-            utils::create_report_string(
-                self.status.as_str(),
-                &lines,
-                &words,
-                &chars,
-                self.url().as_str(),
-            )
-        }
-    }
-
-    /// Create an NDJSON representation of the FeroxResponse
-    ///
-    /// (expanded for clarity)
-    /// ex:
-    /// {
-    ///    "type":"response",
-    ///    "url":"https://localhost.com/images",
-    ///    "path":"/images",
-    ///    "status":301,
-    ///    "content_length":179,
-    ///    "line_count":10,
-    ///    "word_count":16,
-    ///    "headers":{
-    ///       "x-content-type-options":"nosniff",
-    ///       "strict-transport-security":"max-age=31536000; includeSubDomains",
-    ///       "x-frame-options":"SAMEORIGIN",
-    ///       "connection":"keep-alive",
-    ///       "server":"nginx/1.16.1",
-    ///       "content-type":"text/html; charset=UTF-8",
-    ///       "referrer-policy":"origin-when-cross-origin",
-    ///       "content-security-policy":"default-src 'none'",
-    ///       "access-control-allow-headers":"X-Requested-With",
-    ///       "x-xss-protection":"1; mode=block",
-    ///       "content-length":"179",
-    ///       "date":"Mon, 23 Nov 2020 15:33:24 GMT",
-    ///       "location":"/images/",
-    ///       "access-control-allow-origin":"https://localhost.com"
-    ///    }
-    /// }\n
-    fn as_json(&self) -> String {
-        if let Ok(mut json) = serde_json::to_string(&self) {
-            json.push('\n');
-            json
-        } else {
-            format!("{{\"error\":\"could not convert {} to json\"}}", self.url())
-        }
-    }
-}
-
-/// Serialize implementation for FeroxResponse
-impl Serialize for FeroxResponse {
-    /// Function that handles serialization of a FeroxResponse to NDJSON
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        let mut headers = HashMap::new();
-        let mut state = serializer.serialize_struct("FeroxResponse", 7)?;
-
-        // need to convert the HeaderMap to a HashMap in order to pass it to the serializer
-        for (key, value) in &self.headers {
-            let k = key.as_str().to_owned();
-            let v = String::from_utf8_lossy(value.as_bytes());
-            headers.insert(k, v);
-        }
-
-        state.serialize_field("type", "response")?;
-        state.serialize_field("url", self.url.as_str())?;
-        state.serialize_field("path", self.url.path())?;
-        state.serialize_field("wildcard", &self.wildcard)?;
-        state.serialize_field("status", &self.status.as_u16())?;
-        state.serialize_field("content_length", &self.content_length)?;
-        state.serialize_field("line_count", &self.line_count)?;
-        state.serialize_field("word_count", &self.word_count)?;
-        state.serialize_field("headers", &headers)?;
-
-        state.end()
-    }
-}
-
-/// Deserialize implementation for FeroxResponse
-impl<'de> Deserialize<'de> for FeroxResponse {
-    /// Deserialize a FeroxResponse from a serde_json::Value
-    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
-    where
-        D: Deserializer<'de>,
-    {
-        let mut response = Self {
-            url: Url::parse("http://localhost").unwrap(),
-            status: StatusCode::OK,
-            text: String::new(),
-            content_length: 0,
-            headers: HeaderMap::new(),
-            wildcard: false,
-            line_count: 0,
-            word_count: 0,
-        };
-
-        let map: HashMap<String, Value> = HashMap::deserialize(deserializer)?;
-
-        for (key, value) in &map {
-            match key.as_str() {
-                "url" => {
-                    if let Some(url) = value.as_str() {
-                        if let Ok(parsed) = Url::parse(url) {
-                            response.url = parsed;
-                        }
-                    }
-                }
-                "status" => {
-                    if let Some(num) = value.as_u64() {
-                        if let Ok(smaller) = u16::try_from(num) {
-                            if let Ok(status) = StatusCode::from_u16(smaller) {
-                                response.status = status;
-                            }
-                        }
-                    }
-                }
-                "content_length" => {
-                    if let Some(num) = value.as_u64() {
-                        response.content_length = num;
-                    }
-                }
-                "line_count" => {
-                    if let Some(num) = value.as_u64() {
-                        response.line_count = num.try_into().unwrap_or_default();
-                    }
-                }
-                "word_count" => {
-                    if let Some(num) = value.as_u64() {
-                        response.word_count = num.try_into().unwrap_or_default();
-                    }
-                }
-                "headers" => {
-                    let mut headers = HeaderMap::<HeaderValue>::default();
-
-                    if let Some(map_headers) = value.as_object() {
-                        for (h_key, h_value) in map_headers {
-                            let h_value_str = h_value.as_str().unwrap_or("");
-                            let h_name = HeaderName::from_str(h_key)
-                                .unwrap_or_else(|_| HeaderName::from_str("Unknown").unwrap());
-                            let h_value_parsed = HeaderValue::from_str(h_value_str)
-                                .unwrap_or_else(|_| HeaderValue::from_str("Unknown").unwrap());
-                            headers.insert(h_name, h_value_parsed);
-                        }
-                    }
-
-                    response.headers = headers;
-                }
-                "wildcard" => {
-                    if let Some(result) = value.as_bool() {
-                        response.wildcard = result;
-                    }
-                }
-                _ => {}
-            }
-        }
-
-        Ok(response)
-    }
-}
-
-#[derive(Serialize, Deserialize, Default)]
-/// Representation of a log entry, can be represented as a human readable string or JSON
-pub struct FeroxMessage {
-    #[serde(rename = "type")]
-    /// Name of this type of struct, used for serialization, i.e. `{"type":"log"}`
-    kind: String,
-
-    /// The log message
-    pub message: String,
-
-    /// The log level
-    pub level: String,
-
-    /// The number of seconds elapsed since the scan started
-    pub time_offset: f32,
-
-    /// The module from which log::* was called
-    pub module: String,
-}
-
-/// Implementation of FeroxMessage
-impl FeroxSerialize for FeroxMessage {
-    /// Create an NDJSON representation of the log message
-    ///
-    /// (expanded for clarity)
-    /// ex:
-    /// {
-    ///   "type": "log",
-    ///   "message": "Sent https://localhost/api to file handler",
-    ///   "level": "DEBUG",
-    ///   "time_offset": 0.86333454,
-    ///   "module": "feroxbuster::reporter"
-    /// }\n
-    fn as_json(&self) -> String {
-        if let Ok(mut json) = serde_json::to_string(&self) {
-            json.push('\n');
-            json
-        } else {
-            String::from("{\"error\":\"could not convert to json\"}")
-        }
-    }
-
-    /// Create a string representation of the log message
-    ///
-    /// ex:  301       10l       16w      173c https://localhost/api
-    fn as_str(&self) -> String {
-        let (level_name, level_color) = match self.level.as_str() {
-            "ERROR" => ("ERR", Color::Red),
-            "WARN" => ("WRN", Color::Red),
-            "INFO" => ("INF", Color::Cyan),
-            "DEBUG" => ("DBG", Color::Yellow),
-            "TRACE" => ("TRC", Color::Magenta),
-            "WILDCARD" => ("WLD", Color::Cyan),
-            _ => ("UNK", Color::White),
-        };
-
-        format!(
-            "{} {:10.03} {} {}\n",
-            style(level_name).bg(level_color).black(),
-            style(self.time_offset).dim(),
-            self.module,
-            style(&self.message).dim(),
-        )
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -551,46 +116,4 @@ mod tests {
     fn default_version() {
         assert_eq!(VERSION, env!("CARGO_PKG_VERSION"));
     }
-
-    #[test]
-    /// test as_str method of FeroxMessage
-    fn ferox_message_as_str_returns_string_with_newline() {
-        let message = FeroxMessage {
-            message: "message".to_string(),
-            module: "utils".to_string(),
-            time_offset: 1.0,
-            level: "INFO".to_string(),
-            kind: "log".to_string(),
-        };
-        let message_str = message.as_str();
-
-        assert!(message_str.contains("INF"));
-        assert!(message_str.contains("1.000"));
-        assert!(message_str.contains("utils"));
-        assert!(message_str.contains("message"));
-        assert!(message_str.ends_with('\n'));
-    }
-
-    #[test]
-    /// test as_json method of FeroxMessage
-    fn ferox_message_as_json_returns_json_representation_of_ferox_message_with_newline() {
-        let message = FeroxMessage {
-            message: "message".to_string(),
-            module: "utils".to_string(),
-            time_offset: 1.0,
-            level: "INFO".to_string(),
-            kind: "log".to_string(),
-        };
-
-        let message_str = message.as_json();
-
-        let error_margin = f32::EPSILON;
-
-        let json: FeroxMessage = serde_json::from_str(&message_str).unwrap();
-        assert_eq!(json.module, message.module);
-        assert_eq!(json.message, message.message);
-        assert!((json.time_offset - message.time_offset).abs() < error_margin);
-        assert_eq!(json.level, message.level);
-        assert_eq!(json.kind, message.kind);
-    }
 }

src/logger.rs

@@ -1,24 +1,31 @@
-use crate::{
-    config::{CONFIGURATION, PROGRESS_PRINTER},
-    reporter::safe_file_write,
-    utils::open_file,
-    FeroxMessage, FeroxSerialize,
-};
-use env_logger::Builder;
 use std::env;
+use std::fs::OpenOptions;
+use std::io::BufWriter;
+use std::sync::{Arc, RwLock};
 use std::time::Instant;
 
+use anyhow::{Context, Result};
+use env_logger::Builder;
+
+use crate::{
+    config::Configuration,
+    message::FeroxMessage,
+    progress::PROGRESS_PRINTER,
+    traits::FeroxSerialize,
+    utils::{fmt_err, write_to},
+};
+
 /// Create a customized instance of
 /// [env_logger::Logger](https://docs.rs/env_logger/latest/env_logger/struct.Logger.html)
 /// with timer offset/color and set the log level based on `verbosity`
-pub fn initialize(verbosity: u8) {
+pub fn initialize(config: Arc<Configuration>) -> Result<()> {
     // use occurrences of -v on commandline to or verbosity = N in feroxconfig.toml to set
     // log level for the application; respects already specified RUST_LOG environment variable
     match env::var("RUST_LOG") {
         Ok(_) => {} // RUST_LOG found, don't override
         Err(_) => {
             // only set log level based on verbosity when RUST_LOG variable doesn't exist
-            match verbosity {
+            match config.verbosity {
                 0 => (),
                 1 => env::set_var("RUST_LOG", "warn"),
                 2 => env::set_var("RUST_LOG", "info"),
@@ -31,12 +38,22 @@ pub fn initialize(verbosity: u8) {
     let start = Instant::now();
     let mut builder = Builder::from_default_env();
 
-    let debug_file = open_file(&CONFIGURATION.debug_log);
+    let file = if !config.debug_log.is_empty() {
+        let f = OpenOptions::new() // std fs
+            .create(true)
+            .append(true)
+            .open(&config.debug_log)
+            .with_context(|| fmt_err(&format!("Could not open {}", &config.debug_log)))?;
+
+        let mut writer = BufWriter::new(f);
 
-    if let Some(buffered_file) = debug_file.clone() {
         // write out the configuration to the debug file if it exists
-        safe_file_write(&*CONFIGURATION, buffered_file, CONFIGURATION.json);
-    }
+        write_to(&*config, &mut writer, config.json)?;
+
+        Some(Arc::new(RwLock::new(writer)))
+    } else {
+        None
+    };
 
     builder
         .format(move |_, record| {
@@ -50,11 +67,15 @@ pub fn initialize(verbosity: u8) {
 
             PROGRESS_PRINTER.println(&log_entry.as_str());
 
-            if let Some(buffered_file) = debug_file.clone() {
-                safe_file_write(&log_entry, buffered_file, CONFIGURATION.json);
+            if let Some(buffered_file) = file.clone() {
+                if let Ok(mut unlocked) = buffered_file.write() {
+                    let _ = write_to(&log_entry, &mut unlocked, config.json);
+                }
             }
 
             Ok(())
         })
         .init();
+
+    Ok(())
 }

src/macros.rs

@@ -0,0 +1,23 @@
+#![macro_use]
+
+#[macro_export]
+/// wrapper to improve code readability
+macro_rules! send_command {
+    ($tx:expr, $value:expr) => {
+        $tx.send($value).unwrap_or_default();
+    };
+}
+
+#[macro_export]
+/// while looping, check for a Result, if Ok return the value, if Err, continue
+macro_rules! skip_fail {
+    ($res:expr) => {
+        match $res {
+            Ok(val) => val,
+            Err(e) => {
+                log::warn!("{}", fmt_err(&format!("{}; skipping...", e)));
+                continue;
+            }
+        }
+    };
+}

src/main.rs

@@ -1,90 +1,66 @@
-use crossterm::event::{self, Event, KeyCode};
-use feroxbuster::progress::add_bar;
+use std::{
+    env::args,
+    fs::{create_dir, remove_file, File},
+    io::{stderr, BufRead, BufReader},
+    ops::Index,
+    path::Path,
+    process::Command,
+    sync::{atomic::Ordering, Arc},
+};
+
+use anyhow::{bail, Context, Result};
+use futures::StreamExt;
+use tokio::{
+    io,
+    sync::{oneshot, Semaphore},
+};
+use tokio_util::codec::{FramedRead, LinesCodec};
+
 use feroxbuster::{
-    banner,
-    config::{CONFIGURATION, PROGRESS_BAR, PROGRESS_PRINTER},
-    heuristics, logger, reporter,
-    scan_manager::{self, PAUSE_SCAN},
-    scanner::{self, scan_url, RESPONSES, SCANNED_URLS},
-    utils::{ferox_print, get_current_depth, module_colorizer, status_colorizer},
-    FeroxError, FeroxResponse, FeroxResult, FeroxSerialize, SLEEP_DURATION, VERSION,
+    banner::{Banner, UPDATE_URL},
+    config::{Configuration, OutputLevel},
+    event_handlers::{
+        Command::{CreateBar, Exit, JoinTasks, LoadStats, ScanInitialUrls, UpdateWordlist},
+        FiltersHandler, Handles, ScanHandler, StatsHandler, Tasks, TermInputHandler,
+        TermOutHandler, SCAN_COMPLETE,
+    },
+    filters, heuristics, logger,
+    progress::{PROGRESS_BAR, PROGRESS_PRINTER},
+    scan_manager::{self},
+    scanner,
+    utils::{fmt_err, slugify_filename},
 };
 #[cfg(not(target_os = "windows"))]
 use feroxbuster::{utils::set_open_file_limit, DEFAULT_OPEN_FILE_LIMIT};
-use futures::StreamExt;
-use std::convert::TryInto;
-use std::{
-    collections::HashSet,
-    fs::File,
-    io::{stderr, BufRead, BufReader},
-    process,
-    sync::{
-        atomic::{AtomicBool, Ordering},
-        Arc,
-    },
-    time::Duration,
-};
-use tokio::{io, sync::mpsc::UnboundedSender, task::JoinHandle};
-use tokio_util::codec::{FramedRead, LinesCodec};
+use lazy_static::lazy_static;
+use regex::Regex;
 
-/// Atomic boolean flag, used to determine whether or not the terminal input handler should exit
-pub static SCAN_COMPLETE: AtomicBool = AtomicBool::new(false);
-
-/// Handles specific key events triggered by the user over stdin
-fn terminal_input_handler() {
-    log::trace!("enter: terminal_input_handler");
-
-    loop {
-        if event::poll(Duration::from_millis(SLEEP_DURATION)).unwrap_or(false) {
-            // It's guaranteed that the `read()` won't block when the `poll()`
-            // function returns `true`
-
-            if let Ok(key_pressed) = event::read() {
-                if key_pressed == Event::Key(KeyCode::Enter.into()) {
-                    // if the user presses Enter, toggle the value stored in PAUSE_SCAN
-                    // ignore any other keys
-                    let current = PAUSE_SCAN.load(Ordering::Acquire);
-
-                    PAUSE_SCAN.store(!current, Ordering::Release);
-                }
-            }
-        } else {
-            // Timeout expired and no `Event` is available; use the timeout to check SCAN_COMPLETE
-            if SCAN_COMPLETE.load(Ordering::Relaxed) {
-                // scan has been marked complete by main, time to exit the loop
-                break;
-            }
-        }
-    }
-    log::trace!("exit: terminal_input_handler");
+lazy_static! {
+    /// Limits the number of parallel scans active at any given time when using --parallel
+    static ref PARALLEL_LIMITER: Semaphore = Semaphore::new(0);
 }
 
 /// Create a HashSet of Strings from the given wordlist then stores it inside an Arc
-fn get_unique_words_from_wordlist(path: &str) -> FeroxResult<Arc<HashSet<String>>> {
+fn get_unique_words_from_wordlist(path: &str) -> Result<Arc<Vec<String>>> {
     log::trace!("enter: get_unique_words_from_wordlist({})", path);
 
-    let file = match File::open(&path) {
-        Ok(f) => f,
-        Err(e) => {
-            log::error!("Could not open wordlist: {}", e);
-            log::trace!("exit: get_unique_words_from_wordlist -> {}", e);
-
-            return Err(Box::new(e));
-        }
-    };
+    let file = File::open(&path).with_context(|| format!("Could not open {}", path))?;
 
     let reader = BufReader::new(file);
 
-    let mut words = HashSet::new();
+    let mut words = Vec::new();
 
     for line in reader.lines() {
-        let result = line?;
+        let result = match line {
+            Ok(read_line) => read_line,
+            Err(_) => continue,
+        };
 
         if result.starts_with('#') || result.is_empty() {
             continue;
         }
 
-        words.insert(result);
+        words.push(result);
     }
 
     log::trace!(
@@ -96,81 +72,59 @@ fn get_unique_words_from_wordlist(path: &str) -> FeroxResult<Arc<HashSet<String>
 }
 
 /// Determine whether it's a single url scan or urls are coming from stdin, then scan as needed
-async fn scan(
-    targets: Vec<String>,
-    tx_term: UnboundedSender<FeroxResponse>,
-    tx_file: UnboundedSender<FeroxResponse>,
-) -> FeroxResult<()> {
-    log::trace!("enter: scan({:?}, {:?}, {:?})", targets, tx_term, tx_file);
+async fn scan(targets: Vec<String>, handles: Arc<Handles>) -> Result<()> {
+    log::trace!("enter: scan({:?}, {:?})", targets, handles);
     // cloning an Arc is cheap (it's basically a pointer into the heap)
     // so that will allow for cheap/safe sharing of a single wordlist across multi-target scans
     // as well as additional directories found as part of recursion
-    let words =
-        tokio::spawn(async move { get_unique_words_from_wordlist(&CONFIGURATION.wordlist) })
-            .await??;
+
+    let words = get_unique_words_from_wordlist(&handles.config.wordlist)?;
 
     if words.len() == 0 {
-        let mut err = FeroxError::default();
-        err.message = format!("Did not find any words in {}", CONFIGURATION.wordlist);
-        return Err(Box::new(err));
+        bail!("Did not find any words in {}", handles.config.wordlist);
     }
 
-    scanner::initialize(words.len(), &CONFIGURATION);
+    let scanned_urls = handles.ferox_scans()?;
 
-    if CONFIGURATION.resumed {
-        if let Ok(scans) = SCANNED_URLS.scans.lock() {
-            for scan in scans.iter() {
-                if let Ok(locked_scan) = scan.lock() {
-                    if locked_scan.complete {
-                        // these scans are complete, and just need to be shown to the user
-                        let pb = add_bar(
-                            &locked_scan.url,
-                            words.len().try_into().unwrap_or_default(),
-                            false,
-                            true,
-                        );
-                        pb.finish();
-                    }
-                }
-            }
-        }
+    handles.send_scan_command(UpdateWordlist(words.clone()))?;
 
-        if let Ok(responses) = RESPONSES.responses.read() {
-            for response in responses.iter() {
-                PROGRESS_PRINTER.println(response.as_str());
-            }
-        }
+    scanner::initialize(words.len(), handles.clone()).await?;
+
+    // at this point, the stat thread's progress bar can be created; things that needed to happen
+    // first:
+    // - banner gets printed
+    // - scanner initialized (this sent expected requests per directory to the stats thread, which
+    //   having been set, makes it so the progress bar doesn't flash as full before anything has
+    //   even happened
+    if matches!(handles.config.output_level, OutputLevel::Default) {
+        // only create the bar if no --silent|--quiet
+        handles.stats.send(CreateBar)?;
+
+        // blocks until the bar is created / avoids race condition in first two bars
+        handles.stats.sync().await?;
     }
 
-    let mut tasks = vec![];
-
-    for target in targets {
-        let word_clone = words.clone();
-        let term_clone = tx_term.clone();
-        let file_clone = tx_file.clone();
-
-        let task = tokio::spawn(async move {
-            let base_depth = get_current_depth(&target);
-            scan_url(&target, word_clone, base_depth, term_clone, file_clone).await;
-        });
-
-        tasks.push(task);
+    if handles.config.resumed {
+        // display what has already been completed
+        scanned_urls.print_known_responses();
+        scanned_urls.print_completed_bars(words.len())?;
     }
 
-    // drive execution of all accumulated futures
-    futures::future::join_all(tasks).await;
+    log::debug!("sending {:?} to be scanned as initial targets", targets);
+    handles.send_scan_command(ScanInitialUrls(targets))?;
+
     log::trace!("exit: scan");
 
     Ok(())
 }
 
 /// Get targets from either commandline or stdin, pass them back to the caller as a Result<Vec>
-async fn get_targets() -> FeroxResult<Vec<String>> {
-    log::trace!("enter: get_targets");
+async fn get_targets(handles: Arc<Handles>) -> Result<Vec<String>> {
+    log::trace!("enter: get_targets({:?})", handles);
 
     let mut targets = vec![];
 
-    if CONFIGURATION.stdin {
+    if handles.config.stdin {
         // got targets from stdin, i.e. cat sites | ./feroxbuster ...
         // just need to read the targets from stdin and spawn a future for each target found
         let stdin = io::stdin(); // tokio's stdin, not std
@@ -179,24 +133,25 @@ async fn get_targets() -> FeroxResult<Vec<String>> {
         while let Some(line) = reader.next().await {
             targets.push(line?);
         }
-    } else if CONFIGURATION.resumed {
-        // resume-from can't be used with any other flag, making it mutually exclusive from either
-        // of the other two options
-        if let Ok(scans) = SCANNED_URLS.scans.lock() {
+    } else if handles.config.resumed {
+        // resume-from can't be used with --url, and --stdin is marked false for every resumed
+        // scan, making it mutually exclusive from either of the other two options
+        let ferox_scans = handles.ferox_scans()?;
+
+        if let Ok(scans) = ferox_scans.scans.read() {
             for scan in scans.iter() {
-                // SCANNED_URLS gets deserialized scans added to it at program start if --resume-from
+                // ferox_scans gets deserialized scans added to it at program start if --resume-from
                 // is used, so scans that aren't marked complete still need to be scanned
-                if let Ok(locked_scan) = scan.lock() {
-                    if locked_scan.complete {
-                        // this one's already done, ignore it
-                        continue;
-                    }
-                    targets.push(locked_scan.url.to_owned());
+                if scan.is_complete() {
+                    // this one's already done, ignore it
+                    continue;
                 }
+
+                targets.push(scan.url().to_owned());
             }
-        }
+        };
     } else {
-        targets.push(CONFIGURATION.target_url.clone());
+        targets.push(handles.config.target_url.clone());
     }
 
     log::trace!("exit: get_targets -> {:?}", targets);
@@ -206,7 +161,7 @@ async fn get_targets() -> FeroxResult<Vec<String>> {
 
 /// async main called from real main, broken out in this way to allow for some synchronous code
 /// to be executed before bringing the tokio runtime online
-async fn wrapped_main() {
+async fn wrapped_main(config: Arc<Configuration>) -> Result<()> {
     // join can only be called once, otherwise it causes the thread to panic
     tokio::task::spawn_blocking(move || {
         // ok, lazy_static! uses (unsurprisingly in retrospect) a lazy loading model where the
@@ -220,112 +175,275 @@ async fn wrapped_main() {
         PROGRESS_BAR.join().unwrap();
     });
 
+    // spawn all event handlers, expect back a JoinHandle and a *Handle to the specific event
+    let (stats_task, stats_handle) = StatsHandler::initialize(config.clone());
+    let (filters_task, filters_handle) = FiltersHandler::initialize();
+    let (out_task, out_handle) =
+        TermOutHandler::initialize(config.clone(), stats_handle.tx.clone());
+
+    // bundle up all the disparate handles and JoinHandles (tasks)
+    let handles = Arc::new(Handles::new(
+        stats_handle,
+        filters_handle,
+        out_handle,
+        config.clone(),
+    ));
+
+    let (scan_task, scan_handle) = ScanHandler::initialize(handles.clone());
+
+    handles.set_scan_handle(scan_handle); // must be done after Handles initialization
+
+    filters::initialize(handles.clone()).await?; // send user-supplied filters to the handler
+
+    // create new Tasks object, each of these handles is one that will be joined on later
+    let tasks = Tasks::new(out_task, stats_task, filters_task, scan_task);
+
+    if !config.time_limit.is_empty() {
+        // --time-limit value not an empty string, need to kick off the thread that enforces
+        // the limit
+        let time_handles = handles.clone();
+        tokio::spawn(async move { scan_manager::start_max_time_thread(time_handles).await });
+    }
+
     // can't trace main until after logger is initialized and the above task is started
     log::trace!("enter: main");
 
     // spawn a thread that listens for keyboard input on stdin, when a user presses enter
     // the input handler will toggle PAUSE_SCAN, which in turn is used to pause and resume
     // scans that are already running
-    tokio::task::spawn_blocking(terminal_input_handler);
+    // also starts ctrl+c handler
+    TermInputHandler::initialize(handles.clone());
 
-    let save_output = !CONFIGURATION.output.is_empty(); // was -o used?
+    if config.resumed {
+        let scanned_urls = handles.ferox_scans()?;
+        let from_here = config.resume_from.clone();
 
-    let (tx_term, tx_file, term_handle, file_handle) =
-        reporter::initialize(&CONFIGURATION.output, save_output);
+        // populate FeroxScans object with previously seen scans
+        scanned_urls.add_serialized_scans(&from_here)?;
+
+        // populate Stats object with previously known statistics
+        handles.stats.send(LoadStats(from_here))?;
+    }
 
     // get targets from command line or stdin
-    let targets = match get_targets().await {
+    let targets = match get_targets(handles.clone()).await {
         Ok(t) => t,
         Err(e) => {
             // should only happen in the event that there was an error reading from stdin
-            log::error!("{} {}", module_colorizer("main::get_targets"), e);
-            clean_up(tx_term, term_handle, tx_file, file_handle, save_output).await;
-            return;
+            clean_up(handles, tasks).await?;
+            bail!("Could not get determine initial targets: {}", e);
         }
     };
 
-    if !CONFIGURATION.quiet {
-        // only print banner if -q isn't used
+    // --parallel branch
+    if config.parallel > 0 {
+        log::trace!("enter: parallel branch");
+
+        PARALLEL_LIMITER.add_permits(config.parallel);
+
+        let invocation = args();
+
+        let para_regex =
+            Regex::new("--stdin|-q|--quiet|--silent|--verbosity|-v|-vv|-vvv|-vvvv").unwrap();
+
+        // remove stdin since only the original process will process targets
+        // remove quiet and silent so we can force silent later to normalize output
+        let mut original = invocation
+            .filter(|s| !para_regex.is_match(s))
+            .collect::<Vec<String>>();
+
+        original.push("--silent".to_string()); // only output modifier allowed
+
+        // we need remove --parallel from command line so we don't hit this branch over and over
+        // but we must remove --parallel N manually; the filter above never sees --parallel and the
+        // value passed to it at the same time, so can't filter them out in one pass
+
+        // unwrap is fine, as it has to be in the args for us to be in this code branch
+        let parallel_index = original.iter().position(|s| *s == "--parallel").unwrap();
+
+        // remove --parallel
+        original.remove(parallel_index);
+
+        // remove N passed to --parallel (it's the same index again since everything shifts
+        // from removing --parallel)
+        original.remove(parallel_index);
+
+        // to log unique files to a shared folder, we need to first check for the presence
+        // of -o|--output.
+        let out_dir = if !config.output.is_empty() {
+            // -o|--output was used, so we'll attempt to create a directory to store the files
+            let output_path = Path::new(&handles.config.output);
+
+            // this only returns None if the path terminates in `..`. Since I don't want to
+            // hand-hold to that degree, we'll unwrap and fail if the output path ends in `..`
+            let base_name = output_path.file_name().unwrap();
+
+            let new_folder = slugify_filename(&base_name.to_string_lossy(), "", "logs");
+
+            let final_path = output_path.with_file_name(&new_folder);
+
+            // create the directory or fail silently, assuming the reason for failure is that
+            // the path exists already
+            create_dir(&final_path).unwrap_or(());
+
+            final_path.to_string_lossy().to_string()
+        } else {
+            String::new()
+        };
+
+        // unvalidated targets fresh from stdin, just spawn children and let them do all checks
+        for target in targets {
+            // add the current target to the provided command
+            let mut cloned = original.clone();
+
+            if !out_dir.is_empty() {
+                // output directory value is not empty, need to join output directory with
+                // unique scan filename
+
+                // unwrap is ok, we already know -o was used
+                let out_idx = original
+                    .iter()
+                    .position(|s| *s == "--output" || *s == "-o")
+                    .unwrap();
+
+                let filename = slugify_filename(&target, "ferox", "log");
+
+                let full_path = Path::new(&out_dir)
+                    .join(filename)
+                    .to_string_lossy()
+                    .to_string();
+
+                // a +1 to the index is fine here, as clap has already validated that
+                // -o|--output has a value associated with it
+                cloned[out_idx + 1] = full_path;
+            }
+
+            cloned.push("-u".to_string());
+            cloned.push(target);
+
+            let bin = cloned.index(0).to_owned(); // user's path to feroxbuster
+            let args = cloned.index(1..).to_vec(); // and args
+
+            let permit = PARALLEL_LIMITER.acquire().await?;
+
+            log::debug!("parallel exec: {} {}", bin, args.join(" "));
+
+            tokio::task::spawn_blocking(move || {
+                let result = Command::new(bin)
+                    .args(&args)
+                    .spawn()
+                    .expect("failed to spawn a child process")
+                    .wait()
+                    .expect("child process errored during execution");
+
+                drop(permit);
+                result
+            });
+        }
+
+        // the output handler creates an empty file to which it will try to write, because
+        // this happens before we enter the --parallel branch, we need to remove that file
+        // if it's empty
+        let output = handles.config.output.to_owned();
+
+        clean_up(handles, tasks).await?;
+
+        let file = Path::new(&output);
+        if file.exists() {
+            // expectation is that this is always true for the first ferox process
+            if file.metadata()?.len() == 0 {
+                // empty file, attempt to remove it
+                remove_file(file)?;
+            }
+        }
+
+        log::trace!("exit: parallel branch && wrapped main");
+        return Ok(());
+    }
+
+    if matches!(config.output_level, OutputLevel::Default) {
+        // only print banner if output level is default (no banner on --quiet|--silent)
         let std_stderr = stderr(); // std::io::stderr
-        banner::initialize(&targets, &CONFIGURATION, &VERSION, std_stderr).await;
+
+        let mut banner = Banner::new(&targets, &config);
+
+        // only interested in the side-effect that sets banner.update_status
+        let _ = banner.check_for_updates(UPDATE_URL, handles.clone()).await;
+
+        if banner.print_to(std_stderr, config.clone()).is_err() {
+            clean_up(handles, tasks).await?;
+            bail!(fmt_err("Could not print banner"));
+        }
+    }
+
+    {
+        let send_to_file = !config.output.is_empty();
+
+        // The TermOutHandler spawns a FileOutHandler, so errors in the FileOutHandler never bubble
+        // up due to the TermOutHandler never awaiting the result of FileOutHandler::start (that's
+        // done later here in main). sync checks that the tx/rx connection to the file handler works
+        if send_to_file && handles.output.sync(send_to_file).await.is_err() {
+            // output file specified and file handler could not initialize
+            clean_up(handles, tasks).await?;
+            let msg = format!("Couldn't start {} file handler", config.output);
+            bail!(fmt_err(&msg));
+        }
     }
 
     // discard non-responsive targets
-    let live_targets = heuristics::connectivity_test(&targets).await;
+    let live_targets = {
+        let test = heuristics::HeuristicTests::new(handles.clone());
+        let result = test.connectivity(&targets).await;
+        if result.is_err() {
+            clean_up(handles, tasks).await?;
+            bail!(fmt_err(&result.unwrap_err().to_string()));
+        }
+        result?
+    };
 
     if live_targets.is_empty() {
-        clean_up(tx_term, term_handle, tx_file, file_handle, save_output).await;
-        return;
+        clean_up(handles, tasks).await?;
+        bail!(fmt_err("Could not find any live targets to scan"));
     }
 
     // kick off a scan against any targets determined to be responsive
-    match scan(live_targets, tx_term.clone(), tx_file.clone()).await {
-        Ok(_) => {
-            log::info!("All scans complete!");
-        }
+    match scan(live_targets, handles.clone()).await {
+        Ok(_) => {}
         Err(e) => {
-            ferox_print(
-                &format!("{} while scanning: {}", status_colorizer("Error"), e),
-                &PROGRESS_PRINTER,
-            );
-            clean_up(tx_term, term_handle, tx_file, file_handle, save_output).await;
-            process::exit(1);
+            clean_up(handles, tasks).await?;
+            bail!(fmt_err(&format!("Failed while scanning: {}", e)));
         }
-    };
+    }
 
-    clean_up(tx_term, term_handle, tx_file, file_handle, save_output).await;
+    clean_up(handles, tasks).await?;
 
-    log::trace!("exit: main");
+    log::trace!("exit: wrapped_main");
+    Ok(())
 }
 
 /// Single cleanup function that handles all the necessary drops/finishes etc required to gracefully
 /// shutdown the program
-async fn clean_up(
-    tx_term: UnboundedSender<FeroxResponse>,
-    term_handle: JoinHandle<()>,
-    tx_file: UnboundedSender<FeroxResponse>,
-    file_handle: Option<JoinHandle<()>>,
-    save_output: bool,
-) {
-    log::trace!(
-        "enter: clean_up({:?}, {:?}, {:?}, {:?}, {})",
-        tx_term,
-        term_handle,
-        tx_file,
-        file_handle,
-        save_output
-    );
+async fn clean_up(handles: Arc<Handles>, tasks: Tasks) -> Result<()> {
+    log::trace!("enter: clean_up({:?}, {:?})", handles, tasks);
 
-    drop(tx_term);
-    log::trace!("dropped terminal output handler's transmitter");
+    let (tx, rx) = oneshot::channel::<bool>();
+    handles.send_scan_command(JoinTasks(tx))?;
+    rx.await?;
 
-    log::trace!("awaiting terminal output handler's receiver");
-    // after dropping tx, we can await the future where rx lived
-    match term_handle.await {
-        Ok(_) => {}
-        Err(e) => {
-            log::error!("error awaiting terminal output handler's receiver: {}", e);
-        }
-    }
-    log::trace!("done awaiting terminal output handler's receiver");
+    log::info!("All scans complete!");
 
-    log::trace!("tx_file: {:?}", tx_file);
-    // the same drop/await process used on the terminal handler is repeated for the file handler
-    // we drop the file transmitter every time, because it's created no matter what
-    drop(tx_file);
+    // terminal handler closes file handler if one is in use
+    handles.output.send(Exit)?;
+    tasks.terminal.await??;
+    log::trace!("terminal handler closed");
 
-    log::trace!("dropped file output handler's transmitter");
-    if save_output {
-        // but we only await if -o was specified
-        log::trace!("awaiting file output handler's receiver");
-        match file_handle.unwrap().await {
-            Ok(_) => {}
-            Err(e) => {
-                log::error!("error awaiting file output handler's receiver: {}", e);
-            }
-        }
-        log::trace!("done awaiting file output handler's receiver");
-    }
+    handles.filters.send(Exit)?;
+    tasks.filters.await??;
+    log::trace!("filters handler closed");
+
+    handles.stats.send(Exit)?;
+    tasks.stats.await??;
+    log::trace!("stats handler closed");
 
     // mark all scans complete so the terminal input handler will exit cleanly
     SCAN_COMPLETE.store(true, Ordering::Relaxed);
@@ -335,23 +453,36 @@ async fn clean_up(
     PROGRESS_PRINTER.finish();
 
     log::trace!("exit: clean_up");
+    Ok(())
 }
 
-fn main() {
-    // setup logging based on the number of -v's used
-    logger::initialize(CONFIGURATION.verbosity);
+fn main() -> Result<()> {
+    let config = Arc::new(Configuration::new().with_context(|| "Could not create Configuration")?);
 
-    if CONFIGURATION.save_state {
-        // start the ctrl+c handler
-        scan_manager::initialize();
+    // setup logging based on the number of -v's used
+    if matches!(
+        config.output_level,
+        OutputLevel::Default | OutputLevel::Quiet
+    ) {
+        // don't log on --silent
+        logger::initialize(config.clone())?;
     }
 
     // this function uses rlimit, which is not supported on windows
     #[cfg(not(target_os = "windows"))]
     set_open_file_limit(DEFAULT_OPEN_FILE_LIMIT);
 
-    if let Ok(mut runtime) = tokio::runtime::Runtime::new() {
-        let future = wrapped_main();
-        runtime.block_on(future);
+    if let Ok(runtime) = tokio::runtime::Builder::new_multi_thread()
+        .enable_all()
+        .build()
+    {
+        let future = wrapped_main(config);
+        if let Err(e) = runtime.block_on(future) {
+            eprintln!("{}", e);
+        };
     }
+
+    log::trace!("exit: main");
+
+    Ok(())
 }

src/message.rs

@@ -0,0 +1,148 @@
+use anyhow::Context;
+use console::{style, Color};
+use serde::{Deserialize, Serialize};
+
+use crate::traits::FeroxSerialize;
+use crate::utils::fmt_err;
+
+#[derive(Serialize, Deserialize, Default)]
+/// Representation of a log entry, can be represented as a human readable string or JSON
+pub struct FeroxMessage {
+    #[serde(rename = "type")]
+    /// Name of this type of struct, used for serialization, i.e. `{"type":"log"}`
+    pub(crate) kind: String,
+
+    /// The log message
+    pub(crate) message: String,
+
+    /// The log level
+    pub(crate) level: String,
+
+    /// The number of seconds elapsed since the scan started
+    pub(crate) time_offset: f32,
+
+    /// The module from which log::* was called
+    pub(crate) module: String,
+}
+
+/// Implementation of FeroxMessage
+impl FeroxSerialize for FeroxMessage {
+    /// Create a string representation of the log message
+    ///
+    /// ex:  301       10l       16w      173c https://localhost/api
+    fn as_str(&self) -> String {
+        let (level_name, level_color) = match self.level.as_str() {
+            "ERROR" => ("ERR", Color::Red),
+            "WARN" => ("WRN", Color::Red),
+            "INFO" => ("INF", Color::Cyan),
+            "DEBUG" => ("DBG", Color::Yellow),
+            "TRACE" => ("TRC", Color::Magenta),
+            "WILDCARD" => ("WLD", Color::Cyan),
+            _ => ("UNK", Color::White),
+        };
+
+        format!(
+            "{} {:10.03} {} {}\n",
+            style(level_name).bg(level_color).black(),
+            style(self.time_offset).dim(),
+            self.module,
+            style(&self.message).dim(),
+        )
+    }
+
+    /// Create an NDJSON representation of the log message
+    ///
+    /// (expanded for clarity)
+    /// ex:
+    /// {
+    ///   "type": "log",
+    ///   "message": "Sent https://localhost/api to file handler",
+    ///   "level": "DEBUG",
+    ///   "time_offset": 0.86333454,
+    ///   "module": "feroxbuster::reporter"
+    /// }\n
+    fn as_json(&self) -> anyhow::Result<String> {
+        let mut json = serde_json::to_string(&self).with_context(|| {
+            fmt_err(&format!(
+                "Could not convert {}:{} to JSON",
+                self.level, self.message
+            ))
+        })?;
+        json.push('\n');
+        Ok(json)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    /// test as_str method of FeroxMessage
+    fn ferox_message_as_str_returns_string_with_newline() {
+        let message = FeroxMessage {
+            message: "message".to_string(),
+            module: "utils".to_string(),
+            time_offset: 1.0,
+            level: "INFO".to_string(),
+            kind: "log".to_string(),
+        };
+        let message_str = message.as_str();
+
+        assert!(message_str.contains("INF"));
+        assert!(message_str.contains("1.000"));
+        assert!(message_str.contains("utils"));
+        assert!(message_str.contains("message"));
+        assert!(message_str.ends_with('\n'));
+    }
+
+    #[test]
+    /// test as_json method of FeroxMessage
+    fn ferox_message_as_json_returns_json_representation_of_ferox_message_with_newline() {
+        let message = FeroxMessage {
+            message: "message".to_string(),
+            module: "utils".to_string(),
+            time_offset: 1.0,
+            level: "INFO".to_string(),
+            kind: "log".to_string(),
+        };
+
+        let message_str = message.as_json().unwrap();
+
+        let error_margin = f32::EPSILON;
+
+        let json: FeroxMessage = serde_json::from_str(&message_str).unwrap();
+        assert_eq!(json.module, message.module);
+        assert_eq!(json.message, message.message);
+        assert!((json.time_offset - message.time_offset).abs() < error_margin);
+        assert_eq!(json.level, message.level);
+        assert_eq!(json.kind, message.kind);
+    }
+
+    #[test]
+    /// test defaults for coverage
+    fn message_defaults() {
+        let msg = FeroxMessage::default();
+        assert_eq!(msg.level, String::new());
+        assert_eq!(msg.kind, String::new());
+        assert_eq!(msg.message, String::new());
+        assert_eq!(msg.module, String::new());
+        assert!(msg.time_offset < 0.1);
+    }
+
+    #[test]
+    /// ensure WILDCARD messages serialize to WLD and anything not known to UNK
+    fn message_as_str_edges() {
+        let mut msg = FeroxMessage {
+            message: "message".to_string(),
+            module: "utils".to_string(),
+            time_offset: 1.0,
+            level: "WILDCARD".to_string(),
+            kind: "log".to_string(),
+        };
+        assert!(console::strip_ansi_codes(&msg.as_str()).starts_with("WLD"));
+
+        msg.level = "UNKNOWN".to_string();
+        assert!(console::strip_ansi_codes(&msg.as_str()).starts_with("UNK"));
+    }
+}

src/parser.rs

@@ -1,10 +1,25 @@
-use crate::VERSION;
 use clap::{App, Arg, ArgGroup};
+use lazy_static::lazy_static;
+use regex::Regex;
+use std::env;
+use std::process;
+
+lazy_static! {
+    /// Regex used to validate values passed to --time-limit
+    ///
+    /// Examples of expected values that will this regex will match:
+    /// - 30s
+    /// - 20m
+    /// - 1h
+    /// - 1d
+    pub static ref TIMESPEC_REGEX: Regex =
+        Regex::new(r"^(?i)(?P<n>\d+)(?P<m>[smdh])$").expect("Could not compile regex");
+}
 
 /// Create and return an instance of [clap::App](https://docs.rs/clap/latest/clap/struct.App.html), i.e. the Command Line Interface's configuration
 pub fn initialize() -> App<'static, 'static> {
-    App::new("feroxbuster")
-        .version(VERSION)
+    let mut app = App::new("feroxbuster")
+        .version(env!("CARGO_PKG_VERSION"))
         .author("Ben 'epi' Risher (@epi052)")
         .about("A fast, simple, recursive content discovery tool written in Rust")
         .arg(
@@ -55,6 +70,7 @@ pub fn initialize() -> App<'static, 'static> {
                 .long("verbosity")
                 .takes_value(false)
                 .multiple(true)
+                .conflicts_with("silent")
                 .help("Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v's is probably too much)"),
         )
         .arg(
@@ -64,7 +80,7 @@ pub fn initialize() -> App<'static, 'static> {
                 .takes_value(true)
                 .value_name("PROXY")
                 .help(
-                    "Proxy to use for requests (ex: http(s)://host:port, socks5://host:port)",
+                    "Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)",
                 ),
         )
         .arg(
@@ -102,12 +118,32 @@ pub fn initialize() -> App<'static, 'static> {
                     "Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)",
                 ),
         )
+        .arg(
+            Arg::with_name("silent")
+                .long("silent")
+                .takes_value(false)
+                .conflicts_with("quiet")
+                .help("Only print URLs + turn off logging (good for piping a list of urls to other commands)")
+        )
         .arg(
             Arg::with_name("quiet")
                 .short("q")
                 .long("quiet")
                 .takes_value(false)
-                .help("Only print URLs; Don't print status codes, response size, running config, etc...")
+                .help("Hide progress bars and banner (good for tmux windows w/ notifications)")
+        )
+        .arg(
+            Arg::with_name("auto_tune")
+                .long("auto-tune")
+                .takes_value(false)
+                .conflicts_with("auto_bail")
+                .help("Automatically lower scan rate when an excessive amount of errors are encountered")
+        )
+        .arg(
+            Arg::with_name("auto_bail")
+                .long("auto-bail")
+                .takes_value(false)
+                .help("Automatically stop scanning when an excessive amount of errors are encountered")
         )
         .arg(
             Arg::with_name("json")
@@ -136,7 +172,7 @@ pub fn initialize() -> App<'static, 'static> {
                 .long("resume-from")
                 .value_name("STATE_FILE")
                 .help("State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)")
-                .conflicts_with_all(&["wordlist", "url", "threads", "depth", "timeout", "verbosity", "proxy", "replay_proxy", "replay_codes", "status_codes", "quiet", "json", "dont_filter", "output", "debug_log", "user_agent", "redirects", "insecure", "extensions", "headers", "queries", "no_recursion", "add_slash", "stdin", "filter_size", "filter_regex", "filter_words", "filter_lines", "filter_status", "extract_links", "scan_limit"])
+                .conflicts_with("url")
                 .takes_value(true),
         )
         .arg(
@@ -182,6 +218,17 @@ pub fn initialize() -> App<'static, 'static> {
                     "File extension(s) to search for (ex: -x php -x pdf js)",
                 ),
         )
+        .arg(
+            Arg::with_name("url_denylist")
+                .long("dont-scan")
+                .value_name("URL")
+                .takes_value(true)
+                .multiple(true)
+                .use_delimiter(true)
+                .help(
+                    "URL(s) to exclude from recursion/scans",
+                ),
+        )
         .arg(
             Arg::with_name("headers")
                 .short("H")
@@ -288,6 +335,17 @@ pub fn initialize() -> App<'static, 'static> {
                     "Filter out status codes (deny list) (ex: -C 200 -C 401)",
                 ),
         )
+        .arg(
+            Arg::with_name("filter_similar")
+                .long("filter-similar-to")
+                .value_name("UNWANTED_PAGE")
+                .takes_value(true)
+                .multiple(true)
+                .use_delimiter(true)
+                .help(
+                    "Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)",
+                ),
+        )
         .arg(
             Arg::with_name("extract_links")
                 .short("e")
@@ -303,6 +361,30 @@ pub fn initialize() -> App<'static, 'static> {
                 .takes_value(true)
                 .help("Limit total number of concurrent scans (default: 0, i.e. no limit)")
         )
+        .arg(
+            Arg::with_name("parallel")
+                .long("parallel")
+                .value_name("PARALLEL_SCANS")
+                .takes_value(true)
+                .requires("stdin")
+                .help("Run parallel feroxbuster instances (one child process per url passed via stdin)")
+        )
+        .arg(
+            Arg::with_name("rate_limit")
+                .long("rate-limit")
+                .value_name("RATE_LIMIT")
+                .takes_value(true)
+                .conflicts_with("auto_tune")
+                .help("Limit number of requests per second (per directory) (default: 0, i.e. no limit)")
+        )
+        .arg(
+            Arg::with_name("time_limit")
+                .long("time-limit")
+                .value_name("TIME_SPEC")
+                .takes_value(true)
+                .validator(valid_time_spec)
+                .help("Limit total run time of all scans (ex: --time-limit 10m)")
+        )
         .group(ArgGroup::with_name("output_files")
             .args(&["debug_log", "output"])
             .multiple(true)
@@ -325,7 +407,7 @@ EXAMPLES:
         ./feroxbuster -u http://[::1] --no-recursion -vv
 
     Read urls from STDIN; pipe only resulting urls out to another tool
-        cat targets | ./feroxbuster --stdin --quiet -s 200 301 302 --redirects -x js | fff -s 200 -o js-files
+        cat targets | ./feroxbuster --stdin --silent -s 200 301 302 --redirects -x js | fff -s 200 -o js-files
 
     Proxy traffic through Burp
         ./feroxbuster -u http://127.1 --insecure --proxy http://127.0.0.1:8080
@@ -341,7 +423,34 @@ EXAMPLES:
 
     Ludicrous speed... go!
         ./feroxbuster -u http://127.1 -t 200
-    "#)
+    "#);
+
+    for arg in env::args() {
+        // secure-77 noticed that when an incorrect flag/option is used, the short help message is printed
+        // which is fine, but if you add -h|--help, it still errors out on the bad flag/option,
+        // never showing the full help message. This code addresses that behavior
+        if arg == "--help" || arg == "-h" {
+            app.print_long_help().unwrap();
+            println!(); // just a newline to mirror original --help output
+            process::exit(0);
+        }
+    }
+
+    app
+}
+
+/// Validate that a string is formatted as a number followed by s, m, h, or d (10d, 30s, etc...)
+fn valid_time_spec(time_spec: String) -> Result<(), String> {
+    match TIMESPEC_REGEX.is_match(&time_spec) {
+        true => Ok(()),
+        false => {
+            let msg = format!(
+                "Expected a non-negative, whole number followed by s, m, h, or d (case insensitive); received {}",
+                time_spec
+            );
+            Err(msg)
+        }
+    }
 }
 
 #[cfg(test)]
@@ -354,4 +463,37 @@ mod tests {
         let app = initialize();
         assert_eq!(app.get_name(), "feroxbuster");
     }
+
+    #[test]
+    /// sanity checks that valid_time_spec correctly checks and rejects a given string
+    ///
+    /// instead of having a bunch of single tests here, they're all quick and are mostly checking
+    /// that i didn't hose up the regex.  Going to consolidate them into a single test
+    fn validate_valid_time_spec_validation() {
+        let float_rejected = "1.4m";
+        assert!(valid_time_spec(float_rejected.into()).is_err());
+
+        let negative_rejected = "-1m";
+        assert!(valid_time_spec(negative_rejected.into()).is_err());
+
+        let only_number_rejected = "1";
+        assert!(valid_time_spec(only_number_rejected.into()).is_err());
+
+        let only_measurement_rejected = "m";
+        assert!(valid_time_spec(only_measurement_rejected.into()).is_err());
+
+        for accepted_measurement in &["s", "m", "h", "d", "S", "M", "H", "D"] {
+            // all upper/lowercase should be good
+            assert!(valid_time_spec(format!("1{}", *accepted_measurement)).is_ok());
+        }
+
+        let leading_space_rejected = " 14m";
+        assert!(valid_time_spec(leading_space_rejected.into()).is_err());
+
+        let trailing_space_rejected = "14m ";
+        assert!(valid_time_spec(trailing_space_rejected.into()).is_err());
+
+        let space_between_rejected = "1 4m";
+        assert!(valid_time_spec(space_between_rejected.into()).is_err());
+    }
 }

src/progress.rs

@@ -1,29 +1,57 @@
-use crate::config::{CONFIGURATION, PROGRESS_BAR};
-use indicatif::{ProgressBar, ProgressStyle};
+use indicatif::{MultiProgress, ProgressBar, ProgressDrawTarget, ProgressStyle};
+use lazy_static::lazy_static;
+
+lazy_static! {
+    /// Global progress bar that houses other progress bars
+    pub static ref PROGRESS_BAR: MultiProgress = MultiProgress::with_draw_target(ProgressDrawTarget::stdout());
+
+    /// Global progress bar that is only used for printing messages that don't jack up other bars
+    pub static ref PROGRESS_PRINTER: ProgressBar = add_bar("", 0, BarType::Hidden);
+}
+
+/// Types of ProgressBars that can be added to `PROGRESS_BAR`
+#[derive(Copy, Clone)]
+pub enum BarType {
+    /// no template used / not visible
+    Hidden,
+
+    /// normal directory status bar (reqs/sec shown)
+    Default,
+
+    /// similar to `Default`, except `-` is used in place of line/word/char count
+    Message,
+
+    /// bar used to show overall scan metrics
+    Total,
+
+    /// simpler output bar that shows only the directory being scanned (no updating info)
+    Quiet,
+}
 
 /// Add an [indicatif::ProgressBar](https://docs.rs/indicatif/latest/indicatif/struct.ProgressBar.html)
 /// to the global [PROGRESS_BAR](../config/struct.PROGRESS_BAR.html)
-pub fn add_bar(prefix: &str, length: u64, hidden: bool, hide_per_sec: bool) -> ProgressBar {
-    let style = if hidden || CONFIGURATION.quiet {
-        ProgressStyle::default_bar().template("")
-    } else if hide_per_sec {
-        ProgressStyle::default_bar()
-            .template(&format!(
-                "[{{bar:.cyan/blue}}] - {{elapsed:<4}} {{pos:>7}}/{{len:7}} {:7} {{prefix}}",
-                "-"
-            ))
-            .progress_chars("#>-")
-    } else {
-        ProgressStyle::default_bar()
-            .template("[{bar:.cyan/blue}] - {elapsed:<4} {pos:>7}/{len:7} {per_sec:7} {prefix}")
-            .progress_chars("#>-")
+pub fn add_bar(prefix: &str, length: u64, bar_type: BarType) -> ProgressBar {
+    let mut style = ProgressStyle::default_bar().progress_chars("#>-");
+
+    style = match bar_type {
+        BarType::Hidden => style.template(""),
+        BarType::Default => style
+            .template("[{bar:.cyan/blue}] - {elapsed:<4} {pos:>7}/{len:7} {per_sec:7} {prefix}"),
+        BarType::Message => style.template(&format!(
+            "[{{bar:.cyan/blue}}] - {{elapsed:<4}} {{pos:>7}}/{{len:7}} {:7} {{prefix}}",
+            "-"
+        )),
+        BarType::Total => {
+            style.template("[{bar:.yellow/blue}] - {elapsed:<4} {pos:>7}/{len:7} {eta:7} {msg}")
+        }
+        BarType::Quiet => style.template("Scanning: {prefix}"),
     };
 
     let progress_bar = PROGRESS_BAR.add(ProgressBar::new(length));
 
     progress_bar.set_style(style);
 
-    progress_bar.set_prefix(&prefix);
+    progress_bar.set_prefix(prefix);
 
     progress_bar
 }
@@ -35,16 +63,19 @@ mod tests {
     #[test]
     /// hit all code branches for add_bar
     fn add_bar_with_all_configurations() {
-        let p1 = add_bar("prefix", 2, true, false); // hidden
-        let p2 = add_bar("prefix", 2, false, true); // no per second field
-        let p3 = add_bar("prefix", 2, false, false); // normal bar
+        let p1 = add_bar("prefix", 2, BarType::Hidden); // hidden
+        let p2 = add_bar("prefix", 2, BarType::Message); // no per second field
+        let p3 = add_bar("prefix", 2, BarType::Default); // normal bar
+        let p4 = add_bar("prefix", 2, BarType::Total); // totals bar
 
         p1.finish();
         p2.finish();
         p3.finish();
+        p4.finish();
 
         assert!(p1.is_finished());
         assert!(p2.is_finished());
         assert!(p3.is_finished());
+        assert!(p4.is_finished());
     }
 }

src/reporter.rs

@@ -1,227 +0,0 @@
-use crate::{
-    config::{CONFIGURATION, PROGRESS_PRINTER},
-    scanner::RESPONSES,
-    utils::{ferox_print, make_request, open_file},
-    FeroxChannel, FeroxResponse, FeroxSerialize,
-};
-use console::strip_ansi_codes;
-use std::io::Write;
-use std::sync::{Arc, Once, RwLock};
-use std::{fs, io};
-use tokio::sync::mpsc::{self, UnboundedReceiver, UnboundedSender};
-use tokio::task::JoinHandle;
-
-/// Singleton buffered file behind an Arc/RwLock; used for file writes from two locations:
-///     - [logger::initialize](../logger/fn.initialize.html) (specifically a closure on the global logger instance)
-///     - `reporter::spawn_file_handler`
-pub static mut LOCKED_FILE: Option<Arc<RwLock<io::BufWriter<fs::File>>>> = None;
-
-/// An initializer Once variable used to create `LOCKED_FILE`
-static INIT: Once = Once::new();
-
-// Accessing a `static mut` is unsafe much of the time, but if we do so
-// in a synchronized fashion (e.g., write once or read all) then we're
-// good to go!
-//
-// This function will only call `open_file` once, and will
-// otherwise always return the value returned from the first invocation.
-pub fn get_cached_file_handle(filename: &str) -> Option<Arc<RwLock<io::BufWriter<fs::File>>>> {
-    unsafe {
-        INIT.call_once(|| {
-            LOCKED_FILE = open_file(&filename);
-        });
-        LOCKED_FILE.clone()
-    }
-}
-
-/// Creates all required output handlers (terminal, file) and returns
-/// the transmitter sides of each mpsc along with each receiver's future's JoinHandle to be awaited
-///
-/// Any other module that needs to write a Response to stdout or output results to a file should
-/// be passed a clone of the appropriate returned transmitter
-pub fn initialize(
-    output_file: &str,
-    save_output: bool,
-) -> (
-    UnboundedSender<FeroxResponse>,
-    UnboundedSender<FeroxResponse>,
-    JoinHandle<()>,
-    Option<JoinHandle<()>>,
-) {
-    log::trace!("enter: initialize({}, {})", output_file, save_output);
-
-    let (tx_rpt, rx_rpt): FeroxChannel<FeroxResponse> = mpsc::unbounded_channel();
-    let (tx_file, rx_file): FeroxChannel<FeroxResponse> = mpsc::unbounded_channel();
-
-    let file_clone = tx_file.clone();
-
-    let term_reporter =
-        tokio::spawn(async move { spawn_terminal_reporter(rx_rpt, file_clone, save_output).await });
-
-    let file_reporter = if save_output {
-        // -o used, need to spawn the thread for writing to disk
-        let file_clone = output_file.to_string();
-        Some(tokio::spawn(async move {
-            spawn_file_reporter(rx_file, &file_clone).await
-        }))
-    } else {
-        None
-    };
-
-    log::trace!(
-        "exit: initialize -> ({:?}, {:?}, {:?}, {:?})",
-        tx_rpt,
-        tx_file,
-        term_reporter,
-        file_reporter
-    );
-    (tx_rpt, tx_file, term_reporter, file_reporter)
-}
-
-/// Spawn a single consumer task (sc side of mpsc)
-///
-/// The consumer simply receives responses and prints them if they meet the given
-/// reporting criteria
-async fn spawn_terminal_reporter(
-    mut resp_chan: UnboundedReceiver<FeroxResponse>,
-    file_chan: UnboundedSender<FeroxResponse>,
-    save_output: bool,
-) {
-    log::trace!(
-        "enter: spawn_terminal_reporter({:?}, {:?}, {})",
-        resp_chan,
-        file_chan,
-        save_output
-    );
-
-    while let Some(resp) = resp_chan.recv().await {
-        log::trace!("received {} on reporting channel", resp.url());
-
-        let contains_sentry = CONFIGURATION.status_codes.contains(&resp.status().as_u16());
-        let unknown_sentry = !RESPONSES.contains(&resp); // !contains == unknown
-        let should_process_response = contains_sentry && unknown_sentry;
-
-        if should_process_response {
-            // print to stdout
-            ferox_print(&resp.as_str(), &PROGRESS_PRINTER);
-
-            if save_output {
-                // -o used, need to send the report to be written out to disk
-                match file_chan.send(resp.clone()) {
-                    Ok(_) => {
-                        log::debug!("Sent {} to file handler", resp.url());
-                    }
-                    Err(e) => {
-                        log::error!("Could not send {} to file handler: {}", resp.url(), e);
-                    }
-                }
-            }
-        }
-        log::trace!("report complete: {}", resp.url());
-
-        if CONFIGURATION.replay_client.is_some() && should_process_response {
-            // replay proxy specified/client created and this response's status code is one that
-            // should be replayed
-            match make_request(CONFIGURATION.replay_client.as_ref().unwrap(), &resp.url()).await {
-                Ok(_) => {}
-                Err(e) => {
-                    log::error!("{}", e);
-                }
-            }
-        }
-
-        if should_process_response {
-            // add response to RESPONSES for serialization in case of ctrl+c
-            // placed all by its lonesome like this so that RESPONSES can take ownership
-            // of the FeroxResponse
-            RESPONSES.insert(resp);
-        }
-    }
-    log::trace!("exit: spawn_terminal_reporter");
-}
-
-/// Spawn a single consumer task (sc side of mpsc)
-///
-/// The consumer simply receives responses and writes them to the given output file if they meet
-/// the given reporting criteria
-async fn spawn_file_reporter(
-    mut report_channel: UnboundedReceiver<FeroxResponse>,
-    output_file: &str,
-) {
-    let buffered_file = match get_cached_file_handle(&CONFIGURATION.output) {
-        Some(file) => file,
-        None => {
-            log::trace!("exit: spawn_file_reporter");
-            return;
-        }
-    };
-
-    log::trace!(
-        "enter: spawn_file_reporter({:?}, {})",
-        report_channel,
-        output_file
-    );
-
-    log::info!("Writing scan results to {}", output_file);
-
-    while let Some(response) = report_channel.recv().await {
-        safe_file_write(&response, buffered_file.clone(), CONFIGURATION.json);
-    }
-
-    log::trace!("exit: spawn_file_reporter");
-}
-
-/// Given a string and a reference to a locked buffered file, write the contents and flush
-/// the buffer to disk.
-pub fn safe_file_write<T>(
-    value: &T,
-    locked_file: Arc<RwLock<io::BufWriter<fs::File>>>,
-    convert_to_json: bool,
-) where
-    T: FeroxSerialize,
-{
-    // note to future self: adding logging of anything other than error to this function
-    // is a bad idea. we call this function while processing records generated by the logger.
-    // If we then call log::... while already processing some logging output, it results in
-    // the second log entry being injected into the first.
-
-    let contents = if convert_to_json {
-        value.as_json()
-    } else {
-        value.as_str()
-    };
-
-    let contents = strip_ansi_codes(&contents);
-
-    if let Ok(mut handle) = locked_file.write() {
-        // write lock acquired
-        match handle.write(contents.as_bytes()) {
-            Ok(_) => {}
-            Err(e) => {
-                log::error!("could not write report to disk: {}", e);
-            }
-        }
-
-        match handle.flush() {
-            // this function is used within async functions/loops, so i'm flushing so that in
-            // the event of a ctrl+c or w/e results seen so far are saved instead of left lying
-            // around in the buffer
-            Ok(_) => {}
-            Err(e) => {
-                log::error!("error writing to file: {}", e);
-            }
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    #[should_panic]
-    /// asserts that an empty string for a filename returns None
-    fn reporter_get_cached_file_handle_without_filename_returns_none() {
-        let _used = get_cached_file_handle(&"").unwrap();
-    }
-}

src/response.rs

@@ -0,0 +1,640 @@
+use std::{
+    collections::HashMap,
+    convert::{TryFrom, TryInto},
+    fmt,
+    str::FromStr,
+    sync::Arc,
+};
+
+use anyhow::{Context, Result};
+use reqwest::{
+    header::{HeaderMap, HeaderName, HeaderValue},
+    Response, StatusCode, Url,
+};
+use serde::ser::SerializeStruct;
+use serde::{Deserialize, Deserializer, Serialize, Serializer};
+use serde_json::Value;
+
+use crate::{
+    config::OutputLevel,
+    event_handlers::{Command, Handles},
+    traits::FeroxSerialize,
+    url::FeroxUrl,
+    utils::{self, fmt_err, status_colorizer},
+    CommandSender,
+};
+
+/// A `FeroxResponse`, derived from a `Response` to a submitted `Request`
+#[derive(Debug, Clone)]
+pub struct FeroxResponse {
+    /// The final `Url` of this `FeroxResponse`
+    url: Url,
+
+    /// The `StatusCode` of this `FeroxResponse`
+    status: StatusCode,
+
+    /// The full response text
+    text: String,
+
+    /// The content-length of this response, if known
+    content_length: u64,
+
+    /// The number of lines contained in the body of this response, if known
+    line_count: usize,
+
+    /// The number of words contained in the body of this response, if known
+    word_count: usize,
+
+    /// The `Headers` of this `FeroxResponse`
+    headers: HeaderMap,
+
+    /// Wildcard response status
+    wildcard: bool,
+
+    /// whether the user passed --quiet|--silent on the command line
+    pub(crate) output_level: OutputLevel,
+}
+
+/// implement Default trait for FeroxResponse
+impl Default for FeroxResponse {
+    /// return a default reqwest::Url and then normal defaults after that
+    fn default() -> Self {
+        Self {
+            url: Url::parse("http://localhost").unwrap(),
+            status: Default::default(),
+            text: "".to_string(),
+            content_length: 0,
+            line_count: 0,
+            word_count: 0,
+            headers: Default::default(),
+            wildcard: false,
+            output_level: Default::default(),
+        }
+    }
+}
+
+/// Implement Display for FeroxResponse
+impl fmt::Display for FeroxResponse {
+    /// formatter for Display
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(
+            f,
+            "FeroxResponse {{ url: {}, status: {}, content-length: {} }}",
+            self.url(),
+            self.status(),
+            self.content_length()
+        )
+    }
+}
+
+/// `FeroxResponse` implementation
+impl FeroxResponse {
+    /// Get the `StatusCode` of this `FeroxResponse`
+    pub fn status(&self) -> &StatusCode {
+        &self.status
+    }
+
+    /// Get the `wildcard` of this `FeroxResponse`
+    pub fn wildcard(&self) -> bool {
+        self.wildcard
+    }
+
+    /// Get the final `Url` of this `FeroxResponse`.
+    pub fn url(&self) -> &Url {
+        &self.url
+    }
+
+    /// Get the full response text
+    pub fn text(&self) -> &str {
+        &self.text
+    }
+
+    /// Get the `Headers` of this `FeroxResponse`
+    pub fn headers(&self) -> &HeaderMap {
+        &self.headers
+    }
+
+    /// Get the content-length of this response, if known
+    pub fn content_length(&self) -> u64 {
+        self.content_length
+    }
+
+    /// Set `FeroxResponse`'s `url` attribute, has no affect if an error occurs
+    pub fn set_url(&mut self, url: &str) {
+        match Url::parse(url) {
+            Ok(url) => {
+                self.url = url;
+            }
+            Err(e) => {
+                log::warn!("Could not parse {} into a Url: {}", url, e);
+            }
+        };
+    }
+
+    /// set `wildcard` attribute
+    pub fn set_wildcard(&mut self, is_wildcard: bool) {
+        self.wildcard = is_wildcard;
+    }
+
+    /// set `text` attribute; update words/lines/content_length
+    #[cfg(test)]
+    pub fn set_text(&mut self, text: &str) {
+        self.text = String::from(text);
+        self.content_length = self.text.len() as u64;
+        self.line_count = self.text.lines().count();
+        self.word_count = self
+            .text
+            .lines()
+            .map(|s| s.split_whitespace().count())
+            .sum();
+    }
+
+    /// free the `text` data, reducing memory usage
+    pub fn drop_text(&mut self) {
+        self.text = String::new();
+    }
+
+    /// Make a reasonable guess at whether the response is a file or not
+    ///
+    /// Examines the last part of a path to determine if it has an obvious extension
+    /// i.e. http://localhost/some/path/stuff.js where stuff.js indicates a file
+    ///
+    /// Additionally, inspects query parameters, as they're also often indicative of a file
+    pub fn is_file(&self) -> bool {
+        let has_extension = match self.url.path_segments() {
+            Some(path) => {
+                if let Some(last) = path.last() {
+                    last.contains('.') // last segment has some sort of extension, probably
+                } else {
+                    false
+                }
+            }
+            None => false,
+        };
+
+        self.url.query_pairs().count() > 0 || has_extension
+    }
+
+    /// Returns line count of the response text.
+    pub fn line_count(&self) -> usize {
+        self.line_count
+    }
+
+    /// Returns word count of the response text.
+    pub fn word_count(&self) -> usize {
+        self.word_count
+    }
+
+    /// Create a new `FeroxResponse` from the given `Response`
+    pub async fn from(response: Response, read_body: bool, output_level: OutputLevel) -> Self {
+        let url = response.url().clone();
+        let status = response.status();
+        let headers = response.headers().clone();
+        let content_length = response.content_length().unwrap_or(0);
+
+        let text = if read_body {
+            // .text() consumes the response, must be called last
+            // additionally, --extract-links is currently the only place we use the body of the
+            // response, so we forego the processing if not performing extraction
+            match response.text().await {
+                // await the response's body
+                Ok(text) => text,
+                Err(e) => {
+                    log::warn!("Could not parse body from response: {}", e);
+                    String::new()
+                }
+            }
+        } else {
+            String::new()
+        };
+
+        let line_count = text.lines().count();
+        let word_count = text.lines().map(|s| s.split_whitespace().count()).sum();
+
+        FeroxResponse {
+            url,
+            status,
+            content_length,
+            text,
+            headers,
+            line_count,
+            word_count,
+            output_level,
+            wildcard: false,
+        }
+    }
+
+    /// Helper function that determines if the configured maximum recursion depth has been reached
+    ///
+    /// Essentially looks at the Url path and determines how many directories are present in the
+    /// given Url
+    pub(crate) fn reached_max_depth(
+        &self,
+        base_depth: usize,
+        max_depth: usize,
+        handles: Arc<Handles>,
+    ) -> bool {
+        log::trace!(
+            "enter: reached_max_depth({}, {}, {:?})",
+            base_depth,
+            max_depth,
+            handles
+        );
+
+        if max_depth == 0 {
+            // early return, as 0 means recurse forever; no additional processing needed
+            log::trace!("exit: reached_max_depth -> false");
+            return false;
+        }
+        let url = FeroxUrl::from_url(&self.url, handles);
+        let depth = url.depth().unwrap_or_default(); // 0 on error
+
+        if depth - base_depth >= max_depth {
+            return true;
+        }
+
+        log::trace!("exit: reached_max_depth -> false");
+        false
+    }
+
+    /// Helper function to determine suitability for recursion
+    ///
+    /// handles 2xx and 3xx responses by either checking if the url ends with a / (2xx)
+    /// or if the Location header is present and matches the base url + / (3xx)
+    pub fn is_directory(&self) -> bool {
+        log::trace!("enter: is_directory({})", self);
+
+        if self.status().is_redirection() {
+            // status code is 3xx
+            match self.headers().get("Location") {
+                // and has a Location header
+                Some(loc) => {
+                    // get absolute redirect Url based on the already known base url
+                    log::debug!("Location header: {:?}", loc);
+
+                    if let Ok(loc_str) = loc.to_str() {
+                        if let Ok(abs_url) = self.url().join(loc_str) {
+                            if format!("{}/", self.url()) == abs_url.as_str() {
+                                // if current response's Url + / == the absolute redirection
+                                // location, we've found a directory suitable for recursion
+                                log::debug!(
+                                    "found directory suitable for recursion: {}",
+                                    self.url()
+                                );
+                                log::trace!("exit: is_directory -> true");
+                                return true;
+                            }
+                        }
+                    }
+                }
+                None => {
+                    log::debug!("expected Location header, but none was found: {}", self);
+                    log::trace!("exit: is_directory -> false");
+                    return false;
+                }
+            }
+        } else if self.status().is_success() || matches!(self.status(), &StatusCode::FORBIDDEN) {
+            // status code is 2xx or 403, need to check if it ends in /
+
+            if self.url().as_str().ends_with('/') {
+                log::debug!("{} is directory suitable for recursion", self.url());
+                log::trace!("exit: is_directory -> true");
+                return true;
+            }
+        }
+
+        log::trace!("exit: is_directory -> false");
+        false
+    }
+
+    /// Simple helper to send a `FeroxResponse` over the tx side of an `mpsc::unbounded_channel`
+    pub fn send_report(self, report_sender: CommandSender) -> Result<()> {
+        log::trace!("enter: send_report({:?}", report_sender);
+
+        report_sender.send(Command::Report(Box::new(self)))?;
+
+        log::trace!("exit: send_report");
+        Ok(())
+    }
+}
+
+/// Implement FeroxSerialize for FeroxResponse
+impl FeroxSerialize for FeroxResponse {
+    /// Simple wrapper around create_report_string
+    fn as_str(&self) -> String {
+        let lines = self.line_count().to_string();
+        let words = self.word_count().to_string();
+        let chars = self.content_length().to_string();
+        let status = self.status().as_str();
+        let wild_status = status_colorizer("WLD");
+
+        if self.wildcard && matches!(self.output_level, OutputLevel::Default | OutputLevel::Quiet) {
+            // --silent was not used and response is a wildcard, special messages abound when
+            // this is the case...
+
+            // create the base message
+            let mut message = format!(
+                "{} {:>8}l {:>8}w {:>8}c Got {} for {} (url length: {})\n",
+                wild_status,
+                lines,
+                words,
+                chars,
+                status_colorizer(status),
+                self.url(),
+                FeroxUrl::path_length_of_url(&self.url)
+            );
+
+            if self.status().is_redirection() {
+                // when it's a redirect, show where it goes, if possible
+                if let Some(next_loc) = self.headers().get("Location") {
+                    let next_loc_str = next_loc.to_str().unwrap_or("Unknown");
+
+                    let redirect_msg = format!(
+                        "{} {:>9} {:>9} {:>9} {} redirects to => {}\n",
+                        wild_status,
+                        "-",
+                        "-",
+                        "-",
+                        self.url(),
+                        next_loc_str
+                    );
+
+                    message.push_str(&redirect_msg);
+                }
+            }
+
+            // base message + redirection message (if appropriate)
+            message
+        } else {
+            // not a wildcard, just create a normal entry
+            utils::create_report_string(
+                self.status.as_str(),
+                &lines,
+                &words,
+                &chars,
+                self.url().as_str(),
+                self.output_level,
+            )
+        }
+    }
+
+    /// Create an NDJSON representation of the FeroxResponse
+    ///
+    /// (expanded for clarity)
+    /// ex:
+    /// {
+    ///    "type":"response",
+    ///    "url":"https://localhost.com/images",
+    ///    "path":"/images",
+    ///    "status":301,
+    ///    "content_length":179,
+    ///    "line_count":10,
+    ///    "word_count":16,
+    ///    "headers":{
+    ///       "x-content-type-options":"nosniff",
+    ///       "strict-transport-security":"max-age=31536000; includeSubDomains",
+    ///       "x-frame-options":"SAMEORIGIN",
+    ///       "connection":"keep-alive",
+    ///       "server":"nginx/1.16.1",
+    ///       "content-type":"text/html; charset=UTF-8",
+    ///       "referrer-policy":"origin-when-cross-origin",
+    ///       "content-security-policy":"default-src 'none'",
+    ///       "access-control-allow-headers":"X-Requested-With",
+    ///       "x-xss-protection":"1; mode=block",
+    ///       "content-length":"179",
+    ///       "date":"Mon, 23 Nov 2020 15:33:24 GMT",
+    ///       "location":"/images/",
+    ///       "access-control-allow-origin":"https://localhost.com"
+    ///    }
+    /// }\n
+    fn as_json(&self) -> anyhow::Result<String> {
+        let mut json = serde_json::to_string(&self)
+            .with_context(|| fmt_err(&format!("Could not convert {} to JSON", self.url())))?;
+        json.push('\n');
+        Ok(json)
+    }
+}
+
+/// Serialize implementation for FeroxResponse
+impl Serialize for FeroxResponse {
+    /// Function that handles serialization of a FeroxResponse to NDJSON
+    fn serialize<S>(&self, serializer: S) -> anyhow::Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let mut headers = HashMap::new();
+        let mut state = serializer.serialize_struct("FeroxResponse", 7)?;
+
+        // need to convert the HeaderMap to a HashMap in order to pass it to the serializer
+        for (key, value) in &self.headers {
+            let k = key.as_str().to_owned();
+            let v = String::from_utf8_lossy(value.as_bytes());
+            headers.insert(k, v);
+        }
+
+        state.serialize_field("type", "response")?;
+        state.serialize_field("url", self.url.as_str())?;
+        state.serialize_field("path", self.url.path())?;
+        state.serialize_field("wildcard", &self.wildcard)?;
+        state.serialize_field("status", &self.status.as_u16())?;
+        state.serialize_field("content_length", &self.content_length)?;
+        state.serialize_field("line_count", &self.line_count)?;
+        state.serialize_field("word_count", &self.word_count)?;
+        state.serialize_field("headers", &headers)?;
+
+        state.end()
+    }
+}
+
+/// Deserialize implementation for FeroxResponse
+impl<'de> Deserialize<'de> for FeroxResponse {
+    /// Deserialize a FeroxResponse from a serde_json::Value
+    fn deserialize<D>(deserializer: D) -> anyhow::Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let mut response = Self {
+            url: Url::parse("http://localhost").unwrap(),
+            status: StatusCode::OK,
+            text: String::new(),
+            content_length: 0,
+            headers: HeaderMap::new(),
+            wildcard: false,
+            output_level: Default::default(),
+            line_count: 0,
+            word_count: 0,
+        };
+
+        let map: HashMap<String, Value> = HashMap::deserialize(deserializer)?;
+
+        for (key, value) in &map {
+            match key.as_str() {
+                "url" => {
+                    if let Some(url) = value.as_str() {
+                        if let Ok(parsed) = Url::parse(url) {
+                            response.url = parsed;
+                        }
+                    }
+                }
+                "status" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(smaller) = u16::try_from(num) {
+                            if let Ok(status) = StatusCode::from_u16(smaller) {
+                                response.status = status;
+                            }
+                        }
+                    }
+                }
+                "content_length" => {
+                    if let Some(num) = value.as_u64() {
+                        response.content_length = num;
+                    }
+                }
+                "line_count" => {
+                    if let Some(num) = value.as_u64() {
+                        response.line_count = num.try_into().unwrap_or_default();
+                    }
+                }
+                "word_count" => {
+                    if let Some(num) = value.as_u64() {
+                        response.word_count = num.try_into().unwrap_or_default();
+                    }
+                }
+                "headers" => {
+                    let mut headers = HeaderMap::<HeaderValue>::default();
+
+                    if let Some(map_headers) = value.as_object() {
+                        for (h_key, h_value) in map_headers {
+                            let h_value_str = h_value.as_str().unwrap_or("");
+                            let h_name = HeaderName::from_str(h_key)
+                                .unwrap_or_else(|_| HeaderName::from_str("Unknown").unwrap());
+                            let h_value_parsed = HeaderValue::from_str(h_value_str)
+                                .unwrap_or_else(|_| HeaderValue::from_str("Unknown").unwrap());
+                            headers.insert(h_name, h_value_parsed);
+                        }
+                    }
+
+                    response.headers = headers;
+                }
+                "wildcard" => {
+                    if let Some(result) = value.as_bool() {
+                        response.wildcard = result;
+                    }
+                }
+                _ => {}
+            }
+        }
+
+        Ok(response)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    /// call reached_max_depth with max depth of zero, which is infinite recursion, expect false
+    fn reached_max_depth_returns_early_on_zero() {
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+        let url = Url::parse("http://localhost").unwrap();
+        let response = FeroxResponse {
+            url,
+            status: Default::default(),
+            text: "".to_string(),
+            content_length: 0,
+            line_count: 0,
+            word_count: 0,
+            headers: Default::default(),
+            wildcard: false,
+            output_level: Default::default(),
+        };
+        let result = response.reached_max_depth(0, 0, handles);
+        assert!(!result);
+    }
+
+    #[test]
+    /// call reached_max_depth with url depth equal to max depth, expect true
+    fn reached_max_depth_current_depth_equals_max() {
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+
+        let url = Url::parse("http://localhost/one/two").unwrap();
+        let response = FeroxResponse {
+            url,
+            status: Default::default(),
+            text: "".to_string(),
+            content_length: 0,
+            line_count: 0,
+            word_count: 0,
+            headers: Default::default(),
+            wildcard: false,
+            output_level: Default::default(),
+        };
+
+        let result = response.reached_max_depth(0, 2, handles);
+        assert!(result);
+    }
+
+    #[test]
+    /// call reached_max_depth with url dpeth less than max depth, expect false
+    fn reached_max_depth_current_depth_less_than_max() {
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+        let url = Url::parse("http://localhost").unwrap();
+        let response = FeroxResponse {
+            url,
+            status: Default::default(),
+            text: "".to_string(),
+            content_length: 0,
+            line_count: 0,
+            word_count: 0,
+            headers: Default::default(),
+            wildcard: false,
+            output_level: Default::default(),
+        };
+
+        let result = response.reached_max_depth(0, 2, handles);
+        assert!(!result);
+    }
+
+    #[test]
+    /// call reached_max_depth with url of 2, base depth of 2, and max depth of 2, expect false
+    fn reached_max_depth_base_depth_equals_max_depth() {
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+        let url = Url::parse("http://localhost/one/two").unwrap();
+        let response = FeroxResponse {
+            url,
+            status: Default::default(),
+            text: "".to_string(),
+            content_length: 0,
+            line_count: 0,
+            word_count: 0,
+            headers: Default::default(),
+            wildcard: false,
+            output_level: Default::default(),
+        };
+
+        let result = response.reached_max_depth(2, 2, handles);
+        assert!(!result);
+    }
+
+    #[test]
+    /// call reached_max_depth with url depth greater than max depth, expect true
+    fn reached_max_depth_current_greater_than_max() {
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+        let url = Url::parse("http://localhost/one/two/three").unwrap();
+        let response = FeroxResponse {
+            url,
+            status: Default::default(),
+            text: "".to_string(),
+            content_length: 0,
+            line_count: 0,
+            word_count: 0,
+            headers: Default::default(),
+            wildcard: false,
+            output_level: Default::default(),
+        };
+
+        let result = response.reached_max_depth(0, 2, handles);
+        assert!(result);
+    }
+}

src/scan_manager.rs

@@ -1,992 +0,0 @@
-use crate::config::Configuration;
-use crate::reporter::safe_file_write;
-use crate::utils::open_file;
-use crate::{
-    config::{CONFIGURATION, PROGRESS_PRINTER},
-    progress,
-    scanner::{NUMBER_OF_REQUESTS, RESPONSES, SCANNED_URLS},
-    FeroxResponse, FeroxSerialize, SLEEP_DURATION,
-};
-use console::style;
-use indicatif::{ProgressBar, ProgressStyle};
-use lazy_static::lazy_static;
-use serde::{
-    ser::{SerializeSeq, SerializeStruct},
-    Deserialize, Deserializer, Serialize, Serializer,
-};
-use serde_json::Value;
-use std::collections::HashMap;
-use std::{
-    cmp::PartialEq,
-    fmt,
-    fs::File,
-    io::BufReader,
-    sync::{Arc, Mutex, RwLock},
-    time::{SystemTime, UNIX_EPOCH},
-};
-use std::{
-    io::{stderr, Write},
-    sync::atomic::{AtomicBool, AtomicUsize, Ordering},
-};
-use tokio::{task::JoinHandle, time};
-use uuid::Uuid;
-
-lazy_static! {
-    /// A clock spinner protected with a RwLock to allow for a single thread to use at a time
-    // todo remove this when issue #107 is resolved
-    static ref SINGLE_SPINNER: RwLock<ProgressBar> = RwLock::new(get_single_spinner());
-}
-
-/// Single atomic number that gets incremented once, used to track first thread to interact with
-/// when pausing a scan
-static INTERACTIVE_BARRIER: AtomicUsize = AtomicUsize::new(0);
-
-/// Atomic boolean flag, used to determine whether or not a scan should pause or resume
-pub static PAUSE_SCAN: AtomicBool = AtomicBool::new(false);
-
-/// Simple enum used to flag a `FeroxScan` as likely a directory or file
-#[derive(Debug, Serialize, Deserialize)]
-pub enum ScanType {
-    File,
-    Directory,
-}
-
-/// Default implementation for ScanType
-impl Default for ScanType {
-    /// Return ScanType::File as default
-    fn default() -> Self {
-        Self::File
-    }
-}
-
-/// Struct to hold scan-related state
-///
-/// The purpose of this container is to open up the pathway to aborting currently running tasks and
-/// serialization of all scan state into a state file in order to resume scans that were cut short
-#[derive(Debug)]
-pub struct FeroxScan {
-    /// UUID that uniquely ID's the scan
-    pub id: String,
-
-    /// The URL that to be scanned
-    pub url: String,
-
-    /// The type of scan
-    pub scan_type: ScanType,
-
-    /// Whether or not this scan has completed
-    pub complete: bool,
-
-    /// The spawned tokio task performing this scan
-    pub task: Option<JoinHandle<()>>,
-
-    /// The progress bar associated with this scan
-    pub progress_bar: Option<ProgressBar>,
-}
-
-/// Default implementation for FeroxScan
-impl Default for FeroxScan {
-    /// Create a default FeroxScan, populates ID with a new UUID
-    fn default() -> Self {
-        let new_id = Uuid::new_v4().to_simple().to_string();
-
-        FeroxScan {
-            id: new_id,
-            task: None,
-            complete: false,
-            url: String::new(),
-            progress_bar: None,
-            scan_type: ScanType::File,
-        }
-    }
-}
-
-/// Implementation of FeroxScan
-impl FeroxScan {
-    /// Stop a currently running scan
-    pub fn abort(&self) {
-        self.stop_progress_bar();
-
-        if let Some(_task) = &self.task {
-            // task.abort();  todo uncomment once upgraded to tokio 0.3 (issue #107)
-        }
-    }
-
-    /// Simple helper to call .finish on the scan's progress bar
-    fn stop_progress_bar(&self) {
-        if let Some(pb) = &self.progress_bar {
-            pb.finish();
-        }
-    }
-
-    /// Simple helper get a progress bar
-    pub fn progress_bar(&mut self) -> ProgressBar {
-        if let Some(pb) = &self.progress_bar {
-            pb.clone()
-        } else {
-            let num_requests = NUMBER_OF_REQUESTS.load(Ordering::Relaxed);
-            let pb = progress::add_bar(&self.url, num_requests, false, false);
-
-            pb.reset_elapsed();
-
-            self.progress_bar = Some(pb.clone());
-
-            pb
-        }
-    }
-
-    /// Given a URL and ProgressBar, create a new FeroxScan, wrap it in an Arc and return it
-    pub fn new(url: &str, scan_type: ScanType, pb: Option<ProgressBar>) -> Arc<Mutex<Self>> {
-        let mut me = Self::default();
-
-        me.url = url.to_string();
-        me.scan_type = scan_type;
-        me.progress_bar = pb;
-
-        Arc::new(Mutex::new(me))
-    }
-
-    /// Mark the scan as complete and stop the scan's progress bar
-    pub fn finish(&mut self) {
-        self.complete = true;
-        self.stop_progress_bar();
-    }
-}
-
-/// Display implementation
-impl fmt::Display for FeroxScan {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        let complete = if self.complete {
-            style("complete").green()
-        } else {
-            style("incomplete").red()
-        };
-
-        write!(f, "{:10} {}", complete, self.url)
-    }
-}
-
-/// PartialEq implementation; uses FeroxScan.id for comparison
-impl PartialEq for FeroxScan {
-    fn eq(&self, other: &Self) -> bool {
-        self.id == other.id
-    }
-}
-
-/// Serialize implementation for FeroxScan
-impl Serialize for FeroxScan {
-    /// Function that handles serialization of a FeroxScan
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        let mut state = serializer.serialize_struct("FeroxScan", 4)?;
-
-        state.serialize_field("id", &self.id)?;
-        state.serialize_field("url", &self.url)?;
-        state.serialize_field("scan_type", &self.scan_type)?;
-        state.serialize_field("complete", &self.complete)?;
-
-        state.end()
-    }
-}
-
-/// Deserialize implementation for FeroxScan
-impl<'de> Deserialize<'de> for FeroxScan {
-    /// Deserialize a FeroxScan from a serde_json::Value
-    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
-    where
-        D: Deserializer<'de>,
-    {
-        let mut scan = Self::default();
-
-        let map: HashMap<String, Value> = HashMap::deserialize(deserializer)?;
-
-        for (key, value) in &map {
-            match key.as_str() {
-                "id" => {
-                    if let Some(id) = value.as_str() {
-                        scan.id = id.to_string();
-                    }
-                }
-                "scan_type" => {
-                    if let Some(scan_type) = value.as_str() {
-                        scan.scan_type = match scan_type {
-                            "File" => ScanType::File,
-                            "Directory" => ScanType::Directory,
-                            _ => ScanType::File,
-                        }
-                    }
-                }
-                "complete" => {
-                    if let Some(complete) = value.as_bool() {
-                        scan.complete = complete;
-                    }
-                }
-                "url" => {
-                    if let Some(url) = value.as_str() {
-                        scan.url = url.to_string();
-                    }
-                }
-                _ => {}
-            }
-        }
-
-        Ok(scan)
-    }
-}
-
-/// Container around a locked hashset of `FeroxScan`s, adds wrappers for insertion and searching
-#[derive(Debug, Default)]
-pub struct FeroxScans {
-    /// Internal structure: locked hashset of `FeroxScan`s
-    pub scans: Mutex<Vec<Arc<Mutex<FeroxScan>>>>,
-}
-
-/// Serialize implementation for FeroxScans
-impl Serialize for FeroxScans {
-    /// Function that handles serialization of FeroxScans
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        if let Ok(scans) = self.scans.lock() {
-            let mut seq = serializer.serialize_seq(Some(scans.len()))?;
-
-            for scan in scans.iter() {
-                if let Ok(unlocked) = scan.lock() {
-                    seq.serialize_element(&*unlocked)?;
-                }
-            }
-
-            seq.end()
-        } else {
-            // if for some reason we can't unlock the mutex, just write an empty list
-            let seq = serializer.serialize_seq(Some(0))?;
-            seq.end()
-        }
-    }
-}
-
-/// Implementation of `FeroxScans`
-impl FeroxScans {
-    /// Add a `FeroxScan` to the internal container
-    ///
-    /// If the internal container did NOT contain the scan, true is returned; else false
-    pub fn insert(&self, scan: Arc<Mutex<FeroxScan>>) -> bool {
-        let sentry = match scan.lock() {
-            Ok(locked_scan) => {
-                // If the container did contain the scan, set sentry to false
-                // If the container did not contain the scan, set sentry to true
-                !self.contains(&locked_scan.url)
-            }
-            Err(e) => {
-                // poisoned lock
-                log::error!("FeroxScan's ({:?}) mutex is poisoned: {}", self, e);
-                false
-            }
-        };
-
-        if sentry {
-            // can't update the internal container while the scan itself is locked, so first
-            // lock the scan and check the container for the scan's presence, then add if
-            // not found
-            match self.scans.lock() {
-                Ok(mut scans) => {
-                    scans.push(scan);
-                }
-                Err(e) => {
-                    log::error!("FeroxScans' container's mutex is poisoned: {}", e);
-                    return false;
-                }
-            }
-        }
-
-        sentry
-    }
-
-    /// Simple check for whether or not a FeroxScan is contained within the inner container based
-    /// on the given URL
-    pub fn contains(&self, url: &str) -> bool {
-        match self.scans.lock() {
-            Ok(scans) => {
-                for scan in scans.iter() {
-                    if let Ok(locked_scan) = scan.lock() {
-                        if locked_scan.url == url {
-                            return true;
-                        }
-                    }
-                }
-            }
-            Err(e) => {
-                log::error!("FeroxScans' container's mutex is poisoned: {}", e);
-            }
-        }
-        false
-    }
-
-    /// Find and return a `FeroxScan` based on the given URL
-    pub fn get_scan_by_url(&self, url: &str) -> Option<Arc<Mutex<FeroxScan>>> {
-        if let Ok(scans) = self.scans.lock() {
-            for scan in scans.iter() {
-                if let Ok(locked_scan) = scan.lock() {
-                    if locked_scan.url == url {
-                        return Some(scan.clone());
-                    }
-                }
-            }
-        }
-        None
-    }
-
-    /// Print all FeroxScans of type Directory
-    ///
-    /// Example:
-    ///   0: complete   https://10.129.45.20
-    ///   9: complete   https://10.129.45.20/images
-    ///  10: complete   https://10.129.45.20/assets
-    pub fn display_scans(&self) {
-        if let Ok(scans) = self.scans.lock() {
-            for (i, scan) in scans.iter().enumerate() {
-                if let Ok(unlocked_scan) = scan.lock() {
-                    match unlocked_scan.scan_type {
-                        ScanType::Directory => {
-                            PROGRESS_PRINTER.println(format!("{:3}: {}", i, unlocked_scan));
-                        }
-                        ScanType::File => {
-                            // we're only interested in displaying directory scans, as those are
-                            // the only ones that make sense to be stopped
-                        }
-                    }
-                }
-            }
-        }
-    }
-
-    /// Forced the calling thread into a busy loop
-    ///
-    /// Every `SLEEP_DURATION` milliseconds, the function examines the result stored in `PAUSE_SCAN`
-    ///
-    /// When the value stored in `PAUSE_SCAN` becomes `false`, the function returns, exiting the busy
-    /// loop
-    pub async fn pause(&self, get_user_input: bool) {
-        // function uses tokio::time, not std
-
-        // local testing showed a pretty slow increase (less than linear) in CPU usage as # of
-        // concurrent scans rose when SLEEP_DURATION was set to 500, using that as the default for now
-        let mut interval = time::interval(time::Duration::from_millis(SLEEP_DURATION));
-
-        // ignore any error returned
-        let _ = stderr().flush();
-
-        if INTERACTIVE_BARRIER.load(Ordering::Relaxed) == 0 {
-            INTERACTIVE_BARRIER.fetch_add(1, Ordering::Relaxed);
-
-            if get_user_input {
-                self.display_scans();
-
-                let mut user_input = String::new();
-                std::io::stdin().read_line(&mut user_input).unwrap();
-                // todo (issue #107) actual logic for parsing user input in a way that allows for
-                // calling .abort on the scan retrieved based on the input
-            }
-        }
-
-        if SINGLE_SPINNER.read().unwrap().is_finished() {
-            // todo remove this when issue #107 is resolved
-
-            // in order to not leave draw artifacts laying around in the terminal, we call
-            // finish_and_clear on the progress bar when resuming scans. For this reason, we need to
-            // check if the spinner is finished, and repopulate the RwLock with a new spinner if
-            // necessary
-            if let Ok(mut guard) = SINGLE_SPINNER.write() {
-                *guard = get_single_spinner();
-            }
-        }
-
-        if let Ok(spinner) = SINGLE_SPINNER.write() {
-            spinner.enable_steady_tick(120);
-        }
-
-        loop {
-            // first tick happens immediately, all others wait the specified duration
-            interval.tick().await;
-
-            if !PAUSE_SCAN.load(Ordering::Acquire) {
-                // PAUSE_SCAN is false, so we can exit the busy loop
-
-                if INTERACTIVE_BARRIER.load(Ordering::Relaxed) == 1 {
-                    INTERACTIVE_BARRIER.fetch_sub(1, Ordering::Relaxed);
-                }
-
-                if let Ok(spinner) = SINGLE_SPINNER.write() {
-                    // todo remove this when issue #107 is resolved
-                    spinner.finish_and_clear();
-                }
-
-                let _ = stderr().flush();
-
-                log::trace!("exit: pause_scan");
-                return;
-            }
-        }
-    }
-
-    /// Given a url, create a new `FeroxScan` and add it to `FeroxScans`
-    ///
-    /// If `FeroxScans` did not already contain the scan, return true; otherwise return false
-    ///
-    /// Also return a reference to the new `FeroxScan`
-    fn add_scan(&self, url: &str, scan_type: ScanType) -> (bool, Arc<Mutex<FeroxScan>>) {
-        let bar = match scan_type {
-            ScanType::Directory => {
-                let progress_bar = progress::add_bar(
-                    &url,
-                    NUMBER_OF_REQUESTS.load(Ordering::Relaxed),
-                    false,
-                    false,
-                );
-
-                progress_bar.reset_elapsed();
-
-                Some(progress_bar)
-            }
-            ScanType::File => None,
-        };
-
-        let ferox_scan = FeroxScan::new(&url, scan_type, bar);
-
-        // If the set did not contain the scan, true is returned.
-        // If the set did contain the scan, false is returned.
-        let response = self.insert(ferox_scan.clone());
-
-        (response, ferox_scan)
-    }
-
-    /// Given a url, create a new `FeroxScan` and add it to `FeroxScans` as a Directory Scan
-    ///
-    /// If `FeroxScans` did not already contain the scan, return true; otherwise return false
-    ///
-    /// Also return a reference to the new `FeroxScan`
-    pub fn add_directory_scan(&self, url: &str) -> (bool, Arc<Mutex<FeroxScan>>) {
-        self.add_scan(&url, ScanType::Directory)
-    }
-
-    /// Given a url, create a new `FeroxScan` and add it to `FeroxScans` as a File Scan
-    ///
-    /// If `FeroxScans` did not already contain the scan, return true; otherwise return false
-    ///
-    /// Also return a reference to the new `FeroxScan`
-    pub fn add_file_scan(&self, url: &str) -> (bool, Arc<Mutex<FeroxScan>>) {
-        self.add_scan(&url, ScanType::File)
-    }
-}
-
-/// Return a clock spinner, used when scans are paused
-// todo remove this when issue #107 is resolved
-fn get_single_spinner() -> ProgressBar {
-    log::trace!("enter: get_single_spinner");
-
-    let spinner = ProgressBar::new_spinner().with_style(
-        ProgressStyle::default_spinner()
-            .tick_strings(&[
-                "🕛", "🕐", "🕑", "🕒", "🕓", "🕔", "🕕", "🕖", "🕗", "🕘", "🕙", "🕚",
-            ])
-            .template(&format!(
-                "\t-= All Scans {{spinner}} {} =-",
-                style("Paused").red()
-            )),
-    );
-
-    log::trace!("exit: get_single_spinner -> {:?}", spinner);
-    spinner
-}
-
-/// Container around a locked vector of `FeroxResponse`s, adds wrappers for insertion and search
-#[derive(Debug, Default)]
-pub struct FeroxResponses {
-    /// Internal structure: locked hashset of `FeroxScan`s
-    pub responses: Arc<RwLock<Vec<FeroxResponse>>>,
-}
-
-/// Serialize implementation for FeroxResponses
-impl Serialize for FeroxResponses {
-    /// Function that handles serialization of FeroxResponses
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        if let Ok(responses) = self.responses.read() {
-            let mut seq = serializer.serialize_seq(Some(responses.len()))?;
-
-            for response in responses.iter() {
-                seq.serialize_element(response)?;
-            }
-
-            seq.end()
-        } else {
-            // if for some reason we can't unlock the mutex, just write an empty list
-            let seq = serializer.serialize_seq(Some(0))?;
-            seq.end()
-        }
-    }
-}
-
-/// Implementation of `FeroxResponses`
-impl FeroxResponses {
-    /// Add a `FeroxResponse` to the internal container
-    pub fn insert(&self, response: FeroxResponse) {
-        match self.responses.write() {
-            Ok(mut responses) => {
-                responses.push(response);
-            }
-            Err(e) => {
-                log::error!("FeroxResponses' container's mutex is poisoned: {}", e);
-            }
-        }
-    }
-
-    /// Simple check for whether or not a FeroxResponse is contained within the inner container
-    pub fn contains(&self, other: &FeroxResponse) -> bool {
-        match self.responses.read() {
-            Ok(responses) => {
-                for response in responses.iter() {
-                    if response.url == other.url {
-                        return true;
-                    }
-                }
-            }
-            Err(e) => {
-                log::error!("FeroxResponses' container's mutex is poisoned: {}", e);
-            }
-        }
-        false
-    }
-}
-/// Data container for (de)?serialization of multiple items
-#[derive(Serialize, Debug)]
-pub struct FeroxState {
-    /// Known scans
-    scans: &'static FeroxScans,
-
-    /// Current running config
-    config: &'static Configuration,
-
-    /// Known responses
-    responses: &'static FeroxResponses,
-}
-
-/// FeroxSerialize implementation for FeroxState
-impl FeroxSerialize for FeroxState {
-    /// Simply return debug format of FeroxState to satisfy as_str
-    fn as_str(&self) -> String {
-        format!("{:?}", self)
-    }
-
-    /// Simple call to produce a JSON string using the given FeroxState
-    fn as_json(&self) -> String {
-        serde_json::to_string(&self).unwrap_or_default()
-    }
-}
-
-/// Initialize the ctrl+c handler that saves scan state to disk
-pub fn initialize() {
-    log::trace!("enter: initialize");
-
-    let result = ctrlc::set_handler(move || {
-        let ts = SystemTime::now()
-            .duration_since(UNIX_EPOCH)
-            .unwrap()
-            .as_secs();
-
-        let slug = if !CONFIGURATION.target_url.is_empty() {
-            // target url populated
-            CONFIGURATION
-                .target_url
-                .replace("://", "_")
-                .replace("/", "_")
-                .replace(".", "_")
-        } else {
-            // stdin used
-            "stdin".to_string()
-        };
-
-        let filename = format!("ferox-{}-{}.state", slug, ts);
-        let warning = format!(
-            "🚨 Caught {} 🚨 saving scan state to {} ...",
-            style("ctrl+c").yellow(),
-            filename
-        );
-
-        PROGRESS_PRINTER.println(warning);
-
-        let state = FeroxState {
-            config: &CONFIGURATION,
-            scans: &SCANNED_URLS,
-            responses: &RESPONSES,
-        };
-
-        let state_file = open_file(&filename);
-
-        if let Some(buffered_file) = state_file {
-            safe_file_write(&state, buffered_file, true);
-        }
-
-        std::process::exit(1);
-    });
-
-    if result.is_err() {
-        log::error!("Could not set Ctrl+c handler");
-        std::process::exit(1);
-    }
-
-    log::trace!("exit: initialize");
-}
-
-/// Primary logic used to load a Configuration from disk and populate the appropriate data
-/// structures
-pub fn resume_scan(filename: &str) -> Configuration {
-    log::trace!("enter: resume_scan({})", filename);
-
-    let file = File::open(filename).unwrap_or_else(|e| {
-        log::error!("{}", e);
-        log::error!("Could not open state file, exiting");
-        std::process::exit(1);
-    });
-
-    let reader = BufReader::new(file);
-    let state: serde_json::Value = serde_json::from_reader(reader).unwrap();
-
-    let conf = state.get("config").unwrap_or_else(|| {
-        log::error!("Could not load configuration from state file, exiting");
-        std::process::exit(1);
-    });
-
-    let config = serde_json::from_value(conf.clone()).unwrap_or_else(|e| {
-        log::error!("{}", e);
-        log::error!("Could not deserialize configuration found in state file, exiting");
-        std::process::exit(1);
-    });
-
-    // let scans: FeroxScans = serde_json::from_value(state.get("scans").unwrap().clone()).unwrap();
-    if let Some(responses) = state.get("responses") {
-        if let Some(arr_responses) = responses.as_array() {
-            for response in arr_responses {
-                if let Ok(deser_resp) = serde_json::from_value(response.clone()) {
-                    RESPONSES.insert(deser_resp);
-                }
-            }
-        }
-    }
-
-    if let Some(scans) = state.get("scans") {
-        if let Some(arr_scans) = scans.as_array() {
-            for scan in arr_scans {
-                let deser_scan: FeroxScan =
-                    serde_json::from_value(scan.clone()).unwrap_or_default();
-                // need to determine if it's complete and based on that create a progress bar
-                // populate it accordingly based on completion
-                SCANNED_URLS.insert(Arc::new(Mutex::new(deser_scan)));
-            }
-        }
-    }
-
-    log::trace!("exit: resume_scan -> {:?}", config);
-    config
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use predicates::prelude::*;
-
-    #[test]
-    /// test that ScanType's default is File
-    fn default_scantype_is_file() {
-        match ScanType::default() {
-            ScanType::File => {}
-            ScanType::Directory => panic!(),
-        }
-    }
-
-    #[test]
-    /// test that get_single_spinner returns the correct spinner
-    // todo remove this when issue #107 is resolved
-    fn scanner_get_single_spinner_returns_spinner() {
-        let spinner = get_single_spinner();
-        assert!(!spinner.is_finished());
-    }
-
-    #[tokio::test(core_threads = 1)]
-    /// tests that pause_scan pauses execution and releases execution when PAUSE_SCAN is toggled
-    /// the spinner used during the test has had .finish_and_clear called on it, meaning that
-    /// a new one will be created, taking the if branch within the function
-    async fn scanner_pause_scan_with_finished_spinner() {
-        let now = time::Instant::now();
-        let urls = FeroxScans::default();
-
-        PAUSE_SCAN.store(true, Ordering::Relaxed);
-
-        let expected = time::Duration::from_secs(2);
-
-        tokio::spawn(async move {
-            time::delay_for(expected).await;
-            PAUSE_SCAN.store(false, Ordering::Relaxed);
-        });
-
-        urls.pause(false).await;
-
-        assert!(now.elapsed() > expected);
-    }
-
-    #[test]
-    /// add an unknown url to the hashset, expect true
-    fn add_url_to_list_of_scanned_urls_with_unknown_url() {
-        let urls = FeroxScans::default();
-        let url = "http://unknown_url";
-        let (result, _scan) = urls.add_scan(url, ScanType::Directory);
-        assert_eq!(result, true);
-    }
-
-    #[test]
-    /// add a known url to the hashset, with a trailing slash, expect false
-    fn add_url_to_list_of_scanned_urls_with_known_url() {
-        let urls = FeroxScans::default();
-        let pb = ProgressBar::new(1);
-        let url = "http://unknown_url/";
-        let scan = FeroxScan::new(url, ScanType::Directory, Some(pb));
-
-        assert_eq!(urls.insert(scan), true);
-
-        let (result, _scan) = urls.add_scan(url, ScanType::Directory);
-
-        assert_eq!(result, false);
-    }
-
-    #[test]
-    /// abort should call stop_progress_bar, marking it as finished
-    fn abort_stops_progress_bar() {
-        let pb = ProgressBar::new(1);
-        let url = "http://unknown_url/";
-        let scan = FeroxScan::new(url, ScanType::Directory, Some(pb));
-
-        assert_eq!(
-            scan.lock()
-                .unwrap()
-                .progress_bar
-                .as_ref()
-                .unwrap()
-                .is_finished(),
-            false
-        );
-
-        scan.lock().unwrap().abort();
-
-        assert_eq!(
-            scan.lock()
-                .unwrap()
-                .progress_bar
-                .as_ref()
-                .unwrap()
-                .is_finished(),
-            true
-        );
-    }
-
-    #[test]
-    /// add a known url to the hashset, without a trailing slash, expect false
-    fn add_url_to_list_of_scanned_urls_with_known_url_without_slash() {
-        let urls = FeroxScans::default();
-        let url = "http://unknown_url";
-        let scan = FeroxScan::new(url, ScanType::File, None);
-
-        assert_eq!(urls.insert(scan), true);
-
-        let (result, _scan) = urls.add_scan(url, ScanType::File);
-
-        assert_eq!(result, false);
-    }
-
-    #[test]
-    /// just increasing coverage, no real expectations
-    fn call_display_scans() {
-        let urls = FeroxScans::default();
-        let pb = ProgressBar::new(1);
-        let pb_two = ProgressBar::new(2);
-        let url = "http://unknown_url/";
-        let url_two = "http://unknown_url/fa";
-        let scan = FeroxScan::new(url, ScanType::Directory, Some(pb));
-        let scan_two = FeroxScan::new(url_two, ScanType::Directory, Some(pb_two));
-
-        scan_two.lock().unwrap().finish(); // one complete, one incomplete
-
-        assert_eq!(urls.insert(scan), true);
-
-        urls.display_scans();
-    }
-
-    #[test]
-    /// ensure that PartialEq compares FeroxScan.id fields
-    fn partial_eq_compares_the_id_field() {
-        let url = "http://unknown_url/";
-        let scan = FeroxScan::new(url, ScanType::Directory, None);
-        let scan_two = FeroxScan::new(url, ScanType::Directory, None);
-
-        assert!(!scan.lock().unwrap().eq(&scan_two.lock().unwrap()));
-
-        scan_two.lock().unwrap().id = scan.lock().unwrap().id.clone();
-
-        assert!(scan.lock().unwrap().eq(&scan_two.lock().unwrap()));
-    }
-
-    #[test]
-    /// show that a new progress bar is created if one doesn't exist
-    fn ferox_scan_get_progress_bar_when_none_is_set() {
-        let mut scan = FeroxScan::default();
-
-        assert!(scan.progress_bar.is_none()); // no pb exists
-
-        let pb = scan.progress_bar();
-
-        assert!(scan.progress_bar.is_some()); // new pb created
-        assert!(!pb.is_finished()) // not finished
-    }
-
-    #[test]
-    /// given a JSON entry representing a FeroxScan, test that it deserializes into the proper type
-    /// with the right attributes
-    fn ferox_scan_deserialize() {
-        let fs_json = r#"{"id":"057016a14769414aac9a7a62707598cb","url":"https://spiritanimal.com","scan_type":"Directory","complete":true}"#;
-        let fs_json_two = r#"{"id":"057016a14769414aac9a7a62707598cb","url":"https://spiritanimal.com","scan_type":"Not Correct","complete":true}"#;
-
-        let fs: FeroxScan = serde_json::from_str(fs_json).unwrap();
-        let fs_two: FeroxScan = serde_json::from_str(fs_json_two).unwrap();
-        assert_eq!(fs.url, "https://spiritanimal.com");
-
-        match fs.scan_type {
-            ScanType::Directory => {}
-            ScanType::File => {
-                panic!();
-            }
-        }
-        match fs_two.scan_type {
-            ScanType::Directory => {
-                panic!();
-            }
-            ScanType::File => {}
-        }
-
-        match fs.progress_bar {
-            None => {}
-            Some(_) => {
-                panic!();
-            }
-        }
-        assert_eq!(fs.complete, true);
-        assert_eq!(fs.id, "057016a14769414aac9a7a62707598cb");
-    }
-
-    #[test]
-    /// given a FeroxScan, test that it serializes into the proper JSON entry
-    fn ferox_scan_serialize() {
-        let fs = FeroxScan::new("https://spiritanimal.com", ScanType::Directory, None);
-        let fs_json = format!(
-            r#"{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","complete":false}}"#,
-            fs.lock().unwrap().id
-        );
-        assert_eq!(
-            fs_json,
-            serde_json::to_string(&*fs.lock().unwrap()).unwrap()
-        );
-    }
-
-    #[test]
-    /// given a FeroxScans, test that it serializes into the proper JSON entry
-    fn ferox_scans_serialize() {
-        let ferox_scan = FeroxScan::new("https://spiritanimal.com", ScanType::Directory, None);
-        let ferox_scans = FeroxScans::default();
-        let ferox_scans_json = format!(
-            r#"[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","complete":false}}]"#,
-            ferox_scan.lock().unwrap().id
-        );
-        ferox_scans.scans.lock().unwrap().push(ferox_scan);
-        assert_eq!(
-            ferox_scans_json,
-            serde_json::to_string(&ferox_scans).unwrap()
-        );
-    }
-
-    #[test]
-    /// given a FeroxResponses, test that it serializes into the proper JSON entry
-    fn ferox_responses_serialize() {
-        let json_response = r#"{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{"server":"nginx/1.16.1"}}"#;
-        let response: FeroxResponse = serde_json::from_str(json_response).unwrap();
-
-        let responses = FeroxResponses::default();
-        responses.insert(response);
-        // responses has a response now
-
-        // serialized should be a list of responses
-        let expected = format!("[{}]", json_response);
-
-        let serialized = serde_json::to_string(&responses).unwrap();
-        assert_eq!(expected, serialized);
-    }
-
-    #[test]
-    /// given a FeroxResponse, test that it serializes into the proper JSON entry
-    fn ferox_response_serialize_and_deserialize() {
-        // deserialize
-        let json_response = r#"{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{"server":"nginx/1.16.1"}}"#;
-        let response: FeroxResponse = serde_json::from_str(json_response).unwrap();
-
-        assert_eq!(response.url.as_str(), "https://nerdcore.com/css");
-        assert_eq!(response.url.path(), "/css");
-        assert_eq!(response.wildcard, true);
-        assert_eq!(response.status.as_u16(), 301);
-        assert_eq!(response.content_length, 173);
-        assert_eq!(response.line_count, 10);
-        assert_eq!(response.word_count, 16);
-        assert_eq!(response.headers.get("server").unwrap(), "nginx/1.16.1");
-
-        // serialize, however, this can fail when headers are out of order
-        let new_json = serde_json::to_string(&response).unwrap();
-        assert_eq!(json_response, new_json);
-    }
-
-    #[test]
-    /// test FeroxSerialize implementation of FeroxState
-    fn feroxstates_feroxserialize_implementation() {
-        let ferox_scan = FeroxScan::new("https://spiritanimal.com", ScanType::Directory, None);
-        let saved_id = ferox_scan.lock().unwrap().id.clone();
-        SCANNED_URLS.insert(ferox_scan);
-
-        let json_response = r#"{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{"server":"nginx/1.16.1"}}"#;
-        let response: FeroxResponse = serde_json::from_str(json_response).unwrap();
-        RESPONSES.insert(response);
-
-        let ferox_state = FeroxState {
-            scans: &SCANNED_URLS,
-            responses: &RESPONSES,
-            config: &CONFIGURATION,
-        };
-
-        let expected_strs = predicates::str::contains("scans: FeroxScans").and(
-            predicate::str::contains("config: Configuration")
-                .and(predicate::str::contains("responses: FeroxResponses"))
-                .and(predicate::str::contains("nerdcore.com"))
-                .and(predicate::str::contains("/css"))
-                .and(predicate::str::contains("https://spiritanimal.com")),
-        );
-
-        assert!(expected_strs.eval(&ferox_state.as_str()));
-
-        let json_state = ferox_state.as_json();
-        let expected = format!(
-            r#"{{"scans":[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","complete":false}}],"config":{{"type":"configuration","wordlist":"/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt","config":"","proxy":"","replay_proxy":"","target_url":"","status_codes":[200,204,301,302,307,308,401,403,405],"replay_codes":[200,204,301,302,307,308,401,403,405],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"quiet":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/1.9.0","redirects":false,"insecure":false,"extensions":[],"headers":{{}},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":false,"depth":4,"scan_limit":0,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false,"resumed":false,"save_state":true}},"responses":[{{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{{"server":"nginx/1.16.1"}}}}]}}"#,
-            saved_id
-        );
-
-        assert!(predicates::str::similar(expected).eval(&json_state));
-    }
-}

src/scan_manager/menu.rs

@@ -0,0 +1,192 @@
+use crate::progress::PROGRESS_BAR;
+use console::{measure_text_width, pad_str, style, Alignment, Term};
+use indicatif::ProgressDrawTarget;
+
+/// Interactive scan cancellation menu
+#[derive(Debug)]
+pub(super) struct Menu {
+    /// character to use as visual separator of lines
+    separator: String,
+
+    /// name of menu
+    name: String,
+
+    /// header: name surrounded by separators
+    header: String,
+
+    /// instructions
+    instructions: String,
+
+    /// footer: instructions surrounded by separators
+    footer: String,
+
+    /// target for output
+    term: Term,
+}
+
+/// Implementation of Menu
+impl Menu {
+    /// Creates new Menu
+    pub(super) fn new() -> Self {
+        let separator = "─".to_string();
+
+        let instructions = format!(
+            "Enter a {} list of indexes/ranges to {} ({}: 1-4,8,9-13)",
+            style("comma-separated").yellow(),
+            style("cancel").red(),
+            style("ex").cyan(),
+        );
+
+        let name = format!(
+            "{} {} {}",
+            "💀",
+            style("Scan Cancel Menu").bright().yellow(),
+            "💀"
+        );
+
+        let force_msg = format!(
+            "Add {} to {} confirmation ({}: 3-5 -f)",
+            style("-f").yellow(),
+            style("skip").yellow(),
+            style("ex").cyan(),
+        );
+
+        let longest = measure_text_width(&instructions).max(measure_text_width(&name));
+
+        let border = separator.repeat(longest);
+
+        let padded_name = pad_str(&name, longest, Alignment::Center, None);
+        let padded_force = pad_str(&force_msg, longest, Alignment::Center, None);
+
+        let header = format!("{}\n{}\n{}", border, padded_name, border);
+        let footer = format!("{}\n{}\n{}\n{}", border, instructions, padded_force, border);
+
+        Self {
+            separator,
+            name,
+            header,
+            instructions,
+            footer,
+            term: Term::stderr(),
+        }
+    }
+
+    /// print menu header
+    pub(super) fn print_header(&self) {
+        self.println(&self.header);
+    }
+
+    /// print menu footer
+    pub(super) fn print_footer(&self) {
+        self.println(&self.footer);
+    }
+
+    /// set PROGRESS_BAR bar target to hidden
+    pub(super) fn hide_progress_bars(&self) {
+        PROGRESS_BAR.set_draw_target(ProgressDrawTarget::hidden());
+    }
+
+    /// set PROGRESS_BAR bar target to hidden
+    pub(super) fn show_progress_bars(&self) {
+        PROGRESS_BAR.set_draw_target(ProgressDrawTarget::stdout());
+    }
+
+    /// Wrapper around console's Term::clear_screen and flush
+    pub(super) fn clear_screen(&self) {
+        self.term.clear_screen().unwrap_or_default();
+        self.term.flush().unwrap_or_default();
+    }
+
+    /// Wrapper around console's Term::write_line
+    pub(super) fn println(&self, msg: &str) {
+        self.term.write_line(msg).unwrap_or_default();
+    }
+
+    /// Helper for parsing a usize from a str
+    fn str_to_usize(&self, value: &str) -> usize {
+        if value.is_empty() {
+            return 0;
+        }
+
+        value
+            .trim()
+            .to_string()
+            .parse::<usize>()
+            .unwrap_or_else(|e| {
+                self.println(&format!("Found non-numeric input: {}: {:?}", e, value));
+                0
+            })
+    }
+
+    /// split a comma delimited string into vec of usizes
+    pub(super) fn split_to_nums(&self, line: &str) -> Vec<usize> {
+        let mut nums = Vec::new();
+        let values = line.split(',');
+
+        for mut value in values {
+            value = value.trim();
+
+            if value.contains('-') {
+                // range of two values, needs further processing
+
+                let range: Vec<usize> = value
+                    .split('-')
+                    .map(|s| self.str_to_usize(s))
+                    .filter(|m| *m != 0)
+                    .collect();
+
+                if range.len() != 2 {
+                    // expecting [1, 4] or similar, if a 0 was used, we'd be left with a vec of size 1
+                    self.println(&format!("Found invalid range of scans: {}", value));
+                    continue;
+                }
+
+                (range[0]..=range[1]).for_each(|n| {
+                    // iterate from lower to upper bound and add all interim values, skipping
+                    // any already known
+                    if !nums.contains(&n) {
+                        nums.push(n)
+                    }
+                });
+            } else {
+                let value = self.str_to_usize(value);
+
+                if value != 0 && !nums.contains(&value) {
+                    // the zeroth scan is always skipped, skip already known values
+                    nums.push(value);
+                }
+            }
+        }
+
+        nums
+    }
+
+    /// get comma-separated list of scan indexes from the user
+    pub(super) fn get_scans_from_user(&self) -> Option<(Vec<usize>, bool)> {
+        if let Ok(line) = self.term.read_line() {
+            let force = line.contains("-f");
+            let line = line.replace("-f", "");
+            Some((self.split_to_nums(&line), force))
+        } else {
+            None
+        }
+    }
+
+    /// Given a url, confirm with user that we should cancel
+    pub(super) fn confirm_cancellation(&self, url: &str) -> char {
+        self.println(&format!(
+            "You sure you wanna cancel this scan: {}? [Y/n]",
+            url
+        ));
+
+        self.term.read_char().unwrap_or('n')
+    }
+}
+
+/// Default implementation for Menu
+impl Default for Menu {
+    /// return Menu::new as default
+    fn default() -> Menu {
+        Menu::new()
+    }
+}

src/scan_manager/mod.rs

@@ -0,0 +1,17 @@
+mod scan_container;
+mod response_container;
+mod scan;
+mod menu;
+mod utils;
+mod order;
+mod state;
+#[cfg(test)]
+mod tests;
+
+pub(self) use menu::Menu;
+pub use order::ScanOrder;
+pub use response_container::FeroxResponses;
+pub use scan::{FeroxScan, ScanStatus, ScanType};
+pub use scan_container::{FeroxScans, PAUSE_SCAN};
+pub use state::FeroxState;
+pub use utils::{resume_scan, start_max_time_thread};

src/scan_manager/order.rs

@@ -0,0 +1,10 @@
+#[derive(Debug, Copy, Clone)]
+/// Simple enum to designate whether a URL was passed in by the user (Initial) or found during
+/// scanning (Latest)
+pub enum ScanOrder {
+    /// Url was passed in by the user
+    Initial,
+
+    /// Url was found during scanning
+    Latest,
+}

src/scan_manager/response_container.rs

@@ -0,0 +1,55 @@
+use crate::response::FeroxResponse;
+use serde::{ser::SerializeSeq, Serialize, Serializer};
+use std::sync::{Arc, RwLock};
+
+/// Container around a locked vector of `FeroxResponse`s, adds wrappers for insertion and search
+#[derive(Debug, Default)]
+pub struct FeroxResponses {
+    /// Internal structure: locked hashset of `FeroxScan`s
+    pub responses: Arc<RwLock<Vec<FeroxResponse>>>,
+}
+
+/// Serialize implementation for FeroxResponses
+impl Serialize for FeroxResponses {
+    /// Function that handles serialization of FeroxResponses
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        if let Ok(responses) = self.responses.read() {
+            let mut seq = serializer.serialize_seq(Some(responses.len()))?;
+
+            for response in responses.iter() {
+                seq.serialize_element(response)?;
+            }
+
+            seq.end()
+        } else {
+            // if for some reason we can't unlock the mutex, just write an empty list
+            let seq = serializer.serialize_seq(Some(0))?;
+            seq.end()
+        }
+    }
+}
+
+/// Implementation of `FeroxResponses`
+impl FeroxResponses {
+    /// Add a `FeroxResponse` to the internal container
+    pub fn insert(&self, response: FeroxResponse) {
+        if let Ok(mut responses) = self.responses.write() {
+            responses.push(response);
+        }
+    }
+
+    /// Simple check for whether or not a FeroxResponse is contained within the inner container
+    pub fn contains(&self, other: &FeroxResponse) -> bool {
+        if let Ok(responses) = self.responses.read() {
+            for response in responses.iter() {
+                if response.url() == other.url() {
+                    return true;
+                }
+            }
+        }
+        false
+    }
+}

src/scan_manager/scan.rs

@@ -0,0 +1,508 @@
+use super::*;
+use crate::{
+    config::OutputLevel,
+    progress::{add_bar, BarType},
+    scanner::PolicyTrigger,
+};
+use anyhow::Result;
+use console::style;
+use indicatif::ProgressBar;
+use serde::{ser::SerializeStruct, Deserialize, Deserializer, Serialize, Serializer};
+use serde_json::Value;
+use std::{
+    collections::HashMap,
+    fmt,
+    sync::{Arc, Mutex},
+    time::Instant,
+};
+
+use std::sync::atomic::{AtomicUsize, Ordering};
+use tokio::{sync, task::JoinHandle};
+use uuid::Uuid;
+
+/// Struct to hold scan-related state
+///
+/// The purpose of this container is to open up the pathway to aborting currently running tasks and
+/// serialization of all scan state into a state file in order to resume scans that were cut short
+#[derive(Debug)]
+pub struct FeroxScan {
+    /// UUID that uniquely ID's the scan
+    pub(super) id: String,
+
+    /// The URL that to be scanned
+    pub(super) url: String,
+
+    /// The type of scan
+    pub(super) scan_type: ScanType,
+
+    /// The order in which the scan was received
+    pub(crate) scan_order: ScanOrder,
+
+    /// Number of requests to populate the progress bar with
+    pub(super) num_requests: u64,
+
+    /// Status of this scan
+    pub(super) status: Mutex<ScanStatus>,
+
+    /// The spawned tokio task performing this scan (uses tokio::sync::Mutex)
+    pub(super) task: sync::Mutex<Option<JoinHandle<()>>>,
+
+    /// The progress bar associated with this scan
+    pub(super) progress_bar: Mutex<Option<ProgressBar>>,
+
+    /// whether or not the user passed --silent|--quiet on the command line
+    pub(super) output_level: OutputLevel,
+
+    /// tracker for overall number of 403s seen by the FeroxScan instance
+    pub(super) status_403s: AtomicUsize,
+
+    /// tracker for overall number of 429s seen by the FeroxScan instance
+    pub(super) status_429s: AtomicUsize,
+
+    /// tracker for total number of errors encountered by the FeroxScan instance
+    pub(super) errors: AtomicUsize,
+
+    /// tracker for the time at which this scan was started
+    pub(super) start_time: Instant,
+}
+
+/// Default implementation for FeroxScan
+impl Default for FeroxScan {
+    /// Create a default FeroxScan, populates ID with a new UUID
+    fn default() -> Self {
+        let new_id = Uuid::new_v4().to_simple().to_string();
+
+        FeroxScan {
+            id: new_id,
+            task: sync::Mutex::new(None), // tokio mutex
+            status: Mutex::new(ScanStatus::default()),
+            num_requests: 0,
+            scan_order: ScanOrder::Latest,
+            url: String::new(),
+            progress_bar: Mutex::new(None),
+            scan_type: ScanType::File,
+            output_level: Default::default(),
+            errors: Default::default(),
+            status_429s: Default::default(),
+            status_403s: Default::default(),
+            start_time: Instant::now(),
+        }
+    }
+}
+
+/// Implementation of FeroxScan
+impl FeroxScan {
+    /// Stop a currently running scan
+    pub async fn abort(&self) -> Result<()> {
+        log::trace!("enter: abort");
+
+        match self.task.try_lock() {
+            Ok(mut guard) => {
+                if let Some(task) = std::mem::replace(&mut *guard, None) {
+                    log::trace!("aborting {:?}", self);
+                    task.abort();
+                    self.set_status(ScanStatus::Cancelled)?;
+                    self.stop_progress_bar();
+                }
+            }
+            Err(e) => {
+                log::warn!("Could not acquire lock to abort scan (we're already waiting for its results): {:?} {}", self, e);
+            }
+        }
+        log::trace!("exit: abort");
+        Ok(())
+    }
+
+    /// getter for url
+    pub fn url(&self) -> &str {
+        &self.url
+    }
+
+    /// small wrapper to set the JoinHandle
+    pub async fn set_task(&self, task: JoinHandle<()>) -> Result<()> {
+        let mut guard = self.task.lock().await;
+        let _ = std::mem::replace(&mut *guard, Some(task));
+        Ok(())
+    }
+
+    /// small wrapper to set ScanStatus
+    pub fn set_status(&self, status: ScanStatus) -> Result<()> {
+        if let Ok(mut guard) = self.status.lock() {
+            let _ = std::mem::replace(&mut *guard, status);
+        }
+        Ok(())
+    }
+
+    /// Simple helper to call .finish on the scan's progress bar
+    pub(super) fn stop_progress_bar(&self) {
+        if let Ok(guard) = self.progress_bar.lock() {
+            if guard.is_some() {
+                (*guard).as_ref().unwrap().finish_at_current_pos()
+            }
+        }
+    }
+
+    /// Simple helper get a progress bar
+    pub fn progress_bar(&self) -> ProgressBar {
+        match self.progress_bar.lock() {
+            Ok(mut guard) => {
+                if guard.is_some() {
+                    (*guard).as_ref().unwrap().clone()
+                } else {
+                    let bar_type = match self.output_level {
+                        OutputLevel::Default => BarType::Default,
+                        OutputLevel::Quiet => BarType::Quiet,
+                        OutputLevel::Silent => BarType::Hidden,
+                    };
+
+                    let pb = add_bar(&self.url, self.num_requests, bar_type);
+                    pb.reset_elapsed();
+
+                    let _ = std::mem::replace(&mut *guard, Some(pb.clone()));
+
+                    pb
+                }
+            }
+            Err(_) => {
+                log::warn!("Could not unlock progress bar on {:?}", self);
+
+                let bar_type = match self.output_level {
+                    OutputLevel::Default => BarType::Default,
+                    OutputLevel::Quiet => BarType::Quiet,
+                    OutputLevel::Silent => BarType::Hidden,
+                };
+
+                let pb = add_bar(&self.url, self.num_requests, bar_type);
+                pb.reset_elapsed();
+
+                pb
+            }
+        }
+    }
+
+    /// Given a URL and ProgressBar, create a new FeroxScan, wrap it in an Arc and return it
+    pub fn new(
+        url: &str,
+        scan_type: ScanType,
+        scan_order: ScanOrder,
+        num_requests: u64,
+        output_level: OutputLevel,
+        pb: Option<ProgressBar>,
+    ) -> Arc<Self> {
+        Arc::new(Self {
+            url: url.to_string(),
+            scan_type,
+            scan_order,
+            num_requests,
+            output_level,
+            progress_bar: Mutex::new(pb),
+            ..Default::default()
+        })
+    }
+
+    /// Mark the scan as complete and stop the scan's progress bar
+    pub fn finish(&self) -> Result<()> {
+        self.set_status(ScanStatus::Complete)?;
+        self.stop_progress_bar();
+        Ok(())
+    }
+
+    /// small wrapper to inspect ScanType and ScanStatus to see if a Directory scan is running or
+    /// in the queue to be run
+    pub fn is_active(&self) -> bool {
+        if let Ok(guard) = self.status.lock() {
+            return matches!(
+                (self.scan_type, *guard),
+                (ScanType::Directory, ScanStatus::Running)
+                    | (ScanType::Directory, ScanStatus::NotStarted)
+            );
+        }
+        false
+    }
+
+    /// small wrapper to inspect ScanStatus and see if it's Complete
+    pub fn is_complete(&self) -> bool {
+        if let Ok(guard) = self.status.lock() {
+            return matches!(*guard, ScanStatus::Complete);
+        }
+        false
+    }
+
+    /// await a task's completion, similar to a thread's join; perform necessary bookkeeping
+    pub async fn join(&self) {
+        log::trace!("enter join({:?})", self);
+        let mut guard = self.task.lock().await;
+
+        if guard.is_some() {
+            if let Some(task) = std::mem::replace(&mut *guard, None) {
+                task.await.unwrap();
+                self.set_status(ScanStatus::Complete)
+                    .unwrap_or_else(|e| log::warn!("Could not mark scan complete: {}", e))
+            }
+        }
+
+        log::trace!("exit join({:?})", self);
+    }
+    /// increment the value in question by 1
+    pub(crate) fn add_403(&self) {
+        self.status_403s.fetch_add(1, Ordering::Relaxed);
+    }
+
+    /// increment the value in question by 1
+    pub(crate) fn add_429(&self) {
+        self.status_429s.fetch_add(1, Ordering::Relaxed);
+    }
+
+    /// increment the value in question by 1
+    pub(crate) fn add_error(&self) {
+        self.errors.fetch_add(1, Ordering::Relaxed);
+    }
+
+    /// simple wrapper to call the appropriate getter based on the given PolicyTrigger
+    pub fn num_errors(&self, trigger: PolicyTrigger) -> usize {
+        match trigger {
+            PolicyTrigger::Status403 => self.status_403s(),
+            PolicyTrigger::Status429 => self.status_429s(),
+            PolicyTrigger::Errors => self.errors(),
+        }
+    }
+
+    /// return the number of errors seen by this scan
+    fn errors(&self) -> usize {
+        self.errors.load(Ordering::Relaxed)
+    }
+
+    /// return the number of 403s seen by this scan
+    fn status_403s(&self) -> usize {
+        self.status_403s.load(Ordering::Relaxed)
+    }
+
+    /// return the number of 429s seen by this scan
+    fn status_429s(&self) -> usize {
+        self.status_429s.load(Ordering::Relaxed)
+    }
+
+    /// return the number of requests per second performed by this scan's scanner
+    pub fn requests_per_second(&self) -> u64 {
+        if !self.is_active() {
+            return 0;
+        }
+
+        let reqs = self.requests();
+        let seconds = self.start_time.elapsed().as_secs();
+
+        reqs.checked_div(seconds).unwrap_or(0)
+    }
+
+    /// return the number of requests performed by this scan's scanner
+    pub fn requests(&self) -> u64 {
+        self.progress_bar().position()
+    }
+}
+
+/// Display implementation
+impl fmt::Display for FeroxScan {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let status = if let Ok(guard) = self.status.lock() {
+            match *guard {
+                ScanStatus::NotStarted => style("not started").bright().blue(),
+                ScanStatus::Complete => style("complete").green(),
+                ScanStatus::Cancelled => style("cancelled").red(),
+                ScanStatus::Running => style("running").bright().yellow(),
+            }
+        } else {
+            style("unknown").red()
+        };
+
+        write!(f, "{:12} {}", status, self.url)
+    }
+}
+
+/// PartialEq implementation; uses FeroxScan.id for comparison
+impl PartialEq for FeroxScan {
+    fn eq(&self, other: &Self) -> bool {
+        self.id == other.id
+    }
+}
+
+/// Serialize implementation for FeroxScan
+impl Serialize for FeroxScan {
+    /// Function that handles serialization of a FeroxScan
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let mut state = serializer.serialize_struct("FeroxScan", 4)?;
+
+        state.serialize_field("id", &self.id)?;
+        state.serialize_field("url", &self.url)?;
+        state.serialize_field("scan_type", &self.scan_type)?;
+        state.serialize_field("status", &self.status)?;
+        state.serialize_field("num_requests", &self.num_requests)?;
+
+        state.end()
+    }
+}
+
+/// Deserialize implementation for FeroxScan
+impl<'de> Deserialize<'de> for FeroxScan {
+    /// Deserialize a FeroxScan from a serde_json::Value
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let mut scan = Self::default();
+
+        let map: HashMap<String, Value> = HashMap::deserialize(deserializer)?;
+
+        for (key, value) in &map {
+            match key.as_str() {
+                "id" => {
+                    if let Some(id) = value.as_str() {
+                        scan.id = id.to_string();
+                    }
+                }
+                "scan_type" => {
+                    if let Some(scan_type) = value.as_str() {
+                        scan.scan_type = match scan_type {
+                            "File" => ScanType::File,
+                            "Directory" => ScanType::Directory,
+                            _ => ScanType::File,
+                        }
+                    }
+                }
+                "status" => {
+                    if let Some(status) = value.as_str() {
+                        scan.status = Mutex::new(match status {
+                            "NotStarted" => ScanStatus::NotStarted,
+                            "Running" => ScanStatus::Running,
+                            "Complete" => ScanStatus::Complete,
+                            "Cancelled" => ScanStatus::Cancelled,
+                            _ => ScanStatus::default(),
+                        })
+                    }
+                }
+                "url" => {
+                    if let Some(url) = value.as_str() {
+                        scan.url = url.to_string();
+                    }
+                }
+                "num_requests" => {
+                    if let Some(num_requests) = value.as_u64() {
+                        scan.num_requests = num_requests;
+                    }
+                }
+                _ => {}
+            }
+        }
+
+        Ok(scan)
+    }
+}
+
+/// Simple enum used to flag a `FeroxScan` as likely a directory or file
+#[derive(Debug, Serialize, Deserialize, Copy, Clone)]
+pub enum ScanType {
+    /// Just a file being requested
+    File,
+
+    /// A an entire directory that might be scanned
+    Directory,
+}
+
+/// Default implementation for ScanType
+impl Default for ScanType {
+    /// Return ScanType::File as default
+    fn default() -> Self {
+        Self::File
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, Copy, Clone)]
+/// Simple enum to represent a scan's current status ([in]complete, cancelled)
+pub enum ScanStatus {
+    /// Scan hasn't started yet
+    NotStarted,
+
+    /// Scan finished normally
+    Complete,
+
+    /// Scan was cancelled by the user
+    Cancelled,
+
+    /// Scan has started, but hasn't finished, nor been cancelled
+    Running,
+}
+
+/// Default implementation for ScanStatus
+impl Default for ScanStatus {
+    /// Default variant for ScanStatus is NotStarted
+    fn default() -> Self {
+        Self::NotStarted
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::thread::sleep;
+    use tokio::time::Duration;
+
+    #[test]
+    /// ensure that num_errors returns the correct values for the given PolicyTrigger
+    ///
+    /// covers tests for add_[403,429,error] and the related getters in addition to num_errors
+    fn num_errors_returns_correct_values() {
+        let scan = FeroxScan::new(
+            "http://localhost",
+            ScanType::Directory,
+            ScanOrder::Latest,
+            1000,
+            OutputLevel::Default,
+            None,
+        );
+
+        scan.add_error();
+        scan.add_403();
+        scan.add_403();
+        scan.add_429();
+        scan.add_429();
+        scan.add_429();
+
+        assert_eq!(scan.num_errors(PolicyTrigger::Errors), 1);
+        assert_eq!(scan.num_errors(PolicyTrigger::Status403), 2);
+        assert_eq!(scan.num_errors(PolicyTrigger::Status429), 3);
+    }
+
+    #[test]
+    /// ensure that requests_per_second returns the correct values
+    fn requests_per_second_returns_correct_values() {
+        let scan = FeroxScan {
+            id: "".to_string(),
+            url: "".to_string(),
+            scan_type: ScanType::Directory,
+            scan_order: ScanOrder::Initial,
+            num_requests: 0,
+            status: Mutex::new(ScanStatus::Running),
+            task: Default::default(),
+            progress_bar: Mutex::new(None),
+            output_level: Default::default(),
+            status_403s: Default::default(),
+            status_429s: Default::default(),
+            errors: Default::default(),
+            start_time: Instant::now(),
+        };
+
+        let pb = scan.progress_bar();
+        pb.set_position(100);
+
+        sleep(Duration::new(1, 0));
+
+        let req_sec = scan.requests_per_second();
+
+        assert_eq!(req_sec, 100);
+
+        scan.finish().unwrap();
+        assert_eq!(scan.requests_per_second(), 0);
+    }
+}

src/scan_manager/scan_container.rs

@@ -0,0 +1,503 @@
+use super::scan::ScanType;
+use super::*;
+use crate::{
+    config::OutputLevel,
+    progress::PROGRESS_PRINTER,
+    progress::{add_bar, BarType},
+    scanner::RESPONSES,
+    traits::FeroxSerialize,
+    SLEEP_DURATION,
+};
+use anyhow::Result;
+use reqwest::StatusCode;
+use serde::{ser::SerializeSeq, Serialize, Serializer};
+use std::{
+    convert::TryInto,
+    fs::File,
+    io::BufReader,
+    ops::Index,
+    sync::{
+        atomic::{AtomicBool, AtomicUsize, Ordering},
+        Arc, Mutex, RwLock,
+    },
+    thread::sleep,
+};
+use tokio::time::{self, Duration};
+
+/// Single atomic number that gets incremented once, used to track first thread to interact with
+/// when pausing a scan
+static INTERACTIVE_BARRIER: AtomicUsize = AtomicUsize::new(0);
+
+/// Atomic boolean flag, used to determine whether or not a scan should pause or resume
+pub static PAUSE_SCAN: AtomicBool = AtomicBool::new(false);
+
+/// Container around a locked hashset of `FeroxScan`s, adds wrappers for insertion and searching
+#[derive(Debug, Default)]
+pub struct FeroxScans {
+    /// Internal structure: locked hashset of `FeroxScan`s
+    pub scans: RwLock<Vec<Arc<FeroxScan>>>,
+
+    /// menu used for providing a way for users to cancel a scan
+    menu: Menu,
+
+    /// number of requests expected per scan (mirrors the same on Stats); used for initializing
+    /// progress bars and feroxscans
+    bar_length: Mutex<u64>,
+
+    /// whether or not the user passed --silent|--quiet on the command line
+    output_level: OutputLevel,
+}
+
+/// Serialize implementation for FeroxScans
+///
+/// purposefully skips menu attribute
+impl Serialize for FeroxScans {
+    /// Function that handles serialization of FeroxScans
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        if let Ok(scans) = self.scans.read() {
+            let mut seq = serializer.serialize_seq(Some(scans.len()))?;
+            for scan in scans.iter() {
+                seq.serialize_element(&*scan).unwrap_or_default();
+            }
+
+            seq.end()
+        } else {
+            // if for some reason we can't unlock the RwLock, just write an empty list
+            let seq = serializer.serialize_seq(Some(0))?;
+            seq.end()
+        }
+    }
+}
+
+/// Implementation of `FeroxScans`
+impl FeroxScans {
+    /// given an OutputLevel, create a new FeroxScans object
+    pub fn new(output_level: OutputLevel) -> Self {
+        Self {
+            output_level,
+            ..Default::default()
+        }
+    }
+
+    /// Add a `FeroxScan` to the internal container
+    ///
+    /// If the internal container did NOT contain the scan, true is returned; else false
+    pub fn insert(&self, scan: Arc<FeroxScan>) -> bool {
+        // If the container did contain the scan, set sentry to false
+        // If the container did not contain the scan, set sentry to true
+        let sentry = !self.contains(&scan.url);
+
+        if sentry {
+            // can't update the internal container while the scan itself is locked, so first
+            // lock the scan and check the container for the scan's presence, then add if
+            // not found
+            match self.scans.write() {
+                Ok(mut scans) => {
+                    scans.push(scan);
+                }
+                Err(e) => {
+                    log::warn!("FeroxScans' container's mutex is poisoned: {}", e);
+                    return false;
+                }
+            }
+        }
+
+        sentry
+    }
+
+    /// load serialized FeroxScan(s) into this FeroxScans  
+    pub fn add_serialized_scans(&self, filename: &str) -> Result<()> {
+        log::trace!("enter: add_serialized_scans({})", filename);
+        let file = File::open(filename)?;
+
+        let reader = BufReader::new(file);
+        let state: serde_json::Value = serde_json::from_reader(reader)?;
+
+        if let Some(scans) = state.get("scans") {
+            if let Some(arr_scans) = scans.as_array() {
+                for scan in arr_scans {
+                    let mut deser_scan: FeroxScan =
+                        serde_json::from_value(scan.clone()).unwrap_or_default();
+                    // FeroxScans gets -q value from config as usual; the FeroxScans themselves
+                    // rely on that value being passed in. If the user starts a scan without -q
+                    // and resumes the scan but adds -q, FeroxScan will not have the proper value
+                    // without the line below
+                    deser_scan.output_level = self.output_level;
+
+                    log::debug!("added: {}", deser_scan);
+                    self.insert(Arc::new(deser_scan));
+                }
+            }
+        }
+
+        log::trace!("exit: add_serialized_scans");
+        Ok(())
+    }
+
+    /// Simple check for whether or not a FeroxScan is contained within the inner container based
+    /// on the given URL
+    pub fn contains(&self, url: &str) -> bool {
+        if let Ok(scans) = self.scans.read() {
+            for scan in scans.iter() {
+                if scan.url == url {
+                    return true;
+                }
+            }
+        }
+        false
+    }
+
+    /// Find and return a `FeroxScan` based on the given URL
+    pub fn get_scan_by_url(&self, url: &str) -> Option<Arc<FeroxScan>> {
+        if let Ok(guard) = self.scans.read() {
+            for scan in guard.iter() {
+                if scan.url == url {
+                    return Some(scan.clone());
+                }
+            }
+        }
+        None
+    }
+
+    pub(super) fn get_base_scan_by_url(&self, url: &str) -> Option<Arc<FeroxScan>> {
+        log::trace!("enter: get_sub_paths_from_path({})", url);
+
+        // rmatch_indices returns tuples in index, match form, i.e. (10, "/")
+        // with the furthest-right match in the first position in the vector
+        let matches: Vec<_> = url.rmatch_indices('/').collect();
+
+        // iterate from the furthest right matching index and check the given url from the
+        // start to the furthest-right '/' character. compare that slice to the urls associated
+        // with directory scans and return the first match, since it should be the 'deepest'
+        // match.
+        // Example:
+        //   url: http://shmocalhost/src/release/examples/stuff.php
+        //   scans:
+        //      http://shmocalhost/src/statistics
+        //      http://shmocalhost/src/banner
+        //      http://shmocalhost/src/release
+        //      http://shmocalhost/src/release/examples
+        //
+        //  returns: http://shmocalhost/src/release/examples
+        if let Ok(guard) = self.scans.read() {
+            for (idx, _) in &matches {
+                for scan in guard.iter() {
+                    let slice = url.index(0..*idx);
+                    if slice == scan.url || format!("{}/", slice).as_str() == scan.url {
+                        log::trace!("enter: get_sub_paths_from_path -> {}", scan);
+                        return Some(scan.clone());
+                    }
+                }
+            }
+        }
+
+        log::trace!("enter: get_sub_paths_from_path -> None");
+        None
+    }
+    /// add one to either 403 or 429 tracker in the scan related to the given url
+    pub fn increment_status_code(&self, url: &str, code: StatusCode) {
+        if let Some(scan) = self.get_base_scan_by_url(url) {
+            match code {
+                StatusCode::TOO_MANY_REQUESTS => {
+                    scan.add_429();
+                }
+                StatusCode::FORBIDDEN => {
+                    scan.add_403();
+                }
+                _ => {}
+            }
+        }
+    }
+
+    /// add one to either 403 or 429 tracker in the scan related to the given url
+    pub fn increment_error(&self, url: &str) {
+        if let Some(scan) = self.get_base_scan_by_url(url) {
+            scan.add_error();
+        }
+    }
+
+    /// Print all FeroxScans of type Directory
+    ///
+    /// Example:
+    ///   0: complete   https://10.129.45.20
+    ///   9: complete   https://10.129.45.20/images
+    ///  10: complete   https://10.129.45.20/assets
+    pub async fn display_scans(&self) {
+        let scans = {
+            // written this way in order to grab the vector and drop the lock immediately
+            // otherwise the spawned task that this is a part of is no longer Send due to
+            // the scan.task.lock().await below while the lock is held (RwLock is not Send)
+            self.scans
+                .read()
+                .expect("Could not acquire lock in display_scans")
+                .clone()
+        };
+
+        for (i, scan) in scans.iter().enumerate() {
+            if matches!(scan.scan_order, ScanOrder::Initial) || scan.task.try_lock().is_err() {
+                // original target passed in via either -u or --stdin
+                continue;
+            }
+
+            if matches!(scan.scan_type, ScanType::Directory) {
+                // we're only interested in displaying directory scans, as those are
+                // the only ones that make sense to be stopped
+                let scan_msg = format!("{:3}: {}", i, scan);
+                self.menu.println(&scan_msg);
+            }
+        }
+    }
+
+    /// Given a list of indexes, cancel their associated FeroxScans
+    async fn cancel_scans(&self, indexes: Vec<usize>, force: bool) -> usize {
+        let menu_pause_duration = Duration::from_millis(SLEEP_DURATION);
+
+        let mut num_cancelled = 0_usize;
+
+        for num in indexes {
+            let selected = match self.scans.read() {
+                Ok(u_scans) => {
+                    // check if number provided is out of range
+                    if num >= u_scans.len() {
+                        // usize can't be negative, just need to handle exceeding bounds
+                        self.menu
+                            .println(&format!("The number {} is not a valid choice.", num));
+                        sleep(menu_pause_duration);
+                        continue;
+                    }
+                    u_scans.index(num).clone()
+                }
+                Err(..) => continue,
+            };
+
+            let input = if force {
+                'y'
+            } else {
+                self.menu.confirm_cancellation(&selected.url)
+            };
+
+            if input == 'y' || input == '\n' {
+                self.menu.println(&format!("Stopping {}...", selected.url));
+
+                selected
+                    .abort()
+                    .await
+                    .unwrap_or_else(|e| log::warn!("Could not cancel task: {}", e));
+
+                let pb = selected.progress_bar();
+                num_cancelled += pb.length() as usize - pb.position() as usize
+            } else {
+                self.menu.println("Ok, doing nothing...");
+            }
+
+            sleep(menu_pause_duration);
+        }
+
+        num_cancelled
+    }
+
+    /// CLI menu that allows for interactive cancellation of recursed-into directories
+    async fn interactive_menu(&self) -> usize {
+        self.menu.hide_progress_bars();
+        self.menu.clear_screen();
+        self.menu.print_header();
+        self.display_scans().await;
+        self.menu.print_footer();
+
+        let mut num_cancelled = 0_usize;
+
+        if let Some((input, force)) = self.menu.get_scans_from_user() {
+            num_cancelled += self.cancel_scans(input, force).await;
+        };
+
+        self.menu.clear_screen();
+        self.menu.show_progress_bars();
+
+        num_cancelled
+    }
+
+    /// prints all known responses that the scanner has already seen
+    pub fn print_known_responses(&self) {
+        if let Ok(mut responses) = RESPONSES.responses.write() {
+            for response in responses.iter_mut() {
+                if self.output_level != response.output_level {
+                    // set the output_level prior to printing the response to ensure that the
+                    // response's setting aligns with the overall configuration (since we're
+                    // calling this from a resumed state)
+                    response.output_level = self.output_level;
+                }
+                PROGRESS_PRINTER.println(response.as_str());
+            }
+        }
+    }
+
+    /// if a resumed scan is already complete, display a completed progress bar to the user
+    pub fn print_completed_bars(&self, bar_length: usize) -> Result<()> {
+        let bar_type = match self.output_level {
+            OutputLevel::Default => BarType::Message,
+            OutputLevel::Quiet => BarType::Quiet,
+            OutputLevel::Silent => return Ok(()), // fast exit when --silent was used
+        };
+
+        if let Ok(scans) = self.scans.read() {
+            for scan in scans.iter() {
+                if scan.is_complete() {
+                    // these scans are complete, and just need to be shown to the user
+                    let pb = add_bar(
+                        &scan.url,
+                        bar_length.try_into().unwrap_or_default(),
+                        bar_type,
+                    );
+                    pb.finish();
+                }
+            }
+        }
+        Ok(())
+    }
+
+    /// Forced the calling thread into a busy loop
+    ///
+    /// Every `SLEEP_DURATION` milliseconds, the function examines the result stored in `PAUSE_SCAN`
+    ///
+    /// When the value stored in `PAUSE_SCAN` becomes `false`, the function returns, exiting the busy
+    /// loop
+    pub async fn pause(&self, get_user_input: bool) -> usize {
+        // function uses tokio::time, not std
+
+        // local testing showed a pretty slow increase (less than linear) in CPU usage as # of
+        // concurrent scans rose when SLEEP_DURATION was set to 500, using that as the default for now
+        let mut interval = time::interval(time::Duration::from_millis(SLEEP_DURATION));
+        let mut num_cancelled = 0_usize;
+
+        if INTERACTIVE_BARRIER.load(Ordering::Relaxed) == 0 {
+            INTERACTIVE_BARRIER.fetch_add(1, Ordering::Relaxed);
+
+            if get_user_input {
+                num_cancelled += self.interactive_menu().await;
+                PAUSE_SCAN.store(false, Ordering::Relaxed);
+                self.print_known_responses();
+            }
+        }
+
+        loop {
+            // first tick happens immediately, all others wait the specified duration
+            interval.tick().await;
+
+            if !PAUSE_SCAN.load(Ordering::Acquire) {
+                // PAUSE_SCAN is false, so we can exit the busy loop
+
+                if INTERACTIVE_BARRIER.load(Ordering::Relaxed) == 1 {
+                    INTERACTIVE_BARRIER.fetch_sub(1, Ordering::Relaxed);
+                }
+
+                log::trace!("exit: pause_scan -> {}", num_cancelled);
+                return num_cancelled;
+            }
+        }
+    }
+
+    /// set the bar length of FeroxScans
+    pub fn set_bar_length(&self, bar_length: u64) {
+        if let Ok(mut guard) = self.bar_length.lock() {
+            *guard = bar_length;
+        }
+    }
+
+    /// Given a url, create a new `FeroxScan` and add it to `FeroxScans`
+    ///
+    /// If `FeroxScans` did not already contain the scan, return true; otherwise return false
+    ///
+    /// Also return a reference to the new `FeroxScan`
+    pub(super) fn add_scan(
+        &self,
+        url: &str,
+        scan_type: ScanType,
+        scan_order: ScanOrder,
+    ) -> (bool, Arc<FeroxScan>) {
+        let bar_length = if let Ok(guard) = self.bar_length.lock() {
+            *guard
+        } else {
+            0
+        };
+
+        let bar = match scan_type {
+            ScanType::Directory => {
+                let bar_type = match self.output_level {
+                    OutputLevel::Default => BarType::Default,
+                    OutputLevel::Quiet => BarType::Quiet,
+                    OutputLevel::Silent => BarType::Hidden,
+                };
+
+                let progress_bar = add_bar(url, bar_length, bar_type);
+
+                progress_bar.reset_elapsed();
+
+                Some(progress_bar)
+            }
+            ScanType::File => None,
+        };
+
+        let ferox_scan = FeroxScan::new(
+            url,
+            scan_type,
+            scan_order,
+            bar_length,
+            self.output_level,
+            bar,
+        );
+
+        // If the set did not contain the scan, true is returned.
+        // If the set did contain the scan, false is returned.
+        let response = self.insert(ferox_scan.clone());
+
+        (response, ferox_scan)
+    }
+
+    /// Given a url, create a new `FeroxScan` and add it to `FeroxScans` as a Directory Scan
+    ///
+    /// If `FeroxScans` did not already contain the scan, return true; otherwise return false
+    ///
+    /// Also return a reference to the new `FeroxScan`
+    pub fn add_directory_scan(&self, url: &str, scan_order: ScanOrder) -> (bool, Arc<FeroxScan>) {
+        self.add_scan(url, ScanType::Directory, scan_order)
+    }
+
+    /// Given a url, create a new `FeroxScan` and add it to `FeroxScans` as a File Scan
+    ///
+    /// If `FeroxScans` did not already contain the scan, return true; otherwise return false
+    ///
+    /// Also return a reference to the new `FeroxScan`
+    pub fn add_file_scan(&self, url: &str, scan_order: ScanOrder) -> (bool, Arc<FeroxScan>) {
+        self.add_scan(url, ScanType::File, scan_order)
+    }
+
+    /// small helper to determine whether any scans are active or not
+    pub fn has_active_scans(&self) -> bool {
+        if let Ok(guard) = self.scans.read() {
+            for scan in guard.iter() {
+                if scan.is_active() {
+                    return true;
+                }
+            }
+        }
+        false
+    }
+
+    /// Retrieve all active scans
+    pub fn get_active_scans(&self) -> Vec<Arc<FeroxScan>> {
+        let mut scans = vec![];
+
+        if let Ok(guard) = self.scans.read() {
+            for scan in guard.iter() {
+                if !scan.is_active() {
+                    continue;
+                }
+                scans.push(scan.clone());
+            }
+        }
+        scans
+    }
+}

src/scan_manager/state.rs

@@ -0,0 +1,53 @@
+use super::*;
+use crate::{config::Configuration, statistics::Stats, traits::FeroxSerialize, utils::fmt_err};
+use anyhow::{Context, Result};
+use serde::Serialize;
+use std::sync::Arc;
+
+/// Data container for (de)?serialization of multiple items
+#[derive(Serialize, Debug)]
+pub struct FeroxState {
+    /// Known scans
+    scans: Arc<FeroxScans>,
+
+    /// Current running config
+    config: Arc<Configuration>,
+
+    /// Known responses
+    responses: &'static FeroxResponses,
+
+    /// Gathered statistics
+    statistics: Arc<Stats>,
+}
+
+/// implementation of FeroxState
+impl FeroxState {
+    /// create new FeroxState object
+    pub fn new(
+        scans: Arc<FeroxScans>,
+        config: Arc<Configuration>,
+        responses: &'static FeroxResponses,
+        statistics: Arc<Stats>,
+    ) -> Self {
+        Self {
+            scans,
+            config,
+            responses,
+            statistics,
+        }
+    }
+}
+
+/// FeroxSerialize implementation for FeroxState
+impl FeroxSerialize for FeroxState {
+    /// Simply return debug format of FeroxState to satisfy as_str
+    fn as_str(&self) -> String {
+        format!("{:?}", self)
+    }
+
+    /// Simple call to produce a JSON string using the given FeroxState
+    fn as_json(&self) -> Result<String> {
+        serde_json::to_string(&self)
+            .with_context(|| fmt_err("Could not convert scan's running state to JSON"))
+    }
+}

src/scan_manager/tests.rs

@@ -0,0 +1,594 @@
+use super::*;
+use crate::{
+    config::{Configuration, OutputLevel},
+    event_handlers::Handles,
+    response::FeroxResponse,
+    scanner::RESPONSES,
+    statistics::Stats,
+    traits::FeroxSerialize,
+    SLEEP_DURATION, VERSION,
+};
+use indicatif::ProgressBar;
+use predicates::prelude::*;
+use std::sync::{atomic::Ordering, Arc};
+use std::thread::sleep;
+use std::time::Instant;
+use tokio::time::{self, Duration};
+
+#[test]
+/// test that ScanType's default is File
+fn default_scantype_is_file() {
+    match ScanType::default() {
+        ScanType::File => {}
+        ScanType::Directory => panic!(),
+    }
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// tests that pause_scan pauses execution and releases execution when PAUSE_SCAN is toggled
+/// the spinner used during the test has had .finish_and_clear called on it, meaning that
+/// a new one will be created, taking the if branch within the function
+async fn scanner_pause_scan_with_finished_spinner() {
+    let now = time::Instant::now();
+    let urls = FeroxScans::default();
+
+    PAUSE_SCAN.store(true, Ordering::Relaxed);
+
+    let expected = time::Duration::from_secs(2);
+
+    tokio::spawn(async move {
+        time::sleep(expected).await;
+        PAUSE_SCAN.store(false, Ordering::Relaxed);
+    });
+
+    urls.pause(false).await;
+
+    assert!(now.elapsed() > expected);
+}
+
+#[test]
+/// add an unknown url to the hashset, expect true
+fn add_url_to_list_of_scanned_urls_with_unknown_url() {
+    let urls = FeroxScans::default();
+    let url = "http://unknown_url";
+    let (result, _scan) = urls.add_scan(url, ScanType::Directory, ScanOrder::Latest);
+    assert!(result);
+}
+
+#[test]
+/// add a known url to the hashset, with a trailing slash, expect false
+fn add_url_to_list_of_scanned_urls_with_known_url() {
+    let urls = FeroxScans::default();
+    let pb = ProgressBar::new(1);
+    let url = "http://unknown_url/";
+
+    let scan = FeroxScan::new(
+        url,
+        ScanType::Directory,
+        ScanOrder::Latest,
+        pb.length(),
+        OutputLevel::Default,
+        Some(pb),
+    );
+
+    assert!(urls.insert(scan));
+
+    let (result, _scan) = urls.add_scan(url, ScanType::Directory, ScanOrder::Latest);
+
+    assert!(!result);
+}
+
+#[test]
+/// stop_progress_bar should stop the progress bar
+fn stop_progress_bar_stops_bar() {
+    let pb = ProgressBar::new(1);
+    let url = "http://unknown_url/";
+
+    let scan = FeroxScan::new(
+        url,
+        ScanType::Directory,
+        ScanOrder::Latest,
+        pb.length(),
+        OutputLevel::Default,
+        Some(pb),
+    );
+
+    assert!(!scan
+        .progress_bar
+        .lock()
+        .unwrap()
+        .as_ref()
+        .unwrap()
+        .is_finished());
+
+    scan.stop_progress_bar();
+
+    assert!(scan
+        .progress_bar
+        .lock()
+        .unwrap()
+        .as_ref()
+        .unwrap()
+        .is_finished());
+}
+
+#[test]
+/// add a known url to the hashset, without a trailing slash, expect false
+fn add_url_to_list_of_scanned_urls_with_known_url_without_slash() {
+    let urls = FeroxScans::default();
+    let url = "http://unknown_url";
+
+    let scan = FeroxScan::new(
+        url,
+        ScanType::File,
+        ScanOrder::Latest,
+        0,
+        OutputLevel::Default,
+        None,
+    );
+
+    assert!(urls.insert(scan));
+
+    let (result, _scan) = urls.add_scan(url, ScanType::File, ScanOrder::Latest);
+
+    assert!(!result);
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// just increasing coverage, no real expectations
+async fn call_display_scans() {
+    let urls = FeroxScans::default();
+    let pb = ProgressBar::new(1);
+    let pb_two = ProgressBar::new(2);
+    let url = "http://unknown_url/";
+    let url_two = "http://unknown_url/fa";
+    let scan = FeroxScan::new(
+        url,
+        ScanType::Directory,
+        ScanOrder::Latest,
+        pb.length(),
+        OutputLevel::Default,
+        Some(pb),
+    );
+    let scan_two = FeroxScan::new(
+        url_two,
+        ScanType::Directory,
+        ScanOrder::Latest,
+        pb_two.length(),
+        OutputLevel::Default,
+        Some(pb_two),
+    );
+
+    scan_two.finish().unwrap(); // one complete, one incomplete
+    scan_two
+        .set_task(tokio::spawn(async move {
+            sleep(Duration::from_millis(SLEEP_DURATION));
+        }))
+        .await
+        .unwrap();
+
+    assert!(urls.insert(scan));
+    assert!(urls.insert(scan_two));
+
+    urls.display_scans().await;
+}
+
+#[test]
+/// ensure that PartialEq compares FeroxScan.id fields
+fn partial_eq_compares_the_id_field() {
+    let url = "http://unknown_url/";
+    let scan = FeroxScan::new(
+        url,
+        ScanType::Directory,
+        ScanOrder::Latest,
+        0,
+        OutputLevel::Default,
+        None,
+    );
+    let scan_two = FeroxScan::new(
+        url,
+        ScanType::Directory,
+        ScanOrder::Latest,
+        0,
+        OutputLevel::Default,
+        None,
+    );
+
+    assert!(!scan.eq(&scan_two));
+
+    let scan_two = scan.clone();
+
+    assert!(scan.eq(&scan_two));
+}
+
+#[test]
+/// show that a new progress bar is created if one doesn't exist
+fn ferox_scan_get_progress_bar_when_none_is_set() {
+    let scan = FeroxScan::default();
+
+    assert!(scan.progress_bar.lock().unwrap().is_none()); // no pb exists
+
+    let pb = scan.progress_bar();
+
+    assert!(scan.progress_bar.lock().unwrap().is_some()); // new pb created
+    assert!(!pb.is_finished()) // not finished
+}
+
+#[test]
+/// given a JSON entry representing a FeroxScan, test that it deserializes into the proper type
+/// with the right attributes
+fn ferox_scan_deserialize() {
+    let fs_json = r#"{"id":"057016a14769414aac9a7a62707598cb","url":"https://spiritanimal.com","scan_type":"Directory","status":"Complete"}"#;
+    let fs_json_two = r#"{"id":"057016a14769414aac9a7a62707598cb","url":"https://spiritanimal.com","scan_type":"Not Correct","status":"Cancelled"}"#;
+    let fs_json_three = r#"{"id":"057016a14769414aac9a7a62707598cb","url":"https://spiritanimal.com","scan_type":"Not Correct","status":"","num_requests":42}"#;
+
+    let fs: FeroxScan = serde_json::from_str(fs_json).unwrap();
+    let fs_two: FeroxScan = serde_json::from_str(fs_json_two).unwrap();
+    let fs_three: FeroxScan = serde_json::from_str(fs_json_three).unwrap();
+    assert_eq!(fs.url, "https://spiritanimal.com");
+
+    match fs.scan_type {
+        ScanType::Directory => {}
+        ScanType::File => {
+            panic!();
+        }
+    }
+    match fs_two.scan_type {
+        ScanType::Directory => {
+            panic!();
+        }
+        ScanType::File => {}
+    }
+
+    match *fs.progress_bar.lock().unwrap() {
+        None => {}
+        Some(_) => {
+            panic!();
+        }
+    }
+    assert!(matches!(*fs.status.lock().unwrap(), ScanStatus::Complete));
+    assert!(matches!(
+        *fs_two.status.lock().unwrap(),
+        ScanStatus::Cancelled
+    ));
+    assert!(matches!(
+        *fs_three.status.lock().unwrap(),
+        ScanStatus::NotStarted
+    ));
+    assert_eq!(fs_three.num_requests, 42);
+    assert_eq!(fs.id, "057016a14769414aac9a7a62707598cb");
+}
+
+#[test]
+/// given a FeroxScan, test that it serializes into the proper JSON entry
+fn ferox_scan_serialize() {
+    let fs = FeroxScan::new(
+        "https://spiritanimal.com",
+        ScanType::Directory,
+        ScanOrder::Latest,
+        0,
+        OutputLevel::Default,
+        None,
+    );
+    let fs_json = format!(
+        r#"{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","status":"NotStarted","num_requests":0}}"#,
+        fs.id
+    );
+    assert_eq!(fs_json, serde_json::to_string(&*fs).unwrap());
+}
+
+#[test]
+/// given a FeroxScans, test that it serializes into the proper JSON entry
+fn ferox_scans_serialize() {
+    let ferox_scan = FeroxScan::new(
+        "https://spiritanimal.com",
+        ScanType::Directory,
+        ScanOrder::Latest,
+        0,
+        OutputLevel::Default,
+        None,
+    );
+    let ferox_scans = FeroxScans::default();
+    let ferox_scans_json = format!(
+        r#"[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","status":"NotStarted","num_requests":0}}]"#,
+        ferox_scan.id
+    );
+    ferox_scans.scans.write().unwrap().push(ferox_scan);
+    assert_eq!(
+        ferox_scans_json,
+        serde_json::to_string(&ferox_scans).unwrap()
+    );
+}
+
+#[test]
+/// given a FeroxResponses, test that it serializes into the proper JSON entry
+fn ferox_responses_serialize() {
+    let json_response = r#"{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{"server":"nginx/1.16.1"}}"#;
+    let response: FeroxResponse = serde_json::from_str(json_response).unwrap();
+
+    let responses = FeroxResponses::default();
+    responses.insert(response);
+    // responses has a response now
+
+    // serialized should be a list of responses
+    let expected = format!("[{}]", json_response);
+
+    let serialized = serde_json::to_string(&responses).unwrap();
+    assert_eq!(expected, serialized);
+}
+
+#[test]
+/// given a FeroxResponse, test that it serializes into the proper JSON entry
+fn ferox_response_serialize_and_deserialize() {
+    // deserialize
+    let json_response = r#"{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{"server":"nginx/1.16.1"}}"#;
+    let response: FeroxResponse = serde_json::from_str(json_response).unwrap();
+
+    assert_eq!(response.url().as_str(), "https://nerdcore.com/css");
+    assert_eq!(response.url().path(), "/css");
+    assert!(response.wildcard());
+    assert_eq!(response.status().as_u16(), 301);
+    assert_eq!(response.content_length(), 173);
+    assert_eq!(response.line_count(), 10);
+    assert_eq!(response.word_count(), 16);
+    assert_eq!(response.headers().get("server").unwrap(), "nginx/1.16.1");
+
+    // serialize, however, this can fail when headers are out of order
+    let new_json = serde_json::to_string(&response).unwrap();
+    assert_eq!(json_response, new_json);
+}
+
+#[test]
+/// test FeroxSerialize implementation of FeroxState
+fn feroxstates_feroxserialize_implementation() {
+    let ferox_scan = FeroxScan::new(
+        "https://spiritanimal.com",
+        ScanType::Directory,
+        ScanOrder::Latest,
+        0,
+        OutputLevel::Default,
+        None,
+    );
+    let ferox_scans = FeroxScans::default();
+    let saved_id = ferox_scan.id.clone();
+    ferox_scans.insert(ferox_scan);
+
+    let config = Configuration::new().unwrap();
+    let stats = Arc::new(Stats::new(config.extensions.len(), config.json));
+
+    let json_response = r#"{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{"server":"nginx/1.16.1"}}"#;
+    let response: FeroxResponse = serde_json::from_str(json_response).unwrap();
+    RESPONSES.insert(response);
+
+    let ferox_state = FeroxState::new(
+        Arc::new(ferox_scans),
+        Arc::new(Configuration::new().unwrap()),
+        &RESPONSES,
+        stats,
+    );
+
+    let expected_strs = predicates::str::contains("scans: FeroxScans").and(
+        predicate::str::contains("config: Configuration")
+            .and(predicate::str::contains("responses: FeroxResponses"))
+            .and(predicate::str::contains("nerdcore.com"))
+            .and(predicate::str::contains("/css"))
+            .and(predicate::str::contains("https://spiritanimal.com")),
+    );
+
+    assert!(expected_strs.eval(&ferox_state.as_str()));
+
+    let json_state = ferox_state.as_json().unwrap();
+    let expected = format!(
+        r#"{{"scans":[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","status":"NotStarted","num_requests":0}}],"config":{{"type":"configuration","wordlist":"/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt","config":"","proxy":"","replay_proxy":"","target_url":"","status_codes":[200,204,301,302,307,308,401,403,405,500],"replay_codes":[200,204,301,302,307,308,401,403,405,500],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"silent":false,"quiet":false,"auto_bail":false,"auto_tune":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/{}","redirects":false,"insecure":false,"extensions":[],"headers":{{}},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":false,"depth":4,"scan_limit":0,"parallel":0,"rate_limit":0,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false,"resumed":false,"resume_from":"","save_state":false,"time_limit":"","filter_similar":[],"url_denylist":[]}},"responses":[{{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{{"server":"nginx/1.16.1"}}}}]"#,
+        saved_id, VERSION
+    );
+    println!("{}\n{}", expected, json_state);
+    assert!(predicates::str::contains(expected).eval(&json_state));
+}
+
+#[should_panic]
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// call start_max_time_thread with a valid timespec, expect a panic, but only after a certain
+/// number of seconds
+async fn start_max_time_thread_panics_after_delay() {
+    let now = time::Instant::now();
+    let delay = time::Duration::new(3, 0);
+
+    let config = Configuration {
+        time_limit: String::from("3s"),
+        ..Default::default()
+    };
+
+    let handles = Arc::new(Handles::for_testing(None, Some(Arc::new(config))).0);
+
+    start_max_time_thread(handles).await;
+
+    assert!(now.elapsed() > delay);
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// call start_max_time_thread with a timespec that's too large to be parsed correctly, expect
+/// immediate return and no panic, as the sigint handler is never called
+async fn start_max_time_thread_returns_immediately_with_too_large_input() {
+    let now = time::Instant::now();
+    let delay = time::Duration::new(1, 0);
+    let config = Configuration {
+        time_limit: String::from("18446744073709551616m"),
+        ..Default::default()
+    };
+
+    let handles = Arc::new(Handles::for_testing(None, Some(Arc::new(config))).0);
+
+    // pub const MAX: usize = usize::MAX; // 18_446_744_073_709_551_615usize
+    start_max_time_thread(handles).await; // can't fit in dest u64
+
+    assert!(now.elapsed() < delay); // assuming function call will take less than 1second
+}
+
+#[test]
+/// coverage for FeroxScan's Display implementation
+fn feroxscan_display() {
+    let scan = FeroxScan {
+        id: "".to_string(),
+        url: String::from("http://localhost"),
+        scan_order: ScanOrder::Latest,
+        scan_type: Default::default(),
+        num_requests: 0,
+        start_time: Instant::now(),
+        output_level: OutputLevel::Default,
+        status_403s: Default::default(),
+        status_429s: Default::default(),
+        status: Default::default(),
+        task: tokio::sync::Mutex::new(None),
+        progress_bar: std::sync::Mutex::new(None),
+        errors: Default::default(),
+    };
+
+    let not_started = format!("{}", scan);
+
+    assert!(predicate::str::contains("not started")
+        .and(predicate::str::contains("localhost"))
+        .eval(&not_started));
+
+    scan.set_status(ScanStatus::Complete).unwrap();
+    let complete = format!("{}", scan);
+    assert!(predicate::str::contains("complete")
+        .and(predicate::str::contains("localhost"))
+        .eval(&complete));
+
+    scan.set_status(ScanStatus::Cancelled).unwrap();
+    let cancelled = format!("{}", scan);
+    assert!(predicate::str::contains("cancelled")
+        .and(predicate::str::contains("localhost"))
+        .eval(&cancelled));
+
+    scan.set_status(ScanStatus::Running).unwrap();
+    let running = format!("{}", scan);
+    assert!(predicate::str::contains("running")
+        .and(predicate::str::contains("localhost"))
+        .eval(&running));
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// call FeroxScan::abort, ensure status becomes cancelled
+async fn ferox_scan_abort() {
+    let scan = FeroxScan {
+        id: "".to_string(),
+        url: String::from("http://localhost"),
+        scan_order: ScanOrder::Latest,
+        scan_type: Default::default(),
+        num_requests: 0,
+        start_time: Instant::now(),
+        output_level: OutputLevel::Default,
+        status_403s: Default::default(),
+        status_429s: Default::default(),
+        status: std::sync::Mutex::new(ScanStatus::Running),
+        task: tokio::sync::Mutex::new(Some(tokio::spawn(async move {
+            sleep(Duration::from_millis(SLEEP_DURATION * 2));
+        }))),
+        progress_bar: std::sync::Mutex::new(None),
+        errors: Default::default(),
+    };
+
+    scan.abort().await.unwrap();
+
+    assert!(matches!(
+        *scan.status.lock().unwrap(),
+        ScanStatus::Cancelled
+    ));
+}
+
+#[test]
+/// call a few menu functions for coverage's sake
+///
+/// there's not a trivial way to test these programmatically (at least i'm too lazy rn to do it)
+/// and their correctness can be verified easily manually; just calling for now
+fn menu_print_header_and_footer() {
+    let menu = Menu::new();
+    menu.clear_screen();
+    menu.print_header();
+    menu.print_footer();
+    menu.hide_progress_bars();
+    menu.show_progress_bars();
+}
+
+#[test]
+/// ensure spaces are trimmed and numbers are returned from split_to_nums
+fn split_to_nums_is_correct() {
+    let menu = Menu::new();
+
+    let nums = menu.split_to_nums("1, 3,      4, 7 -     12, 10-10, 10-11, 9-12, 12-6, -1, 4-");
+
+    assert_eq!(nums, vec![1, 3, 4, 7, 8, 9, 10, 11, 12]);
+    assert_eq!(menu.split_to_nums("9-12"), vec![9, 10, 11, 12]);
+    assert!(menu.split_to_nums("-12").is_empty());
+    assert!(menu.split_to_nums("12-").is_empty());
+    assert!(menu.split_to_nums("\n").is_empty());
+}
+
+#[test]
+/// given a deep url, find the correct scan
+fn get_base_scan_by_url_finds_correct_scan() {
+    let urls = FeroxScans::default();
+    let url = "http://localhost";
+    let url1 = "http://localhost/stuff";
+    let url2 = "http://shlocalhost/stuff/things";
+    let url3 = "http://shlocalhost/stuff/things/mostuff";
+    let (_, scan) = urls.add_scan(url, ScanType::Directory, ScanOrder::Latest);
+    let (_, scan1) = urls.add_scan(url1, ScanType::Directory, ScanOrder::Latest);
+    let (_, scan2) = urls.add_scan(url2, ScanType::Directory, ScanOrder::Latest);
+    let (_, scan3) = urls.add_scan(url3, ScanType::Directory, ScanOrder::Latest);
+
+    assert_eq!(
+        urls.get_base_scan_by_url("http://localhost/things.php")
+            .unwrap()
+            .id,
+        scan.id
+    );
+    assert_eq!(
+        urls.get_base_scan_by_url("http://localhost/stuff/things.php")
+            .unwrap()
+            .id,
+        scan1.id
+    );
+    assert_eq!(
+        urls.get_base_scan_by_url("http://shlocalhost/stuff/things/mostuff.php")
+            .unwrap()
+            .id,
+        scan2.id
+    );
+    assert_eq!(
+        urls.get_base_scan_by_url("http://shlocalhost/stuff/things/mostuff/mothings.php")
+            .unwrap()
+            .id,
+        scan3.id
+    );
+}
+
+#[test]
+/// given a shallow url without a trailing slash, find the correct scan
+fn get_base_scan_by_url_finds_correct_scan_without_trailing_slash() {
+    let urls = FeroxScans::default();
+    let url = "http://localhost";
+    let (_, scan) = urls.add_scan(url, ScanType::Directory, ScanOrder::Latest);
+    assert_eq!(
+        urls.get_base_scan_by_url("http://localhost/BKPMiherrortBPKcw")
+            .unwrap()
+            .id,
+        scan.id
+    );
+}
+
+#[test]
+/// given a shallow url with a trailing slash, find the correct scan
+fn get_base_scan_by_url_finds_correct_scan_with_trailing_slash() {
+    let urls = FeroxScans::default();
+    let url = "http://127.0.0.1:41971/";
+    let (_, scan) = urls.add_scan(url, ScanType::Directory, ScanOrder::Latest);
+    assert_eq!(
+        urls.get_base_scan_by_url("http://127.0.0.1:41971/BKPMiherrortBPKcw")
+            .unwrap()
+            .id,
+        scan.id
+    );
+}

src/scan_manager/utils.rs

@@ -0,0 +1,92 @@
+#[cfg(not(test))]
+use crate::event_handlers::TermInputHandler;
+use crate::{
+    config::Configuration, event_handlers::Handles, parser::TIMESPEC_REGEX, scanner::RESPONSES,
+};
+
+use std::{fs::File, io::BufReader, sync::Arc};
+use tokio::time;
+
+/// Given a string representing some number of seconds, minutes, hours, or days, convert
+/// that representation to seconds and then wait for those seconds to elapse.  Once that period
+/// of time has elapsed, kill all currently running scans and dump a state file to disk that can
+/// be used to resume any unfinished scan.
+pub async fn start_max_time_thread(handles: Arc<Handles>) {
+    log::trace!("enter: start_max_time_thread({:?})", handles);
+
+    // as this function has already made it through the parser, which calls is_match on
+    // the value passed to --time-limit using TIMESPEC_REGEX; we can safely assume that
+    // the capture groups are populated; can expect something like 10m, 30s, 1h, etc...
+    let captures = TIMESPEC_REGEX.captures(&handles.config.time_limit).unwrap();
+    let length_match = captures.get(1).unwrap();
+    let measurement_match = captures.get(2).unwrap();
+
+    if let Ok(length) = length_match.as_str().parse::<u64>() {
+        let length_in_secs = match measurement_match.as_str().to_ascii_lowercase().as_str() {
+            "s" => length,
+            "m" => length * 60,           // minutes
+            "h" => length * 60 * 60,      // hours
+            "d" => length * 60 * 60 * 24, // days
+            _ => length,
+        };
+
+        log::debug!(
+            "max time limit as string: {} and as seconds: {}",
+            handles.config.time_limit,
+            length_in_secs
+        );
+
+        time::sleep(time::Duration::new(length_in_secs, 0)).await;
+
+        log::trace!("exit: start_max_time_thread");
+
+        #[cfg(test)]
+        panic!("{:?}", handles);
+        #[cfg(not(test))]
+        let _ = TermInputHandler::sigint_handler(handles.clone());
+    }
+
+    log::warn!(
+        "Could not parse the value provided ({}), can't enforce time limit",
+        handles.config.time_limit
+    );
+}
+
+/// Primary logic used to load a Configuration from disk and populate the appropriate data
+/// structures
+pub fn resume_scan(filename: &str) -> Configuration {
+    log::trace!("enter: resume_scan({})", filename);
+
+    let file = File::open(filename).unwrap_or_else(|e| {
+        log::error!("{}", e);
+        log::error!("Could not open state file, exiting");
+        std::process::exit(1);
+    });
+
+    let reader = BufReader::new(file);
+    let state: serde_json::Value = serde_json::from_reader(reader).unwrap();
+
+    let conf = state.get("config").unwrap_or_else(|| {
+        log::error!("Could not load configuration from state file, exiting");
+        std::process::exit(1);
+    });
+
+    let config = serde_json::from_value(conf.clone()).unwrap_or_else(|e| {
+        log::error!("{}", e);
+        log::error!("Could not deserialize configuration found in state file, exiting");
+        std::process::exit(1);
+    });
+
+    if let Some(responses) = state.get("responses") {
+        if let Some(arr_responses) = responses.as_array() {
+            for response in arr_responses {
+                if let Ok(deser_resp) = serde_json::from_value(response.clone()) {
+                    RESPONSES.insert(deser_resp);
+                }
+            }
+        }
+    }
+
+    log::trace!("exit: resume_scan -> {:?}", config);
+    config
+}

src/scanner.rs

@@ -1,798 +0,0 @@
-use crate::{
-    config::{Configuration, CONFIGURATION},
-    extractor::get_links,
-    filters::{
-        FeroxFilter, LinesFilter, RegexFilter, SizeFilter, StatusCodeFilter, WildcardFilter,
-        WordsFilter,
-    },
-    heuristics,
-    scan_manager::{FeroxResponses, FeroxScans, PAUSE_SCAN},
-    utils::{format_url, get_current_depth, make_request},
-    FeroxChannel, FeroxResponse,
-};
-use futures::{
-    future::{BoxFuture, FutureExt},
-    stream, StreamExt,
-};
-use lazy_static::lazy_static;
-use regex::Regex;
-use reqwest::Url;
-#[cfg(not(test))]
-use std::process::exit;
-use std::{
-    collections::HashSet,
-    convert::TryInto,
-    ops::Deref,
-    sync::atomic::{AtomicU64, AtomicUsize, Ordering},
-    sync::{Arc, RwLock},
-};
-use tokio::{
-    sync::{
-        mpsc::{self, UnboundedReceiver, UnboundedSender},
-        Semaphore,
-    },
-    task::JoinHandle,
-};
-
-/// Single atomic number that gets incremented once, used to track first scan vs. all others
-static CALL_COUNT: AtomicUsize = AtomicUsize::new(0);
-
-/// Single atomic number that gets holds the number of requests to be sent per directory scanned
-pub static NUMBER_OF_REQUESTS: AtomicU64 = AtomicU64::new(0);
-
-lazy_static! {
-    /// Set of urls that have been sent to [scan_url](fn.scan_url.html), used for deduplication
-    pub static ref SCANNED_URLS: FeroxScans = FeroxScans::default();
-
-    /// Vector of implementors of the FeroxFilter trait
-    static ref FILTERS: Arc<RwLock<Vec<Box<dyn FeroxFilter>>>> = Arc::new(RwLock::new(Vec::<Box<dyn FeroxFilter>>::new()));
-
-    /// Vector of FeroxResponse objects
-    pub static ref RESPONSES: FeroxResponses = FeroxResponses::default();
-
-    /// Bounded semaphore used as a barrier to limit concurrent scans
-    static ref SCAN_LIMITER: Semaphore = Semaphore::new(CONFIGURATION.scan_limit);
-}
-
-/// Adds the given FeroxFilter to the given list of FeroxFilter implementors
-///
-/// If the given list did not already contain the filter, return true; otherwise return false
-fn add_filter_to_list_of_ferox_filters(
-    filter: Box<dyn FeroxFilter>,
-    ferox_filters: Arc<RwLock<Vec<Box<dyn FeroxFilter>>>>,
-) -> bool {
-    log::trace!(
-        "enter: add_filter_to_list_of_ferox_filters({:?}, {:?})",
-        filter,
-        ferox_filters
-    );
-
-    match ferox_filters.write() {
-        Ok(mut filters) => {
-            // If the set did not contain the assigned filter, true is returned.
-            // If the set did contain the assigned filter, false is returned.
-            if filters.contains(&filter) {
-                log::trace!("exit: add_filter_to_list_of_ferox_filters -> false");
-                return false;
-            }
-
-            filters.push(filter);
-
-            log::trace!("exit: add_filter_to_list_of_ferox_filters -> true");
-            true
-        }
-        Err(e) => {
-            // poisoned lock
-            log::error!("Set of wildcard filters poisoned: {}", e);
-            log::trace!("exit: add_filter_to_list_of_ferox_filters -> false");
-            false
-        }
-    }
-}
-
-/// Spawn a single consumer task (sc side of mpsc)
-///
-/// The consumer simply receives Urls and scans them
-fn spawn_recursion_handler(
-    mut recursion_channel: UnboundedReceiver<String>,
-    wordlist: Arc<HashSet<String>>,
-    base_depth: usize,
-    tx_term: UnboundedSender<FeroxResponse>,
-    tx_file: UnboundedSender<FeroxResponse>,
-) -> BoxFuture<'static, Vec<JoinHandle<()>>> {
-    log::trace!(
-        "enter: spawn_recursion_handler({:?}, wordlist[{} words...], {}, {:?}, {:?})",
-        recursion_channel,
-        wordlist.len(),
-        base_depth,
-        tx_term,
-        tx_file
-    );
-
-    let boxed_future = async move {
-        let mut scans = vec![];
-
-        while let Some(resp) = recursion_channel.recv().await {
-            let (unknown, _) = SCANNED_URLS.add_directory_scan(&resp);
-
-            if !unknown {
-                // not unknown, i.e. we've seen the url before and don't need to scan again
-                continue;
-            }
-
-            log::info!("received {} on recursion channel", resp);
-
-            let term_clone = tx_term.clone();
-            let file_clone = tx_file.clone();
-            let resp_clone = resp.clone();
-            let list_clone = wordlist.clone();
-
-            let future = tokio::spawn(async move {
-                scan_url(
-                    resp_clone.to_owned().as_str(),
-                    list_clone,
-                    base_depth,
-                    term_clone,
-                    file_clone,
-                )
-                .await
-            });
-
-            scans.push(future);
-        }
-        scans
-    }
-    .boxed();
-
-    log::trace!("exit: spawn_recursion_handler -> BoxFuture<'static, Vec<JoinHandle<()>>>");
-    boxed_future
-}
-
-/// Creates a vector of formatted Urls
-///
-/// At least one value will be returned (base_url + word)
-///
-/// If any extensions were passed to the program, each extension will add a
-/// (base_url + word + ext) Url to the vector
-fn create_urls(target_url: &str, word: &str, extensions: &[String]) -> Vec<Url> {
-    log::trace!(
-        "enter: create_urls({}, {}, {:?})",
-        target_url,
-        word,
-        extensions
-    );
-
-    let mut urls = vec![];
-
-    if let Ok(url) = format_url(
-        &target_url,
-        &word,
-        CONFIGURATION.add_slash,
-        &CONFIGURATION.queries,
-        None,
-    ) {
-        urls.push(url); // default request, i.e. no extension
-    }
-
-    for ext in extensions.iter() {
-        if let Ok(url) = format_url(
-            &target_url,
-            &word,
-            CONFIGURATION.add_slash,
-            &CONFIGURATION.queries,
-            Some(ext),
-        ) {
-            urls.push(url); // any extensions passed in
-        }
-    }
-
-    log::trace!("exit: create_urls -> {:?}", urls);
-    urls
-}
-
-/// Helper function to determine suitability for recursion
-///
-/// handles 2xx and 3xx responses by either checking if the url ends with a / (2xx)
-/// or if the Location header is present and matches the base url + / (3xx)
-fn response_is_directory(response: &FeroxResponse) -> bool {
-    log::trace!("enter: is_directory({})", response);
-
-    if response.status().is_redirection() {
-        // status code is 3xx
-        match response.headers().get("Location") {
-            // and has a Location header
-            Some(loc) => {
-                // get absolute redirect Url based on the already known base url
-                log::debug!("Location header: {:?}", loc);
-
-                if let Ok(loc_str) = loc.to_str() {
-                    if let Ok(abs_url) = response.url().join(loc_str) {
-                        if format!("{}/", response.url()) == abs_url.as_str() {
-                            // if current response's Url + / == the absolute redirection
-                            // location, we've found a directory suitable for recursion
-                            log::debug!(
-                                "found directory suitable for recursion: {}",
-                                response.url()
-                            );
-                            log::trace!("exit: is_directory -> true");
-                            return true;
-                        }
-                    }
-                }
-            }
-            None => {
-                log::debug!("expected Location header, but none was found: {}", response);
-                log::trace!("exit: is_directory -> false");
-                return false;
-            }
-        }
-    } else if response.status().is_success() {
-        // status code is 2xx, need to check if it ends in /
-
-        if response.url().as_str().ends_with('/') {
-            log::debug!("{} is directory suitable for recursion", response.url());
-            log::trace!("exit: is_directory -> true");
-            return true;
-        }
-    }
-
-    log::trace!("exit: is_directory -> false");
-    false
-}
-
-/// Helper function that determines if the configured maximum recursion depth has been reached
-///
-/// Essentially looks at the Url path and determines how many directories are present in the
-/// given Url
-fn reached_max_depth(url: &Url, base_depth: usize, max_depth: usize) -> bool {
-    log::trace!(
-        "enter: reached_max_depth({}, {}, {})",
-        url,
-        base_depth,
-        max_depth
-    );
-
-    if max_depth == 0 {
-        // early return, as 0 means recurse forever; no additional processing needed
-        log::trace!("exit: reached_max_depth -> false");
-        return false;
-    }
-
-    let depth = get_current_depth(url.as_str());
-
-    if depth - base_depth >= max_depth {
-        return true;
-    }
-
-    log::trace!("exit: reached_max_depth -> false");
-    false
-}
-
-/// Helper function that wraps logic to check for recursion opportunities
-///
-/// When a recursion opportunity is found, the new url is sent across the recursion channel
-async fn try_recursion(
-    response: &FeroxResponse,
-    base_depth: usize,
-    transmitter: UnboundedSender<String>,
-) {
-    log::trace!(
-        "enter: try_recursion({}, {}, {:?})",
-        response,
-        base_depth,
-        transmitter
-    );
-
-    if !reached_max_depth(response.url(), base_depth, CONFIGURATION.depth)
-        && response_is_directory(&response)
-    {
-        if CONFIGURATION.redirects {
-            // response is 2xx can simply send it because we're following redirects
-            log::info!("Added new directory to recursive scan: {}", response.url());
-
-            match transmitter.send(String::from(response.url().as_str())) {
-                Ok(_) => {
-                    log::debug!("sent {} across channel to begin a new scan", response.url());
-                }
-                Err(e) => {
-                    log::error!(
-                        "Could not send {} to recursion handler: {}",
-                        response.url(),
-                        e
-                    );
-                }
-            }
-        } else {
-            let new_url = String::from(response.url().as_str());
-
-            log::info!("Added new directory to recursive scan: {}", new_url);
-
-            match transmitter.send(new_url) {
-                Ok(_) => {}
-                Err(e) => {
-                    log::error!(
-                        "Could not send {}/ to recursion handler: {}",
-                        response.url(),
-                        e
-                    );
-                }
-            }
-        }
-    }
-    log::trace!("exit: try_recursion");
-}
-
-/// Simple helper to stay DRY; determines whether or not a given `FeroxResponse` should be reported
-/// to the user or not.
-pub fn should_filter_response(response: &FeroxResponse) -> bool {
-    match FILTERS.read() {
-        Ok(filters) => {
-            for filter in filters.iter() {
-                // wildcard.should_filter goes here
-                if filter.should_filter_response(&response) {
-                    return true;
-                }
-            }
-        }
-        Err(e) => {
-            log::error!("{}", e);
-        }
-    }
-    false
-}
-
-/// Wrapper for [make_request](fn.make_request.html)
-///
-/// Handles making multiple requests based on the presence of extensions
-///
-/// Attempts recursion when appropriate and sends Responses to the report handler for processing
-async fn make_requests(
-    target_url: &str,
-    word: &str,
-    base_depth: usize,
-    dir_chan: UnboundedSender<String>,
-    report_chan: UnboundedSender<FeroxResponse>,
-) {
-    log::trace!(
-        "enter: make_requests({}, {}, {}, {:?}, {:?})",
-        target_url,
-        word,
-        base_depth,
-        dir_chan,
-        report_chan
-    );
-
-    let urls = create_urls(&target_url, &word, &CONFIGURATION.extensions);
-
-    for url in urls {
-        if let Ok(response) = make_request(&CONFIGURATION.client, &url).await {
-            // response came back without error, convert it to FeroxResponse
-            let ferox_response = FeroxResponse::from(response, true).await;
-
-            // do recursion if appropriate
-            if !CONFIGURATION.no_recursion {
-                try_recursion(&ferox_response, base_depth, dir_chan.clone()).await;
-            }
-
-            // purposefully doing recursion before filtering. the thought process is that
-            // even though this particular url is filtered, subsequent urls may not
-
-            if should_filter_response(&ferox_response) {
-                continue;
-            }
-
-            if CONFIGURATION.extract_links && !ferox_response.status().is_redirection() {
-                let new_links = get_links(&ferox_response).await;
-
-                for new_link in new_links {
-                    // create a url based on the given command line options, continue on error
-                    let new_url = match format_url(
-                        &new_link,
-                        &"",
-                        CONFIGURATION.add_slash,
-                        &CONFIGURATION.queries,
-                        None,
-                    ) {
-                        Ok(url) => url,
-                        Err(_) => continue,
-                    };
-
-                    if SCANNED_URLS.get_scan_by_url(&new_url.to_string()).is_some() {
-                        //we've seen the url before and don't need to scan again
-                        continue;
-                    }
-
-                    // make the request and store the response
-                    let new_response = match make_request(&CONFIGURATION.client, &new_url).await {
-                        Ok(resp) => resp,
-                        Err(_) => continue,
-                    };
-
-                    let mut new_ferox_response = FeroxResponse::from(new_response, true).await;
-
-                    // filter if necessary
-                    if should_filter_response(&new_ferox_response) {
-                        continue;
-                    }
-
-                    if new_ferox_response.is_file() {
-                        // very likely a file, simply request and report
-                        log::debug!("Singular extraction: {}", new_ferox_response);
-
-                        SCANNED_URLS.add_file_scan(&new_url.to_string());
-
-                        send_report(report_chan.clone(), new_ferox_response);
-
-                        continue;
-                    }
-
-                    if !CONFIGURATION.no_recursion {
-                        log::debug!("Recursive extraction: {}", new_ferox_response);
-
-                        if new_ferox_response.status().is_success()
-                            && !new_ferox_response.url().as_str().ends_with('/')
-                        {
-                            // since all of these are 2xx, recursion is only attempted if the
-                            // url ends in a /. I am actually ok with adding the slash and not
-                            // adding it, as both have merit.  Leaving it in for now to see how
-                            // things turn out (current as of: v1.1.0)
-                            new_ferox_response.set_url(&format!("{}/", new_ferox_response.url()));
-                        }
-
-                        try_recursion(&new_ferox_response, base_depth, dir_chan.clone()).await;
-                    }
-                }
-            }
-
-            // everything else should be reported
-            send_report(report_chan.clone(), ferox_response);
-        }
-    }
-    log::trace!("exit: make_requests");
-}
-
-/// Simple helper to send a `FeroxResponse` over the tx side of an `mpsc::unbounded_channel`
-fn send_report(report_sender: UnboundedSender<FeroxResponse>, response: FeroxResponse) {
-    log::trace!("enter: send_report({:?}, {}", report_sender, response);
-
-    match report_sender.send(response) {
-        Ok(_) => {}
-        Err(e) => {
-            log::error!("{}", e);
-        }
-    }
-
-    log::trace!("exit: send_report");
-}
-
-/// Scan a given url using a given wordlist
-///
-/// This is the primary entrypoint for the scanner
-pub async fn scan_url(
-    target_url: &str,
-    wordlist: Arc<HashSet<String>>,
-    base_depth: usize,
-    tx_term: UnboundedSender<FeroxResponse>,
-    tx_file: UnboundedSender<FeroxResponse>,
-) {
-    log::trace!(
-        "enter: scan_url({:?}, wordlist[{} words...], {}, {:?}, {:?})",
-        target_url,
-        wordlist.len(),
-        base_depth,
-        tx_term,
-        tx_file
-    );
-
-    log::info!("Starting scan against: {}", target_url);
-
-    let (tx_dir, rx_dir): FeroxChannel<String> = mpsc::unbounded_channel();
-
-    if CALL_COUNT.load(Ordering::Relaxed) == 0 {
-        CALL_COUNT.fetch_add(1, Ordering::Relaxed);
-
-        // this protection allows us to add the first scanned url to SCANNED_URLS
-        // from within the scan_url function instead of the recursion handler
-        SCANNED_URLS.add_directory_scan(&target_url);
-    }
-
-    let ferox_scan = match SCANNED_URLS.get_scan_by_url(&target_url) {
-        Some(scan) => scan,
-        None => {
-            log::error!(
-                "Could not find FeroxScan associated with {}; this shouldn't happen... exiting",
-                target_url
-            );
-            return;
-        }
-    };
-
-    let progress_bar = match ferox_scan.lock() {
-        Ok(mut scan) => scan.progress_bar(),
-        Err(e) => {
-            log::error!("FeroxScan's ({:?}) mutex is poisoned: {}", ferox_scan, e);
-            return;
-        }
-    };
-
-    // When acquire is called and the semaphore has remaining permits, the function immediately
-    // returns a permit. However, if no remaining permits are available, acquire (asynchronously)
-    // waits until an outstanding permit is dropped. At this point, the freed permit is assigned
-    // to the caller.
-    let permit = SCAN_LIMITER.acquire().await;
-
-    // Arc clones to be passed around to the various scans
-    let wildcard_bar = progress_bar.clone();
-    let heuristics_term_clone = tx_term.clone();
-    let recurser_term_clone = tx_term.clone();
-    let recurser_file_clone = tx_file.clone();
-    let recurser_words = wordlist.clone();
-    let looping_words = wordlist.clone();
-
-    let recurser = tokio::spawn(async move {
-        spawn_recursion_handler(
-            rx_dir,
-            recurser_words,
-            base_depth,
-            recurser_term_clone,
-            recurser_file_clone,
-        )
-        .await
-    });
-
-    // add any wildcard filters to `FILTERS`
-    let filter =
-        match heuristics::wildcard_test(&target_url, wildcard_bar, heuristics_term_clone).await {
-            Some(f) => Box::new(f),
-            None => Box::new(WildcardFilter::default()),
-        };
-
-    add_filter_to_list_of_ferox_filters(filter, FILTERS.clone());
-
-    // producer tasks (mp of mpsc); responsible for making requests
-    let producers = stream::iter(looping_words.deref().to_owned())
-        .map(|word| {
-            let txd = tx_dir.clone();
-            let txr = tx_term.clone();
-            let pb = progress_bar.clone(); // progress bar is an Arc around internal state
-            let tgt = target_url.to_string(); // done to satisfy 'static lifetime below
-            (
-                tokio::spawn(async move {
-                    if PAUSE_SCAN.load(Ordering::Acquire) {
-                        // for every word in the wordlist, check to see if PAUSE_SCAN is set to true
-                        // when true; enter a busy loop that only exits by setting PAUSE_SCAN back
-                        // to false
-
-                        // todo change to true when issue #107 is resolved
-                        SCANNED_URLS.pause(false).await;
-                    }
-                    make_requests(&tgt, &word, base_depth, txd, txr).await
-                }),
-                pb,
-            )
-        })
-        .for_each_concurrent(CONFIGURATION.threads, |(resp, bar)| async move {
-            match resp.await {
-                Ok(_) => {
-                    bar.inc((CONFIGURATION.extensions.len() + 1) as u64);
-                }
-                Err(e) => {
-                    log::error!("error awaiting a response: {}", e);
-                }
-            }
-        });
-
-    // await tx tasks
-    log::trace!("awaiting scan producers");
-    producers.await;
-    log::trace!("done awaiting scan producers");
-
-    // drop the current permit so the semaphore will allow another scan to proceed
-    drop(permit);
-
-    if let Ok(mut scan) = ferox_scan.lock() {
-        scan.finish();
-    }
-
-    // manually drop tx in order for the rx task's while loops to eval to false
-    log::trace!("dropped recursion handler's transmitter");
-    drop(tx_dir);
-
-    // await rx tasks
-    log::trace!("awaiting recursive scan receiver/scans");
-    futures::future::join_all(recurser.await.unwrap()).await;
-    log::trace!("done awaiting recursive scan receiver/scans");
-
-    log::trace!("exit: scan_url");
-}
-
-/// Perform steps necessary to run scans that only need to be performed once (warming up the
-/// engine, as it were)
-pub fn initialize(num_words: usize, config: &Configuration) {
-    log::trace!("enter: initialize({}, {:?})", num_words, config,);
-
-    // number of requests only needs to be calculated once, and then can be reused
-    let num_reqs_expected: u64 = if config.extensions.is_empty() {
-        num_words.try_into().unwrap()
-    } else {
-        let total = num_words * (config.extensions.len() + 1);
-        total.try_into().unwrap()
-    };
-
-    NUMBER_OF_REQUESTS.store(num_reqs_expected, Ordering::Relaxed);
-
-    // add any status code filters to `FILTERS` (-C|--filter-status)
-    for code_filter in &config.filter_status {
-        let filter = StatusCodeFilter {
-            filter_code: *code_filter,
-        };
-        let boxed_filter = Box::new(filter);
-        add_filter_to_list_of_ferox_filters(boxed_filter, FILTERS.clone());
-    }
-
-    // add any line count filters to `FILTERS` (-N|--filter-lines)
-    for lines_filter in &config.filter_line_count {
-        let filter = LinesFilter {
-            line_count: *lines_filter,
-        };
-        let boxed_filter = Box::new(filter);
-        add_filter_to_list_of_ferox_filters(boxed_filter, FILTERS.clone());
-    }
-
-    // add any line count filters to `FILTERS` (-W|--filter-words)
-    for words_filter in &config.filter_word_count {
-        let filter = WordsFilter {
-            word_count: *words_filter,
-        };
-        let boxed_filter = Box::new(filter);
-        add_filter_to_list_of_ferox_filters(boxed_filter, FILTERS.clone());
-    }
-
-    // add any line count filters to `FILTERS` (-S|--filter-size)
-    for size_filter in &config.filter_size {
-        let filter = SizeFilter {
-            content_length: *size_filter,
-        };
-        let boxed_filter = Box::new(filter);
-        add_filter_to_list_of_ferox_filters(boxed_filter, FILTERS.clone());
-    }
-
-    // add any regex filters to `FILTERS` (-X|--filter-regex)
-    for regex_filter in &config.filter_regex {
-        let raw = regex_filter;
-        let compiled = match Regex::new(&raw) {
-            Ok(regex) => regex,
-            Err(e) => {
-                log::error!("Invalid regular expression: {}", e);
-                #[cfg(test)]
-                panic!();
-                #[cfg(not(test))]
-                exit(1);
-            }
-        };
-
-        let filter = RegexFilter {
-            raw_string: raw.to_owned(),
-            compiled,
-        };
-        let boxed_filter = Box::new(filter);
-        add_filter_to_list_of_ferox_filters(boxed_filter, FILTERS.clone());
-    }
-
-    if config.scan_limit == 0 {
-        // scan_limit == 0 means no limit should be imposed... however, scoping the Semaphore
-        // permit is tricky, so as a workaround, we'll add a ridiculous number of permits to
-        // the semaphore (1,152,921,504,606,846,975 to be exact) and call that 'unlimited'
-        SCAN_LIMITER.add_permits(usize::MAX >> 4);
-    }
-
-    log::trace!("exit: initialize");
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    /// sending url + word without any extensions should get back one url with the joined word
-    fn create_urls_no_extension_returns_base_url_with_word() {
-        let urls = create_urls("http://localhost", "turbo", &[]);
-        assert_eq!(urls, [Url::parse("http://localhost/turbo").unwrap()])
-    }
-
-    #[test]
-    /// sending url + word + 1 extension should get back two urls, one base and one with extension
-    fn create_urls_one_extension_returns_two_urls() {
-        let urls = create_urls("http://localhost", "turbo", &[String::from("js")]);
-        assert_eq!(
-            urls,
-            [
-                Url::parse("http://localhost/turbo").unwrap(),
-                Url::parse("http://localhost/turbo.js").unwrap()
-            ]
-        )
-    }
-
-    #[test]
-    /// sending url + word + multiple extensions should get back n+1 urls
-    fn create_urls_multiple_extensions_returns_n_plus_one_urls() {
-        let ext_vec = vec![
-            vec![String::from("js")],
-            vec![String::from("js"), String::from("php")],
-            vec![String::from("js"), String::from("php"), String::from("pdf")],
-            vec![
-                String::from("js"),
-                String::from("php"),
-                String::from("pdf"),
-                String::from("tar.gz"),
-            ],
-        ];
-
-        let base = Url::parse("http://localhost/turbo").unwrap();
-        let js = Url::parse("http://localhost/turbo.js").unwrap();
-        let php = Url::parse("http://localhost/turbo.php").unwrap();
-        let pdf = Url::parse("http://localhost/turbo.pdf").unwrap();
-        let tar = Url::parse("http://localhost/turbo.tar.gz").unwrap();
-
-        let expected = vec![
-            vec![base.clone(), js.clone()],
-            vec![base.clone(), js.clone(), php.clone()],
-            vec![base.clone(), js.clone(), php.clone(), pdf.clone()],
-            vec![base, js, php, pdf, tar],
-        ];
-
-        for (i, ext_set) in ext_vec.into_iter().enumerate() {
-            let urls = create_urls("http://localhost", "turbo", &ext_set);
-            assert_eq!(urls, expected[i]);
-        }
-    }
-
-    #[test]
-    /// call reached_max_depth with max depth of zero, which is infinite recursion, expect false
-    fn reached_max_depth_returns_early_on_zero() {
-        let url = Url::parse("http://localhost").unwrap();
-        let result = reached_max_depth(&url, 0, 0);
-        assert!(!result);
-    }
-
-    #[test]
-    /// call reached_max_depth with url depth equal to max depth, expect true
-    fn reached_max_depth_current_depth_equals_max() {
-        let url = Url::parse("http://localhost/one/two").unwrap();
-        let result = reached_max_depth(&url, 0, 2);
-        assert!(result);
-    }
-
-    #[test]
-    /// call reached_max_depth with url dpeth less than max depth, expect false
-    fn reached_max_depth_current_depth_less_than_max() {
-        let url = Url::parse("http://localhost").unwrap();
-        let result = reached_max_depth(&url, 0, 2);
-        assert!(!result);
-    }
-
-    #[test]
-    /// call reached_max_depth with url of 2, base depth of 2, and max depth of 2, expect false
-    fn reached_max_depth_base_depth_equals_max_depth() {
-        let url = Url::parse("http://localhost/one/two").unwrap();
-        let result = reached_max_depth(&url, 2, 2);
-        assert!(!result);
-    }
-
-    #[test]
-    /// call reached_max_depth with url depth greater than max depth, expect true
-    fn reached_max_depth_current_greater_than_max() {
-        let url = Url::parse("http://localhost/one/two/three").unwrap();
-        let result = reached_max_depth(&url, 0, 2);
-        assert!(result);
-    }
-
-    #[test]
-    #[should_panic]
-    /// call initialize with a bad regex, triggering a panic
-    fn initialize_panics_on_bad_regex() {
-        let mut config = Configuration::default();
-        config.filter_regex = vec![r"(".to_string()];
-        initialize(1, &config);
-    }
-}

src/scanner/ferox_scanner.rs

@@ -0,0 +1,186 @@
+use std::{ops::Deref, sync::atomic::Ordering, sync::Arc, time::Instant};
+
+use anyhow::{bail, Result};
+use futures::{stream, StreamExt};
+use lazy_static::lazy_static;
+use tokio::sync::Semaphore;
+
+use crate::{
+    event_handlers::{
+        Command::{AddError, AddToF64Field, SubtractFromUsizeField},
+        Handles,
+    },
+    extractor::{ExtractionTarget::RobotsTxt, ExtractorBuilder},
+    heuristics,
+    scan_manager::{FeroxResponses, ScanOrder, ScanStatus, PAUSE_SCAN},
+    statistics::{
+        StatError::Other,
+        StatField::{DirScanTimes, TotalExpected},
+    },
+    utils::fmt_err,
+};
+
+use super::requester::Requester;
+
+lazy_static! {
+    /// Vector of FeroxResponse objects
+    pub static ref RESPONSES: FeroxResponses = FeroxResponses::default();
+    // todo consider removing this
+}
+/// handles the main muscle movement of scanning a url
+pub struct FeroxScanner {
+    /// handles to handlers and config
+    pub(super) handles: Arc<Handles>,
+
+    /// url that will be scanned
+    pub(super) target_url: String,
+
+    /// whether or not this scanner is targeting an initial target specified by the user or one
+    /// found via recursion
+    order: ScanOrder,
+
+    /// wordlist that's already been read from disk
+    wordlist: Arc<Vec<String>>,
+
+    /// limiter that restricts the number of active FeroxScanners  
+    scan_limiter: Arc<Semaphore>,
+}
+
+/// FeroxScanner implementation
+impl FeroxScanner {
+    /// create a new FeroxScanner
+    pub fn new(
+        target_url: &str,
+        order: ScanOrder,
+        wordlist: Arc<Vec<String>>,
+        scan_limiter: Arc<Semaphore>,
+        handles: Arc<Handles>,
+    ) -> Self {
+        Self {
+            order,
+            handles,
+            wordlist,
+            scan_limiter,
+            target_url: target_url.to_string(),
+        }
+    }
+
+    /// Scan a given url using a given wordlist
+    ///
+    /// This is the primary entrypoint for the scanner
+    pub async fn scan_url(&self) -> Result<()> {
+        log::trace!("enter: scan_url");
+        log::info!("Starting scan against: {}", self.target_url);
+
+        let scan_timer = Instant::now();
+
+        if matches!(self.order, ScanOrder::Initial) && self.handles.config.extract_links {
+            // only grab robots.txt on the initial scan_url calls. all fresh dirs will be passed
+            // to try_recursion
+            let extractor = ExtractorBuilder::default()
+                .url(&self.target_url)
+                .handles(self.handles.clone())
+                .target(RobotsTxt)
+                .build()?;
+
+            let links = extractor.extract().await?;
+            extractor.request_links(links).await?;
+        }
+
+        let scanned_urls = self.handles.ferox_scans()?;
+
+        let ferox_scan = match scanned_urls.get_scan_by_url(&self.target_url) {
+            Some(scan) => {
+                scan.set_status(ScanStatus::Running)?;
+                scan
+            }
+            None => {
+                let msg = format!(
+                    "Could not find FeroxScan associated with {}; this shouldn't happen... exiting",
+                    self.target_url
+                );
+                bail!(fmt_err(&msg))
+            }
+        };
+
+        let progress_bar = ferox_scan.progress_bar();
+
+        // When acquire is called and the semaphore has remaining permits, the function immediately
+        // returns a permit. However, if no remaining permits are available, acquire (asynchronously)
+        // waits until an outstanding permit is dropped, at which point, the freed permit is assigned
+        // to the caller.
+        let _permit = self.scan_limiter.acquire().await;
+
+        // Arc clones to be passed around to the various scans
+        let looping_words = self.wordlist.clone();
+
+        {
+            let test = heuristics::HeuristicTests::new(self.handles.clone());
+            if let Ok(num_reqs) = test.wildcard(&self.target_url).await {
+                progress_bar.inc(num_reqs);
+            }
+        }
+
+        let requester = Arc::new(Requester::from(self, ferox_scan.clone())?);
+        let increment_len = (self.handles.config.extensions.len() + 1) as u64;
+
+        // producer tasks (mp of mpsc); responsible for making requests
+        let producers = stream::iter(looping_words.deref().to_owned())
+            .map(|word| {
+                let pb = progress_bar.clone(); // progress bar is an Arc around internal state
+                let scanned_urls_clone = scanned_urls.clone();
+                let requester_clone = requester.clone();
+                let handles_clone = self.handles.clone();
+                (
+                    tokio::spawn(async move {
+                        if PAUSE_SCAN.load(Ordering::Acquire) {
+                            // for every word in the wordlist, check to see if PAUSE_SCAN is set to true
+                            // when true; enter a busy loop that only exits by setting PAUSE_SCAN back
+                            // to false
+                            let num_cancelled = scanned_urls_clone.pause(true).await;
+                            if num_cancelled > 0 {
+                                handles_clone
+                                    .stats
+                                    .send(SubtractFromUsizeField(TotalExpected, num_cancelled))
+                                    .unwrap_or_else(|e| {
+                                        log::warn!("Could not update overall scan bar: {}", e)
+                                    });
+                            }
+                        }
+                        requester_clone
+                            .request(&word)
+                            .await
+                            .unwrap_or_else(|e| log::warn!("Requester encountered an error: {}", e))
+                    }),
+                    pb,
+                )
+            })
+            .for_each_concurrent(self.handles.config.threads, |(resp, bar)| async move {
+                match resp.await {
+                    Ok(_) => {
+                        bar.inc(increment_len);
+                    }
+                    Err(e) => {
+                        log::warn!("error awaiting a response: {}", e);
+                        self.handles.stats.send(AddError(Other)).unwrap_or_default();
+                    }
+                }
+            });
+
+        // await tx tasks
+        log::trace!("awaiting scan producers");
+        producers.await;
+        log::trace!("done awaiting scan producers");
+
+        self.handles.stats.send(AddToF64Field(
+            DirScanTimes,
+            scan_timer.elapsed().as_secs_f64(),
+        ))?;
+
+        ferox_scan.finish()?;
+
+        log::trace!("exit: scan_url");
+
+        Ok(())
+    }
+}

src/scanner/init.rs

@@ -0,0 +1,34 @@
+use crate::{
+    event_handlers::{Command::AddToUsizeField, Handles},
+    statistics::StatField::ExpectedPerScan,
+};
+use anyhow::Result;
+use std::{convert::TryInto, sync::Arc};
+
+/// Perform steps necessary to run scans that only need to be performed once (warming up the
+/// engine, as it were)
+pub async fn initialize(num_words: usize, handles: Arc<Handles>) -> Result<()> {
+    log::trace!("enter: initialize({}, {:?})", num_words, handles);
+
+    // number of requests only needs to be calculated once, and then can be reused
+    let num_reqs_expected: u64 = if handles.config.extensions.is_empty() {
+        num_words.try_into()?
+    } else {
+        let total = num_words * (handles.config.extensions.len() + 1);
+        total.try_into()?
+    };
+
+    {
+        // no real reason to keep the arc around beyond this call
+        let scans = handles.ferox_scans()?;
+        scans.set_bar_length(num_reqs_expected);
+    }
+
+    // tell Stats object about the number of expected requests
+    handles
+        .stats
+        .send(AddToUsizeField(ExpectedPerScan, num_reqs_expected as usize))?;
+
+    log::trace!("exit: initialize");
+    Ok(())
+}

src/scanner/limit_heap.rs

@@ -0,0 +1,171 @@
+use std::fmt::{Debug, Formatter, Result};
+
+/// bespoke variation on an array-backed max-heap
+///
+/// 255 possible values generated from the initial requests/second
+///
+/// when no additional errors are encountered, the left child is taken (increasing req/sec)
+/// if errors have increased since the last interval, the right child is taken (decreasing req/sec)
+///
+/// formula for each child:
+/// - left: (|parent - current|) / 2 + current
+/// - right: current - ((|parent - current|) / 2)
+pub(super) struct LimitHeap {
+    /// backing array, 255 nodes == height of 7 ( 2^(h+1) -1 nodes )
+    pub(super) inner: [i32; 255],
+
+    /// original # of requests / second
+    pub(super) original: i32,
+
+    /// current position w/in the backing array
+    pub(super) current: usize,
+}
+
+/// default implementation of a LimitHeap
+impl Default for LimitHeap {
+    /// zero-initialize the backing array
+    fn default() -> Self {
+        Self {
+            inner: [0; 255],
+            original: 0,
+            current: 0,
+        }
+    }
+}
+
+/// Debug implementation of a LimitHeap
+impl Debug for LimitHeap {
+    /// return debug representation that conforms to <32 elements in array
+    fn fmt(&self, f: &mut Formatter<'_>) -> Result {
+        let msg = format!(
+            "LimitHeap {{ original: {}, current: {}, inner: [{}...] }}",
+            self.original, self.current, self.inner[0]
+        );
+        write!(f, "{}", msg)
+    }
+}
+
+/// implementation of a LimitHeap
+impl LimitHeap {
+    /// move to right child, return node's index from which the move was requested
+    pub(super) fn move_right(&mut self) -> usize {
+        if self.has_children() {
+            let tmp = self.current;
+            self.current = self.current * 2 + 2;
+            return tmp;
+        }
+        self.current
+    }
+
+    /// move to left child, return node's index from which the move was requested
+    pub(super) fn move_left(&mut self) -> usize {
+        if self.has_children() {
+            let tmp = self.current;
+            self.current = self.current * 2 + 1;
+            return tmp;
+        }
+        self.current
+    }
+
+    /// move to parent, return node's index from which the move was requested
+    pub(super) fn move_up(&mut self) -> usize {
+        if self.has_parent() {
+            let tmp = self.current;
+            self.current = (self.current - 1) / 2;
+            return tmp;
+        }
+        self.current
+    }
+
+    /// move directly to the given index
+    pub(super) fn move_to(&mut self, index: usize) {
+        self.current = index;
+    }
+
+    /// get the current node's value
+    pub(super) fn value(&self) -> i32 {
+        self.inner[self.current]
+    }
+
+    /// set the current node's value
+    pub(super) fn set_value(&mut self, value: i32) {
+        self.inner[self.current] = value;
+    }
+
+    /// check that this node has a parent (true for all except root)
+    pub(super) fn has_parent(&self) -> bool {
+        self.current > 0
+    }
+
+    /// get node's parent's value or self.original if at the root
+    pub(super) fn parent_value(&mut self) -> i32 {
+        if self.has_parent() {
+            let current = self.move_up();
+            let val = self.value();
+            self.move_to(current);
+            return val;
+        }
+        self.original
+    }
+
+    /// check if the current node has children
+    pub(super) fn has_children(&self) -> bool {
+        // inner structure is a complete tree, just check for the right child
+        self.current * 2 + 2 <= self.inner.len()
+    }
+
+    /// get current node's right child's value
+    fn right_child_value(&mut self) -> i32 {
+        let tmp = self.move_right();
+        let val = self.value();
+        self.move_to(tmp);
+        val
+    }
+
+    /// set current node's left child's value
+    fn set_left_child(&mut self) {
+        let parent = self.parent_value();
+        let current = self.value();
+        let value = ((parent - current).abs() / 2) + current;
+
+        self.move_left();
+        self.set_value(value);
+        self.move_up();
+    }
+
+    /// set current node's right child's value
+    fn set_right_child(&mut self) {
+        let parent = self.parent_value();
+        let current = self.value();
+        let value = current - ((parent - current).abs() / 2);
+
+        self.move_right();
+        self.set_value(value);
+        self.move_up();
+    }
+
+    /// iterate over the backing array, filling in each child's value based on the original value
+    pub(super) fn build(&mut self) {
+        // ex: original is 400
+        // arr[0] == 200
+        // arr[1] (left child) == 300
+        // arr[2] (right child) == 100
+        let root = self.original / 2;
+
+        self.inner[0] = root; // set root node to half of the original value
+        self.inner[1] = ((self.original - root).abs() / 2) + root;
+        self.inner[2] = root - ((self.original - root).abs() / 2);
+
+        // start with index 1 and fill in each child below that node
+        for i in 1..self.inner.len() {
+            self.move_to(i);
+
+            if self.has_children() && self.right_child_value() == 0 {
+                // this node has an unset child since the rchild is 0
+                self.set_left_child();
+                self.set_right_child();
+            }
+        }
+        self.move_to(0); // reset current index to the root of the tree
+    }
+}

src/scanner/mod.rs

@@ -0,0 +1,12 @@
+mod ferox_scanner;
+mod utils;
+mod init;
+#[cfg(test)]
+mod tests;
+mod limit_heap;
+mod policy_data;
+mod requester;
+
+pub use self::ferox_scanner::{FeroxScanner, RESPONSES};
+pub use self::init::initialize;
+pub use self::utils::PolicyTrigger;

src/scanner/policy_data.rs

@@ -0,0 +1,306 @@
+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+
+use crate::{atomic_load, atomic_store, config::RequesterPolicy};
+
+use super::limit_heap::LimitHeap;
+
+/// data regarding policy and metadata about last enforced trigger etc...
+#[derive(Default, Debug)]
+pub struct PolicyData {
+    /// how to handle exceptional cases such as too many errors / 403s / 429s etc
+    pub(super) policy: RequesterPolicy,
+
+    /// whether or not we're in the middle of a cooldown period
+    pub(super) cooling_down: AtomicBool,
+
+    /// length of time to pause tuning after making an adjustment
+    pub(super) wait_time: u64,
+
+    /// rate limit (at last interval)
+    limit: AtomicUsize,
+
+    /// number of errors (at last interval)
+    pub(super) errors: AtomicUsize,
+
+    /// whether or not the owning Requester should remove the rate_limiter, happens when a scan
+    /// has been limited and moves back up to the point of its original scan speed
+    pub(super) remove_limit: AtomicBool,
+
+    /// heap of values used for adjusting # of requests/second
+    pub(super) heap: std::sync::RwLock<LimitHeap>,
+}
+
+/// implementation of PolicyData
+impl PolicyData {
+    /// given a RequesterPolicy, create a new PolicyData
+    pub fn new(policy: RequesterPolicy, timeout: u64) -> Self {
+        // can use this as a tweak for how aggressively adjustments should be made when tuning
+        let wait_time = ((timeout as f64 / 2.0) * 1000.0) as u64;
+
+        Self {
+            policy,
+            wait_time,
+            ..Default::default()
+        }
+    }
+
+    /// setter for requests / second; populates the underlying heap with values from req/sec seed
+    pub(super) fn set_reqs_sec(&self, reqs_sec: usize) {
+        if let Ok(mut guard) = self.heap.write() {
+            guard.original = reqs_sec as i32;
+            guard.build();
+            self.set_limit(guard.inner[0] as usize); // set limit to 1/2 of current request rate
+        }
+    }
+
+    /// setter for errors
+    pub(super) fn set_errors(&self, errors: usize) {
+        atomic_store!(self.errors, errors);
+    }
+
+    /// setter for limit
+    fn set_limit(&self, limit: usize) {
+        atomic_store!(self.limit, limit);
+    }
+
+    /// getter for limit
+    pub(super) fn get_limit(&self) -> usize {
+        atomic_load!(self.limit)
+    }
+
+    /// adjust the rate of requests per second up (increase rate)
+    pub(super) fn adjust_up(&self, streak_counter: &usize) {
+        if let Ok(mut heap) = self.heap.try_write() {
+            if *streak_counter > 2 {
+                // streak of 3 upward moves in a row, traverse the tree upward instead of to a
+                // higher-valued branch lower in the tree
+                let current = heap.value();
+                heap.move_up();
+                heap.move_up();
+                if current > heap.value() {
+                    // the tree's structure makes it so that sometimes 2 moves up results in a
+                    // value greater than the current node's and other times we need to move 3 up
+                    // to arrive at a greater value
+                    if heap.has_parent() && heap.parent_value() > current {
+                        // all nodes except 0th node (root)
+                        heap.move_up();
+                    } else if !heap.has_parent() {
+                        // been here enough that we can try resuming the scan to its original
+                        // speed (no limiting at all)
+                        atomic_store!(self.remove_limit, true);
+                    }
+                }
+            } else if heap.has_children() {
+                // streak not at 3, just check that we can move down, and do so
+                heap.move_left();
+            } else {
+                // tree bottomed out, need to move back up the tree a bit
+                let current = heap.value();
+                heap.move_up();
+                heap.move_up();
+
+                if current > heap.value() {
+                    heap.move_up();
+                }
+            }
+            self.set_limit(heap.value() as usize);
+        }
+    }
+
+    /// adjust the rate of requests per second down (decrease rate)
+    pub(super) fn adjust_down(&self) {
+        if let Ok(mut heap) = self.heap.try_write() {
+            if heap.has_children() {
+                heap.move_right();
+                self.set_limit(heap.value() as usize);
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    /// PolicyData builds and sets correct values for the inner heap when set_reqs_sec is called
+    fn set_reqs_sec_builds_heap_and_sets_initial_value() {
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        assert_eq!(pd.wait_time, 3500);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+        assert_eq!(pd.heap.read().unwrap().original, 400);
+        assert_eq!(pd.heap.read().unwrap().current, 0);
+        assert_eq!(pd.heap.read().unwrap().inner[0], 200);
+        assert_eq!(pd.heap.read().unwrap().inner[1], 300);
+        assert_eq!(pd.heap.read().unwrap().inner[2], 100);
+    }
+
+    #[test]
+    /// PolicyData setters/getters tests for code coverage / sanity
+    fn policy_data_getters_and_setters() {
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_errors(20);
+        assert_eq!(pd.errors.load(Ordering::Relaxed), 20);
+        pd.set_limit(200);
+        assert_eq!(pd.get_limit(), 200);
+    }
+
+    #[test]
+    /// PolicyData adjust_down sets the limit to the correct value
+    fn policy_data_adjust_down_simple() {
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+        pd.adjust_down();
+        assert_eq!(pd.get_limit(), 100);
+    }
+
+    #[test]
+    /// PolicyData adjust_down sets the limit to the correct value when no child nodes are present
+    fn policy_data_adjust_down_no_children() {
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+        let mut guard = pd.heap.write().unwrap();
+        guard.move_to(250);
+        guard.set_value(27);
+        pd.set_limit(guard.value() as usize);
+        drop(guard);
+
+        pd.adjust_down();
+        assert_eq!(pd.get_limit(), 27);
+    }
+
+    #[test]
+    /// PolicyData adjust_up sets the limit to the correct value
+    fn policy_data_adjust_up_simple() {
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+        pd.adjust_up(&0);
+        assert_eq!(pd.get_limit(), 300);
+    }
+
+    #[test]
+    /// PolicyData adjust_up sets the limit to the correct value
+    fn policy_data_adjust_up_with_streak_and_2_moves() {
+        // original: 400
+        // [200, 300, 100, 350, 250, 150, 50, 375, 325, 275, 225, 175, 125, 75, 25, ...]
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+
+        // 2 moves
+        pd.heap.write().unwrap().move_to(9);
+        assert_eq!(pd.heap.read().unwrap().value(), 275);
+        pd.adjust_up(&3);
+        assert_eq!(pd.heap.read().unwrap().value(), 300);
+        assert_eq!(pd.limit.load(Ordering::Relaxed), 300);
+        assert!(!pd.remove_limit.load(Ordering::Relaxed));
+    }
+
+    #[test]
+    /// PolicyData adjust_up sets the limit to the correct value
+    fn policy_data_adjust_up_with_streak_and_2_moves_to_arrive_at_root() {
+        // original: 400
+        // [200, 300, 100, 350, 250, 150, 50, 375, 325, 275, 225, 175, 125, 75, 25, ...]
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+
+        pd.heap.write().unwrap().move_to(4);
+        assert_eq!(pd.heap.read().unwrap().value(), 250);
+        pd.adjust_up(&3);
+        assert_eq!(pd.heap.read().unwrap().value(), 200);
+        assert_eq!(pd.limit.load(Ordering::Relaxed), 200);
+        assert!(pd.remove_limit.load(Ordering::Relaxed));
+    }
+
+    #[test]
+    /// PolicyData adjust_up sets the limit to the correct value
+    fn policy_data_adjust_up_with_streak_and_2_moves_to_find_less_than_current() {
+        // original: 400
+        // [200, 300, 100, 350, 250, 150, 50, 375, 325, 275, 225, 175, 125, 75, 25, ...]
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+
+        pd.heap.write().unwrap().move_to(15);
+        assert_eq!(pd.heap.read().unwrap().value(), 387);
+        pd.adjust_up(&3);
+        assert_eq!(pd.heap.read().unwrap().value(), 350);
+        assert_eq!(pd.limit.load(Ordering::Relaxed), 350);
+        assert!(!pd.remove_limit.load(Ordering::Relaxed));
+    }
+
+    #[test]
+    /// PolicyData adjust_up sets the limit to the correct value
+    fn policy_data_adjust_up_with_streak_and_3_moves() {
+        // original: 400
+        // [200, 300, 100, 350, 250, 150, 50, 375, 325, 275, 225, 175, 125, 75, 25, ...]
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+
+        pd.heap.write().unwrap().move_to(19);
+        assert_eq!(pd.heap.read().unwrap().value(), 287);
+        pd.adjust_up(&3);
+        assert_eq!(pd.heap.read().unwrap().value(), 300);
+        assert_eq!(pd.limit.load(Ordering::Relaxed), 300);
+        assert!(!pd.remove_limit.load(Ordering::Relaxed));
+    }
+
+    #[test]
+    /// PolicyData adjust_up sets the limit to the correct value
+    fn policy_data_adjust_up_with_no_children_2_moves() {
+        // original: 400
+        // [200, 300, 100, 350, 250, 150, 50, 375, 325, 275, 225, 175, 125, 75, 25, ...]
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+
+        pd.heap.write().unwrap().move_to(241);
+
+        assert_eq!(pd.heap.read().unwrap().value(), 41);
+        pd.adjust_up(&0);
+        assert_eq!(pd.heap.read().unwrap().value(), 43);
+        assert_eq!(pd.limit.load(Ordering::Relaxed), 43);
+        assert!(!pd.remove_limit.load(Ordering::Relaxed));
+    }
+
+    #[test]
+    /// PolicyData adjust_up sets the limit to the correct value
+    fn policy_data_adjust_up_with_no_children_3_moves() {
+        // original: 400
+        // [200, 300, 100, 350, 250, 150, 50, 375, 325, 275, 225, 175, 125, 75, 25, ...]
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+
+        pd.heap.write().unwrap().move_to(240);
+
+        assert_eq!(pd.heap.read().unwrap().value(), 45);
+        pd.adjust_up(&0);
+        assert_eq!(pd.heap.read().unwrap().value(), 37);
+        assert_eq!(pd.limit.load(Ordering::Relaxed), 37);
+        assert!(!pd.remove_limit.load(Ordering::Relaxed));
+    }
+
+    #[test]
+    /// hit some of the out of the way corners of limitheap for coverage
+    fn increase_limit_heap_coverage_by_hitting_edge_cases() {
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+
+        println!("{:?}", pd.heap.read().unwrap()); // debug derivation
+
+        pd.heap.write().unwrap().move_to(240);
+        assert_eq!(pd.heap.write().unwrap().move_right(), 240);
+        assert_eq!(pd.heap.write().unwrap().move_left(), 240);
+
+        pd.heap.write().unwrap().move_to(0);
+        assert_eq!(pd.heap.write().unwrap().move_up(), 0);
+        assert_eq!(pd.heap.write().unwrap().parent_value(), 400);
+    }
+}

src/scanner/tests.rs

@@ -0,0 +1,28 @@
+use std::sync::Arc;
+
+use tokio::sync::Semaphore;
+
+use crate::{
+    config::OutputLevel,
+    event_handlers::Handles,
+    scan_manager::{FeroxScans, ScanOrder},
+};
+
+use super::*;
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+#[should_panic]
+/// try to hit struct field coverage of FileOutHandler
+async fn get_scan_by_url_bails_on_unfound_url() {
+    let sem = Semaphore::new(10);
+    let urls = FeroxScans::new(OutputLevel::Default);
+
+    let scanner = FeroxScanner::new(
+        "http://localhost",
+        ScanOrder::Initial,
+        Arc::new(Default::default()),
+        Arc::new(sem),
+        Arc::new(Handles::for_testing(Some(Arc::new(urls)), None).0),
+    );
+    scanner.scan_url().await.unwrap();
+}

src/scanner/utils.rs

@@ -0,0 +1,12 @@
+#[derive(Copy, Clone, PartialEq, Debug)]
+/// represents different situations where different criteria can trigger auto-tune/bail behavior
+pub enum PolicyTrigger {
+    /// excessive 403 trigger
+    Status403,
+
+    /// excessive 429 trigger
+    Status429,
+
+    /// excessive general errors
+    Errors,
+}

src/statistics/container.rs

@@ -0,0 +1,918 @@
+use std::{
+    collections::HashMap,
+    convert::TryFrom,
+    fs::File,
+    io::BufReader,
+    sync::{
+        atomic::{AtomicUsize, Ordering},
+        Mutex,
+    },
+};
+
+use anyhow::{Context, Result};
+use reqwest::StatusCode;
+use serde::{ser::SerializeStruct, Deserialize, Deserializer, Serialize, Serializer};
+use serde_json::Value;
+
+use crate::{
+    traits::FeroxSerialize,
+    utils::{fmt_err, open_file, write_to},
+};
+
+use super::{error::StatError, field::StatField};
+
+/// Data collection of statistics related to a scan
+#[derive(Default, Debug)]
+pub struct Stats {
+    /// Name of this type of struct, used for serialization, i.e. `{"type":"statistics"}`
+    kind: String,
+
+    /// tracker for number of timeouts seen by the client
+    timeouts: AtomicUsize,
+
+    /// tracker for total number of requests sent by the client
+    pub(crate) requests: AtomicUsize,
+
+    /// tracker for total number of requests expected to send if the scan runs to completion
+    ///
+    /// Note: this is a per-scan expectation; `expected_requests * current # of scans` would be
+    /// indicative of the current expectation at any given time, but is a moving target.  
+    expected_per_scan: AtomicUsize,
+
+    /// tracker for accumulating total number of requests expected (i.e. as a new scan is started
+    /// this value should increase by `expected_requests`
+    total_expected: AtomicUsize,
+
+    /// tracker for total number of errors encountered by the client
+    pub(crate) errors: AtomicUsize,
+
+    /// tracker for overall number of 2xx status codes seen by the client
+    successes: AtomicUsize,
+
+    /// tracker for overall number of 3xx status codes seen by the client
+    redirects: AtomicUsize,
+
+    /// tracker for overall number of 4xx status codes seen by the client
+    client_errors: AtomicUsize,
+
+    /// tracker for overall number of 5xx status codes seen by the client
+    server_errors: AtomicUsize,
+
+    /// tracker for number of scans performed, this directly equates to number of directories
+    /// recursed into and affects the total number of expected requests
+    pub(crate) total_scans: AtomicUsize,
+
+    /// tracker for initial number of requested targets
+    initial_targets: AtomicUsize,
+
+    /// tracker for number of links extracted when `--extract-links` is used; sources are
+    /// response bodies and robots.txt as of v1.11.0
+    links_extracted: AtomicUsize,
+
+    /// tracker for overall number of 200s seen by the client
+    status_200s: AtomicUsize,
+
+    /// tracker for overall number of 301s seen by the client
+    status_301s: AtomicUsize,
+
+    /// tracker for overall number of 302s seen by the client
+    status_302s: AtomicUsize,
+
+    /// tracker for overall number of 401s seen by the client
+    status_401s: AtomicUsize,
+
+    /// tracker for overall number of 403s seen by the client
+    pub(crate) status_403s: AtomicUsize,
+
+    /// tracker for overall number of 429s seen by the client
+    pub(crate) status_429s: AtomicUsize,
+
+    /// tracker for overall number of 500s seen by the client
+    status_500s: AtomicUsize,
+
+    /// tracker for overall number of 503s seen by the client
+    status_503s: AtomicUsize,
+
+    /// tracker for overall number of 504s seen by the client
+    status_504s: AtomicUsize,
+
+    /// tracker for overall number of 508s seen by the client
+    status_508s: AtomicUsize,
+
+    /// tracker for overall number of wildcard urls filtered out by the client
+    wildcards_filtered: AtomicUsize,
+
+    /// tracker for overall number of all filtered responses
+    responses_filtered: AtomicUsize,
+
+    /// tracker for number of files found
+    resources_discovered: AtomicUsize,
+
+    /// tracker for number of errors triggered during URL formatting
+    url_format_errors: AtomicUsize,
+
+    /// tracker for number of errors triggered by the `reqwest::RedirectPolicy`
+    redirection_errors: AtomicUsize,
+
+    /// tracker for number of errors related to the connecting
+    connection_errors: AtomicUsize,
+
+    /// tracker for number of errors related to the request used
+    request_errors: AtomicUsize,
+
+    /// tracker for each directory's total scan time in seconds as a float
+    directory_scan_times: Mutex<Vec<f64>>,
+
+    /// tracker for total runtime
+    total_runtime: Mutex<Vec<f64>>,
+
+    /// tracker for the number of extensions the user specified
+    num_extensions: usize,
+
+    /// tracker for whether to use json during serialization or not
+    json: bool,
+}
+
+/// FeroxSerialize implementation for Stats
+impl FeroxSerialize for Stats {
+    /// Simply return empty string here to disable serializing this to the output file as a string
+    /// due to it looking like garbage
+    fn as_str(&self) -> String {
+        String::new()
+    }
+
+    /// Simple call to produce a JSON string using the given Stats object
+    fn as_json(&self) -> Result<String> {
+        Ok(serde_json::to_string(&self)?)
+    }
+}
+
+/// Serialize implementation for Stats
+impl Serialize for Stats {
+    /// Function that handles serialization of Stats
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let mut state = serializer.serialize_struct("Stats", 32)?;
+
+        state.serialize_field("type", &self.kind)?;
+        state.serialize_field("timeouts", &atomic_load!(self.timeouts))?;
+        state.serialize_field("requests", &atomic_load!(self.requests))?;
+        state.serialize_field("expected_per_scan", &atomic_load!(self.expected_per_scan))?;
+        state.serialize_field("total_expected", &atomic_load!(self.total_expected))?;
+        state.serialize_field("errors", &atomic_load!(self.errors))?;
+        state.serialize_field("successes", &atomic_load!(self.successes))?;
+        state.serialize_field("redirects", &atomic_load!(self.redirects))?;
+        state.serialize_field("client_errors", &atomic_load!(self.client_errors))?;
+        state.serialize_field("server_errors", &atomic_load!(self.server_errors))?;
+        state.serialize_field("total_scans", &atomic_load!(self.total_scans))?;
+        state.serialize_field("initial_targets", &atomic_load!(self.initial_targets))?;
+        state.serialize_field("links_extracted", &atomic_load!(self.links_extracted))?;
+        state.serialize_field("status_200s", &atomic_load!(self.status_200s))?;
+        state.serialize_field("status_301s", &atomic_load!(self.status_301s))?;
+        state.serialize_field("status_302s", &atomic_load!(self.status_302s))?;
+        state.serialize_field("status_401s", &atomic_load!(self.status_401s))?;
+        state.serialize_field("status_403s", &atomic_load!(self.status_403s))?;
+        state.serialize_field("status_429s", &atomic_load!(self.status_429s))?;
+        state.serialize_field("status_500s", &atomic_load!(self.status_500s))?;
+        state.serialize_field("status_503s", &atomic_load!(self.status_503s))?;
+        state.serialize_field("status_504s", &atomic_load!(self.status_504s))?;
+        state.serialize_field("status_508s", &atomic_load!(self.status_508s))?;
+        state.serialize_field("wildcards_filtered", &atomic_load!(self.wildcards_filtered))?;
+        state.serialize_field("responses_filtered", &atomic_load!(self.responses_filtered))?;
+        state.serialize_field(
+            "resources_discovered",
+            &atomic_load!(self.resources_discovered),
+        )?;
+        state.serialize_field("url_format_errors", &atomic_load!(self.url_format_errors))?;
+        state.serialize_field("redirection_errors", &atomic_load!(self.redirection_errors))?;
+        state.serialize_field("connection_errors", &atomic_load!(self.connection_errors))?;
+        state.serialize_field("request_errors", &atomic_load!(self.request_errors))?;
+        state.serialize_field("directory_scan_times", &self.directory_scan_times)?;
+        state.serialize_field("total_runtime", &self.total_runtime)?;
+
+        state.end()
+    }
+}
+
+/// Deserialize implementation for Stats
+impl<'a> Deserialize<'a> for Stats {
+    /// Deserialize a Stats object from a serde_json::Value
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'a>,
+    {
+        let stats = Self::new(0, false);
+
+        let map: HashMap<String, Value> = HashMap::deserialize(deserializer)?;
+
+        for (key, value) in &map {
+            match key.as_str() {
+                "timeouts" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.timeouts, parsed);
+                        }
+                    }
+                }
+                "requests" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.requests, parsed);
+                        }
+                    }
+                }
+                "expected_per_scan" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.expected_per_scan, parsed);
+                        }
+                    }
+                }
+                "total_expected" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.total_expected, parsed);
+                        }
+                    }
+                }
+                "errors" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.errors, parsed);
+                        }
+                    }
+                }
+                "successes" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.successes, parsed);
+                        }
+                    }
+                }
+                "redirects" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.redirects, parsed);
+                        }
+                    }
+                }
+                "client_errors" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.client_errors, parsed);
+                        }
+                    }
+                }
+                "server_errors" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.server_errors, parsed);
+                        }
+                    }
+                }
+                "total_scans" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.total_scans, parsed);
+                        }
+                    }
+                }
+                "initial_targets" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.initial_targets, parsed);
+                        }
+                    }
+                }
+                "links_extracted" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.links_extracted, parsed);
+                        }
+                    }
+                }
+                "status_200s" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.status_200s, parsed);
+                        }
+                    }
+                }
+                "status_301s" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.status_301s, parsed);
+                        }
+                    }
+                }
+                "status_302s" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.status_302s, parsed);
+                        }
+                    }
+                }
+                "status_401s" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.status_401s, parsed);
+                        }
+                    }
+                }
+                "status_403s" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.status_403s, parsed);
+                        }
+                    }
+                }
+                "status_429s" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.status_429s, parsed);
+                        }
+                    }
+                }
+                "status_500s" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.status_500s, parsed);
+                        }
+                    }
+                }
+                "status_503s" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.status_503s, parsed);
+                        }
+                    }
+                }
+                "status_504s" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.status_504s, parsed);
+                        }
+                    }
+                }
+                "status_508s" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.status_508s, parsed);
+                        }
+                    }
+                }
+                "wildcards_filtered" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.wildcards_filtered, parsed);
+                        }
+                    }
+                }
+                "responses_filtered" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.responses_filtered, parsed);
+                        }
+                    }
+                }
+                "resources_discovered" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.resources_discovered, parsed);
+                        }
+                    }
+                }
+                "url_format_errors" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.url_format_errors, parsed);
+                        }
+                    }
+                }
+                "redirection_errors" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.redirection_errors, parsed);
+                        }
+                    }
+                }
+                "connection_errors" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.connection_errors, parsed);
+                        }
+                    }
+                }
+                "request_errors" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.request_errors, parsed);
+                        }
+                    }
+                }
+                "directory_scan_times" => {
+                    if let Some(arr) = value.as_array() {
+                        for val in arr {
+                            if let Some(parsed) = val.as_f64() {
+                                if let Ok(mut guard) = stats.directory_scan_times.lock() {
+                                    guard.push(parsed)
+                                }
+                            }
+                        }
+                    }
+                }
+                "total_runtime" => {
+                    if let Some(arr) = value.as_array() {
+                        for val in arr {
+                            if let Some(parsed) = val.as_f64() {
+                                if let Ok(mut guard) = stats.total_runtime.lock() {
+                                    guard.push(parsed)
+                                }
+                            }
+                        }
+                    }
+                }
+                _ => {}
+            }
+        }
+
+        Ok(stats)
+    }
+}
+
+/// implementation of statistics data collection struct
+impl Stats {
+    /// Small wrapper for default to set `kind` to "statistics" and `total_runtime` to have at least
+    /// one value
+    pub fn new(num_extensions: usize, is_json: bool) -> Self {
+        Self {
+            num_extensions,
+            json: is_json,
+            kind: String::from("statistics"),
+            total_runtime: Mutex::new(vec![0.0]),
+            ..Default::default()
+        }
+    }
+
+    /// public getter for expected_per_scan
+    pub fn expected_per_scan(&self) -> usize {
+        atomic_load!(self.expected_per_scan)
+    }
+
+    /// public getter for resources_discovered
+    pub fn resources_discovered(&self) -> usize {
+        atomic_load!(self.resources_discovered)
+    }
+
+    /// public getter for errors
+    pub fn errors(&self) -> usize {
+        atomic_load!(self.errors)
+    }
+
+    /// public getter for status_403s
+    pub fn status_403s(&self) -> usize {
+        atomic_load!(self.status_403s)
+    }
+
+    /// public getter for status_429s
+    pub fn status_429s(&self) -> usize {
+        atomic_load!(self.status_429s)
+    }
+
+    /// public getter for total_expected
+    pub fn total_expected(&self) -> usize {
+        atomic_load!(self.total_expected)
+    }
+
+    /// public getter for initial_targets
+    pub fn initial_targets(&self) -> usize {
+        atomic_load!(self.initial_targets)
+    }
+
+    /// increment `requests` field by one
+    pub fn add_request(&self) {
+        atomic_increment!(self.requests);
+    }
+
+    /// given an `Instant` update total runtime
+    fn update_runtime(&self, seconds: f64) {
+        if let Ok(mut runtime) = self.total_runtime.lock() {
+            runtime[0] = seconds;
+        }
+    }
+
+    /// save an instance of `Stats` to disk after updating the total runtime for the scan
+    pub fn save(&self, seconds: f64, location: &str) -> Result<()> {
+        let mut file = open_file(location)?;
+
+        self.update_runtime(seconds);
+
+        write_to(self, &mut file, self.json)?;
+
+        Ok(())
+    }
+
+    /// Inspect the given `StatError` and increment the appropriate fields
+    ///
+    /// Implies incrementing:
+    ///     - requests
+    ///     - errors
+    pub fn add_error(&self, error: StatError) {
+        self.add_request();
+        atomic_increment!(self.errors);
+
+        match error {
+            StatError::Timeout => {
+                atomic_increment!(self.timeouts);
+            }
+            StatError::UrlFormat => {
+                atomic_increment!(self.url_format_errors);
+            }
+            StatError::Redirection => {
+                atomic_increment!(self.redirection_errors);
+            }
+            StatError::Connection => {
+                atomic_increment!(self.connection_errors);
+            }
+            StatError::Request => {
+                atomic_increment!(self.request_errors);
+            }
+            _ => {} // no need to hit Other as we always increment self.errors anyway
+        }
+    }
+
+    /// Inspect the given `StatusCode` and increment the appropriate fields
+    ///
+    /// Implies incrementing:
+    ///     - requests
+    ///     - appropriate status_* codes
+    ///     - errors (when code is [45]xx)
+    pub fn add_status_code(&self, status: StatusCode) {
+        self.add_request();
+
+        if status.is_success() {
+            atomic_increment!(self.successes);
+        } else if status.is_redirection() {
+            atomic_increment!(self.redirects);
+        } else if status.is_client_error() {
+            atomic_increment!(self.client_errors);
+        } else if status.is_server_error() {
+            atomic_increment!(self.server_errors);
+        }
+
+        match status {
+            StatusCode::OK => {
+                atomic_increment!(self.status_200s);
+            }
+            StatusCode::MOVED_PERMANENTLY => {
+                atomic_increment!(self.status_301s);
+            }
+            StatusCode::FOUND => {
+                atomic_increment!(self.status_302s);
+            }
+            StatusCode::UNAUTHORIZED => {
+                atomic_increment!(self.status_401s);
+            }
+            StatusCode::FORBIDDEN => {
+                atomic_increment!(self.status_403s);
+            }
+            StatusCode::TOO_MANY_REQUESTS => {
+                atomic_increment!(self.status_429s);
+            }
+            StatusCode::INTERNAL_SERVER_ERROR => {
+                atomic_increment!(self.status_500s);
+            }
+            StatusCode::SERVICE_UNAVAILABLE => {
+                atomic_increment!(self.status_503s);
+            }
+            StatusCode::GATEWAY_TIMEOUT => {
+                atomic_increment!(self.status_504s);
+            }
+            StatusCode::LOOP_DETECTED => {
+                atomic_increment!(self.status_508s);
+            }
+            _ => {} // other status codes ignored for stat gathering
+        }
+    }
+
+    /// Update a `Stats` field of type f64
+    pub fn update_f64_field(&self, field: StatField, value: f64) {
+        if let StatField::DirScanTimes = field {
+            if let Ok(mut locked_times) = self.directory_scan_times.lock() {
+                locked_times.push(value);
+            }
+        }
+    }
+
+    /// subtract a value from the given field
+    pub fn subtract_from_usize_field(&self, field: StatField, value: usize) {
+        if let StatField::TotalExpected = field {
+            self.total_expected.fetch_sub(value, Ordering::Relaxed);
+        }
+    }
+
+    /// Update a `Stats` field of type usize
+    pub fn update_usize_field(&self, field: StatField, value: usize) {
+        match field {
+            StatField::ExpectedPerScan => {
+                atomic_increment!(self.expected_per_scan, value);
+            }
+            StatField::TotalScans => {
+                atomic_increment!(self.total_scans, value);
+                atomic_increment!(
+                    self.total_expected,
+                    value * self.expected_per_scan.load(Ordering::Relaxed)
+                );
+            }
+            StatField::TotalExpected => {
+                atomic_increment!(self.total_expected, value);
+            }
+            StatField::LinksExtracted => {
+                atomic_increment!(self.links_extracted, value);
+            }
+            StatField::WildcardsFiltered => {
+                atomic_increment!(self.wildcards_filtered, value);
+                atomic_increment!(self.responses_filtered, value);
+            }
+            StatField::ResponsesFiltered => {
+                atomic_increment!(self.responses_filtered, value);
+            }
+            StatField::ResourcesDiscovered => {
+                atomic_increment!(self.resources_discovered, value);
+            }
+            StatField::InitialTargets => {
+                atomic_increment!(self.initial_targets, value);
+            }
+            _ => {} // f64 fields
+        }
+    }
+
+    /// Merge a given `Stats` object from a json entry written to disk when handling a Ctrl+c
+    ///
+    /// This is only ever called when resuming a scan from disk
+    pub fn merge_from(&self, filename: &str) -> Result<()> {
+        let file = File::open(filename)
+            .with_context(|| fmt_err(&format!("Could not open {}", filename)))?;
+        let reader = BufReader::new(file);
+        let state: serde_json::Value = serde_json::from_reader(reader)?;
+
+        if let Some(state_stats) = state.get("statistics") {
+            let d_stats = serde_json::from_value::<Stats>(state_stats.clone())?;
+            atomic_increment!(self.successes, atomic_load!(d_stats.successes));
+            atomic_increment!(self.timeouts, atomic_load!(d_stats.timeouts));
+            atomic_increment!(self.requests, atomic_load!(d_stats.requests));
+            atomic_increment!(self.errors, atomic_load!(d_stats.errors));
+            atomic_increment!(self.redirects, atomic_load!(d_stats.redirects));
+            atomic_increment!(self.client_errors, atomic_load!(d_stats.client_errors));
+            atomic_increment!(self.server_errors, atomic_load!(d_stats.server_errors));
+            atomic_increment!(self.links_extracted, atomic_load!(d_stats.links_extracted));
+            atomic_increment!(self.status_200s, atomic_load!(d_stats.status_200s));
+            atomic_increment!(self.status_301s, atomic_load!(d_stats.status_301s));
+            atomic_increment!(self.status_302s, atomic_load!(d_stats.status_302s));
+            atomic_increment!(self.status_401s, atomic_load!(d_stats.status_401s));
+            atomic_increment!(self.status_403s, atomic_load!(d_stats.status_403s));
+            atomic_increment!(self.status_429s, atomic_load!(d_stats.status_429s));
+            atomic_increment!(self.status_500s, atomic_load!(d_stats.status_500s));
+            atomic_increment!(self.status_503s, atomic_load!(d_stats.status_503s));
+            atomic_increment!(self.status_504s, atomic_load!(d_stats.status_504s));
+            atomic_increment!(self.status_508s, atomic_load!(d_stats.status_508s));
+            atomic_increment!(
+                self.wildcards_filtered,
+                atomic_load!(d_stats.wildcards_filtered)
+            );
+            atomic_increment!(
+                self.responses_filtered,
+                atomic_load!(d_stats.responses_filtered)
+            );
+            atomic_increment!(
+                self.resources_discovered,
+                atomic_load!(d_stats.resources_discovered)
+            );
+            atomic_increment!(
+                self.url_format_errors,
+                atomic_load!(d_stats.url_format_errors)
+            );
+            atomic_increment!(
+                self.connection_errors,
+                atomic_load!(d_stats.connection_errors)
+            );
+            atomic_increment!(
+                self.redirection_errors,
+                atomic_load!(d_stats.redirection_errors)
+            );
+            atomic_increment!(self.request_errors, atomic_load!(d_stats.request_errors));
+
+            if let Ok(scan_times) = d_stats.directory_scan_times.lock() {
+                for scan_time in scan_times.iter() {
+                    self.update_f64_field(StatField::DirScanTimes, *scan_time);
+                }
+            };
+        }
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::{config::Configuration, Command};
+    use std::fs::write;
+    use tempfile::NamedTempFile;
+
+    use super::super::*;
+    use super::*;
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// when sent StatCommand::AddRequest, stats object should reflect the change
+    async fn statistics_handler_increments_requests() -> Result<()> {
+        let (task, handle) = setup_stats_test();
+
+        handle.tx.send(Command::AddRequest)?;
+        handle.tx.send(Command::AddRequest)?;
+        handle.tx.send(Command::AddRequest)?;
+
+        teardown_stats_test(handle.tx.clone(), task).await;
+
+        assert_eq!(handle.data.requests.load(Ordering::Relaxed), 3);
+
+        Ok(())
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// when sent StatCommand::AddRequest, stats object should reflect the change
+    ///
+    /// incrementing a 403 (tracked in status_403s) should also increment:
+    ///     - requests
+    ///     - client_errors
+    async fn statistics_handler_increments_403_via_status_code() {
+        let (task, handle) = setup_stats_test();
+
+        let err = Command::AddStatus(reqwest::StatusCode::FORBIDDEN);
+        let err2 = Command::AddStatus(reqwest::StatusCode::FORBIDDEN);
+
+        handle.tx.send(err).unwrap_or_default();
+        handle.tx.send(err2).unwrap_or_default();
+
+        teardown_stats_test(handle.tx.clone(), task).await;
+
+        assert_eq!(handle.data.requests.load(Ordering::Relaxed), 2);
+        assert_eq!(handle.data.status_403s.load(Ordering::Relaxed), 2);
+        assert_eq!(handle.data.client_errors.load(Ordering::Relaxed), 2);
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// when sent StatCommand::AddStatus, stats object should reflect the change
+    ///
+    /// incrementing a 500 (tracked in server_errors) should also increment:
+    ///     - requests
+    async fn statistics_handler_increments_500_via_status_code() -> Result<()> {
+        let (task, handle) = setup_stats_test();
+
+        let err = Command::AddStatus(reqwest::StatusCode::INTERNAL_SERVER_ERROR);
+        let err2 = Command::AddStatus(reqwest::StatusCode::INTERNAL_SERVER_ERROR);
+
+        handle.tx.send(err)?;
+        handle.tx.send(err2)?;
+
+        teardown_stats_test(handle.tx.clone(), task).await;
+
+        assert_eq!(handle.data.requests.load(Ordering::Relaxed), 2);
+        assert_eq!(handle.data.server_errors.load(Ordering::Relaxed), 2);
+
+        Ok(())
+    }
+
+    #[test]
+    /// when Stats::add_error receives StatError::Timeout, it should increment the following:
+    ///     - timeouts
+    ///     - requests
+    ///     - errors
+    fn stats_increments_timeouts() {
+        let config = Configuration::new().unwrap();
+        let stats = Stats::new(config.extensions.len(), config.json);
+
+        stats.add_error(StatError::Timeout);
+        stats.add_error(StatError::Timeout);
+        stats.add_error(StatError::Timeout);
+        stats.add_error(StatError::Timeout);
+
+        assert_eq!(stats.errors.load(Ordering::Relaxed), 4);
+        assert_eq!(stats.requests.load(Ordering::Relaxed), 4);
+        assert_eq!(stats.timeouts.load(Ordering::Relaxed), 4);
+    }
+
+    #[test]
+    /// when Stats::update_usize_field receives StatField::WildcardsFiltered, it should increment
+    /// the following:
+    ///     - responses_filtered
+    fn stats_increments_wildcards() {
+        let config = Configuration::new().unwrap();
+        let stats = Stats::new(config.extensions.len(), config.json);
+
+        assert_eq!(stats.responses_filtered.load(Ordering::Relaxed), 0);
+        assert_eq!(stats.wildcards_filtered.load(Ordering::Relaxed), 0);
+
+        stats.update_usize_field(StatField::WildcardsFiltered, 1);
+        stats.update_usize_field(StatField::WildcardsFiltered, 1);
+
+        assert_eq!(stats.responses_filtered.load(Ordering::Relaxed), 2);
+        assert_eq!(stats.wildcards_filtered.load(Ordering::Relaxed), 2);
+    }
+
+    #[test]
+    /// when Stats::update_usize_field receives StatField::ResponsesFiltered, it should increment
+    fn stats_increments_responses_filtered() {
+        let config = Configuration::new().unwrap();
+        let stats = Stats::new(config.extensions.len(), config.json);
+
+        assert_eq!(stats.responses_filtered.load(Ordering::Relaxed), 0);
+
+        stats.update_usize_field(StatField::ResponsesFiltered, 1);
+        stats.update_usize_field(StatField::ResponsesFiltered, 1);
+        stats.update_usize_field(StatField::ResponsesFiltered, 1);
+
+        assert_eq!(stats.responses_filtered.load(Ordering::Relaxed), 3);
+    }
+
+    #[test]
+    /// Stats::merge_from should properly increment expected fields and ignore others
+    fn stats_merge_from_alters_correct_fields() {
+        let contents = r#"{"statistics":{"type":"statistics","timeouts":1,"requests":9207,"expected_per_scan":707,"total_expected":9191,"errors":3,"successes":720,"redirects":13,"client_errors":8474,"server_errors":2,"total_scans":13,"initial_targets":1,"links_extracted":51,"status_403s":3,"status_200s":720,"status_301s":12,"status_302s":1,"status_401s":4,"status_429s":2,"status_500s":5,"status_503s":9,"status_504s":6,"status_508s":7,"wildcards_filtered":707,"responses_filtered":707,"resources_discovered":27,"directory_scan_times":[2.211973078,1.989015505,1.898675839,3.9714468910000003,4.938152838,5.256073528,6.021986595,6.065740734,6.42633762,7.095142125,7.336982137,5.319785619,4.843649778],"total_runtime":[11.556575456000001],"url_format_errors":17,"redirection_errors":12,"connection_errors":21,"request_errors":4}}"#;
+        let config = Configuration::new().unwrap();
+        let stats = Stats::new(config.extensions.len(), config.json);
+
+        let tfile = NamedTempFile::new().unwrap();
+        write(&tfile, contents).unwrap();
+
+        stats.merge_from(tfile.path().to_str().unwrap()).unwrap();
+
+        // as of 2.1.0; all Stats fields are accounted for whether they're updated in merge_from
+        // or not
+        assert_eq!(atomic_load!(stats.timeouts), 1);
+        assert_eq!(atomic_load!(stats.requests), 9207);
+        assert_eq!(atomic_load!(stats.expected_per_scan), 0); // not updated in merge_from
+        assert_eq!(atomic_load!(stats.total_expected), 0); // not updated in merge_from
+        assert_eq!(atomic_load!(stats.errors), 3);
+        assert_eq!(atomic_load!(stats.successes), 720);
+        assert_eq!(atomic_load!(stats.redirects), 13);
+        assert_eq!(atomic_load!(stats.client_errors), 8474);
+        assert_eq!(atomic_load!(stats.server_errors), 2);
+        assert_eq!(atomic_load!(stats.total_scans), 0); // not updated in merge_from
+        assert_eq!(atomic_load!(stats.initial_targets), 0); // not updated in merge_from
+        assert_eq!(atomic_load!(stats.links_extracted), 51);
+        assert_eq!(atomic_load!(stats.status_200s), 720);
+        assert_eq!(atomic_load!(stats.status_301s), 12);
+        assert_eq!(atomic_load!(stats.status_302s), 1);
+        assert_eq!(atomic_load!(stats.status_401s), 4);
+        assert_eq!(atomic_load!(stats.status_403s), 3);
+        assert_eq!(atomic_load!(stats.status_429s), 2);
+        assert_eq!(atomic_load!(stats.status_500s), 5);
+        assert_eq!(atomic_load!(stats.status_503s), 9);
+        assert_eq!(atomic_load!(stats.status_504s), 6);
+        assert_eq!(atomic_load!(stats.status_508s), 7);
+        assert_eq!(atomic_load!(stats.wildcards_filtered), 707);
+        assert_eq!(atomic_load!(stats.responses_filtered), 707);
+        assert_eq!(atomic_load!(stats.resources_discovered), 27);
+        assert_eq!(atomic_load!(stats.url_format_errors), 17);
+        assert_eq!(atomic_load!(stats.redirection_errors), 12);
+        assert_eq!(atomic_load!(stats.connection_errors), 21);
+        assert_eq!(atomic_load!(stats.request_errors), 4);
+        assert_eq!(stats.directory_scan_times.lock().unwrap().len(), 13);
+        for scan in stats.directory_scan_times.lock().unwrap().iter() {
+            assert!(scan.max(0.0) > 0.0); // all scans are non-zero
+        }
+        // total_runtime not updated in merge_from
+        assert_eq!(stats.total_runtime.lock().unwrap().len(), 1);
+        assert!((stats.total_runtime.lock().unwrap()[0] - 0.0).abs() < f64::EPSILON);
+    }
+
+    #[test]
+    /// ensure update runtime overwrites the default 0th entry
+    fn update_runtime_works() {
+        let config = Configuration::new().unwrap();
+        let stats = Stats::new(config.extensions.len(), config.json);
+
+        assert!((stats.total_runtime.lock().unwrap()[0] - 0.0).abs() < f64::EPSILON);
+        stats.update_runtime(20.2);
+        assert!((stats.total_runtime.lock().unwrap()[0] - 20.2).abs() < f64::EPSILON);
+    }
+
+    #[test]
+    /// ensure status_403s returns the correct value
+    fn status_403s_returns_correct_value() {
+        let config = Configuration::new().unwrap();
+        let stats = Stats::new(config.extensions.len(), config.json);
+        stats.status_403s.store(12, Ordering::Relaxed);
+        assert_eq!(stats.status_403s(), 12);
+    }
+
+    #[test]
+    /// ensure status_403s returns the correct value
+    fn status_429s_returns_correct_value() {
+        let config = Configuration::new().unwrap();
+        let stats = Stats::new(config.extensions.len(), config.json);
+        stats.status_429s.store(141, Ordering::Relaxed);
+        assert_eq!(stats.status_429s(), 141);
+    }
+}

src/statistics/error.rs

@@ -0,0 +1,21 @@
+#[derive(Debug, Copy, Clone)]
+/// Enum variants used to inform the `StatCommand` protocol what `Stats` fields should be updated
+pub enum StatError {
+    /// Represents a timeout error
+    Timeout,
+
+    /// Represents a URL formatting error
+    UrlFormat,
+
+    /// Represents an error encountered during redirection
+    Redirection,
+
+    /// Represents an error encountered during connection
+    Connection,
+
+    /// Represents an error resulting from the client's request
+    Request,
+
+    /// Represents any other error not explicitly defined above
+    Other,
+}

src/statistics/field.rs

@@ -0,0 +1,33 @@
+/// Enum representing fields whose updates need to be performed in batches instead of one at
+/// a time
+#[derive(Debug, Copy, Clone)]
+pub enum StatField {
+    /// Due to the necessary order of events, the number of requests expected to be sent isn't
+    /// known until after `statistics::initialize` is called. This command allows for updating
+    /// the `expected_per_scan` field after initialization
+    ExpectedPerScan,
+
+    /// Translates to `total_scans`
+    TotalScans,
+
+    /// Translates to `links_extracted`
+    LinksExtracted,
+
+    /// Translates to `total_expected`
+    TotalExpected,
+
+    /// Translates to `wildcards_filtered`
+    WildcardsFiltered,
+
+    /// Translates to `responses_filtered`
+    ResponsesFiltered,
+
+    /// Translates to `resources_discovered`
+    ResourcesDiscovered,
+
+    /// Translates to `initial_targets`
+    InitialTargets,
+
+    /// Translates to `directory_scan_times`; assumes a single append to the vector
+    DirScanTimes,
+}