Merge pull request #192 from epi052/190-fix-double-fslash

fixed url parsing issue when word starts with 2 or more /

.github/actions-rs/grcov.yml

@@ -0,0 +1,8 @@
+branch: false
+ignore-not-existing: true
+llvm: true
+output-type: lcov
+output-path: ./lcov.info
+# excl-br-line: "^\\s*((debug_)?assert(_eq|_ne)?!|#\\[derive\\(|log::)"
+ignore:
+  - "../*"

.github/dependabot.yml

@@ -0,0 +1,7 @@
+version: 2
+updates:
+- package-ecosystem: cargo
+  directory: "/"
+  schedule:
+    interval: daily
+  open-pull-requests-limit: 10

.github/pull_request_template.md

@@ -4,14 +4,14 @@ Long form explanations of most of the items below can be found in the [CONTRIBUT
 
 ## Branching checklist
 - [ ] There is an issue associated with your PR (bug, feature, etc.. if not, create one)
-- [ ] Your PR description references the associated issue (i.e. fixes #123)
+- [ ] Your PR description references the associated issue (i.e. fixes #123456)
 - [ ] Code is in its own branch
 - [ ] Branch name is related to the PR contents
 - [ ] PR targets master
 
 ## Static analysis checks
 - [ ] All rust files are formatted using `cargo fmt`
-- [ ] All `clippy` checks pass when running `cargo clippy --all-targets --all-features -- -D warnings -A clippy::unnecessary_unwrap`
+- [ ] All `clippy` checks pass when running `cargo clippy --all-targets --all-features -- -D warnings -A clippy::deref_addrof`
 - [ ] All existing tests pass
 
 ## Documentation

.github/stale.yml

@@ -0,0 +1,17 @@
+# Number of days of inactivity before an issue becomes stale
+daysUntilStale: 14
+# Number of days of inactivity before a stale issue is closed
+daysUntilClose: 7
+# Issues with these labels will never be considered stale
+exemptLabels:
+  - pinned
+  - security
+# Label to use when marking an issue as stale
+staleLabel: stale
+# Comment to post when marking an issue as stale. Set to `false` to disable
+markComment: >
+  This issue has been automatically marked as stale because it has not had
+  recent activity. It will be closed if no further activity occurs. Thank you
+  for your contributions.
+# Comment to post when closing a stale issue. Set to `false` to disable
+closeComment: false

.github/workflows/build.yml

@@ -1,69 +1,8 @@
-name: CI Pipeline
+name: CD Pipeline
 
 on: [push]
 
 jobs:
-  check:
-    name: Check
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-      - uses: actions-rs/cargo@v1
-        with:
-          command: check
-
-  test:
-    name: Test Suite
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-      - uses: actions-rs/cargo@v1
-        with:
-          command: test
-
-  fmt:
-    name: Rust fmt
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-      - run: rustup component add rustfmt
-      - uses: actions-rs/cargo@v1
-        with:
-          command: fmt
-          args: --all -- --check
-
-  clippy:
-    name: Clippy
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-      - run: rustup component add clippy
-      - uses: actions-rs/cargo@v1
-        with:
-          command: clippy
-          args: --all-targets --all-features -- -D warnings -A clippy::unnecessary_unwrap
-
-
   build-nix:
     runs-on: ${{ matrix.os }}
     if: github.ref == 'refs/heads/master'
@@ -102,10 +41,22 @@ jobs:
           use-cross: true
           command: build
           args: --release --target=${{ matrix.target }}
+      - name: Strip symbols from binary
+        run: |
+          strip -s ${{ matrix.path }}
+      - name: Build tar.gz for homebrew installs
+        if: matrix.type == 'ubuntu-x64'
+        run: |
+          tar czf ${{ matrix.name }}.tar.gz -C target/x86_64-unknown-linux-musl/release feroxbuster
       - uses: actions/upload-artifact@v2
         with:
           name: ${{ matrix.name }}
           path: ${{ matrix.path }}
+      - uses: actions/upload-artifact@v2
+        if: matrix.type == 'ubuntu-x64'
+        with:
+          name: ${{ matrix.name }}.tar.gz
+          path: ${{ matrix.name }}.tar.gz
 
   build-deb:
     needs: [build-nix]
@@ -120,18 +71,43 @@ jobs:
           name: feroxbuster_amd64.deb
           path: ./target/x86_64-unknown-linux-musl/debian/*
 
-  build-rest:
+  build-macos:
+    runs-on: macos-latest
+    if: github.ref == 'refs/heads/master'
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          target: x86_64-apple-darwin
+          override: true
+      - uses: actions-rs/cargo@v1
+        with:
+          use-cross: true
+          command: build
+          args: --release --target=x86_64-apple-darwin
+      - name: Strip symbols from binary
+        run: |
+          strip -u -r target/x86_64-apple-darwin/release/feroxbuster
+      - name: Build tar.gz for homebrew installs
+        run: |
+          tar czf x86_64-macos-feroxbuster.tar.gz -C target/x86_64-apple-darwin/release feroxbuster
+      - uses: actions/upload-artifact@v2
+        with:
+          name: x86_64-macos-feroxbuster
+          path: target/x86_64-apple-darwin/release/feroxbuster
+      - uses: actions/upload-artifact@v2
+        with:
+          name: x86_64-macos-feroxbuster.tar.gz
+          path: x86_64-macos-feroxbuster.tar.gz
+
+  build-windows:
     runs-on: ${{ matrix.os }}
     if: github.ref == 'refs/heads/master'
     strategy:
       matrix:
-        type: [windows-x64, windows-x86, macos]
+        type: [windows-x64, windows-x86]
         include:
-          - type: macos
-            os: macos-latest
-            target: x86_64-apple-darwin
-            name: x86_64-macos-feroxbuster
-            path: target/x86_64-apple-darwin/release/feroxbuster
           - type: windows-x64
             os: windows-latest
             target: x86_64-pc-windows-msvc
@@ -158,3 +134,4 @@ jobs:
         with:
           name: ${{ matrix.name }}
           path: ${{ matrix.path }}
+

.github/workflows/check.yml

@@ -0,0 +1,64 @@
+name: CI Pipeline
+
+on: [push, pull_request]
+
+jobs:
+  check:
+    name: Check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+      - uses: actions-rs/cargo@v1
+        with:
+          command: check
+
+  test:
+    name: Test Suite
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+      - uses: actions-rs/cargo@v1
+        with:
+          command: test
+
+  fmt:
+    name: Rust fmt
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+      - run: rustup component add rustfmt
+      - uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --all -- --check
+
+  clippy:
+    name: Clippy
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+      - run: rustup component add clippy
+      - uses: actions-rs/cargo@v1
+        with:
+          command: clippy
+          args: --all-targets --all-features -- -D warnings -A clippy::deref_addrof

.github/workflows/coverage.yml

@@ -0,0 +1,36 @@
+on: [push]
+
+name: Code Coverage Pipeline
+
+jobs:
+  upload-coverage:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: nightly
+          override: true
+      - uses: actions-rs/cargo@v1
+        with:
+          command: clean
+      - uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --all-features --no-fail-fast
+        env:
+          CARGO_INCREMENTAL: '0'
+          RUSTFLAGS: '-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort'
+          RUSTDOCFLAGS: '-Cpanic=abort'
+      - uses: actions-rs/grcov@v0.1
+      - name: Convert lcov to xml
+        run: |
+          curl -O https://raw.githubusercontent.com/eriwen/lcov-to-cobertura-xml/master/lcov_cobertura/lcov_cobertura.py
+          chmod +x lcov_cobertura.py
+          ./lcov_cobertura.py ./lcov.info
+      - uses: codecov/codecov-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          file: ./coverage.xml
+          name: codecov-umbrella
+          fail_ci_if_error: true

.gitignore

@@ -15,3 +15,13 @@ Cargo.lock
 
 # personal feroxbuster config for testing
 ferox-config.toml
+
+# images for the README on github
+img/**
+
+# scripts to check code coverage using nightly compiler
+check-coverage.sh
+lcov_cobertura.py
+
+# dockerignore file that makes it so i can work on the docker config without copying a 4GB manifest or w/e it is
+.dockerignore

.rustfmt.toml

@@ -0,0 +1 @@
+reorder_modules = false

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "feroxbuster"
-version = "0.2.0"
+version = "1.12.2"
 authors = ["Ben 'epi' Risher <epibar052@gmail.com>"]
 license = "MIT"
 edition = "2018"
@@ -10,31 +10,42 @@ description = "A fast, simple, recursive content discovery tool."
 categories = ["command-line-utilities"]
 keywords = ["pentest", "enumeration", "url-bruteforce", "content-discovery", "web"]
 exclude = [".github/*", "img/*", "check-coverage.sh"]
+build = "build.rs"
 
 [badges]
 maintenance = { status = "actively-developed" }
 
+[build-dependencies]
+clap = "2.33"
+regex = "1"
+lazy_static = "1.4"
+
 [dependencies]
 futures = { version = "0.3"}
-tokio = { version = "0.2", features = ["full"] }
-tokio-util = {version = "0.3", features = ["codec"]}
+tokio = { version = "1.0", features = ["full"] }
+tokio-util = {version = "0.6", features = ["codec"]}
 log = "0.4"
-env_logger = "0.7"
-reqwest = { version = "0.10", features = ["socks"] }
-clap = "2"
+env_logger = "0.8"
+reqwest = { version = "0.11", features = ["socks"] }
+clap = "2.33"
 lazy_static = "1.4"
 toml = "0.5"
-serde = { version = "1.0", features = ["derive"] }
+serde = { version = "1.0", features = ["derive", "rc"] }
+serde_json = "1.0"
 uuid = { version = "0.8", features = ["v4"] }
-ansi_term = "0.12"
 indicatif = "0.15"
-console = "0.12"
+console = "0.14"
 openssl = { version = "0.10", features = ["vendored"] }
 dirs = "3.0"
+regex = "1"
+crossterm = "0.19"
+rlimit = "0.5"
+ctrlc = "3.1"
+fuzzyhash = "0.2.1"
 
 [dev-dependencies]
 tempfile = "3.1"
-httpmock = "0.4.5"
+httpmock = "0.5.2"
 assert_cmd = "1.0.1"
 predicates = "1.0.5"
 
@@ -46,4 +57,8 @@ panic = 'abort'
 [package.metadata.deb]
 section = "utility"
 license-file = ["LICENSE", "4"]
-conf-files = ["~/.config/feroxbuster/ferox-config.toml"]
+conf-files = ["/etc/feroxbuster/ferox-config.toml"]
+assets = [
+    ["target/release/feroxbuster", "/usr/bin/", "755"],
+    ["ferox-config.toml.example", "/etc/feroxbuster/ferox-config.toml", "644"],
+]

Dockerfile

@@ -0,0 +1,14 @@
+FROM alpine:latest
+LABEL maintainer="wfnintr@null.net"
+
+RUN sed -i -e 's/v[[:digit:]]\..*\//edge\//g' /etc/apk/repositories && apk upgrade --update-cache --available
+
+# download default wordlists 
+RUN apk add --no-cache --virtual .depends subversion font-noto-emoji && \
+	svn export https://github.com/danielmiessler/SecLists/trunk/Discovery/Web-Content /usr/share/seclists/Discovery/Web-Content && \
+	apk del .depends
+
+# install latest release
+RUN wget https://github.com/epi052/feroxbuster/releases/latest/download/x86_64-linux-feroxbuster.zip -qO feroxbuster.zip && unzip -d /usr/local/bin/ feroxbuster.zip feroxbuster && rm feroxbuster.zip && chmod +x /usr/local/bin/feroxbuster
+
+ENTRYPOINT ["feroxbuster"]

build.rs

@@ -0,0 +1,23 @@
+extern crate clap;
+
+use clap::Shell;
+
+include!("src/parser.rs");
+
+fn main() {
+    println!("cargo:rerun-if-env-changed=src/parser.rs");
+
+    if std::env::var("DOCS_RS").is_ok() {
+        return; // only build when we're not generating docs
+    }
+
+    let outdir = "shell_completions";
+
+    let mut app = initialize();
+
+    let shells: [Shell; 4] = [Shell::Bash, Shell::Fish, Shell::Zsh, Shell::PowerShell];
+
+    for shell in &shells {
+        app.gen_completions("feroxbuster", *shell, outdir);
+    }
+}

ferox-config.toml.example

@@ -8,24 +8,37 @@
 # Any setting used here can be overridden by the corresponding command line option/argument
 #
 # wordlist = "/wordlists/seclists/Discovery/Web-Content/raft-medium-directories.txt"
-# statuscodes = [200, 500]
+# status_codes = [200, 500]
+# filter_status = [301]
 # threads = 1
 # timeout = 5
 # proxy = "http://127.0.0.1:8080"
+# replay_proxy = "http://127.0.0.1:8081"
+# replay_codes = [200, 302]
 # verbosity = 1
+# scan_limit = 6
 # quiet = true
+# json = true
 # output = "/targets/ellingson_mineral_company/gibson.txt"
-# useragent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0"
+# debug_log = "/var/log/find-the-derp.log"
+# user_agent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0"
 # redirects = true
 # insecure = true
 # extensions = ["php", "html"]
-# norecursion = true
-# addslash = true
+# no_recursion = true
+# add_slash = true
 # stdin = true
-# dontfilter = true
+# dont_filter = true
+# extract_links = true
 # depth = 1
-# sizefilters = [5174]
+# filter_size = [5174]
+# filter_regex = ["^ignore me$"]
+# filter_similar = ["https://somesite.com/soft404"]
+# filter_word_count = [993]
+# filter_line_count = [35, 36]
 # queries = [["name","value"], ["rick", "astley"]]
+# save_state = false
+# time_limit = "10m"
 
 # headers can be specified on multiple lines or as an inline table
 #

install-nix.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+BASE_URL=https://github.com/epi052/feroxbuster/releases/latest/download
+
+MAC_ZIP=x86_64-macos-feroxbuster.zip
+MAC_URL="${BASE_URL}/${MAC_ZIP}"
+
+LIN32_ZIP=x86-linux-feroxbuster.zip
+LIN32_URL="${BASE_URL}/${LIN32_ZIP}"
+
+LIN64_ZIP=x86_64-linux-feroxbuster.zip
+LIN64_URL="${BASE_URL}/${LIN64_ZIP}"
+
+EMOJI_URL=https://gist.github.com/epi052/8196b550ea51d0907ad4b93751b1b57d/raw/6112c9f32ae07922983fdc549c54fd3fb9a38e4c/NotoColorEmoji.ttf
+
+echo "[+] Installing feroxbuster!"
+
+if [[ "$(uname)" == "Darwin" ]]; then
+    echo "[=] Found MacOS, downloading from ${MAC_URL}"
+
+    curl -sLO "${MAC_URL}"
+    unzip -o "${MAC_ZIP}" > /dev/null
+    rm "${MAC_ZIP}"
+elif [[ "$(expr substr $(uname -s) 1 5)" == "Linux" ]]; then
+    if [[ $(getconf LONG_BIT) == 32 ]]; then
+        echo "[=] Found 32-bit Linux, downloading from ${LIN32_URL}"
+
+        curl -sLO "${LIN32_URL}"
+        unzip -o "${LIN32_ZIP}" > /dev/null
+        rm "${LIN32_ZIP}"
+    else
+        echo "[=] Found 64-bit Linux, downloading from ${LIN64_URL}"
+
+        curl -sLO "${LIN64_URL}"
+        unzip -o "${LIN64_ZIP}" > /dev/null
+        rm "${LIN64_ZIP}"
+    fi
+
+    if [[ -e ~/.fonts/NotoColorEmoji.ttf ]]; then
+       echo "[=] Found Noto Emoji Font, skipping install" 
+    else 
+        echo "[=] Installing Noto Emoji Font"
+        mkdir -p ~/.fonts
+        pushd ~/.fonts 2>&1 >/dev/null
+
+        curl -sLO "${EMOJI_URL}"
+
+        fc-cache -f -v >/dev/null
+
+        popd 2>&1 >/dev/null
+        echo "[+] Noto Emoji Font installed"
+    fi
+fi
+
+
+chmod +x ./feroxbuster
+
+echo "[+] Installed feroxbuster version $(./feroxbuster -V)"
+
+
+

shell_completions/_feroxbuster

@@ -0,0 +1,96 @@
+#compdef feroxbuster
+
+autoload -U is-at-least
+
+_feroxbuster() {
+    typeset -A opt_args
+    typeset -a _arguments_options
+    local ret=1
+
+    if is-at-least 5.2; then
+        _arguments_options=(-s -S -C)
+    else
+        _arguments_options=(-s -C)
+    fi
+
+    local context curcontext="$curcontext" state line
+    _arguments "${_arguments_options[@]}" \
+'-w+[Path to the wordlist]' \
+'--wordlist=[Path to the wordlist]' \
+'*-u+[The target URL(s) (required, unless --stdin used)]' \
+'*--url=[The target URL(s) (required, unless --stdin used)]' \
+'-t+[Number of concurrent threads (default: 50)]' \
+'--threads=[Number of concurrent threads (default: 50)]' \
+'-d+[Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)]' \
+'--depth=[Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)]' \
+'-T+[Number of seconds before a request times out (default: 7)]' \
+'--timeout=[Number of seconds before a request times out (default: 7)]' \
+'-p+[Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)]' \
+'--proxy=[Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)]' \
+'-P+[Send only unfiltered requests through a Replay Proxy, instead of all requests]' \
+'--replay-proxy=[Send only unfiltered requests through a Replay Proxy, instead of all requests]' \
+'*-R+[Status Codes to send through a Replay Proxy when found (default: --status-codes value)]' \
+'*--replay-codes=[Status Codes to send through a Replay Proxy when found (default: --status-codes value)]' \
+'*-s+[Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)]' \
+'*--status-codes=[Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)]' \
+'-o+[Output file to write results to (use w/ --json for JSON entries)]' \
+'--output=[Output file to write results to (use w/ --json for JSON entries)]' \
+'(-u --url)--resume-from=[State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)]' \
+'--debug-log=[Output file to write log entries (use w/ --json for JSON entries)]' \
+'-a+[Sets the User-Agent (default: feroxbuster/VERSION)]' \
+'--user-agent=[Sets the User-Agent (default: feroxbuster/VERSION)]' \
+'*-x+[File extension(s) to search for (ex: -x php -x pdf js)]' \
+'*--extensions=[File extension(s) to search for (ex: -x php -x pdf js)]' \
+'*-H+[Specify HTTP headers (ex: -H Header:val '\''stuff: things'\'')]' \
+'*--headers=[Specify HTTP headers (ex: -H Header:val '\''stuff: things'\'')]' \
+'*-Q+[Specify URL query parameters (ex: -Q token=stuff -Q secret=key)]' \
+'*--query=[Specify URL query parameters (ex: -Q token=stuff -Q secret=key)]' \
+'*-S+[Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)]' \
+'*--filter-size=[Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)]' \
+'*-X+[Filter out messages via regular expression matching on the response'\''s body (ex: -X '\''^ignore me$'\'')]' \
+'*--filter-regex=[Filter out messages via regular expression matching on the response'\''s body (ex: -X '\''^ignore me$'\'')]' \
+'*-W+[Filter out messages of a particular word count (ex: -W 312 -W 91,82)]' \
+'*--filter-words=[Filter out messages of a particular word count (ex: -W 312 -W 91,82)]' \
+'*-N+[Filter out messages of a particular line count (ex: -N 20 -N 31,30)]' \
+'*--filter-lines=[Filter out messages of a particular line count (ex: -N 20 -N 31,30)]' \
+'*-C+[Filter out status codes (deny list) (ex: -C 200 -C 401)]' \
+'*--filter-status=[Filter out status codes (deny list) (ex: -C 200 -C 401)]' \
+'*--filter-similar-to=[Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)]' \
+'-L+[Limit total number of concurrent scans (default: 0, i.e. no limit)]' \
+'--scan-limit=[Limit total number of concurrent scans (default: 0, i.e. no limit)]' \
+'--time-limit=[Limit total run time of all scans (ex: --time-limit 10m)]' \
+'*-v[Increase verbosity level (use -vv or more for greater effect. \[CAUTION\] 4 -v'\''s is probably too much)]' \
+'*--verbosity[Increase verbosity level (use -vv or more for greater effect. \[CAUTION\] 4 -v'\''s is probably too much)]' \
+'-q[Only print URLs; Don'\''t print status codes, response size, running config, etc...]' \
+'--quiet[Only print URLs; Don'\''t print status codes, response size, running config, etc...]' \
+'--json[Emit JSON logs to --output and --debug-log instead of normal text]' \
+'-D[Don'\''t auto-filter wildcard responses]' \
+'--dont-filter[Don'\''t auto-filter wildcard responses]' \
+'-r[Follow redirects]' \
+'--redirects[Follow redirects]' \
+'-k[Disables TLS certificate validation]' \
+'--insecure[Disables TLS certificate validation]' \
+'-n[Do not scan recursively]' \
+'--no-recursion[Do not scan recursively]' \
+'(-x --extensions)-f[Append / to each request]' \
+'(-x --extensions)--add-slash[Append / to each request]' \
+'(-u --url)--stdin[Read url(s) from STDIN]' \
+'-e[Extract links from response body (html, javascript, etc...); make new requests based on findings (default: false)]' \
+'--extract-links[Extract links from response body (html, javascript, etc...); make new requests based on findings (default: false)]' \
+'-h[Prints help information]' \
+'--help[Prints help information]' \
+'-V[Prints version information]' \
+'--version[Prints version information]' \
+&& ret=0
+    
+}
+
+(( $+functions[_feroxbuster_commands] )) ||
+_feroxbuster_commands() {
+    local commands; commands=(
+        
+    )
+    _describe -t commands 'feroxbuster commands' commands "$@"
+}
+
+_feroxbuster "$@"

shell_completions/_feroxbuster.ps1

@@ -0,0 +1,95 @@
+
+using namespace System.Management.Automation
+using namespace System.Management.Automation.Language
+
+Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
+    param($wordToComplete, $commandAst, $cursorPosition)
+
+    $commandElements = $commandAst.CommandElements
+    $command = @(
+        'feroxbuster'
+        for ($i = 1; $i -lt $commandElements.Count; $i++) {
+            $element = $commandElements[$i]
+            if ($element -isnot [StringConstantExpressionAst] -or
+                $element.StringConstantType -ne [StringConstantType]::BareWord -or
+                $element.Value.StartsWith('-')) {
+                break
+        }
+        $element.Value
+    }) -join ';'
+
+    $completions = @(switch ($command) {
+        'feroxbuster' {
+            [CompletionResult]::new('-w', 'w', [CompletionResultType]::ParameterName, 'Path to the wordlist')
+            [CompletionResult]::new('--wordlist', 'wordlist', [CompletionResultType]::ParameterName, 'Path to the wordlist')
+            [CompletionResult]::new('-u', 'u', [CompletionResultType]::ParameterName, 'The target URL(s) (required, unless --stdin used)')
+            [CompletionResult]::new('--url', 'url', [CompletionResultType]::ParameterName, 'The target URL(s) (required, unless --stdin used)')
+            [CompletionResult]::new('-t', 't', [CompletionResultType]::ParameterName, 'Number of concurrent threads (default: 50)')
+            [CompletionResult]::new('--threads', 'threads', [CompletionResultType]::ParameterName, 'Number of concurrent threads (default: 50)')
+            [CompletionResult]::new('-d', 'd', [CompletionResultType]::ParameterName, 'Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)')
+            [CompletionResult]::new('--depth', 'depth', [CompletionResultType]::ParameterName, 'Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)')
+            [CompletionResult]::new('-T', 'T', [CompletionResultType]::ParameterName, 'Number of seconds before a request times out (default: 7)')
+            [CompletionResult]::new('--timeout', 'timeout', [CompletionResultType]::ParameterName, 'Number of seconds before a request times out (default: 7)')
+            [CompletionResult]::new('-p', 'p', [CompletionResultType]::ParameterName, 'Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)')
+            [CompletionResult]::new('--proxy', 'proxy', [CompletionResultType]::ParameterName, 'Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)')
+            [CompletionResult]::new('-P', 'P', [CompletionResultType]::ParameterName, 'Send only unfiltered requests through a Replay Proxy, instead of all requests')
+            [CompletionResult]::new('--replay-proxy', 'replay-proxy', [CompletionResultType]::ParameterName, 'Send only unfiltered requests through a Replay Proxy, instead of all requests')
+            [CompletionResult]::new('-R', 'R', [CompletionResultType]::ParameterName, 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)')
+            [CompletionResult]::new('--replay-codes', 'replay-codes', [CompletionResultType]::ParameterName, 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)')
+            [CompletionResult]::new('-s', 's', [CompletionResultType]::ParameterName, 'Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)')
+            [CompletionResult]::new('--status-codes', 'status-codes', [CompletionResultType]::ParameterName, 'Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)')
+            [CompletionResult]::new('-o', 'o', [CompletionResultType]::ParameterName, 'Output file to write results to (use w/ --json for JSON entries)')
+            [CompletionResult]::new('--output', 'output', [CompletionResultType]::ParameterName, 'Output file to write results to (use w/ --json for JSON entries)')
+            [CompletionResult]::new('--resume-from', 'resume-from', [CompletionResultType]::ParameterName, 'State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)')
+            [CompletionResult]::new('--debug-log', 'debug-log', [CompletionResultType]::ParameterName, 'Output file to write log entries (use w/ --json for JSON entries)')
+            [CompletionResult]::new('-a', 'a', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/VERSION)')
+            [CompletionResult]::new('--user-agent', 'user-agent', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/VERSION)')
+            [CompletionResult]::new('-x', 'x', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js)')
+            [CompletionResult]::new('--extensions', 'extensions', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js)')
+            [CompletionResult]::new('-H', 'H', [CompletionResultType]::ParameterName, 'Specify HTTP headers (ex: -H Header:val ''stuff: things'')')
+            [CompletionResult]::new('--headers', 'headers', [CompletionResultType]::ParameterName, 'Specify HTTP headers (ex: -H Header:val ''stuff: things'')')
+            [CompletionResult]::new('-Q', 'Q', [CompletionResultType]::ParameterName, 'Specify URL query parameters (ex: -Q token=stuff -Q secret=key)')
+            [CompletionResult]::new('--query', 'query', [CompletionResultType]::ParameterName, 'Specify URL query parameters (ex: -Q token=stuff -Q secret=key)')
+            [CompletionResult]::new('-S', 'S', [CompletionResultType]::ParameterName, 'Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)')
+            [CompletionResult]::new('--filter-size', 'filter-size', [CompletionResultType]::ParameterName, 'Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)')
+            [CompletionResult]::new('-X', 'X', [CompletionResultType]::ParameterName, 'Filter out messages via regular expression matching on the response''s body (ex: -X ''^ignore me$'')')
+            [CompletionResult]::new('--filter-regex', 'filter-regex', [CompletionResultType]::ParameterName, 'Filter out messages via regular expression matching on the response''s body (ex: -X ''^ignore me$'')')
+            [CompletionResult]::new('-W', 'W', [CompletionResultType]::ParameterName, 'Filter out messages of a particular word count (ex: -W 312 -W 91,82)')
+            [CompletionResult]::new('--filter-words', 'filter-words', [CompletionResultType]::ParameterName, 'Filter out messages of a particular word count (ex: -W 312 -W 91,82)')
+            [CompletionResult]::new('-N', 'N', [CompletionResultType]::ParameterName, 'Filter out messages of a particular line count (ex: -N 20 -N 31,30)')
+            [CompletionResult]::new('--filter-lines', 'filter-lines', [CompletionResultType]::ParameterName, 'Filter out messages of a particular line count (ex: -N 20 -N 31,30)')
+            [CompletionResult]::new('-C', 'C', [CompletionResultType]::ParameterName, 'Filter out status codes (deny list) (ex: -C 200 -C 401)')
+            [CompletionResult]::new('--filter-status', 'filter-status', [CompletionResultType]::ParameterName, 'Filter out status codes (deny list) (ex: -C 200 -C 401)')
+            [CompletionResult]::new('--filter-similar-to', 'filter-similar-to', [CompletionResultType]::ParameterName, 'Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)')
+            [CompletionResult]::new('-L', 'L', [CompletionResultType]::ParameterName, 'Limit total number of concurrent scans (default: 0, i.e. no limit)')
+            [CompletionResult]::new('--scan-limit', 'scan-limit', [CompletionResultType]::ParameterName, 'Limit total number of concurrent scans (default: 0, i.e. no limit)')
+            [CompletionResult]::new('--time-limit', 'time-limit', [CompletionResultType]::ParameterName, 'Limit total run time of all scans (ex: --time-limit 10m)')
+            [CompletionResult]::new('-v', 'v', [CompletionResultType]::ParameterName, 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v''s is probably too much)')
+            [CompletionResult]::new('--verbosity', 'verbosity', [CompletionResultType]::ParameterName, 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v''s is probably too much)')
+            [CompletionResult]::new('-q', 'q', [CompletionResultType]::ParameterName, 'Only print URLs; Don''t print status codes, response size, running config, etc...')
+            [CompletionResult]::new('--quiet', 'quiet', [CompletionResultType]::ParameterName, 'Only print URLs; Don''t print status codes, response size, running config, etc...')
+            [CompletionResult]::new('--json', 'json', [CompletionResultType]::ParameterName, 'Emit JSON logs to --output and --debug-log instead of normal text')
+            [CompletionResult]::new('-D', 'D', [CompletionResultType]::ParameterName, 'Don''t auto-filter wildcard responses')
+            [CompletionResult]::new('--dont-filter', 'dont-filter', [CompletionResultType]::ParameterName, 'Don''t auto-filter wildcard responses')
+            [CompletionResult]::new('-r', 'r', [CompletionResultType]::ParameterName, 'Follow redirects')
+            [CompletionResult]::new('--redirects', 'redirects', [CompletionResultType]::ParameterName, 'Follow redirects')
+            [CompletionResult]::new('-k', 'k', [CompletionResultType]::ParameterName, 'Disables TLS certificate validation')
+            [CompletionResult]::new('--insecure', 'insecure', [CompletionResultType]::ParameterName, 'Disables TLS certificate validation')
+            [CompletionResult]::new('-n', 'n', [CompletionResultType]::ParameterName, 'Do not scan recursively')
+            [CompletionResult]::new('--no-recursion', 'no-recursion', [CompletionResultType]::ParameterName, 'Do not scan recursively')
+            [CompletionResult]::new('-f', 'f', [CompletionResultType]::ParameterName, 'Append / to each request')
+            [CompletionResult]::new('--add-slash', 'add-slash', [CompletionResultType]::ParameterName, 'Append / to each request')
+            [CompletionResult]::new('--stdin', 'stdin', [CompletionResultType]::ParameterName, 'Read url(s) from STDIN')
+            [CompletionResult]::new('-e', 'e', [CompletionResultType]::ParameterName, 'Extract links from response body (html, javascript, etc...); make new requests based on findings (default: false)')
+            [CompletionResult]::new('--extract-links', 'extract-links', [CompletionResultType]::ParameterName, 'Extract links from response body (html, javascript, etc...); make new requests based on findings (default: false)')
+            [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Prints help information')
+            [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Prints help information')
+            [CompletionResult]::new('-V', 'V', [CompletionResultType]::ParameterName, 'Prints version information')
+            [CompletionResult]::new('--version', 'version', [CompletionResultType]::ParameterName, 'Prints version information')
+            break
+        }
+    })
+
+    $completions.Where{ $_.CompletionText -like "$wordToComplete*" } |
+        Sort-Object -Property ListItemText
+}

shell_completions/feroxbuster.bash

@@ -0,0 +1,217 @@
+_feroxbuster() {
+    local i cur prev opts cmds
+    COMPREPLY=()
+    cur="${COMP_WORDS[COMP_CWORD]}"
+    prev="${COMP_WORDS[COMP_CWORD-1]}"
+    cmd=""
+    opts=""
+
+    for i in ${COMP_WORDS[@]}
+    do
+        case "${i}" in
+            feroxbuster)
+                cmd="feroxbuster"
+                ;;
+            
+            *)
+                ;;
+        esac
+    done
+
+    case "${cmd}" in
+        feroxbuster)
+            opts=" -v -q -D -r -k -n -f -e -h -V -w -u -t -d -T -p -P -R -s -o -a -x -H -Q -S -X -W -N -C -L  --verbosity --quiet --json --dont-filter --redirects --insecure --no-recursion --add-slash --stdin --extract-links --help --version --wordlist --url --threads --depth --timeout --proxy --replay-proxy --replay-codes --status-codes --output --resume-from --debug-log --user-agent --extensions --headers --query --filter-size --filter-regex --filter-words --filter-lines --filter-status --filter-similar-to --scan-limit --time-limit  "
+            if [[ ${cur} == -* || ${COMP_CWORD} -eq 1 ]] ; then
+                COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
+                return 0
+            fi
+            case "${prev}" in
+                
+                --wordlist)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -w)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --url)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -u)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --threads)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -t)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --depth)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -d)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --timeout)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -T)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --proxy)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -p)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --replay-proxy)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -P)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --replay-codes)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -R)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --status-codes)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -s)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --output)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -o)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --resume-from)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --debug-log)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --user-agent)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -a)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --extensions)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -x)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --headers)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -H)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --query)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -Q)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --filter-size)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -S)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --filter-regex)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -X)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --filter-words)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -W)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --filter-lines)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -N)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --filter-status)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -C)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --filter-similar-to)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --scan-limit)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -L)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --time-limit)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                *)
+                    COMPREPLY=()
+                    ;;
+            esac
+            COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
+            return 0
+            ;;
+        
+    esac
+}
+
+complete -F _feroxbuster -o bashdefault -o default feroxbuster

shell_completions/feroxbuster.fish

@@ -0,0 +1,36 @@
+complete -c feroxbuster -n "__fish_use_subcommand" -s w -l wordlist -d 'Path to the wordlist'
+complete -c feroxbuster -n "__fish_use_subcommand" -s u -l url -d 'The target URL(s) (required, unless --stdin used)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s t -l threads -d 'Number of concurrent threads (default: 50)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s d -l depth -d 'Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s T -l timeout -d 'Number of seconds before a request times out (default: 7)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s p -l proxy -d 'Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s P -l replay-proxy -d 'Send only unfiltered requests through a Replay Proxy, instead of all requests'
+complete -c feroxbuster -n "__fish_use_subcommand" -s R -l replay-codes -d 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s s -l status-codes -d 'Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s o -l output -d 'Output file to write results to (use w/ --json for JSON entries)'
+complete -c feroxbuster -n "__fish_use_subcommand" -l resume-from -d 'State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)'
+complete -c feroxbuster -n "__fish_use_subcommand" -l debug-log -d 'Output file to write log entries (use w/ --json for JSON entries)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s a -l user-agent -d 'Sets the User-Agent (default: feroxbuster/VERSION)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s x -l extensions -d 'File extension(s) to search for (ex: -x php -x pdf js)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s H -l headers -d 'Specify HTTP headers (ex: -H Header:val \'stuff: things\')'
+complete -c feroxbuster -n "__fish_use_subcommand" -s Q -l query -d 'Specify URL query parameters (ex: -Q token=stuff -Q secret=key)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s S -l filter-size -d 'Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s X -l filter-regex -d 'Filter out messages via regular expression matching on the response\'s body (ex: -X \'^ignore me$\')'
+complete -c feroxbuster -n "__fish_use_subcommand" -s W -l filter-words -d 'Filter out messages of a particular word count (ex: -W 312 -W 91,82)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s N -l filter-lines -d 'Filter out messages of a particular line count (ex: -N 20 -N 31,30)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s C -l filter-status -d 'Filter out status codes (deny list) (ex: -C 200 -C 401)'
+complete -c feroxbuster -n "__fish_use_subcommand" -l filter-similar-to -d 'Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s L -l scan-limit -d 'Limit total number of concurrent scans (default: 0, i.e. no limit)'
+complete -c feroxbuster -n "__fish_use_subcommand" -l time-limit -d 'Limit total run time of all scans (ex: --time-limit 10m)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s v -l verbosity -d 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v\'s is probably too much)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s q -l quiet -d 'Only print URLs; Don\'t print status codes, response size, running config, etc...'
+complete -c feroxbuster -n "__fish_use_subcommand" -l json -d 'Emit JSON logs to --output and --debug-log instead of normal text'
+complete -c feroxbuster -n "__fish_use_subcommand" -s D -l dont-filter -d 'Don\'t auto-filter wildcard responses'
+complete -c feroxbuster -n "__fish_use_subcommand" -s r -l redirects -d 'Follow redirects'
+complete -c feroxbuster -n "__fish_use_subcommand" -s k -l insecure -d 'Disables TLS certificate validation'
+complete -c feroxbuster -n "__fish_use_subcommand" -s n -l no-recursion -d 'Do not scan recursively'
+complete -c feroxbuster -n "__fish_use_subcommand" -s f -l add-slash -d 'Append / to each request'
+complete -c feroxbuster -n "__fish_use_subcommand" -l stdin -d 'Read url(s) from STDIN'
+complete -c feroxbuster -n "__fish_use_subcommand" -s e -l extract-links -d 'Extract links from response body (html, javascript, etc...); make new requests based on findings (default: false)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s h -l help -d 'Prints help information'
+complete -c feroxbuster -n "__fish_use_subcommand" -s V -l version -d 'Prints version information'

snapcraft.yaml

@@ -0,0 +1,41 @@
+name: feroxbuster
+version: git
+summary: A simple, fast, recursive content discovery tool written in Rust
+description: |
+  feroxbuster is a tool designed to perform Forced Browsing.
+
+  Forced browsing is an attack where the aim is to enumerate and access resources that are not referenced by the web application, but are still accessible by an attacker.
+
+  feroxbuster uses brute force combined with a wordlist to search for unlinked content in target directories. These resources may store sensitive information about web applications and operational systems, such as source code, credentials, internal network addressing, etc...
+
+  This attack is also known as Predictable Resource Location, File Enumeration, Directory Enumeration, and Resource Enumeration.
+
+
+base: core18
+
+plugs:
+  etc-feroxbuster:
+    interface: system-files
+    read:
+    - /etc/feroxbuster
+  dot-config-feroxbuster:
+    interface: personal-files
+    read:
+    - $HOME/.config/feroxbuster
+
+architectures:
+  - build-on: amd64
+  - build-on: i386
+
+parts:
+  feroxbuster:
+    plugin: rust
+    source: .
+
+apps:
+  feroxbuster:
+    command: bin/feroxbuster
+    plugs:
+      - etc-feroxbuster
+      - dot-config-feroxbuster
+      - network

src/banner.rs

@@ -1,4 +1,13 @@
-use crate::{config::CONFIGURATION, utils::status_colorizer, VERSION};
+use crate::{
+    config::{Configuration, CONFIGURATION},
+    statistics::StatCommand,
+    utils::{make_request, status_colorizer},
+};
+use console::{style, Emoji};
+use reqwest::{Client, Url};
+use serde_json::Value;
+use std::io::Write;
+use tokio::sync::mpsc::UnboundedSender;
 
 /// macro helper to abstract away repetitive string formatting
 macro_rules! format_banner_entry_helper {
@@ -40,203 +49,705 @@ macro_rules! format_banner_entry {
     };
 }
 
+/// Url used to query github's api; specifically used to look for the latest tagged release name
+const UPDATE_URL: &str = "https://api.github.com/repos/epi052/feroxbuster/releases/latest";
+
+/// Simple enum to hold three different update states
+#[derive(Debug)]
+enum UpdateStatus {
+    /// this version and latest release are the same
+    UpToDate,
+
+    /// this version and latest release are not the same
+    OutOfDate,
+
+    /// some error occurred during version check
+    Unknown,
+}
+
+/// Makes a request to the given url, expecting to receive a JSON response that contains a field
+/// named `tag_name` that holds a value representing the latest tagged release of this tool.
+///
+/// ex: v1.1.0
+///
+/// Returns `UpdateStatus`
+async fn needs_update(
+    client: &Client,
+    url: &str,
+    bin_version: &str,
+    tx_stats: UnboundedSender<StatCommand>,
+) -> UpdateStatus {
+    log::trace!("enter: needs_update({:?}, {}, {:?})", client, url, tx_stats);
+
+    let unknown = UpdateStatus::Unknown;
+
+    let api_url = match Url::parse(url) {
+        Ok(url) => url,
+        Err(e) => {
+            log::error!("{}", e);
+            log::trace!("exit: needs_update -> {:?}", unknown);
+            return unknown;
+        }
+    };
+
+    if let Ok(response) = make_request(&client, &api_url, tx_stats.clone()).await {
+        let body = response.text().await.unwrap_or_default();
+
+        let json_response: Value = serde_json::from_str(&body).unwrap_or_default();
+
+        if json_response.is_null() {
+            // unwrap_or_default above should result in a null value for the json_response variable
+            log::error!("Could not parse JSON from response body");
+            log::trace!("exit: needs_update -> {:?}", unknown);
+            return unknown;
+        }
+
+        let latest_version = match json_response["tag_name"].as_str() {
+            Some(tag) => tag.trim_start_matches('v'),
+            None => {
+                log::error!("Could not get version field from JSON response");
+                log::debug!("{}", json_response);
+                log::trace!("exit: needs_update -> {:?}", unknown);
+                return unknown;
+            }
+        };
+
+        // if we've gotten this far, we have a string in the form of X.X.X where X is a number
+        // all that's left is to compare the current version with the version found above
+
+        return if latest_version == bin_version {
+            // there's really only two possible outcomes if we accept that the tag conforms to
+            // the X.X.X pattern:
+            //   1. the version strings match, meaning we're up to date
+            //   2. the version strings do not match, meaning we're out of date
+            //
+            // except for developers working on this code, nobody should ever be in a situation
+            // where they have a version greater than the latest tagged release
+            log::trace!("exit: needs_update -> UpdateStatus::UpToDate");
+            UpdateStatus::UpToDate
+        } else {
+            log::trace!("exit: needs_update -> UpdateStatus::OutOfDate");
+            UpdateStatus::OutOfDate
+        };
+    }
+
+    log::trace!("exit: needs_update -> {:?}", unknown);
+    unknown
+}
+
+/// Simple wrapper for emoji or fallback when terminal doesn't support emoji
+fn format_emoji(emoji: &str) -> String {
+    let width = console::measure_text_width(emoji);
+    let pad_len = width * width;
+    let pad = format!("{:<pad_len$}", "\u{0020}", pad_len = pad_len);
+    Emoji(emoji, &pad).to_string()
+}
+
 /// Prints the banner to stdout.
 ///
 /// Only prints those settings which are either always present, or passed in by the user.
-pub fn initialize(targets: &[String]) {
+pub async fn initialize<W>(
+    targets: &[String],
+    config: &Configuration,
+    version: &str,
+    mut writer: W,
+    tx_stats: UnboundedSender<StatCommand>,
+) where
+    W: Write,
+{
     let artwork = format!(
         r#"
  ___  ___  __   __     __      __         __   ___
 |__  |__  |__) |__) | /  `    /  \ \_/ | |  \ |__
 |    |___ |  \ |  \ | \__,    \__/ / \ | |__/ |___
-by Ben "epi" Risher {}                  ver: {}"#,
-        '\u{1F913}', VERSION
+by Ben "epi" Risher {}                 ver: {}"#,
+        Emoji("🤓", &format!("{:<2}", "\u{0020}")),
+        version
     );
+    let status = needs_update(&CONFIGURATION.client, UPDATE_URL, version, tx_stats).await;
 
     let top = "───────────────────────────┬──────────────────────";
+    let addl_section = "──────────────────────────────────────────────────";
     let bottom = "───────────────────────────┴──────────────────────";
 
-    eprintln!("{}", artwork);
-    eprintln!("{}", top);
+    writeln!(&mut writer, "{}", artwork).unwrap_or_default();
+    writeln!(&mut writer, "{}", top).unwrap_or_default();
 
     // begin with always printed items
     for target in targets {
-        eprintln!(
+        writeln!(
+            &mut writer,
             "{}",
-            format_banner_entry!("\u{1F3af}", "Target Url", target)
-        ); // 🎯
+            format_banner_entry!(format_emoji("🎯"), "Target Url", target)
+        )
+        .unwrap_or_default(); // 🎯
     }
 
     let mut codes = vec![];
 
-    for code in &CONFIGURATION.statuscodes {
+    for code in &config.status_codes {
         codes.push(status_colorizer(&code.to_string()))
     }
 
-    eprintln!(
+    writeln!(
+        &mut writer,
         "{}",
-        format_banner_entry!("\u{1F680}", "Threads", CONFIGURATION.threads)
-    ); // 🚀
-    eprintln!(
+        format_banner_entry!(format_emoji("🚀"), "Threads", config.threads)
+    )
+    .unwrap_or_default(); // 🚀
+
+    writeln!(
+        &mut writer,
         "{}",
-        format_banner_entry!("\u{1f4d6}", "Wordlist", CONFIGURATION.wordlist)
-    ); // 📖
-    eprintln!(
+        format_banner_entry!(format_emoji("📖"), "Wordlist", config.wordlist)
+    )
+    .unwrap_or_default(); // 📖
+
+    writeln!(
+        &mut writer,
         "{}",
         format_banner_entry!(
-            "\u{1F197}",
+            format_emoji("🆗"),
             "Status Codes",
             format!("[{}]", codes.join(", "))
         )
-    ); // 🆗
-    eprintln!(
+    )
+    .unwrap_or_default(); // 🆗
+
+    if !config.filter_status.is_empty() {
+        // exception here for optional print due to me wanting the allows and denys to be printed
+        // one after the other
+        let mut code_filters = vec![];
+
+        for code in &config.filter_status {
+            code_filters.push(status_colorizer(&code.to_string()))
+        }
+
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!(
+                format_emoji("🗑"),
+                "Status Code Filters",
+                format!("[{}]", code_filters.join(", "))
+            )
+        )
+        .unwrap_or_default(); // 🗑
+    }
+
+    writeln!(
+        &mut writer,
         "{}",
-        format_banner_entry!("\u{1f4a5}", "Timeout (secs)", CONFIGURATION.timeout)
-    ); // 💥
-    eprintln!(
+        format_banner_entry!(format_emoji("💥"), "Timeout (secs)", config.timeout)
+    )
+    .unwrap_or_default(); // 💥
+
+    writeln!(
+        &mut writer,
         "{}",
-        format_banner_entry!("\u{1F9a1}", "User-Agent", CONFIGURATION.useragent)
-    ); // 🦡
+        format_banner_entry!(format_emoji("🦡"), "User-Agent", config.user_agent)
+    )
+    .unwrap_or_default(); // 🦡
 
     // followed by the maybe printed or variably displayed values
-    if !CONFIGURATION.config.is_empty() {
-        eprintln!(
+    if !config.config.is_empty() {
+        writeln!(
+            &mut writer,
             "{}",
-            format_banner_entry!("\u{1f489}", "Config File", CONFIGURATION.config)
-        ); // 💉
+            format_banner_entry!(format_emoji("💉"), "Config File", config.config)
+        )
+        .unwrap_or_default(); // 💉
     }
 
-    if !CONFIGURATION.proxy.is_empty() {
-        eprintln!(
+    if !config.proxy.is_empty() {
+        writeln!(
+            &mut writer,
             "{}",
-            format_banner_entry!("\u{1f48e}", "Proxy", CONFIGURATION.proxy)
-        ); // 💎
+            format_banner_entry!(format_emoji("💎"), "Proxy", config.proxy)
+        )
+        .unwrap_or_default(); // 💎
     }
 
-    if !CONFIGURATION.headers.is_empty() {
-        for (name, value) in &CONFIGURATION.headers {
-            eprintln!(
+    if !config.replay_proxy.is_empty() {
+        // i include replay codes logic here because in config.rs, replay codes are set to the
+        // value in status codes, meaning it's never empty
+
+        let mut replay_codes = vec![];
+
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!(format_emoji("🎥"), "Replay Proxy", config.replay_proxy)
+        )
+        .unwrap_or_default(); // 🎥
+
+        for code in &config.replay_codes {
+            replay_codes.push(status_colorizer(&code.to_string()))
+        }
+
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!(
+                format_emoji("📼"),
+                "Replay Proxy Codes",
+                format!("[{}]", replay_codes.join(", "))
+            )
+        )
+        .unwrap_or_default(); // 📼
+    }
+
+    if !config.headers.is_empty() {
+        for (name, value) in &config.headers {
+            writeln!(
+                &mut writer,
                 "{}",
-                format_banner_entry!("\u{1f92f}", "Header", name, value)
-            ); // 🤯
+                format_banner_entry!(format_emoji("🤯"), "Header", name, value)
+            )
+            .unwrap_or_default(); // 🤯
         }
     }
 
-    if !CONFIGURATION.sizefilters.is_empty() {
-        for filter in &CONFIGURATION.sizefilters {
-            eprintln!(
+    if !config.filter_size.is_empty() {
+        for filter in &config.filter_size {
+            writeln!(
+                &mut writer,
                 "{}",
-                format_banner_entry!("\u{1f4a2}", "Size Filter", filter)
-            ); // 💢
+                format_banner_entry!(format_emoji("💢"), "Size Filter", filter)
+            )
+            .unwrap_or_default(); // 💢
         }
     }
 
-    if !CONFIGURATION.queries.is_empty() {
-        for query in &CONFIGURATION.queries {
-            eprintln!(
+    if !config.filter_similar.is_empty() {
+        for filter in &config.filter_similar {
+            writeln!(
+                &mut writer,
+                "{}",
+                format_banner_entry!(format_emoji("💢"), "Similarity Filter", filter)
+            )
+            .unwrap_or_default(); // 💢
+        }
+    }
+
+    for filter in &config.filter_word_count {
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!(format_emoji("💢"), "Word Count Filter", filter)
+        )
+        .unwrap_or_default(); // 💢
+    }
+
+    for filter in &config.filter_line_count {
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!(format_emoji("💢"), "Line Count Filter", filter)
+        )
+        .unwrap_or_default(); // 💢
+    }
+
+    for filter in &config.filter_regex {
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!(format_emoji("💢"), "Regex Filter", filter)
+        )
+        .unwrap_or_default(); // 💢
+    }
+
+    if config.extract_links {
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!(format_emoji("🔎"), "Extract Links", config.extract_links)
+        )
+        .unwrap_or_default(); // 🔎
+    }
+
+    if config.json {
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!(format_emoji("🧔"), "JSON Output", config.json)
+        )
+        .unwrap_or_default(); // 🧔
+    }
+
+    if !config.queries.is_empty() {
+        for query in &config.queries {
+            writeln!(
+                &mut writer,
                 "{}",
                 format_banner_entry!(
-                    "\u{1f914}",
+                    format_emoji("🤔"),
                     "Query Parameter",
                     format!("{}={}", query.0, query.1)
                 )
-            ); // 🤔
+            )
+            .unwrap_or_default(); // 🤔
         }
     }
 
-    if !CONFIGURATION.output.is_empty() {
-        eprintln!(
+    if !config.output.is_empty() {
+        writeln!(
+            &mut writer,
             "{}",
-            format_banner_entry!("\u{1f4be}", "Output File", CONFIGURATION.output)
-        ); // 💾
+            format_banner_entry!(format_emoji("💾"), "Output File", config.output)
+        )
+        .unwrap_or_default(); // 💾
     }
 
-    if !CONFIGURATION.extensions.is_empty() {
-        eprintln!(
+    if !config.debug_log.is_empty() {
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!(format_emoji("🪲"), "Debugging Log", config.debug_log)
+        )
+        .unwrap_or_default(); // 🪲
+    }
+
+    if !config.extensions.is_empty() {
+        writeln!(
+            &mut writer,
             "{}",
             format_banner_entry!(
-                "\u{1f4b2}",
+                format_emoji("💲"),
                 "Extensions",
-                format!("[{}]", CONFIGURATION.extensions.join(", "))
+                format!("[{}]", config.extensions.join(", "))
             )
-        ); // 💲
+        )
+        .unwrap_or_default(); // 💲
     }
 
-    if CONFIGURATION.insecure {
-        eprintln!(
+    if config.insecure {
+        writeln!(
+            &mut writer,
             "{}",
-            format_banner_entry!("\u{1f513}", "Insecure", CONFIGURATION.insecure)
-        ); // 🔓
+            format_banner_entry!(format_emoji("🔓"), "Insecure", config.insecure)
+        )
+        .unwrap_or_default(); // 🔓
     }
 
-    if CONFIGURATION.redirects {
-        eprintln!(
+    if config.redirects {
+        writeln!(
+            &mut writer,
             "{}",
-            format_banner_entry!("\u{1f4cd}", "Follow Redirects", CONFIGURATION.redirects)
-        ); // 📍
+            format_banner_entry!(format_emoji("📍"), "Follow Redirects", config.redirects)
+        )
+        .unwrap_or_default(); // 📍
     }
 
-    if CONFIGURATION.dontfilter {
-        eprintln!(
+    if config.dont_filter {
+        writeln!(
+            &mut writer,
             "{}",
-            format_banner_entry!("\u{1f92a}", "Filter Wildcards", !CONFIGURATION.dontfilter)
-        ); // 🤪
+            format_banner_entry!(format_emoji("🤪"), "Filter Wildcards", !config.dont_filter)
+        )
+        .unwrap_or_default(); // 🤪
     }
 
-    match CONFIGURATION.verbosity {
+    let volume = ["🔈", "🔉", "🔊", "📢"];
+    if let 1..=4 = config.verbosity {
         //speaker medium volume (increasing with verbosity to loudspeaker)
-        1 => {
-            eprintln!(
-                "{}",
-                format_banner_entry!("\u{1f508}", "Verbosity", CONFIGURATION.verbosity)
-            ); // 🔈
-        }
-        2 => {
-            eprintln!(
-                "{}",
-                format_banner_entry!("\u{1f509}", "Verbosity", CONFIGURATION.verbosity)
-            ); // 🔉
-        }
-        3 => {
-            eprintln!(
-                "{}",
-                format_banner_entry!("\u{1f50a}", "Verbosity", CONFIGURATION.verbosity)
-            ); // 🔊
-        }
-        4 => {
-            eprintln!(
-                "{}",
-                format_banner_entry!("\u{1f4e2}", "Verbosity", CONFIGURATION.verbosity)
-            ); // 📢
-        }
-        _ => {}
-    }
-
-    if CONFIGURATION.addslash {
-        eprintln!(
+        writeln!(
+            &mut writer,
             "{}",
-            format_banner_entry!("\u{1fa93}", "Add Slash", CONFIGURATION.addslash)
-        ); // 🪓
+            format_banner_entry!(
+                format_emoji(volume[config.verbosity as usize - 1]),
+                "Verbosity",
+                config.verbosity
+            )
+        )
+        .unwrap_or_default();
     }
 
-    if !CONFIGURATION.norecursion {
-        if CONFIGURATION.depth == 0 {
-            eprintln!(
+    if config.add_slash {
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!(format_emoji("🪓"), "Add Slash", config.add_slash)
+        )
+        .unwrap_or_default(); // 🪓
+    }
+
+    if !config.no_recursion {
+        if config.depth == 0 {
+            writeln!(
+                &mut writer,
                 "{}",
-                format_banner_entry!("\u{1f503}", "Recursion Depth", "INFINITE")
-            ); // 🔃
+                format_banner_entry!(format_emoji("🔃"), "Recursion Depth", "INFINITE")
+            )
+            .unwrap_or_default(); // 🔃
         } else {
-            eprintln!(
+            writeln!(
+                &mut writer,
                 "{}",
-                format_banner_entry!("\u{1f503}", "Recursion Depth", CONFIGURATION.depth)
-            ); // 🔃
+                format_banner_entry!(format_emoji("🔃"), "Recursion Depth", config.depth)
+            )
+            .unwrap_or_default(); // 🔃
         }
     } else {
-        eprintln!(
+        writeln!(
+            &mut writer,
             "{}",
-            format_banner_entry!("\u{1f6ab}", "Do Not Recurse", CONFIGURATION.norecursion)
-        ); // 🚫
+            format_banner_entry!(format_emoji("🚫"), "Do Not Recurse", config.no_recursion)
+        )
+        .unwrap_or_default(); // 🚫
     }
 
-    eprintln!("{}", bottom);
+    if CONFIGURATION.scan_limit > 0 {
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!(
+                format_emoji("🦥"),
+                "Concurrent Scan Limit",
+                config.scan_limit
+            )
+        )
+        .unwrap_or_default(); // 🦥
+    }
+
+    if !CONFIGURATION.time_limit.is_empty() {
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!(format_emoji("🕖"), "Time Limit", config.time_limit)
+        )
+        .unwrap_or_default(); // 🕖
+    }
+
+    if matches!(status, UpdateStatus::OutOfDate) {
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!(
+                format_emoji("🎉"),
+                "New Version Available",
+                "https://github.com/epi052/feroxbuster/releases/latest"
+            )
+        )
+        .unwrap_or_default(); // 🎉
+    }
+
+    writeln!(&mut writer, "{}", bottom).unwrap_or_default();
+    // ⏯
+    writeln!(
+        &mut writer,
+        " {}  Press [{}] to use the {}™",
+        format_emoji("🏁"),
+        style("ENTER").yellow(),
+        style("Scan Cancel Menu").bright().yellow(),
+    )
+    .unwrap_or_default();
+
+    writeln!(&mut writer, "{}", addl_section).unwrap_or_default();
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::{FeroxChannel, VERSION};
+    use httpmock::Method::GET;
+    use httpmock::MockServer;
+    use std::fs::read_to_string;
+    use std::io::stderr;
+    use std::time::Duration;
+    use tempfile::NamedTempFile;
+    use tokio::sync::mpsc;
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// test to hit no execution of targets for loop in banner
+    async fn banner_intialize_without_targets() {
+        let config = Configuration::default();
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
+        initialize(&[], &config, VERSION, stderr(), tx).await;
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// test to hit no execution of statuscode for loop in banner
+    async fn banner_intialize_without_status_codes() {
+        let config = Configuration {
+            status_codes: vec![],
+            ..Default::default()
+        };
+
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
+        initialize(
+            &[String::from("http://localhost")],
+            &config,
+            VERSION,
+            stderr(),
+            tx,
+        )
+        .await;
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// test to hit an empty config file
+    async fn banner_intialize_without_config_file() {
+        let config = Configuration {
+            config: String::new(),
+            ..Default::default()
+        };
+
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
+        initialize(
+            &[String::from("http://localhost")],
+            &config,
+            VERSION,
+            stderr(),
+            tx,
+        )
+        .await;
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// test to hit an empty config file
+    async fn banner_intialize_without_queries() {
+        let config = Configuration {
+            queries: vec![(String::new(), String::new())],
+            ..Default::default()
+        };
+
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
+        initialize(
+            &[String::from("http://localhost")],
+            &config,
+            VERSION,
+            stderr(),
+            tx,
+        )
+        .await;
+    }
+
+    #[ignore]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// test to show that a new version is available for download
+    async fn banner_intialize_with_mismatched_version() {
+        let config = Configuration::default();
+        let file = NamedTempFile::new().unwrap();
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
+        initialize(
+            &[String::from("http://localhost")],
+            &config,
+            "mismatched-version",
+            &file,
+            tx,
+        )
+        .await;
+        let contents = read_to_string(file.path()).unwrap();
+        println!("contents: {}", contents);
+        assert!(contents.contains("New Version Available"));
+        assert!(contents.contains("https://github.com/epi052/feroxbuster/releases/latest"));
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// test that
+    async fn banner_needs_update_returns_unknown_with_bad_url() {
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
+        let result = needs_update(&CONFIGURATION.client, &"", VERSION, tx).await;
+        assert!(matches!(result, UpdateStatus::Unknown));
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// test return value of good url to needs_update
+    async fn banner_needs_update_returns_up_to_date() {
+        let srv = MockServer::start();
+
+        let mock = srv.mock(|when, then| {
+            when.method(GET).path("/latest");
+            then.status(200).body("{\"tag_name\":\"v1.1.0\"}");
+        });
+
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
+        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.1.0", tx).await;
+
+        assert_eq!(mock.hits(), 1);
+        assert!(matches!(result, UpdateStatus::UpToDate));
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// test return value of good url to needs_update that returns a newer version than current
+    async fn banner_needs_update_returns_out_of_date() {
+        let srv = MockServer::start();
+
+        let mock = srv.mock(|when, then| {
+            when.method(GET).path("/latest");
+            then.status(200).body("{\"tag_name\":\"v1.1.0\"}");
+        });
+
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
+        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1", tx).await;
+
+        assert_eq!(mock.hits(), 1);
+        assert!(matches!(result, UpdateStatus::OutOfDate));
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// test return value of good url that times out
+    async fn banner_needs_update_returns_unknown_on_timeout() {
+        let srv = MockServer::start();
+
+        let mock = srv.mock(|when, then| {
+            when.method(GET).path("/latest");
+            then.status(200)
+                .body("{\"tag_name\":\"v1.1.0\"}")
+                .delay(Duration::from_secs(8));
+        });
+
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
+        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1", tx).await;
+
+        assert_eq!(mock.hits(), 1);
+        assert!(matches!(result, UpdateStatus::Unknown));
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// test return value of good url with bad json response
+    async fn banner_needs_update_returns_unknown_on_bad_json_response() {
+        let srv = MockServer::start();
+
+        let mock = srv.mock(|when, then| {
+            when.method(GET).path("/latest");
+            then.status(200).body("not json");
+        });
+
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
+        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1", tx).await;
+
+        assert_eq!(mock.hits(), 1);
+        assert!(matches!(result, UpdateStatus::Unknown));
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// test return value of good url with json response that lacks the tag_name field
+    async fn banner_needs_update_returns_unknown_on_json_without_correct_tag() {
+        let srv = MockServer::start();
+
+        let mock = srv.mock(|when, then| {
+            when.method(GET).path("/latest");
+            then.status(200)
+                .body("{\"no tag_name\": \"doesn't exist\"}");
+        });
+
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
+        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1", tx).await;
+
+        assert_eq!(mock.hits(), 1);
+        assert!(matches!(result, UpdateStatus::Unknown));
+    }
 }

src/client.rs

@@ -1,16 +1,16 @@
-use crate::utils::status_colorizer;
-use ansi_term::Color::Cyan;
+use crate::utils::{module_colorizer, status_colorizer};
 use reqwest::header::HeaderMap;
 use reqwest::{redirect::Policy, Client, Proxy};
 use std::collections::HashMap;
 use std::convert::TryInto;
+#[cfg(not(test))]
 use std::process::exit;
 use std::time::Duration;
 
 /// Create and return an instance of [reqwest::Client](https://docs.rs/reqwest/latest/reqwest/struct.Client.html)
 pub fn initialize(
     timeout: u64,
-    useragent: &str,
+    user_agent: &str,
     redirects: bool,
     insecure: bool,
     headers: &HashMap<String, String>,
@@ -22,64 +22,80 @@ pub fn initialize(
         Policy::none()
     };
 
-    let header_map: HeaderMap = match headers.try_into() {
-        Ok(map) => map,
-        Err(e) => {
-            eprintln!(
-                "{} {} {}",
-                status_colorizer("ERROR"),
-                Cyan.paint("Client::initialize"),
-                e
-            );
-            exit(1);
-        }
-    };
+    // try_into returns infallible as its error, unwrap is safe here
+    let header_map: HeaderMap = headers.try_into().unwrap();
 
     let client = Client::builder()
         .timeout(Duration::new(timeout, 0))
-        .user_agent(useragent)
+        .user_agent(user_agent)
         .danger_accept_invalid_certs(insecure)
         .default_headers(header_map)
         .redirect(policy);
 
-    let client = if proxy.is_some() && !proxy.unwrap().is_empty() {
-        match Proxy::all(proxy.unwrap()) {
-            Ok(proxy_obj) => client.proxy(proxy_obj),
-            Err(e) => {
-                eprintln!(
-                    "{} {} Could not add proxy ({:?}) to Client configuration",
-                    status_colorizer("ERROR"),
-                    Cyan.paint("Client::initialize"),
-                    proxy
-                );
-                eprintln!(
-                    "{} {} {}",
-                    status_colorizer("ERROR"),
-                    Cyan.paint("Client::initialize"),
-                    e
-                );
-                exit(1);
+    let client = match proxy {
+        // a proxy is specified, need to add it to the client
+        Some(some_proxy) => {
+            if !some_proxy.is_empty() {
+                // it's not an empty string
+                match Proxy::all(some_proxy) {
+                    Ok(proxy_obj) => client.proxy(proxy_obj),
+                    Err(e) => {
+                        eprintln!(
+                            "{} {} {}",
+                            status_colorizer("ERROR"),
+                            module_colorizer("Client::initialize"),
+                            e
+                        );
+
+                        #[cfg(test)]
+                        panic!();
+                        #[cfg(not(test))]
+                        exit(1);
+                    }
+                }
+            } else {
+                client // Some("") was used?
             }
         }
-    } else {
-        client
+        // no proxy specified
+        None => client,
     };
 
     match client.build() {
         Ok(client) => client,
         Err(e) => {
-            eprintln!(
-                "{} {} Could not create a Client with the given configuration, exiting.",
-                status_colorizer("ERROR"),
-                Cyan.paint("Client::build")
-            );
             eprintln!(
                 "{} {} {}",
                 status_colorizer("ERROR"),
-                Cyan.paint("Client::build"),
+                module_colorizer("Client::build"),
                 e
             );
+
+            #[cfg(test)]
+            panic!();
+            #[cfg(not(test))]
             exit(1);
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    #[should_panic]
+    /// create client with a bad proxy, expect panic
+    fn client_with_bad_proxy() {
+        let headers = HashMap::new();
+        initialize(0, "stuff", true, false, &headers, Some("not a valid proxy"));
+    }
+
+    #[test]
+    /// create client with a proxy, expect no error
+    fn client_with_good_proxy() {
+        let headers = HashMap::new();
+        let proxy = "http://127.0.0.1:8080";
+        initialize(0, "stuff", true, true, &headers, Some(proxy));
+    }
+}

src/extractor.rs

@@ -0,0 +1,504 @@
+use crate::{
+    client,
+    config::{Configuration, CONFIGURATION},
+    scanner::SCANNED_URLS,
+    statistics::{
+        StatCommand::{self, UpdateUsizeField},
+        StatField::{LinksExtracted, TotalExpected},
+    },
+    utils::{format_url, make_request},
+    FeroxResponse,
+};
+use lazy_static::lazy_static;
+use regex::Regex;
+use reqwest::Url;
+use std::collections::HashSet;
+use tokio::sync::mpsc::UnboundedSender;
+
+/// Regular expression used in [LinkFinder](https://github.com/GerbenJavado/LinkFinder)
+///
+/// Incorporates change from this [Pull Request](https://github.com/GerbenJavado/LinkFinder/pull/66/files)
+const LINKFINDER_REGEX: &str = r#"(?:"|')(((?:[a-zA-Z]{1,10}://|//)[^"'/]{1,}\.[a-zA-Z]{2,}[^"']{0,})|((?:/|\.\./|\./)[^"'><,;| *()(%%$^/\\\[\]][^"'><,;|()]{1,})|([a-zA-Z0-9_\-/]{1,}/[a-zA-Z0-9_\-/]{1,}\.(?:[a-zA-Z]{1,4}|action)(?:[\?|#][^"|']{0,}|))|([a-zA-Z0-9_\-/]{1,}/[a-zA-Z0-9_\-/]{3,}(?:[\?|#][^"|']{0,}|))|([a-zA-Z0-9_\-.]{1,}\.(?:php|asp|aspx|jsp|json|action|html|js|txt|xml)(?:[\?|#][^"|']{0,}|)))(?:"|')"#;
+
+/// Regular expression to pull url paths from robots.txt
+///
+/// ref: https://developers.google.com/search/reference/robots_txt
+const ROBOTS_TXT_REGEX: &str =
+    r#"(?m)^ *(Allow|Disallow): *(?P<url_path>[a-zA-Z0-9._/?#@!&'()+,;%=-]+?)$"#; // multi-line (?m)
+
+lazy_static! {
+    /// `LINKFINDER_REGEX` as a regex::Regex type
+    static ref LINKS_REGEX: Regex = Regex::new(LINKFINDER_REGEX).unwrap();
+
+    /// `ROBOTS_TXT_REGEX` as a regex::Regex type
+    static ref ROBOTS_REGEX: Regex = Regex::new(ROBOTS_TXT_REGEX).unwrap();
+}
+
+/// Iterate over a given path, return a list of every sub-path found
+///
+/// example: `path` contains a link fragment `homepage/assets/img/icons/handshake.svg`
+/// the following fragments would be returned:
+///   - homepage/assets/img/icons/handshake.svg
+///   - homepage/assets/img/icons/
+///   - homepage/assets/img/
+///   - homepage/assets/
+///   - homepage/
+fn get_sub_paths_from_path(path: &str) -> Vec<String> {
+    log::trace!("enter: get_sub_paths_from_path({})", path);
+    let mut paths = vec![];
+
+    // filter out any empty strings caused by .split
+    let mut parts: Vec<&str> = path.split('/').filter(|s| !s.is_empty()).collect();
+
+    let length = parts.len();
+
+    for i in 0..length {
+        // iterate over all parts of the path
+        if parts.is_empty() {
+            // pop left us with an empty vector, we're done
+            break;
+        }
+
+        let mut possible_path = parts.join("/");
+
+        if possible_path.is_empty() {
+            // .join can result in an empty string, which we don't need, ignore
+            continue;
+        }
+
+        if i > 0 {
+            // this isn't the last index of the parts array
+            // ex: /buried/misc/stupidfile.php
+            // this block skips the file but sees all parent folders
+            possible_path = format!("{}/", possible_path);
+        }
+
+        paths.push(possible_path); // good sub-path found
+        parts.pop(); // use .pop() to remove the last part of the path and continue iteration
+    }
+
+    log::trace!("exit: get_sub_paths_from_path -> {:?}", paths);
+    paths
+}
+
+/// simple helper to stay DRY, trys to join a url + fragment and add it to the `links` HashSet
+fn add_link_to_set_of_links(link: &str, url: &Url, links: &mut HashSet<String>) {
+    log::trace!(
+        "enter: add_link_to_set_of_links({}, {}, {:?})",
+        link,
+        url.to_string(),
+        links
+    );
+    match url.join(&link) {
+        Ok(new_url) => {
+            links.insert(new_url.to_string());
+        }
+        Err(e) => {
+            log::error!("Could not join given url to the base url: {}", e);
+        }
+    }
+    log::trace!("exit: add_link_to_set_of_links");
+}
+
+/// Given a `reqwest::Response`, perform the following actions
+///   - parse the response's text for links using the linkfinder regex
+///   - for every link found take its url path and parse each sub-path
+///     - example: Response contains a link fragment `homepage/assets/img/icons/handshake.svg`
+///       with a base url of http://localhost, the following urls would be returned:
+///         - homepage/assets/img/icons/handshake.svg
+///         - homepage/assets/img/icons/
+///         - homepage/assets/img/
+///         - homepage/assets/
+///         - homepage/
+pub async fn get_links(
+    response: &FeroxResponse,
+    tx_stats: UnboundedSender<StatCommand>,
+) -> HashSet<String> {
+    log::trace!(
+        "enter: get_links({}, {:?})",
+        response.url().as_str(),
+        tx_stats
+    );
+
+    let mut links = HashSet::<String>::new();
+
+    let body = response.text();
+
+    for capture in LINKS_REGEX.captures_iter(&body) {
+        // remove single & double quotes from both ends of the capture
+        // capture[0] is the entire match, additional capture groups start at [1]
+        let link = capture[0].trim_matches(|c| c == '\'' || c == '"');
+
+        match Url::parse(link) {
+            Ok(absolute) => {
+                if absolute.domain() != response.url().domain()
+                    || absolute.host() != response.url().host()
+                {
+                    // domains/ips are not the same, don't scan things that aren't part of the original
+                    // target url
+                    continue;
+                }
+
+                add_all_sub_paths(absolute.path(), &response, &mut links);
+            }
+            Err(e) => {
+                // this is the expected error that happens when we try to parse a url fragment
+                //     ex: Url::parse("/login") -> Err("relative URL without a base")
+                // while this is technically an error, these are good results for us
+                if e.to_string().contains("relative URL without a base") {
+                    add_all_sub_paths(link, &response, &mut links);
+                } else {
+                    // unexpected error has occurred
+                    log::error!("Could not parse given url: {}", e);
+                }
+            }
+        }
+    }
+
+    let multiplier = CONFIGURATION.extensions.len().max(1);
+
+    update_stat!(tx_stats, UpdateUsizeField(LinksExtracted, links.len()));
+    update_stat!(
+        tx_stats,
+        UpdateUsizeField(TotalExpected, links.len() * multiplier)
+    );
+
+    log::trace!("exit: get_links -> {:?}", links);
+
+    links
+}
+
+/// take a url fragment like homepage/assets/img/icons/handshake.svg and
+/// incrementally add
+///     - homepage/assets/img/icons/
+///     - homepage/assets/img/
+///     - homepage/assets/
+///     - homepage/
+fn add_all_sub_paths(url_path: &str, response: &FeroxResponse, mut links: &mut HashSet<String>) {
+    log::trace!(
+        "enter: add_all_sub_paths({}, {}, {:?})",
+        url_path,
+        response,
+        links
+    );
+
+    for sub_path in get_sub_paths_from_path(url_path) {
+        log::debug!("Adding {} to {:?}", sub_path, links);
+        add_link_to_set_of_links(&sub_path, &response.url(), &mut links);
+    }
+
+    log::trace!("exit: add_all_sub_paths");
+}
+
+/// Wrapper around link extraction logic
+/// currently used in two places:
+///   - links from response bodys
+///   - links from robots.txt responses
+///
+/// general steps taken:
+///   - create a new Url object based on cli options/args
+///   - check if the new Url has already been seen/scanned -> None
+///   - make a request to the new Url ? -> Some(response) : None
+pub async fn request_feroxresponse_from_new_link(
+    url: &str,
+    tx_stats: UnboundedSender<StatCommand>,
+) -> Option<FeroxResponse> {
+    log::trace!(
+        "enter: request_feroxresponse_from_new_link({}, {:?})",
+        url,
+        tx_stats
+    );
+
+    // create a url based on the given command line options, return None on error
+    let new_url = match format_url(
+        &url,
+        &"",
+        CONFIGURATION.add_slash,
+        &CONFIGURATION.queries,
+        None,
+        tx_stats.clone(),
+    ) {
+        Ok(url) => url,
+        Err(_) => {
+            log::trace!("exit: request_feroxresponse_from_new_link -> None");
+            return None;
+        }
+    };
+
+    if SCANNED_URLS.get_scan_by_url(&new_url.to_string()).is_some() {
+        //we've seen the url before and don't need to scan again
+        log::trace!("exit: request_feroxresponse_from_new_link -> None");
+        return None;
+    }
+
+    // make the request and store the response
+    let new_response = match make_request(&CONFIGURATION.client, &new_url, tx_stats).await {
+        Ok(resp) => resp,
+        Err(_) => {
+            log::trace!("exit: request_feroxresponse_from_new_link -> None");
+            return None;
+        }
+    };
+
+    let new_ferox_response = FeroxResponse::from(new_response, true).await;
+
+    log::trace!(
+        "exit: request_feroxresponse_from_new_link -> {:?}",
+        new_ferox_response
+    );
+    Some(new_ferox_response)
+}
+
+/// helper function that simply requests /robots.txt on the given url's base url
+///
+/// example:
+///     http://localhost/api/users -> http://localhost/robots.txt
+///     
+/// The length of the given path has no effect on what's requested; it's always
+/// base url + /robots.txt
+pub async fn request_robots_txt(
+    base_url: &str,
+    config: &Configuration,
+    tx_stats: UnboundedSender<StatCommand>,
+) -> Option<FeroxResponse> {
+    log::trace!(
+        "enter: get_robots_file({}, CONFIGURATION, {:?})",
+        base_url,
+        tx_stats
+    );
+
+    // more often than not, domain/robots.txt will redirect to www.domain/robots.txt or something
+    // similar; to account for that, create a client that will follow redirects, regardless of
+    // what the user specified for the scanning client. Other than redirects, it will respect
+    // all other user specified settings
+    let follow_redirects = true;
+
+    let proxy = if config.proxy.is_empty() {
+        None
+    } else {
+        Some(config.proxy.as_str())
+    };
+
+    let client = client::initialize(
+        config.timeout,
+        &config.user_agent,
+        follow_redirects,
+        config.insecure,
+        &config.headers,
+        proxy,
+    );
+
+    if let Ok(mut url) = Url::parse(base_url) {
+        url.set_path("/robots.txt"); // overwrite existing path with /robots.txt
+
+        if let Ok(response) = make_request(&client, &url, tx_stats).await {
+            let ferox_response = FeroxResponse::from(response, true).await;
+
+            log::trace!("exit: get_robots_file -> {}", ferox_response);
+            return Some(ferox_response);
+        }
+    }
+
+    None
+}
+
+/// Entry point to perform link extraction from robots.txt
+///
+/// `base_url` can have paths and subpaths, however robots.txt will be requested from the
+/// root of the url
+/// given the url:
+///     http://localhost/stuff/things
+/// this function requests:
+///     http://localhost/robots.txt
+pub async fn extract_robots_txt(
+    base_url: &str,
+    config: &Configuration,
+    tx_stats: UnboundedSender<StatCommand>,
+) -> HashSet<String> {
+    log::trace!(
+        "enter: extract_robots_txt({}, CONFIGURATION, {:?})",
+        base_url,
+        tx_stats
+    );
+    let mut links = HashSet::new();
+
+    if let Some(response) = request_robots_txt(&base_url, &config, tx_stats.clone()).await {
+        for capture in ROBOTS_REGEX.captures_iter(response.text.as_str()) {
+            if let Some(new_path) = capture.name("url_path") {
+                if let Ok(mut new_url) = Url::parse(base_url) {
+                    new_url.set_path(new_path.as_str());
+                    add_all_sub_paths(new_url.path(), &response, &mut links);
+                }
+            }
+        }
+    }
+
+    let multiplier = CONFIGURATION.extensions.len().max(1);
+
+    update_stat!(tx_stats, UpdateUsizeField(LinksExtracted, links.len()));
+    update_stat!(
+        tx_stats,
+        UpdateUsizeField(TotalExpected, links.len() * multiplier)
+    );
+
+    log::trace!("exit: extract_robots_txt -> {:?}", links);
+    links
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::utils::make_request;
+    use crate::FeroxChannel;
+    use httpmock::Method::GET;
+    use httpmock::MockServer;
+    use reqwest::Client;
+    use tokio::sync::mpsc;
+
+    #[test]
+    /// extract sub paths from the given url fragment; expect 4 sub paths and that all are
+    /// in the expected array
+    fn extractor_get_sub_paths_from_path_with_multiple_paths() {
+        let path = "homepage/assets/img/icons/handshake.svg";
+        let paths = get_sub_paths_from_path(&path);
+        let expected = vec![
+            "homepage/",
+            "homepage/assets/",
+            "homepage/assets/img/",
+            "homepage/assets/img/icons/",
+            "homepage/assets/img/icons/handshake.svg",
+        ];
+
+        assert_eq!(paths.len(), expected.len());
+        for expected_path in expected {
+            assert_eq!(paths.contains(&expected_path.to_string()), true);
+        }
+    }
+
+    #[test]
+    /// extract sub paths from the given url fragment; expect 2 sub paths and that all are
+    /// in the expected array. the fragment is wrapped in slashes to ensure no empty strings are
+    /// returned
+    fn extractor_get_sub_paths_from_path_with_enclosing_slashes() {
+        let path = "/homepage/assets/";
+        let paths = get_sub_paths_from_path(&path);
+        let expected = vec!["homepage/", "homepage/assets"];
+
+        assert_eq!(paths.len(), expected.len());
+        for expected_path in expected {
+            assert_eq!(paths.contains(&expected_path.to_string()), true);
+        }
+    }
+
+    #[test]
+    /// extract sub paths from the given url fragment; expect 1 sub path, no forward slashes are
+    /// included
+    fn extractor_get_sub_paths_from_path_with_only_a_word() {
+        let path = "homepage";
+        let paths = get_sub_paths_from_path(&path);
+        let expected = vec!["homepage"];
+
+        assert_eq!(paths.len(), expected.len());
+        for expected_path in expected {
+            assert_eq!(paths.contains(&expected_path.to_string()), true);
+        }
+    }
+
+    #[test]
+    /// extract sub paths from the given url fragment; expect 1 sub path, forward slash removed
+    fn extractor_get_sub_paths_from_path_with_an_absolute_word() {
+        let path = "/homepage";
+        let paths = get_sub_paths_from_path(&path);
+        let expected = vec!["homepage"];
+
+        assert_eq!(paths.len(), expected.len());
+        for expected_path in expected {
+            assert_eq!(paths.contains(&expected_path.to_string()), true);
+        }
+    }
+
+    #[test]
+    /// test that a full url and fragment are joined correctly, then added to the given list
+    /// i.e. the happy path
+    fn extractor_add_link_to_set_of_links_happy_path() {
+        let url = Url::parse("https://localhost").unwrap();
+        let mut links = HashSet::<String>::new();
+        let link = "admin";
+
+        assert_eq!(links.len(), 0);
+        add_link_to_set_of_links(link, &url, &mut links);
+
+        assert_eq!(links.len(), 1);
+        assert!(links.contains("https://localhost/admin"));
+    }
+
+    #[test]
+    /// test that an invalid path fragment doesn't add anything to the set of links
+    fn extractor_add_link_to_set_of_links_with_non_base_url() {
+        let url = Url::parse("https://localhost").unwrap();
+        let mut links = HashSet::<String>::new();
+        let link = "\\\\\\\\";
+
+        assert_eq!(links.len(), 0);
+        add_link_to_set_of_links(link, &url, &mut links);
+
+        assert_eq!(links.len(), 0);
+        assert!(links.is_empty());
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// use make_request to generate a Response, and use the Response to test get_links;
+    /// the response will contain an absolute path to a domain that is not part of the scanned
+    /// domain; expect an empty set returned
+    async fn extractor_get_links_with_absolute_url_that_differs_from_target_domain(
+    ) -> Result<(), Box<dyn std::error::Error>> {
+        let srv = MockServer::start();
+
+        let mock = srv.mock(|when, then|{
+            when.method(GET)
+                .path("/some-path");
+            then.status(200)
+                .body("\"http://defintely.not.a.thing.probably.com/homepage/assets/img/icons/handshake.svg\"");
+        });
+
+        let client = Client::new();
+        let url = Url::parse(&srv.url("/some-path")).unwrap();
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
+        let response = make_request(&client, &url, tx.clone()).await.unwrap();
+
+        let ferox_response = FeroxResponse::from(response, true).await;
+
+        let links = get_links(&ferox_response, tx).await;
+
+        assert!(links.is_empty());
+
+        assert_eq!(mock.hits(), 1);
+        Ok(())
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// test that /robots.txt is correctly requested given a base url (happy path)
+    async fn request_robots_txt_with_and_without_proxy() {
+        let srv = MockServer::start();
+
+        let mock = srv.mock(|when, then| {
+            when.method(GET).path("/robots.txt");
+            then.status(200).body("this is a test");
+        });
+
+        let mut config = Configuration::default();
+
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
+        request_robots_txt(&srv.url("/api/users/stuff/things"), &config, tx.clone()).await;
+
+        // note: the proxy doesn't actually do anything other than hit a different code branch
+        // in this unit test; it would however have an effect on an integration test
+        config.proxy = srv.url("/ima-proxy");
+
+        request_robots_txt(&srv.url("/api/different/path"), &config, tx).await;
+
+        assert_eq!(mock.hits(), 2);
+    }
+}

src/filters/lines.rs

@@ -0,0 +1,33 @@
+use super::*;
+
+/// Simple implementor of FeroxFilter; used to filter out responses based on the number of lines
+/// in a Response body; specified using -N|--filter-lines
+#[derive(Default, Debug, PartialEq)]
+pub struct LinesFilter {
+    /// Number of lines in a Response's body that should be filtered
+    pub line_count: usize,
+}
+
+/// implementation of FeroxFilter for LinesFilter
+impl FeroxFilter for LinesFilter {
+    /// Check `line_count` against what was passed in via -N|--filter-lines
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        let result = response.line_count() == self.line_count;
+
+        log::trace!("exit: should_filter_response -> {}", result);
+
+        result
+    }
+
+    /// Compare one LinesFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/filters/mod.rs

@@ -0,0 +1,24 @@
+//! module containing all of feroxbuster's filters
+mod traits;
+mod wildcard;
+mod status_code;
+mod words;
+mod lines;
+mod size;
+mod regex;
+mod similarity;
+#[cfg(test)]
+mod tests;
+
+pub use self::lines::LinesFilter;
+pub use self::regex::RegexFilter;
+pub use self::similarity::SimilarityFilter;
+pub use self::size::SizeFilter;
+pub use self::status_code::StatusCodeFilter;
+pub use self::traits::FeroxFilter;
+pub use self::wildcard::WildcardFilter;
+pub use self::words::WordsFilter;
+
+use crate::{config::CONFIGURATION, utils::get_url_path_length, FeroxResponse, FeroxSerialize};
+use std::any::Any;
+use std::fmt::Debug;

src/filters/regex.rs

@@ -0,0 +1,46 @@
+use super::*;
+use ::regex::Regex;
+
+/// Simple implementor of FeroxFilter; used to filter out responses based on a given regular
+/// expression; specified using -X|--filter-regex
+#[derive(Debug)]
+pub struct RegexFilter {
+    /// Regular expression to be applied to the response body for filtering, compiled
+    pub compiled: Regex,
+
+    /// Regular expression as passed in on the command line, not compiled
+    pub raw_string: String,
+}
+
+/// implementation of FeroxFilter for RegexFilter
+impl FeroxFilter for RegexFilter {
+    /// Check `expression` against the response body, if the expression matches, the response
+    /// should be filtered out
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        let result = self.compiled.is_match(response.text());
+
+        log::trace!("exit: should_filter_response -> {}", result);
+
+        result
+    }
+
+    /// Compare one SizeFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
+
+/// PartialEq implementation for RegexFilter
+impl PartialEq for RegexFilter {
+    /// Simple comparison of the raw string passed in via the command line
+    fn eq(&self, other: &RegexFilter) -> bool {
+        self.raw_string == other.raw_string
+    }
+}

src/filters/similarity.rs

@@ -0,0 +1,40 @@
+use super::*;
+use fuzzyhash::FuzzyHash;
+
+/// Simple implementor of FeroxFilter; used to filter out responses based on the similarity of a
+/// Response body with a known response; specified using --filter-similar-to
+#[derive(Default, Debug, PartialEq)]
+pub struct SimilarityFilter {
+    /// Response's body to be used for comparison for similarity
+    pub text: String,
+
+    /// Percentage of similarity at which a page is determined to be a near-duplicate of another
+    pub threshold: u32,
+}
+
+/// implementation of FeroxFilter for SimilarityFilter
+impl FeroxFilter for SimilarityFilter {
+    /// Check `FeroxResponse::text` against what was requested from the site passed in via
+    /// --filter-similar-to
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        let other = FuzzyHash::new(&response.text);
+
+        if let Ok(result) = FuzzyHash::compare(&self.text, &other.to_string()) {
+            return result >= self.threshold;
+        }
+
+        // couldn't hash the response, don't filter
+        log::warn!("Could not hash body from {}", response.as_str());
+        false
+    }
+
+    /// Compare one SimilarityFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/filters/size.rs

@@ -0,0 +1,33 @@
+use super::*;
+
+/// Simple implementor of FeroxFilter; used to filter out responses based on the length of a
+/// Response body; specified using -S|--filter-size
+#[derive(Default, Debug, PartialEq)]
+pub struct SizeFilter {
+    /// Overall length of a Response's body that should be filtered
+    pub content_length: u64,
+}
+
+/// implementation of FeroxFilter for SizeFilter
+impl FeroxFilter for SizeFilter {
+    /// Check `content_length` against what was passed in via -S|--filter-size
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        let result = response.content_length() == self.content_length;
+
+        log::trace!("exit: should_filter_response -> {}", result);
+
+        result
+    }
+
+    /// Compare one SizeFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/filters/status_code.rs

@@ -0,0 +1,40 @@
+use super::*;
+
+/// Simple implementor of FeroxFilter; used to filter out status codes specified using
+/// -C|--filter-status
+#[derive(Default, Debug, PartialEq)]
+pub struct StatusCodeFilter {
+    /// Status code that should not be displayed to the user
+    pub filter_code: u16,
+}
+
+/// implementation of FeroxFilter for StatusCodeFilter
+impl FeroxFilter for StatusCodeFilter {
+    /// Check `filter_code` against what was passed in via -C|--filter-status
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        if response.status().as_u16() == self.filter_code {
+            log::debug!(
+                "filtered out {} based on --filter-status of {}",
+                response.url(),
+                self.filter_code
+            );
+            log::trace!("exit: should_filter_response -> true");
+            return true;
+        }
+
+        log::trace!("exit: should_filter_response -> false");
+        false
+    }
+
+    /// Compare one StatusCodeFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/filters/tests.rs

@@ -0,0 +1,188 @@
+use super::*;
+use ::fuzzyhash::FuzzyHash;
+use ::regex::Regex;
+use reqwest::Url;
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn lines_filter_as_any() {
+    let filter = LinesFilter { line_count: 1 };
+
+    assert_eq!(filter.line_count, 1);
+    assert_eq!(
+        *filter.as_any().downcast_ref::<LinesFilter>().unwrap(),
+        filter
+    );
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn words_filter_as_any() {
+    let filter = WordsFilter { word_count: 1 };
+
+    assert_eq!(filter.word_count, 1);
+    assert_eq!(
+        *filter.as_any().downcast_ref::<WordsFilter>().unwrap(),
+        filter
+    );
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn size_filter_as_any() {
+    let filter = SizeFilter { content_length: 1 };
+
+    assert_eq!(filter.content_length, 1);
+    assert_eq!(
+        *filter.as_any().downcast_ref::<SizeFilter>().unwrap(),
+        filter
+    );
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn status_code_filter_as_any() {
+    let filter = StatusCodeFilter { filter_code: 200 };
+
+    assert_eq!(filter.filter_code, 200);
+    assert_eq!(
+        *filter.as_any().downcast_ref::<StatusCodeFilter>().unwrap(),
+        filter
+    );
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn regex_filter_as_any() {
+    let raw = r".*\.txt$";
+    let compiled = Regex::new(raw).unwrap();
+    let filter = RegexFilter {
+        compiled,
+        raw_string: raw.to_string(),
+    };
+
+    assert_eq!(filter.raw_string, r".*\.txt$");
+    assert_eq!(
+        *filter.as_any().downcast_ref::<RegexFilter>().unwrap(),
+        filter
+    );
+}
+
+#[test]
+/// test should_filter on WilcardFilter where static logic matches
+fn wildcard_should_filter_when_static_wildcard_found() {
+    let resp = FeroxResponse {
+        text: String::new(),
+        wildcard: true,
+        url: Url::parse("http://localhost").unwrap(),
+        content_length: 100,
+        word_count: 50,
+        line_count: 25,
+        headers: reqwest::header::HeaderMap::new(),
+        status: reqwest::StatusCode::OK,
+    };
+
+    let filter = WildcardFilter {
+        size: 100,
+        dynamic: 0,
+    };
+
+    assert!(filter.should_filter_response(&resp));
+}
+
+#[test]
+/// test should_filter on WilcardFilter where dynamic logic matches
+fn wildcard_should_filter_when_dynamic_wildcard_found() {
+    let resp = FeroxResponse {
+        text: String::new(),
+        wildcard: true,
+        url: Url::parse("http://localhost/stuff").unwrap(),
+        content_length: 100,
+        word_count: 50,
+        line_count: 25,
+        headers: reqwest::header::HeaderMap::new(),
+        status: reqwest::StatusCode::OK,
+    };
+
+    let filter = WildcardFilter {
+        size: 0,
+        dynamic: 95,
+    };
+
+    assert!(filter.should_filter_response(&resp));
+}
+
+#[test]
+/// test should_filter on RegexFilter where regex matches body
+fn regexfilter_should_filter_when_regex_matches_on_response_body() {
+    let resp = FeroxResponse {
+        text: String::from("im a body response hurr durr!"),
+        wildcard: false,
+        url: Url::parse("http://localhost/stuff").unwrap(),
+        content_length: 100,
+        word_count: 50,
+        line_count: 25,
+        headers: reqwest::header::HeaderMap::new(),
+        status: reqwest::StatusCode::OK,
+    };
+
+    let raw = r"response...rr";
+
+    let filter = RegexFilter {
+        raw_string: raw.to_string(),
+        compiled: Regex::new(raw).unwrap(),
+    };
+
+    assert!(filter.should_filter_response(&resp));
+}
+
+#[test]
+/// a few simple tests for similarity filter
+fn similarity_filter_is_accurate() {
+    let mut resp = FeroxResponse {
+        text: String::from("sitting"),
+        wildcard: false,
+        url: Url::parse("http://localhost/stuff").unwrap(),
+        content_length: 100,
+        word_count: 50,
+        line_count: 25,
+        headers: reqwest::header::HeaderMap::new(),
+        status: reqwest::StatusCode::OK,
+    };
+
+    let mut filter = SimilarityFilter {
+        text: FuzzyHash::new("kitten").to_string(),
+        threshold: 95,
+    };
+
+    // kitten/sitting is 57% similar, so a threshold of 95 should not be filtered
+    assert!(!filter.should_filter_response(&resp));
+
+    resp.text = String::new();
+    filter.text = String::new();
+    filter.threshold = 100;
+
+    // two empty strings are the same, however ssdeep doesn't accept empty strings, expect false
+    assert!(!filter.should_filter_response(&resp));
+
+    resp.text = String::from("some data to hash for the purposes of running a test");
+    filter.text = FuzzyHash::new("some data to hash for the purposes of running a te").to_string();
+    filter.threshold = 17;
+
+    assert!(filter.should_filter_response(&resp));
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn similarity_filter_as_any() {
+    let filter = SimilarityFilter {
+        text: String::from("stuff"),
+        threshold: 95,
+    };
+
+    assert_eq!(filter.text, "stuff");
+    assert_eq!(
+        *filter.as_any().downcast_ref::<SimilarityFilter>().unwrap(),
+        filter
+    );
+}

src/filters/traits.rs

@@ -0,0 +1,27 @@
+use super::*;
+
+// references:
+//   https://dev.to/magnusstrale/rust-trait-objects-in-a-vector-non-trivial-4co5
+//   https://stackoverflow.com/questions/25339603/how-to-test-for-equality-between-trait-objects
+
+/// FeroxFilter trait; represents different types of possible filters that can be applied to
+/// responses
+pub trait FeroxFilter: Debug + Send + Sync {
+    /// Determine whether or not this particular filter should be applied or not
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool;
+
+    /// delegates to the FeroxFilter-implementing type which gives us the actual type of self
+    fn box_eq(&self, other: &dyn Any) -> bool;
+
+    /// gives us `other` as Any in box_eq
+    fn as_any(&self) -> &dyn Any;
+}
+
+/// implementation of PartialEq, necessary long-form due to "trait cannot be made into an object"
+/// error when attempting to derive PartialEq on the trait itself
+impl PartialEq for Box<dyn FeroxFilter> {
+    /// Perform a comparison of two implementors of the FeroxFilter trait
+    fn eq(&self, other: &Box<dyn FeroxFilter>) -> bool {
+        self.box_eq(other.as_any())
+    }
+}

src/filters/wildcard.rs

@@ -0,0 +1,73 @@
+use super::*;
+
+/// Data holder for two pieces of data needed when auto-filtering out wildcard responses
+///
+/// `dynamic` is the size of the response that will later be combined with the length
+/// of the path of the url requested and used to determine interesting pages from custom
+/// 404s where the requested url is reflected back in the response
+///
+/// `size` is size of the response that should be included with filters passed via runtime
+/// configuration and any static wildcard lengths.
+#[derive(Debug, Default, Clone, PartialEq)]
+pub struct WildcardFilter {
+    /// size of the response that will later be combined with the length of the path of the url
+    /// requested
+    pub dynamic: u64,
+
+    /// size of the response that should be included with filters passed via runtime configuration
+    pub size: u64,
+}
+
+/// implementation of FeroxFilter for WildcardFilter
+impl FeroxFilter for WildcardFilter {
+    /// Examine size, dynamic, and content_len to determine whether or not the response received
+    /// is a wildcard response and therefore should be filtered out
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        // quick return if dont_filter is set
+        if CONFIGURATION.dont_filter {
+            // --dont-filter applies specifically to wildcard filters, it is not a 100% catch all
+            // for not filtering anything.  As such, it should live in the implementation of
+            // a wildcard filter
+            return false;
+        }
+
+        if self.size > 0 && self.size == response.content_length() {
+            // static wildcard size found during testing
+            // size isn't default, size equals response length, and auto-filter is on
+            log::debug!("static wildcard: filtered out {}", response.url());
+            log::trace!("exit: should_filter_response -> true");
+            return true;
+        }
+
+        if self.dynamic > 0 {
+            // dynamic wildcard offset found during testing
+
+            // I'm about to manually split this url path instead of using reqwest::Url's
+            // builtin parsing. The reason is that they call .split() on the url path
+            // except that I don't want an empty string taking up the last index in the
+            // event that the url ends with a forward slash.  It's ugly enough to be split
+            // into its own function for readability.
+            let url_len = get_url_path_length(&response.url());
+
+            if url_len + self.dynamic == response.content_length() {
+                log::debug!("dynamic wildcard: filtered out {}", response.url());
+                log::trace!("exit: should_filter_response -> true");
+                return true;
+            }
+        }
+        log::trace!("exit: should_filter_response -> false");
+        false
+    }
+
+    /// Compare one WildcardFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/filters/words.rs

@@ -0,0 +1,33 @@
+use super::*;
+
+/// Simple implementor of FeroxFilter; used to filter out responses based on the number of words
+/// in a Response body; specified using -W|--filter-words
+#[derive(Default, Debug, PartialEq)]
+pub struct WordsFilter {
+    /// Number of words in a Response's body that should be filtered
+    pub word_count: usize,
+}
+
+/// implementation of FeroxFilter for WordsFilter
+impl FeroxFilter for WordsFilter {
+    /// Check `word_count` against what was passed in via -W|--filter-words
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        let result = response.word_count() == self.word_count;
+
+        log::trace!("exit: should_filter_response -> {}", result);
+
+        result
+    }
+
+    /// Compare one WordsFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/heuristics.rs

@@ -1,28 +1,19 @@
-use crate::config::{CONFIGURATION, PROGRESS_PRINTER};
-use crate::utils::{ferox_print, format_url, get_url_path_length, make_request, status_colorizer};
-use ansi_term::Color::{Cyan, Yellow};
+use crate::{
+    config::{CONFIGURATION, PROGRESS_PRINTER},
+    filters::WildcardFilter,
+    scanner::should_filter_response,
+    statistics::StatCommand,
+    utils::{ferox_print, format_url, get_url_path_length, make_request, status_colorizer},
+    FeroxResponse,
+};
+use console::style;
 use indicatif::ProgressBar;
-use reqwest::Response;
-use std::process;
+use tokio::sync::mpsc::UnboundedSender;
 use uuid::Uuid;
 
 /// length of a standard UUID, used when determining wildcard responses
 const UUID_LENGTH: u64 = 32;
 
-/// Data holder for two pieces of data needed when auto-filtering out wildcard responses
-///
-/// `dynamic` is the size of the response that will later be combined with the length
-/// of the path of the url requested and used to determine interesting pages from custom
-/// 404s where the requested url is reflected back in the response
-///
-/// `size` is size of the response that should be included with filters passed via runtime
-/// configuration and any static wildcard lengths.
-#[derive(Default, Debug)]
-pub struct WildcardFilter {
-    pub dynamic: u64,
-    pub size: u64,
-}
-
 /// Simple helper to return a uuid, formatted as lowercase without hyphens
 ///
 /// `length` determines the number of uuids to string together. Each uuid
@@ -46,22 +37,40 @@ fn unique_string(length: usize) -> String {
 ///
 /// In the event that url returns a wildcard response, a
 /// [WildcardFilter](struct.WildcardFilter.html) is created and returned to the caller.
-pub async fn wildcard_test(target_url: &str, bar: ProgressBar) -> Option<WildcardFilter> {
-    log::trace!("enter: wildcard_test({:?})", target_url);
+pub async fn wildcard_test(
+    target_url: &str,
+    bar: ProgressBar,
+    tx_term: UnboundedSender<FeroxResponse>,
+    tx_stats: UnboundedSender<StatCommand>,
+) -> Option<WildcardFilter> {
+    log::trace!(
+        "enter: wildcard_test({:?}, {:?}, {:?}, {:?})",
+        target_url,
+        bar,
+        tx_term,
+        tx_stats
+    );
 
-    if CONFIGURATION.dontfilter {
-        // early return, dontfilter scans don't need tested
+    if CONFIGURATION.dont_filter {
+        // early return, dont_filter scans don't need tested
         log::trace!("exit: wildcard_test -> None");
         return None;
     }
 
-    if let Some(resp_one) = make_wildcard_request(&target_url, 1).await {
+    let tx_term_mwcr1 = tx_term.clone();
+    let tx_term_mwcr2 = tx_term.clone();
+    let tx_stats_mwcr1 = tx_stats.clone();
+    let tx_stats_mwcr2 = tx_stats.clone();
+
+    if let Some(ferox_response) =
+        make_wildcard_request(&target_url, 1, tx_term_mwcr1, tx_stats_mwcr1).await
+    {
         bar.inc(1);
 
         // found a wildcard response
         let mut wildcard = WildcardFilter::default();
 
-        let wc_length = resp_one.content_length().unwrap_or(0);
+        let wc_length = ferox_response.content_length();
 
         if wc_length == 0 {
             log::trace!("exit: wildcard_test -> Some({:?})", wildcard);
@@ -70,42 +79,51 @@ pub async fn wildcard_test(target_url: &str, bar: ProgressBar) -> Option<Wildcar
 
         // content length of wildcard is non-zero, perform additional tests:
         //   make a second request, with a known-sized (64) longer request
-        if let Some(resp_two) = make_wildcard_request(&target_url, 3).await {
+        if let Some(resp_two) =
+            make_wildcard_request(&target_url, 3, tx_term_mwcr2, tx_stats_mwcr2).await
+        {
             bar.inc(1);
 
-            let wc2_length = resp_two.content_length().unwrap_or(0);
+            let wc2_length = resp_two.content_length();
 
             if wc2_length == wc_length + (UUID_LENGTH * 2) {
                 // second length is what we'd expect to see if the requested url is
                 // reflected in the response along with some static content; aka custom 404
-                let url_len = get_url_path_length(&resp_one.url());
-
-                if !CONFIGURATION.quiet {
-                    ferox_print(
-                    &format!(
-                            "{} {:>10} Wildcard response is dynamic; {} ({} + url length) responses; toggle this behavior by using {}",
-                            status_colorizer("WLD"),
-                            wc_length - url_len,
-                            Yellow.paint("auto-filtering"),
-                            Cyan.paint(format!("{}", wc_length - url_len)),
-                            Yellow.paint("--dontfilter")
-                        ), &PROGRESS_PRINTER
-                    );
-                }
+                let url_len = get_url_path_length(&ferox_response.url());
 
                 wildcard.dynamic = wc_length - url_len;
-            } else if wc_length == wc2_length {
+
                 if !CONFIGURATION.quiet {
-                    ferox_print(&format!(
-                        "{} {:>10} Wildcard response is static; {} {} responses; toggle this behavior by using {}",
-                        status_colorizer("WLD"),
-                        wc_length,
-                        Yellow.paint("auto-filtering"),
-                        Cyan.paint(format!("{}", wc_length)),
-                        Yellow.paint("--dontfilter")
-                    ), &PROGRESS_PRINTER);
+                    let msg = format!(
+                            "{} {:>9} {:>9} {:>9} Wildcard response is dynamic; {} ({} + url length) responses; toggle this behavior by using {}\n",
+                            status_colorizer("WLD"),
+                            "-",
+                            "-",
+                            "-",
+                            style("auto-filtering").yellow(),
+                            style(wc_length - url_len).cyan(),
+                            style("--dont-filter").yellow()
+                    );
+
+                    ferox_print(&msg, &PROGRESS_PRINTER);
                 }
+            } else if wc_length == wc2_length {
                 wildcard.size = wc_length;
+
+                if !CONFIGURATION.quiet {
+                    let msg = format!(
+                        "{} {:>9} {:>9} {:>9} Wildcard response is static; {} {} responses; toggle this behavior by using {}\n",
+                        status_colorizer("WLD"),
+                        "-",
+                        "-",
+                        "-",
+                        style("auto-filtering").yellow(),
+                        style(wc_length).cyan(),
+                        style("--dont-filter").yellow()
+                    );
+
+                    ferox_print(&msg, &PROGRESS_PRINTER);
+                }
             }
         } else {
             bar.inc(2);
@@ -125,17 +143,29 @@ pub async fn wildcard_test(target_url: &str, bar: ProgressBar) -> Option<Wildcar
 /// Once the unique url is created, the request is sent to the server. If the server responds
 /// back with a valid status code, the response is considered to be a wildcard response. If that
 /// wildcard response has a 3xx status code, that redirection location is displayed to the user.
-async fn make_wildcard_request(target_url: &str, length: usize) -> Option<Response> {
-    log::trace!("enter: make_wildcard_request({}, {})", target_url, length);
+async fn make_wildcard_request(
+    target_url: &str,
+    length: usize,
+    tx_file: UnboundedSender<FeroxResponse>,
+    tx_stats: UnboundedSender<StatCommand>,
+) -> Option<FeroxResponse> {
+    log::trace!(
+        "enter: make_wildcard_request({}, {}, {:?}, {:?})",
+        target_url,
+        length,
+        tx_file,
+        tx_stats,
+    );
 
     let unique_str = unique_string(length);
 
     let nonexistent = match format_url(
         target_url,
         &unique_str,
-        CONFIGURATION.addslash,
+        CONFIGURATION.add_slash,
         &CONFIGURATION.queries,
         None,
+        tx_stats.clone(),
     ) {
         Ok(url) => url,
         Err(e) => {
@@ -145,63 +175,31 @@ async fn make_wildcard_request(target_url: &str, length: usize) -> Option<Respon
         }
     };
 
-    let wildcard = status_colorizer("WLD");
-
-    match make_request(&CONFIGURATION.client, &nonexistent.to_owned()).await {
+    match make_request(
+        &CONFIGURATION.client,
+        &nonexistent.to_owned(),
+        tx_stats.clone(),
+    )
+    .await
+    {
         Ok(response) => {
             if CONFIGURATION
-                .statuscodes
+                .status_codes
                 .contains(&response.status().as_u16())
             {
                 // found a wildcard response
-                let url_len = get_url_path_length(&response.url());
-                let content_len = response.content_length().unwrap_or(0);
+                let mut ferox_response = FeroxResponse::from(response, true).await;
+                ferox_response.wildcard = true;
 
-                if !CONFIGURATION.quiet {
-                    ferox_print(
-                        &format!(
-                            "{} {:>10} Got {} for {} (url length: {})",
-                            wildcard,
-                            content_len,
-                            status_colorizer(&response.status().as_str()),
-                            response.url(),
-                            url_len
-                        ),
-                        &PROGRESS_PRINTER,
-                    );
+                if !CONFIGURATION.quiet
+                    && !should_filter_response(&ferox_response, tx_stats.clone())
+                    && tx_file.send(ferox_response.clone()).is_err()
+                {
+                    return None;
                 }
-                if response.status().is_redirection() {
-                    // show where it goes, if possible
-                    if let Some(next_loc) = response.headers().get("Location") {
-                        if let Ok(next_loc_str) = next_loc.to_str() {
-                            if !CONFIGURATION.quiet {
-                                ferox_print(
-                                    &format!(
-                                        "{} {:>10} {} redirects to => {}",
-                                        wildcard,
-                                        content_len,
-                                        response.url(),
-                                        next_loc_str
-                                    ),
-                                    &PROGRESS_PRINTER,
-                                );
-                            }
-                        } else if !CONFIGURATION.quiet {
-                            ferox_print(
-                                &format!(
-                                    "{} {:>10} {} redirects to => {:?}",
-                                    wildcard,
-                                    content_len,
-                                    response.url(),
-                                    next_loc
-                                ),
-                                &PROGRESS_PRINTER,
-                            );
-                        }
-                    }
-                }
-                log::trace!("exit: make_wildcard_request -> {:?}", response);
-                return Some(response);
+
+                log::trace!("exit: make_wildcard_request -> {}", ferox_response);
+                return Some(ferox_response);
             }
         }
         Err(e) => {
@@ -210,6 +208,7 @@ async fn make_wildcard_request(target_url: &str, length: usize) -> Option<Respon
             return None;
         }
     }
+
     log::trace!("exit: make_wildcard_request -> None");
     None
 }
@@ -219,8 +218,15 @@ async fn make_wildcard_request(target_url: &str, length: usize) -> Option<Respon
 /// In the event that no sites can be reached, the program will exit.
 ///
 /// Any urls that are found to be alive are returned to the caller.
-pub async fn connectivity_test(target_urls: &[String]) -> Vec<String> {
-    log::trace!("enter: connectivity_test({:?})", target_urls);
+pub async fn connectivity_test(
+    target_urls: &[String],
+    tx_stats: UnboundedSender<StatCommand>,
+) -> Vec<String> {
+    log::trace!(
+        "enter: connectivity_test({:?}, {:?})",
+        target_urls,
+        tx_stats
+    );
 
     let mut good_urls = vec![];
 
@@ -228,9 +234,10 @@ pub async fn connectivity_test(target_urls: &[String]) -> Vec<String> {
         let request = match format_url(
             target_url,
             "",
-            CONFIGURATION.addslash,
+            CONFIGURATION.add_slash,
             &CONFIGURATION.queries,
             None,
+            tx_stats.clone(),
         ) {
             Ok(url) => url,
             Err(e) => {
@@ -239,7 +246,7 @@ pub async fn connectivity_test(target_urls: &[String]) -> Vec<String> {
             }
         };
 
-        match make_request(&CONFIGURATION.client, &request).await {
+        match make_request(&CONFIGURATION.client, &request, tx_stats.clone()).await {
             Ok(_) => {
                 good_urls.push(target_url.to_owned());
             }
@@ -257,14 +264,6 @@ pub async fn connectivity_test(target_urls: &[String]) -> Vec<String> {
 
     if good_urls.is_empty() {
         log::error!("Could not connect to any target provided, exiting.");
-        log::trace!("exit: connectivity_test");
-        eprintln!(
-            "{} {} Could not connect to any target provided",
-            status_colorizer("ERROR"),
-            Cyan.paint("heuristics::connectivity_test"),
-        );
-
-        process::exit(1);
     }
 
     log::trace!("exit: connectivity_test -> {:?}", good_urls);
@@ -277,9 +276,18 @@ mod tests {
     use super::*;
 
     #[test]
-    fn unique_string_returns_correct_length() {
+    /// request a unique string of 32bytes * a value returns correct result
+    fn heuristics_unique_string_returns_correct_length() {
         for i in 0..10 {
             assert_eq!(unique_string(i).len(), i * 32);
         }
     }
+
+    #[test]
+    /// simply test the default values for wildcardfilter, expect 0, 0
+    fn heuristics_wildcardfilter_dafaults() {
+        let wcf = WildcardFilter::default();
+        assert_eq!(wcf.size, 0);
+        assert_eq!(wcf.dynamic, 0);
+    }
 }

src/lib.rs

@@ -1,22 +1,60 @@
+pub mod utils;
 pub mod banner;
 pub mod client;
 pub mod config;
+pub mod extractor;
+pub mod filters;
 pub mod heuristics;
 pub mod logger;
 pub mod parser;
 pub mod progress;
+pub mod reporter;
+pub mod scan_manager;
 pub mod scanner;
-pub mod utils;
+pub mod statistics;
 
-use reqwest::StatusCode;
+use crate::utils::{get_url_path_length, status_colorizer};
+use console::{style, Color};
+use reqwest::header::{HeaderName, HeaderValue};
+use reqwest::{header::HeaderMap, Response, StatusCode, Url};
+use serde::{ser::SerializeStruct, Deserialize, Deserializer, Serialize, Serializer};
+use serde_json::Value;
+use std::collections::HashMap;
+use std::convert::{TryFrom, TryInto};
+use std::str::FromStr;
+use std::{error, fmt};
+use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender};
 
 /// Generic Result type to ease error handling in async contexts
-pub type FeroxResult<T> =
-    std::result::Result<T, Box<dyn std::error::Error + Send + Sync + 'static>>;
+pub type FeroxResult<T> = std::result::Result<T, Box<dyn error::Error + Send + Sync + 'static>>;
+
+/// Simple Error implementation to allow for custom error returns
+#[derive(Debug, Default)]
+pub struct FeroxError {
+    /// fancy string that can be printed via Display
+    pub message: String,
+}
+
+impl error::Error for FeroxError {}
+
+impl fmt::Display for FeroxError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}", &self.message)
+    }
+}
+
+/// Generic mpsc::unbounded_channel type to tidy up some code
+pub type FeroxChannel<T> = (UnboundedSender<T>, UnboundedReceiver<T>);
 
 /// Version pulled from Cargo.toml at compile time
 pub const VERSION: &str = env!("CARGO_PKG_VERSION");
 
+/// Maximum number of file descriptors that can be opened during a scan
+pub const DEFAULT_OPEN_FILE_LIMIT: usize = 8192;
+
+/// Default value used to determine near-duplicate web pages (equivalent to 95%)
+pub const SIMILARITY_THRESHOLD: u32 = 95;
+
 /// Default wordlist to use when `-w|--wordlist` isn't specified and not `wordlist` isn't set
 /// in a [ferox-config.toml](constant.DEFAULT_CONFIG_NAME.html) config file.
 ///
@@ -25,6 +63,9 @@ pub const VERSION: &str = env!("CARGO_PKG_VERSION");
 pub const DEFAULT_WORDLIST: &str =
     "/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt";
 
+/// Number of milliseconds to wait between polls of `PAUSE_SCAN` when user pauses a scan
+pub static SLEEP_DURATION: u64 = 500;
+
 /// Default list of status codes to report
 ///
 /// * 200 Ok
@@ -52,3 +93,508 @@ pub const DEFAULT_STATUS_CODES: [StatusCode; 9] = [
 ///
 /// Expected location is in the same directory as the feroxbuster binary.
 pub const DEFAULT_CONFIG_NAME: &str = "ferox-config.toml";
+
+/// FeroxSerialize trait; represents different types that are Serialize and also implement
+/// as_str / as_json methods
+pub trait FeroxSerialize: Serialize {
+    /// Return a String representation of the object, generally the human readable version of the
+    /// implementor
+    fn as_str(&self) -> String;
+
+    /// Return an NDJSON representation of the object
+    fn as_json(&self) -> String;
+}
+
+/// A `FeroxResponse`, derived from a `Response` to a submitted `Request`
+#[derive(Debug, Clone)]
+pub struct FeroxResponse {
+    /// The final `Url` of this `FeroxResponse`
+    url: Url,
+
+    /// The `StatusCode` of this `FeroxResponse`
+    status: StatusCode,
+
+    /// The full response text
+    text: String,
+
+    /// The content-length of this response, if known
+    content_length: u64,
+
+    /// The number of lines contained in the body of this response, if known
+    line_count: usize,
+
+    /// The number of words contained in the body of this response, if known
+    word_count: usize,
+
+    /// The `Headers` of this `FeroxResponse`
+    headers: HeaderMap,
+
+    /// Wildcard response status
+    wildcard: bool,
+}
+
+/// Implement Display for FeroxResponse
+impl fmt::Display for FeroxResponse {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(
+            f,
+            "FeroxResponse {{ url: {}, status: {}, content-length: {} }}",
+            self.url(),
+            self.status(),
+            self.content_length()
+        )
+    }
+}
+
+/// `FeroxResponse` implementation
+impl FeroxResponse {
+    /// Get the `StatusCode` of this `FeroxResponse`
+    pub fn status(&self) -> &StatusCode {
+        &self.status
+    }
+
+    /// Get the final `Url` of this `FeroxResponse`.
+    pub fn url(&self) -> &Url {
+        &self.url
+    }
+
+    /// Get the full response text
+    pub fn text(&self) -> &str {
+        &self.text
+    }
+
+    /// Get the `Headers` of this `FeroxResponse`
+    pub fn headers(&self) -> &HeaderMap {
+        &self.headers
+    }
+
+    /// Get the content-length of this response, if known
+    pub fn content_length(&self) -> u64 {
+        self.content_length
+    }
+
+    /// Set `FeroxResponse`'s `url` attribute, has no affect if an error occurs
+    pub fn set_url(&mut self, url: &str) {
+        match Url::parse(&url) {
+            Ok(url) => {
+                self.url = url;
+            }
+            Err(e) => {
+                log::error!("Could not parse {} into a Url: {}", url, e);
+            }
+        };
+    }
+
+    /// Make a reasonable guess at whether the response is a file or not
+    ///
+    /// Examines the last part of a path to determine if it has an obvious extension
+    /// i.e. http://localhost/some/path/stuff.js where stuff.js indicates a file
+    ///
+    /// Additionally, inspects query parameters, as they're also often indicative of a file
+    pub fn is_file(&self) -> bool {
+        let has_extension = match self.url.path_segments() {
+            Some(path) => {
+                if let Some(last) = path.last() {
+                    last.contains('.') // last segment has some sort of extension, probably
+                } else {
+                    false
+                }
+            }
+            None => false,
+        };
+
+        self.url.query_pairs().count() > 0 || has_extension
+    }
+
+    /// Returns line count of the response text.
+    pub fn line_count(&self) -> usize {
+        self.line_count
+    }
+
+    /// Returns word count of the response text.
+    pub fn word_count(&self) -> usize {
+        self.word_count
+    }
+
+    /// Create a new `FeroxResponse` from the given `Response`
+    pub async fn from(response: Response, read_body: bool) -> Self {
+        let url = response.url().clone();
+        let status = response.status();
+        let headers = response.headers().clone();
+        let content_length = response.content_length().unwrap_or(0);
+
+        let text = if read_body {
+            // .text() consumes the response, must be called last
+            // additionally, --extract-links is currently the only place we use the body of the
+            // response, so we forego the processing if not performing extraction
+            match response.text().await {
+                // await the response's body
+                Ok(text) => text,
+                Err(e) => {
+                    log::error!("Could not parse body from response: {}", e);
+                    String::new()
+                }
+            }
+        } else {
+            String::new()
+        };
+
+        let line_count = text.lines().count();
+        let word_count = text.lines().map(|s| s.split_whitespace().count()).sum();
+
+        FeroxResponse {
+            url,
+            status,
+            content_length,
+            text,
+            headers,
+            line_count,
+            word_count,
+            wildcard: false,
+        }
+    }
+}
+
+/// Implement FeroxSerialusize::from(ize for FeroxRespons)e
+impl FeroxSerialize for FeroxResponse {
+    /// Simple wrapper around create_report_string
+    fn as_str(&self) -> String {
+        let lines = self.line_count().to_string();
+        let words = self.word_count().to_string();
+        let chars = self.content_length().to_string();
+        let status = self.status().as_str();
+        let wild_status = status_colorizer("WLD");
+
+        if self.wildcard {
+            // response is a wildcard, special messages abound when this is the case...
+
+            // create the base message
+            let mut message = format!(
+                "{} {:>8}l {:>8}w {:>8}c Got {} for {} (url length: {})\n",
+                wild_status,
+                lines,
+                words,
+                chars,
+                status_colorizer(&status),
+                self.url(),
+                get_url_path_length(&self.url())
+            );
+
+            if self.status().is_redirection() {
+                // when it's a redirect, show where it goes, if possible
+                if let Some(next_loc) = self.headers().get("Location") {
+                    let next_loc_str = next_loc.to_str().unwrap_or("Unknown");
+
+                    let redirect_msg = format!(
+                        "{} {:>9} {:>9} {:>9} {} redirects to => {}\n",
+                        wild_status,
+                        "-",
+                        "-",
+                        "-",
+                        self.url(),
+                        next_loc_str
+                    );
+
+                    message.push_str(&redirect_msg);
+                }
+            }
+
+            // base message + redirection message (if appropriate)
+            message
+        } else {
+            // not a wildcard, just create a normal entry
+            utils::create_report_string(
+                self.status.as_str(),
+                &lines,
+                &words,
+                &chars,
+                self.url().as_str(),
+            )
+        }
+    }
+
+    /// Create an NDJSON representation of the FeroxResponse
+    ///
+    /// (expanded for clarity)
+    /// ex:
+    /// {
+    ///    "type":"response",
+    ///    "url":"https://localhost.com/images",
+    ///    "path":"/images",
+    ///    "status":301,
+    ///    "content_length":179,
+    ///    "line_count":10,
+    ///    "word_count":16,
+    ///    "headers":{
+    ///       "x-content-type-options":"nosniff",
+    ///       "strict-transport-security":"max-age=31536000; includeSubDomains",
+    ///       "x-frame-options":"SAMEORIGIN",
+    ///       "connection":"keep-alive",
+    ///       "server":"nginx/1.16.1",
+    ///       "content-type":"text/html; charset=UTF-8",
+    ///       "referrer-policy":"origin-when-cross-origin",
+    ///       "content-security-policy":"default-src 'none'",
+    ///       "access-control-allow-headers":"X-Requested-With",
+    ///       "x-xss-protection":"1; mode=block",
+    ///       "content-length":"179",
+    ///       "date":"Mon, 23 Nov 2020 15:33:24 GMT",
+    ///       "location":"/images/",
+    ///       "access-control-allow-origin":"https://localhost.com"
+    ///    }
+    /// }\n
+    fn as_json(&self) -> String {
+        if let Ok(mut json) = serde_json::to_string(&self) {
+            json.push('\n');
+            json
+        } else {
+            format!("{{\"error\":\"could not convert {} to json\"}}", self.url())
+        }
+    }
+}
+
+/// Serialize implementation for FeroxResponse
+impl Serialize for FeroxResponse {
+    /// Function that handles serialization of a FeroxResponse to NDJSON
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let mut headers = HashMap::new();
+        let mut state = serializer.serialize_struct("FeroxResponse", 7)?;
+
+        // need to convert the HeaderMap to a HashMap in order to pass it to the serializer
+        for (key, value) in &self.headers {
+            let k = key.as_str().to_owned();
+            let v = String::from_utf8_lossy(value.as_bytes());
+            headers.insert(k, v);
+        }
+
+        state.serialize_field("type", "response")?;
+        state.serialize_field("url", self.url.as_str())?;
+        state.serialize_field("path", self.url.path())?;
+        state.serialize_field("wildcard", &self.wildcard)?;
+        state.serialize_field("status", &self.status.as_u16())?;
+        state.serialize_field("content_length", &self.content_length)?;
+        state.serialize_field("line_count", &self.line_count)?;
+        state.serialize_field("word_count", &self.word_count)?;
+        state.serialize_field("headers", &headers)?;
+
+        state.end()
+    }
+}
+
+/// Deserialize implementation for FeroxResponse
+impl<'de> Deserialize<'de> for FeroxResponse {
+    /// Deserialize a FeroxResponse from a serde_json::Value
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let mut response = Self {
+            url: Url::parse("http://localhost").unwrap(),
+            status: StatusCode::OK,
+            text: String::new(),
+            content_length: 0,
+            headers: HeaderMap::new(),
+            wildcard: false,
+            line_count: 0,
+            word_count: 0,
+        };
+
+        let map: HashMap<String, Value> = HashMap::deserialize(deserializer)?;
+
+        for (key, value) in &map {
+            match key.as_str() {
+                "url" => {
+                    if let Some(url) = value.as_str() {
+                        if let Ok(parsed) = Url::parse(url) {
+                            response.url = parsed;
+                        }
+                    }
+                }
+                "status" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(smaller) = u16::try_from(num) {
+                            if let Ok(status) = StatusCode::from_u16(smaller) {
+                                response.status = status;
+                            }
+                        }
+                    }
+                }
+                "content_length" => {
+                    if let Some(num) = value.as_u64() {
+                        response.content_length = num;
+                    }
+                }
+                "line_count" => {
+                    if let Some(num) = value.as_u64() {
+                        response.line_count = num.try_into().unwrap_or_default();
+                    }
+                }
+                "word_count" => {
+                    if let Some(num) = value.as_u64() {
+                        response.word_count = num.try_into().unwrap_or_default();
+                    }
+                }
+                "headers" => {
+                    let mut headers = HeaderMap::<HeaderValue>::default();
+
+                    if let Some(map_headers) = value.as_object() {
+                        for (h_key, h_value) in map_headers {
+                            let h_value_str = h_value.as_str().unwrap_or("");
+                            let h_name = HeaderName::from_str(h_key)
+                                .unwrap_or_else(|_| HeaderName::from_str("Unknown").unwrap());
+                            let h_value_parsed = HeaderValue::from_str(h_value_str)
+                                .unwrap_or_else(|_| HeaderValue::from_str("Unknown").unwrap());
+                            headers.insert(h_name, h_value_parsed);
+                        }
+                    }
+
+                    response.headers = headers;
+                }
+                "wildcard" => {
+                    if let Some(result) = value.as_bool() {
+                        response.wildcard = result;
+                    }
+                }
+                _ => {}
+            }
+        }
+
+        Ok(response)
+    }
+}
+
+#[derive(Serialize, Deserialize, Default)]
+/// Representation of a log entry, can be represented as a human readable string or JSON
+pub struct FeroxMessage {
+    #[serde(rename = "type")]
+    /// Name of this type of struct, used for serialization, i.e. `{"type":"log"}`
+    kind: String,
+
+    /// The log message
+    pub message: String,
+
+    /// The log level
+    pub level: String,
+
+    /// The number of seconds elapsed since the scan started
+    pub time_offset: f32,
+
+    /// The module from which log::* was called
+    pub module: String,
+}
+
+/// Implementation of FeroxMessage
+impl FeroxSerialize for FeroxMessage {
+    /// Create an NDJSON representation of the log message
+    ///
+    /// (expanded for clarity)
+    /// ex:
+    /// {
+    ///   "type": "log",
+    ///   "message": "Sent https://localhost/api to file handler",
+    ///   "level": "DEBUG",
+    ///   "time_offset": 0.86333454,
+    ///   "module": "feroxbuster::reporter"
+    /// }\n
+    fn as_json(&self) -> String {
+        if let Ok(mut json) = serde_json::to_string(&self) {
+            json.push('\n');
+            json
+        } else {
+            String::from("{\"error\":\"could not convert to json\"}")
+        }
+    }
+
+    /// Create a string representation of the log message
+    ///
+    /// ex:  301       10l       16w      173c https://localhost/api
+    fn as_str(&self) -> String {
+        let (level_name, level_color) = match self.level.as_str() {
+            "ERROR" => ("ERR", Color::Red),
+            "WARN" => ("WRN", Color::Red),
+            "INFO" => ("INF", Color::Cyan),
+            "DEBUG" => ("DBG", Color::Yellow),
+            "TRACE" => ("TRC", Color::Magenta),
+            "WILDCARD" => ("WLD", Color::Cyan),
+            _ => ("UNK", Color::White),
+        };
+
+        format!(
+            "{} {:10.03} {} {}\n",
+            style(level_name).bg(level_color).black(),
+            style(self.time_offset).dim(),
+            self.module,
+            style(&self.message).dim(),
+        )
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    /// asserts default config name is correct
+    fn default_config_name() {
+        assert_eq!(DEFAULT_CONFIG_NAME, "ferox-config.toml");
+    }
+
+    #[test]
+    /// asserts default wordlist is correct
+    fn default_wordlist() {
+        assert_eq!(
+            DEFAULT_WORDLIST,
+            "/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt"
+        );
+    }
+
+    #[test]
+    /// asserts default version is correct
+    fn default_version() {
+        assert_eq!(VERSION, env!("CARGO_PKG_VERSION"));
+    }
+
+    #[test]
+    /// test as_str method of FeroxMessage
+    fn ferox_message_as_str_returns_string_with_newline() {
+        let message = FeroxMessage {
+            message: "message".to_string(),
+            module: "utils".to_string(),
+            time_offset: 1.0,
+            level: "INFO".to_string(),
+            kind: "log".to_string(),
+        };
+        let message_str = message.as_str();
+
+        assert!(message_str.contains("INF"));
+        assert!(message_str.contains("1.000"));
+        assert!(message_str.contains("utils"));
+        assert!(message_str.contains("message"));
+        assert!(message_str.ends_with('\n'));
+    }
+
+    #[test]
+    /// test as_json method of FeroxMessage
+    fn ferox_message_as_json_returns_json_representation_of_ferox_message_with_newline() {
+        let message = FeroxMessage {
+            message: "message".to_string(),
+            module: "utils".to_string(),
+            time_offset: 1.0,
+            level: "INFO".to_string(),
+            kind: "log".to_string(),
+        };
+
+        let message_str = message.as_json();
+
+        let error_margin = f32::EPSILON;
+
+        let json: FeroxMessage = serde_json::from_str(&message_str).unwrap();
+        assert_eq!(json.module, message.module);
+        assert_eq!(json.message, message.message);
+        assert!((json.time_offset - message.time_offset).abs() < error_margin);
+        assert_eq!(json.level, message.level);
+        assert_eq!(json.kind, message.kind);
+    }
+}

src/logger.rs

@@ -1,5 +1,9 @@
-use crate::config::PROGRESS_PRINTER;
-use console::{style, Color};
+use crate::{
+    config::{CONFIGURATION, PROGRESS_PRINTER},
+    reporter::safe_file_write,
+    utils::open_file,
+    FeroxMessage, FeroxSerialize,
+};
 use env_logger::Builder;
 use std::env;
 use std::time::Instant;
@@ -18,8 +22,8 @@ pub fn initialize(verbosity: u8) {
                 0 => (),
                 1 => env::set_var("RUST_LOG", "warn"),
                 2 => env::set_var("RUST_LOG", "info"),
-                3 => env::set_var("RUST_LOG", "debug,hyper=info,reqwest=info"),
-                _ => env::set_var("RUST_LOG", "trace,hyper=info,reqwest=info"),
+                3 => env::set_var("RUST_LOG", "feroxbuster=debug,info"),
+                _ => env::set_var("RUST_LOG", "feroxbuster=trace,info"),
             }
         }
     }
@@ -27,27 +31,29 @@ pub fn initialize(verbosity: u8) {
     let start = Instant::now();
     let mut builder = Builder::from_default_env();
 
+    let debug_file = open_file(&CONFIGURATION.debug_log);
+
+    if let Some(buffered_file) = debug_file.clone() {
+        // write out the configuration to the debug file if it exists
+        safe_file_write(&*CONFIGURATION, buffered_file, CONFIGURATION.json);
+    }
+
     builder
         .format(move |_, record| {
-            let t = start.elapsed().as_secs_f32();
-            let level = record.level();
-
-            let (level_name, level_color) = match level {
-                log::Level::Error => ("ERR", Color::Red),
-                log::Level::Warn => ("WRN", Color::Red),
-                log::Level::Info => ("INF", Color::Cyan),
-                log::Level::Debug => ("DBG", Color::Yellow),
-                log::Level::Trace => ("TRC", Color::Magenta),
+            let log_entry = FeroxMessage {
+                message: record.args().to_string(),
+                level: record.level().to_string(),
+                time_offset: start.elapsed().as_secs_f32(),
+                module: record.target().to_string(),
+                kind: "log".to_string(),
             };
 
-            let msg = format!(
-                "{} {:10.03} {}",
-                style(level_name).bg(level_color).black(),
-                style(t).dim(),
-                style(record.args()).dim(),
-            );
+            PROGRESS_PRINTER.println(&log_entry.as_str());
+
+            if let Some(buffered_file) = debug_file.clone() {
+                safe_file_write(&log_entry, buffered_file, CONFIGURATION.json);
+            }
 
-            PROGRESS_PRINTER.println(msg);
             Ok(())
         })
         .init();

src/main.rs

@@ -1,17 +1,76 @@
-use ansi_term::Color::Cyan;
-use feroxbuster::config::{CONFIGURATION, PROGRESS_PRINTER};
-use feroxbuster::scanner::scan_url;
-use feroxbuster::utils::{get_current_depth, status_colorizer};
-use feroxbuster::{banner, heuristics, logger, FeroxResult};
+use crossterm::event::{self, Event, KeyCode};
+use feroxbuster::{
+    banner,
+    config::{CONFIGURATION, PROGRESS_BAR, PROGRESS_PRINTER},
+    heuristics, logger,
+    progress::{add_bar, BarType},
+    reporter,
+    scan_manager::{self, ScanStatus, PAUSE_SCAN},
+    scanner::{self, scan_url, SCANNED_URLS},
+    statistics::{
+        self,
+        StatCommand::{self, CreateBar, LoadStats, UpdateUsizeField},
+        StatField::InitialTargets,
+        Stats,
+    },
+    update_stat,
+    utils::{ferox_print, get_current_depth, module_colorizer, status_colorizer},
+    FeroxError, FeroxResponse, FeroxResult, SLEEP_DURATION, VERSION,
+};
+#[cfg(not(target_os = "windows"))]
+use feroxbuster::{utils::set_open_file_limit, DEFAULT_OPEN_FILE_LIMIT};
 use futures::StreamExt;
-use std::collections::HashSet;
-use std::fs::File;
-use std::io::{BufRead, BufReader};
-use std::process;
-use std::sync::Arc;
-use tokio::io;
+use std::{
+    collections::HashSet,
+    convert::TryInto,
+    fs::File,
+    io::{stderr, BufRead, BufReader},
+    process,
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        Arc,
+    },
+    thread::sleep,
+    time::Duration,
+};
+use tokio::{io, sync::mpsc::UnboundedSender, task::JoinHandle};
 use tokio_util::codec::{FramedRead, LinesCodec};
 
+/// Atomic boolean flag, used to determine whether or not the terminal input handler should exit
+pub static SCAN_COMPLETE: AtomicBool = AtomicBool::new(false);
+
+/// Handles specific key events triggered by the user over stdin
+fn terminal_input_handler() {
+    log::trace!("enter: terminal_input_handler");
+
+    loop {
+        if PAUSE_SCAN.load(Ordering::Relaxed) {
+            // if the scan is already paused, we don't want this event poller fighting the user
+            // over stdin
+            sleep(Duration::from_millis(SLEEP_DURATION));
+        } else if event::poll(Duration::from_millis(SLEEP_DURATION)).unwrap_or(false) {
+            // It's guaranteed that the `read()` won't block when the `poll()`
+            // function returns `true`
+
+            if let Ok(key_pressed) = event::read() {
+                // ignore any other keys
+                if key_pressed == Event::Key(KeyCode::Enter.into()) {
+                    // if the user presses Enter, set PAUSE_SCAN to true. The interactive menu
+                    // will be triggered and will handle setting PAUSE_SCAN to false
+                    PAUSE_SCAN.store(true, Ordering::Release);
+                }
+            }
+        } else {
+            // Timeout expired and no `Event` is available; use the timeout to check SCAN_COMPLETE
+            if SCAN_COMPLETE.load(Ordering::Relaxed) {
+                // scan has been marked complete by main, time to exit the loop
+                break;
+            }
+        }
+    }
+    log::trace!("exit: terminal_input_handler");
+}
+
 /// Create a HashSet of Strings from the given wordlist then stores it inside an Arc
 fn get_unique_words_from_wordlist(path: &str) -> FeroxResult<Arc<HashSet<String>>> {
     log::trace!("enter: get_unique_words_from_wordlist({})", path);
@@ -19,12 +78,6 @@ fn get_unique_words_from_wordlist(path: &str) -> FeroxResult<Arc<HashSet<String>
     let file = match File::open(&path) {
         Ok(f) => f,
         Err(e) => {
-            eprintln!(
-                "{} {} {}",
-                status_colorizer("ERROR"),
-                Cyan.paint("main::get_unique_words_from_wordlist"),
-                e
-            );
             log::error!("Could not open wordlist: {}", e);
             log::trace!("exit: get_unique_words_from_wordlist -> {}", e);
 
@@ -37,26 +90,39 @@ fn get_unique_words_from_wordlist(path: &str) -> FeroxResult<Arc<HashSet<String>
     let mut words = HashSet::new();
 
     for line in reader.lines() {
-        match line {
-            Ok(word) => {
-                words.insert(word);
-            }
-            Err(e) => {
-                log::warn!("Could not parse current line from wordlist : {}", e);
-            }
+        let result = line?;
+
+        if result.starts_with('#') || result.is_empty() {
+            continue;
         }
+
+        words.insert(result);
     }
 
     log::trace!(
         "exit: get_unique_words_from_wordlist -> Arc<wordlist[{} words...]>",
         words.len()
     );
+
     Ok(Arc::new(words))
 }
 
 /// Determine whether it's a single url scan or urls are coming from stdin, then scan as needed
-async fn scan(targets: Vec<String>) -> FeroxResult<()> {
-    log::trace!("enter: scan");
+async fn scan(
+    targets: Vec<String>,
+    stats: Arc<Stats>,
+    tx_term: UnboundedSender<FeroxResponse>,
+    tx_file: UnboundedSender<FeroxResponse>,
+    tx_stats: UnboundedSender<StatCommand>,
+) -> FeroxResult<()> {
+    log::trace!(
+        "enter: scan({:?}, {:?}, {:?}, {:?}, {:?})",
+        targets,
+        stats,
+        tx_term,
+        tx_file,
+        tx_stats
+    );
     // cloning an Arc is cheap (it's basically a pointer into the heap)
     // so that will allow for cheap/safe sharing of a single wordlist across multi-target scans
     // as well as additional directories found as part of recursion
@@ -65,23 +131,66 @@ async fn scan(targets: Vec<String>) -> FeroxResult<()> {
             .await??;
 
     if words.len() == 0 {
-        eprintln!(
-            "{} {} Did not find any words in {}",
-            status_colorizer("ERROR"),
-            Cyan.paint("main::scan"),
-            CONFIGURATION.wordlist
-        );
-        process::exit(1);
+        let err = FeroxError {
+            message: format!("Did not find any words in {}", CONFIGURATION.wordlist),
+        };
+
+        return Err(Box::new(err));
+    }
+
+    scanner::initialize(words.len(), &CONFIGURATION, tx_stats.clone()).await;
+
+    // at this point, the stat thread's progress bar can be created; things that needed to happen
+    // first:
+    // - banner gets printed
+    // - scanner initialized (this sent expected requests per directory to the stats thread, which
+    //   having been set, makes it so the progress bar doesn't flash as full before anything has
+    //   even happened
+    update_stat!(tx_stats, CreateBar);
+
+    if CONFIGURATION.resumed {
+        update_stat!(tx_stats, LoadStats(CONFIGURATION.resume_from.clone()));
+
+        SCANNED_URLS.print_known_responses();
+
+        if let Ok(scans) = SCANNED_URLS.scans.lock() {
+            for scan in scans.iter() {
+                if let Ok(locked_scan) = scan.lock() {
+                    if matches!(locked_scan.status, ScanStatus::Complete) {
+                        // these scans are complete, and just need to be shown to the user
+                        let pb = add_bar(
+                            &locked_scan.url,
+                            words.len().try_into().unwrap_or_default(),
+                            BarType::Message,
+                        );
+                        pb.finish();
+                    }
+                }
+            }
+        }
     }
 
     let mut tasks = vec![];
 
     for target in targets {
-        let wordclone = words.clone();
+        let word_clone = words.clone();
+        let term_clone = tx_term.clone();
+        let file_clone = tx_file.clone();
+        let tx_stats_clone = tx_stats.clone();
+        let stats_clone = stats.clone();
 
         let task = tokio::spawn(async move {
             let base_depth = get_current_depth(&target);
-            scan_url(&target, wordclone, base_depth).await;
+            scan_url(
+                &target,
+                word_clone,
+                base_depth,
+                stats_clone,
+                term_clone,
+                file_clone,
+                tx_stats_clone,
+            )
+            .await;
         });
 
         tasks.push(task);
@@ -94,7 +203,8 @@ async fn scan(targets: Vec<String>) -> FeroxResult<()> {
     Ok(())
 }
 
-async fn get_targets() -> Vec<String> {
+/// Get targets from either commandline or stdin, pass them back to the caller as a Result<Vec>
+async fn get_targets() -> FeroxResult<Vec<String>> {
     log::trace!("enter: get_targets");
 
     let mut targets = vec![];
@@ -106,12 +216,21 @@ async fn get_targets() -> Vec<String> {
         let mut reader = FramedRead::new(stdin, LinesCodec::new());
 
         while let Some(line) = reader.next().await {
-            match line {
-                Ok(target) => {
-                    targets.push(target);
-                }
-                Err(e) => {
-                    log::error!("{}", e);
+            targets.push(line?);
+        }
+    } else if CONFIGURATION.resumed {
+        // resume-from can't be used with --url, and --stdin is marked false for every resumed
+        // scan, making it mutually exclusive from either of the other two options
+        if let Ok(scans) = SCANNED_URLS.scans.lock() {
+            for scan in scans.iter() {
+                // SCANNED_URLS gets deserialized scans added to it at program start if --resume-from
+                // is used, so scans that aren't marked complete still need to be scanned
+                if let Ok(locked_scan) = scan.lock() {
+                    if matches!(locked_scan.status, ScanStatus::Complete) {
+                        // this one's already done, ignore it
+                        continue;
+                    }
+                    targets.push(locked_scan.url.to_owned());
                 }
             }
         }
@@ -121,35 +240,236 @@ async fn get_targets() -> Vec<String> {
 
     log::trace!("exit: get_targets -> {:?}", targets);
 
-    targets
+    Ok(targets)
 }
 
-#[tokio::main]
-async fn main() {
-    logger::initialize(CONFIGURATION.verbosity);
+/// async main called from real main, broken out in this way to allow for some synchronous code
+/// to be executed before bringing the tokio runtime online
+async fn wrapped_main() {
+    // join can only be called once, otherwise it causes the thread to panic
+    tokio::task::spawn_blocking(move || {
+        // ok, lazy_static! uses (unsurprisingly in retrospect) a lazy loading model where the
+        // thing obtained through deref isn't actually created until it's used. This created a
+        // problem when initializing the logger as it relied on PROGRESS_PRINTER which may or may
+        // not have been created by the time it was needed for logging (really only occurred in
+        // heuristics / banner / main). In order to initialize logging properly, we need to ensure
+        // PROGRESS_PRINTER and PROGRESS_BAR have been used at least once.  This call satisfies
+        // that constraint
+        PROGRESS_PRINTER.println("");
+        PROGRESS_BAR.join().unwrap();
+    });
 
+    let (stats, tx_stats, stats_handle) = statistics::initialize();
+
+    if !CONFIGURATION.time_limit.is_empty() {
+        // --time-limit value not an empty string, need to kick off the thread that enforces
+        // the limit
+
+        let max_time_stats = stats.clone();
+
+        tokio::spawn(async move {
+            scan_manager::start_max_time_thread(&CONFIGURATION.time_limit, max_time_stats).await
+        });
+    }
+
+    // can't trace main until after logger is initialized and the above task is started
     log::trace!("enter: main");
-    log::debug!("{:#?}", *CONFIGURATION);
+
+    // spawn a thread that listens for keyboard input on stdin, when a user presses enter
+    // the input handler will toggle PAUSE_SCAN, which in turn is used to pause and resume
+    // scans that are already running
+    tokio::task::spawn_blocking(terminal_input_handler);
+
+    let save_output = !CONFIGURATION.output.is_empty(); // was -o used?
+
+    if CONFIGURATION.save_state {
+        // start the ctrl+c handler
+        scan_manager::initialize(stats.clone());
+    }
+
+    let (tx_term, tx_file, term_handle, file_handle) =
+        reporter::initialize(&CONFIGURATION.output, save_output, tx_stats.clone());
 
     // get targets from command line or stdin
-    let targets = get_targets().await;
+    let targets = match get_targets().await {
+        Ok(t) => t,
+        Err(e) => {
+            // should only happen in the event that there was an error reading from stdin
+            log::error!("{} {}", module_colorizer("main::get_targets"), e);
+            clean_up(
+                tx_term,
+                term_handle,
+                tx_file,
+                file_handle,
+                tx_stats,
+                stats_handle,
+                save_output,
+            )
+            .await;
+            return;
+        }
+    };
+
+    update_stat!(tx_stats, UpdateUsizeField(InitialTargets, targets.len()));
 
     if !CONFIGURATION.quiet {
         // only print banner if -q isn't used
-        banner::initialize(&targets);
+        let std_stderr = stderr(); // std::io::stderr
+        banner::initialize(
+            &targets,
+            &CONFIGURATION,
+            &VERSION,
+            std_stderr,
+            tx_stats.clone(),
+        )
+        .await;
     }
 
     // discard non-responsive targets
-    let live_targets = heuristics::connectivity_test(&targets).await;
+    let live_targets = heuristics::connectivity_test(&targets, tx_stats.clone()).await;
 
-    match scan(live_targets).await {
+    if live_targets.is_empty() {
+        clean_up(
+            tx_term,
+            term_handle,
+            tx_file,
+            file_handle,
+            tx_stats,
+            stats_handle,
+            save_output,
+        )
+        .await;
+        return;
+    }
+
+    // kick off a scan against any targets determined to be responsive
+    match scan(
+        live_targets,
+        stats,
+        tx_term.clone(),
+        tx_file.clone(),
+        tx_stats.clone(),
+    )
+    .await
+    {
         Ok(_) => {
-            log::info!("Done");
+            log::info!("All scans complete!");
+        }
+        Err(e) => {
+            ferox_print(
+                &format!("{} while scanning: {}", status_colorizer("Error"), e),
+                &PROGRESS_PRINTER,
+            );
+            clean_up(
+                tx_term,
+                term_handle,
+                tx_file,
+                file_handle,
+                tx_stats,
+                stats_handle,
+                save_output,
+            )
+            .await;
+            process::exit(1);
         }
-        Err(e) => log::error!("An error occurred: {}", e),
     };
 
+    clean_up(
+        tx_term,
+        term_handle,
+        tx_file,
+        file_handle,
+        tx_stats,
+        stats_handle,
+        save_output,
+    )
+    .await;
+
+    log::trace!("exit: wrapped_main");
+}
+
+/// Single cleanup function that handles all the necessary drops/finishes etc required to gracefully
+/// shutdown the program
+async fn clean_up(
+    tx_term: UnboundedSender<FeroxResponse>,
+    term_handle: JoinHandle<()>,
+    tx_file: UnboundedSender<FeroxResponse>,
+    file_handle: Option<JoinHandle<()>>,
+    tx_stats: UnboundedSender<StatCommand>,
+    stats_handle: JoinHandle<()>,
+    save_output: bool,
+) {
+    log::trace!(
+        "enter: clean_up({:?}, {:?}, {:?}, {:?}, {:?}, {:?}, {})",
+        tx_term,
+        term_handle,
+        tx_file,
+        file_handle,
+        tx_stats,
+        stats_handle,
+        save_output
+    );
+    drop(tx_term);
+    log::trace!("dropped terminal output handler's transmitter");
+
+    log::trace!("awaiting terminal output handler's receiver");
+    // after dropping tx, we can await the future where rx lived
+    match term_handle.await {
+        Ok(_) => {}
+        Err(e) => {
+            log::error!("error awaiting terminal output handler's receiver: {}", e);
+        }
+    }
+    log::trace!("done awaiting terminal output handler's receiver");
+
+    log::trace!("tx_file: {:?}", tx_file);
+    // the same drop/await process used on the terminal handler is repeated for the file handler
+    // we drop the file transmitter every time, because it's created no matter what
+    drop(tx_file);
+
+    log::trace!("dropped file output handler's transmitter");
+    if save_output {
+        // but we only await if -o was specified
+        log::trace!("awaiting file output handler's receiver");
+        match file_handle.unwrap().await {
+            Ok(_) => {}
+            Err(e) => {
+                log::error!("error awaiting file output handler's receiver: {}", e);
+            }
+        }
+        log::trace!("done awaiting file output handler's receiver");
+    }
+
+    update_stat!(tx_stats, StatCommand::Exit); // send exit command and await the end of the future
+    stats_handle.await.unwrap_or_default();
+
+    // mark all scans complete so the terminal input handler will exit cleanly
+    SCAN_COMPLETE.store(true, Ordering::Relaxed);
+
+    // clean-up function for the MultiProgress bar; must be called last in order to still see
+    // the final trace messages above
     PROGRESS_PRINTER.finish();
 
+    drop(tx_stats);
+
+    log::trace!("exit: clean_up");
+}
+
+fn main() {
+    // setup logging based on the number of -v's used
+    logger::initialize(CONFIGURATION.verbosity);
+
+    // this function uses rlimit, which is not supported on windows
+    #[cfg(not(target_os = "windows"))]
+    set_open_file_limit(DEFAULT_OPEN_FILE_LIMIT);
+
+    if let Ok(runtime) = tokio::runtime::Builder::new_multi_thread()
+        .enable_all()
+        .build()
+    {
+        let future = wrapped_main();
+        runtime.block_on(future);
+    }
+
     log::trace!("exit: main");
 }

src/parser.rs

@@ -1,10 +1,23 @@
-use crate::VERSION;
-use clap::{App, Arg};
+use clap::{App, Arg, ArgGroup};
+use lazy_static::lazy_static;
+use regex::Regex;
+
+lazy_static! {
+    /// Regex used to validate values passed to --time-limit
+    ///
+    /// Examples of expected values that will this regex will match:
+    /// - 30s
+    /// - 20m
+    /// - 1h
+    /// - 1d
+    pub static ref TIMESPEC_REGEX: Regex =
+        Regex::new(r"^(?i)(?P<n>\d+)(?P<m>[smdh])$").expect("Could not compile regex");
+}
 
 /// Create and return an instance of [clap::App](https://docs.rs/clap/latest/clap/struct.App.html), i.e. the Command Line Interface's configuration
 pub fn initialize() -> App<'static, 'static> {
     App::new("feroxbuster")
-        .version(VERSION)
+        .version(env!("CARGO_PKG_VERSION"))
         .author("Ben 'epi' Risher (@epi052)")
         .about("A fast, simple, recursive content discovery tool written in Rust")
         .arg(
@@ -19,7 +32,7 @@ pub fn initialize() -> App<'static, 'static> {
             Arg::with_name("url")
                 .short("u")
                 .long("url")
-                .required_unless("stdin")
+                .required_unless_one(&["stdin", "resume_from"])
                 .value_name("URL")
                 .multiple(true)
                 .use_delimiter(true)
@@ -55,7 +68,7 @@ pub fn initialize() -> App<'static, 'static> {
                 .long("verbosity")
                 .takes_value(false)
                 .multiple(true)
-                .help("Increase verbosity level (use -vv or more for greater effect)"),
+                .help("Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v's is probably too much)"),
         )
         .arg(
             Arg::with_name("proxy")
@@ -64,19 +77,42 @@ pub fn initialize() -> App<'static, 'static> {
                 .takes_value(true)
                 .value_name("PROXY")
                 .help(
-                    "Proxy to use for requests (ex: http(s)://host:port, socks5://host:port)",
+                    "Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)",
                 ),
         )
         .arg(
-            Arg::with_name("statuscodes")
+            Arg::with_name("replay_proxy")
+                .short("P")
+                .long("replay-proxy")
+                .takes_value(true)
+                .value_name("REPLAY_PROXY")
+                .help(
+                    "Send only unfiltered requests through a Replay Proxy, instead of all requests",
+                ),
+        )
+        .arg(
+            Arg::with_name("replay_codes")
+                .short("R")
+                .long("replay-codes")
+                .value_name("REPLAY_CODE")
+                .takes_value(true)
+                .multiple(true)
+                .use_delimiter(true)
+                .requires("replay_proxy")
+                .help(
+                    "Status Codes to send through a Replay Proxy when found (default: --status-codes value)",
+                ),
+        )
+        .arg(
+            Arg::with_name("status_codes")
                 .short("s")
-                .long("statuscodes")
+                .long("status-codes")
                 .value_name("STATUS_CODE")
                 .takes_value(true)
                 .multiple(true)
                 .use_delimiter(true)
                 .help(
-                    "Status Codes of interest (default: 200 204 301 302 307 308 401 403 405)",
+                    "Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)",
                 ),
         )
         .arg(
@@ -87,9 +123,16 @@ pub fn initialize() -> App<'static, 'static> {
                 .help("Only print URLs; Don't print status codes, response size, running config, etc...")
         )
         .arg(
-            Arg::with_name("dontfilter")
+            Arg::with_name("json")
+                .long("json")
+                .takes_value(false)
+                .requires("output_files")
+                .help("Emit JSON logs to --output and --debug-log instead of normal text")
+        )
+        .arg(
+            Arg::with_name("dont_filter")
                 .short("D")
-                .long("dontfilter")
+                .long("dont-filter")
                 .takes_value(false)
                 .help("Don't auto-filter wildcard responses")
         )
@@ -98,13 +141,28 @@ pub fn initialize() -> App<'static, 'static> {
                 .short("o")
                 .long("output")
                 .value_name("FILE")
-                .help("Output file to write results to (default: stdout)")
+                .help("Output file to write results to (use w/ --json for JSON entries)")
                 .takes_value(true),
         )
         .arg(
-            Arg::with_name("useragent")
+            Arg::with_name("resume_from")
+                .long("resume-from")
+                .value_name("STATE_FILE")
+                .help("State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)")
+                .conflicts_with("url")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name("debug_log")
+                .long("debug-log")
+                .value_name("FILE")
+                .help("Output file to write log entries (use w/ --json for JSON entries)")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name("user_agent")
                 .short("a")
-                .long("useragent")
+                .long("user-agent")
                 .value_name("USER_AGENT")
                 .takes_value(true)
                 .help(
@@ -162,16 +220,16 @@ pub fn initialize() -> App<'static, 'static> {
                 ),
         )
         .arg(
-            Arg::with_name("norecursion")
+            Arg::with_name("no_recursion")
                 .short("n")
-                .long("norecursion")
+                .long("no-recursion")
                 .takes_value(false)
                 .help("Do not scan recursively")
         )
         .arg(
-            Arg::with_name("addslash")
+            Arg::with_name("add_slash")
                 .short("f")
-                .long("addslash")
+                .long("add-slash")
                 .takes_value(false)
                 .conflicts_with("extensions")
                 .help("Append / to each request")
@@ -184,9 +242,9 @@ pub fn initialize() -> App<'static, 'static> {
                 .conflicts_with("url")
         )
         .arg(
-            Arg::with_name("sizefilters")
+            Arg::with_name("filter_size")
                 .short("S")
-                .long("sizefilter")
+                .long("filter-size")
                 .value_name("SIZE")
                 .takes_value(true)
                 .multiple(true)
@@ -195,7 +253,92 @@ pub fn initialize() -> App<'static, 'static> {
                     "Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)",
                 ),
         )
-
+        .arg(
+            Arg::with_name("filter_regex")
+                .short("X")
+                .long("filter-regex")
+                .value_name("REGEX")
+                .takes_value(true)
+                .multiple(true)
+                .use_delimiter(true)
+                .help(
+                    "Filter out messages via regular expression matching on the response's body (ex: -X '^ignore me$')",
+                ),
+        )
+        .arg(
+            Arg::with_name("filter_words")
+                .short("W")
+                .long("filter-words")
+                .value_name("WORDS")
+                .takes_value(true)
+                .multiple(true)
+                .use_delimiter(true)
+                .help(
+                    "Filter out messages of a particular word count (ex: -W 312 -W 91,82)",
+                ),
+        )
+        .arg(
+            Arg::with_name("filter_lines")
+                .short("N")
+                .long("filter-lines")
+                .value_name("LINES")
+                .takes_value(true)
+                .multiple(true)
+                .use_delimiter(true)
+                .help(
+                    "Filter out messages of a particular line count (ex: -N 20 -N 31,30)",
+                ),
+        )
+        .arg(
+            Arg::with_name("filter_status")
+                .short("C")
+                .long("filter-status")
+                .value_name("STATUS_CODE")
+                .takes_value(true)
+                .multiple(true)
+                .use_delimiter(true)
+                .help(
+                    "Filter out status codes (deny list) (ex: -C 200 -C 401)",
+                ),
+        )
+        .arg(
+            Arg::with_name("filter_similar")
+                .long("filter-similar-to")
+                .value_name("UNWANTED_PAGE")
+                .takes_value(true)
+                .multiple(true)
+                .use_delimiter(true)
+                .help(
+                    "Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)",
+                ),
+        )
+        .arg(
+            Arg::with_name("extract_links")
+                .short("e")
+                .long("extract-links")
+                .takes_value(false)
+                .help("Extract links from response body (html, javascript, etc...); make new requests based on findings (default: false)")
+        )
+        .arg(
+            Arg::with_name("scan_limit")
+                .short("L")
+                .long("scan-limit")
+                .value_name("SCAN_LIMIT")
+                .takes_value(true)
+                .help("Limit total number of concurrent scans (default: 0, i.e. no limit)")
+        )
+        .arg(
+            Arg::with_name("time_limit")
+                .long("time-limit")
+                .value_name("TIME_SPEC")
+                .takes_value(true)
+                .validator(valid_time_spec)
+                .help("Limit total run time of all scans (ex: --time-limit 10m)")
+        )
+        .group(ArgGroup::with_name("output_files")
+            .args(&["debug_log", "output"])
+            .multiple(true)
+        )
         .after_help(r#"NOTE:
     Options that take multiple values are very flexible.  Consider the following ways of specifying
     extensions:
@@ -211,7 +354,7 @@ EXAMPLES:
         ./feroxbuster -u http://127.1 -H Accept:application/json "Authorization: Bearer {token}"
 
     IPv6, non-recursive scan with INFO-level logging enabled:
-        ./feroxbuster -u http://[::1] --norecursion -vv
+        ./feroxbuster -u http://[::1] --no-recursion -vv
 
     Read urls from STDIN; pipe only resulting urls out to another tool
         cat targets | ./feroxbuster --stdin --quiet -s 200 301 302 --redirects -x js | fff -s 200 -o js-files
@@ -225,7 +368,69 @@ EXAMPLES:
     Pass auth token via query parameter
         ./feroxbuster -u http://127.1 --query token=0123456789ABCDEF
 
+    Find links in javascript/html and make additional requests based on results
+        ./feroxbuster -u http://127.1 --extract-links
+
     Ludicrous speed... go!
         ./feroxbuster -u http://127.1 -t 200
     "#)
 }
+
+/// Validate that a string is formatted as a number followed by s, m, h, or d (10d, 30s, etc...)
+fn valid_time_spec(time_spec: String) -> Result<(), String> {
+    match TIMESPEC_REGEX.is_match(&time_spec) {
+        true => Ok(()),
+        false => {
+            let msg = format!(
+                "Expected a non-negative, whole number followed by s, m, h, or d (case insensitive); received {}",
+                time_spec
+            );
+            Err(msg)
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    /// initalize parser, expect a clap::App returned
+    fn parser_initialize_gives_defaults() {
+        let app = initialize();
+        assert_eq!(app.get_name(), "feroxbuster");
+    }
+
+    #[test]
+    /// sanity checks that valid_time_spec correctly checks and rejects a given string
+    ///
+    /// instead of having a bunch of single tests here, they're all quick and are mostly checking
+    /// that i didn't hose up the regex.  Going to consolidate them into a single test
+    fn validate_valid_time_spec_validation() {
+        let float_rejected = "1.4m";
+        assert!(valid_time_spec(float_rejected.into()).is_err());
+
+        let negative_rejected = "-1m";
+        assert!(valid_time_spec(negative_rejected.into()).is_err());
+
+        let only_number_rejected = "1";
+        assert!(valid_time_spec(only_number_rejected.into()).is_err());
+
+        let only_measurement_rejected = "m";
+        assert!(valid_time_spec(only_measurement_rejected.into()).is_err());
+
+        for accepted_measurement in &["s", "m", "h", "d", "S", "M", "H", "D"] {
+            // all upper/lowercase should be good
+            assert!(valid_time_spec(format!("1{}", *accepted_measurement)).is_ok());
+        }
+
+        let leading_space_rejected = " 14m";
+        assert!(valid_time_spec(leading_space_rejected.into()).is_err());
+
+        let trailing_space_rejected = "14m ";
+        assert!(valid_time_spec(trailing_space_rejected.into()).is_err());
+
+        let space_between_rejected = "1 4m";
+        assert!(valid_time_spec(space_between_rejected.into()).is_err());
+    }
+}

src/progress.rs

@@ -1,15 +1,42 @@
 use crate::config::{CONFIGURATION, PROGRESS_BAR};
 use indicatif::{ProgressBar, ProgressStyle};
 
+/// Types of ProgressBars that can be added to `PROGRESS_BAR`
+pub enum BarType {
+    /// no template used / not visible
+    Hidden,
+
+    /// normal directory status bar (reqs/sec shown)
+    Default,
+
+    /// similar to `Default`, except `-` is used in place of line/word/char count
+    Message,
+
+    /// bar used to show overall scan metrics
+    Total,
+}
+
 /// Add an [indicatif::ProgressBar](https://docs.rs/indicatif/latest/indicatif/struct.ProgressBar.html)
 /// to the global [PROGRESS_BAR](../config/struct.PROGRESS_BAR.html)
-pub fn add_bar(prefix: &str, length: u64, hidden: bool) -> ProgressBar {
-    let style = if hidden || CONFIGURATION.quiet {
-        ProgressStyle::default_bar().template("")
+pub fn add_bar(prefix: &str, length: u64, bar_type: BarType) -> ProgressBar {
+    let mut style = ProgressStyle::default_bar().progress_chars("#>-");
+
+    style = if CONFIGURATION.quiet {
+        style.template("")
     } else {
-        ProgressStyle::default_bar()
-            .template("[{bar:.cyan/blue}] - {elapsed:<4} {pos:>7}/{len:7} {per_sec:7} {prefix}")
-            .progress_chars("#>-")
+        match bar_type {
+            BarType::Hidden => style.template(""),
+            BarType::Default => style.template(
+                "[{bar:.cyan/blue}] - {elapsed:<4} {pos:>7}/{len:7} {per_sec:7} {prefix}",
+            ),
+            BarType::Message => style.template(&format!(
+                "[{{bar:.cyan/blue}}] - {{elapsed:<4}} {{pos:>7}}/{{len:7}} {:7} {{prefix}}",
+                "-"
+            )),
+            BarType::Total => {
+                style.template("[{bar:.yellow/blue}] - {elapsed:<4} {pos:>7}/{len:7} {eta:7} {msg}")
+            }
+        }
     };
 
     let progress_bar = PROGRESS_BAR.add(ProgressBar::new(length));
@@ -20,3 +47,27 @@ pub fn add_bar(prefix: &str, length: u64, hidden: bool) -> ProgressBar {
 
     progress_bar
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    /// hit all code branches for add_bar
+    fn add_bar_with_all_configurations() {
+        let p1 = add_bar("prefix", 2, BarType::Hidden); // hidden
+        let p2 = add_bar("prefix", 2, BarType::Message); // no per second field
+        let p3 = add_bar("prefix", 2, BarType::Default); // normal bar
+        let p4 = add_bar("prefix", 2, BarType::Total); // totals bar
+
+        p1.finish();
+        p2.finish();
+        p3.finish();
+        p4.finish();
+
+        assert!(p1.is_finished());
+        assert!(p2.is_finished());
+        assert!(p3.is_finished());
+        assert!(p4.is_finished());
+    }
+}

src/reporter.rs

@@ -0,0 +1,261 @@
+use crate::{
+    config::{CONFIGURATION, PROGRESS_PRINTER},
+    scanner::RESPONSES,
+    statistics::{
+        StatCommand::{self, UpdateUsizeField},
+        StatField::ResourcesDiscovered,
+    },
+    utils::{ferox_print, make_request, open_file},
+    FeroxChannel, FeroxResponse, FeroxSerialize,
+};
+use console::strip_ansi_codes;
+use std::{
+    fs, io,
+    io::Write,
+    sync::{Arc, Once, RwLock},
+};
+use tokio::{
+    sync::mpsc::{self, UnboundedReceiver, UnboundedSender},
+    task::JoinHandle,
+};
+
+/// Singleton buffered file behind an Arc/RwLock; used for file writes from two locations:
+///     - [logger::initialize](../logger/fn.initialize.html) (specifically a closure on the global logger instance)
+///     - `reporter::spawn_file_handler`
+pub static mut LOCKED_FILE: Option<Arc<RwLock<io::BufWriter<fs::File>>>> = None;
+
+/// An initializer Once variable used to create `LOCKED_FILE`
+static INIT: Once = Once::new();
+
+// Accessing a `static mut` is unsafe much of the time, but if we do so
+// in a synchronized fashion (e.g., write once or read all) then we're
+// good to go!
+//
+// This function will only call `open_file` once, and will
+// otherwise always return the value returned from the first invocation.
+pub fn get_cached_file_handle(filename: &str) -> Option<Arc<RwLock<io::BufWriter<fs::File>>>> {
+    unsafe {
+        INIT.call_once(|| {
+            LOCKED_FILE = open_file(&filename);
+        });
+        LOCKED_FILE.clone()
+    }
+}
+
+/// Creates all required output handlers (terminal, file) and returns
+/// the transmitter sides of each mpsc along with each receiver's future's JoinHandle to be awaited
+///
+/// Any other module that needs to write a Response to stdout or output results to a file should
+/// be passed a clone of the appropriate returned transmitter
+pub fn initialize(
+    output_file: &str,
+    save_output: bool,
+    tx_stats: UnboundedSender<StatCommand>,
+) -> (
+    UnboundedSender<FeroxResponse>,
+    UnboundedSender<FeroxResponse>,
+    JoinHandle<()>,
+    Option<JoinHandle<()>>,
+) {
+    log::trace!(
+        "enter: initialize({}, {}, {:?})",
+        output_file,
+        save_output,
+        tx_stats
+    );
+
+    let (tx_rpt, rx_rpt): FeroxChannel<FeroxResponse> = mpsc::unbounded_channel();
+    let (tx_file, rx_file): FeroxChannel<FeroxResponse> = mpsc::unbounded_channel();
+
+    let file_clone = tx_file.clone();
+    let stats_clone = tx_stats.clone();
+
+    let term_reporter = tokio::spawn(async move {
+        spawn_terminal_reporter(rx_rpt, file_clone, stats_clone, save_output).await
+    });
+
+    let file_reporter = if save_output {
+        // -o used, need to spawn the thread for writing to disk
+        let file_clone = output_file.to_string();
+        Some(tokio::spawn(async move {
+            spawn_file_reporter(rx_file, tx_stats, &file_clone).await
+        }))
+    } else {
+        None
+    };
+
+    log::trace!(
+        "exit: initialize -> ({:?}, {:?}, {:?}, {:?})",
+        tx_rpt,
+        tx_file,
+        term_reporter,
+        file_reporter
+    );
+    (tx_rpt, tx_file, term_reporter, file_reporter)
+}
+
+/// Spawn a single consumer task (sc side of mpsc)
+///
+/// The consumer simply receives responses and prints them if they meet the given
+/// reporting criteria
+async fn spawn_terminal_reporter(
+    mut resp_chan: UnboundedReceiver<FeroxResponse>,
+    file_chan: UnboundedSender<FeroxResponse>,
+    tx_stats: UnboundedSender<StatCommand>,
+    save_output: bool,
+) {
+    log::trace!(
+        "enter: spawn_terminal_reporter({:?}, {:?}, {:?}, {})",
+        resp_chan,
+        file_chan,
+        tx_stats,
+        save_output
+    );
+
+    while let Some(mut resp) = resp_chan.recv().await {
+        log::trace!("received {} on reporting channel", resp.url());
+
+        let contains_sentry = CONFIGURATION.status_codes.contains(&resp.status().as_u16());
+        let unknown_sentry = !RESPONSES.contains(&resp); // !contains == unknown
+        let should_process_response = contains_sentry && unknown_sentry;
+
+        if should_process_response {
+            // print to stdout
+            ferox_print(&resp.as_str(), &PROGRESS_PRINTER);
+
+            update_stat!(tx_stats, UpdateUsizeField(ResourcesDiscovered, 1));
+
+            if save_output {
+                // -o used, need to send the report to be written out to disk
+                match file_chan.send(resp.clone()) {
+                    Ok(_) => {
+                        log::debug!("Sent {} to file handler", resp.url());
+                    }
+                    Err(e) => {
+                        log::error!("Could not send {} to file handler: {}", resp.url(), e);
+                    }
+                }
+            }
+        }
+        log::trace!("report complete: {}", resp.url());
+
+        if CONFIGURATION.replay_client.is_some() && should_process_response {
+            // replay proxy specified/client created and this response's status code is one that
+            // should be replayed
+            match make_request(
+                CONFIGURATION.replay_client.as_ref().unwrap(),
+                &resp.url(),
+                tx_stats.clone(),
+            )
+            .await
+            {
+                Ok(_) => {}
+                Err(e) => {
+                    log::error!("{}", e);
+                }
+            }
+        }
+
+        if should_process_response {
+            // add response to RESPONSES for serialization in case of ctrl+c
+            // placed all by its lonesome like this so that RESPONSES can take ownership
+            // of the FeroxResponse
+
+            // before ownership is transferred, there's no real reason to keep the body anymore
+            // so we can free that piece of data, reducing memory usage
+            resp.text = String::new();
+
+            RESPONSES.insert(resp);
+        }
+    }
+    log::trace!("exit: spawn_terminal_reporter");
+}
+
+/// Spawn a single consumer task (sc side of mpsc)
+///
+/// The consumer simply receives responses and writes them to the given output file if they meet
+/// the given reporting criteria
+async fn spawn_file_reporter(
+    mut report_channel: UnboundedReceiver<FeroxResponse>,
+    tx_stats: UnboundedSender<StatCommand>,
+    output_file: &str,
+) {
+    let buffered_file = match get_cached_file_handle(&CONFIGURATION.output) {
+        Some(file) => file,
+        None => {
+            log::trace!("exit: spawn_file_reporter");
+            return;
+        }
+    };
+
+    log::trace!(
+        "enter: spawn_file_reporter({:?}, {})",
+        report_channel,
+        output_file
+    );
+
+    log::info!("Writing scan results to {}", output_file);
+
+    while let Some(response) = report_channel.recv().await {
+        safe_file_write(&response, buffered_file.clone(), CONFIGURATION.json);
+    }
+
+    update_stat!(tx_stats, StatCommand::Save);
+
+    log::trace!("exit: spawn_file_reporter");
+}
+
+/// Given a string and a reference to a locked buffered file, write the contents and flush
+/// the buffer to disk.
+pub fn safe_file_write<T>(
+    value: &T,
+    locked_file: Arc<RwLock<io::BufWriter<fs::File>>>,
+    convert_to_json: bool,
+) where
+    T: FeroxSerialize,
+{
+    // note to future self: adding logging of anything other than error to this function
+    // is a bad idea. we call this function while processing records generated by the logger.
+    // If we then call log::... while already processing some logging output, it results in
+    // the second log entry being injected into the first.
+
+    let contents = if convert_to_json {
+        value.as_json()
+    } else {
+        value.as_str()
+    };
+
+    let contents = strip_ansi_codes(&contents);
+
+    if let Ok(mut handle) = locked_file.write() {
+        // write lock acquired
+        match handle.write(contents.as_bytes()) {
+            Ok(_) => {}
+            Err(e) => {
+                log::error!("could not write report to disk: {}", e);
+            }
+        }
+
+        match handle.flush() {
+            // this function is used within async functions/loops, so i'm flushing so that in
+            // the event of a ctrl+c or w/e results seen so far are saved instead of left lying
+            // around in the buffer
+            Ok(_) => {}
+            Err(e) => {
+                log::error!("error writing to file: {}", e);
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    #[should_panic]
+    /// asserts that an empty string for a filename returns None
+    fn reporter_get_cached_file_handle_without_filename_returns_none() {
+        let _used = get_cached_file_handle(&"").unwrap();
+    }
+}

src/statistics.rs

@@ -0,0 +1,833 @@
+use crate::{
+    config::CONFIGURATION,
+    progress::{add_bar, BarType},
+    reporter::{get_cached_file_handle, safe_file_write},
+    FeroxChannel, FeroxSerialize,
+};
+use console::style;
+use indicatif::ProgressBar;
+use reqwest::StatusCode;
+use serde::{Deserialize, Serialize};
+use std::{
+    fs::File,
+    io::BufReader,
+    sync::{
+        atomic::{AtomicUsize, Ordering},
+        Arc, Mutex,
+    },
+    time::Instant,
+};
+use tokio::{
+    sync::mpsc::{self, UnboundedReceiver, UnboundedSender},
+    task::JoinHandle,
+};
+
+/// Wrapper `Atomic*.fetch_add` to save me from writing Ordering::Relaxed a bajillion times
+///
+/// default is to increment by 1, second arg can be used to increment by a different value
+macro_rules! atomic_increment {
+    ($metric:expr) => {
+        $metric.fetch_add(1, Ordering::Relaxed);
+    };
+
+    ($metric:expr, $value:expr) => {
+        $metric.fetch_add($value, Ordering::Relaxed);
+    };
+}
+
+/// Wrapper around `Atomic*.load` to save me from writing Ordering::Relaxed a bajillion times
+macro_rules! atomic_load {
+    ($metric:expr) => {
+        $metric.load(Ordering::Relaxed);
+    };
+}
+
+/// Data collection of statistics related to a scan
+#[derive(Default, Deserialize, Debug, Serialize)]
+pub struct Stats {
+    #[serde(rename = "type")]
+    /// Name of this type of struct, used for serialization, i.e. `{"type":"statistics"}`
+    kind: String,
+
+    /// tracker for number of timeouts seen by the client
+    timeouts: AtomicUsize,
+
+    /// tracker for total number of requests sent by the client
+    requests: AtomicUsize,
+
+    /// tracker for total number of requests expected to send if the scan runs to completion
+    ///
+    /// Note: this is a per-scan expectation; `expected_requests * current # of scans` would be
+    /// indicative of the current expectation at any given time, but is a moving target.  
+    pub expected_per_scan: AtomicUsize,
+
+    /// tracker for accumulating total number of requests expected (i.e. as a new scan is started
+    /// this value should increase by `expected_requests`
+    total_expected: AtomicUsize,
+
+    /// tracker for total number of errors encountered by the client
+    errors: AtomicUsize,
+
+    /// tracker for overall number of 2xx status codes seen by the client
+    successes: AtomicUsize,
+
+    /// tracker for overall number of 3xx status codes seen by the client
+    redirects: AtomicUsize,
+
+    /// tracker for overall number of 4xx status codes seen by the client
+    client_errors: AtomicUsize,
+
+    /// tracker for overall number of 5xx status codes seen by the client
+    server_errors: AtomicUsize,
+
+    /// tracker for number of scans performed, this directly equates to number of directories
+    /// recursed into and affects the total number of expected requests
+    total_scans: AtomicUsize,
+
+    /// tracker for initial number of requested targets
+    pub initial_targets: AtomicUsize,
+
+    /// tracker for number of links extracted when `--extract-links` is used; sources are
+    /// response bodies and robots.txt as of v1.11.0
+    links_extracted: AtomicUsize,
+
+    /// tracker for overall number of 200s seen by the client
+    status_200s: AtomicUsize,
+
+    /// tracker for overall number of 301s seen by the client
+    status_301s: AtomicUsize,
+
+    /// tracker for overall number of 302s seen by the client
+    status_302s: AtomicUsize,
+
+    /// tracker for overall number of 401s seen by the client
+    status_401s: AtomicUsize,
+
+    /// tracker for overall number of 403s seen by the client
+    status_403s: AtomicUsize,
+
+    /// tracker for overall number of 429s seen by the client
+    status_429s: AtomicUsize,
+
+    /// tracker for overall number of 500s seen by the client
+    status_500s: AtomicUsize,
+
+    /// tracker for overall number of 503s seen by the client
+    status_503s: AtomicUsize,
+
+    /// tracker for overall number of 504s seen by the client
+    status_504s: AtomicUsize,
+
+    /// tracker for overall number of 508s seen by the client
+    status_508s: AtomicUsize,
+
+    /// tracker for overall number of wildcard urls filtered out by the client
+    wildcards_filtered: AtomicUsize,
+
+    /// tracker for overall number of all filtered responses
+    responses_filtered: AtomicUsize,
+
+    /// tracker for number of files found
+    resources_discovered: AtomicUsize,
+
+    /// tracker for number of errors triggered during URL formatting
+    url_format_errors: AtomicUsize,
+
+    /// tracker for number of errors triggered by the `reqwest::RedirectPolicy`
+    redirection_errors: AtomicUsize,
+
+    /// tracker for number of errors related to the connecting
+    connection_errors: AtomicUsize,
+
+    /// tracker for number of errors related to the request used
+    request_errors: AtomicUsize,
+
+    /// tracker for each directory's total scan time in seconds as a float
+    directory_scan_times: Mutex<Vec<f64>>,
+
+    /// tracker for total runtime
+    total_runtime: Mutex<Vec<f64>>,
+}
+
+/// FeroxSerialize implementation for Stats
+impl FeroxSerialize for Stats {
+    /// Simply return empty string here to disable serializing this to the output file as a string
+    /// due to it looking like garbage
+    fn as_str(&self) -> String {
+        String::new()
+    }
+
+    /// Simple call to produce a JSON string using the given Stats object
+    fn as_json(&self) -> String {
+        serde_json::to_string(&self).unwrap_or_default()
+    }
+}
+
+/// implementation of statistics data collection struct
+impl Stats {
+    /// Small wrapper for default to set `kind` to "statistics" and `total_runtime` to have at least
+    /// one value
+    pub fn new() -> Self {
+        Self {
+            kind: String::from("statistics"),
+            total_runtime: Mutex::new(vec![0.0]),
+            ..Default::default()
+        }
+    }
+
+    /// increment `requests` field by one
+    fn add_request(&self) {
+        atomic_increment!(self.requests);
+    }
+
+    /// given an `Instant` update total runtime
+    fn update_runtime(&self, seconds: f64) {
+        if let Ok(mut runtime) = self.total_runtime.lock() {
+            runtime[0] = seconds;
+        }
+    }
+
+    /// save an instance of `Stats` to disk after updating the total runtime for the scan
+    fn save(&self, seconds: f64, location: &str) {
+        let buffered_file = match get_cached_file_handle(location) {
+            Some(file) => file,
+            None => {
+                return;
+            }
+        };
+
+        self.update_runtime(seconds);
+
+        safe_file_write(self, buffered_file, CONFIGURATION.json);
+    }
+
+    /// Inspect the given `StatError` and increment the appropriate fields
+    ///
+    /// Implies incrementing:
+    ///     - requests
+    ///     - errors
+    pub fn add_error(&self, error: StatError) {
+        self.add_request();
+        atomic_increment!(self.errors);
+
+        match error {
+            StatError::Timeout => {
+                atomic_increment!(self.timeouts);
+            }
+            StatError::Status403 => {
+                atomic_increment!(self.status_403s);
+                atomic_increment!(self.client_errors);
+            }
+            StatError::UrlFormat => {
+                atomic_increment!(self.url_format_errors);
+            }
+            StatError::Redirection => {
+                atomic_increment!(self.redirection_errors);
+            }
+            StatError::Connection => {
+                atomic_increment!(self.connection_errors);
+            }
+            StatError::Request => {
+                atomic_increment!(self.request_errors);
+            }
+            StatError::Other => {
+                atomic_increment!(self.errors);
+            }
+        }
+    }
+
+    /// Inspect the given `StatusCode` and increment the appropriate fields
+    ///
+    /// Implies incrementing:
+    ///     - requests
+    ///     - status_403s (when code is 403)
+    ///     - errors (when code is [45]xx)
+    fn add_status_code(&self, status: StatusCode) {
+        self.add_request();
+
+        if status.is_success() {
+            atomic_increment!(self.successes);
+        } else if status.is_redirection() {
+            atomic_increment!(self.redirects);
+        } else if status.is_client_error() {
+            atomic_increment!(self.client_errors);
+        } else if status.is_server_error() {
+            atomic_increment!(self.server_errors);
+        }
+
+        match status {
+            StatusCode::FORBIDDEN => {
+                atomic_increment!(self.status_403s);
+            }
+            StatusCode::OK => {
+                atomic_increment!(self.status_200s);
+            }
+            StatusCode::MOVED_PERMANENTLY => {
+                atomic_increment!(self.status_301s);
+            }
+            StatusCode::FOUND => {
+                atomic_increment!(self.status_302s);
+            }
+            StatusCode::UNAUTHORIZED => {
+                atomic_increment!(self.status_401s);
+            }
+            StatusCode::TOO_MANY_REQUESTS => {
+                atomic_increment!(self.status_429s);
+            }
+            StatusCode::INTERNAL_SERVER_ERROR => {
+                atomic_increment!(self.status_500s);
+            }
+            StatusCode::SERVICE_UNAVAILABLE => {
+                atomic_increment!(self.status_503s);
+            }
+            StatusCode::GATEWAY_TIMEOUT => {
+                atomic_increment!(self.status_504s);
+            }
+            StatusCode::LOOP_DETECTED => {
+                atomic_increment!(self.status_508s);
+            }
+            _ => {} // other status codes ignored for stat gathering
+        }
+    }
+
+    /// Update a `Stats` field of type f64
+    fn update_f64_field(&self, field: StatField, value: f64) {
+        if let StatField::DirScanTimes = field {
+            if let Ok(mut locked_times) = self.directory_scan_times.lock() {
+                locked_times.push(value);
+            }
+        }
+    }
+
+    /// Update a `Stats` field of type usize
+    fn update_usize_field(&self, field: StatField, value: usize) {
+        match field {
+            StatField::ExpectedPerScan => {
+                atomic_increment!(self.expected_per_scan, value);
+            }
+            StatField::TotalScans => {
+                let multiplier = CONFIGURATION.extensions.len().max(1);
+
+                atomic_increment!(self.total_scans, value);
+                atomic_increment!(
+                    self.total_expected,
+                    value * self.expected_per_scan.load(Ordering::Relaxed) * multiplier
+                );
+            }
+            StatField::TotalExpected => {
+                atomic_increment!(self.total_expected, value);
+            }
+            StatField::LinksExtracted => {
+                atomic_increment!(self.links_extracted, value);
+            }
+            StatField::WildcardsFiltered => {
+                atomic_increment!(self.wildcards_filtered, value);
+                atomic_increment!(self.responses_filtered, value);
+            }
+            StatField::ResponsesFiltered => {
+                atomic_increment!(self.responses_filtered, value);
+            }
+            StatField::ResourcesDiscovered => {
+                atomic_increment!(self.resources_discovered, value);
+            }
+            StatField::InitialTargets => {
+                atomic_increment!(self.initial_targets, value);
+            }
+            _ => {} // f64 fields
+        }
+    }
+
+    /// Merge a given `Stats` object from a json entry written to disk when handling a Ctrl+c
+    ///
+    /// This is only ever called when resuming a scan from disk
+    pub fn merge_from(&self, filename: &str) {
+        if let Ok(file) = File::open(filename) {
+            let reader = BufReader::new(file);
+            let state: serde_json::Value = serde_json::from_reader(reader).unwrap();
+
+            if let Some(state_stats) = state.get("statistics") {
+                if let Ok(d_stats) = serde_json::from_value::<Stats>(state_stats.clone()) {
+                    atomic_increment!(self.successes, atomic_load!(d_stats.successes));
+                    atomic_increment!(self.timeouts, atomic_load!(d_stats.timeouts));
+                    atomic_increment!(self.requests, atomic_load!(d_stats.requests));
+                    atomic_increment!(self.errors, atomic_load!(d_stats.errors));
+                    atomic_increment!(self.redirects, atomic_load!(d_stats.redirects));
+                    atomic_increment!(self.client_errors, atomic_load!(d_stats.client_errors));
+                    atomic_increment!(self.server_errors, atomic_load!(d_stats.server_errors));
+                    atomic_increment!(self.links_extracted, atomic_load!(d_stats.links_extracted));
+                    atomic_increment!(self.status_200s, atomic_load!(d_stats.status_200s));
+                    atomic_increment!(self.status_301s, atomic_load!(d_stats.status_301s));
+                    atomic_increment!(self.status_302s, atomic_load!(d_stats.status_302s));
+                    atomic_increment!(self.status_401s, atomic_load!(d_stats.status_401s));
+                    atomic_increment!(self.status_403s, atomic_load!(d_stats.status_403s));
+                    atomic_increment!(self.status_429s, atomic_load!(d_stats.status_429s));
+                    atomic_increment!(self.status_500s, atomic_load!(d_stats.status_500s));
+                    atomic_increment!(self.status_503s, atomic_load!(d_stats.status_503s));
+                    atomic_increment!(self.status_504s, atomic_load!(d_stats.status_504s));
+                    atomic_increment!(self.status_508s, atomic_load!(d_stats.status_508s));
+                    atomic_increment!(
+                        self.wildcards_filtered,
+                        atomic_load!(d_stats.wildcards_filtered)
+                    );
+                    atomic_increment!(
+                        self.responses_filtered,
+                        atomic_load!(d_stats.responses_filtered)
+                    );
+                    atomic_increment!(
+                        self.resources_discovered,
+                        atomic_load!(d_stats.resources_discovered)
+                    );
+                    atomic_increment!(
+                        self.url_format_errors,
+                        atomic_load!(d_stats.url_format_errors)
+                    );
+                    atomic_increment!(
+                        self.connection_errors,
+                        atomic_load!(d_stats.connection_errors)
+                    );
+                    atomic_increment!(
+                        self.redirection_errors,
+                        atomic_load!(d_stats.redirection_errors)
+                    );
+                    atomic_increment!(self.request_errors, atomic_load!(d_stats.request_errors));
+
+                    if let Ok(scan_times) = d_stats.directory_scan_times.lock() {
+                        for scan_time in scan_times.iter() {
+                            self.update_f64_field(StatField::DirScanTimes, *scan_time);
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+#[derive(Debug)]
+/// Enum variants used to inform the `StatCommand` protocol what `Stats` fields should be updated
+pub enum StatError {
+    /// Represents a 403 response code
+    Status403,
+
+    /// Represents a timeout error
+    Timeout,
+
+    /// Represents a URL formatting error
+    UrlFormat,
+
+    /// Represents an error encountered during redirection
+    Redirection,
+
+    /// Represents an error encountered during connection
+    Connection,
+
+    /// Represents an error resulting from the client's request
+    Request,
+
+    /// Represents any other error not explicitly defined above
+    Other,
+}
+
+/// Protocol definition for updating a Stats object via mpsc
+#[derive(Debug)]
+pub enum StatCommand {
+    /// Add one to the total number of requests
+    AddRequest,
+
+    /// Add one to the proper field(s) based on the given `StatError`
+    AddError(StatError),
+
+    /// Add one to the proper field(s) based on the given `StatusCode`
+    AddStatus(StatusCode),
+
+    /// Create the progress bar (`BarType::Total`) that is updated from the stats thread
+    CreateBar,
+
+    /// Update a `Stats` field that corresponds to the given `StatField` by the given `usize` value
+    UpdateUsizeField(StatField, usize),
+
+    /// Update a `Stats` field that corresponds to the given `StatField` by the given `f64` value
+    UpdateF64Field(StatField, f64),
+
+    /// Save a `Stats` object to disk using `reporter::get_cached_file_handle`
+    Save,
+
+    /// Load a `Stats` object from disk
+    LoadStats(String),
+
+    /// Break out of the (infinite) mpsc receive loop
+    Exit,
+}
+
+/// Enum representing fields whose updates need to be performed in batches instead of one at
+/// a time
+#[derive(Debug)]
+pub enum StatField {
+    /// Due to the necessary order of events, the number of requests expected to be sent isn't
+    /// known until after `statistics::initialize` is called. This command allows for updating
+    /// the `expected_per_scan` field after initialization
+    ExpectedPerScan,
+
+    /// Translates to `total_scans`
+    TotalScans,
+
+    /// Translates to `links_extracted`
+    LinksExtracted,
+
+    /// Translates to `total_expected`
+    TotalExpected,
+
+    /// Translates to `wildcards_filtered`
+    WildcardsFiltered,
+
+    /// Translates to `responses_filtered`
+    ResponsesFiltered,
+
+    /// Translates to `resources_discovered`
+    ResourcesDiscovered,
+
+    /// Translates to `initial_targets`
+    InitialTargets,
+
+    /// Translates to `directory_scan_times`; assumes a single append to the vector
+    DirScanTimes,
+}
+
+/// Spawn a single consumer task (sc side of mpsc)
+///
+/// The consumer simply receives `StatCommands` and updates the given `Stats` object as appropriate
+pub async fn spawn_statistics_handler(
+    mut rx_stats: UnboundedReceiver<StatCommand>,
+    stats: Arc<Stats>,
+    tx_stats: UnboundedSender<StatCommand>,
+) {
+    log::trace!(
+        "enter: spawn_statistics_handler({:?}, {:?}, {:?})",
+        rx_stats,
+        stats,
+        tx_stats
+    );
+
+    // will be updated later via StatCommand; delay is for banner to print first
+    let mut bar = ProgressBar::hidden();
+
+    let start = Instant::now();
+
+    while let Some(command) = rx_stats.recv().await {
+        match command as StatCommand {
+            StatCommand::AddError(err) => {
+                stats.add_error(err);
+                increment_bar(&bar, stats.clone());
+            }
+            StatCommand::AddStatus(status) => {
+                stats.add_status_code(status);
+                increment_bar(&bar, stats.clone());
+            }
+            StatCommand::AddRequest => {
+                stats.add_request();
+                increment_bar(&bar, stats.clone());
+            }
+            StatCommand::Save => stats.save(start.elapsed().as_secs_f64(), &CONFIGURATION.output),
+            StatCommand::UpdateUsizeField(field, value) => {
+                let update_len = matches!(field, StatField::TotalScans);
+                stats.update_usize_field(field, value);
+
+                if update_len {
+                    bar.set_length(atomic_load!(stats.total_expected) as u64)
+                }
+            }
+            StatCommand::UpdateF64Field(field, value) => stats.update_f64_field(field, value),
+            StatCommand::CreateBar => {
+                bar = add_bar(
+                    "",
+                    atomic_load!(stats.total_expected) as u64,
+                    BarType::Total,
+                );
+            }
+            StatCommand::LoadStats(filename) => {
+                stats.merge_from(&filename);
+            }
+            StatCommand::Exit => break,
+        }
+    }
+
+    bar.finish();
+
+    log::debug!("{:#?}", *stats);
+    log::trace!("exit: spawn_statistics_handler")
+}
+
+/// Wrapper around incrementing the overall scan's progress bar
+fn increment_bar(bar: &ProgressBar, stats: Arc<Stats>) {
+    let msg = format!(
+        "{}:{:<7} {}:{:<7}",
+        style("found").green(),
+        atomic_load!(stats.resources_discovered),
+        style("errors").red(),
+        atomic_load!(stats.errors),
+    );
+
+    bar.set_message(&msg);
+    bar.inc(1);
+}
+
+/// Initialize new `Stats` object and the sc side of an mpsc channel that is responsible for
+/// updates to the aforementioned object.
+pub fn initialize() -> (Arc<Stats>, UnboundedSender<StatCommand>, JoinHandle<()>) {
+    log::trace!("enter: initialize");
+
+    let stats_tracker = Arc::new(Stats::new());
+    let stats_cloned = stats_tracker.clone();
+    let (tx_stats, rx_stats): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+    let tx_stats_cloned = tx_stats.clone();
+    let stats_thread = tokio::spawn(async move {
+        spawn_statistics_handler(rx_stats, stats_cloned, tx_stats_cloned).await
+    });
+
+    log::trace!(
+        "exit: initialize -> ({:?}, {:?}, {:?})",
+        stats_tracker,
+        tx_stats,
+        stats_thread
+    );
+
+    (stats_tracker, tx_stats, stats_thread)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::fs::write;
+    use tempfile::NamedTempFile;
+
+    /// simple helper to reduce code reuse
+    fn setup_stats_test() -> (Arc<Stats>, UnboundedSender<StatCommand>, JoinHandle<()>) {
+        initialize()
+    }
+
+    /// another helper to stay DRY; must be called after any sent commands and before any checks
+    /// performed against the Stats object
+    async fn teardown_stats_test(sender: UnboundedSender<StatCommand>, handle: JoinHandle<()>) {
+        // send exit and await, once the await completes, stats should be updated
+        sender.send(StatCommand::Exit).unwrap_or_default();
+        handle.await.unwrap();
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// when sent StatCommand::Exit, function should exit its while loop (runs forever otherwise)
+    async fn statistics_handler_exits() {
+        let (_, sender, handle) = setup_stats_test();
+
+        sender.send(StatCommand::Exit).unwrap_or_default();
+
+        handle.await.unwrap(); // blocks on the handler's while loop
+
+        // if we've made it here, the test has succeeded
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// when sent StatCommand::AddRequest, stats object should reflect the change
+    async fn statistics_handler_increments_requests() {
+        let (stats, tx, handle) = setup_stats_test();
+
+        tx.send(StatCommand::AddRequest).unwrap_or_default();
+        tx.send(StatCommand::AddRequest).unwrap_or_default();
+        tx.send(StatCommand::AddRequest).unwrap_or_default();
+
+        teardown_stats_test(tx, handle).await;
+
+        assert_eq!(stats.requests.load(Ordering::Relaxed), 3);
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// when sent StatCommand::AddRequest, stats object should reflect the change
+    ///
+    /// incrementing a 403 (tracked in status_403s) should also increment:
+    ///     - errors
+    ///     - requests
+    ///     - client_errors
+    async fn statistics_handler_increments_403() {
+        let (stats, tx, handle) = setup_stats_test();
+
+        let err = StatCommand::AddError(StatError::Status403);
+        let err2 = StatCommand::AddError(StatError::Status403);
+
+        tx.send(err).unwrap_or_default();
+        tx.send(err2).unwrap_or_default();
+
+        teardown_stats_test(tx, handle).await;
+
+        assert_eq!(stats.errors.load(Ordering::Relaxed), 2);
+        assert_eq!(stats.requests.load(Ordering::Relaxed), 2);
+        assert_eq!(stats.status_403s.load(Ordering::Relaxed), 2);
+        assert_eq!(stats.client_errors.load(Ordering::Relaxed), 2);
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// when sent StatCommand::AddRequest, stats object should reflect the change
+    ///
+    /// incrementing a 403 (tracked in status_403s) should also increment:
+    ///     - requests
+    ///     - client_errors
+    async fn statistics_handler_increments_403_via_status_code() {
+        let (stats, tx, handle) = setup_stats_test();
+
+        let err = StatCommand::AddStatus(reqwest::StatusCode::FORBIDDEN);
+        let err2 = StatCommand::AddStatus(reqwest::StatusCode::FORBIDDEN);
+
+        tx.send(err).unwrap_or_default();
+        tx.send(err2).unwrap_or_default();
+
+        teardown_stats_test(tx, handle).await;
+
+        assert_eq!(stats.requests.load(Ordering::Relaxed), 2);
+        assert_eq!(stats.status_403s.load(Ordering::Relaxed), 2);
+        assert_eq!(stats.client_errors.load(Ordering::Relaxed), 2);
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// when sent StatCommand::AddStatus, stats object should reflect the change
+    ///
+    /// incrementing a 500 (tracked in server_errors) should also increment:
+    ///     - requests
+    async fn statistics_handler_increments_500_via_status_code() {
+        let (stats, tx, handle) = setup_stats_test();
+
+        let err = StatCommand::AddStatus(reqwest::StatusCode::INTERNAL_SERVER_ERROR);
+        let err2 = StatCommand::AddStatus(reqwest::StatusCode::INTERNAL_SERVER_ERROR);
+
+        tx.send(err).unwrap_or_default();
+        tx.send(err2).unwrap_or_default();
+
+        teardown_stats_test(tx, handle).await;
+
+        assert_eq!(stats.requests.load(Ordering::Relaxed), 2);
+        assert_eq!(stats.server_errors.load(Ordering::Relaxed), 2);
+    }
+
+    #[test]
+    /// when Stats::add_error receives StatError::Timeout, it should increment the following:
+    ///     - timeouts
+    ///     - requests
+    ///     - errors
+    fn stats_increments_timeouts() {
+        let stats = Stats::new();
+        stats.add_error(StatError::Timeout);
+        stats.add_error(StatError::Timeout);
+        stats.add_error(StatError::Timeout);
+        stats.add_error(StatError::Timeout);
+
+        assert_eq!(stats.errors.load(Ordering::Relaxed), 4);
+        assert_eq!(stats.requests.load(Ordering::Relaxed), 4);
+        assert_eq!(stats.timeouts.load(Ordering::Relaxed), 4);
+    }
+
+    #[test]
+    /// when Stats::update_usize_field receives StatField::WildcardsFiltered, it should increment
+    /// the following:
+    ///     - responses_filtered
+    fn stats_increments_wildcards() {
+        let stats = Stats::new();
+        assert_eq!(stats.responses_filtered.load(Ordering::Relaxed), 0);
+        assert_eq!(stats.wildcards_filtered.load(Ordering::Relaxed), 0);
+
+        stats.update_usize_field(StatField::WildcardsFiltered, 1);
+        stats.update_usize_field(StatField::WildcardsFiltered, 1);
+
+        assert_eq!(stats.responses_filtered.load(Ordering::Relaxed), 2);
+        assert_eq!(stats.wildcards_filtered.load(Ordering::Relaxed), 2);
+    }
+
+    #[test]
+    /// when Stats::update_usize_field receives StatField::ResponsesFiltered, it should increment
+    fn stats_increments_responses_filtered() {
+        let stats = Stats::new();
+        assert_eq!(stats.responses_filtered.load(Ordering::Relaxed), 0);
+
+        stats.update_usize_field(StatField::ResponsesFiltered, 1);
+        stats.update_usize_field(StatField::ResponsesFiltered, 1);
+        stats.update_usize_field(StatField::ResponsesFiltered, 1);
+
+        assert_eq!(stats.responses_filtered.load(Ordering::Relaxed), 3);
+    }
+
+    #[test]
+    /// Stats::merge_from should properly incrememnt expected fields and ignore others
+    fn stats_merge_from_alters_correct_fields() {
+        let contents = r#"{"statistics":{"type":"statistics","timeouts":1,"requests":9207,"expected_per_scan":707,"total_expected":9191,"errors":3,"successes":720,"redirects":13,"client_errors":8474,"server_errors":2,"total_scans":13,"initial_targets":1,"links_extracted":51,"status_403s":3,"status_200s":720,"status_301s":12,"status_302s":1,"status_401s":4,"status_429s":2,"status_500s":5,"status_503s":9,"status_504s":6,"status_508s":7,"wildcards_filtered":707,"responses_filtered":707,"resources_discovered":27,"directory_scan_times":[2.211973078,1.989015505,1.898675839,3.9714468910000003,4.938152838,5.256073528,6.021986595,6.065740734,6.42633762,7.095142125,7.336982137,5.319785619,4.843649778],"total_runtime":[11.556575456000001],"url_format_errors":17,"redirection_errors":12,"connection_errors":21,"request_errors":4}}"#;
+        let stats = Stats::new();
+        let tfile = NamedTempFile::new().unwrap();
+        write(&tfile, contents).unwrap();
+
+        stats.merge_from(tfile.path().to_str().unwrap());
+
+        // as of 1.11.1; all Stats fields are accounted for whether they're updated in merge_from
+        // or not
+        assert_eq!(atomic_load!(stats.timeouts), 1);
+        assert_eq!(atomic_load!(stats.requests), 9207);
+        assert_eq!(atomic_load!(stats.expected_per_scan), 0); // not updated in merge_from
+        assert_eq!(atomic_load!(stats.total_expected), 0); // not updated in merge_from
+        assert_eq!(atomic_load!(stats.errors), 3);
+        assert_eq!(atomic_load!(stats.successes), 720);
+        assert_eq!(atomic_load!(stats.redirects), 13);
+        assert_eq!(atomic_load!(stats.client_errors), 8474);
+        assert_eq!(atomic_load!(stats.server_errors), 2);
+        assert_eq!(atomic_load!(stats.total_scans), 0); // not updated in merge_from
+        assert_eq!(atomic_load!(stats.initial_targets), 0); // not updated in merge_from
+        assert_eq!(atomic_load!(stats.links_extracted), 51);
+        assert_eq!(atomic_load!(stats.status_200s), 720);
+        assert_eq!(atomic_load!(stats.status_301s), 12);
+        assert_eq!(atomic_load!(stats.status_302s), 1);
+        assert_eq!(atomic_load!(stats.status_401s), 4);
+        assert_eq!(atomic_load!(stats.status_403s), 3);
+        assert_eq!(atomic_load!(stats.status_429s), 2);
+        assert_eq!(atomic_load!(stats.status_500s), 5);
+        assert_eq!(atomic_load!(stats.status_503s), 9);
+        assert_eq!(atomic_load!(stats.status_504s), 6);
+        assert_eq!(atomic_load!(stats.status_508s), 7);
+        assert_eq!(atomic_load!(stats.wildcards_filtered), 707);
+        assert_eq!(atomic_load!(stats.responses_filtered), 707);
+        assert_eq!(atomic_load!(stats.resources_discovered), 27);
+        assert_eq!(atomic_load!(stats.url_format_errors), 17);
+        assert_eq!(atomic_load!(stats.redirection_errors), 12);
+        assert_eq!(atomic_load!(stats.connection_errors), 21);
+        assert_eq!(atomic_load!(stats.request_errors), 4);
+        assert_eq!(stats.directory_scan_times.lock().unwrap().len(), 13);
+        for scan in stats.directory_scan_times.lock().unwrap().iter() {
+            assert!(scan.max(0.0) > 0.0); // all scans are non-zero
+        }
+        // total_runtime not updated in merge_from
+        assert_eq!(stats.total_runtime.lock().unwrap().len(), 1);
+        assert!((stats.total_runtime.lock().unwrap()[0] - 0.0).abs() < f64::EPSILON);
+    }
+
+    #[test]
+    /// ensure update runtime overwrites the default 0th entry
+    fn update_runtime_works() {
+        let stats = Stats::new();
+        assert!((stats.total_runtime.lock().unwrap()[0] - 0.0).abs() < f64::EPSILON);
+        stats.update_runtime(20.2);
+        assert!((stats.total_runtime.lock().unwrap()[0] - 20.2).abs() < f64::EPSILON);
+    }
+
+    #[test]
+    /// Stats::save should write contents of Stats to disk
+    fn save_writes_stats_object_to_disk() {
+        let stats = Stats::new();
+        stats.add_request();
+        stats.add_request();
+        stats.add_request();
+        stats.add_request();
+        stats.add_error(StatError::Timeout);
+        stats.add_error(StatError::Timeout);
+        stats.add_error(StatError::Timeout);
+        stats.add_error(StatError::Timeout);
+        stats.add_status_code(StatusCode::OK);
+        stats.add_status_code(StatusCode::OK);
+        stats.add_status_code(StatusCode::OK);
+        let outfile = "/tmp/stuff";
+        stats.save(174.33, outfile);
+        assert!(stats.as_json().contains("statistics"));
+        assert!(stats.as_json().contains("11")); // requests made
+        assert!(stats.as_str().is_empty());
+    }
+}

src/utils.rs

@@ -1,10 +1,47 @@
-use crate::FeroxResult;
-use ansi_term::Color::{Blue, Cyan, Green, Red, Yellow};
-use console::{strip_ansi_codes, user_attended};
+#![macro_use]
+use crate::{
+    config::{CONFIGURATION, PROGRESS_PRINTER},
+    statistics::{
+        StatCommand::{self, AddError, AddStatus},
+        StatError::{Connection, Other, Redirection, Request, Timeout, UrlFormat},
+    },
+    FeroxError, FeroxResult,
+};
+use console::{strip_ansi_codes, style, user_attended};
 use indicatif::ProgressBar;
-use reqwest::Url;
-use reqwest::{Client, Response};
+use reqwest::{Client, Response, Url};
+#[cfg(not(target_os = "windows"))]
+use rlimit::{getrlimit, setrlimit, Resource, Rlim};
 use std::convert::TryInto;
+use std::sync::{Arc, RwLock};
+use std::{fs, io};
+use tokio::sync::mpsc::UnboundedSender;
+
+/// Given the path to a file, open the file in append mode (create it if it doesn't exist) and
+/// return a reference to the file that is buffered and locked
+pub fn open_file(filename: &str) -> Option<Arc<RwLock<io::BufWriter<fs::File>>>> {
+    log::trace!("enter: open_file({})", filename);
+
+    match fs::OpenOptions::new() // std fs
+        .create(true)
+        .append(true)
+        .open(filename)
+    {
+        Ok(file) => {
+            let writer = io::BufWriter::new(file); // std io
+
+            let locked_file = Some(Arc::new(RwLock::new(writer)));
+
+            log::trace!("exit: open_file -> {:?}", locked_file);
+            locked_file
+        }
+        Err(e) => {
+            log::error!("{}", e);
+            log::trace!("exit: open_file -> None");
+            None
+        }
+    }
+}
 
 /// Helper function that determines the current depth of a given url
 ///
@@ -20,13 +57,7 @@ use std::convert::TryInto;
 pub fn get_current_depth(target: &str) -> usize {
     log::trace!("enter: get_current_depth({})", target);
 
-    let target = if !target.ends_with('/') {
-        // target url doesn't end with a /, for the purposes of determining depth, we'll normalize
-        // all urls to end in a / and then calculate accordingly
-        format!("{}/", target)
-    } else {
-        String::from(target)
-    };
+    let target = normalize_url(target);
 
     match Url::parse(&target) {
         Ok(url) => {
@@ -63,17 +94,24 @@ pub fn get_current_depth(target: &str) -> usize {
 /// Takes in a string and examines the first character to return a color version of the same string
 pub fn status_colorizer(status: &str) -> String {
     match status.chars().next() {
-        Some('1') => Blue.paint(status).to_string(), // informational
-        Some('2') => Green.bold().paint(status).to_string(), // success
-        Some('3') => Yellow.paint(status).to_string(), // redirects
-        Some('4') => Red.paint(status).to_string(),  // client error
-        Some('5') => Red.paint(status).to_string(),  // server error
-        Some('W') => Cyan.paint(status).to_string(), // wildcard
-        Some('E') => Red.paint(status).to_string(),  // error
-        _ => status.to_string(),                     // ¯\_(ツ)_/¯
+        Some('1') => style(status).blue().to_string(), // informational
+        Some('2') => style(status).green().to_string(), // success
+        Some('3') => style(status).yellow().to_string(), // redirects
+        Some('4') => style(status).red().to_string(),  // client error
+        Some('5') => style(status).red().to_string(),  // server error
+        Some('W') => style(status).cyan().to_string(), // wildcard
+        Some('E') => style(status).red().to_string(),  // error
+        _ => status.to_string(),                       // ¯\_(ツ)_/¯
     }
 }
 
+/// Takes in a string and colors it using console::style
+///
+/// mainly putting this here in case i want to change the color later, making any changes easy
+pub fn module_colorizer(modname: &str) -> String {
+    style(modname).cyan().to_string()
+}
+
 /// Gets the length of a url's path
 ///
 /// example: http://localhost/stuff -> 5
@@ -82,8 +120,8 @@ pub fn get_url_path_length(url: &Url) -> u64 {
 
     let path = url.path();
 
-    let segments = if path.starts_with('/') {
-        path[1..].split_terminator('/')
+    let segments = if let Some(split) = path.strip_prefix('/') {
+        split.split_terminator('/')
     } else {
         log::trace!("exit: get_url_path_length -> 0");
         return 0;
@@ -128,25 +166,57 @@ pub fn ferox_print(msg: &str, bar: &ProgressBar) {
     }
 }
 
+#[macro_export]
+/// wrapper to improve code readability
+macro_rules! update_stat {
+    ($tx:expr, $value:expr) => {
+        $tx.send($value).unwrap_or_default();
+    };
+}
+
 /// Simple helper to generate a `Url`
 ///
 /// Errors during parsing `url` or joining `word` are propagated up the call stack
 pub fn format_url(
     url: &str,
     word: &str,
-    addslash: bool,
+    add_slash: bool,
     queries: &[(String, String)],
     extension: Option<&str>,
+    tx_stats: UnboundedSender<StatCommand>,
 ) -> FeroxResult<Url> {
     log::trace!(
-        "enter: format_url({}, {}, {}, {:?} {:?})",
+        "enter: format_url({}, {}, {}, {:?} {:?}, {:?})",
         url,
         word,
-        addslash,
+        add_slash,
         queries,
-        extension
+        extension,
+        tx_stats
     );
 
+    if Url::parse(&word).is_ok() {
+        // when a full url is passed in as a word to be joined to a base url using
+        // reqwest::Url::join, the result is that the word (url) completely overwrites the base
+        // url, potentially resulting in requests to places that aren't actually the target
+        // specified.
+        //
+        // in order to resolve the issue, we check if the word from the wordlist is a parsable URL
+        // and if so, don't do any further processing
+        let message = format!(
+            "word ({}) from the wordlist is actually a URL, skipping...",
+            word
+        );
+        log::warn!("{}", message);
+
+        let err = FeroxError { message };
+
+        update_stat!(tx_stats, AddError(UrlFormat));
+
+        log::trace!("exit: format_url -> {}", err);
+        return Err(Box::new(err));
+    }
+
     // from reqwest::Url::join
     //   Note: a trailing slash is significant. Without it, the last path component
     //   is considered to be a “file” name to be removed to get at the “directory”
@@ -154,7 +224,11 @@ pub fn format_url(
     //
     // the transforms that occur here will need to keep this in mind, i.e. add a slash to preserve
     // the current directory sent as part of the url
-    let url = if !url.ends_with('/') {
+    let url = if word.is_empty() {
+        // v1.0.6: added during --extract-links feature implementation to support creating urls
+        // that were extracted from response bodies, i.e. http://localhost/some/path/js/main.js
+        url.to_string()
+    } else if !url.ends_with('/') {
         format!("{}/", url)
     } else {
         url.to_string()
@@ -165,9 +239,18 @@ pub fn format_url(
     // extensions and slashes are mutually exclusive cases
     let word = if extension.is_some() {
         format!("{}.{}", word, extension.unwrap())
-    } else if addslash && !word.ends_with('/') {
+    } else if add_slash && !word.ends_with('/') {
         // -f used, and word doesn't already end with a /
         format!("{}/", word)
+    } else if word.starts_with("//") {
+        // bug ID'd by @Sicks3c, when a wordlist contains words that begin with 2 forward slashes
+        // i.e. //1_40_0/static/js, it gets joined onto the base url in a surprising way
+        // ex: https://localhost/ + //1_40_0/static/js -> https://1_40_0/static/js
+        // this is due to the fact that //... is a valid url. The fix is introduced here in 1.12.2
+        // and simply removes prefixed forward slashes if there are two of them. Additionally,
+        // trim_start_matches will trim the pattern until it's gone, so even if there are more than
+        // 2 /'s, they'll still be trimmed
+        word.trim_start_matches('/').to_string()
     } else {
         String::from(word)
     };
@@ -198,6 +281,7 @@ pub fn format_url(
             }
         }
         Err(e) => {
+            update_stat!(tx_stats, AddError(UrlFormat));
             log::trace!("exit: format_url -> {}", e);
             log::error!("Could not join {} with {}", word, base_url);
             Err(Box::new(e))
@@ -206,86 +290,391 @@ pub fn format_url(
 }
 
 /// Initiate request to the given `Url` using `Client`
-pub async fn make_request(client: &Client, url: &Url) -> FeroxResult<Response> {
-    log::trace!("enter: make_request(CONFIGURATION.Client, {})", url);
+pub async fn make_request(
+    client: &Client,
+    url: &Url,
+    tx_stats: UnboundedSender<StatCommand>,
+) -> FeroxResult<Response> {
+    log::trace!(
+        "enter: make_request(CONFIGURATION.Client, {}, {:?})",
+        url,
+        tx_stats
+    );
 
     match client.get(url.to_owned()).send().await {
-        Ok(resp) => {
-            log::debug!("requested Url: {}", resp.url());
-            log::trace!("exit: make_request -> {:?}", resp);
-            Ok(resp)
-        }
         Err(e) => {
+            let mut log_level = log::Level::Error;
+
             log::trace!("exit: make_request -> {}", e);
-            log::error!("Error while making request: {}", e);
+            if e.is_timeout() {
+                // only warn for timeouts, while actual errors are still left as errors
+                log_level = log::Level::Warn;
+                update_stat!(tx_stats, AddError(Timeout));
+            } else if e.is_redirect() {
+                if let Some(last_redirect) = e.url() {
+                    // get where we were headed (last_redirect) and where we came from (url)
+                    let fancy_message = format!("{} !=> {}", url, last_redirect);
+
+                    let report = if let Some(msg_status) = e.status() {
+                        update_stat!(tx_stats, AddStatus(msg_status));
+                        create_report_string(msg_status.as_str(), "-1", "-1", "-1", &fancy_message)
+                    } else {
+                        create_report_string("UNK", "-1", "-1", "-1", &fancy_message)
+                    };
+
+                    update_stat!(tx_stats, AddError(Redirection));
+
+                    ferox_print(&report, &PROGRESS_PRINTER)
+                };
+            } else if e.is_connect() {
+                update_stat!(tx_stats, AddError(Connection));
+            } else if e.is_request() {
+                update_stat!(tx_stats, AddError(Request));
+            } else {
+                update_stat!(tx_stats, AddError(Other));
+            }
+
+            if matches!(log_level, log::Level::Error) {
+                log::error!("Error while making request: {}", e);
+            } else {
+                log::warn!("Error while making request: {}", e);
+            }
+
             Err(Box::new(e))
         }
+        Ok(resp) => {
+            log::trace!("exit: make_request -> {:?}", resp);
+            update_stat!(tx_stats, AddStatus(resp.status()));
+            Ok(resp)
+        }
     }
 }
 
+/// Helper to create the standard line for output to file/terminal
+///
+/// example output:
+/// 200      127l      283w     4134c http://localhost/faq
+pub fn create_report_string(
+    status: &str,
+    line_count: &str,
+    word_count: &str,
+    content_length: &str,
+    url: &str,
+) -> String {
+    if CONFIGURATION.quiet {
+        // -q used, just need the url
+        format!("{}\n", url)
+    } else {
+        // normal printing with status and sizes
+        let color_status = status_colorizer(status);
+        format!(
+            "{} {:>8}l {:>8}w {:>8}c {}\n",
+            color_status, line_count, word_count, content_length, url
+        )
+    }
+}
+
+/// Attempts to set the soft limit for the RLIMIT_NOFILE resource
+///
+/// RLIMIT_NOFILE is the maximum number of file descriptors that can be opened by this process
+///
+/// The soft limit is the value that the kernel enforces for the corresponding resource.
+/// The hard limit acts as a ceiling for the soft limit: an unprivileged process may set only its
+/// soft limit to a value in the range from 0 up to the hard limit, and (irreversibly) lower its
+/// hard limit.
+///
+/// A child process created via fork(2) inherits its parent's resource limits. Resource limits are
+/// per-process attributes that are shared by all of the threads in a process.
+///
+/// Based on the above information, no attempt is made to restore the limit to its pre-scan value
+/// as the adjustment made here is only valid for the scan itself (and any child processes, of which
+/// there are none).
+#[cfg(not(target_os = "windows"))]
+pub fn set_open_file_limit(limit: usize) -> bool {
+    log::trace!("enter: set_open_file_limit");
+
+    if let Ok((soft, hard)) = getrlimit(Resource::NOFILE) {
+        if hard.as_usize() > limit {
+            // our default open file limit is less than the current hard limit, this means we can
+            // set the soft limit to our default
+            let new_soft_limit = Rlim::from_usize(limit);
+
+            if setrlimit(Resource::NOFILE, new_soft_limit, hard).is_ok() {
+                log::debug!("set open file descriptor limit to {}", limit);
+
+                log::trace!("exit: set_open_file_limit -> {}", true);
+                return true;
+            }
+        } else if soft != hard {
+            // hard limit is lower than our default, the next best option is to set the soft limit as
+            // high as the hard limit will allow
+            if setrlimit(Resource::NOFILE, hard, hard).is_ok() {
+                log::debug!("set open file descriptor limit to {}", limit);
+
+                log::trace!("exit: set_open_file_limit -> {}", true);
+                return true;
+            }
+        }
+    }
+
+    // failed to set a new limit, as limit adjustments are a 'nice to have', we'll just log
+    // and move along
+    log::warn!("could not set open file descriptor limit to {}", limit);
+
+    log::trace!("exit: set_open_file_limit -> {}", false);
+    false
+}
+
+/// Simple helper to abstract away adding a forward-slash to a url if not present
+///
+/// used mostly for deduplication purposes and url state tracking
+pub fn normalize_url(url: &str) -> String {
+    log::trace!("enter: normalize_url({})", url);
+
+    let normalized = if url.ends_with('/') {
+        url.to_string()
+    } else {
+        format!("{}/", url)
+    };
+
+    log::trace!("exit: normalize_url -> {}", normalized);
+    normalized
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::FeroxChannel;
+    use tokio::sync::mpsc;
 
     #[test]
+    /// set_open_file_limit with a low requested limit succeeds
+    fn utils_set_open_file_limit_with_low_requested_limit() {
+        let (_, hard) = getrlimit(Resource::NOFILE).unwrap();
+        let lower_limit = hard.as_usize() - 1;
+        assert!(set_open_file_limit(lower_limit));
+    }
+
+    #[test]
+    /// set_open_file_limit with a high requested limit succeeds
+    fn utils_set_open_file_limit_with_high_requested_limit() {
+        let (_, hard) = getrlimit(Resource::NOFILE).unwrap();
+        let higher_limit = hard.as_usize() + 1;
+        // calculate a new soft to ensure soft != hard and hit that logic branch
+        let new_soft = Rlim::from_usize(hard.as_usize() - 1);
+        setrlimit(Resource::NOFILE, new_soft, hard).unwrap();
+        assert!(set_open_file_limit(higher_limit));
+    }
+
+    #[test]
+    /// set_open_file_limit should fail when hard == soft
+    fn utils_set_open_file_limit_with_fails_when_both_limits_are_equal() {
+        let (_, hard) = getrlimit(Resource::NOFILE).unwrap();
+        // calculate a new soft to ensure soft == hard and hit the failure logic branch
+        setrlimit(Resource::NOFILE, hard, hard).unwrap();
+        assert!(!set_open_file_limit(hard.as_usize())); // returns false
+    }
+
+    #[test]
+    /// base url returns 1
     fn get_current_depth_base_url_returns_1() {
         let depth = get_current_depth("http://localhost");
         assert_eq!(depth, 1);
     }
 
     #[test]
+    /// base url with slash returns 1
     fn get_current_depth_base_url_with_slash_returns_1() {
         let depth = get_current_depth("http://localhost/");
         assert_eq!(depth, 1);
     }
 
     #[test]
+    /// base url + 1 dir returns 2
     fn get_current_depth_one_dir_returns_2() {
         let depth = get_current_depth("http://localhost/src");
         assert_eq!(depth, 2);
     }
 
     #[test]
+    /// base url + 1 dir and slash returns 2
     fn get_current_depth_one_dir_with_slash_returns_2() {
         let depth = get_current_depth("http://localhost/src/");
         assert_eq!(depth, 2);
     }
 
     #[test]
+    /// base url + 1 dir and slash returns 2
+    fn get_current_depth_single_forward_slash_is_zero() {
+        let depth = get_current_depth("");
+        assert_eq!(depth, 0);
+    }
+
+    #[test]
+    /// base url + 1 word + no slash + no extension
     fn format_url_normal() {
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
         assert_eq!(
-            format_url("http://localhost", "stuff", false, &Vec::new(), None).unwrap(),
+            format_url("http://localhost", "stuff", false, &Vec::new(), None, tx).unwrap(),
             reqwest::Url::parse("http://localhost/stuff").unwrap()
         );
     }
 
     #[test]
+    /// base url + no word + no slash + no extension
     fn format_url_no_word() {
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
         assert_eq!(
-            format_url("http://localhost", "", false, &Vec::new(), None).unwrap(),
+            format_url("http://localhost", "", false, &Vec::new(), None, tx).unwrap(),
             reqwest::Url::parse("http://localhost").unwrap()
         );
     }
 
     #[test]
-    #[should_panic]
-    fn format_url_no_url() {
-        format_url("", "stuff", false, &Vec::new(), None).unwrap();
+    /// base url + word + no slash + no extension + queries
+    fn format_url_joins_queries() {
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+        assert_eq!(
+            format_url(
+                "http://localhost",
+                "lazer",
+                false,
+                &[(String::from("stuff"), String::from("things"))],
+                None,
+                tx
+            )
+            .unwrap(),
+            reqwest::Url::parse("http://localhost/lazer?stuff=things").unwrap()
+        );
     }
 
     #[test]
-    fn format_url_word_with_preslash() {
+    /// base url + no word + no slash + no extension + queries
+    fn format_url_without_word_joins_queries() {
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
         assert_eq!(
-            format_url("http://localhost", "/stuff", false, &Vec::new(), None).unwrap(),
+            format_url(
+                "http://localhost",
+                "",
+                false,
+                &[(String::from("stuff"), String::from("things"))],
+                None,
+                tx
+            )
+            .unwrap(),
+            reqwest::Url::parse("http://localhost/?stuff=things").unwrap()
+        );
+    }
+
+    #[test]
+    #[should_panic]
+    /// no base url is an error
+    fn format_url_no_url() {
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+        format_url("", "stuff", false, &Vec::new(), None, tx).unwrap();
+    }
+
+    #[test]
+    /// word prepended with slash is adjusted for correctness
+    fn format_url_word_with_preslash() {
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+        assert_eq!(
+            format_url("http://localhost", "/stuff", false, &Vec::new(), None, tx).unwrap(),
             reqwest::Url::parse("http://localhost/stuff").unwrap()
         );
     }
 
     #[test]
+    /// word with appended slash allows the slash to persist
     fn format_url_word_with_postslash() {
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
         assert_eq!(
-            format_url("http://localhost", "stuff/", false, &Vec::new(), None).unwrap(),
+            format_url("http://localhost", "stuff/", false, &Vec::new(), None, tx).unwrap(),
             reqwest::Url::parse("http://localhost/stuff/").unwrap()
         );
     }
+
+    #[test]
+    /// word with two prepended slashes doesn't discard the entire domain
+    fn format_url_word_with_two_prepended_slashes() {
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
+        let result = format_url(
+            "http://localhost",
+            "//upload/img",
+            false,
+            &Vec::new(),
+            None,
+            tx,
+        )
+        .unwrap();
+
+        assert_eq!(
+            result,
+            reqwest::Url::parse("http://localhost/upload/img").unwrap()
+        );
+    }
+
+    #[test]
+    /// word that is a fully formed url, should return an error
+    fn format_url_word_that_is_a_url() {
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+        let url = format_url(
+            "http://localhost",
+            "http://schmocalhost",
+            false,
+            &Vec::new(),
+            None,
+            tx,
+        );
+        assert!(url.is_err());
+    }
+
+    #[test]
+    /// status colorizer uses red for 500s
+    fn status_colorizer_uses_red_for_500s() {
+        assert_eq!(status_colorizer("500"), style("500").red().to_string());
+    }
+
+    #[test]
+    /// status colorizer uses red for 400s
+    fn status_colorizer_uses_red_for_400s() {
+        assert_eq!(status_colorizer("400"), style("400").red().to_string());
+    }
+
+    #[test]
+    /// status colorizer uses red for errors
+    fn status_colorizer_uses_red_for_errors() {
+        assert_eq!(status_colorizer("ERROR"), style("ERROR").red().to_string());
+    }
+
+    #[test]
+    /// status colorizer uses cyan for wildcards
+    fn status_colorizer_uses_cyan_for_wildcards() {
+        assert_eq!(status_colorizer("WLD"), style("WLD").cyan().to_string());
+    }
+
+    #[test]
+    /// status colorizer uses blue for 100s
+    fn status_colorizer_uses_blue_for_100s() {
+        assert_eq!(status_colorizer("100"), style("100").blue().to_string());
+    }
+
+    #[test]
+    /// status colorizer uses green for 200s
+    fn status_colorizer_uses_green_for_200s() {
+        assert_eq!(status_colorizer("200"), style("200").green().to_string());
+    }
+
+    #[test]
+    /// status colorizer uses yellow for 300s
+    fn status_colorizer_uses_yellow_for_300s() {
+        assert_eq!(status_colorizer("300"), style("300").yellow().to_string());
+    }
+
+    #[test]
+    /// status colorizer doesnt color anything else
+    fn status_colorizer_returns_as_is() {
+        assert_eq!(status_colorizer("farfignewton"), "farfignewton".to_string());
+    }
 }

tests/test_banner.rs

@@ -0,0 +1,839 @@
+mod utils;
+use assert_cmd::Command;
+use predicates::prelude::*;
+use utils::{setup_tmp_directory, teardown_tmp_directory};
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + proxy
+fn banner_prints_proxy() -> Result<(), Box<dyn std::error::Error>> {
+    let urls = vec![
+        String::from("http://localhost"),
+        String::from("http://schmocalhost"),
+    ];
+    let (tmp_dir, file) = setup_tmp_directory(&urls, "wordlist")?;
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--stdin")
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--proxy")
+        .arg("http://127.0.0.1:8080")
+        .pipe_stdin(file)
+        .unwrap()
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("http://schmocalhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Proxy"))
+                .and(predicate::str::contains("http://127.0.0.1:8080"))
+                .and(predicate::str::contains("─┴─")),
+        );
+
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + replay proxy
+fn banner_prints_replay_proxy() -> Result<(), Box<dyn std::error::Error>> {
+    let urls = vec![
+        String::from("http://localhost"),
+        String::from("http://schmocalhost"),
+    ];
+    let (tmp_dir, file) = setup_tmp_directory(&urls, "wordlist")?;
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--stdin")
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--replay-proxy")
+        .arg("http://127.0.0.1:8081")
+        .pipe_stdin(file)
+        .unwrap()
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("http://schmocalhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Replay Proxy"))
+                .and(predicate::str::contains("http://127.0.0.1:8081"))
+                .and(predicate::str::contains("─┴─")),
+        );
+
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + multiple headers
+fn banner_prints_headers() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--headers")
+        .arg("stuff:things")
+        .arg("-H")
+        .arg("mostuff:mothings")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Header"))
+                .and(predicate::str::contains("stuff: things"))
+                .and(predicate::str::contains("mostuff: mothings"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + multiple size filters
+fn banner_prints_filter_sizes() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-S")
+        .arg("789456123")
+        .arg("--filter-size")
+        .arg("44444444")
+        .arg("-N")
+        .arg("678")
+        .arg("--filter-lines")
+        .arg("679")
+        .arg("-W")
+        .arg("93")
+        .arg("--filter-words")
+        .arg("94")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Size Filter"))
+                .and(predicate::str::contains("Word Count Filter"))
+                .and(predicate::str::contains("Line Count Filter"))
+                .and(predicate::str::contains("789456123"))
+                .and(predicate::str::contains("44444444"))
+                .and(predicate::str::contains("93"))
+                .and(predicate::str::contains("94"))
+                .and(predicate::str::contains("678"))
+                .and(predicate::str::contains("679"))
+                .and(predicate::str::contains("─┴─")),
+        );
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + queries
+fn banner_prints_queries() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-Q")
+        .arg("token=supersecret")
+        .arg("--query")
+        .arg("stuff=things")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Query Parameter"))
+                .and(predicate::str::contains("token=supersecret"))
+                .and(predicate::str::contains("stuff=things"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + status codes
+fn banner_prints_status_codes() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-s")
+        .arg("201,301,401")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("[201, 301, 401]"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + replay codes
+fn banner_prints_replay_codes() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--replay-codes")
+        .arg("200,302")
+        .arg("--replay-proxy")
+        .arg("http://localhost:8081")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Replay Proxy"))
+                .and(predicate::str::contains("http://localhost:8081"))
+                .and(predicate::str::contains("Replay Proxy Codes"))
+                .and(predicate::str::contains("[200, 302]"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + output file
+fn banner_prints_output_file() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--output")
+        .arg("/super/cool/path")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Output File"))
+                .and(predicate::str::contains("/super/cool/path"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + insecure
+fn banner_prints_insecure() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-k")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Insecure"))
+                .and(predicate::str::contains("true"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + follow redirects
+fn banner_prints_redirects() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-r")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Follow Redirects"))
+                .and(predicate::str::contains("true"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + extensions
+fn banner_prints_extensions() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-x")
+        .arg("js")
+        .arg("--extensions")
+        .arg("pdf")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Extensions"))
+                .and(predicate::str::contains("[js, pdf]"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + dont_filter
+fn banner_prints_dont_filter() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--dont-filter")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Filter Wildcards"))
+                .and(predicate::str::contains("false"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + verbosity=1
+fn banner_prints_verbosity_one() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-v")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Verbosity"))
+                .and(predicate::str::contains("│ 1"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + verbosity=2
+fn banner_prints_verbosity_two() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-vv")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Verbosity"))
+                .and(predicate::str::contains("│ 2"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + verbosity=3
+fn banner_prints_verbosity_three() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-vvv")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Verbosity"))
+                .and(predicate::str::contains("│ 3"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + verbosity=4
+fn banner_prints_verbosity_four() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-vvvv")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Verbosity"))
+                .and(predicate::str::contains("│ 4"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + add slash
+fn banner_prints_add_slash() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-f")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Add Slash"))
+                .and(predicate::str::contains("true"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + INFINITE recursion
+fn banner_prints_infinite_depth() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--depth")
+        .arg("0")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Recursion Depth"))
+                .and(predicate::str::contains("INFINITE"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + recursion depth
+fn banner_prints_recursion_depth() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--depth")
+        .arg("343214")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Recursion Depth"))
+                .and(predicate::str::contains("343214"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + no recursion
+fn banner_prints_no_recursion() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-n")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Do Not Recurse"))
+                .and(predicate::str::contains("true"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see nothing
+fn banner_doesnt_print() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-q")
+        .assert()
+        .success()
+        .stderr(predicate::str::is_empty());
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + extract-links
+fn banner_prints_extract_links() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-e")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Extract Links"))
+                .and(predicate::str::contains("true"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + scan-limit
+fn banner_prints_scan_limit() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-L")
+        .arg("4")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Concurrent Scan Limit"))
+                .and(predicate::str::contains("│ 4"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + filter-status
+fn banner_prints_filter_status() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-C")
+        .arg("200")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Status Code Filters"))
+                .and(predicate::str::contains("│ [200]"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + json
+fn banner_prints_json() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--json")
+        .arg("--output")
+        .arg("/dev/null")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("JSON Output"))
+                .and(predicate::str::contains("│ true"))
+                .and(predicate::str::contains("─┴─")),
+        );
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + json
+fn banner_prints_debug_log() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--debug-log")
+        .arg("/dev/null")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Debugging Log"))
+                .and(predicate::str::contains("│ /dev/null"))
+                .and(predicate::str::contains("─┴─")),
+        );
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + regex filters
+fn banner_prints_filter_regex() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--filter-regex")
+        .arg("^ignore me$")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Regex Filter"))
+                .and(predicate::str::contains("│ ^ignore me$"))
+                .and(predicate::str::contains("─┴─")),
+        );
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + time limit
+fn banner_prints_time_limit() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--time-limit")
+        .arg("10m")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Time Limit"))
+                .and(predicate::str::contains("│ 10m"))
+                .and(predicate::str::contains("─┴─")),
+        );
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + similarity filter
+fn banner_prints_similarity_filter() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--filter-similar-to")
+        .arg("https://somesite.com")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Similarity Filter"))
+                .and(predicate::str::contains("│ https://somesite.com"))
+                .and(predicate::str::contains("─┴─")),
+        );
+}

tests/test_config.rs

@@ -0,0 +1,27 @@
+mod utils;
+use assert_cmd::prelude::*;
+use predicates::prelude::*;
+use std::process::Command;
+use utils::{setup_tmp_directory, teardown_tmp_directory};
+
+#[test]
+/// send a single valid request, expect a 200 response
+fn read_in_config_file_for_settings() -> Result<(), Box<dyn std::error::Error>> {
+    let (tmp_dir, file) = setup_tmp_directory(&["threads = 37".to_string()], "ferox-config.toml")?;
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .current_dir(&tmp_dir)
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-vvvv")
+        .assert()
+        .success()
+        .stderr(predicate::str::contains("│ 37"));
+
+    teardown_tmp_directory(tmp_dir);
+
+    Ok(())
+}

tests/test_extractor.rs

@@ -0,0 +1,348 @@
+mod utils;
+use assert_cmd::prelude::*;
+use httpmock::Method::GET;
+use httpmock::MockServer;
+use predicates::prelude::*;
+use std::process::Command;
+use utils::{setup_tmp_directory, teardown_tmp_directory};
+
+#[test]
+/// send a request to a page that contains a relative link, --extract-links should find the link
+/// and make a request to the new link
+fn extractor_finds_absolute_url() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200)
+            .body(&srv.url("'/homepage/assets/img/icons/handshake.svg'"));
+    });
+
+    let mock_two = srv.mock(|when, then| {
+        when.method(GET)
+            .path("/homepage/assets/img/icons/handshake.svg");
+        then.status(200);
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--extract-links")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .and(predicate::str::contains("200"))
+            .and(predicate::str::contains(
+                "/homepage/assets/img/icons/handshake.svg",
+            )),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_two.hits(), 1);
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// send a request to a page that contains an absolute link to another domain, scanner should not
+/// follow
+fn extractor_finds_absolute_url_to_different_domain() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200)
+            .body("\"http://localhost/homepage/assets/img/icons/handshake.svg\"");
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--extract-links")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .and(predicate::str::contains("200"))
+            .and(predicate::str::contains(
+                "/homepage/assets/img/icons/handshake.svg",
+            ))
+            .not(),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// send a request to a page that contains a relative link, should follow
+fn extractor_finds_relative_url() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200)
+            .body("\"/homepage/assets/img/icons/handshake.svg\"");
+    });
+
+    let mock_two = srv.mock(|when, then| {
+        when.method(GET)
+            .path("/homepage/assets/img/icons/handshake.svg");
+        then.status(200);
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--extract-links")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .and(predicate::str::contains("200"))
+            .and(predicate::str::contains(
+                "/homepage/assets/img/icons/handshake.svg",
+            )),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_two.hits(), 1);
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// send a request to a page that contains an relative link, follow it, and find the same link again
+/// should follow then filter
+fn extractor_finds_same_relative_url_twice() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) =
+        setup_tmp_directory(&["LICENSE".to_string(), "README".to_string()], "wordlist").unwrap();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200)
+            .body(&srv.url("\"/homepage/assets/img/icons/handshake.svg\""));
+    });
+
+    let mock_two = srv.mock(|when, then| {
+        when.method(GET).path("/README");
+        then.status(200)
+            .body(&srv.url("\"/homepage/assets/img/icons/handshake.svg\""));
+    });
+
+    let mock_three = srv.mock(|when, then| {
+        when.method(GET)
+            .path("/homepage/assets/img/icons/handshake.svg");
+        then.status(200);
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--extract-links")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .and(predicate::str::contains("200"))
+            // .count(1) asserts that we only see the endpoint reported once, even though there
+            // is the potential to request the same url twice
+            .and(predicate::str::contains("/homepage/assets/img/icons/handshake.svg").count(1)),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_two.hits(), 1);
+    assert!(mock_three.hits() <= 2);
+    teardown_tmp_directory(tmp_dir);
+}
+
+#[test]
+/// send a request to a page that contains an absolute link that leads to a page with a filter_size
+/// that should filter it out, expect not to see the second response reported
+fn extractor_finds_filtered_content() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) =
+        setup_tmp_directory(&["LICENSE".to_string(), "README".to_string()], "wordlist")?;
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200)
+            .body(&srv.url("\"/homepage/assets/img/icons/handshake.svg\""));
+    });
+
+    let mock_two = srv.mock(|when, then| {
+        when.method(GET)
+            .path("/homepage/assets/img/icons/handshake.svg");
+        then.status(200).body("im a little teapot");
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--extract-links")
+        .arg("--filter-size")
+        .arg("18")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .and(predicate::str::contains("200"))
+            .and(predicate::str::contains(
+                "/homepage/assets/img/icons/handshake.svg",
+            ))
+            .not(),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_two.hits(), 1);
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// serve a robots.txt with a file and and a folder link contained within it. ferox should
+/// find both links and request each one. Additionally, a scan should start with the directory
+/// link found, meaning the wordlist will be thrown at the sub directory
+fn extractor_finds_robots_txt_links_and_displays_files_or_scans_directories() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("im a little teapot"); // 18
+    });
+
+    let mock_two = srv.mock(|when, then| {
+        when.method(GET).path("/robots.txt");
+        then.status(200).body(
+            r#"
+            User-agent: *
+            Crawl-delay: 10
+            # CSS, JS, Images
+            Allow: /misc/*.css$
+            Disallow: /misc/stupidfile.php
+               Disallow: /disallowed-subdir/
+            "#,
+        );
+    });
+
+    let mock_file = srv.mock(|when, then| {
+        when.method(GET).path("/misc/stupidfile.php");
+        then.status(200).body("im a little teapot too"); // 22
+    });
+
+    let mock_scanned_file = srv.mock(|when, then| {
+        when.method(GET).path("/misc/LICENSE");
+        then.status(200).body("i too, am a container for tea"); // 29
+    });
+
+    let mock_dir = srv.mock(|when, _| {
+        when.method(GET).path("/misc/");
+    });
+
+    let mock_disallowed = srv.mock(|when, then| {
+        when.method(GET).path("/disallowed-subdir");
+        then.status(404);
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--extract-links")
+        .arg("-vvvv")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE") // 2 directories contain LICENSE
+            .count(2)
+            .and(predicate::str::contains("18c"))
+            .and(predicate::str::contains("/misc/stupidfile.php"))
+            .and(predicate::str::contains("22c"))
+            .and(predicate::str::contains("/misc/LICENSE"))
+            .and(predicate::str::contains("29c"))
+            .and(predicate::str::contains("200").count(3)),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_dir.hits(), 1);
+    assert_eq!(mock_two.hits(), 1);
+    assert_eq!(mock_file.hits(), 1);
+    assert_eq!(mock_disallowed.hits(), 1);
+    assert_eq!(mock_scanned_file.hits(), 1);
+    teardown_tmp_directory(tmp_dir);
+}
+
+#[test]
+/// send a request to a page that contains a link that contains a directory that returns a 403
+/// --extract-links should find the link and make recurse into the 403 directory, finding LICENSE
+fn extractor_recurses_into_403_directories() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200)
+            .body(&srv.url("'/homepage/assets/img/icons/handshake.svg'"));
+    });
+
+    let mock_two = srv.mock(|when, then| {
+        when.method(GET).path("/homepage/assets/img/icons/LICENSE");
+        then.status(200).body("that's just like, your opinion man");
+    });
+
+    let forbidden_dir = srv.mock(|when, then| {
+        when.method(GET).path("/homepage/assets/img/icons/");
+        then.status(403);
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--extract-links")
+        .arg("--depth") // need to go past default 4 directories
+        .arg("0")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .count(2)
+            .and(predicate::str::contains("1w")) // link in /LICENSE
+            .and(predicate::str::contains("34c")) // recursed LICENSE
+            .and(predicate::str::contains(
+                "/homepage/assets/img/icons/LICENSE",
+            )),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_two.hits(), 1);
+    assert_eq!(forbidden_dir.hits(), 1);
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}

tests/test_filters.rs

@@ -0,0 +1,249 @@
+mod utils;
+use assert_cmd::prelude::*;
+use httpmock::Method::GET;
+use httpmock::MockServer;
+use predicates::prelude::*;
+use std::process::Command;
+use utils::{setup_tmp_directory, teardown_tmp_directory};
+
+#[test]
+/// create a FeroxResponse that should elicit a true from
+/// StatusCodeFilter::should_filter_response
+fn filters_status_code_should_filter_response() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) =
+        setup_tmp_directory(&["LICENSE".to_string(), "file.js".to_string()], "wordlist").unwrap();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(302).body("this is a test");
+    });
+
+    let mock_two = srv.mock(|when, then| {
+        when.method(GET).path("/file.js");
+        then.status(200).body("this is also a test of some import");
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-vvvv")
+        .arg("--filter-status")
+        .arg("302")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .not()
+            .and(predicate::str::contains("302"))
+            .not()
+            .and(predicate::str::contains("14c"))
+            .not()
+            .and(predicate::str::contains("/file.js"))
+            .and(predicate::str::contains("200"))
+            .and(predicate::str::contains("34c")),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_two.hits(), 1);
+    teardown_tmp_directory(tmp_dir);
+}
+
+#[test]
+/// create a FeroxResponse that should elicit a true from
+/// LinesFilter::should_filter_response
+fn filters_lines_should_filter_response() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) =
+        setup_tmp_directory(&["LICENSE".to_string(), "file.js".to_string()], "wordlist").unwrap();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(302).body("this is a test");
+    });
+
+    let mock_two = srv.mock(|when, then| {
+        when.method(GET).path("/file.js");
+        then.status(200)
+            .body("this is also a test of some import\nwith 2 lines, no less");
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--filter-lines")
+        .arg("2")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .and(predicate::str::contains("302"))
+            .and(predicate::str::contains("14"))
+            .and(predicate::str::contains("/file.js"))
+            .not()
+            .and(predicate::str::contains("200"))
+            .not()
+            .and(predicate::str::contains("2l"))
+            .not(),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_two.hits(), 1);
+    teardown_tmp_directory(tmp_dir);
+}
+
+#[test]
+/// create a FeroxResponse that should elicit a true from
+/// WordsFilter::should_filter_response
+fn filters_words_should_filter_response() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) =
+        setup_tmp_directory(&["LICENSE".to_string(), "file.js".to_string()], "wordlist").unwrap();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(302).body("this is a test");
+    });
+
+    let mock_two = srv.mock(|when, then| {
+        when.method(GET).path("/file.js");
+        then.status(200)
+            .body("this is also a test of some import\nwith 2 lines, no less");
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--filter-words")
+        .arg("13")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .and(predicate::str::contains("302"))
+            .and(predicate::str::contains("14"))
+            .and(predicate::str::contains("/file.js"))
+            .not()
+            .and(predicate::str::contains("200"))
+            .not()
+            .and(predicate::str::contains("13w"))
+            .not(),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_two.hits(), 1);
+    teardown_tmp_directory(tmp_dir);
+}
+
+#[test]
+/// create a FeroxResponse that should elicit a true from
+/// SizeFilter::should_filter_response
+fn filters_size_should_filter_response() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) =
+        setup_tmp_directory(&["LICENSE".to_string(), "file.js".to_string()], "wordlist").unwrap();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(302).body("this is a test");
+    });
+
+    let mock_two = srv.mock(|when, then| {
+        when.method(GET).path("/file.js");
+        then.status(200)
+            .body("this is also a test of some import\nwith 2 lines, no less");
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--filter-size")
+        .arg("56")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .and(predicate::str::contains("302"))
+            .and(predicate::str::contains("14"))
+            .and(predicate::str::contains("/file.js"))
+            .not()
+            .and(predicate::str::contains("200"))
+            .not()
+            .and(predicate::str::contains("56c"))
+            .not(),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_two.hits(), 1);
+    teardown_tmp_directory(tmp_dir);
+}
+
+#[test]
+/// create a FeroxResponse that should elicit a true from
+/// SimilarityFilter::should_filter_response
+fn filters_similar_should_filter_response() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(
+        &["not-similar".to_string(), "similar".to_string()],
+        "wordlist",
+    )
+    .unwrap();
+
+    // ''.join(random.choices(string.ascii_letters + string.digits + string.whitespace, k=4096))
+    let content = "VCiYFr0HKsEIK6r\r1hJLYnOr90Aji\rDWAjQA3LVAzrluN48FuSPrRpm\n \tV\x0cx\nSCc5sX\nTB\x0c6Of7ns\t2HDwQCduKTqG8gG\x0beszazwljW01H60HMOLziOKwQwEYV7CbrLWQiLeCWKVxX\rvag\nAAEOhjER7gURuGXw\nMyY\t8mSw\x0b\x0bK0Z9G0Pt\x0bJZItAIqAq FxeaoOeLqWVFvxtDFfko0YVYt1I\rNmSXZ4lnOoiBCLbu6TLb80lClhY\tPN7Lp36F786I\nglwRK2oD45EtN SWW IF6uqKdf\x0czAcVycf\x0cBzHYnn1HAkU2Jluos0qwMGJ2m74z\nLd3\x0cIUVZmnRmHHWQGd1u2xmsZR\x0bfnml10ur6J\x0ba8xOZatiY 15Aq3KOGWdD3xQwqo\r5SKnnxH5tqU\rO\rZpJ\n7t7UUgfE\niWFgqWDpMeOG 1248M I\ro5B9Yed\r2aq2\tXxLn31s3hCV WEfQd60DKp6eFhUeUSeXDq6qjgTnWigoCZQERf\rXp7s2L37 iOEMl3\r41\nBShOjLfD8Kj0\rbu0ENreRjP\nY77jsrsaYgOsUrEzw\x0bw3OLi\n8fkddcaOvJeutTy B\rsDMkK\x0cnx2S0N\x0cDaY\x0c9iyo6p4IL\tOC1qgNlWP4VLg\tWmPG46ZMCirth5h4FwkS\nD2WsiEA2Z\n0xbLd7Uww hUQC6 3V\r1SsWem4UcQxG\rfuVvWl\nD9\nDpZQFFgiqhQiq1I0LMAR\r\rKBmj4iurrxaoMHTl9oj\x0b0N3AfD17gyqZiJ67bgizvecsRGeB1f\x0c\nYRvieJqIVHDKOOR\ruhqnVZz4BQ5FFBusz\x0cZl5\x0bt\tbdOUhAAAKyA6Jwl 7OjzojiRHGD6dl ncsgndsKURhFv4\tV5d\n73iPzbT\t8v6IrJtnq\nJuFl7A\x0b\rVnnsjTW0Y4QB1BgCy3B\x0cma7\tpPt5jmcJH7v5J\tYKEXh UqRChBFY5nbFbmXjJYxevPYJmSHC\rDQ4j9de\rTMZ\rtWaPAzkJjH\x0c\nyrEuf9WaMM\trFlKo9r9w\r\nQkQqIEu8Gfr\t aRzvN\r2oZhCyB4fa\np37\tXQi4Wa\no7gHUDQLoRvkK1dy2K3ydrI0O6\rFTGS7oHA\x0bajFOd\rcS5W25tFGhocwxM0\nuugNGDLjBQ\tWGdJV0\x0c\r7bNLs\x0cr deAWt35A4co\x0bPCuYmQ ExxtK\rvpckCyJxLrK5xULK\x0cvqtiGyovHQW8aDjV3rhXhR\nmQvmK\x0czLx\x0cECSYSF5jP35zN VkaRzQ lZ4 l06X4HHpsVn 8y8fGbIP\tRWFUAeFI24\rqN\x0cBW7u7WPMv36BmkgzQ\x0c2\x0cyLf\tYo8iRjE7zMsceym4ZnWg7EsOedh2cES\rz2n\x0cJi52uIPfSkAPzW\rEekjgWdb8y 285F4xae8\n8AiIkT4l3AOy\rT4yeXgaRMCI4t3PkHeFZ\rEb6R4FNCE \nbVil\x0c6qxSVPnU\nh\ttFMNE4\x0c\rwF\t\x0cW5vebbRWG\x0biVZLP\x0ct\x0c5gQ4CJ9KJl\x0cwyIfSIYaCvi4m1r\tJbYqmI0NVO36A\t8BSPNlaKbR73l9mxZxoqD4yca\n5h\r7a0z\tVm34aTy\tnLj5nSrh8er5lN0J7hcjmUk2DL\nyWEVNXTF8RWfC\x0bpcgBQXOQzidyYO\x0bh76UyUPAjELmNoECgGq06hiFGDI LiPZcofhcm\r62fEixIoyG\tmI\x0cYLQvBCbCluGgbm\x0c7GI6\n19il8PdPqss2uQqA5KgkHMIb hh211YuqV9kdmVnwyD63pz3p t58q6kHX\r\teYBrg6eDh\x0bx8\x0cI1SOV3Gt5qubmixHR\rApbgkTQJQ\tX0t11IP55hys2d\x0bF dh7j7G0Ac\x0bQMNvkSU9AV\x0b8mcIPHy9d\x0cyINf5qu\x0cdiBFrhiNRmCZ4r\tSx4N5VOm6KCp2T8bOVEjOR6otPAN5e\n\x0csyJ3giBjkgg 9dYQKq5P75AG5\x0bfD6zZO4DxQ44uX7Kz50dv4ncXQA\rqgHT\rLRcsRl\rW\t7We\tpAEJHMChxwVK\x0cprVvINvolf7hj\tUrob\rW3pXlqKIEQT8t7\x0bGODJanb328OiQCxE\rPfW4j\rl3p\x0cRXDB55u0MN7isBL\ty3UvE1 7I\nfuoZVPzk7az1\rMzA2FROXu0k\rFq pby6pHMqfTQT7iTw izlk0CUpyoUaq5w3UPFK7\rMOPw2cZ6FsVITbCoPhT\rIvuImCFGqmYpE hNevWkPCtwwnx2sX\x0c7oKzBExp32ZpdY\tstuDjSzfalsO1M\x0bNMUegnBDr3Liv3Lv\x0b\n37VZT2LEJ9fNYDi9r\t\x0bYC\rHSt0oJbk\x0b\x0bUdS8eB\nMXBPDEppZjHR7vGZYqX7yFm t1i682AXWf VPTzYTvm6mhOre8\x0bk0spJNYuI\tk\tC1B1N0 AYYDWH\t\tX1TjinXdkXcbFTlIiBLzx\rmUoyx9b7paJSVMX\tfLo8hU1Dmuluyk8R8\x0c4\x0cBe\nCrIMlyek4i\x0bFwuE9\nXUqpVxikH0PZspopUwPM9Kcue\rBh2Mf\rme3h4qelC\x0bEH\x0bkkxi6U\x0cE\x0ctqBgN93 V4ovmocLrK6\ngCQlf\x0cshRVvrPq\x0cOjgbjhSEK8PIx8OYqjjDDkJ0AgLhfbdGw2\nLMv2M0E08PGXnqUyVsjN\t C 4\n80 Fia g\x0b5dEFvyl5Y80U6sMAdHgk2nzC5ElDBhgcBprXC\x0bIMKXyt\x0ce5SkYcRartfblLqD1 A5\nre\x0bj67lJYCs\t8b50xA69eMHqGDLLP8sJceN19kkonjLj\t\rS\tk9sMOeewQHbT \x0cp53aMX9\x0bDYCZWAtdA6h\rAFHDEYFBE1MzdOxMO\x0cvDE7QfLb3jq4s\tI3aVTmDDOQAnuvWb2AGUUP\rf2HinUAiF13LKEfpqcD06S8aQC0Kyl729L7a6CbuoB0GRlJx tD yuTVqD62HuXpfKrDsbejEdp3\rxjc\x0bn4lLNaViizec\rWR\x0cTT5aZ\ny9\rO1qB1XGQPnES\nUhJtU Ll7t3Zglj1IAEx 8Rh3V\x0bfmUSC4\x0bVR9l33LS3bPAJpLbH3Q2\nv2fqMeIt3nGR\x0cgCixM4qzVSx7Yb192a1HWx8nnuWQIEK7QHL6p\x0cD3d0Y1FoZqsmY2U\rspvt3gwKOHR6RaZlmhX\n3bmIEF6\x0b\x0bMXJKOnXPgjkdhun4aGDBw\x0cOEW\repDYTcc48oZ4lg7PukNq7TU\tWP0ZJbzVKK\rxAMaZujwTqQXsXODiE2DdwnstAa6CMYfzj7J\x0c2Q\tY2764IYCy 3Fqm0\x0ckbe7VvfqWUh0\tUlubxZ\rX59MfNSfCfcH8GFZIGIRPt\rZVXfra1 H7VI2yJ\x0cspGDCi\rcgHfZa8528CP9tilUx0ifWPGqskLVDPLJP\nciNxodMQSrJXp\ro\r9aBFHCV\x0cR\rrp\x0bmMfxg5rG\tSuWonbJQlmHQ\ri34w8S\x0cN9Ezj2k2OmLH\x0cEcVUDjXNZIFCtlA843I44p GZyhlOctwpd7 OZnUxk4uacN\r8NihNGO\n9eXy5l6gQe5srySxxvuX5jtCzuJ35xvCfEXYa\x0b2lTDBOAaSYpnl v9L\x0cY8RLg2oE7xeCUbD\tSHKZgeXHZIzYAmA7bsmiZUfzmo5ZZUhtBh4F\x0bTx1\x0bz zQov5mYwfpWJTR2Q\x0bLRXMuBzj\x0bZC\x0b pFNPj8ixWJQggQlr9eNW6SHLJk731nc\x0cBn\x0ckQxg2BdRT\x0bp6lf7G\x0bnIMDeY8w6fUf\x0cjGE1Pfsekv7EYEIHsOAsZb3lBfBPO9\tXpHPBMRmRtzMc5WoX6C5cc\x0cBuTPtPOgXnap1Y3xq7pcMcgu55xblsXEAJKsojjR7aDB\tU84kUKRNEj\n8mcqEyOmvq1WA\na6bhzYf9VQv2aj9KLfByVqUKNFVIc4Mkha\x0c0aCPQSKe0GGwPlSfbtNXhdhxAb3RLf1J\x0cshJzjQe4DCmlRmjt\tlB0BwzBpkg2hTYM\r S\x0cux\x0bj6IcEZ\n\ngQ\rKKgg \rrv4sUMy5sfY1aatjK1MmUyXR\rRHk\x0cqq\x0cD1fy4C0\n\x0byd4SFKOyKJqx2mzI74vPxLLo\x0c0OamjXuUu\nWGkiA70nuf0PGRfwLEBPCMeyneJI1HcIXH\nCTFEIMiAq6fT\rmJgC hXEU\rriAhCm3OzgbcDgvQgDSyUw5jl\x0cTaLOPuFseq\x0cj2npTd57itktTdWBY7sqlOGKNSc\x0ctx2mUoHi31EF3l5lvYPDeG6bIPFwIn7\tG6G \x0bgNkSn89flvqcvI73RA";
+    let mutated = "VCiYFr0HKsEIK6r\r1hJLYnOr90Aji\rDWAjQA3LVAzrluN484327FuSPrRpm\n \tV\x0cx\nSCc5sX\nTB\x0c6Of7ns\t2HDwQCduKTqG8gG\x0beszazwljW01H60HMOLziOKwQwEYV7CbrLWQiLeCWKVxX\rvag\nAAEOhjER7gURuGXw\nMyY\t8mSw\x0b\x0bK0Z9G0Pt\x0bJZItAIqAq FxeaoOeLqWVFvxtDFfko0YVYt1I\rNmSXZ4lnOoiBCLbu6TLb80lClhY\tPN7Lp36F786I\nglwRK2oD45EtN SWW IF6uqKdf\x0czAcVycf\x0cBzHYnn1HAkU2Jluos0qwMGJ2m74z\nLd3\x0cIUVZmnRmHHWQGd1u2xmsZR\x0bfnml10ur6J\x0ba8xOZatiY 15Aq3KOGWdD3xQwqo\r5SKnnxH5tqU\rO\rZpJ\n7t7UUgfE\niWFgqWDpMeOG 1248M I\ro5B9Yed\r2aq2\tXxLn31s3hCV WEfQd60DKp6eFhUeUSeXDq6qjgTnWigoCZQERf\rXp7s2L37 iOEMl3\r41\nBShOjLfD8Kj0\rbu0ENreRjP\nY77jsrsaYgOsUrEzw\x0bw3OLi\n8fkddcaOvJeutTy B\rsDMkK\x0cnx2S0N\x0cDaY\x0c9iyo6p4IL\tOC1qgNlWP4VLg\tWmPG46ZMCirth5h4FwkS\nD2WsiEA2Z\n0xbLd7Uww hUQC6 3V\r1SsWem4UcQxG\rfuVvWl\nD9\nDpZQFFgiqhQiq1I0LMAR\r\rKBmj4iurrxaoMHTl9oj\x0b0N3AfD17gyqZiJ67bgizvecsRGeB1f\x0c\nYRvieJqIVHDKOOR\ruhqnVZz4BQ5FFBusz\x0cZl5\x0bt\tbdOUhAAAKyA6Jwl 7OjzojiRHGD6dl ncsgndsKURhFv4\tV5d\n73iPzbT\t8v6IrJtnq\nJuFl7A\x0b\rVnnsjTW0Y4QB1BgCy3B\x0cma7\tpPt5jmcJH7v5J\tYKEXh UqRChBFY5nbFbmXjJYxevPYJmSHC\rDQ4j9de\rTMZ\rtWaPAzkJjH\x0c\nyrEuf9WaMM\trFlKo9r9w\r\nQkQqIEu8Gfr\t aRzvN\r2oZhCyB4fa\np37\tXQi4Wa\no7gHUDQLoRvkK1dy2K3ydrI0O6\rFTGS7oHA\x0bajFOd\rcS5W25tFGhocwxM0\nuugNGDLjBQ\tWGdJV0\x0c\r7bNLs\x0cr deAWt35A4co\x0bPCuYmQ ExxtK\rvpckCyJxLrK5xULK\x0cvqtiGyovHQW8aDjV3rhXhR\nmQvmK\x0czLx\x0cECSYSF5jP35zN VkaRzQ lZ4 l06X4HHpsVn 8y8fGbIP\tRWFUAeFI24\rqN\x0cBW7u7WPMv36BmkgzQ\x0c2\x0cyLf\tYo8iRjE7zMsceym4ZnWg7EsOedh2cES\rz2n\x0cJi52uIPfSkAPzW\rEekjgWdb8y 285F4xae8\n8AiIkT4l3AOy\rT4yeXgaRMCI4t3PkHeFZ\rEb6R4FNCE \nbVil\x0c6qxSVPnU\nh\ttFMNE4\x0c\rwF\t\x0cW5vebbRWG\x0biVZLP\x0ct\x0c5gQ4CJ9KJl\x0cwyIfSIYaCvi4m1r\tJbYqmI0NVO36A\t8BSPNlaKbR73l9mxZxoqD4yca\n5h\r7a0z\tVm34aTy\tnLj5nSrh8er5lN0J7hcjmUk2DL\nyWEVNXTF8RWfC\x0bpcgBQXOQzidyYO\x0bh76UyUPAjELmNoECgGq06hiFGDI LiPZcofhcm\r62fEixIoyG\tmI\x0cYLQvBCbCluGgbm\x0c7GI6\n19il8PdPqss2uQqA5KgkHMIb hh211YuqV9kdmVnwyD63pz3p t58q6kHX\r\teYBrg6eDh\x0bx8\x0cI1SOV3Gt5qubmixHR\rApbgkTQJQ\tX0t11IP55hys2d\x0bF dh7j7G0Ac\x0bQMNvkSU9AV\x0b8mcIPHy9d\x0cyINf5qu\x0cdiBFrhiNRmCZ4r\tSx4N5VOm6KCp2T8bOVEjOR6otPAN5e\n\x0csyJ3giBjkgg 9dYQKq5P75AG5\x0bfD6zZO4DxQ44uX7Kz50dv4ncXQA\rqgHT\rLRcsRl\rW\t7We\tpAEJHMChxwVK\x0cprVvINvolf7hj\tUrob\rW3pXlqKIEQT8t7\x0bGODJanb328OiQCxE\rPfW4j\rl3p\x0cRXDB55u0MN7isBL\ty3UvE1 7I\nfuoZVPzk7az1\rMzA2FROXu0k\rFq pby6pHMqfTQT7iTw izlk0CUpyoUaq5w3UPFK7\rMOPw2cZ6FsVITbCoPhT\rIvuImCFGqmYpE hNevWkPCtwwnx2sX\x0c7oKzBExp32ZpdY\tstuDjSzfalsO1M\x0bNMUegnBDr3Liv3Lv\x0b\n37VZT2LEJ9fNYDi9r\t\x0bYC\rHSt0oJbk\x0b\x0bUdS8eB\nMXBPDEppZjHR7vGZYqX7yFm t1i682AXWf VPTzYTvm6mhOre8\x0bk0spJNYuI\tk\tC1B1N0 AYYDWH\t\tX1TjinXdkXcbFTlIiBLzx\rmUoyx9b7paJSVMX\tfLo8hU1Dmuluyk8R8\x0c4\x0cBe\nCrIMlyek4i\x0bFwuE9\nXUqpVxikH0PZspopUwPM9Kcue\rBh2Mf\rme3h4qelC\x0bEH\x0bkkxi6U\x0cE\x0ctqBgN93 V4ovmocLrK6\ngCQlf\x0cshRVvrPq\x0cOjgbjhSEK8PIx8OYqjjDDkJ0AgLhfbdGw2\nLMv2M0E08PGXnqUyVsjN\t C 4\n80 Fia g\x0b5dEFvyl5Y80U6sMAdHgk2nzC5ElDBhgcBprXC\x0bIMKXyt\x0ce5SkYcRartfblLqD1 A5\nre\x0bj67lJYCs\t8b50xA69eMHqGDLLP8sJceN19kkonjLj\t\rS\tk9sMOeewQHbT \x0cp53aMX9\x0bDYCZWAtdA6h\rAFHDEYFBE1MzdOxMO\x0cvDE7QfLb3jq4s\tI3aVTmDDOQAnuvWb2AGUUP\rf2HinUAiF13LKEfpqcD06S8aQC0Kyl729L7a6CbuoB0GRlJx tD yuTVqD62HuXpfKrDsbejEdp3\rxjc\x0bn4lLNaViizec\rWR\x0cTT5aZ\ny9\rO1qB1XGQPnES\nUhJtU Ll7t3Zglj1IAEx 8Rh3V\x0bfmUSC4\x0bVR9l33LS3bPAJpLbH3Q2\nv2fqMeIt3nGR\x0cgCixM4qzVSx7Yb192a1HWx8nnuWQIEK7QHL6p\x0cD3d0Y1FoZqsmY2U\rspvt3gwKOHR6RaZlmhX\n3bmIEF6\x0b\x0bMXJKOnXPgjkdhun4aGDBw\x0cOEW\repDYTcc48oZ4lg7PukNq7TU\tWP0ZJbzVKK\rxAMaZujwTqQXsXODiE2DdwnstAa6CMYfzj7J\x0c2Q\tY2764IYCy 3Fqm0\x0ckbe7VvfqWUh0\tUlubxZ\rX59MfNSfCfcH8GFZIGIRPt\rZVXfra1 H7VI2yJ\x0cspGDCi\rcgHfZa8528CP9tilUx0ifWPGqskLVDPLJP\nciNxodMQSrJXp\ro\r9aBFHCV\x0cR\rrp\x0bmMfxg5rG\tSuWonbJQlmHQ\ri34w8S\x0cN9Ezj2k2OmLH\x0cEcVUDjXNZIFCtlA843I44p GZyhlOctwpd7 OZnUxk4uacN\r8NihNGO\n9eXy5l6gQe5srySxxvuX5jtCzuJ35xvCfEXYa\x0b2lTDBOAaSYpnl v9L\x0cY8RLg2oE7xeCUbD\tSHKZgeXHZIzYAmA7bsmiZUfzmo5ZZUhtBh4F\x0bTx1\x0bz zQov5mYwfpWJTR2Q\x0bLRXMuBzj\x0bZC\x0b pFNPj8ixWJQggQlr9eNW6SHLJk731nc\x0cBn\x0ckQxg2BdRT\x0bp6lf7G\x0bnIMDeY8w6fUf\x0cjGE1Pfsekv7EYEIHsOAsZb3lBfBPO9\tXpHPBMRmRtzMc5WoX6C5cc\x0cBuTPtPOgXnap1Y3xq7pcMcgu55xblsXEAJKsojjR7aDB\tU84kUKRNEj\n8mcqEyOmvq1WA\na6bhzYf9VQv2aj9KLfByVqUKNFVIc4Mkha\x0c0aCPQSKe0GGwPlSfbtNXhdhxAb3RLf1J\x0cshJzjQe4DCmlRmjt\tlB0BwzBpkg2hTYM\r S\x0cux\x0bj6IcEZ\n\ngQ\rKKgg \rrv4sUMy5sfY1aatjK1MmUyXR\rRHk\x0cqq\x0cD1fy4C0\n\x0byd4SFKOyKJqx2mzI74vPxLLo\x0c0OamjXuUu\nWGkiA70nuf0PGRfwLEBPCMeyneJI1HcIXH\nCTFEIMiAq6fT\rmJgC hXEU\rriAhCm3OzgbcDgvQgDSyUw5jl\x0cTaimauFseq\x0cj2npTd57itktTdWBY7sqlOGKNSc\x0ctx2mUoHi31EF3l5lvYPDeG6bIPFwIn7\tG6G \x0bgNkSn89flvqcvI73RA";
+
+    let canary = srv.mock(|when, then| {
+        when.method(GET).path("/canary");
+        then.status(200).body(content);
+    });
+
+    // not similar, should see results in output
+    let not_similar = srv.mock(|when, then| {
+        when.method(GET).path("/not-similar");
+        then.status(302).body("this is a test");
+    });
+
+    // similar, should not see results
+    let similar = srv.mock(|when, then| {
+        when.method(GET).path("/similar");
+        then.status(200).body(mutated);
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--filter-similar-to")
+        .arg(srv.url("/canary"))
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICfdafdsafdsafadsENSE")
+            .and(predicate::str::contains("302"))
+            .and(predicate::str::contains("14c"))
+            .and(predicate::str::contains("/similar"))
+            .not()
+            .and(predicate::str::contains("4100c"))
+            .not(),
+    );
+
+    assert_eq!(canary.hits(), 1);
+    assert_eq!(similar.hits(), 1);
+    assert_eq!(not_similar.hits(), 1);
+    teardown_tmp_directory(tmp_dir);
+}

tests/test_heuristics.rs

@@ -2,7 +2,7 @@ mod utils;
 use assert_cmd::prelude::*;
 use assert_cmd::Command;
 use httpmock::Method::GET;
-use httpmock::{Mock, MockServer, Regex};
+use httpmock::{MockServer, Regex};
 use predicates::prelude::*;
 use utils::{setup_tmp_directory, teardown_tmp_directory};
 
@@ -10,19 +10,19 @@ use utils::{setup_tmp_directory, teardown_tmp_directory};
 /// test passes one bad target via -u to the scanner, expected result is that the
 /// scanner dies
 fn test_single_target_cannot_connect() -> Result<(), Box<dyn std::error::Error>> {
-    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()])?;
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
-    let cmd = std::panic::catch_unwind(|| {
-        Command::cargo_bin("feroxbuster")
-            .unwrap()
-            .arg("--url")
-            .arg("http://fjdksafjkdsajfkdsajkfdsajkfsdjkdsfdsafdsafdsajkr3l2ajfdskafdsjk")
-            .arg("--wordlist")
-            .arg(file.as_os_str())
-            .unwrap()
-    });
-
-    assert!(cmd.is_err());
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://fjdksafjkdsajfkdsajkfdsajkfsdjkdsfdsafdsafdsajkr3l2ajfdskafdsjk")
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .assert()
+        .success()
+        .stdout(
+            predicate::str::contains("Could not connect to http://fjdksafjkdsajfkdsajkfdsajkfsdjkdsfdsafdsafdsajkr3l2ajfdskafdsjk, skipping...", )
+        );
 
     teardown_tmp_directory(tmp_dir);
     Ok(())
@@ -35,20 +35,20 @@ fn test_two_targets_cannot_connect() -> Result<(), Box<dyn std::error::Error>> {
     let not_real =
         String::from("http://fjdksafjkdsajfkdsajkfdsajkfsdjkdsfdsafdsafdsajkr3l2ajfdskafdsjk");
     let urls = vec![not_real.clone(), not_real];
-    let (tmp_dir, file) = setup_tmp_directory(&urls)?;
+    let (tmp_dir, file) = setup_tmp_directory(&urls, "wordlist")?;
 
-    let cmd = std::panic::catch_unwind(|| {
-        Command::cargo_bin("feroxbuster")
-            .unwrap()
-            .arg("--stdin")
-            .arg("--wordlist")
-            .arg(file.as_os_str())
-            .pipe_stdin(file)
-            .unwrap()
-            .unwrap()
-    });
-
-    assert!(cmd.is_err());
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--stdin")
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .pipe_stdin(file)
+        .unwrap()
+        .assert()
+        .success()
+        .stdout(
+            predicate::str::contains("Could not connect to http://fjdksafjkdsajfkdsajkfdsajkfsdjkdsfdsafdsafdsajkr3l2ajfdskafdsjk, skipping...", )
+        );
 
     teardown_tmp_directory(tmp_dir);
     Ok(())
@@ -63,14 +63,12 @@ fn test_one_good_and_one_bad_target_scan_succeeds() -> Result<(), Box<dyn std::e
     let not_real =
         String::from("http://fjdksafjkdsajfkdsajkfdsajkfsdjkdsfdsafdsafdsajkr3l2ajfdskafdsjk");
     let urls = vec![not_real, srv.url("/"), String::from("LICENSE")];
-    let (tmp_dir, file) = setup_tmp_directory(&urls)?;
+    let (tmp_dir, file) = setup_tmp_directory(&urls, "wordlist")?;
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/LICENSE")
-        .return_status(200)
-        .return_body("this is a test")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
 
     let mut cmd = Command::cargo_bin("feroxbuster").unwrap();
 
@@ -86,7 +84,51 @@ fn test_one_good_and_one_bad_target_scan_succeeds() -> Result<(), Box<dyn std::e
                 .and(predicate::str::contains("200"))
                 .and(predicate::str::contains("14")),
         );
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
+
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// test pipes two good targets to the scanner, expected result is that both targets
+/// are scanned successfully and no error is reported (result of issue #169)
+fn test_two_good_targets_scan_succeeds() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let srv2 = MockServer::start();
+
+    let urls = vec![srv.url("/"), srv2.url("/"), String::from("LICENSE")];
+    let (tmp_dir, file) = setup_tmp_directory(&urls, "wordlist")?;
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
+
+    let mock2 = srv2.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(403).body("this also is a test");
+    });
+
+    let mut cmd = Command::cargo_bin("feroxbuster").unwrap();
+
+    cmd.arg("--stdin")
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .pipe_stdin(file)
+        .unwrap()
+        .assert()
+        .success()
+        .stdout(
+            predicate::str::contains("/LICENSE")
+                .and(predicate::str::contains("200"))
+                .and(predicate::str::contains("403"))
+                .and(predicate::str::contains("14c"))
+                .and(predicate::str::contains("19c")),
+        );
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock2.hits(), 1);
 
     teardown_tmp_directory(tmp_dir);
     Ok(())
@@ -96,14 +138,13 @@ fn test_one_good_and_one_bad_target_scan_succeeds() -> Result<(), Box<dyn std::e
 /// test finds a static wildcard and reports as much to stdout
 fn test_static_wildcard_request_found() -> Result<(), Box<dyn std::error::Error>> {
     let srv = MockServer::start();
-    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()])?;
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap())
-        .return_status(200)
-        .return_body("this is a test")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap());
+        then.status(200).body("this is a test");
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -111,7 +152,7 @@ fn test_static_wildcard_request_found() -> Result<(), Box<dyn std::error::Error>
         .arg(srv.url("/"))
         .arg("--wordlist")
         .arg(file.as_os_str())
-        .arg("--addslash")
+        .arg("--add-slash")
         .unwrap();
 
     teardown_tmp_directory(tmp_dir);
@@ -123,29 +164,28 @@ fn test_static_wildcard_request_found() -> Result<(), Box<dyn std::error::Error>
             .and(predicate::str::contains("(url length: 32)")),
     );
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
     Ok(())
 }
 
 #[test]
-/// test finds a dynamic wildcard and reports as much to stdout
-fn test_dynamic_wildcard_request_found() -> Result<(), Box<dyn std::error::Error>> {
+/// test finds a dynamic wildcard and reports as much to stdout and a file
+fn test_dynamic_wildcard_request_found() {
     let srv = MockServer::start();
-    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()])?;
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
+    let outfile = tmp_dir.path().join("outfile");
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap())
-        .return_status(200)
-        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap());
+        then.status(200)
+            .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+    });
 
-    let mock2 = Mock::new()
-        .expect_method(GET)
-        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap())
-        .return_status(200)
-        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
-        .create_on(&srv);
+    let mock2 = srv.mock(|when, then| {
+        when.method(GET).path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap());
+        then.status(200).body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -153,7 +193,87 @@ fn test_dynamic_wildcard_request_found() -> Result<(), Box<dyn std::error::Error
         .arg(srv.url("/"))
         .arg("--wordlist")
         .arg(file.as_os_str())
-        .arg("--addslash")
+        .arg("--add-slash")
+        .arg("--output")
+        .arg(outfile.as_os_str())
+        .unwrap();
+
+    let contents = std::fs::read_to_string(outfile).unwrap();
+
+    teardown_tmp_directory(tmp_dir);
+
+    assert_eq!(contents.contains("WLD"), true);
+    assert_eq!(contents.contains("Got"), true);
+    assert_eq!(contents.contains("200"), true);
+    assert_eq!(contents.contains("(url length: 32)"), true);
+    assert_eq!(contents.contains("(url length: 96)"), true);
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("WLD")
+            .and(predicate::str::contains("Got"))
+            .and(predicate::str::contains("200"))
+            .and(predicate::str::contains("(url length: 32)"))
+            .and(predicate::str::contains("(url length: 96)")),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock2.hits(), 1);
+}
+
+#[test]
+/// uses dont_filter, so the normal wildcard test should never happen
+fn heuristics_static_wildcard_request_with_dont_filter() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap());
+        then.status(200).body("this is a test");
+    });
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--dont-filter")
+        .unwrap();
+
+    teardown_tmp_directory(tmp_dir);
+
+    assert_eq!(mock.hits(), 0);
+    Ok(())
+}
+
+#[test]
+/// test finds a static wildcard and reports as much to stdout
+fn heuristics_wildcard_test_with_two_static_wildcards() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap());
+        then.status(200)
+            .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+    });
+
+    let mock2 = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap());
+        then.status(200)
+            .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--add-slash")
         .unwrap();
 
     teardown_tmp_directory(tmp_dir);
@@ -164,14 +284,168 @@ fn test_dynamic_wildcard_request_found() -> Result<(), Box<dyn std::error::Error
             .and(predicate::str::contains("200"))
             .and(predicate::str::contains("(url length: 32)"))
             .and(predicate::str::contains("(url length: 96)"))
-            .and(predicate::str::contains("Wildcard response is dynamic;"))
-            .and(predicate::str::contains("auto-filtering"))
             .and(predicate::str::contains(
-                "(14 + url length) responses; toggle this behavior by using",
+                "Wildcard response is static; auto-filtering 46",
             )),
     );
 
-    assert_eq!(mock.times_called(), 1);
-    assert_eq!(mock2.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock2.hits(), 1);
+}
+
+#[test]
+/// test finds a static wildcard and reports nothing to stdout
+fn heuristics_wildcard_test_with_two_static_wildcards_with_quiet_enabled(
+) -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap());
+        then.status(200)
+            .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+    });
+
+    let mock2 = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap());
+        then.status(200)
+            .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--add-slash")
+        .arg("-q")
+        .unwrap();
+
+    teardown_tmp_directory(tmp_dir);
+
+    cmd.assert().success().stdout(predicate::str::is_empty());
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock2.hits(), 1);
+    Ok(())
+}
+
+#[test]
+/// test finds a static wildcard and reports as much to stdout and a file
+fn heuristics_wildcard_test_with_two_static_wildcards_and_output_to_file() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
+    let outfile = tmp_dir.path().join("outfile");
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap());
+        then.status(200)
+            .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+    });
+
+    let mock2 = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap());
+        then.status(200)
+            .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--add-slash")
+        .arg("--output")
+        .arg(outfile.as_os_str())
+        .unwrap();
+
+    let contents = std::fs::read_to_string(outfile).unwrap();
+
+    teardown_tmp_directory(tmp_dir);
+
+    assert_eq!(contents.contains("WLD"), true);
+    assert_eq!(contents.contains("Got"), true);
+    assert_eq!(contents.contains("200"), true);
+    assert_eq!(contents.contains("(url length: 32)"), true);
+    assert_eq!(contents.contains("(url length: 96)"), true);
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("WLD")
+            .and(predicate::str::contains("Got"))
+            .and(predicate::str::contains("200"))
+            .and(predicate::str::contains("(url length: 32)"))
+            .and(predicate::str::contains("(url length: 96)"))
+            .and(predicate::str::contains(
+                "Wildcard response is static; auto-filtering 46",
+            )),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock2.hits(), 1);
+}
+
+#[test]
+/// test finds a static wildcard that returns 3xx, expect redirects to => in response as well as
+/// in the output file
+fn heuristics_wildcard_test_with_redirect_as_response_code(
+) -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+    let outfile = tmp_dir.path().join("outfile");
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap());
+        then.status(301)
+            .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+    });
+
+    let mock2 = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap());
+        then.status(301)
+            .header("Location", &srv.url("/some-redirect"))
+            .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--add-slash")
+        .arg("--output")
+        .arg(outfile.as_os_str())
+        .unwrap();
+
+    let contents = std::fs::read_to_string(outfile).unwrap();
+
+    teardown_tmp_directory(tmp_dir);
+
+    assert_eq!(contents.contains("WLD"), true);
+    assert_eq!(contents.contains("301"), true);
+    assert_eq!(contents.contains("/some-redirect"), true);
+    assert_eq!(contents.contains("redirects to => "), true);
+    assert_eq!(contents.contains(&srv.url("/")), true);
+    assert_eq!(contents.contains("(url length: 32)"), true);
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("redirects to => ")
+            .and(predicate::str::contains("/some-redirect"))
+            .and(predicate::str::contains("301"))
+            .and(predicate::str::contains(srv.url("/")))
+            .and(predicate::str::contains("(url length: 32)"))
+            .and(predicate::str::contains("WLD")),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock2.hits(), 1);
     Ok(())
 }

tests/test_main.rs

@@ -0,0 +1,88 @@
+mod utils;
+use assert_cmd::Command;
+use httpmock::Method::GET;
+use httpmock::MockServer;
+use predicates::prelude::*;
+use utils::{setup_tmp_directory, teardown_tmp_directory};
+
+#[test]
+/// send the function a file to which we dont have permission in order to execute error branch
+fn main_use_root_owned_file_as_wordlist() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/");
+        then.status(200).body("this is a test");
+    });
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg("/etc/shadow")
+        .arg("-vvvv")
+        .assert()
+        .failure()
+        .stdout(predicate::str::contains("Permission denied (os error 13)"));
+
+    // connectivity test hits it once
+    assert_eq!(mock.hits(), 1);
+    Ok(())
+}
+
+#[test]
+/// send the function an empty file
+fn main_use_empty_wordlist() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&[], "wordlist")?;
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/");
+        then.status(200).body("this is a test");
+    });
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-vvvv")
+        .assert()
+        .failure()
+        .stdout(predicate::str::contains("Did not find any words in"));
+
+    assert_eq!(mock.hits(), 1);
+
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// send nothing over stdin, expect heuristics to be upset during connectivity test
+fn main_use_empty_stdin_targets() -> Result<(), Box<dyn std::error::Error>> {
+    let (tmp_dir, file) = setup_tmp_directory(&[], "wordlist")?;
+
+    // get_targets is called before scan, so the empty wordlist shouldn't trigger
+    // the 'Did not find any words' error
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--stdin")
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-vvv")
+        .pipe_stdin(file)
+        .unwrap()
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("Could not connect to any target provided")
+                .and(predicate::str::contains("Target Url"))
+                .not(), // no target url found
+        );
+
+    teardown_tmp_directory(tmp_dir);
+
+    Ok(())
+}

tests/test_scan_manager.rs

@@ -0,0 +1,130 @@
+mod utils;
+use assert_cmd::Command;
+use httpmock::Method::GET;
+use httpmock::MockServer;
+use predicates::prelude::*;
+use std::fs::{read_to_string, write};
+use std::path::Path;
+use std::time;
+use utils::{setup_tmp_directory, teardown_tmp_directory};
+
+#[test]
+/// pass a known serialized scan with 1 scan complete and 1 not. expect the incomplete scan to
+/// start and the complete to not start. expect the responses, scans, and configuration structures
+/// to be populated based off the contents of the given state file
+fn resume_scan_works() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) =
+        setup_tmp_directory(&["css".to_string(), "stuff".to_string()], "wordlist").unwrap();
+
+    // localhost:PORT/ <- complete
+    // localhost:PORT/js <- will get scanned with /css and /stuff
+    let complete_scan = format!(
+        r#"{{"id":"057016a14769414aac9a7a62707598cb","url":"{}","scan_type":"Directory","status":"Complete"}}"#,
+        srv.url("/")
+    );
+    let incomplete_scan = format!(
+        r#"{{"id":"400b2323a16f43468a04ffcbbeba34c6","url":"{}","scan_type":"Directory","status":"NotStarted"}}"#,
+        srv.url("/js")
+    );
+    let scans = format!(r#""scans":[{},{}]"#, complete_scan, incomplete_scan);
+
+    let config = format!(
+        r#""config": {{"type":"configuration","wordlist":"{}","config":"","proxy":"","replay_proxy":"","target_url":"{}","status_codes":[200,204,301,302,307,308,401,403,405],"replay_codes":[200,204,301,302,307,308,401,403,405],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"quiet":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/1.9.0","redirects":false,"insecure":false,"extensions":[],"headers":{{}},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":false,"depth":2,"scan_limit":1,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false}}"#,
+        file.to_string_lossy(),
+        srv.url("/")
+    );
+
+    // // localhost:PORT/js/css has already been seen, expect not to be scanned
+    let response = format!(
+        r#"{{"type":"response","url":"{}","path":"/js/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{{"server":"nginx/1.16.1"}}}}"#,
+        srv.url("/js/css")
+    );
+    let responses = format!(r#""responses":[{}]"#, response);
+
+    // not scanned because /js is not complete, and /js/stuff response is not known
+    let not_scanned_yet = srv.mock(|when, then| {
+        when.method(GET).path("/js/stuff");
+        then.status(200).body("i expect to be scanned");
+    });
+
+    // will get scanned because /js is not complete, but because response of /js/css is known, the
+    // response will not be in stdout
+    let already_scanned = srv.mock(|when, then| {
+        when.method(GET).path("/js/css");
+        then.status(200);
+    });
+
+    // already scanned because scan on / is complete
+    let also_already_scanned = srv.mock(|when, then| {
+        when.method(GET).path("/css");
+        then.status(200).body("two words");
+    });
+
+    let state_file_contents = format!("{{{},{},{}}}", scans, config, responses);
+    let (tmp_dir2, state_file) = setup_tmp_directory(&[state_file_contents], "state-file").unwrap();
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--resume-from")
+        .arg(state_file.as_os_str())
+        .assert()
+        .success()
+        .stdout(
+            predicate::str::contains("/js/stuff")
+                .and(predicate::str::contains("22c"))
+                .and(predicate::str::contains("5w"))
+                .and(predicate::str::contains("/js/css"))
+                .not()
+                .and(predicate::str::contains("2w"))
+                .not()
+                .and(predicate::str::contains("9c"))
+                .not(),
+        );
+
+    teardown_tmp_directory(tmp_dir);
+    teardown_tmp_directory(tmp_dir2);
+
+    assert_eq!(already_scanned.hits(), 1);
+    assert_eq!(also_already_scanned.hits(), 0);
+    assert_eq!(not_scanned_yet.hits(), 1);
+}
+
+#[test]
+/// kick off scan with a time limit;  
+fn time_limit_enforced_when_specified() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) =
+        setup_tmp_directory(&["css".to_string(), "stuff".to_string()], "wordlist").unwrap();
+
+    // ensure the command will run long enough by adding crap to the wordlist
+    let more_words = read_to_string(Path::new("tests/extra-words")).unwrap();
+    write(&file, more_words).unwrap();
+
+    assert!(file.metadata().unwrap().len() > 100); // sanity check on wordlist size
+
+    let now = time::Instant::now();
+    let lower_bound = time::Duration::new(5, 0);
+    let upper_bound = time::Duration::new(6, 0);
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--time-limit")
+        .arg("5s")
+        .assert()
+        .failure();
+
+    // expected run time is somewhere in the 30 seconds ballpark (real    0m37.376s)
+    // so if the cmd returns in a significantly shorter amount of time, the test will have
+    // succeeded
+
+    // --time-limit is 5 seconds, so elapsed should be in a window that is greater than 5
+    // but significantly less than 30ish
+    assert!(now.elapsed() > lower_bound && now.elapsed() < upper_bound);
+
+    teardown_tmp_directory(tmp_dir);
+}

tests/test_scanner.rs

@@ -1,22 +1,574 @@
 mod utils;
 use assert_cmd::prelude::*;
 use httpmock::Method::GET;
-use httpmock::{Mock, MockServer};
+use httpmock::MockServer;
 use predicates::prelude::*;
 use std::process::Command;
 use utils::{setup_tmp_directory, teardown_tmp_directory};
 
 #[test]
-fn test_single_request_scan() -> Result<(), Box<dyn std::error::Error>> {
+/// send a single valid request, expect a 200 response
+fn scanner_single_request_scan() -> Result<(), Box<dyn std::error::Error>> {
     let srv = MockServer::start();
-    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()])?;
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/LICENSE")
-        .return_status(200)
-        .return_body("this is a test")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-vvvv")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .and(predicate::str::contains("200"))
+            .and(predicate::str::contains("14")),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// send a valid request, follow redirects into new directories, expect 301/200 responses
+fn scanner_recursive_request_scan() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let urls = [
+        "js".to_string(),
+        "prod".to_string(),
+        "dev".to_string(),
+        "file.js".to_string(),
+    ];
+    let (tmp_dir, file) = setup_tmp_directory(&urls, "wordlist")?;
+
+    let js_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js");
+        then.status(301).header("Location", &srv.url("/js/"));
+    });
+
+    let js_prod_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js/prod");
+        then.status(301).header("Location", &srv.url("/js/prod/"));
+    });
+
+    let js_dev_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js/dev");
+        then.status(301).header("Location", &srv.url("/js/dev/"));
+    });
+
+    let js_dev_file_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js/dev/file.js");
+        then.status(200)
+            .body("this is a test and is more bytes than other ones");
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-vvvv")
+        .arg("-t")
+        .arg("1")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::is_match("301.*js")
+            .unwrap()
+            .and(predicate::str::is_match("301.*js/prod").unwrap())
+            .and(predicate::str::is_match("301.*js/dev").unwrap())
+            .and(predicate::str::is_match("200.*js/dev/file.js").unwrap()),
+    );
+
+    assert_eq!(js_mock.hits(), 1);
+    assert_eq!(js_prod_mock.hits(), 1);
+    assert_eq!(js_dev_mock.hits(), 1);
+    assert_eq!(js_dev_file_mock.hits(), 1);
+
+    teardown_tmp_directory(tmp_dir);
+
+    Ok(())
+}
+
+#[test]
+/// send a valid request, follow 200s into new directories, expect 200 responses
+fn scanner_recursive_request_scan_using_only_success_responses(
+) -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let urls = [
+        "js/".to_string(),
+        "prod/".to_string(),
+        "dev/".to_string(),
+        "file.js".to_string(),
+    ];
+    let (tmp_dir, file) = setup_tmp_directory(&urls, "wordlist")?;
+
+    let js_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js/");
+        then.status(200).header("Location", &srv.url("/js/"));
+    });
+
+    let js_prod_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js/prod/");
+        then.status(200).header("Location", &srv.url("/js/prod/"));
+    });
+
+    let js_dev_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js/dev/");
+        then.status(200).header("Location", &srv.url("/js/dev/"));
+    });
+
+    let js_dev_file_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js/dev/file.js");
+        then.status(200)
+            .body("this is a test and is more bytes than other ones");
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-vvvv")
+        .arg("-t")
+        .arg("1")
+        .arg("--redirects")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::is_match("200.*js")
+            .unwrap()
+            .and(predicate::str::is_match("200.*js/prod").unwrap())
+            .and(predicate::str::is_match("200.*js/dev").unwrap())
+            .and(predicate::str::is_match("200.*js/dev/file.js").unwrap()),
+    );
+
+    assert_eq!(js_mock.hits(), 1);
+    assert_eq!(js_prod_mock.hits(), 1);
+    assert_eq!(js_dev_mock.hits(), 1);
+    assert_eq!(js_dev_file_mock.hits(), 1);
+
+    teardown_tmp_directory(tmp_dir);
+
+    Ok(())
+}
+
+#[test]
+/// send a single valid request, get a response, and write it to disk
+fn scanner_single_request_scan_with_file_output() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
+
+    let outfile = tmp_dir.path().join("output");
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-vvvv")
+        .arg("-o")
+        .arg(outfile.as_os_str())
+        .unwrap();
+
+    let contents = std::fs::read_to_string(outfile)?;
+
+    assert!(contents.contains("/LICENSE"));
+    assert!(contents.contains("200"));
+    assert!(contents.contains("14"));
+
+    assert_eq!(mock.hits(), 1);
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// send a single valid request with -q, get a response, and write only the url to disk
+fn scanner_single_request_scan_with_file_output_and_tack_q(
+) -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
+
+    let outfile = tmp_dir.path().join("output");
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-vvvv")
+        .arg("-q")
+        .arg("-o")
+        .arg(outfile.as_os_str())
+        .unwrap();
+
+    let contents = std::fs::read_to_string(outfile)?;
+
+    let url = srv.url("/LICENSE");
+    assert!(contents.contains(&url));
+
+    assert_eq!(mock.hits(), 1);
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// send an invalid output file, expect nothing to be written to disk
+fn scanner_single_request_scan_with_invalid_file_output() -> Result<(), Box<dyn std::error::Error>>
+{
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
+
+    let outfile = tmp_dir.path(); // outfile is a directory
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-vvvv")
+        .arg("-q")
+        .arg("-o")
+        .arg(outfile.as_os_str())
+        .unwrap();
+
+    let contents = std::fs::read_to_string(outfile);
+    assert!(contents.is_err());
+
+    assert_eq!(mock.hits(), 1);
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// send a single valid request using -q, expect only the url on stdout
+fn scanner_single_request_quiet_scan() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-x")
+        .arg("js,html")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains(srv.url("/LICENSE"))
+            .and(predicate::str::contains("200"))
+            .not()
+            .and(predicate::str::contains("14"))
+            .not(),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// send single valid request, get back a 301 without a Location header
+/// expect response_is_directory to return false when called
+fn scanner_single_request_returns_301_without_location_header(
+) -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(301).body("this is a test");
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--timeout")
+        .arg("5")
+        .arg("--user-agent")
+        .arg("some-user-agent-string")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains(srv.url("/LICENSE"))
+            .and(predicate::str::contains("301"))
+            .and(predicate::str::contains("14")),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// send a single valid request, expect a 200 response that then gets routed to the replay
+/// proxy
+fn scanner_single_request_replayed_to_proxy() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let proxy = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
+
+    let mock_two = proxy.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--replay-proxy")
+        .arg(format!("http://{}", proxy.address().to_string()))
+        .arg("--replay-codes")
+        .arg("200")
+        .unwrap();
+
+    cmd.assert()
+        .success()
+        .stdout(
+            predicate::str::contains("/LICENSE")
+                .and(predicate::str::contains("200"))
+                .and(predicate::str::contains("14c")),
+        )
+        .stderr(predicate::str::contains("Replay Proxy Codes"));
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_two.hits(), 1);
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// send a single valid request, filter the size of the response, expect one out of 2 urls
+fn scanner_single_request_scan_with_filtered_result() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) =
+        setup_tmp_directory(&["LICENSE".to_string(), "ignored".to_string()], "wordlist")?;
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a not a test");
+    });
+
+    let filtered_mock = srv.mock(|when, then| {
+        when.method(GET).path("/ignored");
+        then.status(200).body("this is a test");
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-n")
+        .arg("-S")
+        .arg("14")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .and(predicate::str::contains("200"))
+            .and(predicate::str::contains("20"))
+            .and(predicate::str::contains("ignored"))
+            .not()
+            .and(predicate::str::contains(" 14 "))
+            .not(),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(filtered_mock.hits(), 1);
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// send a single valid request, get a response, and write the logging messages to disk
+fn scanner_single_request_scan_with_debug_logging() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
+
+    let outfile = tmp_dir.path().join("debug.log");
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-vvvv")
+        .arg("--debug-log")
+        .arg(outfile.as_os_str())
+        .unwrap();
+
+    let contents = std::fs::read_to_string(outfile).unwrap();
+    println!("{}", contents);
+    assert!(contents.starts_with("Configuration {"));
+    assert!(contents.contains("TRC"));
+    assert!(contents.contains("DBG"));
+    assert!(contents.contains("INF"));
+    assert!(contents.contains("feroxbuster All scans complete!"));
+    assert!(contents.contains("feroxbuster exit: terminal_input_handler"));
+
+    assert_eq!(mock.hits(), 1);
+    teardown_tmp_directory(tmp_dir);
+}
+
+#[test]
+/// send a single valid request, get a response, and write the logging messages to disk as NDJSON
+fn scanner_single_request_scan_with_debug_logging_as_json() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
+
+    let outfile = tmp_dir.path().join("debug.log");
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-vvvv")
+        .arg("--debug-log")
+        .arg(outfile.as_os_str())
+        .arg("--json")
+        .unwrap();
+
+    let contents = std::fs::read_to_string(outfile).unwrap();
+    println!("{}", contents);
+    assert!(contents.starts_with("{\"type\":\"configuration\""));
+    assert!(contents.contains("\"level\":\"TRACE\""));
+    assert!(contents.contains("\"level\":\"DEBUG\""));
+    assert!(contents.contains("\"level\":\"INFO\""));
+    assert!(contents.contains("time_offset"));
+    assert!(contents.contains("\"module\":\"feroxbuster::scanner\""));
+    assert!(contents.contains("All scans complete!"));
+    assert!(contents.contains("exit: terminal_input_handler"));
+
+    assert_eq!(mock.hits(), 1);
+    teardown_tmp_directory(tmp_dir);
+}
+
+#[test]
+/// send a single valid request, filter the response by regex, expect one out of 2 urls
+fn scanner_single_request_scan_with_regex_filtered_result() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) =
+        setup_tmp_directory(&["LICENSE".to_string(), "ignored".to_string()], "wordlist").unwrap();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
+
+    let filtered_mock = srv.mock(|when, then| {
+        when.method(GET).path("/ignored");
+        then.status(200)
+            .body("this is a test\nThat rug really tied the room together");
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--filter-regex")
+        .arg("'That rug.*together$'")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .and(predicate::str::contains("200"))
+            .and(predicate::str::contains("20"))
+            .and(predicate::str::contains("ignored"))
+            .not()
+            .and(predicate::str::contains(" 14 "))
+            .not(),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(filtered_mock.hits(), 1);
+    teardown_tmp_directory(tmp_dir);
+}
+
+#[test]
+/// send a request to a 403 directory, expect recursion to work into the 403
+fn scanner_recursion_works_with_403_directories() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) =
+        setup_tmp_directory(&["LICENSE".to_string(), "ignored/".to_string()], "wordlist").unwrap();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
+
+    let forbidden_dir = srv.mock(|when, then| {
+        when.method(GET).path("/ignored/");
+        then.status(403);
+    });
+
+    let found_anyway = srv.mock(|when, then| {
+        when.method(GET).path("/ignored/LICENSE");
+        then.status(200)
+            .body("this is a test\nThat rug really tied the room together");
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -28,11 +580,19 @@ fn test_single_request_scan() -> Result<(), Box<dyn std::error::Error>> {
 
     cmd.assert().success().stdout(
         predicate::str::contains("/LICENSE")
-            .and(predicate::str::contains("200"))
-            .and(predicate::str::contains("14")),
+            .count(2)
+            .and(predicate::str::contains("200").count(2))
+            .and(predicate::str::contains("403"))
+            .and(predicate::str::contains("53c"))
+            .and(predicate::str::contains("14c"))
+            .and(predicate::str::contains("0c"))
+            .and(predicate::str::contains("ignored").count(2))
+            .and(predicate::str::contains("/ignored/LICENSE")),
     );
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(found_anyway.hits(), 1);
+    assert_eq!(forbidden_dir.hits(), 1);
+
     teardown_tmp_directory(tmp_dir);
-    Ok(())
 }

tests/utils/mod.rs

@@ -3,12 +3,13 @@ use std::path::PathBuf;
 use tempfile::TempDir;
 
 /// integration test helper: creates a temp directory, and writes `words` to
-/// a file named `wordlist` in the temp directory
+/// a file named `filename` in the temp directory
 pub fn setup_tmp_directory(
     words: &[String],
+    filename: &str,
 ) -> Result<(TempDir, PathBuf), Box<dyn std::error::Error>> {
     let tmp_dir = TempDir::new()?;
-    let file = tmp_dir.path().join("wordlist");
+    let file = tmp_dir.path().join(&filename);
     write(&file, words.join("\n"))?;
     Ok((tmp_dir, file))
 }