Merge pull request #192 from epi052/190-fix-double-fslash

fixed url parsing issue when word starts with 2 or more /

.github/dependabot.yml

@@ -0,0 +1,7 @@
+version: 2
+updates:
+- package-ecosystem: cargo
+  directory: "/"
+  schedule:
+    interval: daily
+  open-pull-requests-limit: 10

.github/pull_request_template.md

@@ -4,14 +4,14 @@ Long form explanations of most of the items below can be found in the [CONTRIBUT
 
 ## Branching checklist
 - [ ] There is an issue associated with your PR (bug, feature, etc.. if not, create one)
-- [ ] Your PR description references the associated issue (i.e. fixes #123)
+- [ ] Your PR description references the associated issue (i.e. fixes #123456)
 - [ ] Code is in its own branch
 - [ ] Branch name is related to the PR contents
 - [ ] PR targets master
 
 ## Static analysis checks
 - [ ] All rust files are formatted using `cargo fmt`
-- [ ] All `clippy` checks pass when running `cargo clippy --all-targets --all-features -- -D warnings -A clippy::unnecessary_unwrap`
+- [ ] All `clippy` checks pass when running `cargo clippy --all-targets --all-features -- -D warnings -A clippy::deref_addrof`
 - [ ] All existing tests pass
 
 ## Documentation

.github/stale.yml

@@ -0,0 +1,17 @@
+# Number of days of inactivity before an issue becomes stale
+daysUntilStale: 14
+# Number of days of inactivity before a stale issue is closed
+daysUntilClose: 7
+# Issues with these labels will never be considered stale
+exemptLabels:
+  - pinned
+  - security
+# Label to use when marking an issue as stale
+staleLabel: stale
+# Comment to post when marking an issue as stale. Set to `false` to disable
+markComment: >
+  This issue has been automatically marked as stale because it has not had
+  recent activity. It will be closed if no further activity occurs. Thank you
+  for your contributions.
+# Comment to post when closing a stale issue. Set to `false` to disable
+closeComment: false

.github/workflows/check.yml

@@ -1,6 +1,6 @@
 name: CI Pipeline
 
-on: [push]
+on: [push, pull_request]
 
 jobs:
   check:
@@ -61,4 +61,4 @@ jobs:
       - uses: actions-rs/cargo@v1
         with:
           command: clippy
-          args: --all-targets --all-features -- -D warnings -A clippy::unnecessary_unwrap
+          args: --all-targets --all-features -- -D warnings -A clippy::deref_addrof

.gitignore

@@ -22,3 +22,6 @@ img/**
 # scripts to check code coverage using nightly compiler
 check-coverage.sh
 lcov_cobertura.py
+
+# dockerignore file that makes it so i can work on the docker config without copying a 4GB manifest or w/e it is
+.dockerignore

.rustfmt.toml

@@ -0,0 +1 @@
+reorder_modules = false

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "feroxbuster"
-version = "1.5.3"
+version = "1.12.2"
 authors = ["Ben 'epi' Risher <epibar052@gmail.com>"]
 license = "MIT"
 edition = "2018"
@@ -10,34 +10,42 @@ description = "A fast, simple, recursive content discovery tool."
 categories = ["command-line-utilities"]
 keywords = ["pentest", "enumeration", "url-bruteforce", "content-discovery", "web"]
 exclude = [".github/*", "img/*", "check-coverage.sh"]
+build = "build.rs"
 
 [badges]
 maintenance = { status = "actively-developed" }
 
+[build-dependencies]
+clap = "2.33"
+regex = "1"
+lazy_static = "1.4"
+
 [dependencies]
 futures = { version = "0.3"}
-tokio = { version = "0.2", features = ["full"] }
-tokio-util = {version = "0.3", features = ["codec"]}
+tokio = { version = "1.0", features = ["full"] }
+tokio-util = {version = "0.6", features = ["codec"]}
 log = "0.4"
 env_logger = "0.8"
-reqwest = { version = "0.10", features = ["socks"] }
-clap = "2"
+reqwest = { version = "0.11", features = ["socks"] }
+clap = "2.33"
 lazy_static = "1.4"
 toml = "0.5"
-serde = { version = "1.0", features = ["derive"] }
+serde = { version = "1.0", features = ["derive", "rc"] }
 serde_json = "1.0"
 uuid = { version = "0.8", features = ["v4"] }
 indicatif = "0.15"
-console = "0.12"
+console = "0.14"
 openssl = { version = "0.10", features = ["vendored"] }
 dirs = "3.0"
 regex = "1"
-crossterm = "0.18"
+crossterm = "0.19"
 rlimit = "0.5"
+ctrlc = "3.1"
+fuzzyhash = "0.2.1"
 
 [dev-dependencies]
 tempfile = "3.1"
-httpmock = "0.4.5"
+httpmock = "0.5.2"
 assert_cmd = "1.0.1"
 predicates = "1.0.5"
 

Dockerfile

@@ -1,8 +1,10 @@
 FROM alpine:latest
 LABEL maintainer="wfnintr@null.net"
 
+RUN sed -i -e 's/v[[:digit:]]\..*\//edge\//g' /etc/apk/repositories && apk upgrade --update-cache --available
+
 # download default wordlists 
-RUN apk add --no-cache --virtual .depends subversion && \
+RUN apk add --no-cache --virtual .depends subversion font-noto-emoji && \
 	svn export https://github.com/danielmiessler/SecLists/trunk/Discovery/Web-Content /usr/share/seclists/Discovery/Web-Content && \
 	apk del .depends
 

README.md

@@ -22,7 +22,7 @@
   <a href="https://crates.io/crates/feroxbuster">
     <img src="https://img.shields.io/crates/v/feroxbuster?color=blue&label=version&logo=rust">
   </a>
- 
+
   <a href="https://crates.io/crates/feroxbuster">
     <img src="https://img.shields.io/crates/d/feroxbuster?label=downloads&logo=rust&color=inactive">
   </a>
@@ -45,22 +45,29 @@
 
 ## 😕 What the heck is a ferox anyway?
 
-Ferox is short for Ferric Oxide. Ferric Oxide, simply put, is rust.  The name rustbuster was taken, so I decided on a variation.  🤷	
+Ferox is short for Ferric Oxide. Ferric Oxide, simply put, is rust. The name rustbuster was taken, so I decided on a
+variation. 🤷
 
-## 🤔 What's it do tho? 
+## 🤔 What's it do tho?
 
-`feroxbuster` is a tool designed to perform [Forced Browsing](https://owasp.org/www-community/attacks/Forced_browsing).  
+`feroxbuster` is a tool designed to perform [Forced Browsing](https://owasp.org/www-community/attacks/Forced_browsing).
 
-Forced browsing is an attack where the aim is to enumerate and access resources that are not referenced by the web application, but are still accessible by an attacker.
+Forced browsing is an attack where the aim is to enumerate and access resources that are not referenced by the web
+application, but are still accessible by an attacker.
 
-`feroxbuster` uses brute force combined with a wordlist to search for unlinked content in target directories. These resources may store sensitive information about web applications and operational systems, such as source code, credentials, internal network addressing, etc...
+`feroxbuster` uses brute force combined with a wordlist to search for unlinked content in target directories. These
+resources may store sensitive information about web applications and operational systems, such as source code,
+credentials, internal network addressing, etc...
 
-This attack is also known as Predictable Resource Location, File Enumeration, Directory Enumeration, and Resource Enumeration.
+This attack is also known as Predictable Resource Location, File Enumeration, Directory Enumeration, and Resource
+Enumeration.
 
 📖 Table of Contents
 -----------------
+
 - [Installation](#-installation)
     - [Download a Release](#download-a-release)
+    - [Snap Install](#snap-install)
     - [Homebrew on MacOS and Linux](#homebrew-on-macos-and-linux)
     - [Cargo Install](#cargo-install)
     - [apt Install](#apt-install)
@@ -68,33 +75,50 @@ This attack is also known as Predictable Resource Location, File Enumeration, Di
     - [Docker Install](#docker-install)
 - [Configuration](#%EF%B8%8F-configuration)
     - [Default Values](#default-values)
+    - [Threads and Connection Limits At A High-Level](#threads-and-connection-limits-at-a-high-level)
     - [ferox-config.toml](#ferox-configtoml)
     - [Command Line Parsing](#command-line-parsing)
+- [Scan's Display Explained](#-scans-display-explained)
+    - [Discovered Resource](#discovered-resource)
+    - [Overall Scan Progress Bar](#overall-scan-progress-bar)
+    - [Directory Scan Progress Bar](#directory-scan-progress-bar)
 - [Example Usage](#-example-usage)
-    - [Pause and Resume Scans (new in `v1.4.0`)](#pause-and-resume-scans-new-in-v140)
     - [Multiple Values](#multiple-values)
-    - [Extract Links from Response Body (new in `v1.1.0`)](#extract-links-from-response-body-new-in-v110)
     - [Include Headers](#include-headers)
     - [IPv6, Non-recursive scan with INFO logging enabled](#ipv6-non-recursive-scan-with-info-level-logging-enabled)
     - [Read urls from STDIN; pipe only resulting urls out to another tool](#read-urls-from-stdin-pipe-only-resulting-urls-out-to-another-tool)
     - [Proxy traffic through Burp](#proxy-traffic-through-burp)
-    - [Proxy traffic through a SOCKS proxy](#proxy-traffic-through-a-socks-proxy)
+    - [Proxy traffic through a SOCKS proxy (including DNS lookups)](#proxy-traffic-through-a-socks-proxy-including-dns-lookups)
     - [Pass auth token via query parameter](#pass-auth-token-via-query-parameter)
+    - [Extract Links from Response Body (new in `v1.1.0`)](#extract-links-from-response-body-new-in-v110)
     - [Limit Total Number of Concurrent Scans (new in `v1.2.0`)](#limit-total-number-of-concurrent-scans-new-in-v120)
     - [Filter Response by Status Code  (new in `v1.3.0`)](#filter-response-by-status-code--new-in-v130)
+    - [Pause an Active Scan (new in `v1.4.0`)](#pause-an-active-scan-new-in-v140)
     - [Replay Responses to a Proxy based on Status Code (new in `v1.5.0`)](#replay-responses-to-a-proxy-based-on-status-code-new-in-v150)
+    - [Filter Response by Word Count & Line Count  (new in `v1.6.0`)](#filter-response-by-word-count--line-count--new-in-v160)
+    - [Filter Response Using a Regular Expression (new in `v1.8.0`)](#filter-response-using-a-regular-expression-new-in-v180)
+    - [Stop and Resume Scans (save scan's state to disk) (new in `v1.9.0`)](#stop-and-resume-scans---resume-from-file-new-in-v190)
+    - [Enforce a Time Limit on Your Scan (new in `v1.10.0`)](#enforce-a-time-limit-on-your-scan-new-in-v1100)
+    - [Extract Links from robots.txt (New in `v1.10.2`)](#extract-links-from-robotstxt-new-in-v1102)
+    - [Filter Response by Similarity to A Given Page (fuzzy filter) (new in `v1.11.0`)](#filter-response-by-similarity-to-a-given-page-fuzzy-filter-new-in-v1110)
+    - [Cancel a Recursive Scan Interactively (new in `v1.12.0`)](#cancel-a-recursive-scan-interactively-new-in-v1120)
 - [Comparison w/ Similar Tools](#-comparison-w-similar-tools)
 - [Common Problems/Issues (FAQ)](#-common-problemsissues-faq)
     - [No file descriptors available](#no-file-descriptors-available)
     - [Progress bars print one line at a time](#progress-bars-print-one-line-at-a-time)
+    - [What do each of the numbers beside the URL mean?](#what-do-each-of-the-numbers-beside-the-url-mean)
+    - [Connection closed before message completed](#connection-closed-before-message-completed)
+    - [SSL Error routines:tls_process_server_certificate:certificate verify failed](#ssl-error-routinestls_process_server_certificatecertificate-verify-failed)
 
 ## 💿 Installation
 
 ### Download a Release
 
-Releases for multiple architectures can be found in the [Releases](https://github.com/epi052/feroxbuster/releases) section.  The latest release for each of the following systems can be downloaded and executed as shown below.
+Releases for multiple architectures can be found in the [Releases](https://github.com/epi052/feroxbuster/releases)
+section. The latest release for each of the following systems can be downloaded and executed as shown below.
 
 #### Linux (32 and 64-bit) & MacOS
+
 ```
 curl -sL https://raw.githubusercontent.com/epi052/feroxbuster/master/install-nix.sh | bash
 ```
@@ -115,9 +139,34 @@ Expand-Archive .\feroxbuster.zip
 .\feroxbuster\feroxbuster.exe -V
 ```
 
+### Snap Install
+
+Install using `snap`
+
+```
+sudo snap install feroxbuster
+```
+
+The only gotcha here is that the snap package can only read wordlists from a few specific locations. There are a few
+possible solutions, of which two are shown below.
+
+If the wordlist is on the same partition as your home directory, it can be hard-linked into `~/snap/feroxbuster/common`
+
+```
+ln /path/to/the/wordlist ~/snap/feroxbuster/common
+./feroxbuster -u http://localhost -w ~/snap/feroxbuster/common/wordlist
+``` 
+
+If the wordlist is on a separate partition, hard-linking won't work. You'll need to copy it into the snap directory.
+
+```
+cp /path/to/the/wordlist ~/snap/feroxbuster/common
+./feroxbuster -u http://localhost -w ~/snap/feroxbuster/common/wordlist
+``` 
+
 ### Homebrew on MacOS and Linux
 
-Installable by Homebrew throughout own formulas:
+Install using Homebrew via tap
 
 🍏 [MacOS](https://github.com/TGotwig/homebrew-feroxbuster/blob/main/feroxbuster.rb)
 
@@ -143,7 +192,8 @@ cargo install feroxbuster
 
 ### apt Install
 
-Download `feroxbuster_amd64.deb` from the [Releases](https://github.com/epi052/feroxbuster/releases) section.  After that, use your favorite package manager to install the `.deb`.
+Download `feroxbuster_amd64.deb` from the [Releases](https://github.com/epi052/feroxbuster/releases) section. After
+that, use your favorite package manager to install the `.deb`.
 
 ```
 wget -sLO https://github.com/epi052/feroxbuster/releases/latest/download/feroxbuster_amd64.deb.zip
@@ -192,7 +242,9 @@ cat targets.txt | sudo docker run --net=host --init -i feroxbuster --stdin -x js
 
 #### Mount a volume to pass in `ferox-config.toml`
 
-You've got some options available if you want to pass in a config file.  [`ferox-buster.toml`](#ferox-configtoml) can live in multiple locations and still be valid, so it's up to you how you'd like to pass it in.  Below are a few valid examples:
+You've got some options available if you want to pass in a config file.  [`ferox-buster.toml`](#ferox-configtoml) can
+live in multiple locations and still be valid, so it's up to you how you'd like to pass it in. Below are a few valid
+examples:
 
 ```
 sudo docker run --init -v $(pwd)/ferox-config.toml:/etc/feroxbuster/ferox-config.toml -it feroxbuster -u http://example.com
@@ -215,7 +267,9 @@ alias feroxbuster="sudo docker run --init -v ~/.config/feroxbuster:/root/.config
 ```
 
 ## ⚙️ Configuration
+
 ### Default Values
+
 Configuration begins with with the following built-in default values baked into the binary:
 
 - timeout: `7` seconds
@@ -229,12 +283,44 @@ Configuration begins with with the following built-in default values baked into
 - recursion depth: `4`
 - auto-filter wildcards - `true`
 - output: `stdout`
+- save_state: `true` (create a state file in cwd when `Ctrl+C` is received)
+
+### Threads and Connection Limits At A High-Level
+
+This section explains how the `-t` and `-L` options work together to determine the overall aggressiveness of a scan. The
+combination of the two values set by these options determines how hard your target will get hit and to some extent also
+determines how many resources will be consumed on your local machine.
+
+#### A Note on Green Threads
+
+`feroxbuster` uses so-called [green threads](https://en.wikipedia.org/wiki/Green_threads) as opposed to traditional
+kernel/OS threads. This means (at a high-level) that the threads are implemented entirely in userspace, within a single
+running process. As a result, a scan with 30 green threads will appear to the OS to be a single process with no
+additional light-weight processes associated with it as far as the kernel is concerned. As such, there will not be any
+impact to process (`nproc`) limits when specifying larger values for `-t`. However, these threads will still consume
+file descriptors, so you will need to ensure that you have a suitable `nlimit` set when scaling up the amount of
+threads. More detailed documentation on setting appropriate `nlimit` values can be found in
+the [No File Descriptors Available](#no-file-descriptors-available) section of the FAQ
+
+#### Threads and Connection Limits: The Implementation
+
+* Threads: The `-t` option specifies the maximum amount of active threads *per-directory* during a scan
+* Connection Limits: The `-L` option specifies the maximum amount of active connections per thread
+
+#### Threads and Connection Limits: Examples
+
+To truly have only 30 active requests to a site at any given time, `-t 30 -L 1` is necessary. Using `-t 30 -L 2` will
+result in a maximum of 60 total requests being processed at any given time for that site. And so on. For a conversation
+on this, please see [Issue #126](https://github.com/epi052/feroxbuster/issues/126) which may provide more (or less)
+clarity :wink:
 
 ### ferox-config.toml
+
 After setting built-in default values, any values defined in a `ferox-config.toml` config file will override the
-built-in defaults.  
+built-in defaults.
 
 `feroxbuster` searches for `ferox-config.toml` in the following locations (in the order shown):
+
 - `/etc/feroxbuster/` (global)
 - `CONFIG_DIR/ferxobuster/` (per-user)
 - The same directory as the `feroxbuster` executable (per-user)
@@ -245,14 +331,15 @@ built-in defaults.
 > - MacOs: `$HOME/Library/Application Support` i.e. `/Users/bob/Library/Application Support`
 > - Windows: `{FOLDERID_RoamingAppData}` i.e. `C:\Users\Bob\AppData\Roaming`
 
-If more than one valid configuration file is found, each one overwrites the values found previously.  
+If more than one valid configuration file is found, each one overwrites the values found previously.
 
 If no configuration file is found, nothing happens at this stage.
 
-As an example, let's say that we prefer to use a different wordlist as our default when scanning; we can
-set the `wordlist` value in the config file to override the baked-in default.
+As an example, let's say that we prefer to use a different wordlist as our default when scanning; we can set
+the `wordlist` value in the config file to override the baked-in default.
 
 Notes of interest:
+
 - it's ok to only specify values you want to change without specifying anything else
 - variable names in `ferox-config.toml` must match their command-line counterpart
 
@@ -263,6 +350,7 @@ wordlist = "/wordlists/jhaddix/all.txt"
 ```
 
 A pre-made configuration file with examples of all available settings can be found in `ferox-config.toml.example`.
+
 ```toml
 # ferox-config.toml
 # Example configuration for feroxbuster
@@ -277,15 +365,17 @@ A pre-made configuration file with examples of all available settings can be fou
 # wordlist = "/wordlists/jhaddix/all.txt"
 # status_codes = [200, 500]
 # filter_status = [301]
-# replay_codes = [301]
 # threads = 1
 # timeout = 5
 # proxy = "http://127.0.0.1:8080"
 # replay_proxy = "http://127.0.0.1:8081"
+# replay_codes = [200, 302]
 # verbosity = 1
 # scan_limit = 6
 # quiet = true
+# json = true
 # output = "/targets/ellingson_mineral_company/gibson.txt"
+# debug_log = "/var/log/find-the-derp.log"
 # user_agent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0"
 # redirects = true
 # insecure = true
@@ -297,7 +387,13 @@ A pre-made configuration file with examples of all available settings can be fou
 # extract_links = true
 # depth = 1
 # filter_size = [5174]
+# filter_regex = ["^ignore me$"]
+# filter_similar = ["https://somesite.com/soft404"]
+# filter_word_count = [993]
+# filter_line_count = [35, 36]
 # queries = [["name","value"], ["rick", "astley"]]
+# save_state = false
+# time_limit = 10m
 
 # headers can be specified on multiple lines or as an inline table
 #
@@ -314,7 +410,9 @@ A pre-made configuration file with examples of all available settings can be fou
 ```
 
 ### Command Line Parsing
-Finally, after parsing the available config file, any options/arguments given on the commandline will override any values that were set as a built-in or config-file value.
+
+Finally, after parsing the available config file, any options/arguments given on the commandline will override any
+values that were set as a built-in or config-file value.
 
 ```
 USAGE:
@@ -327,47 +425,76 @@ FLAGS:
                            findings (default: false)
     -h, --help             Prints help information
     -k, --insecure         Disables TLS certificate validation
+        --json             Emit JSON logs to --output and --debug-log instead of normal text
     -n, --no-recursion     Do not scan recursively
     -q, --quiet            Only print URLs; Don't print status codes, response size, running config, etc...
     -r, --redirects        Follow redirects
         --stdin            Read url(s) from STDIN
     -V, --version          Prints version information
-    -v, --verbosity        Increase verbosity level (use -vv or more for greater effect)
+    -v, --verbosity        Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v's is probably
+                           too much)
 
 OPTIONS:
+        --debug-log <FILE>                  Output file to write log entries (use w/ --json for JSON entries)
     -d, --depth <RECURSION_DEPTH>           Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)
     -x, --extensions <FILE_EXTENSION>...    File extension(s) to search for (ex: -x php -x pdf js)
+    -N, --filter-lines <LINES>...           Filter out messages of a particular line count (ex: -N 20 -N 31,30)
+    -X, --filter-regex <REGEX>...           Filter out messages via regular expression matching on the response's body
+                                            (ex: -X '^ignore me$')
     -S, --filter-size <SIZE>...             Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)
     -C, --filter-status <STATUS_CODE>...    Filter out status codes (deny list) (ex: -C 200 -C 401)
+    -W, --filter-words <WORDS>...           Filter out messages of a particular word count (ex: -W 312 -W 91,82)
     -H, --headers <HEADER>...               Specify HTTP headers (ex: -H Header:val 'stuff: things')
-    -o, --output <FILE>                     Output file to write results to (default: stdout)
-    -p, --proxy <PROXY>                     Proxy to use for requests (ex: http(s)://host:port, socks5://host:port)
+    -o, --output <FILE>                     Output file to write results to (use w/ --json for JSON entries)
+    -p, --proxy <PROXY>                     Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)
     -Q, --query <QUERY>...                  Specify URL query parameters (ex: -Q token=stuff -Q secret=key)
-    -R, --replay-codes <REPLAY_CODE>...     Status Codes to send through a Replay Proxy when found (default: --status
-                                            -codes value)
+    -R, --replay-codes <REPLAY_CODE>...     Status Codes to send through a Replay Proxy when found (default: --status-
+                                            codes value)
     -P, --replay-proxy <REPLAY_PROXY>       Send only unfiltered requests through a Replay Proxy, instead of all
                                             requests
+        --resume-from <STATE_FILE>          State file from which to resume a partially complete scan (ex. --resume-from
+                                            ferox-1606586780.state)
     -L, --scan-limit <SCAN_LIMIT>           Limit total number of concurrent scans (default: 0, i.e. no limit)
     -s, --status-codes <STATUS_CODE>...     Status Codes to include (allow list) (default: 200 204 301 302 307 308 401
                                             403 405)
     -t, --threads <THREADS>                 Number of concurrent threads (default: 50)
+        --time-limit <TIME_SPEC>            Limit total run time of all scans (ex: --time-limit 10m)
     -T, --timeout <SECONDS>                 Number of seconds before a request times out (default: 7)
     -u, --url <URL>...                      The target URL(s) (required, unless --stdin used)
     -a, --user-agent <USER_AGENT>           Sets the User-Agent (default: feroxbuster/VERSION)
     -w, --wordlist <FILE>                   Path to the wordlist
 ```
 
+## 📊 Scan's Display Explained
+
+`feroxbuster` attempts to be intuitive and easy to understand, however, if you are wondering about any of the scan's
+output and what it means, this is the section for you!  
+
+### Discovered Resource
+
+When `feroxbuster` finds a response that you haven't filtered out, it's reported above the progress bars and looks similar to what's pictured below.  
+
+The number of lines, words, and bytes shown here can be used to [filter those responses](#filter-response-by-word-count--line-count--new-in-v160)
+
+![response-bar-explained](img/response-bar-explained.png)
+
+### Overall Scan Progress Bar
+
+The top progress bar, colored yellow, tracks the overall scan status.  Its fields are described in the image below.
+
+![total-bar-explained](img/total-bar-explained.png)
+
+### Directory Scan Progress Bar
+
+All other progress bars, colored cyan, represent a scan of one particular directory and will look similar to what's below.   
+
+![dir-scan-bar-explained](img/dir-scan-bar-explained.png)
+
 ## 🧰 Example Usage
 
-### Pause and Resume Scans (new in `v1.4.0`)
-
-Scans can be paused and resumed by pressing the ENTER key (shown below)
-
-![pause-resume-demo](img/pause-resume-demo.gif)
-
 ### Multiple Values
 
-Options that take multiple values are very flexible.  Consider the following ways of specifying extensions:
+Options that take multiple values are very flexible. Consider the following ways of specifying extensions:
 
 ```
 ./feroxbuster -u http://127.1 -x pdf -x js,html -x php txt json,docx
@@ -375,7 +502,8 @@ Options that take multiple values are very flexible.  Consider the following way
 
 The command above adds .pdf, .js, .html, .php, .txt, .json, and .docx to each url
 
-All of the methods above (multiple flags, space separated, comma separated, etc...) are valid and interchangeable.  The same goes for urls, headers, status codes, queries, and size filters.
+All of the methods above (multiple flags, space separated, comma separated, etc...) are valid and interchangeable. The
+same goes for urls, headers, status codes, queries, and size filters.
 
 ### Include Headers
 
@@ -383,37 +511,6 @@ All of the methods above (multiple flags, space separated, comma separated, etc.
 ./feroxbuster -u http://127.1 -H Accept:application/json "Authorization: Bearer {token}"
 ```
 
-### Extract Links from Response Body (New in `v1.1.0`) 
-
-Search through the body of valid responses (html, javascript, etc...) for additional endpoints to scan. This turns
-`feroxbuster` into a hybrid that looks for both linked and unlinked content. 
-
-Example request/response with `--extract-links` enabled:
-- Make request to `http://example.com/index.html`
-- Receive, and read in, the `body` of the response
-- Search the `body` for absolute and relative links (i.e. `homepage/assets/img/icons/handshake.svg`)
-- Add the following directories for recursive scanning:
-    - `http://example.com/homepage`
-    - `http://example.com/homepage/assets`
-    - `http://example.com/homepage/assets/img`
-    - `http://example.com/homepage/assets/img/icons`
-- Make a single request to `http://example.com/homepage/assets/img/icons/handshake.svg`
-
-```
-./feroxbuster -u http://127.1 --extract-links
-```
-
-Here's a comparison of a wordlist-only scan vs `--extract-links` using [Feline](https://www.hackthebox.eu/home/machines/profile/274) from Hack the Box:
-
-Wordlist only
-
-![normal-scan-cmp-extract](img/normal-scan-cmp-extract.gif)
-
-With `--extract-links`
-
-![extract-scan-cmp-normal](img/extract-scan-cmp-normal.gif)
-
-
 ### IPv6, non-recursive scan with INFO-level logging enabled
 
 ```
@@ -432,22 +529,54 @@ cat targets | ./feroxbuster --stdin --quiet -s 200 301 302 --redirects -x js | f
 ./feroxbuster -u http://127.1 --insecure --proxy http://127.0.0.1:8080
 ```
 
-### Proxy traffic through a SOCKS proxy
+### Proxy traffic through a SOCKS proxy (including DNS lookups)
 
 ```
-./feroxbuster -u http://127.1 --proxy socks5://127.0.0.1:9050
+./feroxbuster -u http://127.1 --proxy socks5h://127.0.0.1:9050
 ```
 
-### Pass auth token via query parameter 
+### Pass auth token via query parameter
 
 ```
 ./feroxbuster -u http://127.1 --query token=0123456789ABCDEF
 ```
 
+### Extract Links from Response Body (New in `v1.1.0`)
+
+Search through the body of valid responses (html, javascript, etc...) for additional endpoints to scan. This turns
+`feroxbuster` into a hybrid that looks for both linked and unlinked content.
+
+Example request/response with `--extract-links` enabled:
+
+- Make request to `http://example.com/index.html`
+- Receive, and read in, the `body` of the response
+- Search the `body` for absolute and relative links (i.e. `homepage/assets/img/icons/handshake.svg`)
+- Add the following directories for recursive scanning:
+    - `http://example.com/homepage`
+    - `http://example.com/homepage/assets`
+    - `http://example.com/homepage/assets/img`
+    - `http://example.com/homepage/assets/img/icons`
+- Make a single request to `http://example.com/homepage/assets/img/icons/handshake.svg`
+
+```
+./feroxbuster -u http://127.1 --extract-links
+```
+
+Here's a comparison of a wordlist-only scan vs `--extract-links`
+using [Feline](https://www.hackthebox.eu/home/machines/profile/274) from Hack the Box:
+
+Wordlist only
+
+![normal-scan-cmp-extract](img/normal-scan-cmp-extract.gif)
+
+With `--extract-links`
+
+![extract-scan-cmp-normal](img/extract-scan-cmp-normal.gif)
+
 ### Limit Total Number of Concurrent Scans (new in `v1.2.0`)
 
-Limit the number of scans permitted to run at any given time.  Recursion will still identify new directories, but newly
-discovered directories can only begin scanning when the total number of active scans drops below the value passed to 
+Limit the number of scans permitted to run at any given time. Recursion will still identify new directories, but newly
+discovered directories can only begin scanning when the total number of active scans drops below the value passed to
 `--scan-limit`.
 
 ```
@@ -458,70 +587,264 @@ discovered directories can only begin scanning when the total number of active s
 
 ### Filter Response by Status Code  (new in `v1.3.0`)
 
-Version 1.3.0 included an overhaul to the filtering system which will allow for a wide array of filters to be added 
-with minimal effort. The first such filter is a Status Code Filter. As responses come back from the scanned server,
-each one is checked against a list of known filters and either displayed or not according to which filters are set.
+Version 1.3.0 included an overhaul to the filtering system which will allow for a wide array of filters to be added with
+minimal effort. The first such filter is a Status Code Filter. As responses come back from the scanned server, each one
+is checked against a list of known filters and either displayed or not according to which filters are set.
 
 ```
 ./feroxbuster -u http://127.1 --filter-status 301
 ```
 
+### Pause an Active Scan (new in `v1.4.0`)
+
+**NOTE**: [v1.12.0](#cancel-a-recursive-scan-interactively-new-in-v1120) added an interactive menu to the pause/resume
+functionality.  Active scans can still be paused, however, now you're presented with the option to cancel a scan instead
+of simply seeing a spinner.
+
+Scans can be paused and resumed by pressing the ENTER key (~~shown below~~, please see [v1.12.0](#cancel-a-recursive-scan-interactively-new-in-v1120)'s entry for the latest visual representation)
+
 ### Replay Responses to a Proxy based on Status Code (new in `v1.5.0`)
 
-The `--replay-proxy` and `--replay-codes` options were added as a way to only send a select few responses to a proxy.  This is in stark contrast to `--proxy` which proxies EVERY request.  
+The `--replay-proxy` and `--replay-codes` options were added as a way to only send a select few responses to a proxy.
+This is in stark contrast to `--proxy` which proxies EVERY request.
 
-Imagine you only care about proxying responses that have either the status code `200` or `302` (or you just don't want to clutter up your Burp history).  These two options will allow you to fine-tune what gets proxied and what doesn't.  
+Imagine you only care about proxying responses that have either the status code `200` or `302` (or you just don't want
+to clutter up your Burp history). These two options will allow you to fine-tune what gets proxied and what doesn't.
 
 ```
 ./feroxbuster -u http://127.1 --replay-proxy http://localhost:8080 --replay-codes 200 302 --insecure
 ```
 
-Of note: this means that for every response that matches your replay criteria, you'll end up sending the request that generated that response a second time.  Depending on the target and your engagement terms (if any), it may not make sense from a traffic generated perspective.
+Of note: this means that for every response that matches your replay criteria, you'll end up sending the request that
+generated that response a second time. Depending on the target and your engagement terms (if any), it may not make sense
+from a traffic generated perspective.
 
 ![replay-proxy-demo](img/replay-proxy-demo.gif)
 
+### Filter Response by Word Count & Line Count  (new in `v1.6.0`)
+
+In addition to filtering on the size of a response, version 1.6.0 added the ability to filter out responses based on the
+number of lines and/or words contained within the response body. This change drove a change to the information displayed
+to the user as well. This section will detail the new information and how to make use of it with the new filters
+provided.
+
+Example output:
+
+```
+200        10l        212w       38437c https://example-site.com/index.html
+```
+
+There are five columns of output above:
+
+- column 1: status code - can be filtered with `-C|--filter-status`
+- column 2: number of lines - can be filtered with `-N|--filter-lines`
+- column 3: number of words - can be filtered with `-W|--filter-words`
+- column 4: number of bytes (overall size) - can be filtered with `-S|--filter-size`
+- column 5: url to discovered resource
+
+### Filter Response Using a Regular Expression (new in `v1.8.0`)
+
+Version 1.3.0 included an overhaul to the filtering system which will allow for a wide array of filters to be added with
+minimal effort. The latest addition is a Regular Expression Filter. As responses come back from the scanned server,
+the **body** of the response is checked against the filter's regular expression. If the expression is found in the body,
+then that response is filtered out.
+
+**NOTE: Using regular expressions to filter large responses or many regular expressions may negatively impact
+performance.**
+
+```
+./feroxbuster -u http://127.1 --filter-regex '[aA]ccess [dD]enied.?' --output results.txt --json
+```
+
+### Stop and Resume Scans (`--resume-from FILE`) (new in `v1.9.0`)
+
+Version 1.9.0 adds a few features that allow for completely stopping a scan, and resuming that same scan from a file on
+disk.
+
+A simple `Ctrl+C` during a scan will create a file that contains information about the scan that was cancelled.
+
+![save-state](img/save-state.png)
+
+```json
+// example snippet of state file
+
+{
+  "scans": [
+    {
+      "id": "057016a14769414aac9a7a62707598cb",
+      "url": "https://localhost.com",
+      "scan_type": "Directory",
+      "complete": true
+    },
+    {
+      "id": "400b2323a16f43468a04ffcbbeba34c6",
+      "url": "https://localhost.com/css",
+      "scan_type": "Directory",
+      "complete": false
+    }
+  ],
+  "config": {
+    "wordlist": "/wordlists/seclists/Discovery/Web-Content/common.txt",
+    "...": "..."
+  },
+  "responses": [
+    {
+      "type": "response",
+      "url": "https://localhost.com/Login",
+      "path": "/Login",
+      "wildcard": false,
+      "status": 302,
+      "content_length": 0,
+      "line_count": 0,
+      "word_count": 0,
+      "headers": {
+        "content-length": "0",
+        "server": "nginx/1.16.1"
+      }
+    }
+  ]
+},
+```
+
+Based on the example image above, the same scan can be resumed by
+using `feroxbuster --resume-from ferox-http_localhost-1606947491.state`. Directories that were already complete are not
+rescanned, however partially complete scans are started from the beginning.
+
+![resumed-scan](img/resumed-scan.gif)
+
+In order to prevent state file creation when `Ctrl+C` is pressed, you can simply add the entry below to
+your `ferox-config.toml`.
+
+```toml
+# ferox-config.toml
+
+save_state = false
+```
+
+### Enforce a Time Limit on Your Scan (new in `v1.10.0`)
+
+Version 1.10.0 adds the ability to set a maximum runtime, or time limit, on your scan. The usage is pretty simple: a
+number followed directly by a single character representing seconds, minutes, hours, or days.  `feroxbuster` refers to
+this combination as a time_spec.
+
+Examples of possible time_specs:
+
+- `30s` - 30 seconds
+- `20m` - 20 minutes
+- `1h`  - 1 hour
+- `1d`  - 1 day (why??)
+
+A valid time_spec can be passed to `--time-limit` in order to force a shutdown after the given time has elapsed.
+
+![time-limit](img/time-limit.gif)
+
+### Extract Links from robots.txt (New in `v1.10.2`)
+
+In addition to [extracting links from the response body](#extract-links-from-response-body-new-in-v110), using
+`--extract-links` makes a request to `/robots.txt` and examines all `Allow` and `Disallow` entries. Directory entries
+are added to the scan queue, while file entries are requested and then reported if appropriate.
+
+### Filter Response by Similarity to A Given Page (fuzzy filter) (new in `v1.11.0`)
+
+Version 1.11.0 adds the ability to specify an example page for filtering pages that are similar to the given example.
+
+For example, consider a site that attempts to redirect new users to a `/register` endpoint. The `/register` page has a
+CSRF token that alters the page's response slightly with each new request (sometimes affecting overall length). This
+means that a simple line/word/char filter won't be able to filter all responses. In order to filter those redirects out,
+one could use a command like this:
+
+```
+./feroxbuster -u https://somesite.xyz --filter-similar-to https://somesite.xyz/register
+```
+
+`--filter-similar-to` requests the page passed to it via CLI (`https://somesite.xyz/register`), after which it hashes 
+the response body using the [SSDeep algorithm](https://ssdeep-project.github.io/ssdeep/index.html).  All subsequent 
+pages are hashed and compared to the original request's hash. If the comparison of the two hashes meets a certain 
+percentage of similarity (currently 95%), then that request will be filtered out.
+
+SSDeep was selected as it does a good job of identifying near-duplicate pages once content-length reaches a certain 
+size, while remaining performant.  Other algorithms were tested but resulted in huge performance hits (orders of 
+magnitude slower on requests/second).
+
+**NOTE**
+- SSDeep/`--filter-similar-to` does not do well at detecting similarity of very small responses
+  - The lack of accuracy with very small responses is considered a fair trade-off for not negatively impacting performance
+- Using a bunch of `--filter-similar-to` values **may** negatively impact performance
+
+### Cancel a Recursive Scan Interactively (new in `v1.12.0`)
+
+Version 1.12.0 expanded the pause/resume functionality introduced in [v1.4.0](#pause-an-active-scan-new-in-v140) by 
+adding an interactive menu from which currently running recursive scans can be cancelled, without affecting the overall scan.  Scans can still be paused indefinitely by pressing `ENTER`, however, the   
+
+Scans that are started via `-u` or passed in through `--stdin` cannot be cancelled, only scans found via `--extract-links` or recursion are eligible.
+
+Below is an example of the Scan Cancel Menu™.
+
+![cancel-menu](img/cancel-menu.png)
+
+Using the menu is pretty simple:
+- Press `ENTER` to view the menu
+- Choose a scan to cancel by entering its scan index (`1`)
+  - more than one scan can be selected by using a comma-separated list (`1,2,3` ... etc)
+- Confirm selections, after which all non-cancelled scans will resume
+
+Here is a short demonstration of cancelling two in-progress scans found via recursion.
+
+![cancel-scan](img/cancel-scan.gif)
+
 ## 🧐 Comparison w/ Similar Tools
 
-There are quite a few similar tools for forced browsing/content discovery.  Burp Suite Pro, Dirb, Dirbuster, etc... 
-However, in my opinion, there are two that set the standard: [gobuster](https://github.com/OJ/gobuster) and 
-[ffuf](https://github.com/ffuf/ffuf).  Both are mature, feature-rich, and all-around incredible tools to use.
+There are quite a few similar tools for forced browsing/content discovery. Burp Suite Pro, Dirb, Dirbuster, etc...
+However, in my opinion, there are two that set the standard: [gobuster](https://github.com/OJ/gobuster) and
+[ffuf](https://github.com/ffuf/ffuf). Both are mature, feature-rich, and all-around incredible tools to use.
 
-So, why would you ever want to use feroxbuster over ffuf/gobuster?  In most cases, you probably won't.  ffuf in particular
-can do the vast majority of things that feroxbuster can, while still offering boatloads more functionality.  Here are
-a few of the use-cases in which feroxbuster may be a better fit:
+So, why would you ever want to use feroxbuster over ffuf/gobuster? In most cases, you probably won't. ffuf in particular
+can do the vast majority of things that feroxbuster can, while still offering boatloads more functionality. Here are a
+few of the use-cases in which feroxbuster may be a better fit:
 
 - You want a **simple** tool usage experience
 - You want to be able to run your content discovery as part of some crazy 12 command unix **pipeline extravaganza**
 - You want to scan through a **SOCKS** proxy
 - You want **auto-filtering** of Wildcard responses by default
-- You want an integrated **link extractor** to increase discovered endpoints
+- You want an integrated **link extractor/robots.txt parser** to increase discovered endpoints
 - You want **recursion** along with some other thing mentioned above (ffuf also does recursion)
 - You want a **configuration file** option for overriding built-in default values for your scans
 
-|                                                                  | feroxbuster | gobuster | ffuf |
-|------------------------------------------------------------------|---|---|---|
-| fast                                                             | ✔ | ✔ | ✔ |
-| easy to use                                                      | ✔ | ✔ |   |
-| filter out responses by status code (new in `v1.3.0`)            | ✔ | ✔ | ✔ |
-| allows recursion                                                 | ✔ |   | ✔ |
-| can specify query parameters                                     | ✔ |   | ✔ |
-| SOCKS proxy support                                              | ✔ |   |   |
-| extracts links from response body to increase scan coverage      | ✔ |   |   |
-| multiple target scan (via stdin or multiple -u)                  | ✔ |   | ✔ |
-| configuration file for default value override                    | ✔ |   | ✔ |
-| can accept urls via STDIN as part of a pipeline                  | ✔ |   | ✔ |
-| can accept wordlists via STDIN                                   |   | ✔ | ✔ |
-| filter by response size                                          | ✔ |   | ✔ |
-| auto-filter wildcard responses                                   | ✔ |   | ✔ |
-| performs other scans (vhost, dns, etc)                           |   | ✔ | ✔ |
-| time delay / rate limiting                                       |   | ✔ | ✔ |
-| **huge** number of other options                                 |   |   | ✔ |
+|                                                          | feroxbuster | gobuster | ffuf |
+|------------------------------------------------------------------------------|---|---|---|
+| fast                                                                         | ✔ | ✔ | ✔ |
+| easy to use                                                                  | ✔ | ✔ |   |
+| allows recursion                                                             | ✔ |   | ✔ |
+| can specify query parameters                                                 | ✔ |   | ✔ |
+| SOCKS proxy support                                                          | ✔ |   |   |
+| multiple target scan (via stdin or multiple -u)                              | ✔ |   | ✔ |
+| configuration file for default value override                                | ✔ |   | ✔ |
+| can accept urls via STDIN as part of a pipeline                              | ✔ |   | ✔ |
+| can accept wordlists via STDIN                                               |   | ✔ | ✔ |
+| filter based on response size, wordcount, and linecount                      | ✔ |   | ✔ |
+| auto-filter wildcard responses                                               | ✔ |   | ✔ |
+| performs other scans (vhost, dns, etc)                                       |   | ✔ | ✔ |
+| time delay / rate limiting                                                   |   | ✔ | ✔ |
+| extracts links from response body to increase scan coverage (`v1.1.0`)       | ✔ |   |   |
+| limit number of concurrent recursive scans (`v1.2.0`)                        | ✔ |   |   |
+| filter out responses by status code (`v1.3.0`)                               | ✔ | ✔ | ✔ |
+| interactive pause and resume of active scan (`v1.4.0`)                       | ✔ |   |   |
+| replay only matched requests to a proxy (`v1.5.0`)                           | ✔ |   | ✔ |
+| filter out responses by line & word count (`v1.6.0`)                         | ✔ |   | ✔ |
+| json output (ffuf supports other formats as well) (`v1.7.0`)                 | ✔ |   | ✔ |
+| filter out responses by regular expression (`v1.8.0`)                        | ✔ |   | ✔ |
+| save scan's state to disk (can pick up where it left off) (`v1.9.0`)         | ✔ |   |   |
+| maximum run time limit (`v1.10.0`)                                           | ✔ |   | ✔ |
+| use robots.txt to increase scan coverage (`v1.10.2`)                         | ✔ |   |   |
+| use example page's response to fuzzily filter similar pages  (`v1.11.0`)     | ✔ |   |   |
+| cancel a recursive scan interactively (`v1.12.0`)                            | ✔ |   |   |
+| **huge** number of other options                                             |   |   | ✔ |
 
-Of note, there's another written-in-rust content discovery tool, [rustbuster](https://github.com/phra/rustbuster). I 
-came across rustbuster when I was naming my tool (😢). I don't have any experience using it, but it appears to 
-be able to do POST requests with an HTTP body, has SOCKS support, and has an 8.3 shortname scanner (in addition to vhost
-dns, directory, etc...).  In short, it definitely looks interesting and may be what you're looking for as it has some 
-capability I haven't seen in similar tools.  
+Of note, there's another written-in-rust content discovery tool, [rustbuster](https://github.com/phra/rustbuster). I
+came across rustbuster when I was naming my tool (😢). I don't have any experience using it, but it appears to be able
+to do POST requests with an HTTP body, has SOCKS support, and has an 8.3 shortname scanner (in addition to vhost dns,
+directory, etc...). In short, it definitely looks interesting and may be what you're looking for as it has some
+capability I haven't seen in similar tools.
 
 ## 🤯 Common Problems/Issues (FAQ)
 
@@ -531,21 +854,24 @@ Why do I get a bunch of `No file descriptors available (os error 24)` errors?
 
 ---
 
-There are a few potential causes of this error.  The simplest is that your operating system sets an open file limit that is aggressively low.  Through personal testing, I've found that `4096` is a reasonable open file limit (this will vary based on your exact setup).
+There are a few potential causes of this error. The simplest is that your operating system sets an open file limit that
+is aggressively low. Through personal testing, I've found that `4096` is a reasonable open file limit (this will vary
+based on your exact setup).
 
-There are quite a few options to solve this particular problem, of which a handful are shown below.  
+There are quite a few options to solve this particular problem, of which a handful are shown below.
 
 #### Increase the Number of Open Files
 
-We'll start by increasing the number of open files the OS allows. On my Kali install, the default was `1024`, and I know some MacOS installs use `256` 😕.
+We'll start by increasing the number of open files the OS allows. On my Kali install, the default was `1024`, and I know
+some MacOS installs use `256` 😕.
 
 ##### Edit `/etc/security/limits.conf`
 
-One option to up the limit is to edit `/etc/security/limits.conf` so that it includes the two lines below.  
+One option to up the limit is to edit `/etc/security/limits.conf` so that it includes the two lines below.
 
 - `*` represents all users
-- `hard` and `soft` indicate the hard and soft limits for the OS 
-- `nofile` is the number of open files option. 
+- `hard` and `soft` indicate the hard and soft limits for the OS
+- `nofile` is the number of open files option.
 
 ```
 /etc/security/limits.conf
@@ -566,20 +892,25 @@ ulimit -n 4096
 
 #### Additional Tweaks (may not be needed)
 
-If you still find yourself hitting the file limit with the above changes, there are a few additional tweaks that may help.  
+If you still find yourself hitting the file limit with the above changes, there are a few additional tweaks that may
+help.
 
-> This section was shamelessly stolen from this [stackoverflow answer](https://stackoverflow.com/a/3923785).  More information is included in that post and is recommended reading if you end up needing to use this section.
+> This section was shamelessly stolen from this [stackoverflow answer](https://stackoverflow.com/a/3923785). More information is included in that post and is recommended reading if you end up needing to use this section.
 
-✨ Special thanks to HTB user [@sparkla](https://www.hackthebox.eu/home/users/profile/221599) for their help with identifying these additional tweaks ✨
+✨ Special thanks to HTB user [@sparkla](https://www.hackthebox.eu/home/users/profile/221599) for their help with
+identifying these additional tweaks ✨
 
 ##### Increase the ephemeral port range, and decrease the tcp_fin_timeout.
 
-The ephermal port range defines the maximum number of outbound sockets a host can create from a particular I.P. address. The fin_timeout defines the minimum time these sockets will stay in TIME_WAIT state (unusable after being used once). Usual system defaults are
+The ephermal port range defines the maximum number of outbound sockets a host can create from a particular I.P. address.
+The fin_timeout defines the minimum time these sockets will stay in TIME_WAIT state (unusable after being used once).
+Usual system defaults are
 
 - `net.ipv4.ip_local_port_range = 32768   61000`
 - `net.ipv4.tcp_fin_timeout = 60`
 
-This basically means your system cannot consistently guarantee more than `(61000 - 32768) / 60 = 470` sockets per second.
+This basically means your system cannot consistently guarantee more than `(61000 - 32768) / 60 = 470` sockets per
+second.
 
 ```
 sudo sysctl net.ipv4.ip_local_port_range="15000 61000"
@@ -588,7 +919,9 @@ sudo sysctl net.ipv4.tcp_fin_timeout=30
 
 ##### Allow socket reuse while in a `TIME_WAIT` status
 
-This allows fast cycling of sockets in time_wait state and re-using them. Make sure to read post [Coping with the TCP TIME-WAIT](https://vincent.bernat.ch/en/blog/2014-tcp-time-wait-state-linux) from Vincent Bernat to understand the implications.
+This allows fast cycling of sockets in time_wait state and re-using them. Make sure to read
+post [Coping with the TCP TIME-WAIT](https://vincent.bernat.ch/en/blog/2014-tcp-time-wait-state-linux) from Vincent
+Bernat to understand the implications.
 
 ```
 sudo sysctl net.ipv4.tcp_tw_reuse=1 
@@ -596,9 +929,47 @@ sudo sysctl net.ipv4.tcp_tw_reuse=1
 
 ### Progress bars print one line at a time
 
-`feroxbuster` needs a terminal width of at least the size of what's being printed in order to do progress bar printing correctly.  If your width is too small, you may see output like what's shown below.
+`feroxbuster` needs a terminal width of at least the size of what's being printed in order to do progress bar printing
+correctly. If your width is too small, you may see output like what's shown below.
 
 ![small-term](img/small-term.png)
 
-If you can, simply make the terminal wider and rerun.  If you're unable to make your terminal wider
-consider using `-q` to suppress the progress bars.
+If you can, simply make the terminal wider and rerun. If you're unable to make your terminal wider consider using `-q`
+to suppress the progress bars.
+
+### What do each of the numbers beside the URL mean?
+
+Please refer to [this section](#filter-response-by-word-count--line-count--new-in-v160) where each number's meaning and
+how to use it to filter responses is discussed.
+
+### Connection closed before message completed
+
+The error in question can be boiled down to 'networking stuff'. `feroxbuster`
+uses [reqwest](https://docs.rs/reqwest/latest/) which uses [hyper](https://docs.rs/hyper/latest/hyper/) to make requests
+to the server. [This issue report](https://github.com/hyperium/hyper/issues/2136#issuecomment-589345238) to the hyper
+project explains what is happening (quoted below to save you a click). This isn't a bug so much as it's a
+target-specific tuning issue. When lowering the `-t` value, the error doesn't occur (or happens much less frequently).
+
+This isn't a bug. Simply slow down the scan. A `-t` value of 50 was chosen as a sane default that's still quite fast out
+of the box. However, network related errors may occur when the client and/or server become over-saturated.
+The [Threads and Connection Limits At A High-Level](#threads-and-connection-limits-at-a-high-level) section details how
+to accomplish per-target tuning.
+
+> This is just due to the racy nature of networking.
+>
+> hyper has a connection pool of idle connections, and it selected one to send your request. Most of the time, hyper will receive the server's FIN and drop the dead connection from its pool. But occasionally, a connection will be selected from the pool and written to at the same time the server is deciding to close the connection. Since hyper already wrote some of the request, it can't really retry it automatically on a new connection, since the server may have acted already.
+
+### SSL Error routines:tls_process_server_certificate:certificate verify failed
+
+In the event you see an error similar to
+
+![self-signed](img/insecure.png)
+
+```
+error trying to connect: error:1416F086:SSL routines:tls_process_server_certificate:certificate verify failed:ssl/statem/statem_clnt.c:1913: (self signed certificate)
+```
+
+You just need to add the `-k|--insecure` flag to your command.
+
+`feroxbuster` rejects self-signed certs and other "insecure" certificates/site configurations by default. You can choose
+to scan these services anyway by telling `feroxbuster` to ignore insecure server certs.

build.rs

@@ -0,0 +1,23 @@
+extern crate clap;
+
+use clap::Shell;
+
+include!("src/parser.rs");
+
+fn main() {
+    println!("cargo:rerun-if-env-changed=src/parser.rs");
+
+    if std::env::var("DOCS_RS").is_ok() {
+        return; // only build when we're not generating docs
+    }
+
+    let outdir = "shell_completions";
+
+    let mut app = initialize();
+
+    let shells: [Shell; 4] = [Shell::Bash, Shell::Fish, Shell::Zsh, Shell::PowerShell];
+
+    for shell in &shells {
+        app.gen_completions("feroxbuster", *shell, outdir);
+    }
+}

ferox-config.toml.example

@@ -18,7 +18,9 @@
 # verbosity = 1
 # scan_limit = 6
 # quiet = true
+# json = true
 # output = "/targets/ellingson_mineral_company/gibson.txt"
+# debug_log = "/var/log/find-the-derp.log"
 # user_agent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0"
 # redirects = true
 # insecure = true
@@ -30,7 +32,13 @@
 # extract_links = true
 # depth = 1
 # filter_size = [5174]
+# filter_regex = ["^ignore me$"]
+# filter_similar = ["https://somesite.com/soft404"]
+# filter_word_count = [993]
+# filter_line_count = [35, 36]
 # queries = [["name","value"], ["rick", "astley"]]
+# save_state = false
+# time_limit = "10m"
 
 # headers can be specified on multiple lines or as an inline table
 #

install-nix.sh

@@ -36,18 +36,23 @@ elif [[ "$(expr substr $(uname -s) 1 5)" == "Linux" ]]; then
         rm "${LIN64_ZIP}"
     fi
 
-    echo "[=] Installing Noto Emoji Font"
-    mkdir -p ~/.fonts
-    pushd ~/.fonts 2>&1 >/dev/null
+    if [[ -e ~/.fonts/NotoColorEmoji.ttf ]]; then
+       echo "[=] Found Noto Emoji Font, skipping install" 
+    else 
+        echo "[=] Installing Noto Emoji Font"
+        mkdir -p ~/.fonts
+        pushd ~/.fonts 2>&1 >/dev/null
 
-    curl -sLO "${EMOJI_URL}"
+        curl -sLO "${EMOJI_URL}"
 
-    fc-cache -f -v >/dev/null
+        fc-cache -f -v >/dev/null
 
-    popd 2>&1 >/dev/null
-    echo "[+] Noto Emoji Font installed"
+        popd 2>&1 >/dev/null
+        echo "[+] Noto Emoji Font installed"
+    fi
 fi
 
+
 chmod +x ./feroxbuster
 
 echo "[+] Installed feroxbuster version $(./feroxbuster -V)"

shell_completions/_feroxbuster

@@ -0,0 +1,96 @@
+#compdef feroxbuster
+
+autoload -U is-at-least
+
+_feroxbuster() {
+    typeset -A opt_args
+    typeset -a _arguments_options
+    local ret=1
+
+    if is-at-least 5.2; then
+        _arguments_options=(-s -S -C)
+    else
+        _arguments_options=(-s -C)
+    fi
+
+    local context curcontext="$curcontext" state line
+    _arguments "${_arguments_options[@]}" \
+'-w+[Path to the wordlist]' \
+'--wordlist=[Path to the wordlist]' \
+'*-u+[The target URL(s) (required, unless --stdin used)]' \
+'*--url=[The target URL(s) (required, unless --stdin used)]' \
+'-t+[Number of concurrent threads (default: 50)]' \
+'--threads=[Number of concurrent threads (default: 50)]' \
+'-d+[Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)]' \
+'--depth=[Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)]' \
+'-T+[Number of seconds before a request times out (default: 7)]' \
+'--timeout=[Number of seconds before a request times out (default: 7)]' \
+'-p+[Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)]' \
+'--proxy=[Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)]' \
+'-P+[Send only unfiltered requests through a Replay Proxy, instead of all requests]' \
+'--replay-proxy=[Send only unfiltered requests through a Replay Proxy, instead of all requests]' \
+'*-R+[Status Codes to send through a Replay Proxy when found (default: --status-codes value)]' \
+'*--replay-codes=[Status Codes to send through a Replay Proxy when found (default: --status-codes value)]' \
+'*-s+[Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)]' \
+'*--status-codes=[Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)]' \
+'-o+[Output file to write results to (use w/ --json for JSON entries)]' \
+'--output=[Output file to write results to (use w/ --json for JSON entries)]' \
+'(-u --url)--resume-from=[State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)]' \
+'--debug-log=[Output file to write log entries (use w/ --json for JSON entries)]' \
+'-a+[Sets the User-Agent (default: feroxbuster/VERSION)]' \
+'--user-agent=[Sets the User-Agent (default: feroxbuster/VERSION)]' \
+'*-x+[File extension(s) to search for (ex: -x php -x pdf js)]' \
+'*--extensions=[File extension(s) to search for (ex: -x php -x pdf js)]' \
+'*-H+[Specify HTTP headers (ex: -H Header:val '\''stuff: things'\'')]' \
+'*--headers=[Specify HTTP headers (ex: -H Header:val '\''stuff: things'\'')]' \
+'*-Q+[Specify URL query parameters (ex: -Q token=stuff -Q secret=key)]' \
+'*--query=[Specify URL query parameters (ex: -Q token=stuff -Q secret=key)]' \
+'*-S+[Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)]' \
+'*--filter-size=[Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)]' \
+'*-X+[Filter out messages via regular expression matching on the response'\''s body (ex: -X '\''^ignore me$'\'')]' \
+'*--filter-regex=[Filter out messages via regular expression matching on the response'\''s body (ex: -X '\''^ignore me$'\'')]' \
+'*-W+[Filter out messages of a particular word count (ex: -W 312 -W 91,82)]' \
+'*--filter-words=[Filter out messages of a particular word count (ex: -W 312 -W 91,82)]' \
+'*-N+[Filter out messages of a particular line count (ex: -N 20 -N 31,30)]' \
+'*--filter-lines=[Filter out messages of a particular line count (ex: -N 20 -N 31,30)]' \
+'*-C+[Filter out status codes (deny list) (ex: -C 200 -C 401)]' \
+'*--filter-status=[Filter out status codes (deny list) (ex: -C 200 -C 401)]' \
+'*--filter-similar-to=[Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)]' \
+'-L+[Limit total number of concurrent scans (default: 0, i.e. no limit)]' \
+'--scan-limit=[Limit total number of concurrent scans (default: 0, i.e. no limit)]' \
+'--time-limit=[Limit total run time of all scans (ex: --time-limit 10m)]' \
+'*-v[Increase verbosity level (use -vv or more for greater effect. \[CAUTION\] 4 -v'\''s is probably too much)]' \
+'*--verbosity[Increase verbosity level (use -vv or more for greater effect. \[CAUTION\] 4 -v'\''s is probably too much)]' \
+'-q[Only print URLs; Don'\''t print status codes, response size, running config, etc...]' \
+'--quiet[Only print URLs; Don'\''t print status codes, response size, running config, etc...]' \
+'--json[Emit JSON logs to --output and --debug-log instead of normal text]' \
+'-D[Don'\''t auto-filter wildcard responses]' \
+'--dont-filter[Don'\''t auto-filter wildcard responses]' \
+'-r[Follow redirects]' \
+'--redirects[Follow redirects]' \
+'-k[Disables TLS certificate validation]' \
+'--insecure[Disables TLS certificate validation]' \
+'-n[Do not scan recursively]' \
+'--no-recursion[Do not scan recursively]' \
+'(-x --extensions)-f[Append / to each request]' \
+'(-x --extensions)--add-slash[Append / to each request]' \
+'(-u --url)--stdin[Read url(s) from STDIN]' \
+'-e[Extract links from response body (html, javascript, etc...); make new requests based on findings (default: false)]' \
+'--extract-links[Extract links from response body (html, javascript, etc...); make new requests based on findings (default: false)]' \
+'-h[Prints help information]' \
+'--help[Prints help information]' \
+'-V[Prints version information]' \
+'--version[Prints version information]' \
+&& ret=0
+    
+}
+
+(( $+functions[_feroxbuster_commands] )) ||
+_feroxbuster_commands() {
+    local commands; commands=(
+        
+    )
+    _describe -t commands 'feroxbuster commands' commands "$@"
+}
+
+_feroxbuster "$@"

shell_completions/_feroxbuster.ps1

@@ -0,0 +1,95 @@
+
+using namespace System.Management.Automation
+using namespace System.Management.Automation.Language
+
+Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
+    param($wordToComplete, $commandAst, $cursorPosition)
+
+    $commandElements = $commandAst.CommandElements
+    $command = @(
+        'feroxbuster'
+        for ($i = 1; $i -lt $commandElements.Count; $i++) {
+            $element = $commandElements[$i]
+            if ($element -isnot [StringConstantExpressionAst] -or
+                $element.StringConstantType -ne [StringConstantType]::BareWord -or
+                $element.Value.StartsWith('-')) {
+                break
+        }
+        $element.Value
+    }) -join ';'
+
+    $completions = @(switch ($command) {
+        'feroxbuster' {
+            [CompletionResult]::new('-w', 'w', [CompletionResultType]::ParameterName, 'Path to the wordlist')
+            [CompletionResult]::new('--wordlist', 'wordlist', [CompletionResultType]::ParameterName, 'Path to the wordlist')
+            [CompletionResult]::new('-u', 'u', [CompletionResultType]::ParameterName, 'The target URL(s) (required, unless --stdin used)')
+            [CompletionResult]::new('--url', 'url', [CompletionResultType]::ParameterName, 'The target URL(s) (required, unless --stdin used)')
+            [CompletionResult]::new('-t', 't', [CompletionResultType]::ParameterName, 'Number of concurrent threads (default: 50)')
+            [CompletionResult]::new('--threads', 'threads', [CompletionResultType]::ParameterName, 'Number of concurrent threads (default: 50)')
+            [CompletionResult]::new('-d', 'd', [CompletionResultType]::ParameterName, 'Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)')
+            [CompletionResult]::new('--depth', 'depth', [CompletionResultType]::ParameterName, 'Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)')
+            [CompletionResult]::new('-T', 'T', [CompletionResultType]::ParameterName, 'Number of seconds before a request times out (default: 7)')
+            [CompletionResult]::new('--timeout', 'timeout', [CompletionResultType]::ParameterName, 'Number of seconds before a request times out (default: 7)')
+            [CompletionResult]::new('-p', 'p', [CompletionResultType]::ParameterName, 'Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)')
+            [CompletionResult]::new('--proxy', 'proxy', [CompletionResultType]::ParameterName, 'Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)')
+            [CompletionResult]::new('-P', 'P', [CompletionResultType]::ParameterName, 'Send only unfiltered requests through a Replay Proxy, instead of all requests')
+            [CompletionResult]::new('--replay-proxy', 'replay-proxy', [CompletionResultType]::ParameterName, 'Send only unfiltered requests through a Replay Proxy, instead of all requests')
+            [CompletionResult]::new('-R', 'R', [CompletionResultType]::ParameterName, 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)')
+            [CompletionResult]::new('--replay-codes', 'replay-codes', [CompletionResultType]::ParameterName, 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)')
+            [CompletionResult]::new('-s', 's', [CompletionResultType]::ParameterName, 'Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)')
+            [CompletionResult]::new('--status-codes', 'status-codes', [CompletionResultType]::ParameterName, 'Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)')
+            [CompletionResult]::new('-o', 'o', [CompletionResultType]::ParameterName, 'Output file to write results to (use w/ --json for JSON entries)')
+            [CompletionResult]::new('--output', 'output', [CompletionResultType]::ParameterName, 'Output file to write results to (use w/ --json for JSON entries)')
+            [CompletionResult]::new('--resume-from', 'resume-from', [CompletionResultType]::ParameterName, 'State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)')
+            [CompletionResult]::new('--debug-log', 'debug-log', [CompletionResultType]::ParameterName, 'Output file to write log entries (use w/ --json for JSON entries)')
+            [CompletionResult]::new('-a', 'a', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/VERSION)')
+            [CompletionResult]::new('--user-agent', 'user-agent', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/VERSION)')
+            [CompletionResult]::new('-x', 'x', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js)')
+            [CompletionResult]::new('--extensions', 'extensions', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js)')
+            [CompletionResult]::new('-H', 'H', [CompletionResultType]::ParameterName, 'Specify HTTP headers (ex: -H Header:val ''stuff: things'')')
+            [CompletionResult]::new('--headers', 'headers', [CompletionResultType]::ParameterName, 'Specify HTTP headers (ex: -H Header:val ''stuff: things'')')
+            [CompletionResult]::new('-Q', 'Q', [CompletionResultType]::ParameterName, 'Specify URL query parameters (ex: -Q token=stuff -Q secret=key)')
+            [CompletionResult]::new('--query', 'query', [CompletionResultType]::ParameterName, 'Specify URL query parameters (ex: -Q token=stuff -Q secret=key)')
+            [CompletionResult]::new('-S', 'S', [CompletionResultType]::ParameterName, 'Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)')
+            [CompletionResult]::new('--filter-size', 'filter-size', [CompletionResultType]::ParameterName, 'Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)')
+            [CompletionResult]::new('-X', 'X', [CompletionResultType]::ParameterName, 'Filter out messages via regular expression matching on the response''s body (ex: -X ''^ignore me$'')')
+            [CompletionResult]::new('--filter-regex', 'filter-regex', [CompletionResultType]::ParameterName, 'Filter out messages via regular expression matching on the response''s body (ex: -X ''^ignore me$'')')
+            [CompletionResult]::new('-W', 'W', [CompletionResultType]::ParameterName, 'Filter out messages of a particular word count (ex: -W 312 -W 91,82)')
+            [CompletionResult]::new('--filter-words', 'filter-words', [CompletionResultType]::ParameterName, 'Filter out messages of a particular word count (ex: -W 312 -W 91,82)')
+            [CompletionResult]::new('-N', 'N', [CompletionResultType]::ParameterName, 'Filter out messages of a particular line count (ex: -N 20 -N 31,30)')
+            [CompletionResult]::new('--filter-lines', 'filter-lines', [CompletionResultType]::ParameterName, 'Filter out messages of a particular line count (ex: -N 20 -N 31,30)')
+            [CompletionResult]::new('-C', 'C', [CompletionResultType]::ParameterName, 'Filter out status codes (deny list) (ex: -C 200 -C 401)')
+            [CompletionResult]::new('--filter-status', 'filter-status', [CompletionResultType]::ParameterName, 'Filter out status codes (deny list) (ex: -C 200 -C 401)')
+            [CompletionResult]::new('--filter-similar-to', 'filter-similar-to', [CompletionResultType]::ParameterName, 'Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)')
+            [CompletionResult]::new('-L', 'L', [CompletionResultType]::ParameterName, 'Limit total number of concurrent scans (default: 0, i.e. no limit)')
+            [CompletionResult]::new('--scan-limit', 'scan-limit', [CompletionResultType]::ParameterName, 'Limit total number of concurrent scans (default: 0, i.e. no limit)')
+            [CompletionResult]::new('--time-limit', 'time-limit', [CompletionResultType]::ParameterName, 'Limit total run time of all scans (ex: --time-limit 10m)')
+            [CompletionResult]::new('-v', 'v', [CompletionResultType]::ParameterName, 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v''s is probably too much)')
+            [CompletionResult]::new('--verbosity', 'verbosity', [CompletionResultType]::ParameterName, 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v''s is probably too much)')
+            [CompletionResult]::new('-q', 'q', [CompletionResultType]::ParameterName, 'Only print URLs; Don''t print status codes, response size, running config, etc...')
+            [CompletionResult]::new('--quiet', 'quiet', [CompletionResultType]::ParameterName, 'Only print URLs; Don''t print status codes, response size, running config, etc...')
+            [CompletionResult]::new('--json', 'json', [CompletionResultType]::ParameterName, 'Emit JSON logs to --output and --debug-log instead of normal text')
+            [CompletionResult]::new('-D', 'D', [CompletionResultType]::ParameterName, 'Don''t auto-filter wildcard responses')
+            [CompletionResult]::new('--dont-filter', 'dont-filter', [CompletionResultType]::ParameterName, 'Don''t auto-filter wildcard responses')
+            [CompletionResult]::new('-r', 'r', [CompletionResultType]::ParameterName, 'Follow redirects')
+            [CompletionResult]::new('--redirects', 'redirects', [CompletionResultType]::ParameterName, 'Follow redirects')
+            [CompletionResult]::new('-k', 'k', [CompletionResultType]::ParameterName, 'Disables TLS certificate validation')
+            [CompletionResult]::new('--insecure', 'insecure', [CompletionResultType]::ParameterName, 'Disables TLS certificate validation')
+            [CompletionResult]::new('-n', 'n', [CompletionResultType]::ParameterName, 'Do not scan recursively')
+            [CompletionResult]::new('--no-recursion', 'no-recursion', [CompletionResultType]::ParameterName, 'Do not scan recursively')
+            [CompletionResult]::new('-f', 'f', [CompletionResultType]::ParameterName, 'Append / to each request')
+            [CompletionResult]::new('--add-slash', 'add-slash', [CompletionResultType]::ParameterName, 'Append / to each request')
+            [CompletionResult]::new('--stdin', 'stdin', [CompletionResultType]::ParameterName, 'Read url(s) from STDIN')
+            [CompletionResult]::new('-e', 'e', [CompletionResultType]::ParameterName, 'Extract links from response body (html, javascript, etc...); make new requests based on findings (default: false)')
+            [CompletionResult]::new('--extract-links', 'extract-links', [CompletionResultType]::ParameterName, 'Extract links from response body (html, javascript, etc...); make new requests based on findings (default: false)')
+            [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Prints help information')
+            [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Prints help information')
+            [CompletionResult]::new('-V', 'V', [CompletionResultType]::ParameterName, 'Prints version information')
+            [CompletionResult]::new('--version', 'version', [CompletionResultType]::ParameterName, 'Prints version information')
+            break
+        }
+    })
+
+    $completions.Where{ $_.CompletionText -like "$wordToComplete*" } |
+        Sort-Object -Property ListItemText
+}

shell_completions/feroxbuster.bash

@@ -0,0 +1,217 @@
+_feroxbuster() {
+    local i cur prev opts cmds
+    COMPREPLY=()
+    cur="${COMP_WORDS[COMP_CWORD]}"
+    prev="${COMP_WORDS[COMP_CWORD-1]}"
+    cmd=""
+    opts=""
+
+    for i in ${COMP_WORDS[@]}
+    do
+        case "${i}" in
+            feroxbuster)
+                cmd="feroxbuster"
+                ;;
+            
+            *)
+                ;;
+        esac
+    done
+
+    case "${cmd}" in
+        feroxbuster)
+            opts=" -v -q -D -r -k -n -f -e -h -V -w -u -t -d -T -p -P -R -s -o -a -x -H -Q -S -X -W -N -C -L  --verbosity --quiet --json --dont-filter --redirects --insecure --no-recursion --add-slash --stdin --extract-links --help --version --wordlist --url --threads --depth --timeout --proxy --replay-proxy --replay-codes --status-codes --output --resume-from --debug-log --user-agent --extensions --headers --query --filter-size --filter-regex --filter-words --filter-lines --filter-status --filter-similar-to --scan-limit --time-limit  "
+            if [[ ${cur} == -* || ${COMP_CWORD} -eq 1 ]] ; then
+                COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
+                return 0
+            fi
+            case "${prev}" in
+                
+                --wordlist)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -w)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --url)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -u)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --threads)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -t)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --depth)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -d)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --timeout)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -T)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --proxy)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -p)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --replay-proxy)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -P)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --replay-codes)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -R)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --status-codes)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -s)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --output)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -o)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --resume-from)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --debug-log)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --user-agent)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -a)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --extensions)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -x)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --headers)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -H)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --query)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -Q)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --filter-size)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -S)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --filter-regex)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -X)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --filter-words)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -W)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --filter-lines)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -N)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --filter-status)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -C)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --filter-similar-to)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --scan-limit)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                    -L)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --time-limit)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                *)
+                    COMPREPLY=()
+                    ;;
+            esac
+            COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
+            return 0
+            ;;
+        
+    esac
+}
+
+complete -F _feroxbuster -o bashdefault -o default feroxbuster

shell_completions/feroxbuster.fish

@@ -0,0 +1,36 @@
+complete -c feroxbuster -n "__fish_use_subcommand" -s w -l wordlist -d 'Path to the wordlist'
+complete -c feroxbuster -n "__fish_use_subcommand" -s u -l url -d 'The target URL(s) (required, unless --stdin used)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s t -l threads -d 'Number of concurrent threads (default: 50)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s d -l depth -d 'Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s T -l timeout -d 'Number of seconds before a request times out (default: 7)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s p -l proxy -d 'Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s P -l replay-proxy -d 'Send only unfiltered requests through a Replay Proxy, instead of all requests'
+complete -c feroxbuster -n "__fish_use_subcommand" -s R -l replay-codes -d 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s s -l status-codes -d 'Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s o -l output -d 'Output file to write results to (use w/ --json for JSON entries)'
+complete -c feroxbuster -n "__fish_use_subcommand" -l resume-from -d 'State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)'
+complete -c feroxbuster -n "__fish_use_subcommand" -l debug-log -d 'Output file to write log entries (use w/ --json for JSON entries)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s a -l user-agent -d 'Sets the User-Agent (default: feroxbuster/VERSION)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s x -l extensions -d 'File extension(s) to search for (ex: -x php -x pdf js)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s H -l headers -d 'Specify HTTP headers (ex: -H Header:val \'stuff: things\')'
+complete -c feroxbuster -n "__fish_use_subcommand" -s Q -l query -d 'Specify URL query parameters (ex: -Q token=stuff -Q secret=key)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s S -l filter-size -d 'Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s X -l filter-regex -d 'Filter out messages via regular expression matching on the response\'s body (ex: -X \'^ignore me$\')'
+complete -c feroxbuster -n "__fish_use_subcommand" -s W -l filter-words -d 'Filter out messages of a particular word count (ex: -W 312 -W 91,82)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s N -l filter-lines -d 'Filter out messages of a particular line count (ex: -N 20 -N 31,30)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s C -l filter-status -d 'Filter out status codes (deny list) (ex: -C 200 -C 401)'
+complete -c feroxbuster -n "__fish_use_subcommand" -l filter-similar-to -d 'Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s L -l scan-limit -d 'Limit total number of concurrent scans (default: 0, i.e. no limit)'
+complete -c feroxbuster -n "__fish_use_subcommand" -l time-limit -d 'Limit total run time of all scans (ex: --time-limit 10m)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s v -l verbosity -d 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v\'s is probably too much)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s q -l quiet -d 'Only print URLs; Don\'t print status codes, response size, running config, etc...'
+complete -c feroxbuster -n "__fish_use_subcommand" -l json -d 'Emit JSON logs to --output and --debug-log instead of normal text'
+complete -c feroxbuster -n "__fish_use_subcommand" -s D -l dont-filter -d 'Don\'t auto-filter wildcard responses'
+complete -c feroxbuster -n "__fish_use_subcommand" -s r -l redirects -d 'Follow redirects'
+complete -c feroxbuster -n "__fish_use_subcommand" -s k -l insecure -d 'Disables TLS certificate validation'
+complete -c feroxbuster -n "__fish_use_subcommand" -s n -l no-recursion -d 'Do not scan recursively'
+complete -c feroxbuster -n "__fish_use_subcommand" -s f -l add-slash -d 'Append / to each request'
+complete -c feroxbuster -n "__fish_use_subcommand" -l stdin -d 'Read url(s) from STDIN'
+complete -c feroxbuster -n "__fish_use_subcommand" -s e -l extract-links -d 'Extract links from response body (html, javascript, etc...); make new requests based on findings (default: false)'
+complete -c feroxbuster -n "__fish_use_subcommand" -s h -l help -d 'Prints help information'
+complete -c feroxbuster -n "__fish_use_subcommand" -s V -l version -d 'Prints version information'

snapcraft.yaml

@@ -0,0 +1,41 @@
+name: feroxbuster
+version: git
+summary: A simple, fast, recursive content discovery tool written in Rust
+description: |
+  feroxbuster is a tool designed to perform Forced Browsing.
+
+  Forced browsing is an attack where the aim is to enumerate and access resources that are not referenced by the web application, but are still accessible by an attacker.
+
+  feroxbuster uses brute force combined with a wordlist to search for unlinked content in target directories. These resources may store sensitive information about web applications and operational systems, such as source code, credentials, internal network addressing, etc...
+
+  This attack is also known as Predictable Resource Location, File Enumeration, Directory Enumeration, and Resource Enumeration.
+
+
+base: core18
+
+plugs:
+  etc-feroxbuster:
+    interface: system-files
+    read:
+    - /etc/feroxbuster
+  dot-config-feroxbuster:
+    interface: personal-files
+    read:
+    - $HOME/.config/feroxbuster
+
+architectures:
+  - build-on: amd64
+  - build-on: i386
+
+parts:
+  feroxbuster:
+    plugin: rust
+    source: .
+
+apps:
+  feroxbuster:
+    command: bin/feroxbuster
+    plugs:
+      - etc-feroxbuster
+      - dot-config-feroxbuster
+      - network

src/banner.rs

@@ -1,9 +1,13 @@
-use crate::config::{Configuration, CONFIGURATION};
-use crate::utils::{make_request, status_colorizer};
-use console::style;
+use crate::{
+    config::{Configuration, CONFIGURATION},
+    statistics::StatCommand,
+    utils::{make_request, status_colorizer},
+};
+use console::{style, Emoji};
 use reqwest::{Client, Url};
 use serde_json::Value;
 use std::io::Write;
+use tokio::sync::mpsc::UnboundedSender;
 
 /// macro helper to abstract away repetitive string formatting
 macro_rules! format_banner_entry_helper {
@@ -67,8 +71,13 @@ enum UpdateStatus {
 /// ex: v1.1.0
 ///
 /// Returns `UpdateStatus`
-async fn needs_update(client: &Client, url: &str, bin_version: &str) -> UpdateStatus {
-    log::trace!("enter: needs_update({:?}, {})", client, url);
+async fn needs_update(
+    client: &Client,
+    url: &str,
+    bin_version: &str,
+    tx_stats: UnboundedSender<StatCommand>,
+) -> UpdateStatus {
+    log::trace!("enter: needs_update({:?}, {}, {:?})", client, url, tx_stats);
 
     let unknown = UpdateStatus::Unknown;
 
@@ -81,7 +90,7 @@ async fn needs_update(client: &Client, url: &str, bin_version: &str) -> UpdateSt
         }
     };
 
-    if let Ok(response) = make_request(&client, &api_url).await {
+    if let Ok(response) = make_request(&client, &api_url, tx_stats.clone()).await {
         let body = response.text().await.unwrap_or_default();
 
         let json_response: Value = serde_json::from_str(&body).unwrap_or_default();
@@ -126,11 +135,24 @@ async fn needs_update(client: &Client, url: &str, bin_version: &str) -> UpdateSt
     unknown
 }
 
+/// Simple wrapper for emoji or fallback when terminal doesn't support emoji
+fn format_emoji(emoji: &str) -> String {
+    let width = console::measure_text_width(emoji);
+    let pad_len = width * width;
+    let pad = format!("{:<pad_len$}", "\u{0020}", pad_len = pad_len);
+    Emoji(emoji, &pad).to_string()
+}
+
 /// Prints the banner to stdout.
 ///
 /// Only prints those settings which are either always present, or passed in by the user.
-pub async fn initialize<W>(targets: &[String], config: &Configuration, version: &str, mut writer: W)
-where
+pub async fn initialize<W>(
+    targets: &[String],
+    config: &Configuration,
+    version: &str,
+    mut writer: W,
+    tx_stats: UnboundedSender<StatCommand>,
+) where
     W: Write,
 {
     let artwork = format!(
@@ -138,11 +160,11 @@ where
  ___  ___  __   __     __      __         __   ___
 |__  |__  |__) |__) | /  `    /  \ \_/ | |  \ |__
 |    |___ |  \ |  \ | \__,    \__/ / \ | |__/ |___
-by Ben "epi" Risher {}                  ver: {}"#,
-        '\u{1F913}', version
+by Ben "epi" Risher {}                 ver: {}"#,
+        Emoji("🤓", &format!("{:<2}", "\u{0020}")),
+        version
     );
-
-    let status = needs_update(&CONFIGURATION.client, UPDATE_URL, version).await;
+    let status = needs_update(&CONFIGURATION.client, UPDATE_URL, version, tx_stats).await;
 
     let top = "───────────────────────────┬──────────────────────";
     let addl_section = "──────────────────────────────────────────────────";
@@ -156,7 +178,7 @@ by Ben "epi" Risher {}                  ver: {}"#,
         writeln!(
             &mut writer,
             "{}",
-            format_banner_entry!("\u{1F3af}", "Target Url", target)
+            format_banner_entry!(format_emoji("🎯"), "Target Url", target)
         )
         .unwrap_or_default(); // 🎯
     }
@@ -170,14 +192,14 @@ by Ben "epi" Risher {}                  ver: {}"#,
     writeln!(
         &mut writer,
         "{}",
-        format_banner_entry!("\u{1F680}", "Threads", config.threads)
+        format_banner_entry!(format_emoji("🚀"), "Threads", config.threads)
     )
     .unwrap_or_default(); // 🚀
 
     writeln!(
         &mut writer,
         "{}",
-        format_banner_entry!("\u{1f4d6}", "Wordlist", config.wordlist)
+        format_banner_entry!(format_emoji("📖"), "Wordlist", config.wordlist)
     )
     .unwrap_or_default(); // 📖
 
@@ -185,7 +207,7 @@ by Ben "epi" Risher {}                  ver: {}"#,
         &mut writer,
         "{}",
         format_banner_entry!(
-            "\u{1F197}",
+            format_emoji("🆗"),
             "Status Codes",
             format!("[{}]", codes.join(", "))
         )
@@ -205,7 +227,7 @@ by Ben "epi" Risher {}                  ver: {}"#,
             &mut writer,
             "{}",
             format_banner_entry!(
-                "\u{1f5d1}",
+                format_emoji("🗑"),
                 "Status Code Filters",
                 format!("[{}]", code_filters.join(", "))
             )
@@ -216,14 +238,14 @@ by Ben "epi" Risher {}                  ver: {}"#,
     writeln!(
         &mut writer,
         "{}",
-        format_banner_entry!("\u{1f4a5}", "Timeout (secs)", config.timeout)
+        format_banner_entry!(format_emoji("💥"), "Timeout (secs)", config.timeout)
     )
     .unwrap_or_default(); // 💥
 
     writeln!(
         &mut writer,
         "{}",
-        format_banner_entry!("\u{1F9a1}", "User-Agent", config.user_agent)
+        format_banner_entry!(format_emoji("🦡"), "User-Agent", config.user_agent)
     )
     .unwrap_or_default(); // 🦡
 
@@ -232,7 +254,7 @@ by Ben "epi" Risher {}                  ver: {}"#,
         writeln!(
             &mut writer,
             "{}",
-            format_banner_entry!("\u{1f489}", "Config File", config.config)
+            format_banner_entry!(format_emoji("💉"), "Config File", config.config)
         )
         .unwrap_or_default(); // 💉
     }
@@ -241,7 +263,7 @@ by Ben "epi" Risher {}                  ver: {}"#,
         writeln!(
             &mut writer,
             "{}",
-            format_banner_entry!("\u{1f48e}", "Proxy", config.proxy)
+            format_banner_entry!(format_emoji("💎"), "Proxy", config.proxy)
         )
         .unwrap_or_default(); // 💎
     }
@@ -255,7 +277,7 @@ by Ben "epi" Risher {}                  ver: {}"#,
         writeln!(
             &mut writer,
             "{}",
-            format_banner_entry!("\u{1f3a5}", "Replay Proxy", config.replay_proxy)
+            format_banner_entry!(format_emoji("🎥"), "Replay Proxy", config.replay_proxy)
         )
         .unwrap_or_default(); // 🎥
 
@@ -267,7 +289,7 @@ by Ben "epi" Risher {}                  ver: {}"#,
             &mut writer,
             "{}",
             format_banner_entry!(
-                "\u{1f4fc}",
+                format_emoji("📼"),
                 "Replay Proxy Codes",
                 format!("[{}]", replay_codes.join(", "))
             )
@@ -280,7 +302,7 @@ by Ben "epi" Risher {}                  ver: {}"#,
             writeln!(
                 &mut writer,
                 "{}",
-                format_banner_entry!("\u{1f92f}", "Header", name, value)
+                format_banner_entry!(format_emoji("🤯"), "Header", name, value)
             )
             .unwrap_or_default(); // 🤯
         }
@@ -291,28 +313,75 @@ by Ben "epi" Risher {}                  ver: {}"#,
             writeln!(
                 &mut writer,
                 "{}",
-                format_banner_entry!("\u{1f4a2}", "Size Filter", filter)
+                format_banner_entry!(format_emoji("💢"), "Size Filter", filter)
             )
             .unwrap_or_default(); // 💢
         }
     }
 
+    if !config.filter_similar.is_empty() {
+        for filter in &config.filter_similar {
+            writeln!(
+                &mut writer,
+                "{}",
+                format_banner_entry!(format_emoji("💢"), "Similarity Filter", filter)
+            )
+            .unwrap_or_default(); // 💢
+        }
+    }
+
+    for filter in &config.filter_word_count {
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!(format_emoji("💢"), "Word Count Filter", filter)
+        )
+        .unwrap_or_default(); // 💢
+    }
+
+    for filter in &config.filter_line_count {
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!(format_emoji("💢"), "Line Count Filter", filter)
+        )
+        .unwrap_or_default(); // 💢
+    }
+
+    for filter in &config.filter_regex {
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!(format_emoji("💢"), "Regex Filter", filter)
+        )
+        .unwrap_or_default(); // 💢
+    }
+
     if config.extract_links {
         writeln!(
             &mut writer,
             "{}",
-            format_banner_entry!("\u{1F50E}", "Extract Links", config.extract_links)
+            format_banner_entry!(format_emoji("🔎"), "Extract Links", config.extract_links)
         )
         .unwrap_or_default(); // 🔎
     }
 
+    if config.json {
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!(format_emoji("🧔"), "JSON Output", config.json)
+        )
+        .unwrap_or_default(); // 🧔
+    }
+
     if !config.queries.is_empty() {
         for query in &config.queries {
             writeln!(
                 &mut writer,
                 "{}",
                 format_banner_entry!(
-                    "\u{1f914}",
+                    format_emoji("🤔"),
                     "Query Parameter",
                     format!("{}={}", query.0, query.1)
                 )
@@ -325,17 +394,26 @@ by Ben "epi" Risher {}                  ver: {}"#,
         writeln!(
             &mut writer,
             "{}",
-            format_banner_entry!("\u{1f4be}", "Output File", config.output)
+            format_banner_entry!(format_emoji("💾"), "Output File", config.output)
         )
         .unwrap_or_default(); // 💾
     }
 
+    if !config.debug_log.is_empty() {
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!(format_emoji("🪲"), "Debugging Log", config.debug_log)
+        )
+        .unwrap_or_default(); // 🪲
+    }
+
     if !config.extensions.is_empty() {
         writeln!(
             &mut writer,
             "{}",
             format_banner_entry!(
-                "\u{1f4b2}",
+                format_emoji("💲"),
                 "Extensions",
                 format!("[{}]", config.extensions.join(", "))
             )
@@ -347,7 +425,7 @@ by Ben "epi" Risher {}                  ver: {}"#,
         writeln!(
             &mut writer,
             "{}",
-            format_banner_entry!("\u{1f513}", "Insecure", config.insecure)
+            format_banner_entry!(format_emoji("🔓"), "Insecure", config.insecure)
         )
         .unwrap_or_default(); // 🔓
     }
@@ -356,7 +434,7 @@ by Ben "epi" Risher {}                  ver: {}"#,
         writeln!(
             &mut writer,
             "{}",
-            format_banner_entry!("\u{1f4cd}", "Follow Redirects", config.redirects)
+            format_banner_entry!(format_emoji("📍"), "Follow Redirects", config.redirects)
         )
         .unwrap_or_default(); // 📍
     }
@@ -365,53 +443,31 @@ by Ben "epi" Risher {}                  ver: {}"#,
         writeln!(
             &mut writer,
             "{}",
-            format_banner_entry!("\u{1f92a}", "Filter Wildcards", !config.dont_filter)
+            format_banner_entry!(format_emoji("🤪"), "Filter Wildcards", !config.dont_filter)
         )
         .unwrap_or_default(); // 🤪
     }
 
-    match config.verbosity {
+    let volume = ["🔈", "🔉", "🔊", "📢"];
+    if let 1..=4 = config.verbosity {
         //speaker medium volume (increasing with verbosity to loudspeaker)
-        1 => {
-            writeln!(
-                &mut writer,
-                "{}",
-                format_banner_entry!("\u{1f508}", "Verbosity", config.verbosity)
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!(
+                format_emoji(volume[config.verbosity as usize - 1]),
+                "Verbosity",
+                config.verbosity
             )
-            .unwrap_or_default(); // 🔈
-        }
-        2 => {
-            writeln!(
-                &mut writer,
-                "{}",
-                format_banner_entry!("\u{1f509}", "Verbosity", config.verbosity)
-            )
-            .unwrap_or_default(); // 🔉
-        }
-        3 => {
-            writeln!(
-                &mut writer,
-                "{}",
-                format_banner_entry!("\u{1f50a}", "Verbosity", config.verbosity)
-            )
-            .unwrap_or_default(); // 🔊
-        }
-        4 => {
-            writeln!(
-                &mut writer,
-                "{}",
-                format_banner_entry!("\u{1f4e2}", "Verbosity", config.verbosity)
-            )
-            .unwrap_or_default(); // 📢
-        }
-        _ => {}
+        )
+        .unwrap_or_default();
     }
 
     if config.add_slash {
         writeln!(
             &mut writer,
             "{}",
-            format_banner_entry!("\u{1fa93}", "Add Slash", config.add_slash)
+            format_banner_entry!(format_emoji("🪓"), "Add Slash", config.add_slash)
         )
         .unwrap_or_default(); // 🪓
     }
@@ -421,14 +477,14 @@ by Ben "epi" Risher {}                  ver: {}"#,
             writeln!(
                 &mut writer,
                 "{}",
-                format_banner_entry!("\u{1f503}", "Recursion Depth", "INFINITE")
+                format_banner_entry!(format_emoji("🔃"), "Recursion Depth", "INFINITE")
             )
             .unwrap_or_default(); // 🔃
         } else {
             writeln!(
                 &mut writer,
                 "{}",
-                format_banner_entry!("\u{1f503}", "Recursion Depth", config.depth)
+                format_banner_entry!(format_emoji("🔃"), "Recursion Depth", config.depth)
             )
             .unwrap_or_default(); // 🔃
         }
@@ -436,7 +492,7 @@ by Ben "epi" Risher {}                  ver: {}"#,
         writeln!(
             &mut writer,
             "{}",
-            format_banner_entry!("\u{1f6ab}", "Do Not Recurse", config.no_recursion)
+            format_banner_entry!(format_emoji("🚫"), "Do Not Recurse", config.no_recursion)
         )
         .unwrap_or_default(); // 🚫
     }
@@ -445,17 +501,30 @@ by Ben "epi" Risher {}                  ver: {}"#,
         writeln!(
             &mut writer,
             "{}",
-            format_banner_entry!("\u{1f9a5}", "Concurrent Scan Limit", config.scan_limit)
+            format_banner_entry!(
+                format_emoji("🦥"),
+                "Concurrent Scan Limit",
+                config.scan_limit
+            )
         )
         .unwrap_or_default(); // 🦥
     }
 
+    if !CONFIGURATION.time_limit.is_empty() {
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!(format_emoji("🕖"), "Time Limit", config.time_limit)
+        )
+        .unwrap_or_default(); // 🕖
+    }
+
     if matches!(status, UpdateStatus::OutOfDate) {
         writeln!(
             &mut writer,
             "{}",
             format_banner_entry!(
-                "\u{1f389}",
+                format_emoji("🎉"),
                 "New Version Available",
                 "https://github.com/epi052/feroxbuster/releases/latest"
             )
@@ -467,85 +536,111 @@ by Ben "epi" Risher {}                  ver: {}"#,
     // ⏯
     writeln!(
         &mut writer,
-        " \u{23ef}   Press [{}] to {}|{} your scan",
+        " {}  Press [{}] to use the {}™",
+        format_emoji("🏁"),
         style("ENTER").yellow(),
-        style("pause").red(),
-        style("resume").green()
+        style("Scan Cancel Menu").bright().yellow(),
     )
     .unwrap_or_default();
+
     writeln!(&mut writer, "{}", addl_section).unwrap_or_default();
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::VERSION;
+    use crate::{FeroxChannel, VERSION};
     use httpmock::Method::GET;
-    use httpmock::{Mock, MockServer};
+    use httpmock::MockServer;
     use std::fs::read_to_string;
     use std::io::stderr;
     use std::time::Duration;
     use tempfile::NamedTempFile;
+    use tokio::sync::mpsc;
 
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// test to hit no execution of targets for loop in banner
     async fn banner_intialize_without_targets() {
         let config = Configuration::default();
-        initialize(&[], &config, VERSION, stderr()).await;
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
+        initialize(&[], &config, VERSION, stderr(), tx).await;
     }
 
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// test to hit no execution of statuscode for loop in banner
     async fn banner_intialize_without_status_codes() {
-        let mut config = Configuration::default();
-        config.status_codes = vec![];
+        let config = Configuration {
+            status_codes: vec![],
+            ..Default::default()
+        };
+
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
         initialize(
             &[String::from("http://localhost")],
             &config,
             VERSION,
             stderr(),
+            tx,
         )
         .await;
     }
 
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// test to hit an empty config file
     async fn banner_intialize_without_config_file() {
-        let mut config = Configuration::default();
-        config.config = String::new();
+        let config = Configuration {
+            config: String::new(),
+            ..Default::default()
+        };
+
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
         initialize(
             &[String::from("http://localhost")],
             &config,
             VERSION,
             stderr(),
+            tx,
         )
         .await;
     }
 
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// test to hit an empty config file
     async fn banner_intialize_without_queries() {
-        let mut config = Configuration::default();
-        config.queries = vec![(String::new(), String::new())];
+        let config = Configuration {
+            queries: vec![(String::new(), String::new())],
+            ..Default::default()
+        };
+
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
         initialize(
             &[String::from("http://localhost")],
             &config,
             VERSION,
             stderr(),
+            tx,
         )
         .await;
     }
 
-    #[tokio::test(core_threads = 1)]
+    #[ignore]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// test to show that a new version is available for download
     async fn banner_intialize_with_mismatched_version() {
         let config = Configuration::default();
         let file = NamedTempFile::new().unwrap();
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
         initialize(
             &[String::from("http://localhost")],
             &config,
             "mismatched-version",
             &file,
+            tx,
         )
         .await;
         let contents = read_to_string(file.path()).unwrap();
@@ -554,101 +649,105 @@ mod tests {
         assert!(contents.contains("https://github.com/epi052/feroxbuster/releases/latest"));
     }
 
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// test that
     async fn banner_needs_update_returns_unknown_with_bad_url() {
-        let result = needs_update(&CONFIGURATION.client, &"", VERSION).await;
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
+        let result = needs_update(&CONFIGURATION.client, &"", VERSION, tx).await;
         assert!(matches!(result, UpdateStatus::Unknown));
     }
 
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// test return value of good url to needs_update
     async fn banner_needs_update_returns_up_to_date() {
         let srv = MockServer::start();
 
-        let mock = Mock::new()
-            .expect_method(GET)
-            .expect_path("/latest")
-            .return_status(200)
-            .return_body("{\"tag_name\":\"v1.1.0\"}")
-            .create_on(&srv);
+        let mock = srv.mock(|when, then| {
+            when.method(GET).path("/latest");
+            then.status(200).body("{\"tag_name\":\"v1.1.0\"}");
+        });
 
-        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.1.0").await;
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
 
-        assert_eq!(mock.times_called(), 1);
+        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.1.0", tx).await;
+
+        assert_eq!(mock.hits(), 1);
         assert!(matches!(result, UpdateStatus::UpToDate));
     }
 
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// test return value of good url to needs_update that returns a newer version than current
     async fn banner_needs_update_returns_out_of_date() {
         let srv = MockServer::start();
 
-        let mock = Mock::new()
-            .expect_method(GET)
-            .expect_path("/latest")
-            .return_status(200)
-            .return_body("{\"tag_name\":\"v1.1.0\"}")
-            .create_on(&srv);
+        let mock = srv.mock(|when, then| {
+            when.method(GET).path("/latest");
+            then.status(200).body("{\"tag_name\":\"v1.1.0\"}");
+        });
 
-        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1").await;
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
 
-        assert_eq!(mock.times_called(), 1);
+        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1", tx).await;
+
+        assert_eq!(mock.hits(), 1);
         assert!(matches!(result, UpdateStatus::OutOfDate));
     }
 
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// test return value of good url that times out
     async fn banner_needs_update_returns_unknown_on_timeout() {
         let srv = MockServer::start();
 
-        let mock = Mock::new()
-            .expect_method(GET)
-            .expect_path("/latest")
-            .return_status(200)
-            .return_body("{\"tag_name\":\"v1.1.0\"}")
-            .return_with_delay(Duration::from_secs(8))
-            .create_on(&srv);
+        let mock = srv.mock(|when, then| {
+            when.method(GET).path("/latest");
+            then.status(200)
+                .body("{\"tag_name\":\"v1.1.0\"}")
+                .delay(Duration::from_secs(8));
+        });
 
-        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1").await;
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
 
-        assert_eq!(mock.times_called(), 1);
+        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1", tx).await;
+
+        assert_eq!(mock.hits(), 1);
         assert!(matches!(result, UpdateStatus::Unknown));
     }
 
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// test return value of good url with bad json response
     async fn banner_needs_update_returns_unknown_on_bad_json_response() {
         let srv = MockServer::start();
 
-        let mock = Mock::new()
-            .expect_method(GET)
-            .expect_path("/latest")
-            .return_status(200)
-            .return_body("not json")
-            .create_on(&srv);
+        let mock = srv.mock(|when, then| {
+            when.method(GET).path("/latest");
+            then.status(200).body("not json");
+        });
 
-        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1").await;
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
 
-        assert_eq!(mock.times_called(), 1);
+        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1", tx).await;
+
+        assert_eq!(mock.hits(), 1);
         assert!(matches!(result, UpdateStatus::Unknown));
     }
 
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// test return value of good url with json response that lacks the tag_name field
     async fn banner_needs_update_returns_unknown_on_json_without_correct_tag() {
         let srv = MockServer::start();
 
-        let mock = Mock::new()
-            .expect_method(GET)
-            .expect_path("/latest")
-            .return_status(200)
-            .return_body("{\"no tag_name\": \"doesn't exist\"}")
-            .create_on(&srv);
+        let mock = srv.mock(|when, then| {
+            when.method(GET).path("/latest");
+            then.status(200)
+                .body("{\"no tag_name\": \"doesn't exist\"}");
+        });
 
-        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1").await;
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
 
-        assert_eq!(mock.times_called(), 1);
+        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1", tx).await;
+
+        assert_eq!(mock.hits(), 1);
         assert!(matches!(result, UpdateStatus::Unknown));
     }
 }

src/client.rs

@@ -32,31 +32,33 @@ pub fn initialize(
         .default_headers(header_map)
         .redirect(policy);
 
-    let client = if proxy.is_some() && !proxy.unwrap().is_empty() {
-        match Proxy::all(proxy.unwrap()) {
-            Ok(proxy_obj) => client.proxy(proxy_obj),
-            Err(e) => {
-                eprintln!(
-                    "{} {} Could not add proxy ({:?}) to Client configuration",
-                    status_colorizer("ERROR"),
-                    module_colorizer("Client::initialize"),
-                    proxy
-                );
-                eprintln!(
-                    "{} {} {}",
-                    status_colorizer("ERROR"),
-                    module_colorizer("Client::initialize"),
-                    e
-                );
+    let client = match proxy {
+        // a proxy is specified, need to add it to the client
+        Some(some_proxy) => {
+            if !some_proxy.is_empty() {
+                // it's not an empty string
+                match Proxy::all(some_proxy) {
+                    Ok(proxy_obj) => client.proxy(proxy_obj),
+                    Err(e) => {
+                        eprintln!(
+                            "{} {} {}",
+                            status_colorizer("ERROR"),
+                            module_colorizer("Client::initialize"),
+                            e
+                        );
 
-                #[cfg(test)]
-                panic!();
-                #[cfg(not(test))]
-                exit(1);
+                        #[cfg(test)]
+                        panic!();
+                        #[cfg(not(test))]
+                        exit(1);
+                    }
+                }
+            } else {
+                client // Some("") was used?
             }
         }
-    } else {
-        client
+        // no proxy specified
+        None => client,
     };
 
     match client.build() {

src/config.rs

@@ -1,17 +1,23 @@
-use crate::utils::{module_colorizer, status_colorizer};
-use crate::{client, parser, progress};
-use crate::{DEFAULT_CONFIG_NAME, DEFAULT_STATUS_CODES, DEFAULT_WORDLIST, VERSION};
-use clap::value_t;
+use crate::{
+    client, parser,
+    progress::{add_bar, BarType},
+    scan_manager::resume_scan,
+    utils::{module_colorizer, status_colorizer},
+    FeroxSerialize, DEFAULT_CONFIG_NAME, DEFAULT_STATUS_CODES, DEFAULT_WORDLIST, VERSION,
+};
+use clap::{value_t, ArgMatches};
 use indicatif::{MultiProgress, ProgressBar, ProgressDrawTarget};
 use lazy_static::lazy_static;
 use reqwest::{Client, StatusCode};
-use serde::Deserialize;
-use std::collections::HashMap;
-use std::env::{current_dir, current_exe};
-use std::fs::read_to_string;
-use std::path::PathBuf;
+use serde::{Deserialize, Serialize};
 #[cfg(not(test))]
 use std::process::exit;
+use std::{
+    collections::HashMap,
+    env::{current_dir, current_exe},
+    fs::read_to_string,
+    path::PathBuf,
+};
 
 lazy_static! {
     /// Global configuration state
@@ -21,7 +27,33 @@ lazy_static! {
     pub static ref PROGRESS_BAR: MultiProgress = MultiProgress::with_draw_target(ProgressDrawTarget::stdout());
 
     /// Global progress bar that is only used for printing messages that don't jack up other bars
-    pub static ref PROGRESS_PRINTER: ProgressBar = progress::add_bar("", 0, true);
+    pub static ref PROGRESS_PRINTER: ProgressBar = add_bar("", 0, BarType::Hidden);
+}
+
+/// macro helper to abstract away repetitive configuration updates
+macro_rules! update_config_if_present {
+    ($c:expr, $m:ident, $v:expr, $t:ty) => {
+        match value_t!($m, $v, $t) {
+            Ok(value) => *$c = value, // Update value
+            Err(clap::Error {
+                kind: clap::ErrorKind::ArgumentNotFound,
+                message: _,
+                info: _,
+            }) => {
+                // Do nothing if argument not found
+            }
+            Err(e) => e.exit(), // Exit with error on parse error
+        }
+    };
+}
+
+/// macro helper to abstract away repetitive if not default: update checks
+macro_rules! update_if_not_default {
+    ($old:expr, $new:expr, $default:expr) => {
+        if $new != $default {
+            *$old = $new;
+        }
+    };
 }
 
 /// simple helper to clean up some code reuse below; panics under test / exits in prod
@@ -49,8 +81,12 @@ fn report_and_exit(err: &str) -> ! {
 /// In that order.
 ///
 /// Inspired by and derived from https://github.com/PhilipDaniels/rust-config-example
-#[derive(Debug, Clone, Deserialize)]
+#[derive(Debug, Clone, Deserialize, Serialize)]
 pub struct Configuration {
+    #[serde(rename = "type", default = "serialized_type")]
+    /// Name of this type of struct, used for serialization, i.e. `{"type":"configuration"}`
+    pub kind: String,
+
     /// Path to the wordlist
     #[serde(default = "wordlist")]
     pub wordlist: String,
@@ -59,11 +95,11 @@ pub struct Configuration {
     #[serde(default)]
     pub config: String,
 
-    /// Proxy to use for requests (ex: http(s)://host:port, socks5://host:port)
+    /// Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)
     #[serde(default)]
     pub proxy: String,
 
-    /// Replay Proxy to use for requests (ex: http(s)://host:port, socks5://host:port)
+    /// Replay Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)
     #[serde(default)]
     pub replay_proxy: String,
 
@@ -76,7 +112,7 @@ pub struct Configuration {
     pub status_codes: Vec<u16>,
 
     /// Status Codes to replay to the Replay Proxy (default: whatever is passed to --status-code)
-    #[serde(default)]
+    #[serde(default = "status_codes")]
     pub replay_codes: Vec<u16>,
 
     /// Status Codes to filter out (deny list)
@@ -107,10 +143,19 @@ pub struct Configuration {
     #[serde(default)]
     pub quiet: bool,
 
+    /// Store log output as NDJSON
+    #[serde(default)]
+    pub json: bool,
+
     /// Output file to write results to (default: stdout)
     #[serde(default)]
     pub output: String,
 
+    /// File in which to store debug output, used in conjunction with verbosity to dictate which
+    /// logs are written
+    #[serde(default)]
+    pub debug_log: String,
+
     /// Sets the User-Agent (default: feroxbuster/VERSION)
     #[serde(default = "user_agent")]
     pub user_agent: String,
@@ -163,20 +208,65 @@ pub struct Configuration {
     #[serde(default)]
     pub filter_size: Vec<u64>,
 
+    /// Filter out messages of a particular line count
+    #[serde(default)]
+    pub filter_line_count: Vec<usize>,
+
+    /// Filter out messages of a particular word count
+    #[serde(default)]
+    pub filter_word_count: Vec<usize>,
+
+    /// Filter out messages by regular expression
+    #[serde(default)]
+    pub filter_regex: Vec<String>,
+
     /// Don't auto-filter wildcard responses
     #[serde(default)]
     pub dont_filter: bool,
+
+    /// Scan started from a state file, not from CLI args
+    #[serde(default)]
+    pub resumed: bool,
+
+    /// Resume scan from this file
+    #[serde(default)]
+    pub resume_from: String,
+
+    /// Whether or not a scan's current state should be saved when user presses Ctrl+C
+    ///
+    /// Not configurable from CLI; can only be set from a config file
+    #[serde(default = "save_state")]
+    pub save_state: bool,
+
+    /// The maximum runtime for a scan, expressed as N[smdh] where N can be parsed into a
+    /// non-negative integer and the next character is either s, m, h, or d (case insensitive)
+    #[serde(default)]
+    pub time_limit: String,
+
+    /// Filter out response bodies that meet a certain threshold of similarity
+    #[serde(default)]
+    pub filter_similar: Vec<String>,
 }
 
-// functions timeout, threads, status_codes, user_agent, wordlist, and depth are used to provide
+// functions timeout, threads, status_codes, user_agent, wordlist, save_state, and depth are used to provide
 // defaults in the event that a ferox-config.toml is found but one or more of the values below
 // aren't listed in the config.  This way, we get the correct defaults upon Deserialization
 
+/// default Configuration type for use in json output
+fn serialized_type() -> String {
+    String::from("configuration")
+}
+
 /// default timeout value
 fn timeout() -> u64 {
     7
 }
 
+/// default save_state value
+fn save_state() -> bool {
+    true
+}
+
 /// default threads value
 fn threads() -> usize {
     50
@@ -214,8 +304,10 @@ impl Default for Configuration {
         let replay_client = None;
         let status_codes = status_codes();
         let replay_codes = status_codes.clone();
+        let kind = serialized_type();
 
         Configuration {
+            kind,
             client,
             timeout,
             user_agent,
@@ -224,7 +316,9 @@ impl Default for Configuration {
             replay_client,
             dont_filter: false,
             quiet: false,
+            resumed: false,
             stdin: false,
+            json: false,
             verbosity: 0,
             scan_limit: 0,
             add_slash: false,
@@ -232,15 +326,23 @@ impl Default for Configuration {
             redirects: false,
             no_recursion: false,
             extract_links: false,
+            save_state: true,
             proxy: String::new(),
             config: String::new(),
             output: String::new(),
+            debug_log: String::new(),
             target_url: String::new(),
+            time_limit: String::new(),
+            resume_from: String::new(),
             replay_proxy: String::new(),
             queries: Vec::new(),
             extensions: Vec::new(),
             filter_size: Vec::new(),
+            filter_regex: Vec::new(),
+            filter_line_count: Vec::new(),
+            filter_word_count: Vec::new(),
             filter_status: Vec::new(),
+            filter_similar: Vec::new(),
             headers: HashMap::new(),
             depth: depth(),
             threads: threads(),
@@ -265,19 +367,27 @@ impl Configuration {
     /// - **status_codes**: [`DEFAULT_RESPONSE_CODES`](constant.DEFAULT_RESPONSE_CODES.html)
     /// - **filter_status**: `None`
     /// - **output**: `None` (print to stdout)
+    /// - **debug_log**: `None`
     /// - **quiet**: `false`
-    /// - **user_agent**: `feroxer/VERSION`
+    /// - **save_state**: `true`
+    /// - **user_agent**: `feroxbuster/VERSION`
     /// - **insecure**: `false` (don't be insecure, i.e. don't allow invalid certs)
     /// - **extensions**: `None`
     /// - **filter_size**: `None`
+    /// - **filter_similar**: `None`
+    /// - **filter_regex**: `None`
+    /// - **filter_word_count**: `None`
+    /// - **filter_line_count**: `None`
     /// - **headers**: `None`
     /// - **queries**: `None`
     /// - **no_recursion**: `false` (recursively scan enumerated sub-directories)
     /// - **add_slash**: `false`
     /// - **stdin**: `false`
+    /// - **json**: `false`
     /// - **dont_filter**: `false` (auto filter wildcard responses)
     /// - **depth**: `4` (maximum recursion depth)
     /// - **scan_limit**: `0` (no limit on concurrent scans imposed)
+    /// - **time_limit**: `None` (no limit on length of scan imposed)
     /// - **replay_proxy**: `None` (no limit on concurrent scans imposed)
     /// - **replay_codes**: [`DEFAULT_RESPONSE_CODES`](constant.DEFAULT_RESPONSE_CODES.html)
     ///
@@ -301,13 +411,67 @@ impl Configuration {
     pub fn new() -> Self {
         // when compiling for test, we want to eliminate the runtime dependency of the parser
         if cfg!(test) {
-            return Configuration::default();
+            let test_config = Configuration {
+                save_state: false, // don't clutter up junk when testing
+                ..Default::default()
+            };
+            return test_config;
         }
 
+        let args = parser::initialize().get_matches();
+
         // Get the default configuration, this is what will apply if nothing
         // else is specified.
         let mut config = Configuration::default();
 
+        // read in all config files
+        Self::parse_config_files(&mut config);
+
+        // read in the user provided options, this produces a separate instance of Configuration
+        // in order to allow for potentially merging into a --resume-from Configuration
+        let cli_config = Self::parse_cli_args(&args);
+
+        // --resume-from used, need to first read the Configuration from disk, and then
+        // merge the cli_config into the resumed config
+        if let Some(filename) = args.value_of("resume_from") {
+            // when resuming a scan, instead of normal configuration loading, we just
+            // load the config from disk by calling resume_scan
+            let mut previous_config = resume_scan(filename);
+
+            // if any other arguments were passed on the command line, the theory is that the
+            // user meant to modify the previously cancelled/saved scan in some way that we
+            // should take into account
+            Self::merge_config(&mut previous_config, cli_config);
+
+            // the resumed flag isn't printed in the banner and really has no business being
+            // serialized or included in much of the usual config logic; simply setting it to true
+            // here and being done with it
+            previous_config.resumed = true;
+
+            // if the user used --stdin, we already have all the scans started (or complete), we
+            // need to flip stdin to false so that the 'read from stdin' logic doesn't fire (if
+            // not flipped to false, the program hangs waiting for input from stdin again)
+            previous_config.stdin = false;
+
+            // clients aren't serialized, have to remake them from the previous config
+            Self::try_rebuild_clients(&mut previous_config);
+
+            return previous_config;
+        }
+
+        // if we've gotten to this point in the code, --resume-from was not used, so we need to
+        // merge the cli options into the config file options and return the result
+        Self::merge_config(&mut config, cli_config);
+
+        // rebuild clients is the last step in either code branch
+        Self::try_rebuild_clients(&mut config);
+
+        config
+    }
+
+    /// Parse all possible versions of the ferox-config.toml file, adhering to the order of
+    /// precedence outlined above
+    fn parse_config_files(mut config: &mut Self) {
         // Next, we parse the ferox-config.toml file, if present and set the values
         // therein to overwrite our default values. Deserialized defaults are specified
         // in the Configuration struct so that we don't change anything that isn't
@@ -349,39 +513,24 @@ impl Configuration {
             let config_file = cwd.join(DEFAULT_CONFIG_NAME);
             Self::parse_and_merge_config(config_file, &mut config);
         }
+    }
 
-        let args = parser::initialize().get_matches();
+    /// Given a set of ArgMatches read from the CLI, update and return the default Configuration
+    /// settings
+    fn parse_cli_args(args: &ArgMatches) -> Self {
+        let mut config = Configuration::default();
 
-        // the .is_some appears clunky, but it allows default values to be incrementally
-        // overwritten from Struct defaults, to file config, to command line args, soooo ¯\_(ツ)_/¯
-        if args.value_of("threads").is_some() {
-            let threads = value_t!(args.value_of("threads"), usize).unwrap_or_else(|e| e.exit());
-            config.threads = threads;
-        }
+        update_config_if_present!(&mut config.threads, args, "threads", usize);
+        update_config_if_present!(&mut config.depth, args, "depth", usize);
+        update_config_if_present!(&mut config.scan_limit, args, "scan_limit", usize);
+        update_config_if_present!(&mut config.wordlist, args, "wordlist", String);
+        update_config_if_present!(&mut config.output, args, "output", String);
+        update_config_if_present!(&mut config.debug_log, args, "debug_log", String);
+        update_config_if_present!(&mut config.time_limit, args, "time_limit", String);
+        update_config_if_present!(&mut config.resume_from, args, "resume_from", String);
 
-        if args.value_of("depth").is_some() {
-            let depth = value_t!(args.value_of("depth"), usize).unwrap_or_else(|e| e.exit());
-            config.depth = depth;
-        }
-
-        if args.value_of("scan_limit").is_some() {
-            let scan_limit =
-                value_t!(args.value_of("scan_limit"), usize).unwrap_or_else(|e| e.exit());
-            config.scan_limit = scan_limit;
-        }
-
-        if args.value_of("wordlist").is_some() {
-            config.wordlist = String::from(args.value_of("wordlist").unwrap());
-        }
-
-        if args.value_of("output").is_some() {
-            config.output = String::from(args.value_of("output").unwrap());
-        }
-
-        if args.values_of("status_codes").is_some() {
-            config.status_codes = args
-                .values_of("status_codes")
-                .unwrap() // already known good
+        if let Some(arg) = args.values_of("status_codes") {
+            config.status_codes = arg
                 .map(|code| {
                     StatusCode::from_bytes(code.as_bytes())
                         .unwrap_or_else(|e| report_and_exit(&e.to_string()))
@@ -390,11 +539,9 @@ impl Configuration {
                 .collect();
         }
 
-        if args.values_of("replay_codes").is_some() {
+        if let Some(arg) = args.values_of("replay_codes") {
             // replay codes passed in by the user
-            config.replay_codes = args
-                .values_of("replay_codes")
-                .unwrap() // already known good
+            config.replay_codes = arg
                 .map(|code| {
                     StatusCode::from_bytes(code.as_bytes())
                         .unwrap_or_else(|e| report_and_exit(&e.to_string()))
@@ -406,10 +553,8 @@ impl Configuration {
             config.replay_codes = config.status_codes.clone();
         }
 
-        if args.values_of("filter_status").is_some() {
-            config.filter_status = args
-                .values_of("filter_status")
-                .unwrap() // already known good
+        if let Some(arg) = args.values_of("filter_status") {
+            config.filter_status = arg
                 .map(|code| {
                     StatusCode::from_bytes(code.as_bytes())
                         .unwrap_or_else(|e| report_and_exit(&e.to_string()))
@@ -418,20 +563,40 @@ impl Configuration {
                 .collect();
         }
 
-        if args.values_of("extensions").is_some() {
-            config.extensions = args
-                .values_of("extensions")
-                .unwrap()
-                .map(|val| val.to_string())
+        if let Some(arg) = args.values_of("extensions") {
+            config.extensions = arg.map(|val| val.to_string()).collect();
+        }
+
+        if let Some(arg) = args.values_of("filter_regex") {
+            config.filter_regex = arg.map(|val| val.to_string()).collect();
+        }
+
+        if let Some(arg) = args.values_of("filter_similar") {
+            config.filter_similar = arg.map(|val| val.to_string()).collect();
+        }
+
+        if let Some(arg) = args.values_of("filter_size") {
+            config.filter_size = arg
+                .map(|size| {
+                    size.parse::<u64>()
+                        .unwrap_or_else(|e| report_and_exit(&e.to_string()))
+                })
                 .collect();
         }
 
-        if args.values_of("filter_size").is_some() {
-            config.filter_size = args
-                .values_of("filter_size")
-                .unwrap() // already known good
+        if let Some(arg) = args.values_of("filter_words") {
+            config.filter_word_count = arg
                 .map(|size| {
-                    size.parse::<u64>()
+                    size.parse::<usize>()
+                        .unwrap_or_else(|e| report_and_exit(&e.to_string()))
+                })
+                .collect();
+        }
+
+        if let Some(arg) = args.values_of("filter_lines") {
+            config.filter_line_count = arg
+                .map(|size| {
+                    size.parse::<usize>()
                         .unwrap_or_else(|e| report_and_exit(&e.to_string()))
                 })
                 .collect();
@@ -442,11 +607,11 @@ impl Configuration {
             // consider a user specifying quiet = true in ferox-config.toml
             // if the line below is outside of the if, we'd overwrite true with
             // false if no -q is used on the command line
-            config.quiet = args.is_present("quiet");
+            config.quiet = true;
         }
 
         if args.is_present("dont_filter") {
-            config.dont_filter = args.is_present("dont_filter");
+            config.dont_filter = true;
         }
 
         if args.occurrences_of("verbosity") > 0 {
@@ -456,53 +621,45 @@ impl Configuration {
         }
 
         if args.is_present("no_recursion") {
-            config.no_recursion = args.is_present("no_recursion");
+            config.no_recursion = true;
         }
 
         if args.is_present("add_slash") {
-            config.add_slash = args.is_present("add_slash");
+            config.add_slash = true;
         }
 
         if args.is_present("extract_links") {
-            config.extract_links = args.is_present("extract_links");
+            config.extract_links = true;
+        }
+
+        if args.is_present("json") {
+            config.json = true;
         }
 
         if args.is_present("stdin") {
-            config.stdin = args.is_present("stdin");
-        } else {
-            config.target_url = String::from(args.value_of("url").unwrap());
+            config.stdin = true;
+        } else if let Some(url) = args.value_of("url") {
+            config.target_url = String::from(url);
         }
 
         ////
         // organizational breakpoint; all options below alter the Client configuration
         ////
-        if args.value_of("proxy").is_some() {
-            config.proxy = String::from(args.value_of("proxy").unwrap());
-        }
-
-        if args.value_of("replay_proxy").is_some() {
-            config.replay_proxy = String::from(args.value_of("replay_proxy").unwrap());
-        }
-
-        if args.value_of("user_agent").is_some() {
-            config.user_agent = String::from(args.value_of("user_agent").unwrap());
-        }
-
-        if args.value_of("timeout").is_some() {
-            let timeout = value_t!(args.value_of("timeout"), u64).unwrap_or_else(|e| e.exit());
-            config.timeout = timeout;
-        }
+        update_config_if_present!(&mut config.proxy, args, "proxy", String);
+        update_config_if_present!(&mut config.replay_proxy, args, "replay_proxy", String);
+        update_config_if_present!(&mut config.user_agent, args, "user_agent", String);
+        update_config_if_present!(&mut config.timeout, args, "timeout", u64);
 
         if args.is_present("redirects") {
-            config.redirects = args.is_present("redirects");
+            config.redirects = true;
         }
 
         if args.is_present("insecure") {
-            config.insecure = args.is_present("insecure");
+            config.insecure = true;
         }
 
-        if args.values_of("headers").is_some() {
-            for val in args.values_of("headers").unwrap() {
+        if let Some(headers) = args.values_of("headers") {
+            for val in headers {
                 let mut split_val = val.split(':');
 
                 // explicitly take first split value as header's name
@@ -515,8 +672,8 @@ impl Configuration {
             }
         }
 
-        if args.values_of("queries").is_some() {
-            for val in args.values_of("queries").unwrap() {
+        if let Some(queries) = args.values_of("queries") {
+            for val in queries {
                 // same basic logic used as reading in the headers HashMap above
                 let mut split_val = val.split('=');
 
@@ -528,50 +685,53 @@ impl Configuration {
             }
         }
 
-        // this if statement determines if we've gotten a Client configuration change from
-        // either the config file or command line arguments; if we have, we need to rebuild
-        // the client and store it in the config struct
-        if !config.proxy.is_empty()
-            || config.timeout != timeout()
-            || config.user_agent != user_agent()
-            || config.redirects
-            || config.insecure
-            || !config.headers.is_empty()
+        config
+    }
+
+    /// this function determines if we've gotten a Client configuration change from
+    /// either the config file or command line arguments; if we have, we need to rebuild
+    /// the client and store it in the config struct
+    fn try_rebuild_clients(configuration: &mut Configuration) {
+        if !configuration.proxy.is_empty()
+            || configuration.timeout != timeout()
+            || configuration.user_agent != user_agent()
+            || configuration.redirects
+            || configuration.insecure
+            || !configuration.headers.is_empty()
+            || configuration.resumed
         {
-            if config.proxy.is_empty() {
-                config.client = client::initialize(
-                    config.timeout,
-                    &config.user_agent,
-                    config.redirects,
-                    config.insecure,
-                    &config.headers,
+            if configuration.proxy.is_empty() {
+                configuration.client = client::initialize(
+                    configuration.timeout,
+                    &configuration.user_agent,
+                    configuration.redirects,
+                    configuration.insecure,
+                    &configuration.headers,
                     None,
                 )
             } else {
-                config.client = client::initialize(
-                    config.timeout,
-                    &config.user_agent,
-                    config.redirects,
-                    config.insecure,
-                    &config.headers,
-                    Some(&config.proxy),
+                configuration.client = client::initialize(
+                    configuration.timeout,
+                    &configuration.user_agent,
+                    configuration.redirects,
+                    configuration.insecure,
+                    &configuration.headers,
+                    Some(&configuration.proxy),
                 )
             }
         }
 
-        if !config.replay_proxy.is_empty() {
+        if !configuration.replay_proxy.is_empty() {
             // only set replay_client when replay_proxy is set
-            config.replay_client = Some(client::initialize(
-                config.timeout,
-                &config.user_agent,
-                config.redirects,
-                config.insecure,
-                &config.headers,
-                Some(&config.replay_proxy),
+            configuration.replay_client = Some(client::initialize(
+                configuration.timeout,
+                &configuration.user_agent,
+                configuration.redirects,
+                configuration.insecure,
+                &configuration.headers,
+                Some(&configuration.replay_proxy),
             ));
         }
-
-        config
     }
 
     /// Given a configuration file's location and an instance of `Configuration`, read in
@@ -595,32 +755,70 @@ impl Configuration {
     }
 
     /// Given two Configurations, overwrite `settings` with the fields found in `settings_to_merge`
-    fn merge_config(settings: &mut Self, settings_to_merge: Self) {
-        settings.threads = settings_to_merge.threads;
-        settings.wordlist = settings_to_merge.wordlist;
-        settings.status_codes = settings_to_merge.status_codes;
-        settings.proxy = settings_to_merge.proxy;
-        settings.timeout = settings_to_merge.timeout;
-        settings.verbosity = settings_to_merge.verbosity;
-        settings.quiet = settings_to_merge.quiet;
-        settings.output = settings_to_merge.output;
-        settings.user_agent = settings_to_merge.user_agent;
-        settings.redirects = settings_to_merge.redirects;
-        settings.insecure = settings_to_merge.insecure;
-        settings.extract_links = settings_to_merge.extract_links;
-        settings.extensions = settings_to_merge.extensions;
-        settings.headers = settings_to_merge.headers;
-        settings.queries = settings_to_merge.queries;
-        settings.no_recursion = settings_to_merge.no_recursion;
-        settings.add_slash = settings_to_merge.add_slash;
-        settings.stdin = settings_to_merge.stdin;
-        settings.depth = settings_to_merge.depth;
-        settings.filter_size = settings_to_merge.filter_size;
-        settings.filter_status = settings_to_merge.filter_status;
-        settings.dont_filter = settings_to_merge.dont_filter;
-        settings.scan_limit = settings_to_merge.scan_limit;
-        settings.replay_proxy = settings_to_merge.replay_proxy;
-        settings.replay_codes = settings_to_merge.replay_codes;
+    fn merge_config(conf: &mut Self, new: Self) {
+        // does not include the following Configuration fields, as they don't make sense here
+        //  - kind
+        //  - client
+        //  - replay_client
+        //  - resumed
+        //  - config
+        update_if_not_default!(&mut conf.target_url, new.target_url, "");
+        update_if_not_default!(&mut conf.time_limit, new.time_limit, "");
+        update_if_not_default!(&mut conf.proxy, new.proxy, "");
+        update_if_not_default!(&mut conf.verbosity, new.verbosity, 0);
+        update_if_not_default!(&mut conf.quiet, new.quiet, false);
+        update_if_not_default!(&mut conf.output, new.output, "");
+        update_if_not_default!(&mut conf.redirects, new.redirects, false);
+        update_if_not_default!(&mut conf.insecure, new.insecure, false);
+        update_if_not_default!(&mut conf.extract_links, new.extract_links, false);
+        update_if_not_default!(&mut conf.extensions, new.extensions, Vec::<String>::new());
+        update_if_not_default!(&mut conf.headers, new.headers, HashMap::new());
+        update_if_not_default!(&mut conf.queries, new.queries, Vec::new());
+        update_if_not_default!(&mut conf.no_recursion, new.no_recursion, false);
+        update_if_not_default!(&mut conf.add_slash, new.add_slash, false);
+        update_if_not_default!(&mut conf.stdin, new.stdin, false);
+        update_if_not_default!(&mut conf.filter_size, new.filter_size, Vec::<u64>::new());
+        update_if_not_default!(
+            &mut conf.filter_regex,
+            new.filter_regex,
+            Vec::<String>::new()
+        );
+        update_if_not_default!(
+            &mut conf.filter_similar,
+            new.filter_similar,
+            Vec::<String>::new()
+        );
+        update_if_not_default!(
+            &mut conf.filter_word_count,
+            new.filter_word_count,
+            Vec::<usize>::new()
+        );
+        update_if_not_default!(
+            &mut conf.filter_line_count,
+            new.filter_line_count,
+            Vec::<usize>::new()
+        );
+        update_if_not_default!(
+            &mut conf.filter_status,
+            new.filter_status,
+            Vec::<u16>::new()
+        );
+        update_if_not_default!(&mut conf.dont_filter, new.dont_filter, false);
+        update_if_not_default!(&mut conf.scan_limit, new.scan_limit, 0);
+        update_if_not_default!(&mut conf.replay_proxy, new.replay_proxy, "");
+        update_if_not_default!(&mut conf.debug_log, new.debug_log, "");
+        update_if_not_default!(&mut conf.resume_from, new.resume_from, "");
+        update_if_not_default!(&mut conf.json, new.json, false);
+
+        update_if_not_default!(&mut conf.timeout, new.timeout, timeout());
+        update_if_not_default!(&mut conf.user_agent, new.user_agent, user_agent());
+        update_if_not_default!(&mut conf.threads, new.threads, threads());
+        update_if_not_default!(&mut conf.depth, new.depth, depth());
+        update_if_not_default!(&mut conf.wordlist, new.wordlist, wordlist());
+        update_if_not_default!(&mut conf.status_codes, new.status_codes, status_codes());
+        // status_codes() is the default for replay_codes, if they're not provided
+        update_if_not_default!(&mut conf.replay_codes, new.replay_codes, status_codes());
+        update_if_not_default!(&mut conf.save_state, new.save_state, save_state());
     }
 
     /// If present, read in `DEFAULT_CONFIG_NAME` and deserialize the specified values
@@ -646,6 +844,47 @@ impl Configuration {
     }
 }
 
+/// Implementation of FeroxMessage
+impl FeroxSerialize for Configuration {
+    /// Simple wrapper around create_report_string
+    fn as_str(&self) -> String {
+        format!("{:#?}\n", *self)
+    }
+
+    /// Create an NDJSON representation of the current scan's Configuration
+    ///
+    /// (expanded for clarity)
+    /// ex:
+    /// {
+    ///    "type":"configuration",
+    ///    "wordlist":"test",
+    ///    "config":"/home/epi/.config/feroxbuster/ferox-config.toml",
+    ///    "proxy":"",
+    ///    "replay_proxy":"",
+    ///    "target_url":"https://localhost.com",
+    ///    "status_codes":[
+    ///       200,
+    ///       204,
+    ///       301,
+    ///       302,
+    ///       307,
+    ///       308,
+    ///       401,
+    ///       403,
+    ///       405
+    ///    ],
+    /// ...
+    /// }\n
+    fn as_json(&self) -> String {
+        if let Ok(mut json) = serde_json::to_string(&self) {
+            json.push('\n');
+            json
+        } else {
+            String::from("{\"error\":\"could not Configuration convert to json\"}")
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -665,7 +904,10 @@ mod tests {
             quiet = true
             verbosity = 1
             scan_limit = 6
+            time_limit = "10m"
             output = "/some/otherpath"
+            debug_log = "/yet/anotherpath"
+            resume_from = "/some/state/file"
             redirects = true
             insecure = true
             extensions = ["html", "php", "js"]
@@ -676,8 +918,14 @@ mod tests {
             stdin = true
             dont_filter = true
             extract_links = true
+            json = true
+            save_state = false
             depth = 1
             filter_size = [4120]
+            filter_regex = ["^ignore me$"]
+            filter_similar = ["https://somesite.com/soft404"]
+            filter_word_count = [994, 992]
+            filter_line_count = [34]
             filter_status = [201]
         "#;
         let tmp_dir = TempDir::new().unwrap();
@@ -693,6 +941,9 @@ mod tests {
         assert_eq!(config.wordlist, wordlist());
         assert_eq!(config.proxy, String::new());
         assert_eq!(config.target_url, String::new());
+        assert_eq!(config.time_limit, String::new());
+        assert_eq!(config.resume_from, String::new());
+        assert_eq!(config.debug_log, String::new());
         assert_eq!(config.config, String::new());
         assert_eq!(config.replay_proxy, String::new());
         assert_eq!(config.status_codes, status_codes());
@@ -706,6 +957,8 @@ mod tests {
         assert_eq!(config.quiet, false);
         assert_eq!(config.dont_filter, false);
         assert_eq!(config.no_recursion, false);
+        assert_eq!(config.json, false);
+        assert_eq!(config.save_state, true);
         assert_eq!(config.stdin, false);
         assert_eq!(config.add_slash, false);
         assert_eq!(config.redirects, false);
@@ -714,6 +967,10 @@ mod tests {
         assert_eq!(config.queries, Vec::new());
         assert_eq!(config.extensions, Vec::<String>::new());
         assert_eq!(config.filter_size, Vec::<u64>::new());
+        assert_eq!(config.filter_regex, Vec::<String>::new());
+        assert_eq!(config.filter_similar, Vec::<String>::new());
+        assert_eq!(config.filter_word_count, Vec::<usize>::new());
+        assert_eq!(config.filter_line_count, Vec::<usize>::new());
         assert_eq!(config.filter_status, Vec::<u16>::new());
         assert_eq!(config.headers, HashMap::new());
     }
@@ -725,6 +982,13 @@ mod tests {
         assert_eq!(config.wordlist, "/some/path");
     }
 
+    #[test]
+    /// parse the test config and see that the value parsed is correct
+    fn config_reads_debug_log() {
+        let config = setup_config_test();
+        assert_eq!(config.debug_log, "/yet/anotherpath");
+    }
+
     #[test]
     /// parse the test config and see that the value parsed is correct
     fn config_reads_status_codes() {
@@ -788,6 +1052,13 @@ mod tests {
         assert_eq!(config.quiet, true);
     }
 
+    #[test]
+    /// parse the test config and see that the value parsed is correct
+    fn config_reads_json() {
+        let config = setup_config_test();
+        assert_eq!(config.json, true);
+    }
+
     #[test]
     /// parse the test config and see that the value parsed is correct
     fn config_reads_verbosity() {
@@ -858,6 +1129,20 @@ mod tests {
         assert_eq!(config.extensions, vec!["html", "php", "js"]);
     }
 
+    #[test]
+    /// parse the test config and see that the value parsed is correct
+    fn config_reads_filter_regex() {
+        let config = setup_config_test();
+        assert_eq!(config.filter_regex, vec!["^ignore me$"]);
+    }
+
+    #[test]
+    /// parse the test config and see that the value parsed is correct
+    fn config_reads_filter_similar() {
+        let config = setup_config_test();
+        assert_eq!(config.filter_similar, vec!["https://somesite.com/soft404"]);
+    }
+
     #[test]
     /// parse the test config and see that the value parsed is correct
     fn config_reads_filter_size() {
@@ -865,6 +1150,20 @@ mod tests {
         assert_eq!(config.filter_size, vec![4120]);
     }
 
+    #[test]
+    /// parse the test config and see that the value parsed is correct
+    fn config_reads_filter_word_count() {
+        let config = setup_config_test();
+        assert_eq!(config.filter_word_count, vec![994, 992]);
+    }
+
+    #[test]
+    /// parse the test config and see that the value parsed is correct
+    fn config_reads_filter_line_count() {
+        let config = setup_config_test();
+        assert_eq!(config.filter_line_count, vec![34]);
+    }
+
     #[test]
     /// parse the test config and see that the value parsed is correct
     fn config_reads_filter_status() {
@@ -872,6 +1171,27 @@ mod tests {
         assert_eq!(config.filter_status, vec![201]);
     }
 
+    #[test]
+    /// parse the test config and see that the value parsed is correct
+    fn config_reads_save_state() {
+        let config = setup_config_test();
+        assert_eq!(config.save_state, false);
+    }
+
+    #[test]
+    /// parse the test config and see that the value parsed is correct
+    fn config_reads_time_limit() {
+        let config = setup_config_test();
+        assert_eq!(config.time_limit, "10m");
+    }
+
+    #[test]
+    /// parse the test config and see that the value parsed is correct
+    fn config_reads_resume_from() {
+        let config = setup_config_test();
+        assert_eq!(config.resume_from, "/some/state/file");
+    }
+
     #[test]
     /// parse the test config and see that the values parsed are correct
     fn config_reads_headers() {
@@ -898,4 +1218,32 @@ mod tests {
     fn config_report_and_exit_works() {
         report_and_exit("some message");
     }
+
+    #[test]
+    /// test as_str method of Configuration
+    fn as_str_returns_string_with_newline() {
+        let config = Configuration::new();
+        let config_str = config.as_str();
+        println!("{}", config_str);
+        assert!(config_str.starts_with("Configuration {"));
+        assert!(config_str.ends_with("}\n"));
+        assert!(config_str.contains("replay_codes:"));
+        assert!(config_str.contains("client: Client {"));
+        assert!(config_str.contains("user_agent: \"feroxbuster"));
+    }
+
+    #[test]
+    /// test as_json method of Configuration
+    fn as_json_returns_json_representation_of_configuration_with_newline() {
+        let mut config = Configuration::new();
+        config.timeout = 12;
+        config.depth = 2;
+        let config_str = config.as_json();
+        let json: Configuration = serde_json::from_str(&config_str).unwrap();
+        assert_eq!(json.config, config.config);
+        assert_eq!(json.wordlist, config.wordlist);
+        assert_eq!(json.replay_codes, config.replay_codes);
+        assert_eq!(json.timeout, config.timeout);
+        assert_eq!(json.depth, config.depth);
+    }
 }

src/extractor.rs

@@ -1,17 +1,37 @@
-use crate::FeroxResponse;
+use crate::{
+    client,
+    config::{Configuration, CONFIGURATION},
+    scanner::SCANNED_URLS,
+    statistics::{
+        StatCommand::{self, UpdateUsizeField},
+        StatField::{LinksExtracted, TotalExpected},
+    },
+    utils::{format_url, make_request},
+    FeroxResponse,
+};
 use lazy_static::lazy_static;
 use regex::Regex;
 use reqwest::Url;
 use std::collections::HashSet;
+use tokio::sync::mpsc::UnboundedSender;
 
 /// Regular expression used in [LinkFinder](https://github.com/GerbenJavado/LinkFinder)
 ///
 /// Incorporates change from this [Pull Request](https://github.com/GerbenJavado/LinkFinder/pull/66/files)
 const LINKFINDER_REGEX: &str = r#"(?:"|')(((?:[a-zA-Z]{1,10}://|//)[^"'/]{1,}\.[a-zA-Z]{2,}[^"']{0,})|((?:/|\.\./|\./)[^"'><,;| *()(%%$^/\\\[\]][^"'><,;|()]{1,})|([a-zA-Z0-9_\-/]{1,}/[a-zA-Z0-9_\-/]{1,}\.(?:[a-zA-Z]{1,4}|action)(?:[\?|#][^"|']{0,}|))|([a-zA-Z0-9_\-/]{1,}/[a-zA-Z0-9_\-/]{3,}(?:[\?|#][^"|']{0,}|))|([a-zA-Z0-9_\-.]{1,}\.(?:php|asp|aspx|jsp|json|action|html|js|txt|xml)(?:[\?|#][^"|']{0,}|)))(?:"|')"#;
 
+/// Regular expression to pull url paths from robots.txt
+///
+/// ref: https://developers.google.com/search/reference/robots_txt
+const ROBOTS_TXT_REGEX: &str =
+    r#"(?m)^ *(Allow|Disallow): *(?P<url_path>[a-zA-Z0-9._/?#@!&'()+,;%=-]+?)$"#; // multi-line (?m)
+
 lazy_static! {
     /// `LINKFINDER_REGEX` as a regex::Regex type
-    static ref REGEX: Regex = Regex::new(LINKFINDER_REGEX).unwrap();
+    static ref LINKS_REGEX: Regex = Regex::new(LINKFINDER_REGEX).unwrap();
+
+    /// `ROBOTS_TXT_REGEX` as a regex::Regex type
+    static ref ROBOTS_REGEX: Regex = Regex::new(ROBOTS_TXT_REGEX).unwrap();
 }
 
 /// Iterate over a given path, return a list of every sub-path found
@@ -32,20 +52,27 @@ fn get_sub_paths_from_path(path: &str) -> Vec<String> {
 
     let length = parts.len();
 
-    for _ in 0..length {
+    for i in 0..length {
         // iterate over all parts of the path
         if parts.is_empty() {
             // pop left us with an empty vector, we're done
             break;
         }
 
-        let possible_path = parts.join("/");
+        let mut possible_path = parts.join("/");
 
         if possible_path.is_empty() {
             // .join can result in an empty string, which we don't need, ignore
             continue;
         }
 
+        if i > 0 {
+            // this isn't the last index of the parts array
+            // ex: /buried/misc/stupidfile.php
+            // this block skips the file but sees all parent folders
+            possible_path = format!("{}/", possible_path);
+        }
+
         paths.push(possible_path); // good sub-path found
         parts.pop(); // use .pop() to remove the last part of the path and continue iteration
     }
@@ -83,14 +110,21 @@ fn add_link_to_set_of_links(link: &str, url: &Url, links: &mut HashSet<String>)
 ///         - homepage/assets/img/
 ///         - homepage/assets/
 ///         - homepage/
-pub async fn get_links(response: &FeroxResponse) -> HashSet<String> {
-    log::trace!("enter: get_links({})", response.url().as_str());
+pub async fn get_links(
+    response: &FeroxResponse,
+    tx_stats: UnboundedSender<StatCommand>,
+) -> HashSet<String> {
+    log::trace!(
+        "enter: get_links({}, {:?})",
+        response.url().as_str(),
+        tx_stats
+    );
 
     let mut links = HashSet::<String>::new();
 
     let body = response.text();
 
-    for capture in REGEX.captures_iter(&body) {
+    for capture in LINKS_REGEX.captures_iter(&body) {
         // remove single & double quotes from both ends of the capture
         // capture[0] is the entire match, additional capture groups start at [1]
         let link = capture[0].trim_matches(|c| c == '\'' || c == '"');
@@ -105,27 +139,14 @@ pub async fn get_links(response: &FeroxResponse) -> HashSet<String> {
                     continue;
                 }
 
-                for sub_path in get_sub_paths_from_path(absolute.path()) {
-                    // take a url fragment like homepage/assets/img/icons/handshake.svg and
-                    // incrementally add
-                    //     - homepage/assets/img/icons/
-                    //     - homepage/assets/img/
-                    //     - homepage/assets/
-                    //     - homepage/
-                    log::debug!("Adding {} to {:?}", sub_path, links);
-                    add_link_to_set_of_links(&sub_path, &response.url(), &mut links);
-                }
+                add_all_sub_paths(absolute.path(), &response, &mut links);
             }
             Err(e) => {
                 // this is the expected error that happens when we try to parse a url fragment
                 //     ex: Url::parse("/login") -> Err("relative URL without a base")
                 // while this is technically an error, these are good results for us
                 if e.to_string().contains("relative URL without a base") {
-                    for sub_path in get_sub_paths_from_path(link) {
-                        // incrementally save all sub-paths that led to the relative url's resource
-                        log::debug!("Adding {} to {:?}", sub_path, links);
-                        add_link_to_set_of_links(&sub_path, &response.url(), &mut links);
-                    }
+                    add_all_sub_paths(link, &response, &mut links);
                 } else {
                     // unexpected error has occurred
                     log::error!("Could not parse given url: {}", e);
@@ -134,7 +155,193 @@ pub async fn get_links(response: &FeroxResponse) -> HashSet<String> {
         }
     }
 
+    let multiplier = CONFIGURATION.extensions.len().max(1);
+
+    update_stat!(tx_stats, UpdateUsizeField(LinksExtracted, links.len()));
+    update_stat!(
+        tx_stats,
+        UpdateUsizeField(TotalExpected, links.len() * multiplier)
+    );
+
     log::trace!("exit: get_links -> {:?}", links);
+
+    links
+}
+
+/// take a url fragment like homepage/assets/img/icons/handshake.svg and
+/// incrementally add
+///     - homepage/assets/img/icons/
+///     - homepage/assets/img/
+///     - homepage/assets/
+///     - homepage/
+fn add_all_sub_paths(url_path: &str, response: &FeroxResponse, mut links: &mut HashSet<String>) {
+    log::trace!(
+        "enter: add_all_sub_paths({}, {}, {:?})",
+        url_path,
+        response,
+        links
+    );
+
+    for sub_path in get_sub_paths_from_path(url_path) {
+        log::debug!("Adding {} to {:?}", sub_path, links);
+        add_link_to_set_of_links(&sub_path, &response.url(), &mut links);
+    }
+
+    log::trace!("exit: add_all_sub_paths");
+}
+
+/// Wrapper around link extraction logic
+/// currently used in two places:
+///   - links from response bodys
+///   - links from robots.txt responses
+///
+/// general steps taken:
+///   - create a new Url object based on cli options/args
+///   - check if the new Url has already been seen/scanned -> None
+///   - make a request to the new Url ? -> Some(response) : None
+pub async fn request_feroxresponse_from_new_link(
+    url: &str,
+    tx_stats: UnboundedSender<StatCommand>,
+) -> Option<FeroxResponse> {
+    log::trace!(
+        "enter: request_feroxresponse_from_new_link({}, {:?})",
+        url,
+        tx_stats
+    );
+
+    // create a url based on the given command line options, return None on error
+    let new_url = match format_url(
+        &url,
+        &"",
+        CONFIGURATION.add_slash,
+        &CONFIGURATION.queries,
+        None,
+        tx_stats.clone(),
+    ) {
+        Ok(url) => url,
+        Err(_) => {
+            log::trace!("exit: request_feroxresponse_from_new_link -> None");
+            return None;
+        }
+    };
+
+    if SCANNED_URLS.get_scan_by_url(&new_url.to_string()).is_some() {
+        //we've seen the url before and don't need to scan again
+        log::trace!("exit: request_feroxresponse_from_new_link -> None");
+        return None;
+    }
+
+    // make the request and store the response
+    let new_response = match make_request(&CONFIGURATION.client, &new_url, tx_stats).await {
+        Ok(resp) => resp,
+        Err(_) => {
+            log::trace!("exit: request_feroxresponse_from_new_link -> None");
+            return None;
+        }
+    };
+
+    let new_ferox_response = FeroxResponse::from(new_response, true).await;
+
+    log::trace!(
+        "exit: request_feroxresponse_from_new_link -> {:?}",
+        new_ferox_response
+    );
+    Some(new_ferox_response)
+}
+
+/// helper function that simply requests /robots.txt on the given url's base url
+///
+/// example:
+///     http://localhost/api/users -> http://localhost/robots.txt
+///     
+/// The length of the given path has no effect on what's requested; it's always
+/// base url + /robots.txt
+pub async fn request_robots_txt(
+    base_url: &str,
+    config: &Configuration,
+    tx_stats: UnboundedSender<StatCommand>,
+) -> Option<FeroxResponse> {
+    log::trace!(
+        "enter: get_robots_file({}, CONFIGURATION, {:?})",
+        base_url,
+        tx_stats
+    );
+
+    // more often than not, domain/robots.txt will redirect to www.domain/robots.txt or something
+    // similar; to account for that, create a client that will follow redirects, regardless of
+    // what the user specified for the scanning client. Other than redirects, it will respect
+    // all other user specified settings
+    let follow_redirects = true;
+
+    let proxy = if config.proxy.is_empty() {
+        None
+    } else {
+        Some(config.proxy.as_str())
+    };
+
+    let client = client::initialize(
+        config.timeout,
+        &config.user_agent,
+        follow_redirects,
+        config.insecure,
+        &config.headers,
+        proxy,
+    );
+
+    if let Ok(mut url) = Url::parse(base_url) {
+        url.set_path("/robots.txt"); // overwrite existing path with /robots.txt
+
+        if let Ok(response) = make_request(&client, &url, tx_stats).await {
+            let ferox_response = FeroxResponse::from(response, true).await;
+
+            log::trace!("exit: get_robots_file -> {}", ferox_response);
+            return Some(ferox_response);
+        }
+    }
+
+    None
+}
+
+/// Entry point to perform link extraction from robots.txt
+///
+/// `base_url` can have paths and subpaths, however robots.txt will be requested from the
+/// root of the url
+/// given the url:
+///     http://localhost/stuff/things
+/// this function requests:
+///     http://localhost/robots.txt
+pub async fn extract_robots_txt(
+    base_url: &str,
+    config: &Configuration,
+    tx_stats: UnboundedSender<StatCommand>,
+) -> HashSet<String> {
+    log::trace!(
+        "enter: extract_robots_txt({}, CONFIGURATION, {:?})",
+        base_url,
+        tx_stats
+    );
+    let mut links = HashSet::new();
+
+    if let Some(response) = request_robots_txt(&base_url, &config, tx_stats.clone()).await {
+        for capture in ROBOTS_REGEX.captures_iter(response.text.as_str()) {
+            if let Some(new_path) = capture.name("url_path") {
+                if let Ok(mut new_url) = Url::parse(base_url) {
+                    new_url.set_path(new_path.as_str());
+                    add_all_sub_paths(new_url.path(), &response, &mut links);
+                }
+            }
+        }
+    }
+
+    let multiplier = CONFIGURATION.extensions.len().max(1);
+
+    update_stat!(tx_stats, UpdateUsizeField(LinksExtracted, links.len()));
+    update_stat!(
+        tx_stats,
+        UpdateUsizeField(TotalExpected, links.len() * multiplier)
+    );
+
+    log::trace!("exit: extract_robots_txt -> {:?}", links);
     links
 }
 
@@ -142,9 +349,11 @@ pub async fn get_links(response: &FeroxResponse) -> HashSet<String> {
 mod tests {
     use super::*;
     use crate::utils::make_request;
+    use crate::FeroxChannel;
     use httpmock::Method::GET;
-    use httpmock::{Mock, MockServer};
+    use httpmock::MockServer;
     use reqwest::Client;
+    use tokio::sync::mpsc;
 
     #[test]
     /// extract sub paths from the given url fragment; expect 4 sub paths and that all are
@@ -153,10 +362,10 @@ mod tests {
         let path = "homepage/assets/img/icons/handshake.svg";
         let paths = get_sub_paths_from_path(&path);
         let expected = vec![
-            "homepage",
-            "homepage/assets",
-            "homepage/assets/img",
-            "homepage/assets/img/icons",
+            "homepage/",
+            "homepage/assets/",
+            "homepage/assets/img/",
+            "homepage/assets/img/icons/",
             "homepage/assets/img/icons/handshake.svg",
         ];
 
@@ -173,7 +382,7 @@ mod tests {
     fn extractor_get_sub_paths_from_path_with_enclosing_slashes() {
         let path = "/homepage/assets/";
         let paths = get_sub_paths_from_path(&path);
-        let expected = vec!["homepage", "homepage/assets"];
+        let expected = vec!["homepage/", "homepage/assets"];
 
         assert_eq!(paths.len(), expected.len());
         for expected_path in expected {
@@ -237,7 +446,7 @@ mod tests {
         assert!(links.is_empty());
     }
 
-    #[tokio::test(core_threads = 1)]
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// use make_request to generate a Response, and use the Response to test get_links;
     /// the response will contain an absolute path to a domain that is not part of the scanned
     /// domain; expect an empty set returned
@@ -245,25 +454,51 @@ mod tests {
     ) -> Result<(), Box<dyn std::error::Error>> {
         let srv = MockServer::start();
 
-        let mock = Mock::new()
-            .expect_method(GET)
-            .expect_path("/some-path")
-            .return_status(200)
-            .return_body("\"http://defintely.not.a.thing.probably.com/homepage/assets/img/icons/handshake.svg\"")
-            .create_on(&srv);
+        let mock = srv.mock(|when, then|{
+            when.method(GET)
+                .path("/some-path");
+            then.status(200)
+                .body("\"http://defintely.not.a.thing.probably.com/homepage/assets/img/icons/handshake.svg\"");
+        });
 
         let client = Client::new();
         let url = Url::parse(&srv.url("/some-path")).unwrap();
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
 
-        let response = make_request(&client, &url).await.unwrap();
+        let response = make_request(&client, &url, tx.clone()).await.unwrap();
 
         let ferox_response = FeroxResponse::from(response, true).await;
 
-        let links = get_links(&ferox_response).await;
+        let links = get_links(&ferox_response, tx).await;
 
         assert!(links.is_empty());
 
-        assert_eq!(mock.times_called(), 1);
+        assert_eq!(mock.hits(), 1);
         Ok(())
     }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// test that /robots.txt is correctly requested given a base url (happy path)
+    async fn request_robots_txt_with_and_without_proxy() {
+        let srv = MockServer::start();
+
+        let mock = srv.mock(|when, then| {
+            when.method(GET).path("/robots.txt");
+            then.status(200).body("this is a test");
+        });
+
+        let mut config = Configuration::default();
+
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
+        request_robots_txt(&srv.url("/api/users/stuff/things"), &config, tx.clone()).await;
+
+        // note: the proxy doesn't actually do anything other than hit a different code branch
+        // in this unit test; it would however have an effect on an integration test
+        config.proxy = srv.url("/ima-proxy");
+
+        request_robots_txt(&srv.url("/api/different/path"), &config, tx).await;
+
+        assert_eq!(mock.hits(), 2);
+    }
 }

src/filters/lines.rs

@@ -0,0 +1,33 @@
+use super::*;
+
+/// Simple implementor of FeroxFilter; used to filter out responses based on the number of lines
+/// in a Response body; specified using -N|--filter-lines
+#[derive(Default, Debug, PartialEq)]
+pub struct LinesFilter {
+    /// Number of lines in a Response's body that should be filtered
+    pub line_count: usize,
+}
+
+/// implementation of FeroxFilter for LinesFilter
+impl FeroxFilter for LinesFilter {
+    /// Check `line_count` against what was passed in via -N|--filter-lines
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        let result = response.line_count() == self.line_count;
+
+        log::trace!("exit: should_filter_response -> {}", result);
+
+        result
+    }
+
+    /// Compare one LinesFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/filters/mod.rs

@@ -0,0 +1,24 @@
+//! module containing all of feroxbuster's filters
+mod traits;
+mod wildcard;
+mod status_code;
+mod words;
+mod lines;
+mod size;
+mod regex;
+mod similarity;
+#[cfg(test)]
+mod tests;
+
+pub use self::lines::LinesFilter;
+pub use self::regex::RegexFilter;
+pub use self::similarity::SimilarityFilter;
+pub use self::size::SizeFilter;
+pub use self::status_code::StatusCodeFilter;
+pub use self::traits::FeroxFilter;
+pub use self::wildcard::WildcardFilter;
+pub use self::words::WordsFilter;
+
+use crate::{config::CONFIGURATION, utils::get_url_path_length, FeroxResponse, FeroxSerialize};
+use std::any::Any;
+use std::fmt::Debug;

src/filters/regex.rs

@@ -0,0 +1,46 @@
+use super::*;
+use ::regex::Regex;
+
+/// Simple implementor of FeroxFilter; used to filter out responses based on a given regular
+/// expression; specified using -X|--filter-regex
+#[derive(Debug)]
+pub struct RegexFilter {
+    /// Regular expression to be applied to the response body for filtering, compiled
+    pub compiled: Regex,
+
+    /// Regular expression as passed in on the command line, not compiled
+    pub raw_string: String,
+}
+
+/// implementation of FeroxFilter for RegexFilter
+impl FeroxFilter for RegexFilter {
+    /// Check `expression` against the response body, if the expression matches, the response
+    /// should be filtered out
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        let result = self.compiled.is_match(response.text());
+
+        log::trace!("exit: should_filter_response -> {}", result);
+
+        result
+    }
+
+    /// Compare one SizeFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
+
+/// PartialEq implementation for RegexFilter
+impl PartialEq for RegexFilter {
+    /// Simple comparison of the raw string passed in via the command line
+    fn eq(&self, other: &RegexFilter) -> bool {
+        self.raw_string == other.raw_string
+    }
+}

src/filters/similarity.rs

@@ -0,0 +1,40 @@
+use super::*;
+use fuzzyhash::FuzzyHash;
+
+/// Simple implementor of FeroxFilter; used to filter out responses based on the similarity of a
+/// Response body with a known response; specified using --filter-similar-to
+#[derive(Default, Debug, PartialEq)]
+pub struct SimilarityFilter {
+    /// Response's body to be used for comparison for similarity
+    pub text: String,
+
+    /// Percentage of similarity at which a page is determined to be a near-duplicate of another
+    pub threshold: u32,
+}
+
+/// implementation of FeroxFilter for SimilarityFilter
+impl FeroxFilter for SimilarityFilter {
+    /// Check `FeroxResponse::text` against what was requested from the site passed in via
+    /// --filter-similar-to
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        let other = FuzzyHash::new(&response.text);
+
+        if let Ok(result) = FuzzyHash::compare(&self.text, &other.to_string()) {
+            return result >= self.threshold;
+        }
+
+        // couldn't hash the response, don't filter
+        log::warn!("Could not hash body from {}", response.as_str());
+        false
+    }
+
+    /// Compare one SimilarityFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/filters/size.rs

@@ -0,0 +1,33 @@
+use super::*;
+
+/// Simple implementor of FeroxFilter; used to filter out responses based on the length of a
+/// Response body; specified using -S|--filter-size
+#[derive(Default, Debug, PartialEq)]
+pub struct SizeFilter {
+    /// Overall length of a Response's body that should be filtered
+    pub content_length: u64,
+}
+
+/// implementation of FeroxFilter for SizeFilter
+impl FeroxFilter for SizeFilter {
+    /// Check `content_length` against what was passed in via -S|--filter-size
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        let result = response.content_length() == self.content_length;
+
+        log::trace!("exit: should_filter_response -> {}", result);
+
+        result
+    }
+
+    /// Compare one SizeFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/filters/status_code.rs

@@ -0,0 +1,40 @@
+use super::*;
+
+/// Simple implementor of FeroxFilter; used to filter out status codes specified using
+/// -C|--filter-status
+#[derive(Default, Debug, PartialEq)]
+pub struct StatusCodeFilter {
+    /// Status code that should not be displayed to the user
+    pub filter_code: u16,
+}
+
+/// implementation of FeroxFilter for StatusCodeFilter
+impl FeroxFilter for StatusCodeFilter {
+    /// Check `filter_code` against what was passed in via -C|--filter-status
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        if response.status().as_u16() == self.filter_code {
+            log::debug!(
+                "filtered out {} based on --filter-status of {}",
+                response.url(),
+                self.filter_code
+            );
+            log::trace!("exit: should_filter_response -> true");
+            return true;
+        }
+
+        log::trace!("exit: should_filter_response -> false");
+        false
+    }
+
+    /// Compare one StatusCodeFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/filters/tests.rs

@@ -0,0 +1,188 @@
+use super::*;
+use ::fuzzyhash::FuzzyHash;
+use ::regex::Regex;
+use reqwest::Url;
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn lines_filter_as_any() {
+    let filter = LinesFilter { line_count: 1 };
+
+    assert_eq!(filter.line_count, 1);
+    assert_eq!(
+        *filter.as_any().downcast_ref::<LinesFilter>().unwrap(),
+        filter
+    );
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn words_filter_as_any() {
+    let filter = WordsFilter { word_count: 1 };
+
+    assert_eq!(filter.word_count, 1);
+    assert_eq!(
+        *filter.as_any().downcast_ref::<WordsFilter>().unwrap(),
+        filter
+    );
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn size_filter_as_any() {
+    let filter = SizeFilter { content_length: 1 };
+
+    assert_eq!(filter.content_length, 1);
+    assert_eq!(
+        *filter.as_any().downcast_ref::<SizeFilter>().unwrap(),
+        filter
+    );
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn status_code_filter_as_any() {
+    let filter = StatusCodeFilter { filter_code: 200 };
+
+    assert_eq!(filter.filter_code, 200);
+    assert_eq!(
+        *filter.as_any().downcast_ref::<StatusCodeFilter>().unwrap(),
+        filter
+    );
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn regex_filter_as_any() {
+    let raw = r".*\.txt$";
+    let compiled = Regex::new(raw).unwrap();
+    let filter = RegexFilter {
+        compiled,
+        raw_string: raw.to_string(),
+    };
+
+    assert_eq!(filter.raw_string, r".*\.txt$");
+    assert_eq!(
+        *filter.as_any().downcast_ref::<RegexFilter>().unwrap(),
+        filter
+    );
+}
+
+#[test]
+/// test should_filter on WilcardFilter where static logic matches
+fn wildcard_should_filter_when_static_wildcard_found() {
+    let resp = FeroxResponse {
+        text: String::new(),
+        wildcard: true,
+        url: Url::parse("http://localhost").unwrap(),
+        content_length: 100,
+        word_count: 50,
+        line_count: 25,
+        headers: reqwest::header::HeaderMap::new(),
+        status: reqwest::StatusCode::OK,
+    };
+
+    let filter = WildcardFilter {
+        size: 100,
+        dynamic: 0,
+    };
+
+    assert!(filter.should_filter_response(&resp));
+}
+
+#[test]
+/// test should_filter on WilcardFilter where dynamic logic matches
+fn wildcard_should_filter_when_dynamic_wildcard_found() {
+    let resp = FeroxResponse {
+        text: String::new(),
+        wildcard: true,
+        url: Url::parse("http://localhost/stuff").unwrap(),
+        content_length: 100,
+        word_count: 50,
+        line_count: 25,
+        headers: reqwest::header::HeaderMap::new(),
+        status: reqwest::StatusCode::OK,
+    };
+
+    let filter = WildcardFilter {
+        size: 0,
+        dynamic: 95,
+    };
+
+    assert!(filter.should_filter_response(&resp));
+}
+
+#[test]
+/// test should_filter on RegexFilter where regex matches body
+fn regexfilter_should_filter_when_regex_matches_on_response_body() {
+    let resp = FeroxResponse {
+        text: String::from("im a body response hurr durr!"),
+        wildcard: false,
+        url: Url::parse("http://localhost/stuff").unwrap(),
+        content_length: 100,
+        word_count: 50,
+        line_count: 25,
+        headers: reqwest::header::HeaderMap::new(),
+        status: reqwest::StatusCode::OK,
+    };
+
+    let raw = r"response...rr";
+
+    let filter = RegexFilter {
+        raw_string: raw.to_string(),
+        compiled: Regex::new(raw).unwrap(),
+    };
+
+    assert!(filter.should_filter_response(&resp));
+}
+
+#[test]
+/// a few simple tests for similarity filter
+fn similarity_filter_is_accurate() {
+    let mut resp = FeroxResponse {
+        text: String::from("sitting"),
+        wildcard: false,
+        url: Url::parse("http://localhost/stuff").unwrap(),
+        content_length: 100,
+        word_count: 50,
+        line_count: 25,
+        headers: reqwest::header::HeaderMap::new(),
+        status: reqwest::StatusCode::OK,
+    };
+
+    let mut filter = SimilarityFilter {
+        text: FuzzyHash::new("kitten").to_string(),
+        threshold: 95,
+    };
+
+    // kitten/sitting is 57% similar, so a threshold of 95 should not be filtered
+    assert!(!filter.should_filter_response(&resp));
+
+    resp.text = String::new();
+    filter.text = String::new();
+    filter.threshold = 100;
+
+    // two empty strings are the same, however ssdeep doesn't accept empty strings, expect false
+    assert!(!filter.should_filter_response(&resp));
+
+    resp.text = String::from("some data to hash for the purposes of running a test");
+    filter.text = FuzzyHash::new("some data to hash for the purposes of running a te").to_string();
+    filter.threshold = 17;
+
+    assert!(filter.should_filter_response(&resp));
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn similarity_filter_as_any() {
+    let filter = SimilarityFilter {
+        text: String::from("stuff"),
+        threshold: 95,
+    };
+
+    assert_eq!(filter.text, "stuff");
+    assert_eq!(
+        *filter.as_any().downcast_ref::<SimilarityFilter>().unwrap(),
+        filter
+    );
+}

src/filters/traits.rs

@@ -0,0 +1,27 @@
+use super::*;
+
+// references:
+//   https://dev.to/magnusstrale/rust-trait-objects-in-a-vector-non-trivial-4co5
+//   https://stackoverflow.com/questions/25339603/how-to-test-for-equality-between-trait-objects
+
+/// FeroxFilter trait; represents different types of possible filters that can be applied to
+/// responses
+pub trait FeroxFilter: Debug + Send + Sync {
+    /// Determine whether or not this particular filter should be applied or not
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool;
+
+    /// delegates to the FeroxFilter-implementing type which gives us the actual type of self
+    fn box_eq(&self, other: &dyn Any) -> bool;
+
+    /// gives us `other` as Any in box_eq
+    fn as_any(&self) -> &dyn Any;
+}
+
+/// implementation of PartialEq, necessary long-form due to "trait cannot be made into an object"
+/// error when attempting to derive PartialEq on the trait itself
+impl PartialEq for Box<dyn FeroxFilter> {
+    /// Perform a comparison of two implementors of the FeroxFilter trait
+    fn eq(&self, other: &Box<dyn FeroxFilter>) -> bool {
+        self.box_eq(other.as_any())
+    }
+}

src/filters/wildcard.rs

@@ -1,34 +1,4 @@
-use crate::config::CONFIGURATION;
-use crate::utils::get_url_path_length;
-use crate::FeroxResponse;
-use std::any::Any;
-use std::fmt::Debug;
-
-// references:
-//   https://dev.to/magnusstrale/rust-trait-objects-in-a-vector-non-trivial-4co5
-//   https://stackoverflow.com/questions/25339603/how-to-test-for-equality-between-trait-objects
-
-/// FeroxFilter trait; represents different types of possible filters that can be applied to
-/// responses
-pub trait FeroxFilter: Debug + Send + Sync {
-    /// Determine whether or not this particular filter should be applied or not
-    fn should_filter_response(&self, response: &FeroxResponse) -> bool;
-
-    /// delegates to the FeroxFilter-implementing type which gives us the actual type of self
-    fn box_eq(&self, other: &dyn Any) -> bool;
-
-    /// gives us `other` as Any in box_eq
-    fn as_any(&self) -> &dyn Any;
-}
-
-/// implementation of PartialEq, necessary long-form due to "trait cannot be made into an object"
-/// error when attempting to derive PartialEq on the trait itself
-impl PartialEq for Box<dyn FeroxFilter> {
-    /// Perform a comparison of two implementors of the FeroxFilter trait
-    fn eq(&self, other: &Box<dyn FeroxFilter>) -> bool {
-        self.box_eq(other.as_any())
-    }
-}
+use super::*;
 
 /// Data holder for two pieces of data needed when auto-filtering out wildcard responses
 ///
@@ -53,7 +23,7 @@ impl FeroxFilter for WildcardFilter {
     /// Examine size, dynamic, and content_len to determine whether or not the response received
     /// is a wildcard response and therefore should be filtered out
     fn should_filter_response(&self, response: &FeroxResponse) -> bool {
-        log::trace!("enter: should_filter_response({:?} {:?})", self, response);
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
 
         // quick return if dont_filter is set
         if CONFIGURATION.dont_filter {
@@ -101,42 +71,3 @@ impl FeroxFilter for WildcardFilter {
         self
     }
 }
-
-/// Simple implementor of FeroxFilter; used to filter out status codes specified using
-/// -C|--filter-status
-#[derive(Default, Debug, PartialEq)]
-pub struct StatusCodeFilter {
-    /// Status code that should not be displayed to the user
-    pub filter_code: u16,
-}
-
-/// implementation of FeroxFilter for StatusCodeFilter
-impl FeroxFilter for StatusCodeFilter {
-    /// Check `filter_code` against what was passed in via -C|--filter-status
-    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
-        log::trace!("enter: should_filter_response({:?} {:?})", self, response);
-
-        if response.status().as_u16() == self.filter_code {
-            log::debug!(
-                "filtered out {} based on --filter-status of {}",
-                response.url(),
-                self.filter_code
-            );
-            log::trace!("exit: should_filter_response -> true");
-            return true;
-        }
-
-        log::trace!("exit: should_filter_response -> false");
-        false
-    }
-
-    /// Compare one StatusCodeFilter to another
-    fn box_eq(&self, other: &dyn Any) -> bool {
-        other.downcast_ref::<Self>().map_or(false, |a| self == a)
-    }
-
-    /// Return self as Any for dynamic dispatch purposes
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-}

src/filters/words.rs

@@ -0,0 +1,33 @@
+use super::*;
+
+/// Simple implementor of FeroxFilter; used to filter out responses based on the number of words
+/// in a Response body; specified using -W|--filter-words
+#[derive(Default, Debug, PartialEq)]
+pub struct WordsFilter {
+    /// Number of words in a Response's body that should be filtered
+    pub word_count: usize,
+}
+
+/// implementation of FeroxFilter for WordsFilter
+impl FeroxFilter for WordsFilter {
+    /// Check `word_count` against what was passed in via -W|--filter-words
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        let result = response.word_count() == self.word_count;
+
+        log::trace!("exit: should_filter_response -> {}", result);
+
+        result
+    }
+
+    /// Compare one WordsFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/heuristics.rs

@@ -2,10 +2,8 @@ use crate::{
     config::{CONFIGURATION, PROGRESS_PRINTER},
     filters::WildcardFilter,
     scanner::should_filter_response,
-    utils::{
-        ferox_print, format_url, get_url_path_length, make_request, module_colorizer,
-        status_colorizer,
-    },
+    statistics::StatCommand,
+    utils::{ferox_print, format_url, get_url_path_length, make_request, status_colorizer},
     FeroxResponse,
 };
 use console::style;
@@ -42,13 +40,15 @@ fn unique_string(length: usize) -> String {
 pub async fn wildcard_test(
     target_url: &str,
     bar: ProgressBar,
-    tx_file: UnboundedSender<String>,
+    tx_term: UnboundedSender<FeroxResponse>,
+    tx_stats: UnboundedSender<StatCommand>,
 ) -> Option<WildcardFilter> {
     log::trace!(
-        "enter: wildcard_test({:?}, {:?}, {:?})",
+        "enter: wildcard_test({:?}, {:?}, {:?}, {:?})",
         target_url,
         bar,
-        tx_file
+        tx_term,
+        tx_stats
     );
 
     if CONFIGURATION.dont_filter {
@@ -57,10 +57,14 @@ pub async fn wildcard_test(
         return None;
     }
 
-    let clone_req_one = tx_file.clone();
-    let clone_req_two = tx_file.clone();
+    let tx_term_mwcr1 = tx_term.clone();
+    let tx_term_mwcr2 = tx_term.clone();
+    let tx_stats_mwcr1 = tx_stats.clone();
+    let tx_stats_mwcr2 = tx_stats.clone();
 
-    if let Some(ferox_response) = make_wildcard_request(&target_url, 1, clone_req_one).await {
+    if let Some(ferox_response) =
+        make_wildcard_request(&target_url, 1, tx_term_mwcr1, tx_stats_mwcr1).await
+    {
         bar.inc(1);
 
         // found a wildcard response
@@ -75,7 +79,9 @@ pub async fn wildcard_test(
 
         // content length of wildcard is non-zero, perform additional tests:
         //   make a second request, with a known-sized (64) longer request
-        if let Some(resp_two) = make_wildcard_request(&target_url, 3, clone_req_two).await {
+        if let Some(resp_two) =
+            make_wildcard_request(&target_url, 3, tx_term_mwcr2, tx_stats_mwcr2).await
+        {
             bar.inc(1);
 
             let wc2_length = resp_two.content_length();
@@ -89,42 +95,34 @@ pub async fn wildcard_test(
 
                 if !CONFIGURATION.quiet {
                     let msg = format!(
-                            "{} {:>10} Wildcard response is dynamic; {} ({} + url length) responses; toggle this behavior by using {}\n",
+                            "{} {:>9} {:>9} {:>9} Wildcard response is dynamic; {} ({} + url length) responses; toggle this behavior by using {}\n",
                             status_colorizer("WLD"),
-                            wildcard.dynamic,
+                            "-",
+                            "-",
+                            "-",
                             style("auto-filtering").yellow(),
                             style(wc_length - url_len).cyan(),
                             style("--dont-filter").yellow()
-                        );
+                    );
 
                     ferox_print(&msg, &PROGRESS_PRINTER);
-
-                    try_send_message_to_file(
-                        &msg,
-                        tx_file.clone(),
-                        !CONFIGURATION.output.is_empty(),
-                    );
                 }
             } else if wc_length == wc2_length {
                 wildcard.size = wc_length;
 
                 if !CONFIGURATION.quiet {
                     let msg = format!(
-                        "{} {:>10} Wildcard response is static; {} {} responses; toggle this behavior by using {}\n",
+                        "{} {:>9} {:>9} {:>9} Wildcard response is static; {} {} responses; toggle this behavior by using {}\n",
                         status_colorizer("WLD"),
-                        wc_length,
+                        "-",
+                        "-",
+                        "-",
                         style("auto-filtering").yellow(),
                         style(wc_length).cyan(),
                         style("--dont-filter").yellow()
                     );
 
                     ferox_print(&msg, &PROGRESS_PRINTER);
-
-                    try_send_message_to_file(
-                        &msg,
-                        tx_file.clone(),
-                        !CONFIGURATION.output.is_empty(),
-                    );
                 }
             }
         } else {
@@ -148,13 +146,15 @@ pub async fn wildcard_test(
 async fn make_wildcard_request(
     target_url: &str,
     length: usize,
-    tx_file: UnboundedSender<String>,
+    tx_file: UnboundedSender<FeroxResponse>,
+    tx_stats: UnboundedSender<StatCommand>,
 ) -> Option<FeroxResponse> {
     log::trace!(
-        "enter: make_wildcard_request({}, {}, {:?})",
+        "enter: make_wildcard_request({}, {}, {:?}, {:?})",
         target_url,
         length,
-        tx_file
+        tx_file,
+        tx_stats,
     );
 
     let unique_str = unique_string(length);
@@ -165,6 +165,7 @@ async fn make_wildcard_request(
         CONFIGURATION.add_slash,
         &CONFIGURATION.queries,
         None,
+        tx_stats.clone(),
     ) {
         Ok(url) => url,
         Err(e) => {
@@ -174,62 +175,30 @@ async fn make_wildcard_request(
         }
     };
 
-    let wildcard = status_colorizer("WLD");
-
-    match make_request(&CONFIGURATION.client, &nonexistent.to_owned()).await {
+    match make_request(
+        &CONFIGURATION.client,
+        &nonexistent.to_owned(),
+        tx_stats.clone(),
+    )
+    .await
+    {
         Ok(response) => {
             if CONFIGURATION
                 .status_codes
                 .contains(&response.status().as_u16())
             {
                 // found a wildcard response
-                let ferox_response = FeroxResponse::from(response, false).await;
-                let url_len = get_url_path_length(&ferox_response.url());
-                let content_len = ferox_response.content_length();
+                let mut ferox_response = FeroxResponse::from(response, true).await;
+                ferox_response.wildcard = true;
 
-                if !CONFIGURATION.quiet && !should_filter_response(&ferox_response) {
-                    let msg = format!(
-                        "{} {:>10} Got {} for {} (url length: {})\n",
-                        wildcard,
-                        content_len,
-                        status_colorizer(&ferox_response.status().as_str()),
-                        ferox_response.url(),
-                        url_len
-                    );
-
-                    ferox_print(&msg, &PROGRESS_PRINTER);
-
-                    try_send_message_to_file(
-                        &msg,
-                        tx_file.clone(),
-                        !CONFIGURATION.output.is_empty(),
-                    );
+                if !CONFIGURATION.quiet
+                    && !should_filter_response(&ferox_response, tx_stats.clone())
+                    && tx_file.send(ferox_response.clone()).is_err()
+                {
+                    return None;
                 }
 
-                if ferox_response.status().is_redirection() {
-                    // show where it goes, if possible
-                    if let Some(next_loc) = ferox_response.headers().get("Location") {
-                        let next_loc_str = next_loc.to_str().unwrap_or("Unknown");
-                        if !CONFIGURATION.quiet && !should_filter_response(&ferox_response) {
-                            let msg = format!(
-                                "{} {:>10} {} redirects to => {}\n",
-                                wildcard,
-                                content_len,
-                                ferox_response.url(),
-                                next_loc_str
-                            );
-
-                            ferox_print(&msg, &PROGRESS_PRINTER);
-
-                            try_send_message_to_file(
-                                &msg,
-                                tx_file.clone(),
-                                !CONFIGURATION.output.is_empty(),
-                            );
-                        }
-                    }
-                }
-                log::trace!("exit: make_wildcard_request -> {:?}", ferox_response);
+                log::trace!("exit: make_wildcard_request -> {}", ferox_response);
                 return Some(ferox_response);
             }
         }
@@ -239,6 +208,7 @@ async fn make_wildcard_request(
             return None;
         }
     }
+
     log::trace!("exit: make_wildcard_request -> None");
     None
 }
@@ -248,8 +218,15 @@ async fn make_wildcard_request(
 /// In the event that no sites can be reached, the program will exit.
 ///
 /// Any urls that are found to be alive are returned to the caller.
-pub async fn connectivity_test(target_urls: &[String]) -> Vec<String> {
-    log::trace!("enter: connectivity_test({:?})", target_urls);
+pub async fn connectivity_test(
+    target_urls: &[String],
+    tx_stats: UnboundedSender<StatCommand>,
+) -> Vec<String> {
+    log::trace!(
+        "enter: connectivity_test({:?}, {:?})",
+        target_urls,
+        tx_stats
+    );
 
     let mut good_urls = vec![];
 
@@ -260,6 +237,7 @@ pub async fn connectivity_test(target_urls: &[String]) -> Vec<String> {
             CONFIGURATION.add_slash,
             &CONFIGURATION.queries,
             None,
+            tx_stats.clone(),
         ) {
             Ok(url) => url,
             Err(e) => {
@@ -268,7 +246,7 @@ pub async fn connectivity_test(target_urls: &[String]) -> Vec<String> {
             }
         };
 
-        match make_request(&CONFIGURATION.client, &request).await {
+        match make_request(&CONFIGURATION.client, &request, tx_stats.clone()).await {
             Ok(_) => {
                 good_urls.push(target_url.to_owned());
             }
@@ -293,35 +271,9 @@ pub async fn connectivity_test(target_urls: &[String]) -> Vec<String> {
     good_urls
 }
 
-/// simple helper to keep DRY; sends a message using the transmitter side of the given mpsc channel
-/// the receiver is expected to be the side that saves the message to CONFIGURATION.output.
-fn try_send_message_to_file(msg: &str, tx_file: UnboundedSender<String>, save_output: bool) {
-    log::trace!("enter: try_send_message_to_file({}, {:?})", msg, tx_file);
-
-    if save_output {
-        match tx_file.send(msg.to_string()) {
-            Ok(_) => {
-                log::trace!(
-                    "sent message from heuristics::try_send_message_to_file to file handler"
-                );
-            }
-            Err(e) => {
-                log::error!(
-                    "{} {}",
-                    module_colorizer("heuristics::try_send_message_to_file"),
-                    e
-                );
-            }
-        }
-    }
-    log::trace!("exit: try_send_message_to_file");
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::FeroxChannel;
-    use tokio::sync::mpsc;
 
     #[test]
     /// request a unique string of 32bytes * a value returns correct result
@@ -338,41 +290,4 @@ mod tests {
         assert_eq!(wcf.size, 0);
         assert_eq!(wcf.dynamic, 0);
     }
-
-    #[tokio::test(core_threads = 1)]
-    /// tests that given a message and transmitter, the function sends the message across the
-    /// channel
-    async fn heuristics_try_send_message_to_file_sends_when_true() {
-        let (tx, mut rx): FeroxChannel<String> = mpsc::unbounded_channel();
-        let msg = "It really tied the room together.";
-        let should_save = true;
-        try_send_message_to_file(&msg, tx, should_save);
-
-        assert_eq!(rx.recv().await.unwrap(), msg);
-    }
-
-    #[tokio::test(core_threads = 1)]
-    #[should_panic]
-    /// tests that when save_output is false, nothing is sent to the receiver
-    async fn heuristics_try_send_message_to_file_sends_when_false() {
-        let (tx, mut rx): FeroxChannel<String> = mpsc::unbounded_channel();
-        let msg = "I'm the Dude, so that's what you call me.";
-        let should_save = false;
-        try_send_message_to_file(&msg, tx, should_save);
-
-        assert_ne!(rx.recv().await.unwrap(), msg);
-    }
-
-    #[tokio::test(core_threads = 1)]
-    /// tests that when save_output is true, but the receiver is closed, nothing is sent to the receiver
-    /// this test doesn't assert anything, but reaches the error block of the given function and
-    /// can be verified with --nocapture and RUST_LOG being set
-    async fn heuristics_try_send_message_to_file_sends_with_closed_receiver() {
-        env_logger::init();
-        let (tx, mut rx): FeroxChannel<String> = mpsc::unbounded_channel();
-        let msg = "Hey, nice marmot.";
-        let should_save = true;
-        rx.close();
-        try_send_message_to_file(&msg, tx, should_save);
-    }
 }

src/lib.rs

@@ -1,3 +1,4 @@
+pub mod utils;
 pub mod banner;
 pub mod client;
 pub mod config;
@@ -8,13 +9,19 @@ pub mod logger;
 pub mod parser;
 pub mod progress;
 pub mod reporter;
+pub mod scan_manager;
 pub mod scanner;
-pub mod utils;
+pub mod statistics;
 
-use reqwest::{
-    header::HeaderMap,
-    {Response, StatusCode, Url},
-};
+use crate::utils::{get_url_path_length, status_colorizer};
+use console::{style, Color};
+use reqwest::header::{HeaderName, HeaderValue};
+use reqwest::{header::HeaderMap, Response, StatusCode, Url};
+use serde::{ser::SerializeStruct, Deserialize, Deserializer, Serialize, Serializer};
+use serde_json::Value;
+use std::collections::HashMap;
+use std::convert::{TryFrom, TryInto};
+use std::str::FromStr;
 use std::{error, fmt};
 use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender};
 
@@ -45,6 +52,9 @@ pub const VERSION: &str = env!("CARGO_PKG_VERSION");
 /// Maximum number of file descriptors that can be opened during a scan
 pub const DEFAULT_OPEN_FILE_LIMIT: usize = 8192;
 
+/// Default value used to determine near-duplicate web pages (equivalent to 95%)
+pub const SIMILARITY_THRESHOLD: u32 = 95;
+
 /// Default wordlist to use when `-w|--wordlist` isn't specified and not `wordlist` isn't set
 /// in a [ferox-config.toml](constant.DEFAULT_CONFIG_NAME.html) config file.
 ///
@@ -84,6 +94,17 @@ pub const DEFAULT_STATUS_CODES: [StatusCode; 9] = [
 /// Expected location is in the same directory as the feroxbuster binary.
 pub const DEFAULT_CONFIG_NAME: &str = "ferox-config.toml";
 
+/// FeroxSerialize trait; represents different types that are Serialize and also implement
+/// as_str / as_json methods
+pub trait FeroxSerialize: Serialize {
+    /// Return a String representation of the object, generally the human readable version of the
+    /// implementor
+    fn as_str(&self) -> String;
+
+    /// Return an NDJSON representation of the object
+    fn as_json(&self) -> String;
+}
+
 /// A `FeroxResponse`, derived from a `Response` to a submitted `Request`
 #[derive(Debug, Clone)]
 pub struct FeroxResponse {
@@ -99,8 +120,30 @@ pub struct FeroxResponse {
     /// The content-length of this response, if known
     content_length: u64,
 
+    /// The number of lines contained in the body of this response, if known
+    line_count: usize,
+
+    /// The number of words contained in the body of this response, if known
+    word_count: usize,
+
     /// The `Headers` of this `FeroxResponse`
     headers: HeaderMap,
+
+    /// Wildcard response status
+    wildcard: bool,
+}
+
+/// Implement Display for FeroxResponse
+impl fmt::Display for FeroxResponse {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(
+            f,
+            "FeroxResponse {{ url: {}, status: {}, content-length: {} }}",
+            self.url(),
+            self.status(),
+            self.content_length()
+        )
+    }
 }
 
 /// `FeroxResponse` implementation
@@ -163,6 +206,16 @@ impl FeroxResponse {
         self.url.query_pairs().count() > 0 || has_extension
     }
 
+    /// Returns line count of the response text.
+    pub fn line_count(&self) -> usize {
+        self.line_count
+    }
+
+    /// Returns word count of the response text.
+    pub fn word_count(&self) -> usize {
+        self.word_count
+    }
+
     /// Create a new `FeroxResponse` from the given `Response`
     pub async fn from(response: Response, read_body: bool) -> Self {
         let url = response.url().clone();
@@ -186,16 +239,298 @@ impl FeroxResponse {
             String::new()
         };
 
+        let line_count = text.lines().count();
+        let word_count = text.lines().map(|s| s.split_whitespace().count()).sum();
+
         FeroxResponse {
             url,
             status,
             content_length,
             text,
             headers,
+            line_count,
+            word_count,
+            wildcard: false,
         }
     }
 }
 
+/// Implement FeroxSerialusize::from(ize for FeroxRespons)e
+impl FeroxSerialize for FeroxResponse {
+    /// Simple wrapper around create_report_string
+    fn as_str(&self) -> String {
+        let lines = self.line_count().to_string();
+        let words = self.word_count().to_string();
+        let chars = self.content_length().to_string();
+        let status = self.status().as_str();
+        let wild_status = status_colorizer("WLD");
+
+        if self.wildcard {
+            // response is a wildcard, special messages abound when this is the case...
+
+            // create the base message
+            let mut message = format!(
+                "{} {:>8}l {:>8}w {:>8}c Got {} for {} (url length: {})\n",
+                wild_status,
+                lines,
+                words,
+                chars,
+                status_colorizer(&status),
+                self.url(),
+                get_url_path_length(&self.url())
+            );
+
+            if self.status().is_redirection() {
+                // when it's a redirect, show where it goes, if possible
+                if let Some(next_loc) = self.headers().get("Location") {
+                    let next_loc_str = next_loc.to_str().unwrap_or("Unknown");
+
+                    let redirect_msg = format!(
+                        "{} {:>9} {:>9} {:>9} {} redirects to => {}\n",
+                        wild_status,
+                        "-",
+                        "-",
+                        "-",
+                        self.url(),
+                        next_loc_str
+                    );
+
+                    message.push_str(&redirect_msg);
+                }
+            }
+
+            // base message + redirection message (if appropriate)
+            message
+        } else {
+            // not a wildcard, just create a normal entry
+            utils::create_report_string(
+                self.status.as_str(),
+                &lines,
+                &words,
+                &chars,
+                self.url().as_str(),
+            )
+        }
+    }
+
+    /// Create an NDJSON representation of the FeroxResponse
+    ///
+    /// (expanded for clarity)
+    /// ex:
+    /// {
+    ///    "type":"response",
+    ///    "url":"https://localhost.com/images",
+    ///    "path":"/images",
+    ///    "status":301,
+    ///    "content_length":179,
+    ///    "line_count":10,
+    ///    "word_count":16,
+    ///    "headers":{
+    ///       "x-content-type-options":"nosniff",
+    ///       "strict-transport-security":"max-age=31536000; includeSubDomains",
+    ///       "x-frame-options":"SAMEORIGIN",
+    ///       "connection":"keep-alive",
+    ///       "server":"nginx/1.16.1",
+    ///       "content-type":"text/html; charset=UTF-8",
+    ///       "referrer-policy":"origin-when-cross-origin",
+    ///       "content-security-policy":"default-src 'none'",
+    ///       "access-control-allow-headers":"X-Requested-With",
+    ///       "x-xss-protection":"1; mode=block",
+    ///       "content-length":"179",
+    ///       "date":"Mon, 23 Nov 2020 15:33:24 GMT",
+    ///       "location":"/images/",
+    ///       "access-control-allow-origin":"https://localhost.com"
+    ///    }
+    /// }\n
+    fn as_json(&self) -> String {
+        if let Ok(mut json) = serde_json::to_string(&self) {
+            json.push('\n');
+            json
+        } else {
+            format!("{{\"error\":\"could not convert {} to json\"}}", self.url())
+        }
+    }
+}
+
+/// Serialize implementation for FeroxResponse
+impl Serialize for FeroxResponse {
+    /// Function that handles serialization of a FeroxResponse to NDJSON
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let mut headers = HashMap::new();
+        let mut state = serializer.serialize_struct("FeroxResponse", 7)?;
+
+        // need to convert the HeaderMap to a HashMap in order to pass it to the serializer
+        for (key, value) in &self.headers {
+            let k = key.as_str().to_owned();
+            let v = String::from_utf8_lossy(value.as_bytes());
+            headers.insert(k, v);
+        }
+
+        state.serialize_field("type", "response")?;
+        state.serialize_field("url", self.url.as_str())?;
+        state.serialize_field("path", self.url.path())?;
+        state.serialize_field("wildcard", &self.wildcard)?;
+        state.serialize_field("status", &self.status.as_u16())?;
+        state.serialize_field("content_length", &self.content_length)?;
+        state.serialize_field("line_count", &self.line_count)?;
+        state.serialize_field("word_count", &self.word_count)?;
+        state.serialize_field("headers", &headers)?;
+
+        state.end()
+    }
+}
+
+/// Deserialize implementation for FeroxResponse
+impl<'de> Deserialize<'de> for FeroxResponse {
+    /// Deserialize a FeroxResponse from a serde_json::Value
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let mut response = Self {
+            url: Url::parse("http://localhost").unwrap(),
+            status: StatusCode::OK,
+            text: String::new(),
+            content_length: 0,
+            headers: HeaderMap::new(),
+            wildcard: false,
+            line_count: 0,
+            word_count: 0,
+        };
+
+        let map: HashMap<String, Value> = HashMap::deserialize(deserializer)?;
+
+        for (key, value) in &map {
+            match key.as_str() {
+                "url" => {
+                    if let Some(url) = value.as_str() {
+                        if let Ok(parsed) = Url::parse(url) {
+                            response.url = parsed;
+                        }
+                    }
+                }
+                "status" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(smaller) = u16::try_from(num) {
+                            if let Ok(status) = StatusCode::from_u16(smaller) {
+                                response.status = status;
+                            }
+                        }
+                    }
+                }
+                "content_length" => {
+                    if let Some(num) = value.as_u64() {
+                        response.content_length = num;
+                    }
+                }
+                "line_count" => {
+                    if let Some(num) = value.as_u64() {
+                        response.line_count = num.try_into().unwrap_or_default();
+                    }
+                }
+                "word_count" => {
+                    if let Some(num) = value.as_u64() {
+                        response.word_count = num.try_into().unwrap_or_default();
+                    }
+                }
+                "headers" => {
+                    let mut headers = HeaderMap::<HeaderValue>::default();
+
+                    if let Some(map_headers) = value.as_object() {
+                        for (h_key, h_value) in map_headers {
+                            let h_value_str = h_value.as_str().unwrap_or("");
+                            let h_name = HeaderName::from_str(h_key)
+                                .unwrap_or_else(|_| HeaderName::from_str("Unknown").unwrap());
+                            let h_value_parsed = HeaderValue::from_str(h_value_str)
+                                .unwrap_or_else(|_| HeaderValue::from_str("Unknown").unwrap());
+                            headers.insert(h_name, h_value_parsed);
+                        }
+                    }
+
+                    response.headers = headers;
+                }
+                "wildcard" => {
+                    if let Some(result) = value.as_bool() {
+                        response.wildcard = result;
+                    }
+                }
+                _ => {}
+            }
+        }
+
+        Ok(response)
+    }
+}
+
+#[derive(Serialize, Deserialize, Default)]
+/// Representation of a log entry, can be represented as a human readable string or JSON
+pub struct FeroxMessage {
+    #[serde(rename = "type")]
+    /// Name of this type of struct, used for serialization, i.e. `{"type":"log"}`
+    kind: String,
+
+    /// The log message
+    pub message: String,
+
+    /// The log level
+    pub level: String,
+
+    /// The number of seconds elapsed since the scan started
+    pub time_offset: f32,
+
+    /// The module from which log::* was called
+    pub module: String,
+}
+
+/// Implementation of FeroxMessage
+impl FeroxSerialize for FeroxMessage {
+    /// Create an NDJSON representation of the log message
+    ///
+    /// (expanded for clarity)
+    /// ex:
+    /// {
+    ///   "type": "log",
+    ///   "message": "Sent https://localhost/api to file handler",
+    ///   "level": "DEBUG",
+    ///   "time_offset": 0.86333454,
+    ///   "module": "feroxbuster::reporter"
+    /// }\n
+    fn as_json(&self) -> String {
+        if let Ok(mut json) = serde_json::to_string(&self) {
+            json.push('\n');
+            json
+        } else {
+            String::from("{\"error\":\"could not convert to json\"}")
+        }
+    }
+
+    /// Create a string representation of the log message
+    ///
+    /// ex:  301       10l       16w      173c https://localhost/api
+    fn as_str(&self) -> String {
+        let (level_name, level_color) = match self.level.as_str() {
+            "ERROR" => ("ERR", Color::Red),
+            "WARN" => ("WRN", Color::Red),
+            "INFO" => ("INF", Color::Cyan),
+            "DEBUG" => ("DBG", Color::Yellow),
+            "TRACE" => ("TRC", Color::Magenta),
+            "WILDCARD" => ("WLD", Color::Cyan),
+            _ => ("UNK", Color::White),
+        };
+
+        format!(
+            "{} {:10.03} {} {}\n",
+            style(level_name).bg(level_color).black(),
+            style(self.time_offset).dim(),
+            self.module,
+            style(&self.message).dim(),
+        )
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -220,4 +555,46 @@ mod tests {
     fn default_version() {
         assert_eq!(VERSION, env!("CARGO_PKG_VERSION"));
     }
+
+    #[test]
+    /// test as_str method of FeroxMessage
+    fn ferox_message_as_str_returns_string_with_newline() {
+        let message = FeroxMessage {
+            message: "message".to_string(),
+            module: "utils".to_string(),
+            time_offset: 1.0,
+            level: "INFO".to_string(),
+            kind: "log".to_string(),
+        };
+        let message_str = message.as_str();
+
+        assert!(message_str.contains("INF"));
+        assert!(message_str.contains("1.000"));
+        assert!(message_str.contains("utils"));
+        assert!(message_str.contains("message"));
+        assert!(message_str.ends_with('\n'));
+    }
+
+    #[test]
+    /// test as_json method of FeroxMessage
+    fn ferox_message_as_json_returns_json_representation_of_ferox_message_with_newline() {
+        let message = FeroxMessage {
+            message: "message".to_string(),
+            module: "utils".to_string(),
+            time_offset: 1.0,
+            level: "INFO".to_string(),
+            kind: "log".to_string(),
+        };
+
+        let message_str = message.as_json();
+
+        let error_margin = f32::EPSILON;
+
+        let json: FeroxMessage = serde_json::from_str(&message_str).unwrap();
+        assert_eq!(json.module, message.module);
+        assert_eq!(json.message, message.message);
+        assert!((json.time_offset - message.time_offset).abs() < error_margin);
+        assert_eq!(json.level, message.level);
+        assert_eq!(json.kind, message.kind);
+    }
 }

src/logger.rs

@@ -1,6 +1,9 @@
-use crate::config::{CONFIGURATION, PROGRESS_PRINTER};
-use crate::reporter::{get_cached_file_handle, safe_file_write};
-use console::{style, Color};
+use crate::{
+    config::{CONFIGURATION, PROGRESS_PRINTER},
+    reporter::safe_file_write,
+    utils::open_file,
+    FeroxMessage, FeroxSerialize,
+};
 use env_logger::Builder;
 use std::env;
 use std::time::Instant;
@@ -19,8 +22,8 @@ pub fn initialize(verbosity: u8) {
                 0 => (),
                 1 => env::set_var("RUST_LOG", "warn"),
                 2 => env::set_var("RUST_LOG", "info"),
-                3 => env::set_var("RUST_LOG", "debug,hyper=info,reqwest=info"),
-                _ => env::set_var("RUST_LOG", "trace,hyper=info,reqwest=info"),
+                3 => env::set_var("RUST_LOG", "feroxbuster=debug,info"),
+                _ => env::set_var("RUST_LOG", "feroxbuster=trace,info"),
             }
         }
     }
@@ -28,43 +31,27 @@ pub fn initialize(verbosity: u8) {
     let start = Instant::now();
     let mut builder = Builder::from_default_env();
 
-    // I REALLY wanted the logger to also use the reporting channels found in the `reporter`
-    // module. However, in order to properly clean up the channels, all references to the
-    // transmitter side of a channel need to go out of scope, then you can await the future into
-    // which the receiver was moved.
-    //
-    // The problem was that putting a transmitter reference in this closure, which gets initialized
-    // as part of the global logger, made it so that I couldn't destroy/leak/take/swap the last
-    // reference to allow the channels to gracefully close.
-    //
-    // The workaround was to have a RwLock around the file and allow both the logger and the
-    // file handler to both write independent of each other.
-    let locked_file = get_cached_file_handle(&CONFIGURATION.output);
+    let debug_file = open_file(&CONFIGURATION.debug_log);
+
+    if let Some(buffered_file) = debug_file.clone() {
+        // write out the configuration to the debug file if it exists
+        safe_file_write(&*CONFIGURATION, buffered_file, CONFIGURATION.json);
+    }
 
     builder
         .format(move |_, record| {
-            let t = start.elapsed().as_secs_f32();
-            let level = record.level();
-
-            let (level_name, level_color) = match level {
-                log::Level::Error => ("ERR", Color::Red),
-                log::Level::Warn => ("WRN", Color::Red),
-                log::Level::Info => ("INF", Color::Cyan),
-                log::Level::Debug => ("DBG", Color::Yellow),
-                log::Level::Trace => ("TRC", Color::Magenta),
+            let log_entry = FeroxMessage {
+                message: record.args().to_string(),
+                level: record.level().to_string(),
+                time_offset: start.elapsed().as_secs_f32(),
+                module: record.target().to_string(),
+                kind: "log".to_string(),
             };
 
-            let msg = format!(
-                "{} {:10.03} {}\n",
-                style(level_name).bg(level_color).black(),
-                style(t).dim(),
-                style(record.args()).dim(),
-            );
+            PROGRESS_PRINTER.println(&log_entry.as_str());
 
-            PROGRESS_PRINTER.println(&msg);
-
-            if let Some(buffered_file) = locked_file.clone() {
-                safe_file_write(&msg, buffered_file);
+            if let Some(buffered_file) = debug_file.clone() {
+                safe_file_write(&log_entry, buffered_file, CONFIGURATION.json);
             }
 
             Ok(())

src/main.rs

@@ -2,8 +2,18 @@ use crossterm::event::{self, Event, KeyCode};
 use feroxbuster::{
     banner,
     config::{CONFIGURATION, PROGRESS_BAR, PROGRESS_PRINTER},
-    heuristics, logger, reporter,
-    scanner::{scan_url, PAUSE_SCAN},
+    heuristics, logger,
+    progress::{add_bar, BarType},
+    reporter,
+    scan_manager::{self, ScanStatus, PAUSE_SCAN},
+    scanner::{self, scan_url, SCANNED_URLS},
+    statistics::{
+        self,
+        StatCommand::{self, CreateBar, LoadStats, UpdateUsizeField},
+        StatField::InitialTargets,
+        Stats,
+    },
+    update_stat,
     utils::{ferox_print, get_current_depth, module_colorizer, status_colorizer},
     FeroxError, FeroxResponse, FeroxResult, SLEEP_DURATION, VERSION,
 };
@@ -12,6 +22,7 @@ use feroxbuster::{utils::set_open_file_limit, DEFAULT_OPEN_FILE_LIMIT};
 use futures::StreamExt;
 use std::{
     collections::HashSet,
+    convert::TryInto,
     fs::File,
     io::{stderr, BufRead, BufReader},
     process,
@@ -19,6 +30,7 @@ use std::{
         atomic::{AtomicBool, Ordering},
         Arc,
     },
+    thread::sleep,
     time::Duration,
 };
 use tokio::{io, sync::mpsc::UnboundedSender, task::JoinHandle};
@@ -32,17 +44,20 @@ fn terminal_input_handler() {
     log::trace!("enter: terminal_input_handler");
 
     loop {
-        if event::poll(Duration::from_millis(SLEEP_DURATION)).unwrap_or(false) {
+        if PAUSE_SCAN.load(Ordering::Relaxed) {
+            // if the scan is already paused, we don't want this event poller fighting the user
+            // over stdin
+            sleep(Duration::from_millis(SLEEP_DURATION));
+        } else if event::poll(Duration::from_millis(SLEEP_DURATION)).unwrap_or(false) {
             // It's guaranteed that the `read()` won't block when the `poll()`
             // function returns `true`
 
             if let Ok(key_pressed) = event::read() {
+                // ignore any other keys
                 if key_pressed == Event::Key(KeyCode::Enter.into()) {
-                    // if the user presses Enter, toggle the value stored in PAUSE_SCAN
-                    // ignore any other keys
-                    let current = PAUSE_SCAN.load(Ordering::Acquire);
-
-                    PAUSE_SCAN.store(!current, Ordering::Release);
+                    // if the user presses Enter, set PAUSE_SCAN to true. The interactive menu
+                    // will be triggered and will handle setting PAUSE_SCAN to false
+                    PAUSE_SCAN.store(true, Ordering::Release);
                 }
             }
         } else {
@@ -95,10 +110,19 @@ fn get_unique_words_from_wordlist(path: &str) -> FeroxResult<Arc<HashSet<String>
 /// Determine whether it's a single url scan or urls are coming from stdin, then scan as needed
 async fn scan(
     targets: Vec<String>,
+    stats: Arc<Stats>,
     tx_term: UnboundedSender<FeroxResponse>,
-    tx_file: UnboundedSender<String>,
+    tx_file: UnboundedSender<FeroxResponse>,
+    tx_stats: UnboundedSender<StatCommand>,
 ) -> FeroxResult<()> {
-    log::trace!("enter: scan({:?}, {:?}, {:?})", targets, tx_term, tx_file);
+    log::trace!(
+        "enter: scan({:?}, {:?}, {:?}, {:?}, {:?})",
+        targets,
+        stats,
+        tx_term,
+        tx_file,
+        tx_stats
+    );
     // cloning an Arc is cheap (it's basically a pointer into the heap)
     // so that will allow for cheap/safe sharing of a single wordlist across multi-target scans
     // as well as additional directories found as part of recursion
@@ -107,21 +131,66 @@ async fn scan(
             .await??;
 
     if words.len() == 0 {
-        let mut err = FeroxError::default();
-        err.message = format!("Did not find any words in {}", CONFIGURATION.wordlist);
+        let err = FeroxError {
+            message: format!("Did not find any words in {}", CONFIGURATION.wordlist),
+        };
+
         return Err(Box::new(err));
     }
 
+    scanner::initialize(words.len(), &CONFIGURATION, tx_stats.clone()).await;
+
+    // at this point, the stat thread's progress bar can be created; things that needed to happen
+    // first:
+    // - banner gets printed
+    // - scanner initialized (this sent expected requests per directory to the stats thread, which
+    //   having been set, makes it so the progress bar doesn't flash as full before anything has
+    //   even happened
+    update_stat!(tx_stats, CreateBar);
+
+    if CONFIGURATION.resumed {
+        update_stat!(tx_stats, LoadStats(CONFIGURATION.resume_from.clone()));
+
+        SCANNED_URLS.print_known_responses();
+
+        if let Ok(scans) = SCANNED_URLS.scans.lock() {
+            for scan in scans.iter() {
+                if let Ok(locked_scan) = scan.lock() {
+                    if matches!(locked_scan.status, ScanStatus::Complete) {
+                        // these scans are complete, and just need to be shown to the user
+                        let pb = add_bar(
+                            &locked_scan.url,
+                            words.len().try_into().unwrap_or_default(),
+                            BarType::Message,
+                        );
+                        pb.finish();
+                    }
+                }
+            }
+        }
+    }
+
     let mut tasks = vec![];
 
     for target in targets {
         let word_clone = words.clone();
         let term_clone = tx_term.clone();
         let file_clone = tx_file.clone();
+        let tx_stats_clone = tx_stats.clone();
+        let stats_clone = stats.clone();
 
         let task = tokio::spawn(async move {
             let base_depth = get_current_depth(&target);
-            scan_url(&target, word_clone, base_depth, term_clone, file_clone).await;
+            scan_url(
+                &target,
+                word_clone,
+                base_depth,
+                stats_clone,
+                term_clone,
+                file_clone,
+                tx_stats_clone,
+            )
+            .await;
         });
 
         tasks.push(task);
@@ -149,6 +218,22 @@ async fn get_targets() -> FeroxResult<Vec<String>> {
         while let Some(line) = reader.next().await {
             targets.push(line?);
         }
+    } else if CONFIGURATION.resumed {
+        // resume-from can't be used with --url, and --stdin is marked false for every resumed
+        // scan, making it mutually exclusive from either of the other two options
+        if let Ok(scans) = SCANNED_URLS.scans.lock() {
+            for scan in scans.iter() {
+                // SCANNED_URLS gets deserialized scans added to it at program start if --resume-from
+                // is used, so scans that aren't marked complete still need to be scanned
+                if let Ok(locked_scan) = scan.lock() {
+                    if matches!(locked_scan.status, ScanStatus::Complete) {
+                        // this one's already done, ignore it
+                        continue;
+                    }
+                    targets.push(locked_scan.url.to_owned());
+                }
+            }
+        }
     } else {
         targets.push(CONFIGURATION.target_url.clone());
     }
@@ -174,9 +259,21 @@ async fn wrapped_main() {
         PROGRESS_BAR.join().unwrap();
     });
 
+    let (stats, tx_stats, stats_handle) = statistics::initialize();
+
+    if !CONFIGURATION.time_limit.is_empty() {
+        // --time-limit value not an empty string, need to kick off the thread that enforces
+        // the limit
+
+        let max_time_stats = stats.clone();
+
+        tokio::spawn(async move {
+            scan_manager::start_max_time_thread(&CONFIGURATION.time_limit, max_time_stats).await
+        });
+    }
+
     // can't trace main until after logger is initialized and the above task is started
     log::trace!("enter: main");
-    log::debug!("{:#?}", *CONFIGURATION);
 
     // spawn a thread that listens for keyboard input on stdin, when a user presses enter
     // the input handler will toggle PAUSE_SCAN, which in turn is used to pause and resume
@@ -185,8 +282,13 @@ async fn wrapped_main() {
 
     let save_output = !CONFIGURATION.output.is_empty(); // was -o used?
 
+    if CONFIGURATION.save_state {
+        // start the ctrl+c handler
+        scan_manager::initialize(stats.clone());
+    }
+
     let (tx_term, tx_file, term_handle, file_handle) =
-        reporter::initialize(&CONFIGURATION.output, save_output);
+        reporter::initialize(&CONFIGURATION.output, save_output, tx_stats.clone());
 
     // get targets from command line or stdin
     let targets = match get_targets().await {
@@ -194,27 +296,62 @@ async fn wrapped_main() {
         Err(e) => {
             // should only happen in the event that there was an error reading from stdin
             log::error!("{} {}", module_colorizer("main::get_targets"), e);
-            clean_up(tx_term, term_handle, tx_file, file_handle, save_output).await;
+            clean_up(
+                tx_term,
+                term_handle,
+                tx_file,
+                file_handle,
+                tx_stats,
+                stats_handle,
+                save_output,
+            )
+            .await;
             return;
         }
     };
 
+    update_stat!(tx_stats, UpdateUsizeField(InitialTargets, targets.len()));
+
     if !CONFIGURATION.quiet {
         // only print banner if -q isn't used
         let std_stderr = stderr(); // std::io::stderr
-        banner::initialize(&targets, &CONFIGURATION, &VERSION, std_stderr).await;
+        banner::initialize(
+            &targets,
+            &CONFIGURATION,
+            &VERSION,
+            std_stderr,
+            tx_stats.clone(),
+        )
+        .await;
     }
 
     // discard non-responsive targets
-    let live_targets = heuristics::connectivity_test(&targets).await;
+    let live_targets = heuristics::connectivity_test(&targets, tx_stats.clone()).await;
 
     if live_targets.is_empty() {
-        clean_up(tx_term, term_handle, tx_file, file_handle, save_output).await;
+        clean_up(
+            tx_term,
+            term_handle,
+            tx_file,
+            file_handle,
+            tx_stats,
+            stats_handle,
+            save_output,
+        )
+        .await;
         return;
     }
 
     // kick off a scan against any targets determined to be responsive
-    match scan(live_targets, tx_term.clone(), tx_file.clone()).await {
+    match scan(
+        live_targets,
+        stats,
+        tx_term.clone(),
+        tx_file.clone(),
+        tx_stats.clone(),
+    )
+    .await
+    {
         Ok(_) => {
             log::info!("All scans complete!");
         }
@@ -223,14 +360,32 @@ async fn wrapped_main() {
                 &format!("{} while scanning: {}", status_colorizer("Error"), e),
                 &PROGRESS_PRINTER,
             );
-            clean_up(tx_term, term_handle, tx_file, file_handle, save_output).await;
+            clean_up(
+                tx_term,
+                term_handle,
+                tx_file,
+                file_handle,
+                tx_stats,
+                stats_handle,
+                save_output,
+            )
+            .await;
             process::exit(1);
         }
     };
 
-    clean_up(tx_term, term_handle, tx_file, file_handle, save_output).await;
+    clean_up(
+        tx_term,
+        term_handle,
+        tx_file,
+        file_handle,
+        tx_stats,
+        stats_handle,
+        save_output,
+    )
+    .await;
 
-    log::trace!("exit: main");
+    log::trace!("exit: wrapped_main");
 }
 
 /// Single cleanup function that handles all the necessary drops/finishes etc required to gracefully
@@ -238,19 +393,22 @@ async fn wrapped_main() {
 async fn clean_up(
     tx_term: UnboundedSender<FeroxResponse>,
     term_handle: JoinHandle<()>,
-    tx_file: UnboundedSender<String>,
+    tx_file: UnboundedSender<FeroxResponse>,
     file_handle: Option<JoinHandle<()>>,
+    tx_stats: UnboundedSender<StatCommand>,
+    stats_handle: JoinHandle<()>,
     save_output: bool,
 ) {
     log::trace!(
-        "enter: clean_up({:?}, {:?}, {:?}, {:?}, {}",
+        "enter: clean_up({:?}, {:?}, {:?}, {:?}, {:?}, {:?}, {})",
         tx_term,
         term_handle,
         tx_file,
         file_handle,
+        tx_stats,
+        stats_handle,
         save_output
     );
-
     drop(tx_term);
     log::trace!("dropped terminal output handler's transmitter");
 
@@ -282,6 +440,9 @@ async fn clean_up(
         log::trace!("done awaiting file output handler's receiver");
     }
 
+    update_stat!(tx_stats, StatCommand::Exit); // send exit command and await the end of the future
+    stats_handle.await.unwrap_or_default();
+
     // mark all scans complete so the terminal input handler will exit cleanly
     SCAN_COMPLETE.store(true, Ordering::Relaxed);
 
@@ -289,6 +450,8 @@ async fn clean_up(
     // the final trace messages above
     PROGRESS_PRINTER.finish();
 
+    drop(tx_stats);
+
     log::trace!("exit: clean_up");
 }
 
@@ -300,8 +463,13 @@ fn main() {
     #[cfg(not(target_os = "windows"))]
     set_open_file_limit(DEFAULT_OPEN_FILE_LIMIT);
 
-    if let Ok(mut runtime) = tokio::runtime::Runtime::new() {
+    if let Ok(runtime) = tokio::runtime::Builder::new_multi_thread()
+        .enable_all()
+        .build()
+    {
         let future = wrapped_main();
         runtime.block_on(future);
     }
+
+    log::trace!("exit: main");
 }

src/parser.rs

@@ -1,10 +1,23 @@
-use crate::VERSION;
-use clap::{App, Arg};
+use clap::{App, Arg, ArgGroup};
+use lazy_static::lazy_static;
+use regex::Regex;
+
+lazy_static! {
+    /// Regex used to validate values passed to --time-limit
+    ///
+    /// Examples of expected values that will this regex will match:
+    /// - 30s
+    /// - 20m
+    /// - 1h
+    /// - 1d
+    pub static ref TIMESPEC_REGEX: Regex =
+        Regex::new(r"^(?i)(?P<n>\d+)(?P<m>[smdh])$").expect("Could not compile regex");
+}
 
 /// Create and return an instance of [clap::App](https://docs.rs/clap/latest/clap/struct.App.html), i.e. the Command Line Interface's configuration
 pub fn initialize() -> App<'static, 'static> {
     App::new("feroxbuster")
-        .version(VERSION)
+        .version(env!("CARGO_PKG_VERSION"))
         .author("Ben 'epi' Risher (@epi052)")
         .about("A fast, simple, recursive content discovery tool written in Rust")
         .arg(
@@ -19,7 +32,7 @@ pub fn initialize() -> App<'static, 'static> {
             Arg::with_name("url")
                 .short("u")
                 .long("url")
-                .required_unless("stdin")
+                .required_unless_one(&["stdin", "resume_from"])
                 .value_name("URL")
                 .multiple(true)
                 .use_delimiter(true)
@@ -55,7 +68,7 @@ pub fn initialize() -> App<'static, 'static> {
                 .long("verbosity")
                 .takes_value(false)
                 .multiple(true)
-                .help("Increase verbosity level (use -vv or more for greater effect)"),
+                .help("Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v's is probably too much)"),
         )
         .arg(
             Arg::with_name("proxy")
@@ -64,7 +77,7 @@ pub fn initialize() -> App<'static, 'static> {
                 .takes_value(true)
                 .value_name("PROXY")
                 .help(
-                    "Proxy to use for requests (ex: http(s)://host:port, socks5://host:port)",
+                    "Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)",
                 ),
         )
         .arg(
@@ -109,6 +122,13 @@ pub fn initialize() -> App<'static, 'static> {
                 .takes_value(false)
                 .help("Only print URLs; Don't print status codes, response size, running config, etc...")
         )
+        .arg(
+            Arg::with_name("json")
+                .long("json")
+                .takes_value(false)
+                .requires("output_files")
+                .help("Emit JSON logs to --output and --debug-log instead of normal text")
+        )
         .arg(
             Arg::with_name("dont_filter")
                 .short("D")
@@ -121,7 +141,22 @@ pub fn initialize() -> App<'static, 'static> {
                 .short("o")
                 .long("output")
                 .value_name("FILE")
-                .help("Output file to write results to (default: stdout)")
+                .help("Output file to write results to (use w/ --json for JSON entries)")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name("resume_from")
+                .long("resume-from")
+                .value_name("STATE_FILE")
+                .help("State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)")
+                .conflicts_with("url")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name("debug_log")
+                .long("debug-log")
+                .value_name("FILE")
+                .help("Output file to write log entries (use w/ --json for JSON entries)")
                 .takes_value(true),
         )
         .arg(
@@ -218,6 +253,42 @@ pub fn initialize() -> App<'static, 'static> {
                     "Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)",
                 ),
         )
+        .arg(
+            Arg::with_name("filter_regex")
+                .short("X")
+                .long("filter-regex")
+                .value_name("REGEX")
+                .takes_value(true)
+                .multiple(true)
+                .use_delimiter(true)
+                .help(
+                    "Filter out messages via regular expression matching on the response's body (ex: -X '^ignore me$')",
+                ),
+        )
+        .arg(
+            Arg::with_name("filter_words")
+                .short("W")
+                .long("filter-words")
+                .value_name("WORDS")
+                .takes_value(true)
+                .multiple(true)
+                .use_delimiter(true)
+                .help(
+                    "Filter out messages of a particular word count (ex: -W 312 -W 91,82)",
+                ),
+        )
+        .arg(
+            Arg::with_name("filter_lines")
+                .short("N")
+                .long("filter-lines")
+                .value_name("LINES")
+                .takes_value(true)
+                .multiple(true)
+                .use_delimiter(true)
+                .help(
+                    "Filter out messages of a particular line count (ex: -N 20 -N 31,30)",
+                ),
+        )
         .arg(
             Arg::with_name("filter_status")
                 .short("C")
@@ -230,6 +301,17 @@ pub fn initialize() -> App<'static, 'static> {
                     "Filter out status codes (deny list) (ex: -C 200 -C 401)",
                 ),
         )
+        .arg(
+            Arg::with_name("filter_similar")
+                .long("filter-similar-to")
+                .value_name("UNWANTED_PAGE")
+                .takes_value(true)
+                .multiple(true)
+                .use_delimiter(true)
+                .help(
+                    "Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)",
+                ),
+        )
         .arg(
             Arg::with_name("extract_links")
                 .short("e")
@@ -245,6 +327,18 @@ pub fn initialize() -> App<'static, 'static> {
                 .takes_value(true)
                 .help("Limit total number of concurrent scans (default: 0, i.e. no limit)")
         )
+        .arg(
+            Arg::with_name("time_limit")
+                .long("time-limit")
+                .value_name("TIME_SPEC")
+                .takes_value(true)
+                .validator(valid_time_spec)
+                .help("Limit total run time of all scans (ex: --time-limit 10m)")
+        )
+        .group(ArgGroup::with_name("output_files")
+            .args(&["debug_log", "output"])
+            .multiple(true)
+        )
         .after_help(r#"NOTE:
     Options that take multiple values are very flexible.  Consider the following ways of specifying
     extensions:
@@ -282,6 +376,20 @@ EXAMPLES:
     "#)
 }
 
+/// Validate that a string is formatted as a number followed by s, m, h, or d (10d, 30s, etc...)
+fn valid_time_spec(time_spec: String) -> Result<(), String> {
+    match TIMESPEC_REGEX.is_match(&time_spec) {
+        true => Ok(()),
+        false => {
+            let msg = format!(
+                "Expected a non-negative, whole number followed by s, m, h, or d (case insensitive); received {}",
+                time_spec
+            );
+            Err(msg)
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -292,4 +400,37 @@ mod tests {
         let app = initialize();
         assert_eq!(app.get_name(), "feroxbuster");
     }
+
+    #[test]
+    /// sanity checks that valid_time_spec correctly checks and rejects a given string
+    ///
+    /// instead of having a bunch of single tests here, they're all quick and are mostly checking
+    /// that i didn't hose up the regex.  Going to consolidate them into a single test
+    fn validate_valid_time_spec_validation() {
+        let float_rejected = "1.4m";
+        assert!(valid_time_spec(float_rejected.into()).is_err());
+
+        let negative_rejected = "-1m";
+        assert!(valid_time_spec(negative_rejected.into()).is_err());
+
+        let only_number_rejected = "1";
+        assert!(valid_time_spec(only_number_rejected.into()).is_err());
+
+        let only_measurement_rejected = "m";
+        assert!(valid_time_spec(only_measurement_rejected.into()).is_err());
+
+        for accepted_measurement in &["s", "m", "h", "d", "S", "M", "H", "D"] {
+            // all upper/lowercase should be good
+            assert!(valid_time_spec(format!("1{}", *accepted_measurement)).is_ok());
+        }
+
+        let leading_space_rejected = " 14m";
+        assert!(valid_time_spec(leading_space_rejected.into()).is_err());
+
+        let trailing_space_rejected = "14m ";
+        assert!(valid_time_spec(trailing_space_rejected.into()).is_err());
+
+        let space_between_rejected = "1 4m";
+        assert!(valid_time_spec(space_between_rejected.into()).is_err());
+    }
 }

src/progress.rs

@@ -1,15 +1,42 @@
 use crate::config::{CONFIGURATION, PROGRESS_BAR};
 use indicatif::{ProgressBar, ProgressStyle};
 
+/// Types of ProgressBars that can be added to `PROGRESS_BAR`
+pub enum BarType {
+    /// no template used / not visible
+    Hidden,
+
+    /// normal directory status bar (reqs/sec shown)
+    Default,
+
+    /// similar to `Default`, except `-` is used in place of line/word/char count
+    Message,
+
+    /// bar used to show overall scan metrics
+    Total,
+}
+
 /// Add an [indicatif::ProgressBar](https://docs.rs/indicatif/latest/indicatif/struct.ProgressBar.html)
 /// to the global [PROGRESS_BAR](../config/struct.PROGRESS_BAR.html)
-pub fn add_bar(prefix: &str, length: u64, hidden: bool) -> ProgressBar {
-    let style = if hidden || CONFIGURATION.quiet {
-        ProgressStyle::default_bar().template("")
+pub fn add_bar(prefix: &str, length: u64, bar_type: BarType) -> ProgressBar {
+    let mut style = ProgressStyle::default_bar().progress_chars("#>-");
+
+    style = if CONFIGURATION.quiet {
+        style.template("")
     } else {
-        ProgressStyle::default_bar()
-            .template("[{bar:.cyan/blue}] - {elapsed:<4} {pos:>7}/{len:7} {per_sec:7} {prefix}")
-            .progress_chars("#>-")
+        match bar_type {
+            BarType::Hidden => style.template(""),
+            BarType::Default => style.template(
+                "[{bar:.cyan/blue}] - {elapsed:<4} {pos:>7}/{len:7} {per_sec:7} {prefix}",
+            ),
+            BarType::Message => style.template(&format!(
+                "[{{bar:.cyan/blue}}] - {{elapsed:<4}} {{pos:>7}}/{{len:7}} {:7} {{prefix}}",
+                "-"
+            )),
+            BarType::Total => {
+                style.template("[{bar:.yellow/blue}] - {elapsed:<4} {pos:>7}/{len:7} {eta:7} {msg}")
+            }
+        }
     };
 
     let progress_bar = PROGRESS_BAR.add(ProgressBar::new(length));
@@ -20,3 +47,27 @@ pub fn add_bar(prefix: &str, length: u64, hidden: bool) -> ProgressBar {
 
     progress_bar
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    /// hit all code branches for add_bar
+    fn add_bar_with_all_configurations() {
+        let p1 = add_bar("prefix", 2, BarType::Hidden); // hidden
+        let p2 = add_bar("prefix", 2, BarType::Message); // no per second field
+        let p3 = add_bar("prefix", 2, BarType::Default); // normal bar
+        let p4 = add_bar("prefix", 2, BarType::Total); // totals bar
+
+        p1.finish();
+        p2.finish();
+        p3.finish();
+        p4.finish();
+
+        assert!(p1.is_finished());
+        assert!(p2.is_finished());
+        assert!(p3.is_finished());
+        assert!(p4.is_finished());
+    }
+}

src/reporter.rs

@@ -1,12 +1,23 @@
-use crate::config::{CONFIGURATION, PROGRESS_PRINTER};
-use crate::utils::{ferox_print, make_request, status_colorizer};
-use crate::{FeroxChannel, FeroxResponse};
+use crate::{
+    config::{CONFIGURATION, PROGRESS_PRINTER},
+    scanner::RESPONSES,
+    statistics::{
+        StatCommand::{self, UpdateUsizeField},
+        StatField::ResourcesDiscovered,
+    },
+    utils::{ferox_print, make_request, open_file},
+    FeroxChannel, FeroxResponse, FeroxSerialize,
+};
 use console::strip_ansi_codes;
-use std::io::Write;
-use std::sync::{Arc, Once, RwLock};
-use std::{fs, io};
-use tokio::sync::mpsc::{self, UnboundedReceiver, UnboundedSender};
-use tokio::task::JoinHandle;
+use std::{
+    fs, io,
+    io::Write,
+    sync::{Arc, Once, RwLock},
+};
+use tokio::{
+    sync::mpsc::{self, UnboundedReceiver, UnboundedSender},
+    task::JoinHandle,
+};
 
 /// Singleton buffered file behind an Arc/RwLock; used for file writes from two locations:
 ///     - [logger::initialize](../logger/fn.initialize.html) (specifically a closure on the global logger instance)
@@ -39,27 +50,35 @@ pub fn get_cached_file_handle(filename: &str) -> Option<Arc<RwLock<io::BufWriter
 pub fn initialize(
     output_file: &str,
     save_output: bool,
+    tx_stats: UnboundedSender<StatCommand>,
 ) -> (
     UnboundedSender<FeroxResponse>,
-    UnboundedSender<String>,
+    UnboundedSender<FeroxResponse>,
     JoinHandle<()>,
     Option<JoinHandle<()>>,
 ) {
-    log::trace!("enter: initialize({}, {})", output_file, save_output);
+    log::trace!(
+        "enter: initialize({}, {}, {:?})",
+        output_file,
+        save_output,
+        tx_stats
+    );
 
     let (tx_rpt, rx_rpt): FeroxChannel<FeroxResponse> = mpsc::unbounded_channel();
-    let (tx_file, rx_file): FeroxChannel<String> = mpsc::unbounded_channel();
+    let (tx_file, rx_file): FeroxChannel<FeroxResponse> = mpsc::unbounded_channel();
 
     let file_clone = tx_file.clone();
+    let stats_clone = tx_stats.clone();
 
-    let term_reporter =
-        tokio::spawn(async move { spawn_terminal_reporter(rx_rpt, file_clone, save_output).await });
+    let term_reporter = tokio::spawn(async move {
+        spawn_terminal_reporter(rx_rpt, file_clone, stats_clone, save_output).await
+    });
 
     let file_reporter = if save_output {
         // -o used, need to spawn the thread for writing to disk
         let file_clone = output_file.to_string();
         Some(tokio::spawn(async move {
-            spawn_file_reporter(rx_file, &file_clone).await
+            spawn_file_reporter(rx_file, tx_stats, &file_clone).await
         }))
     } else {
         None
@@ -81,42 +100,34 @@ pub fn initialize(
 /// reporting criteria
 async fn spawn_terminal_reporter(
     mut resp_chan: UnboundedReceiver<FeroxResponse>,
-    file_chan: UnboundedSender<String>,
+    file_chan: UnboundedSender<FeroxResponse>,
+    tx_stats: UnboundedSender<StatCommand>,
     save_output: bool,
 ) {
     log::trace!(
-        "enter: spawn_terminal_reporter({:?}, {:?}, {})",
+        "enter: spawn_terminal_reporter({:?}, {:?}, {:?}, {})",
         resp_chan,
         file_chan,
+        tx_stats,
         save_output
     );
 
-    while let Some(resp) = resp_chan.recv().await {
+    while let Some(mut resp) = resp_chan.recv().await {
         log::trace!("received {} on reporting channel", resp.url());
 
-        if CONFIGURATION.status_codes.contains(&resp.status().as_u16()) {
-            let report = if CONFIGURATION.quiet {
-                // -q used, just need the url
-                format!("{}\n", resp.url())
-            } else {
-                // normal printing with status and size
-                let status = status_colorizer(&resp.status().as_str());
-                format!(
-                    // example output
-                    // 200       3280 https://localhost.com/FAQ
-                    "{} {:>10} {}\n",
-                    status,
-                    resp.content_length(),
-                    resp.url()
-                )
-            };
+        let contains_sentry = CONFIGURATION.status_codes.contains(&resp.status().as_u16());
+        let unknown_sentry = !RESPONSES.contains(&resp); // !contains == unknown
+        let should_process_response = contains_sentry && unknown_sentry;
 
+        if should_process_response {
             // print to stdout
-            ferox_print(&report, &PROGRESS_PRINTER);
+            ferox_print(&resp.as_str(), &PROGRESS_PRINTER);
+
+            update_stat!(tx_stats, UpdateUsizeField(ResourcesDiscovered, 1));
 
             if save_output {
                 // -o used, need to send the report to be written out to disk
-                match file_chan.send(report.to_string()) {
+                match file_chan.send(resp.clone()) {
                     Ok(_) => {
                         log::debug!("Sent {} to file handler", resp.url());
                     }
@@ -128,18 +139,34 @@ async fn spawn_terminal_reporter(
         }
         log::trace!("report complete: {}", resp.url());
 
-        if CONFIGURATION.replay_client.is_some()
-            && CONFIGURATION.replay_codes.contains(&resp.status().as_u16())
-        {
+        if CONFIGURATION.replay_client.is_some() && should_process_response {
             // replay proxy specified/client created and this response's status code is one that
             // should be replayed
-            match make_request(CONFIGURATION.replay_client.as_ref().unwrap(), &resp.url()).await {
+            match make_request(
+                CONFIGURATION.replay_client.as_ref().unwrap(),
+                &resp.url(),
+                tx_stats.clone(),
+            )
+            .await
+            {
                 Ok(_) => {}
                 Err(e) => {
                     log::error!("{}", e);
                 }
             }
         }
+
+        if should_process_response {
+            // add response to RESPONSES for serialization in case of ctrl+c
+            // placed all by its lonesome like this so that RESPONSES can take ownership
+            // of the FeroxResponse
+
+            // before ownership is transferred, there's no real reason to keep the body anymore
+            // so we can free that piece of data, reducing memory usage
+            resp.text = String::new();
+
+            RESPONSES.insert(resp);
+        }
     }
     log::trace!("exit: spawn_terminal_reporter");
 }
@@ -148,7 +175,11 @@ async fn spawn_terminal_reporter(
 ///
 /// The consumer simply receives responses and writes them to the given output file if they meet
 /// the given reporting criteria
-async fn spawn_file_reporter(mut report_channel: UnboundedReceiver<String>, output_file: &str) {
+async fn spawn_file_reporter(
+    mut report_channel: UnboundedReceiver<FeroxResponse>,
+    tx_stats: UnboundedSender<StatCommand>,
+    output_file: &str,
+) {
     let buffered_file = match get_cached_file_handle(&CONFIGURATION.output) {
         Some(file) => file,
         None => {
@@ -165,47 +196,35 @@ async fn spawn_file_reporter(mut report_channel: UnboundedReceiver<String>, outp
 
     log::info!("Writing scan results to {}", output_file);
 
-    while let Some(report) = report_channel.recv().await {
-        safe_file_write(&report, buffered_file.clone());
+    while let Some(response) = report_channel.recv().await {
+        safe_file_write(&response, buffered_file.clone(), CONFIGURATION.json);
     }
 
+    update_stat!(tx_stats, StatCommand::Save);
+
     log::trace!("exit: spawn_file_reporter");
 }
 
-/// Given the path to a file, open the file in append mode (create it if it doesn't exist) and
-/// return a reference to the file that is buffered and locked
-fn open_file(filename: &str) -> Option<Arc<RwLock<io::BufWriter<fs::File>>>> {
-    log::trace!("enter: open_file({})", filename);
-
-    match fs::OpenOptions::new() // std fs
-        .create(true)
-        .append(true)
-        .open(filename)
-    {
-        Ok(file) => {
-            let writer = io::BufWriter::new(file); // std io
-
-            let locked_file = Some(Arc::new(RwLock::new(writer)));
-
-            log::trace!("exit: open_file -> {:?}", locked_file);
-            locked_file
-        }
-        Err(e) => {
-            log::error!("{}", e);
-            log::trace!("exit: open_file -> None");
-            None
-        }
-    }
-}
-
 /// Given a string and a reference to a locked buffered file, write the contents and flush
 /// the buffer to disk.
-pub fn safe_file_write(contents: &str, locked_file: Arc<RwLock<io::BufWriter<fs::File>>>) {
+pub fn safe_file_write<T>(
+    value: &T,
+    locked_file: Arc<RwLock<io::BufWriter<fs::File>>>,
+    convert_to_json: bool,
+) where
+    T: FeroxSerialize,
+{
     // note to future self: adding logging of anything other than error to this function
     // is a bad idea. we call this function while processing records generated by the logger.
     // If we then call log::... while already processing some logging output, it results in
     // the second log entry being injected into the first.
 
+    let contents = if convert_to_json {
+        value.as_json()
+    } else {
+        value.as_str()
+    };
+
     let contents = strip_ansi_codes(&contents);
 
     if let Ok(mut handle) = locked_file.write() {

src/scanner.rs

@@ -1,26 +1,37 @@
 use crate::{
-    config::CONFIGURATION,
-    extractor::get_links,
-    filters::{FeroxFilter, StatusCodeFilter, WildcardFilter},
-    heuristics, progress,
+    config::{Configuration, CONFIGURATION},
+    extractor::{extract_robots_txt, get_links, request_feroxresponse_from_new_link},
+    filters::{
+        FeroxFilter, LinesFilter, RegexFilter, SimilarityFilter, SizeFilter, StatusCodeFilter,
+        WildcardFilter, WordsFilter,
+    },
+    heuristics,
+    scan_manager::{FeroxResponses, FeroxScans, ScanStatus, PAUSE_SCAN},
+    statistics::{
+        StatCommand::{self, UpdateF64Field, UpdateUsizeField},
+        StatField::{DirScanTimes, ExpectedPerScan, TotalScans, WildcardsFiltered},
+        Stats,
+    },
     utils::{format_url, get_current_depth, make_request},
-    FeroxChannel, FeroxResponse, SLEEP_DURATION,
+    FeroxChannel, FeroxResponse, SIMILARITY_THRESHOLD,
 };
-use console::style;
 use futures::{
     future::{BoxFuture, FutureExt},
     stream, StreamExt,
 };
-use indicatif::{ProgressBar, ProgressStyle};
+use fuzzyhash::FuzzyHash;
 use lazy_static::lazy_static;
-use reqwest::Url;
+use regex::Regex;
+use reqwest::{StatusCode, Url};
+#[cfg(not(test))]
+use std::process::exit;
 use std::{
     collections::HashSet,
     convert::TryInto,
-    io::{stderr, Write},
     ops::Deref,
-    sync::atomic::{AtomicBool, AtomicUsize, Ordering},
+    sync::atomic::{AtomicUsize, Ordering},
     sync::{Arc, RwLock},
+    time::Instant,
 };
 use tokio::{
     sync::{
@@ -28,122 +39,29 @@ use tokio::{
         Semaphore,
     },
     task::JoinHandle,
-    time,
 };
 
-/// Single atomic number that gets incremented once, used to track first scan vs. all others
+/// Single atomic number that gets incremented at least once, used to track first scan(s) vs. all
+/// others found during recursion
+///
+/// -u means this will be incremented once
+/// --stdin means this will be incremented by the number of targets passed via STDIN
 static CALL_COUNT: AtomicUsize = AtomicUsize::new(0);
 
-/// Atomic boolean flag, used to determine whether or not a scan should pause or resume
-pub static PAUSE_SCAN: AtomicBool = AtomicBool::new(false);
-
 lazy_static! {
     /// Set of urls that have been sent to [scan_url](fn.scan_url.html), used for deduplication
-    static ref SCANNED_URLS: RwLock<HashSet<String>> = RwLock::new(HashSet::new());
-
-    /// A clock spinner protected with a RwLock to allow for a single thread to use at a time
-    static ref SINGLE_SPINNER: RwLock<ProgressBar> = RwLock::new(get_single_spinner());
+    pub static ref SCANNED_URLS: FeroxScans = FeroxScans::default();
 
     /// Vector of implementors of the FeroxFilter trait
     static ref FILTERS: Arc<RwLock<Vec<Box<dyn FeroxFilter>>>> = Arc::new(RwLock::new(Vec::<Box<dyn FeroxFilter>>::new()));
 
+    /// Vector of FeroxResponse objects
+    pub static ref RESPONSES: FeroxResponses = FeroxResponses::default();
+
     /// Bounded semaphore used as a barrier to limit concurrent scans
     static ref SCAN_LIMITER: Semaphore = Semaphore::new(CONFIGURATION.scan_limit);
-}
 
-/// Return a clock spinner, used when scans are paused
-fn get_single_spinner() -> ProgressBar {
-    log::trace!("enter: get_single_spinner");
 
-    let spinner = ProgressBar::new_spinner().with_style(
-        ProgressStyle::default_spinner()
-            .tick_strings(&[
-                "🕛", "🕐", "🕑", "🕒", "🕓", "🕔", "🕕", "🕖", "🕗", "🕘", "🕙", "🕚",
-            ])
-            .template(&format!(
-                "\t-= All Scans {{spinner}} {} =-",
-                style("Paused").red()
-            )),
-    );
-
-    log::trace!("exit: get_single_spinner -> {:?}", spinner);
-    spinner
-}
-
-/// Forced the calling thread into a busy loop
-///
-/// Every `SLEEP_DURATION` milliseconds, the function examines the result stored in `PAUSE_SCAN`
-///
-/// When the value stored in `PAUSE_SCAN` becomes `false`, the function returns, exiting the busy
-/// loop
-async fn pause_scan() {
-    log::trace!("enter: pause_scan");
-    // function uses tokio::time, not std
-
-    // local testing showed a pretty slow increase (less than linear) in CPU usage as # of
-    // concurrent scans rose when SLEEP_DURATION was set to 500, using that as the default for now
-    let mut interval = time::interval(time::Duration::from_millis(SLEEP_DURATION));
-
-    // ignore any error returned
-    let _ = stderr().flush();
-
-    if SINGLE_SPINNER.read().unwrap().is_finished() {
-        // in order to not leave draw artifacts laying around in the terminal, we call
-        // finish_and_clear on the progress bar when resuming scans. For this reason, we need to
-        // check if the spinner is finished, and repopulate the RwLock with a new spinner if
-        // necessary
-        if let Ok(mut guard) = SINGLE_SPINNER.write() {
-            *guard = get_single_spinner();
-        }
-    }
-
-    if let Ok(spinner) = SINGLE_SPINNER.write() {
-        spinner.enable_steady_tick(120);
-    }
-
-    loop {
-        // first tick happens immediately, all others wait the specified duration
-        interval.tick().await;
-
-        if !PAUSE_SCAN.load(Ordering::Acquire) {
-            // PAUSE_SCAN is false, so we can exit the busy loop
-            if let Ok(spinner) = SINGLE_SPINNER.write() {
-                spinner.finish_and_clear();
-            }
-            let _ = stderr().flush();
-            log::trace!("exit: pause_scan");
-            return;
-        }
-    }
-}
-
-/// Adds the given url to `SCANNED_URLS`
-///
-/// If `SCANNED_URLS` did not already contain the url, return true; otherwise return false
-fn add_url_to_list_of_scanned_urls(resp: &str, scanned_urls: &RwLock<HashSet<String>>) -> bool {
-    log::trace!(
-        "enter: add_url_to_list_of_scanned_urls({}, {:?})",
-        resp,
-        scanned_urls
-    );
-
-    match scanned_urls.write() {
-        // check new url against what's already been scanned
-        Ok(mut urls) => {
-            // If the set did not contain resp, true is returned.
-            // If the set did contain resp, false is returned.
-            let response = urls.insert(resp.to_string());
-
-            log::trace!("exit: add_url_to_list_of_scanned_urls -> {}", response);
-            response
-        }
-        Err(e) => {
-            // poisoned lock
-            log::error!("Set of scanned urls poisoned: {}", e);
-            log::trace!("exit: add_url_to_list_of_scanned_urls -> false");
-            false
-        }
-    }
 }
 
 /// Adds the given FeroxFilter to the given list of FeroxFilter implementors
@@ -189,46 +107,64 @@ fn spawn_recursion_handler(
     mut recursion_channel: UnboundedReceiver<String>,
     wordlist: Arc<HashSet<String>>,
     base_depth: usize,
+    stats: Arc<Stats>,
     tx_term: UnboundedSender<FeroxResponse>,
-    tx_file: UnboundedSender<String>,
-) -> BoxFuture<'static, Vec<JoinHandle<()>>> {
+    tx_file: UnboundedSender<FeroxResponse>,
+    tx_stats: UnboundedSender<StatCommand>,
+) -> BoxFuture<'static, Vec<Arc<JoinHandle<()>>>> {
     log::trace!(
-        "enter: spawn_recursion_handler({:?}, wordlist[{} words...], {}, {:?}, {:?})",
+        "enter: spawn_recursion_handler({:?}, wordlist[{} words...], {}, {:?}, {:?}, {:?}, {:?})",
         recursion_channel,
         wordlist.len(),
         base_depth,
+        stats,
         tx_term,
-        tx_file
+        tx_file,
+        tx_stats
     );
 
     let boxed_future = async move {
         let mut scans = vec![];
 
         while let Some(resp) = recursion_channel.recv().await {
-            let unknown = add_url_to_list_of_scanned_urls(&resp, &SCANNED_URLS);
+            let (unknown, scan) = SCANNED_URLS.add_directory_scan(&resp, stats.clone());
 
             if !unknown {
                 // not unknown, i.e. we've seen the url before and don't need to scan again
                 continue;
             }
 
+            update_stat!(tx_stats, UpdateUsizeField(TotalScans, 1));
+
             log::info!("received {} on recursion channel", resp);
 
             let term_clone = tx_term.clone();
             let file_clone = tx_file.clone();
+            let tx_stats_clone = tx_stats.clone();
+            let stats_clone = stats.clone();
             let resp_clone = resp.clone();
             let list_clone = wordlist.clone();
 
-            scans.push(tokio::spawn(async move {
+            let future = tokio::spawn(async move {
                 scan_url(
                     resp_clone.to_owned().as_str(),
                     list_clone,
                     base_depth,
+                    stats_clone,
                     term_clone,
                     file_clone,
+                    tx_stats_clone,
                 )
                 .await
-            }));
+            });
+
+            let shared_task = Arc::new(future);
+
+            if let Ok(mut u_scan) = scan.lock() {
+                u_scan.task = Some(shared_task.clone());
+            }
+
+            scans.push(shared_task);
         }
         scans
     }
@@ -244,12 +180,18 @@ fn spawn_recursion_handler(
 ///
 /// If any extensions were passed to the program, each extension will add a
 /// (base_url + word + ext) Url to the vector
-fn create_urls(target_url: &str, word: &str, extensions: &[String]) -> Vec<Url> {
+fn create_urls(
+    target_url: &str,
+    word: &str,
+    extensions: &[String],
+    tx_stats: UnboundedSender<StatCommand>,
+) -> Vec<Url> {
     log::trace!(
-        "enter: create_urls({}, {}, {:?})",
+        "enter: create_urls({}, {}, {:?}, {:?})",
         target_url,
         word,
-        extensions
+        extensions,
+        tx_stats
     );
 
     let mut urls = vec![];
@@ -260,6 +202,7 @@ fn create_urls(target_url: &str, word: &str, extensions: &[String]) -> Vec<Url>
         CONFIGURATION.add_slash,
         &CONFIGURATION.queries,
         None,
+        tx_stats.clone(),
     ) {
         urls.push(url); // default request, i.e. no extension
     }
@@ -271,6 +214,7 @@ fn create_urls(target_url: &str, word: &str, extensions: &[String]) -> Vec<Url>
             CONFIGURATION.add_slash,
             &CONFIGURATION.queries,
             Some(ext),
+            tx_stats.clone(),
         ) {
             urls.push(url); // any extensions passed in
         }
@@ -285,7 +229,7 @@ fn create_urls(target_url: &str, word: &str, extensions: &[String]) -> Vec<Url>
 /// handles 2xx and 3xx responses by either checking if the url ends with a / (2xx)
 /// or if the Location header is present and matches the base url + / (3xx)
 fn response_is_directory(response: &FeroxResponse) -> bool {
-    log::trace!("enter: is_directory({:?})", response);
+    log::trace!("enter: is_directory({})", response);
 
     if response.status().is_redirection() {
         // status code is 3xx
@@ -311,16 +255,14 @@ fn response_is_directory(response: &FeroxResponse) -> bool {
                 }
             }
             None => {
-                log::debug!(
-                    "expected Location header, but none was found: {:?}",
-                    response
-                );
+                log::debug!("expected Location header, but none was found: {}", response);
                 log::trace!("exit: is_directory -> false");
                 return false;
             }
         }
-    } else if response.status().is_success() {
-        // status code is 2xx, need to check if it ends in /
+    } else if response.status().is_success() || matches!(response.status(), &StatusCode::FORBIDDEN)
+    {
+        // status code is 2xx or 403, need to check if it ends in /
 
         if response.url().as_str().ends_with('/') {
             log::debug!("{} is directory suitable for recursion", response.url());
@@ -370,10 +312,10 @@ async fn try_recursion(
     transmitter: UnboundedSender<String>,
 ) {
     log::trace!(
-        "enter: try_recursion({:?}, {}, {:?})",
+        "enter: try_recursion({}, {}, {:?})",
         response,
         base_depth,
-        transmitter
+        transmitter,
     );
 
     if !reached_max_depth(response.url(), base_depth, CONFIGURATION.depth)
@@ -417,22 +359,18 @@ async fn try_recursion(
 
 /// Simple helper to stay DRY; determines whether or not a given `FeroxResponse` should be reported
 /// to the user or not.
-pub fn should_filter_response(response: &FeroxResponse) -> bool {
-    if CONFIGURATION
-        .filter_size
-        .contains(&response.content_length())
-    {
-        // filtered value from --filter-size, size filters and wildcards are two separate filters
-        // and are applied independently
-        log::debug!("size filter: filtered out {}", response.url());
-        return true;
-    }
-
+pub fn should_filter_response(
+    response: &FeroxResponse,
+    tx_stats: UnboundedSender<StatCommand>,
+) -> bool {
     match FILTERS.read() {
         Ok(filters) => {
             for filter in filters.iter() {
                 // wildcard.should_filter goes here
                 if filter.should_filter_response(&response) {
+                    if filter.as_any().downcast_ref::<WildcardFilter>().is_some() {
+                        update_stat!(tx_stats, UpdateUsizeField(WildcardsFiltered, 1))
+                    }
                     return true;
                 }
             }
@@ -453,24 +391,33 @@ async fn make_requests(
     target_url: &str,
     word: &str,
     base_depth: usize,
+    stats: Arc<Stats>,
     dir_chan: UnboundedSender<String>,
     report_chan: UnboundedSender<FeroxResponse>,
+    tx_stats: UnboundedSender<StatCommand>,
 ) {
     log::trace!(
-        "enter: make_requests({}, {}, {}, {:?}, {:?})",
+        "enter: make_requests({}, {}, {}, {:?}, {:?}, {:?}, {:?})",
         target_url,
         word,
         base_depth,
+        stats,
         dir_chan,
-        report_chan
+        report_chan,
+        tx_stats
     );
 
-    let urls = create_urls(&target_url, &word, &CONFIGURATION.extensions);
+    let urls = create_urls(
+        &target_url,
+        &word,
+        &CONFIGURATION.extensions,
+        tx_stats.clone(),
+    );
 
     for url in urls {
-        if let Ok(response) = make_request(&CONFIGURATION.client, &url).await {
+        if let Ok(response) = make_request(&CONFIGURATION.client, &url, tx_stats.clone()).await {
             // response came back without error, convert it to FeroxResponse
-            let ferox_response = FeroxResponse::from(response, CONFIGURATION.extract_links).await;
+            let ferox_response = FeroxResponse::from(response, true).await;
 
             // do recursion if appropriate
             if !CONFIGURATION.no_recursion {
@@ -480,54 +427,35 @@ async fn make_requests(
             // purposefully doing recursion before filtering. the thought process is that
             // even though this particular url is filtered, subsequent urls may not
 
-            if should_filter_response(&ferox_response) {
+            if should_filter_response(&ferox_response, tx_stats.clone()) {
                 continue;
             }
 
             if CONFIGURATION.extract_links && !ferox_response.status().is_redirection() {
-                let new_links = get_links(&ferox_response).await;
+                let new_links = get_links(&ferox_response, tx_stats.clone()).await;
 
                 for new_link in new_links {
-                    let unknown = add_url_to_list_of_scanned_urls(&new_link, &SCANNED_URLS);
-
-                    if !unknown {
-                        // not unknown, i.e. we've seen the url before and don't need to scan again
-                        continue;
-                    }
-
-                    // create a url based on the given command line options, continue on error
-                    let new_url = match format_url(
+                    let mut new_ferox_response = match request_feroxresponse_from_new_link(
                         &new_link,
-                        &"",
-                        CONFIGURATION.add_slash,
-                        &CONFIGURATION.queries,
-                        None,
-                    ) {
-                        Ok(url) => url,
-                        Err(_) => continue,
+                        tx_stats.clone(),
+                    )
+                    .await
+                    {
+                        Some(resp) => resp,
+                        None => continue,
                     };
 
-                    // make the request and store the response
-                    let new_response = match make_request(&CONFIGURATION.client, &new_url).await {
-                        Ok(resp) => resp,
-                        Err(_) => continue,
-                    };
-
-                    let mut new_ferox_response =
-                        FeroxResponse::from(new_response, CONFIGURATION.extract_links).await;
-
                     // filter if necessary
-                    if should_filter_response(&new_ferox_response) {
+                    if should_filter_response(&new_ferox_response, tx_stats.clone()) {
                         continue;
                     }
 
                     if new_ferox_response.is_file() {
                         // very likely a file, simply request and report
-                        log::debug!(
-                            "Singular extraction: {} ({})",
-                            new_ferox_response.url(),
-                            new_ferox_response.status().as_str(),
-                        );
+                        log::debug!("Singular extraction: {}", new_ferox_response);
+
+                        SCANNED_URLS
+                            .add_file_scan(&new_ferox_response.url().to_string(), stats.clone());
 
                         send_report(report_chan.clone(), new_ferox_response);
 
@@ -535,16 +463,16 @@ async fn make_requests(
                     }
 
                     if !CONFIGURATION.no_recursion {
-                        log::debug!(
-                            "Recursive extraction: {} ({})",
-                            new_ferox_response.url(),
-                            new_ferox_response.status().as_str()
-                        );
+                        log::debug!("Recursive extraction: {}", new_ferox_response);
 
-                        if new_ferox_response.status().is_success()
-                            && !new_ferox_response.url().as_str().ends_with('/')
+                        if !new_ferox_response.url().as_str().ends_with('/')
+                            && (new_ferox_response.status().is_success()
+                                || matches!(new_ferox_response.status(), &StatusCode::FORBIDDEN))
                         {
-                            // since all of these are 2xx, recursion is only attempted if the
+                            // if the url doesn't end with a /
+                            // and the response code is either a 2xx or 403
+
+                            // since all of these are 2xx or 403, recursion is only attempted if the
                             // url ends in a /. I am actually ok with adding the slash and not
                             // adding it, as both have merit.  Leaving it in for now to see how
                             // things turn out (current as of: v1.1.0)
@@ -564,8 +492,8 @@ async fn make_requests(
 }
 
 /// Simple helper to send a `FeroxResponse` over the tx side of an `mpsc::unbounded_channel`
-fn send_report(report_sender: UnboundedSender<FeroxResponse>, response: FeroxResponse) {
-    log::trace!("enter: send_report({:?}, {:?}", report_sender, response);
+pub fn send_report(report_sender: UnboundedSender<FeroxResponse>, response: FeroxResponse) {
+    log::trace!("enter: send_report({:?}, {}", report_sender, response);
 
     match report_sender.send(response) {
         Ok(_) => {}
@@ -577,6 +505,61 @@ fn send_report(report_sender: UnboundedSender<FeroxResponse>, response: FeroxRes
     log::trace!("exit: send_report");
 }
 
+/// Request /robots.txt from given url
+async fn scan_robots_txt(
+    target_url: &str,
+    base_depth: usize,
+    stats: Arc<Stats>,
+    tx_term: UnboundedSender<FeroxResponse>,
+    tx_dir: UnboundedSender<String>,
+    tx_stats: UnboundedSender<StatCommand>,
+) {
+    log::trace!(
+        "enter: scan_robots_txt({}, {}, {:?}, {:?}, {:?}, {:?})",
+        target_url,
+        base_depth,
+        stats,
+        tx_term,
+        tx_dir,
+        tx_stats
+    );
+
+    let robots_links = extract_robots_txt(&target_url, &CONFIGURATION, tx_stats.clone()).await;
+
+    for robot_link in robots_links {
+        // create a url based on the given command line options, continue on error
+        let mut ferox_response =
+            match request_feroxresponse_from_new_link(&robot_link, tx_stats.clone()).await {
+                Some(resp) => resp,
+                None => continue,
+            };
+
+        if should_filter_response(&ferox_response, tx_stats.clone()) {
+            continue;
+        }
+
+        if ferox_response.is_file() {
+            log::debug!("File extracted from robots.txt: {}", ferox_response);
+            SCANNED_URLS.add_file_scan(&robot_link, stats.clone());
+            send_report(tx_term.clone(), ferox_response);
+        } else if !CONFIGURATION.no_recursion {
+            log::debug!("Directory extracted from robots.txt: {}", ferox_response);
+            // todo this code is essentially the same as another piece around ~467 of this file
+            if !ferox_response.url().as_str().ends_with('/')
+                && (ferox_response.status().is_success()
+                    || matches!(ferox_response.status(), &StatusCode::FORBIDDEN))
+            {
+                // if the url doesn't end with a /
+                // and the response code is either a 2xx or 403
+                ferox_response.set_url(&format!("{}/", ferox_response.url()));
+            }
+
+            try_recursion(&ferox_response, base_depth, tx_dir.clone()).await;
+        }
+    }
+    log::trace!("exit: scan_robots_txt");
+}
+
 /// Scan a given url using a given wordlist
 ///
 /// This is the primary entrypoint for the scanner
@@ -584,47 +567,76 @@ pub async fn scan_url(
     target_url: &str,
     wordlist: Arc<HashSet<String>>,
     base_depth: usize,
+    stats: Arc<Stats>,
     tx_term: UnboundedSender<FeroxResponse>,
-    tx_file: UnboundedSender<String>,
+    tx_file: UnboundedSender<FeroxResponse>,
+    tx_stats: UnboundedSender<StatCommand>,
 ) {
     log::trace!(
-        "enter: scan_url({:?}, wordlist[{} words...], {}, {:?}, {:?})",
+        "enter: scan_url({:?}, wordlist[{} words...], {}, {:?}, {:?}, {:?}, {:?})",
         target_url,
         wordlist.len(),
         base_depth,
+        stats,
         tx_term,
-        tx_file
+        tx_file,
+        tx_stats
     );
 
     log::info!("Starting scan against: {}", target_url);
 
+    let scan_timer = Instant::now();
+
     let (tx_dir, rx_dir): FeroxChannel<String> = mpsc::unbounded_channel();
 
-    let num_reqs_expected: u64 = if CONFIGURATION.extensions.is_empty() {
-        wordlist.len().try_into().unwrap()
-    } else {
-        let total = wordlist.len() * (CONFIGURATION.extensions.len() + 1);
-        total.try_into().unwrap()
-    };
-
-    let progress_bar = progress::add_bar(&target_url, num_reqs_expected, false);
-    progress_bar.reset_elapsed();
-
-    if CALL_COUNT.load(Ordering::Relaxed) == 0 {
+    if CALL_COUNT.load(Ordering::Relaxed) < stats.initial_targets.load(Ordering::Relaxed) {
         CALL_COUNT.fetch_add(1, Ordering::Relaxed);
 
+        if CONFIGURATION.extract_links {
+            // only grab robots.txt on the initial scan_url calls. all fresh dirs will be passed
+            // to try_recursion
+            scan_robots_txt(
+                target_url,
+                base_depth,
+                stats.clone(),
+                tx_term.clone(),
+                tx_dir.clone(),
+                tx_stats.clone(),
+            )
+            .await;
+        }
+
+        update_stat!(tx_stats, UpdateUsizeField(TotalScans, 1));
+
         // this protection allows us to add the first scanned url to SCANNED_URLS
         // from within the scan_url function instead of the recursion handler
-        add_url_to_list_of_scanned_urls(&target_url, &SCANNED_URLS);
-
-        if CONFIGURATION.scan_limit == 0 {
-            // scan_limit == 0 means no limit should be imposed... however, scoping the Semaphore
-            // permit is tricky, so as a workaround, we'll add a ridiculous number of permits to
-            // the semaphore (1,152,921,504,606,846,975 to be exact) and call that 'unlimited'
-            SCAN_LIMITER.add_permits(usize::MAX >> 4);
-        }
+        SCANNED_URLS.add_directory_scan(&target_url, stats.clone());
     }
 
+    let ferox_scan = match SCANNED_URLS.get_scan_by_url(&target_url) {
+        Some(scan) => {
+            if let Ok(mut u_scan) = scan.lock() {
+                u_scan.status = ScanStatus::Running;
+            }
+            scan
+        }
+        None => {
+            log::error!(
+                "Could not find FeroxScan associated with {}; this shouldn't happen... exiting",
+                target_url
+            );
+            return;
+        }
+    };
+
+    let progress_bar = match ferox_scan.lock() {
+        Ok(mut scan) => scan.progress_bar(),
+        Err(e) => {
+            log::error!("FeroxScan's ({:?}) mutex is poisoned: {}", ferox_scan, e);
+            return;
+        }
+    };
+
     // When acquire is called and the semaphore has remaining permits, the function immediately
     // returns a permit. However, if no remaining permits are available, acquire (asynchronously)
     // waits until an outstanding permit is dropped. At this point, the freed permit is assigned
@@ -633,57 +645,61 @@ pub async fn scan_url(
 
     // Arc clones to be passed around to the various scans
     let wildcard_bar = progress_bar.clone();
-    let heuristics_file_clone = tx_file.clone();
+    let heuristics_term_clone = tx_term.clone();
+    let heuristics_stats_clone = tx_stats.clone();
     let recurser_term_clone = tx_term.clone();
     let recurser_file_clone = tx_file.clone();
+    let recurser_stats_clone = tx_stats.clone();
     let recurser_words = wordlist.clone();
     let looping_words = wordlist.clone();
+    let looping_stats = stats.clone();
 
     let recurser = tokio::spawn(async move {
         spawn_recursion_handler(
             rx_dir,
             recurser_words,
             base_depth,
+            stats.clone(),
             recurser_term_clone,
             recurser_file_clone,
+            recurser_stats_clone,
         )
         .await
     });
 
     // add any wildcard filters to `FILTERS`
-    let filter =
-        match heuristics::wildcard_test(&target_url, wildcard_bar, heuristics_file_clone).await {
-            Some(f) => Box::new(f),
-            None => Box::new(WildcardFilter::default()),
-        };
+    let filter = match heuristics::wildcard_test(
+        &target_url,
+        wildcard_bar,
+        heuristics_term_clone,
+        heuristics_stats_clone,
+    )
+    .await
+    {
+        Some(f) => Box::new(f),
+        None => Box::new(WildcardFilter::default()),
+    };
 
     add_filter_to_list_of_ferox_filters(filter, FILTERS.clone());
 
-    // add any status code filters to `FILTERS`
-    for code_filter in &CONFIGURATION.filter_status {
-        let filter = StatusCodeFilter {
-            filter_code: *code_filter,
-        };
-        let boxed_filter = Box::new(filter);
-        add_filter_to_list_of_ferox_filters(boxed_filter, FILTERS.clone());
-    }
-
     // producer tasks (mp of mpsc); responsible for making requests
     let producers = stream::iter(looping_words.deref().to_owned())
         .map(|word| {
             let txd = tx_dir.clone();
             let txr = tx_term.clone();
+            let txs = tx_stats.clone();
             let pb = progress_bar.clone(); // progress bar is an Arc around internal state
             let tgt = target_url.to_string(); // done to satisfy 'static lifetime below
+            let lst = looping_stats.clone();
             (
                 tokio::spawn(async move {
                     if PAUSE_SCAN.load(Ordering::Acquire) {
                         // for every word in the wordlist, check to see if PAUSE_SCAN is set to true
                         // when true; enter a busy loop that only exits by setting PAUSE_SCAN back
                         // to false
-                        pause_scan().await;
+                        SCANNED_URLS.pause(true).await;
                     }
-                    make_requests(&tgt, &word, base_depth, txd, txr).await
+                    make_requests(&tgt, &word, base_depth, lst, txd, txr, txs).await
                 }),
                 pb,
             )
@@ -704,23 +720,156 @@ pub async fn scan_url(
     producers.await;
     log::trace!("done awaiting scan producers");
 
+    update_stat!(
+        tx_stats,
+        UpdateF64Field(DirScanTimes, scan_timer.elapsed().as_secs_f64())
+    );
+
     // drop the current permit so the semaphore will allow another scan to proceed
     drop(permit);
 
-    progress_bar.finish();
+    if let Ok(mut scan) = ferox_scan.lock() {
+        scan.finish();
+    }
 
     // manually drop tx in order for the rx task's while loops to eval to false
     log::trace!("dropped recursion handler's transmitter");
     drop(tx_dir);
 
-    // await rx tasks
-    log::trace!("awaiting recursive scan receiver/scans");
-    futures::future::join_all(recurser.await.unwrap()).await;
-    log::trace!("done awaiting recursive scan receiver/scans");
+    // note: in v1.11.2 i removed the join_all call that used to handle the recurser handles.
+    // nothing appears to change by having them removed, however, if ever a revert is needed
+    // this is the place and anything prior to 1.11.2 will have the code to do so
+    let _ = recurser.await.unwrap_or_default();
 
     log::trace!("exit: scan_url");
 }
 
+/// Perform steps necessary to run scans that only need to be performed once (warming up the
+/// engine, as it were)
+pub async fn initialize(
+    num_words: usize,
+    config: &Configuration,
+    tx_stats: UnboundedSender<StatCommand>,
+) {
+    log::trace!(
+        "enter: initialize({}, {:?}, {:?})",
+        num_words,
+        config,
+        tx_stats
+    );
+
+    // number of requests only needs to be calculated once, and then can be reused
+    let num_reqs_expected: u64 = if config.extensions.is_empty() {
+        num_words.try_into().unwrap()
+    } else {
+        let total = num_words * (config.extensions.len() + 1);
+        total.try_into().unwrap()
+    };
+
+    // tell Stats object about the number of expected requests
+    update_stat!(
+        tx_stats,
+        UpdateUsizeField(ExpectedPerScan, num_reqs_expected as usize)
+    );
+
+    // add any status code filters to `FILTERS` (-C|--filter-status)
+    for code_filter in &config.filter_status {
+        let filter = StatusCodeFilter {
+            filter_code: *code_filter,
+        };
+        let boxed_filter = Box::new(filter);
+        add_filter_to_list_of_ferox_filters(boxed_filter, FILTERS.clone());
+    }
+
+    // add any line count filters to `FILTERS` (-N|--filter-lines)
+    for lines_filter in &config.filter_line_count {
+        let filter = LinesFilter {
+            line_count: *lines_filter,
+        };
+        let boxed_filter = Box::new(filter);
+        add_filter_to_list_of_ferox_filters(boxed_filter, FILTERS.clone());
+    }
+
+    // add any line count filters to `FILTERS` (-W|--filter-words)
+    for words_filter in &config.filter_word_count {
+        let filter = WordsFilter {
+            word_count: *words_filter,
+        };
+        let boxed_filter = Box::new(filter);
+        add_filter_to_list_of_ferox_filters(boxed_filter, FILTERS.clone());
+    }
+
+    // add any line count filters to `FILTERS` (-S|--filter-size)
+    for size_filter in &config.filter_size {
+        let filter = SizeFilter {
+            content_length: *size_filter,
+        };
+        let boxed_filter = Box::new(filter);
+        add_filter_to_list_of_ferox_filters(boxed_filter, FILTERS.clone());
+    }
+
+    // add any regex filters to `FILTERS` (-X|--filter-regex)
+    for regex_filter in &config.filter_regex {
+        let raw = regex_filter;
+        let compiled = match Regex::new(&raw) {
+            Ok(regex) => regex,
+            Err(e) => {
+                log::error!("Invalid regular expression: {}", e);
+                #[cfg(test)]
+                panic!();
+                #[cfg(not(test))]
+                exit(1);
+            }
+        };
+
+        let filter = RegexFilter {
+            raw_string: raw.to_owned(),
+            compiled,
+        };
+        let boxed_filter = Box::new(filter);
+        add_filter_to_list_of_ferox_filters(boxed_filter, FILTERS.clone());
+    }
+
+    // add any similarity filters to `FILTERS` (--filter-similar-to)
+    for similarity_filter in &config.filter_similar {
+        // url as-is based on input, ignores user-specified url manipulation options (add-slash etc)
+        if let Ok(url) = format_url(
+            &similarity_filter,
+            &"",
+            false,
+            &Vec::new(),
+            None,
+            tx_stats.clone(),
+        ) {
+            // attempt to request the given url
+            if let Ok(resp) = make_request(&CONFIGURATION.client, &url, tx_stats.clone()).await {
+                // if successful, create a filter based on the response's body
+                let fr = FeroxResponse::from(resp, true).await;
+
+                // hash the response body and store the resulting hash in the filter object
+                let hash = FuzzyHash::new(&fr.text()).to_string();
+
+                let filter = SimilarityFilter {
+                    text: hash,
+                    threshold: SIMILARITY_THRESHOLD,
+                };
+
+                let boxed_filter = Box::new(filter);
+                add_filter_to_list_of_ferox_filters(boxed_filter, FILTERS.clone());
+            }
+        }
+    }
+
+    if config.scan_limit == 0 {
+        // scan_limit == 0 means no limit should be imposed... however, scoping the Semaphore
+        // permit is tricky, so as a workaround, we'll add a ridiculous number of permits to
+        // the semaphore (1,152,921,504,606,846,975 to be exact) and call that 'unlimited'
+        SCAN_LIMITER.add_permits(usize::MAX >> 4);
+    }
+
+    log::trace!("exit: initialize");
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -728,14 +877,16 @@ mod tests {
     #[test]
     /// sending url + word without any extensions should get back one url with the joined word
     fn create_urls_no_extension_returns_base_url_with_word() {
-        let urls = create_urls("http://localhost", "turbo", &[]);
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+        let urls = create_urls("http://localhost", "turbo", &[], tx);
         assert_eq!(urls, [Url::parse("http://localhost/turbo").unwrap()])
     }
 
     #[test]
     /// sending url + word + 1 extension should get back two urls, one base and one with extension
     fn create_urls_one_extension_returns_two_urls() {
-        let urls = create_urls("http://localhost", "turbo", &[String::from("js")]);
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+        let urls = create_urls("http://localhost", "turbo", &[String::from("js")], tx);
         assert_eq!(
             urls,
             [
@@ -773,8 +924,10 @@ mod tests {
             vec![base, js, php, pdf, tar],
         ];
 
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
         for (i, ext_set) in ext_vec.into_iter().enumerate() {
-            let urls = create_urls("http://localhost", "turbo", &ext_set);
+            let urls = create_urls("http://localhost", "turbo", &ext_set, tx.clone());
             assert_eq!(urls, expected[i]);
         }
     }
@@ -819,67 +972,15 @@ mod tests {
         assert!(result);
     }
 
-    #[test]
-    /// add an unknown url to the hashset, expect true
-    fn add_url_to_list_of_scanned_urls_with_unknown_url() {
-        let urls = RwLock::new(HashSet::<String>::new());
-        let url = "http://unknown_url";
-        assert_eq!(add_url_to_list_of_scanned_urls(url, &urls), true);
-    }
-
-    #[test]
-    /// add a known url to the hashset, with a trailing slash, expect false
-    fn add_url_to_list_of_scanned_urls_with_known_url() {
-        let urls = RwLock::new(HashSet::<String>::new());
-        let url = "http://unknown_url/";
-
-        assert_eq!(urls.write().unwrap().insert(url.to_string()), true);
-
-        assert_eq!(add_url_to_list_of_scanned_urls(url, &urls), false);
-    }
-
-    #[test]
-    /// add a known url to the hashset, without a trailing slash, expect false
-    fn add_url_to_list_of_scanned_urls_with_known_url_without_slash() {
-        let urls = RwLock::new(HashSet::<String>::new());
-        let url = "http://unknown_url";
-
-        assert_eq!(
-            urls.write()
-                .unwrap()
-                .insert("http://unknown_url".to_string()),
-            true
-        );
-
-        assert_eq!(add_url_to_list_of_scanned_urls(url, &urls), false);
-    }
-
-    #[test]
-    /// test that get_single_spinner returns the correct spinner
-    fn scanner_get_single_spinner_returns_spinner() {
-        let spinner = get_single_spinner();
-        assert!(!spinner.is_finished());
-    }
-
-    #[tokio::test(core_threads = 1)]
-    /// tests that pause_scan pauses execution and releases execution when PAUSE_SCAN is toggled
-    /// the spinner used during the test has had .finish_and_clear called on it, meaning that
-    /// a new one will be created, taking the if branch within the function
-    async fn scanner_pause_scan_with_finished_spinner() {
-        let now = time::Instant::now();
-
-        PAUSE_SCAN.store(true, Ordering::Relaxed);
-        SINGLE_SPINNER.write().unwrap().finish_and_clear();
-
-        let expected = time::Duration::from_secs(2);
-
-        tokio::spawn(async move {
-            time::delay_for(expected).await;
-            PAUSE_SCAN.store(false, Ordering::Relaxed);
-        });
-
-        pause_scan().await;
-
-        assert!(now.elapsed() > expected);
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    #[should_panic]
+    /// call initialize with a bad regex, triggering a panic
+    async fn initialize_panics_on_bad_regex() {
+        let config = Configuration {
+            filter_regex: vec![r"(".to_string()],
+            ..Default::default()
+        };
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+        initialize(1, &config, tx).await;
     }
 }

src/statistics.rs

@@ -0,0 +1,833 @@
+use crate::{
+    config::CONFIGURATION,
+    progress::{add_bar, BarType},
+    reporter::{get_cached_file_handle, safe_file_write},
+    FeroxChannel, FeroxSerialize,
+};
+use console::style;
+use indicatif::ProgressBar;
+use reqwest::StatusCode;
+use serde::{Deserialize, Serialize};
+use std::{
+    fs::File,
+    io::BufReader,
+    sync::{
+        atomic::{AtomicUsize, Ordering},
+        Arc, Mutex,
+    },
+    time::Instant,
+};
+use tokio::{
+    sync::mpsc::{self, UnboundedReceiver, UnboundedSender},
+    task::JoinHandle,
+};
+
+/// Wrapper `Atomic*.fetch_add` to save me from writing Ordering::Relaxed a bajillion times
+///
+/// default is to increment by 1, second arg can be used to increment by a different value
+macro_rules! atomic_increment {
+    ($metric:expr) => {
+        $metric.fetch_add(1, Ordering::Relaxed);
+    };
+
+    ($metric:expr, $value:expr) => {
+        $metric.fetch_add($value, Ordering::Relaxed);
+    };
+}
+
+/// Wrapper around `Atomic*.load` to save me from writing Ordering::Relaxed a bajillion times
+macro_rules! atomic_load {
+    ($metric:expr) => {
+        $metric.load(Ordering::Relaxed);
+    };
+}
+
+/// Data collection of statistics related to a scan
+#[derive(Default, Deserialize, Debug, Serialize)]
+pub struct Stats {
+    #[serde(rename = "type")]
+    /// Name of this type of struct, used for serialization, i.e. `{"type":"statistics"}`
+    kind: String,
+
+    /// tracker for number of timeouts seen by the client
+    timeouts: AtomicUsize,
+
+    /// tracker for total number of requests sent by the client
+    requests: AtomicUsize,
+
+    /// tracker for total number of requests expected to send if the scan runs to completion
+    ///
+    /// Note: this is a per-scan expectation; `expected_requests * current # of scans` would be
+    /// indicative of the current expectation at any given time, but is a moving target.  
+    pub expected_per_scan: AtomicUsize,
+
+    /// tracker for accumulating total number of requests expected (i.e. as a new scan is started
+    /// this value should increase by `expected_requests`
+    total_expected: AtomicUsize,
+
+    /// tracker for total number of errors encountered by the client
+    errors: AtomicUsize,
+
+    /// tracker for overall number of 2xx status codes seen by the client
+    successes: AtomicUsize,
+
+    /// tracker for overall number of 3xx status codes seen by the client
+    redirects: AtomicUsize,
+
+    /// tracker for overall number of 4xx status codes seen by the client
+    client_errors: AtomicUsize,
+
+    /// tracker for overall number of 5xx status codes seen by the client
+    server_errors: AtomicUsize,
+
+    /// tracker for number of scans performed, this directly equates to number of directories
+    /// recursed into and affects the total number of expected requests
+    total_scans: AtomicUsize,
+
+    /// tracker for initial number of requested targets
+    pub initial_targets: AtomicUsize,
+
+    /// tracker for number of links extracted when `--extract-links` is used; sources are
+    /// response bodies and robots.txt as of v1.11.0
+    links_extracted: AtomicUsize,
+
+    /// tracker for overall number of 200s seen by the client
+    status_200s: AtomicUsize,
+
+    /// tracker for overall number of 301s seen by the client
+    status_301s: AtomicUsize,
+
+    /// tracker for overall number of 302s seen by the client
+    status_302s: AtomicUsize,
+
+    /// tracker for overall number of 401s seen by the client
+    status_401s: AtomicUsize,
+
+    /// tracker for overall number of 403s seen by the client
+    status_403s: AtomicUsize,
+
+    /// tracker for overall number of 429s seen by the client
+    status_429s: AtomicUsize,
+
+    /// tracker for overall number of 500s seen by the client
+    status_500s: AtomicUsize,
+
+    /// tracker for overall number of 503s seen by the client
+    status_503s: AtomicUsize,
+
+    /// tracker for overall number of 504s seen by the client
+    status_504s: AtomicUsize,
+
+    /// tracker for overall number of 508s seen by the client
+    status_508s: AtomicUsize,
+
+    /// tracker for overall number of wildcard urls filtered out by the client
+    wildcards_filtered: AtomicUsize,
+
+    /// tracker for overall number of all filtered responses
+    responses_filtered: AtomicUsize,
+
+    /// tracker for number of files found
+    resources_discovered: AtomicUsize,
+
+    /// tracker for number of errors triggered during URL formatting
+    url_format_errors: AtomicUsize,
+
+    /// tracker for number of errors triggered by the `reqwest::RedirectPolicy`
+    redirection_errors: AtomicUsize,
+
+    /// tracker for number of errors related to the connecting
+    connection_errors: AtomicUsize,
+
+    /// tracker for number of errors related to the request used
+    request_errors: AtomicUsize,
+
+    /// tracker for each directory's total scan time in seconds as a float
+    directory_scan_times: Mutex<Vec<f64>>,
+
+    /// tracker for total runtime
+    total_runtime: Mutex<Vec<f64>>,
+}
+
+/// FeroxSerialize implementation for Stats
+impl FeroxSerialize for Stats {
+    /// Simply return empty string here to disable serializing this to the output file as a string
+    /// due to it looking like garbage
+    fn as_str(&self) -> String {
+        String::new()
+    }
+
+    /// Simple call to produce a JSON string using the given Stats object
+    fn as_json(&self) -> String {
+        serde_json::to_string(&self).unwrap_or_default()
+    }
+}
+
+/// implementation of statistics data collection struct
+impl Stats {
+    /// Small wrapper for default to set `kind` to "statistics" and `total_runtime` to have at least
+    /// one value
+    pub fn new() -> Self {
+        Self {
+            kind: String::from("statistics"),
+            total_runtime: Mutex::new(vec![0.0]),
+            ..Default::default()
+        }
+    }
+
+    /// increment `requests` field by one
+    fn add_request(&self) {
+        atomic_increment!(self.requests);
+    }
+
+    /// given an `Instant` update total runtime
+    fn update_runtime(&self, seconds: f64) {
+        if let Ok(mut runtime) = self.total_runtime.lock() {
+            runtime[0] = seconds;
+        }
+    }
+
+    /// save an instance of `Stats` to disk after updating the total runtime for the scan
+    fn save(&self, seconds: f64, location: &str) {
+        let buffered_file = match get_cached_file_handle(location) {
+            Some(file) => file,
+            None => {
+                return;
+            }
+        };
+
+        self.update_runtime(seconds);
+
+        safe_file_write(self, buffered_file, CONFIGURATION.json);
+    }
+
+    /// Inspect the given `StatError` and increment the appropriate fields
+    ///
+    /// Implies incrementing:
+    ///     - requests
+    ///     - errors
+    pub fn add_error(&self, error: StatError) {
+        self.add_request();
+        atomic_increment!(self.errors);
+
+        match error {
+            StatError::Timeout => {
+                atomic_increment!(self.timeouts);
+            }
+            StatError::Status403 => {
+                atomic_increment!(self.status_403s);
+                atomic_increment!(self.client_errors);
+            }
+            StatError::UrlFormat => {
+                atomic_increment!(self.url_format_errors);
+            }
+            StatError::Redirection => {
+                atomic_increment!(self.redirection_errors);
+            }
+            StatError::Connection => {
+                atomic_increment!(self.connection_errors);
+            }
+            StatError::Request => {
+                atomic_increment!(self.request_errors);
+            }
+            StatError::Other => {
+                atomic_increment!(self.errors);
+            }
+        }
+    }
+
+    /// Inspect the given `StatusCode` and increment the appropriate fields
+    ///
+    /// Implies incrementing:
+    ///     - requests
+    ///     - status_403s (when code is 403)
+    ///     - errors (when code is [45]xx)
+    fn add_status_code(&self, status: StatusCode) {
+        self.add_request();
+
+        if status.is_success() {
+            atomic_increment!(self.successes);
+        } else if status.is_redirection() {
+            atomic_increment!(self.redirects);
+        } else if status.is_client_error() {
+            atomic_increment!(self.client_errors);
+        } else if status.is_server_error() {
+            atomic_increment!(self.server_errors);
+        }
+
+        match status {
+            StatusCode::FORBIDDEN => {
+                atomic_increment!(self.status_403s);
+            }
+            StatusCode::OK => {
+                atomic_increment!(self.status_200s);
+            }
+            StatusCode::MOVED_PERMANENTLY => {
+                atomic_increment!(self.status_301s);
+            }
+            StatusCode::FOUND => {
+                atomic_increment!(self.status_302s);
+            }
+            StatusCode::UNAUTHORIZED => {
+                atomic_increment!(self.status_401s);
+            }
+            StatusCode::TOO_MANY_REQUESTS => {
+                atomic_increment!(self.status_429s);
+            }
+            StatusCode::INTERNAL_SERVER_ERROR => {
+                atomic_increment!(self.status_500s);
+            }
+            StatusCode::SERVICE_UNAVAILABLE => {
+                atomic_increment!(self.status_503s);
+            }
+            StatusCode::GATEWAY_TIMEOUT => {
+                atomic_increment!(self.status_504s);
+            }
+            StatusCode::LOOP_DETECTED => {
+                atomic_increment!(self.status_508s);
+            }
+            _ => {} // other status codes ignored for stat gathering
+        }
+    }
+
+    /// Update a `Stats` field of type f64
+    fn update_f64_field(&self, field: StatField, value: f64) {
+        if let StatField::DirScanTimes = field {
+            if let Ok(mut locked_times) = self.directory_scan_times.lock() {
+                locked_times.push(value);
+            }
+        }
+    }
+
+    /// Update a `Stats` field of type usize
+    fn update_usize_field(&self, field: StatField, value: usize) {
+        match field {
+            StatField::ExpectedPerScan => {
+                atomic_increment!(self.expected_per_scan, value);
+            }
+            StatField::TotalScans => {
+                let multiplier = CONFIGURATION.extensions.len().max(1);
+
+                atomic_increment!(self.total_scans, value);
+                atomic_increment!(
+                    self.total_expected,
+                    value * self.expected_per_scan.load(Ordering::Relaxed) * multiplier
+                );
+            }
+            StatField::TotalExpected => {
+                atomic_increment!(self.total_expected, value);
+            }
+            StatField::LinksExtracted => {
+                atomic_increment!(self.links_extracted, value);
+            }
+            StatField::WildcardsFiltered => {
+                atomic_increment!(self.wildcards_filtered, value);
+                atomic_increment!(self.responses_filtered, value);
+            }
+            StatField::ResponsesFiltered => {
+                atomic_increment!(self.responses_filtered, value);
+            }
+            StatField::ResourcesDiscovered => {
+                atomic_increment!(self.resources_discovered, value);
+            }
+            StatField::InitialTargets => {
+                atomic_increment!(self.initial_targets, value);
+            }
+            _ => {} // f64 fields
+        }
+    }
+
+    /// Merge a given `Stats` object from a json entry written to disk when handling a Ctrl+c
+    ///
+    /// This is only ever called when resuming a scan from disk
+    pub fn merge_from(&self, filename: &str) {
+        if let Ok(file) = File::open(filename) {
+            let reader = BufReader::new(file);
+            let state: serde_json::Value = serde_json::from_reader(reader).unwrap();
+
+            if let Some(state_stats) = state.get("statistics") {
+                if let Ok(d_stats) = serde_json::from_value::<Stats>(state_stats.clone()) {
+                    atomic_increment!(self.successes, atomic_load!(d_stats.successes));
+                    atomic_increment!(self.timeouts, atomic_load!(d_stats.timeouts));
+                    atomic_increment!(self.requests, atomic_load!(d_stats.requests));
+                    atomic_increment!(self.errors, atomic_load!(d_stats.errors));
+                    atomic_increment!(self.redirects, atomic_load!(d_stats.redirects));
+                    atomic_increment!(self.client_errors, atomic_load!(d_stats.client_errors));
+                    atomic_increment!(self.server_errors, atomic_load!(d_stats.server_errors));
+                    atomic_increment!(self.links_extracted, atomic_load!(d_stats.links_extracted));
+                    atomic_increment!(self.status_200s, atomic_load!(d_stats.status_200s));
+                    atomic_increment!(self.status_301s, atomic_load!(d_stats.status_301s));
+                    atomic_increment!(self.status_302s, atomic_load!(d_stats.status_302s));
+                    atomic_increment!(self.status_401s, atomic_load!(d_stats.status_401s));
+                    atomic_increment!(self.status_403s, atomic_load!(d_stats.status_403s));
+                    atomic_increment!(self.status_429s, atomic_load!(d_stats.status_429s));
+                    atomic_increment!(self.status_500s, atomic_load!(d_stats.status_500s));
+                    atomic_increment!(self.status_503s, atomic_load!(d_stats.status_503s));
+                    atomic_increment!(self.status_504s, atomic_load!(d_stats.status_504s));
+                    atomic_increment!(self.status_508s, atomic_load!(d_stats.status_508s));
+                    atomic_increment!(
+                        self.wildcards_filtered,
+                        atomic_load!(d_stats.wildcards_filtered)
+                    );
+                    atomic_increment!(
+                        self.responses_filtered,
+                        atomic_load!(d_stats.responses_filtered)
+                    );
+                    atomic_increment!(
+                        self.resources_discovered,
+                        atomic_load!(d_stats.resources_discovered)
+                    );
+                    atomic_increment!(
+                        self.url_format_errors,
+                        atomic_load!(d_stats.url_format_errors)
+                    );
+                    atomic_increment!(
+                        self.connection_errors,
+                        atomic_load!(d_stats.connection_errors)
+                    );
+                    atomic_increment!(
+                        self.redirection_errors,
+                        atomic_load!(d_stats.redirection_errors)
+                    );
+                    atomic_increment!(self.request_errors, atomic_load!(d_stats.request_errors));
+
+                    if let Ok(scan_times) = d_stats.directory_scan_times.lock() {
+                        for scan_time in scan_times.iter() {
+                            self.update_f64_field(StatField::DirScanTimes, *scan_time);
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+#[derive(Debug)]
+/// Enum variants used to inform the `StatCommand` protocol what `Stats` fields should be updated
+pub enum StatError {
+    /// Represents a 403 response code
+    Status403,
+
+    /// Represents a timeout error
+    Timeout,
+
+    /// Represents a URL formatting error
+    UrlFormat,
+
+    /// Represents an error encountered during redirection
+    Redirection,
+
+    /// Represents an error encountered during connection
+    Connection,
+
+    /// Represents an error resulting from the client's request
+    Request,
+
+    /// Represents any other error not explicitly defined above
+    Other,
+}
+
+/// Protocol definition for updating a Stats object via mpsc
+#[derive(Debug)]
+pub enum StatCommand {
+    /// Add one to the total number of requests
+    AddRequest,
+
+    /// Add one to the proper field(s) based on the given `StatError`
+    AddError(StatError),
+
+    /// Add one to the proper field(s) based on the given `StatusCode`
+    AddStatus(StatusCode),
+
+    /// Create the progress bar (`BarType::Total`) that is updated from the stats thread
+    CreateBar,
+
+    /// Update a `Stats` field that corresponds to the given `StatField` by the given `usize` value
+    UpdateUsizeField(StatField, usize),
+
+    /// Update a `Stats` field that corresponds to the given `StatField` by the given `f64` value
+    UpdateF64Field(StatField, f64),
+
+    /// Save a `Stats` object to disk using `reporter::get_cached_file_handle`
+    Save,
+
+    /// Load a `Stats` object from disk
+    LoadStats(String),
+
+    /// Break out of the (infinite) mpsc receive loop
+    Exit,
+}
+
+/// Enum representing fields whose updates need to be performed in batches instead of one at
+/// a time
+#[derive(Debug)]
+pub enum StatField {
+    /// Due to the necessary order of events, the number of requests expected to be sent isn't
+    /// known until after `statistics::initialize` is called. This command allows for updating
+    /// the `expected_per_scan` field after initialization
+    ExpectedPerScan,
+
+    /// Translates to `total_scans`
+    TotalScans,
+
+    /// Translates to `links_extracted`
+    LinksExtracted,
+
+    /// Translates to `total_expected`
+    TotalExpected,
+
+    /// Translates to `wildcards_filtered`
+    WildcardsFiltered,
+
+    /// Translates to `responses_filtered`
+    ResponsesFiltered,
+
+    /// Translates to `resources_discovered`
+    ResourcesDiscovered,
+
+    /// Translates to `initial_targets`
+    InitialTargets,
+
+    /// Translates to `directory_scan_times`; assumes a single append to the vector
+    DirScanTimes,
+}
+
+/// Spawn a single consumer task (sc side of mpsc)
+///
+/// The consumer simply receives `StatCommands` and updates the given `Stats` object as appropriate
+pub async fn spawn_statistics_handler(
+    mut rx_stats: UnboundedReceiver<StatCommand>,
+    stats: Arc<Stats>,
+    tx_stats: UnboundedSender<StatCommand>,
+) {
+    log::trace!(
+        "enter: spawn_statistics_handler({:?}, {:?}, {:?})",
+        rx_stats,
+        stats,
+        tx_stats
+    );
+
+    // will be updated later via StatCommand; delay is for banner to print first
+    let mut bar = ProgressBar::hidden();
+
+    let start = Instant::now();
+
+    while let Some(command) = rx_stats.recv().await {
+        match command as StatCommand {
+            StatCommand::AddError(err) => {
+                stats.add_error(err);
+                increment_bar(&bar, stats.clone());
+            }
+            StatCommand::AddStatus(status) => {
+                stats.add_status_code(status);
+                increment_bar(&bar, stats.clone());
+            }
+            StatCommand::AddRequest => {
+                stats.add_request();
+                increment_bar(&bar, stats.clone());
+            }
+            StatCommand::Save => stats.save(start.elapsed().as_secs_f64(), &CONFIGURATION.output),
+            StatCommand::UpdateUsizeField(field, value) => {
+                let update_len = matches!(field, StatField::TotalScans);
+                stats.update_usize_field(field, value);
+
+                if update_len {
+                    bar.set_length(atomic_load!(stats.total_expected) as u64)
+                }
+            }
+            StatCommand::UpdateF64Field(field, value) => stats.update_f64_field(field, value),
+            StatCommand::CreateBar => {
+                bar = add_bar(
+                    "",
+                    atomic_load!(stats.total_expected) as u64,
+                    BarType::Total,
+                );
+            }
+            StatCommand::LoadStats(filename) => {
+                stats.merge_from(&filename);
+            }
+            StatCommand::Exit => break,
+        }
+    }
+
+    bar.finish();
+
+    log::debug!("{:#?}", *stats);
+    log::trace!("exit: spawn_statistics_handler")
+}
+
+/// Wrapper around incrementing the overall scan's progress bar
+fn increment_bar(bar: &ProgressBar, stats: Arc<Stats>) {
+    let msg = format!(
+        "{}:{:<7} {}:{:<7}",
+        style("found").green(),
+        atomic_load!(stats.resources_discovered),
+        style("errors").red(),
+        atomic_load!(stats.errors),
+    );
+
+    bar.set_message(&msg);
+    bar.inc(1);
+}
+
+/// Initialize new `Stats` object and the sc side of an mpsc channel that is responsible for
+/// updates to the aforementioned object.
+pub fn initialize() -> (Arc<Stats>, UnboundedSender<StatCommand>, JoinHandle<()>) {
+    log::trace!("enter: initialize");
+
+    let stats_tracker = Arc::new(Stats::new());
+    let stats_cloned = stats_tracker.clone();
+    let (tx_stats, rx_stats): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+    let tx_stats_cloned = tx_stats.clone();
+    let stats_thread = tokio::spawn(async move {
+        spawn_statistics_handler(rx_stats, stats_cloned, tx_stats_cloned).await
+    });
+
+    log::trace!(
+        "exit: initialize -> ({:?}, {:?}, {:?})",
+        stats_tracker,
+        tx_stats,
+        stats_thread
+    );
+
+    (stats_tracker, tx_stats, stats_thread)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::fs::write;
+    use tempfile::NamedTempFile;
+
+    /// simple helper to reduce code reuse
+    fn setup_stats_test() -> (Arc<Stats>, UnboundedSender<StatCommand>, JoinHandle<()>) {
+        initialize()
+    }
+
+    /// another helper to stay DRY; must be called after any sent commands and before any checks
+    /// performed against the Stats object
+    async fn teardown_stats_test(sender: UnboundedSender<StatCommand>, handle: JoinHandle<()>) {
+        // send exit and await, once the await completes, stats should be updated
+        sender.send(StatCommand::Exit).unwrap_or_default();
+        handle.await.unwrap();
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// when sent StatCommand::Exit, function should exit its while loop (runs forever otherwise)
+    async fn statistics_handler_exits() {
+        let (_, sender, handle) = setup_stats_test();
+
+        sender.send(StatCommand::Exit).unwrap_or_default();
+
+        handle.await.unwrap(); // blocks on the handler's while loop
+
+        // if we've made it here, the test has succeeded
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// when sent StatCommand::AddRequest, stats object should reflect the change
+    async fn statistics_handler_increments_requests() {
+        let (stats, tx, handle) = setup_stats_test();
+
+        tx.send(StatCommand::AddRequest).unwrap_or_default();
+        tx.send(StatCommand::AddRequest).unwrap_or_default();
+        tx.send(StatCommand::AddRequest).unwrap_or_default();
+
+        teardown_stats_test(tx, handle).await;
+
+        assert_eq!(stats.requests.load(Ordering::Relaxed), 3);
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// when sent StatCommand::AddRequest, stats object should reflect the change
+    ///
+    /// incrementing a 403 (tracked in status_403s) should also increment:
+    ///     - errors
+    ///     - requests
+    ///     - client_errors
+    async fn statistics_handler_increments_403() {
+        let (stats, tx, handle) = setup_stats_test();
+
+        let err = StatCommand::AddError(StatError::Status403);
+        let err2 = StatCommand::AddError(StatError::Status403);
+
+        tx.send(err).unwrap_or_default();
+        tx.send(err2).unwrap_or_default();
+
+        teardown_stats_test(tx, handle).await;
+
+        assert_eq!(stats.errors.load(Ordering::Relaxed), 2);
+        assert_eq!(stats.requests.load(Ordering::Relaxed), 2);
+        assert_eq!(stats.status_403s.load(Ordering::Relaxed), 2);
+        assert_eq!(stats.client_errors.load(Ordering::Relaxed), 2);
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// when sent StatCommand::AddRequest, stats object should reflect the change
+    ///
+    /// incrementing a 403 (tracked in status_403s) should also increment:
+    ///     - requests
+    ///     - client_errors
+    async fn statistics_handler_increments_403_via_status_code() {
+        let (stats, tx, handle) = setup_stats_test();
+
+        let err = StatCommand::AddStatus(reqwest::StatusCode::FORBIDDEN);
+        let err2 = StatCommand::AddStatus(reqwest::StatusCode::FORBIDDEN);
+
+        tx.send(err).unwrap_or_default();
+        tx.send(err2).unwrap_or_default();
+
+        teardown_stats_test(tx, handle).await;
+
+        assert_eq!(stats.requests.load(Ordering::Relaxed), 2);
+        assert_eq!(stats.status_403s.load(Ordering::Relaxed), 2);
+        assert_eq!(stats.client_errors.load(Ordering::Relaxed), 2);
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// when sent StatCommand::AddStatus, stats object should reflect the change
+    ///
+    /// incrementing a 500 (tracked in server_errors) should also increment:
+    ///     - requests
+    async fn statistics_handler_increments_500_via_status_code() {
+        let (stats, tx, handle) = setup_stats_test();
+
+        let err = StatCommand::AddStatus(reqwest::StatusCode::INTERNAL_SERVER_ERROR);
+        let err2 = StatCommand::AddStatus(reqwest::StatusCode::INTERNAL_SERVER_ERROR);
+
+        tx.send(err).unwrap_or_default();
+        tx.send(err2).unwrap_or_default();
+
+        teardown_stats_test(tx, handle).await;
+
+        assert_eq!(stats.requests.load(Ordering::Relaxed), 2);
+        assert_eq!(stats.server_errors.load(Ordering::Relaxed), 2);
+    }
+
+    #[test]
+    /// when Stats::add_error receives StatError::Timeout, it should increment the following:
+    ///     - timeouts
+    ///     - requests
+    ///     - errors
+    fn stats_increments_timeouts() {
+        let stats = Stats::new();
+        stats.add_error(StatError::Timeout);
+        stats.add_error(StatError::Timeout);
+        stats.add_error(StatError::Timeout);
+        stats.add_error(StatError::Timeout);
+
+        assert_eq!(stats.errors.load(Ordering::Relaxed), 4);
+        assert_eq!(stats.requests.load(Ordering::Relaxed), 4);
+        assert_eq!(stats.timeouts.load(Ordering::Relaxed), 4);
+    }
+
+    #[test]
+    /// when Stats::update_usize_field receives StatField::WildcardsFiltered, it should increment
+    /// the following:
+    ///     - responses_filtered
+    fn stats_increments_wildcards() {
+        let stats = Stats::new();
+        assert_eq!(stats.responses_filtered.load(Ordering::Relaxed), 0);
+        assert_eq!(stats.wildcards_filtered.load(Ordering::Relaxed), 0);
+
+        stats.update_usize_field(StatField::WildcardsFiltered, 1);
+        stats.update_usize_field(StatField::WildcardsFiltered, 1);
+
+        assert_eq!(stats.responses_filtered.load(Ordering::Relaxed), 2);
+        assert_eq!(stats.wildcards_filtered.load(Ordering::Relaxed), 2);
+    }
+
+    #[test]
+    /// when Stats::update_usize_field receives StatField::ResponsesFiltered, it should increment
+    fn stats_increments_responses_filtered() {
+        let stats = Stats::new();
+        assert_eq!(stats.responses_filtered.load(Ordering::Relaxed), 0);
+
+        stats.update_usize_field(StatField::ResponsesFiltered, 1);
+        stats.update_usize_field(StatField::ResponsesFiltered, 1);
+        stats.update_usize_field(StatField::ResponsesFiltered, 1);
+
+        assert_eq!(stats.responses_filtered.load(Ordering::Relaxed), 3);
+    }
+
+    #[test]
+    /// Stats::merge_from should properly incrememnt expected fields and ignore others
+    fn stats_merge_from_alters_correct_fields() {
+        let contents = r#"{"statistics":{"type":"statistics","timeouts":1,"requests":9207,"expected_per_scan":707,"total_expected":9191,"errors":3,"successes":720,"redirects":13,"client_errors":8474,"server_errors":2,"total_scans":13,"initial_targets":1,"links_extracted":51,"status_403s":3,"status_200s":720,"status_301s":12,"status_302s":1,"status_401s":4,"status_429s":2,"status_500s":5,"status_503s":9,"status_504s":6,"status_508s":7,"wildcards_filtered":707,"responses_filtered":707,"resources_discovered":27,"directory_scan_times":[2.211973078,1.989015505,1.898675839,3.9714468910000003,4.938152838,5.256073528,6.021986595,6.065740734,6.42633762,7.095142125,7.336982137,5.319785619,4.843649778],"total_runtime":[11.556575456000001],"url_format_errors":17,"redirection_errors":12,"connection_errors":21,"request_errors":4}}"#;
+        let stats = Stats::new();
+        let tfile = NamedTempFile::new().unwrap();
+        write(&tfile, contents).unwrap();
+
+        stats.merge_from(tfile.path().to_str().unwrap());
+
+        // as of 1.11.1; all Stats fields are accounted for whether they're updated in merge_from
+        // or not
+        assert_eq!(atomic_load!(stats.timeouts), 1);
+        assert_eq!(atomic_load!(stats.requests), 9207);
+        assert_eq!(atomic_load!(stats.expected_per_scan), 0); // not updated in merge_from
+        assert_eq!(atomic_load!(stats.total_expected), 0); // not updated in merge_from
+        assert_eq!(atomic_load!(stats.errors), 3);
+        assert_eq!(atomic_load!(stats.successes), 720);
+        assert_eq!(atomic_load!(stats.redirects), 13);
+        assert_eq!(atomic_load!(stats.client_errors), 8474);
+        assert_eq!(atomic_load!(stats.server_errors), 2);
+        assert_eq!(atomic_load!(stats.total_scans), 0); // not updated in merge_from
+        assert_eq!(atomic_load!(stats.initial_targets), 0); // not updated in merge_from
+        assert_eq!(atomic_load!(stats.links_extracted), 51);
+        assert_eq!(atomic_load!(stats.status_200s), 720);
+        assert_eq!(atomic_load!(stats.status_301s), 12);
+        assert_eq!(atomic_load!(stats.status_302s), 1);
+        assert_eq!(atomic_load!(stats.status_401s), 4);
+        assert_eq!(atomic_load!(stats.status_403s), 3);
+        assert_eq!(atomic_load!(stats.status_429s), 2);
+        assert_eq!(atomic_load!(stats.status_500s), 5);
+        assert_eq!(atomic_load!(stats.status_503s), 9);
+        assert_eq!(atomic_load!(stats.status_504s), 6);
+        assert_eq!(atomic_load!(stats.status_508s), 7);
+        assert_eq!(atomic_load!(stats.wildcards_filtered), 707);
+        assert_eq!(atomic_load!(stats.responses_filtered), 707);
+        assert_eq!(atomic_load!(stats.resources_discovered), 27);
+        assert_eq!(atomic_load!(stats.url_format_errors), 17);
+        assert_eq!(atomic_load!(stats.redirection_errors), 12);
+        assert_eq!(atomic_load!(stats.connection_errors), 21);
+        assert_eq!(atomic_load!(stats.request_errors), 4);
+        assert_eq!(stats.directory_scan_times.lock().unwrap().len(), 13);
+        for scan in stats.directory_scan_times.lock().unwrap().iter() {
+            assert!(scan.max(0.0) > 0.0); // all scans are non-zero
+        }
+        // total_runtime not updated in merge_from
+        assert_eq!(stats.total_runtime.lock().unwrap().len(), 1);
+        assert!((stats.total_runtime.lock().unwrap()[0] - 0.0).abs() < f64::EPSILON);
+    }
+
+    #[test]
+    /// ensure update runtime overwrites the default 0th entry
+    fn update_runtime_works() {
+        let stats = Stats::new();
+        assert!((stats.total_runtime.lock().unwrap()[0] - 0.0).abs() < f64::EPSILON);
+        stats.update_runtime(20.2);
+        assert!((stats.total_runtime.lock().unwrap()[0] - 20.2).abs() < f64::EPSILON);
+    }
+
+    #[test]
+    /// Stats::save should write contents of Stats to disk
+    fn save_writes_stats_object_to_disk() {
+        let stats = Stats::new();
+        stats.add_request();
+        stats.add_request();
+        stats.add_request();
+        stats.add_request();
+        stats.add_error(StatError::Timeout);
+        stats.add_error(StatError::Timeout);
+        stats.add_error(StatError::Timeout);
+        stats.add_error(StatError::Timeout);
+        stats.add_status_code(StatusCode::OK);
+        stats.add_status_code(StatusCode::OK);
+        stats.add_status_code(StatusCode::OK);
+        let outfile = "/tmp/stuff";
+        stats.save(174.33, outfile);
+        assert!(stats.as_json().contains("statistics"));
+        assert!(stats.as_json().contains("11")); // requests made
+        assert!(stats.as_str().is_empty());
+    }
+}

src/utils.rs

@@ -1,11 +1,47 @@
-use crate::{FeroxError, FeroxResult};
+#![macro_use]
+use crate::{
+    config::{CONFIGURATION, PROGRESS_PRINTER},
+    statistics::{
+        StatCommand::{self, AddError, AddStatus},
+        StatError::{Connection, Other, Redirection, Request, Timeout, UrlFormat},
+    },
+    FeroxError, FeroxResult,
+};
 use console::{strip_ansi_codes, style, user_attended};
 use indicatif::ProgressBar;
-use reqwest::Url;
-use reqwest::{Client, Response};
+use reqwest::{Client, Response, Url};
 #[cfg(not(target_os = "windows"))]
 use rlimit::{getrlimit, setrlimit, Resource, Rlim};
 use std::convert::TryInto;
+use std::sync::{Arc, RwLock};
+use std::{fs, io};
+use tokio::sync::mpsc::UnboundedSender;
+
+/// Given the path to a file, open the file in append mode (create it if it doesn't exist) and
+/// return a reference to the file that is buffered and locked
+pub fn open_file(filename: &str) -> Option<Arc<RwLock<io::BufWriter<fs::File>>>> {
+    log::trace!("enter: open_file({})", filename);
+
+    match fs::OpenOptions::new() // std fs
+        .create(true)
+        .append(true)
+        .open(filename)
+    {
+        Ok(file) => {
+            let writer = io::BufWriter::new(file); // std io
+
+            let locked_file = Some(Arc::new(RwLock::new(writer)));
+
+            log::trace!("exit: open_file -> {:?}", locked_file);
+            locked_file
+        }
+        Err(e) => {
+            log::error!("{}", e);
+            log::trace!("exit: open_file -> None");
+            None
+        }
+    }
+}
 
 /// Helper function that determines the current depth of a given url
 ///
@@ -21,13 +57,7 @@ use std::convert::TryInto;
 pub fn get_current_depth(target: &str) -> usize {
     log::trace!("enter: get_current_depth({})", target);
 
-    let target = if !target.ends_with('/') {
-        // target url doesn't end with a /, for the purposes of determining depth, we'll normalize
-        // all urls to end in a / and then calculate accordingly
-        format!("{}/", target)
-    } else {
-        String::from(target)
-    };
+    let target = normalize_url(target);
 
     match Url::parse(&target) {
         Ok(url) => {
@@ -90,8 +120,8 @@ pub fn get_url_path_length(url: &Url) -> u64 {
 
     let path = url.path();
 
-    let segments = if path.starts_with('/') {
-        path[1..].split_terminator('/')
+    let segments = if let Some(split) = path.strip_prefix('/') {
+        split.split_terminator('/')
     } else {
         log::trace!("exit: get_url_path_length -> 0");
         return 0;
@@ -136,6 +166,14 @@ pub fn ferox_print(msg: &str, bar: &ProgressBar) {
     }
 }
 
+#[macro_export]
+/// wrapper to improve code readability
+macro_rules! update_stat {
+    ($tx:expr, $value:expr) => {
+        $tx.send($value).unwrap_or_default();
+    };
+}
+
 /// Simple helper to generate a `Url`
 ///
 /// Errors during parsing `url` or joining `word` are propagated up the call stack
@@ -145,14 +183,16 @@ pub fn format_url(
     add_slash: bool,
     queries: &[(String, String)],
     extension: Option<&str>,
+    tx_stats: UnboundedSender<StatCommand>,
 ) -> FeroxResult<Url> {
     log::trace!(
-        "enter: format_url({}, {}, {}, {:?} {:?})",
+        "enter: format_url({}, {}, {}, {:?} {:?}, {:?})",
         url,
         word,
         add_slash,
         queries,
-        extension
+        extension,
+        tx_stats
     );
 
     if Url::parse(&word).is_ok() {
@@ -169,8 +209,9 @@ pub fn format_url(
         );
         log::warn!("{}", message);
 
-        let mut err = FeroxError::default();
-        err.message = message;
+        let err = FeroxError { message };
+
+        update_stat!(tx_stats, AddError(UrlFormat));
 
         log::trace!("exit: format_url -> {}", err);
         return Err(Box::new(err));
@@ -184,7 +225,7 @@ pub fn format_url(
     // the transforms that occur here will need to keep this in mind, i.e. add a slash to preserve
     // the current directory sent as part of the url
     let url = if word.is_empty() {
-        // v1.0.6: added during --extract-links feature inplementation to support creating urls
+        // v1.0.6: added during --extract-links feature implementation to support creating urls
         // that were extracted from response bodies, i.e. http://localhost/some/path/js/main.js
         url.to_string()
     } else if !url.ends_with('/') {
@@ -201,6 +242,15 @@ pub fn format_url(
     } else if add_slash && !word.ends_with('/') {
         // -f used, and word doesn't already end with a /
         format!("{}/", word)
+    } else if word.starts_with("//") {
+        // bug ID'd by @Sicks3c, when a wordlist contains words that begin with 2 forward slashes
+        // i.e. //1_40_0/static/js, it gets joined onto the base url in a surprising way
+        // ex: https://localhost/ + //1_40_0/static/js -> https://1_40_0/static/js
+        // this is due to the fact that //... is a valid url. The fix is introduced here in 1.12.2
+        // and simply removes prefixed forward slashes if there are two of them. Additionally,
+        // trim_start_matches will trim the pattern until it's gone, so even if there are more than
+        // 2 /'s, they'll still be trimmed
+        word.trim_start_matches('/').to_string()
     } else {
         String::from(word)
     };
@@ -231,6 +281,7 @@ pub fn format_url(
             }
         }
         Err(e) => {
+            update_stat!(tx_stats, AddError(UrlFormat));
             log::trace!("exit: format_url -> {}", e);
             log::error!("Could not join {} with {}", word, base_url);
             Err(Box::new(e))
@@ -239,25 +290,87 @@ pub fn format_url(
 }
 
 /// Initiate request to the given `Url` using `Client`
-pub async fn make_request(client: &Client, url: &Url) -> FeroxResult<Response> {
-    log::trace!("enter: make_request(CONFIGURATION.Client, {})", url);
+pub async fn make_request(
+    client: &Client,
+    url: &Url,
+    tx_stats: UnboundedSender<StatCommand>,
+) -> FeroxResult<Response> {
+    log::trace!(
+        "enter: make_request(CONFIGURATION.Client, {}, {:?})",
+        url,
+        tx_stats
+    );
 
     match client.get(url.to_owned()).send().await {
-        Ok(resp) => {
-            log::debug!("requested Url: {}", resp.url());
-            log::trace!("exit: make_request -> {:?}", resp);
-            Ok(resp)
-        }
         Err(e) => {
+            let mut log_level = log::Level::Error;
+
             log::trace!("exit: make_request -> {}", e);
-            if e.to_string().contains("operation timed out") {
+            if e.is_timeout() {
                 // only warn for timeouts, while actual errors are still left as errors
-                log::warn!("Error while making request: {}", e);
+                log_level = log::Level::Warn;
+                update_stat!(tx_stats, AddError(Timeout));
+            } else if e.is_redirect() {
+                if let Some(last_redirect) = e.url() {
+                    // get where we were headed (last_redirect) and where we came from (url)
+                    let fancy_message = format!("{} !=> {}", url, last_redirect);
+
+                    let report = if let Some(msg_status) = e.status() {
+                        update_stat!(tx_stats, AddStatus(msg_status));
+                        create_report_string(msg_status.as_str(), "-1", "-1", "-1", &fancy_message)
+                    } else {
+                        create_report_string("UNK", "-1", "-1", "-1", &fancy_message)
+                    };
+
+                    update_stat!(tx_stats, AddError(Redirection));
+
+                    ferox_print(&report, &PROGRESS_PRINTER)
+                };
+            } else if e.is_connect() {
+                update_stat!(tx_stats, AddError(Connection));
+            } else if e.is_request() {
+                update_stat!(tx_stats, AddError(Request));
             } else {
-                log::error!("Error while making request: {}", e);
+                update_stat!(tx_stats, AddError(Other));
             }
+
+            if matches!(log_level, log::Level::Error) {
+                log::error!("Error while making request: {}", e);
+            } else {
+                log::warn!("Error while making request: {}", e);
+            }
+
             Err(Box::new(e))
         }
+        Ok(resp) => {
+            log::trace!("exit: make_request -> {:?}", resp);
+            update_stat!(tx_stats, AddStatus(resp.status()));
+            Ok(resp)
+        }
+    }
+}
+
+/// Helper to create the standard line for output to file/terminal
+///
+/// example output:
+/// 200      127l      283w     4134c http://localhost/faq
+pub fn create_report_string(
+    status: &str,
+    line_count: &str,
+    word_count: &str,
+    content_length: &str,
+    url: &str,
+) -> String {
+    if CONFIGURATION.quiet {
+        // -q used, just need the url
+        format!("{}\n", url)
+    } else {
+        // normal printing with status and sizes
+        let color_status = status_colorizer(status);
+        format!(
+            "{} {:>8}l {:>8}w {:>8}c {}\n",
+            color_status, line_count, word_count, content_length, url
+        )
     }
 }
 
@@ -312,9 +425,27 @@ pub fn set_open_file_limit(limit: usize) -> bool {
     false
 }
 
+/// Simple helper to abstract away adding a forward-slash to a url if not present
+///
+/// used mostly for deduplication purposes and url state tracking
+pub fn normalize_url(url: &str) -> String {
+    log::trace!("enter: normalize_url({})", url);
+
+    let normalized = if url.ends_with('/') {
+        url.to_string()
+    } else {
+        format!("{}/", url)
+    };
+
+    log::trace!("exit: normalize_url -> {}", normalized);
+    normalized
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::FeroxChannel;
+    use tokio::sync::mpsc;
 
     #[test]
     /// set_open_file_limit with a low requested limit succeeds
@@ -382,8 +513,9 @@ mod tests {
     #[test]
     /// base url + 1 word + no slash + no extension
     fn format_url_normal() {
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
         assert_eq!(
-            format_url("http://localhost", "stuff", false, &Vec::new(), None).unwrap(),
+            format_url("http://localhost", "stuff", false, &Vec::new(), None, tx).unwrap(),
             reqwest::Url::parse("http://localhost/stuff").unwrap()
         );
     }
@@ -391,8 +523,9 @@ mod tests {
     #[test]
     /// base url + no word + no slash + no extension
     fn format_url_no_word() {
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
         assert_eq!(
-            format_url("http://localhost", "", false, &Vec::new(), None).unwrap(),
+            format_url("http://localhost", "", false, &Vec::new(), None, tx).unwrap(),
             reqwest::Url::parse("http://localhost").unwrap()
         );
     }
@@ -400,13 +533,15 @@ mod tests {
     #[test]
     /// base url + word + no slash + no extension + queries
     fn format_url_joins_queries() {
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
         assert_eq!(
             format_url(
                 "http://localhost",
                 "lazer",
                 false,
                 &[(String::from("stuff"), String::from("things"))],
-                None
+                None,
+                tx
             )
             .unwrap(),
             reqwest::Url::parse("http://localhost/lazer?stuff=things").unwrap()
@@ -416,13 +551,15 @@ mod tests {
     #[test]
     /// base url + no word + no slash + no extension + queries
     fn format_url_without_word_joins_queries() {
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
         assert_eq!(
             format_url(
                 "http://localhost",
                 "",
                 false,
                 &[(String::from("stuff"), String::from("things"))],
-                None
+                None,
+                tx
             )
             .unwrap(),
             reqwest::Url::parse("http://localhost/?stuff=things").unwrap()
@@ -433,14 +570,16 @@ mod tests {
     #[should_panic]
     /// no base url is an error
     fn format_url_no_url() {
-        format_url("", "stuff", false, &Vec::new(), None).unwrap();
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+        format_url("", "stuff", false, &Vec::new(), None, tx).unwrap();
     }
 
     #[test]
     /// word prepended with slash is adjusted for correctness
     fn format_url_word_with_preslash() {
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
         assert_eq!(
-            format_url("http://localhost", "/stuff", false, &Vec::new(), None).unwrap(),
+            format_url("http://localhost", "/stuff", false, &Vec::new(), None, tx).unwrap(),
             reqwest::Url::parse("http://localhost/stuff").unwrap()
         );
     }
@@ -448,21 +587,45 @@ mod tests {
     #[test]
     /// word with appended slash allows the slash to persist
     fn format_url_word_with_postslash() {
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
         assert_eq!(
-            format_url("http://localhost", "stuff/", false, &Vec::new(), None).unwrap(),
+            format_url("http://localhost", "stuff/", false, &Vec::new(), None, tx).unwrap(),
             reqwest::Url::parse("http://localhost/stuff/").unwrap()
         );
     }
 
+    #[test]
+    /// word with two prepended slashes doesn't discard the entire domain
+    fn format_url_word_with_two_prepended_slashes() {
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
+
+        let result = format_url(
+            "http://localhost",
+            "//upload/img",
+            false,
+            &Vec::new(),
+            None,
+            tx,
+        )
+        .unwrap();
+
+        assert_eq!(
+            result,
+            reqwest::Url::parse("http://localhost/upload/img").unwrap()
+        );
+    }
+
     #[test]
     /// word that is a fully formed url, should return an error
     fn format_url_word_that_is_a_url() {
+        let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
         let url = format_url(
             "http://localhost",
             "http://schmocalhost",
             false,
             &Vec::new(),
             None,
+            tx,
         );
         assert!(url.is_err());
     }

tests/test_banner.rs

@@ -117,7 +117,7 @@ fn banner_prints_headers() -> Result<(), Box<dyn std::error::Error>> {
 #[test]
 /// test allows non-existent wordlist to trigger the banner printing to stderr
 /// expect to see all mandatory prints + multiple size filters
-fn banner_prints_filter_sizes() -> Result<(), Box<dyn std::error::Error>> {
+fn banner_prints_filter_sizes() {
     Command::cargo_bin("feroxbuster")
         .unwrap()
         .arg("--url")
@@ -126,6 +126,14 @@ fn banner_prints_filter_sizes() -> Result<(), Box<dyn std::error::Error>> {
         .arg("789456123")
         .arg("--filter-size")
         .arg("44444444")
+        .arg("-N")
+        .arg("678")
+        .arg("--filter-lines")
+        .arg("679")
+        .arg("-W")
+        .arg("93")
+        .arg("--filter-words")
+        .arg("94")
         .assert()
         .success()
         .stderr(
@@ -138,11 +146,16 @@ fn banner_prints_filter_sizes() -> Result<(), Box<dyn std::error::Error>> {
                 .and(predicate::str::contains("Timeout (secs)"))
                 .and(predicate::str::contains("User-Agent"))
                 .and(predicate::str::contains("Size Filter"))
+                .and(predicate::str::contains("Word Count Filter"))
+                .and(predicate::str::contains("Line Count Filter"))
                 .and(predicate::str::contains("789456123"))
                 .and(predicate::str::contains("44444444"))
+                .and(predicate::str::contains("93"))
+                .and(predicate::str::contains("94"))
+                .and(predicate::str::contains("678"))
+                .and(predicate::str::contains("679"))
                 .and(predicate::str::contains("─┴─")),
         );
-    Ok(())
 }
 
 #[test]
@@ -688,3 +701,139 @@ fn banner_prints_filter_status() -> Result<(), Box<dyn std::error::Error>> {
         );
     Ok(())
 }
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + json
+fn banner_prints_json() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--json")
+        .arg("--output")
+        .arg("/dev/null")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("JSON Output"))
+                .and(predicate::str::contains("│ true"))
+                .and(predicate::str::contains("─┴─")),
+        );
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + json
+fn banner_prints_debug_log() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--debug-log")
+        .arg("/dev/null")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Debugging Log"))
+                .and(predicate::str::contains("│ /dev/null"))
+                .and(predicate::str::contains("─┴─")),
+        );
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + regex filters
+fn banner_prints_filter_regex() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--filter-regex")
+        .arg("^ignore me$")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Regex Filter"))
+                .and(predicate::str::contains("│ ^ignore me$"))
+                .and(predicate::str::contains("─┴─")),
+        );
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + time limit
+fn banner_prints_time_limit() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--time-limit")
+        .arg("10m")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Time Limit"))
+                .and(predicate::str::contains("│ 10m"))
+                .and(predicate::str::contains("─┴─")),
+        );
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + similarity filter
+fn banner_prints_similarity_filter() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--filter-similar-to")
+        .arg("https://somesite.com")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Similarity Filter"))
+                .and(predicate::str::contains("│ https://somesite.com"))
+                .and(predicate::str::contains("─┴─")),
+        );
+}

tests/test_extractor.rs

@@ -1,7 +1,7 @@
 mod utils;
 use assert_cmd::prelude::*;
 use httpmock::Method::GET;
-use httpmock::{Mock, MockServer};
+use httpmock::MockServer;
 use predicates::prelude::*;
 use std::process::Command;
 use utils::{setup_tmp_directory, teardown_tmp_directory};
@@ -13,18 +13,17 @@ fn extractor_finds_absolute_url() -> Result<(), Box<dyn std::error::Error>> {
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/LICENSE")
-        .return_status(200)
-        .return_body(&srv.url("'/homepage/assets/img/icons/handshake.svg'"))
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200)
+            .body(&srv.url("'/homepage/assets/img/icons/handshake.svg'"));
+    });
 
-    let mock_two = Mock::new()
-        .expect_method(GET)
-        .expect_path("/homepage/assets/img/icons/handshake.svg")
-        .return_status(200)
-        .create_on(&srv);
+    let mock_two = srv.mock(|when, then| {
+        when.method(GET)
+            .path("/homepage/assets/img/icons/handshake.svg");
+        then.status(200);
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -43,8 +42,8 @@ fn extractor_finds_absolute_url() -> Result<(), Box<dyn std::error::Error>> {
             )),
     );
 
-    assert_eq!(mock.times_called(), 1);
-    assert_eq!(mock_two.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_two.hits(), 1);
     teardown_tmp_directory(tmp_dir);
     Ok(())
 }
@@ -56,12 +55,11 @@ fn extractor_finds_absolute_url_to_different_domain() -> Result<(), Box<dyn std:
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/LICENSE")
-        .return_status(200)
-        .return_body("\"http://localhost/homepage/assets/img/icons/handshake.svg\"")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200)
+            .body("\"http://localhost/homepage/assets/img/icons/handshake.svg\"");
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -81,7 +79,7 @@ fn extractor_finds_absolute_url_to_different_domain() -> Result<(), Box<dyn std:
             .not(),
     );
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
     teardown_tmp_directory(tmp_dir);
     Ok(())
 }
@@ -92,18 +90,17 @@ fn extractor_finds_relative_url() -> Result<(), Box<dyn std::error::Error>> {
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/LICENSE")
-        .return_status(200)
-        .return_body("\"/homepage/assets/img/icons/handshake.svg\"")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200)
+            .body("\"/homepage/assets/img/icons/handshake.svg\"");
+    });
 
-    let mock_two = Mock::new()
-        .expect_method(GET)
-        .expect_path("/homepage/assets/img/icons/handshake.svg")
-        .return_status(200)
-        .create_on(&srv);
+    let mock_two = srv.mock(|when, then| {
+        when.method(GET)
+            .path("/homepage/assets/img/icons/handshake.svg");
+        then.status(200);
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -122,8 +119,8 @@ fn extractor_finds_relative_url() -> Result<(), Box<dyn std::error::Error>> {
             )),
     );
 
-    assert_eq!(mock.times_called(), 1);
-    assert_eq!(mock_two.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_two.hits(), 1);
     teardown_tmp_directory(tmp_dir);
     Ok(())
 }
@@ -131,30 +128,28 @@ fn extractor_finds_relative_url() -> Result<(), Box<dyn std::error::Error>> {
 #[test]
 /// send a request to a page that contains an relative link, follow it, and find the same link again
 /// should follow then filter
-fn extractor_finds_same_relative_url_twice() -> Result<(), Box<dyn std::error::Error>> {
+fn extractor_finds_same_relative_url_twice() {
     let srv = MockServer::start();
     let (tmp_dir, file) =
-        setup_tmp_directory(&["LICENSE".to_string(), "README".to_string()], "wordlist")?;
+        setup_tmp_directory(&["LICENSE".to_string(), "README".to_string()], "wordlist").unwrap();
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/LICENSE")
-        .return_status(200)
-        .return_body(&srv.url("\"/homepage/assets/img/icons/handshake.svg\""))
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200)
+            .body(&srv.url("\"/homepage/assets/img/icons/handshake.svg\""));
+    });
 
-    let mock_two = Mock::new()
-        .expect_method(GET)
-        .expect_path("/README")
-        .return_body(&srv.url("\"/homepage/assets/img/icons/handshake.svg\""))
-        .return_status(200)
-        .create_on(&srv);
+    let mock_two = srv.mock(|when, then| {
+        when.method(GET).path("/README");
+        then.status(200)
+            .body(&srv.url("\"/homepage/assets/img/icons/handshake.svg\""));
+    });
 
-    let mock_three = Mock::new()
-        .expect_method(GET)
-        .expect_path("/homepage/assets/img/icons/handshake.svg")
-        .return_status(200)
-        .create_on(&srv);
+    let mock_three = srv.mock(|when, then| {
+        when.method(GET)
+            .path("/homepage/assets/img/icons/handshake.svg");
+        then.status(200);
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -168,16 +163,15 @@ fn extractor_finds_same_relative_url_twice() -> Result<(), Box<dyn std::error::E
     cmd.assert().success().stdout(
         predicate::str::contains("/LICENSE")
             .and(predicate::str::contains("200"))
-            .and(predicate::str::contains(
-                "/homepage/assets/img/icons/handshake.svg",
-            )),
+            // .count(1) asserts that we only see the endpoint reported once, even though there
+            // is the potential to request the same url twice
+            .and(predicate::str::contains("/homepage/assets/img/icons/handshake.svg").count(1)),
     );
 
-    assert_eq!(mock.times_called(), 1);
-    assert_eq!(mock_two.times_called(), 1);
-    assert_eq!(mock_three.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_two.hits(), 1);
+    assert!(mock_three.hits() <= 2);
     teardown_tmp_directory(tmp_dir);
-    Ok(())
 }
 
 #[test]
@@ -188,19 +182,17 @@ fn extractor_finds_filtered_content() -> Result<(), Box<dyn std::error::Error>>
     let (tmp_dir, file) =
         setup_tmp_directory(&["LICENSE".to_string(), "README".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/LICENSE")
-        .return_status(200)
-        .return_body(&srv.url("\"/homepage/assets/img/icons/handshake.svg\""))
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200)
+            .body(&srv.url("\"/homepage/assets/img/icons/handshake.svg\""));
+    });
 
-    let mock_two = Mock::new()
-        .expect_method(GET)
-        .expect_path("/homepage/assets/img/icons/handshake.svg")
-        .return_body("im a little teapot")
-        .return_status(200)
-        .create_on(&srv);
+    let mock_two = srv.mock(|when, then| {
+        when.method(GET)
+            .path("/homepage/assets/img/icons/handshake.svg");
+        then.status(200).body("im a little teapot");
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -222,8 +214,135 @@ fn extractor_finds_filtered_content() -> Result<(), Box<dyn std::error::Error>>
             .not(),
     );
 
-    assert_eq!(mock.times_called(), 1);
-    assert_eq!(mock_two.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_two.hits(), 1);
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// serve a robots.txt with a file and and a folder link contained within it. ferox should
+/// find both links and request each one. Additionally, a scan should start with the directory
+/// link found, meaning the wordlist will be thrown at the sub directory
+fn extractor_finds_robots_txt_links_and_displays_files_or_scans_directories() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("im a little teapot"); // 18
+    });
+
+    let mock_two = srv.mock(|when, then| {
+        when.method(GET).path("/robots.txt");
+        then.status(200).body(
+            r#"
+            User-agent: *
+            Crawl-delay: 10
+            # CSS, JS, Images
+            Allow: /misc/*.css$
+            Disallow: /misc/stupidfile.php
+               Disallow: /disallowed-subdir/
+            "#,
+        );
+    });
+
+    let mock_file = srv.mock(|when, then| {
+        when.method(GET).path("/misc/stupidfile.php");
+        then.status(200).body("im a little teapot too"); // 22
+    });
+
+    let mock_scanned_file = srv.mock(|when, then| {
+        when.method(GET).path("/misc/LICENSE");
+        then.status(200).body("i too, am a container for tea"); // 29
+    });
+
+    let mock_dir = srv.mock(|when, _| {
+        when.method(GET).path("/misc/");
+    });
+
+    let mock_disallowed = srv.mock(|when, then| {
+        when.method(GET).path("/disallowed-subdir");
+        then.status(404);
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--extract-links")
+        .arg("-vvvv")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE") // 2 directories contain LICENSE
+            .count(2)
+            .and(predicate::str::contains("18c"))
+            .and(predicate::str::contains("/misc/stupidfile.php"))
+            .and(predicate::str::contains("22c"))
+            .and(predicate::str::contains("/misc/LICENSE"))
+            .and(predicate::str::contains("29c"))
+            .and(predicate::str::contains("200").count(3)),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_dir.hits(), 1);
+    assert_eq!(mock_two.hits(), 1);
+    assert_eq!(mock_file.hits(), 1);
+    assert_eq!(mock_disallowed.hits(), 1);
+    assert_eq!(mock_scanned_file.hits(), 1);
+    teardown_tmp_directory(tmp_dir);
+}
+
+#[test]
+/// send a request to a page that contains a link that contains a directory that returns a 403
+/// --extract-links should find the link and make recurse into the 403 directory, finding LICENSE
+fn extractor_recurses_into_403_directories() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200)
+            .body(&srv.url("'/homepage/assets/img/icons/handshake.svg'"));
+    });
+
+    let mock_two = srv.mock(|when, then| {
+        when.method(GET).path("/homepage/assets/img/icons/LICENSE");
+        then.status(200).body("that's just like, your opinion man");
+    });
+
+    let forbidden_dir = srv.mock(|when, then| {
+        when.method(GET).path("/homepage/assets/img/icons/");
+        then.status(403);
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--extract-links")
+        .arg("--depth") // need to go past default 4 directories
+        .arg("0")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .count(2)
+            .and(predicate::str::contains("1w")) // link in /LICENSE
+            .and(predicate::str::contains("34c")) // recursed LICENSE
+            .and(predicate::str::contains(
+                "/homepage/assets/img/icons/LICENSE",
+            )),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_two.hits(), 1);
+    assert_eq!(forbidden_dir.hits(), 1);
     teardown_tmp_directory(tmp_dir);
     Ok(())
 }

tests/test_filters.rs

@@ -1,7 +1,7 @@
 mod utils;
 use assert_cmd::prelude::*;
 use httpmock::Method::GET;
-use httpmock::{Mock, MockServer};
+use httpmock::MockServer;
 use predicates::prelude::*;
 use std::process::Command;
 use utils::{setup_tmp_directory, teardown_tmp_directory};
@@ -14,19 +14,15 @@ fn filters_status_code_should_filter_response() {
     let (tmp_dir, file) =
         setup_tmp_directory(&["LICENSE".to_string(), "file.js".to_string()], "wordlist").unwrap();
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/LICENSE")
-        .return_status(302)
-        .return_body("this is a test")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(302).body("this is a test");
+    });
 
-    let mock_two = Mock::new()
-        .expect_method(GET)
-        .expect_path("/file.js")
-        .return_status(200)
-        .return_body("this is also a test of some import")
-        .create_on(&srv);
+    let mock_two = srv.mock(|when, then| {
+        when.method(GET).path("/file.js");
+        then.status(200).body("this is also a test of some import");
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -44,14 +40,210 @@ fn filters_status_code_should_filter_response() {
             .not()
             .and(predicate::str::contains("302"))
             .not()
-            .and(predicate::str::contains("14"))
+            .and(predicate::str::contains("14c"))
             .not()
             .and(predicate::str::contains("/file.js"))
             .and(predicate::str::contains("200"))
-            .and(predicate::str::contains("34")),
+            .and(predicate::str::contains("34c")),
     );
 
-    assert_eq!(mock.times_called(), 1);
-    assert_eq!(mock_two.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_two.hits(), 1);
+    teardown_tmp_directory(tmp_dir);
+}
+
+#[test]
+/// create a FeroxResponse that should elicit a true from
+/// LinesFilter::should_filter_response
+fn filters_lines_should_filter_response() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) =
+        setup_tmp_directory(&["LICENSE".to_string(), "file.js".to_string()], "wordlist").unwrap();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(302).body("this is a test");
+    });
+
+    let mock_two = srv.mock(|when, then| {
+        when.method(GET).path("/file.js");
+        then.status(200)
+            .body("this is also a test of some import\nwith 2 lines, no less");
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--filter-lines")
+        .arg("2")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .and(predicate::str::contains("302"))
+            .and(predicate::str::contains("14"))
+            .and(predicate::str::contains("/file.js"))
+            .not()
+            .and(predicate::str::contains("200"))
+            .not()
+            .and(predicate::str::contains("2l"))
+            .not(),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_two.hits(), 1);
+    teardown_tmp_directory(tmp_dir);
+}
+
+#[test]
+/// create a FeroxResponse that should elicit a true from
+/// WordsFilter::should_filter_response
+fn filters_words_should_filter_response() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) =
+        setup_tmp_directory(&["LICENSE".to_string(), "file.js".to_string()], "wordlist").unwrap();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(302).body("this is a test");
+    });
+
+    let mock_two = srv.mock(|when, then| {
+        when.method(GET).path("/file.js");
+        then.status(200)
+            .body("this is also a test of some import\nwith 2 lines, no less");
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--filter-words")
+        .arg("13")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .and(predicate::str::contains("302"))
+            .and(predicate::str::contains("14"))
+            .and(predicate::str::contains("/file.js"))
+            .not()
+            .and(predicate::str::contains("200"))
+            .not()
+            .and(predicate::str::contains("13w"))
+            .not(),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_two.hits(), 1);
+    teardown_tmp_directory(tmp_dir);
+}
+
+#[test]
+/// create a FeroxResponse that should elicit a true from
+/// SizeFilter::should_filter_response
+fn filters_size_should_filter_response() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) =
+        setup_tmp_directory(&["LICENSE".to_string(), "file.js".to_string()], "wordlist").unwrap();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(302).body("this is a test");
+    });
+
+    let mock_two = srv.mock(|when, then| {
+        when.method(GET).path("/file.js");
+        then.status(200)
+            .body("this is also a test of some import\nwith 2 lines, no less");
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--filter-size")
+        .arg("56")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .and(predicate::str::contains("302"))
+            .and(predicate::str::contains("14"))
+            .and(predicate::str::contains("/file.js"))
+            .not()
+            .and(predicate::str::contains("200"))
+            .not()
+            .and(predicate::str::contains("56c"))
+            .not(),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_two.hits(), 1);
+    teardown_tmp_directory(tmp_dir);
+}
+
+#[test]
+/// create a FeroxResponse that should elicit a true from
+/// SimilarityFilter::should_filter_response
+fn filters_similar_should_filter_response() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(
+        &["not-similar".to_string(), "similar".to_string()],
+        "wordlist",
+    )
+    .unwrap();
+
+    // ''.join(random.choices(string.ascii_letters + string.digits + string.whitespace, k=4096))
+    let content = "VCiYFr0HKsEIK6r\r1hJLYnOr90Aji\rDWAjQA3LVAzrluN48FuSPrRpm\n \tV\x0cx\nSCc5sX\nTB\x0c6Of7ns\t2HDwQCduKTqG8gG\x0beszazwljW01H60HMOLziOKwQwEYV7CbrLWQiLeCWKVxX\rvag\nAAEOhjER7gURuGXw\nMyY\t8mSw\x0b\x0bK0Z9G0Pt\x0bJZItAIqAq FxeaoOeLqWVFvxtDFfko0YVYt1I\rNmSXZ4lnOoiBCLbu6TLb80lClhY\tPN7Lp36F786I\nglwRK2oD45EtN SWW IF6uqKdf\x0czAcVycf\x0cBzHYnn1HAkU2Jluos0qwMGJ2m74z\nLd3\x0cIUVZmnRmHHWQGd1u2xmsZR\x0bfnml10ur6J\x0ba8xOZatiY 15Aq3KOGWdD3xQwqo\r5SKnnxH5tqU\rO\rZpJ\n7t7UUgfE\niWFgqWDpMeOG 1248M I\ro5B9Yed\r2aq2\tXxLn31s3hCV WEfQd60DKp6eFhUeUSeXDq6qjgTnWigoCZQERf\rXp7s2L37 iOEMl3\r41\nBShOjLfD8Kj0\rbu0ENreRjP\nY77jsrsaYgOsUrEzw\x0bw3OLi\n8fkddcaOvJeutTy B\rsDMkK\x0cnx2S0N\x0cDaY\x0c9iyo6p4IL\tOC1qgNlWP4VLg\tWmPG46ZMCirth5h4FwkS\nD2WsiEA2Z\n0xbLd7Uww hUQC6 3V\r1SsWem4UcQxG\rfuVvWl\nD9\nDpZQFFgiqhQiq1I0LMAR\r\rKBmj4iurrxaoMHTl9oj\x0b0N3AfD17gyqZiJ67bgizvecsRGeB1f\x0c\nYRvieJqIVHDKOOR\ruhqnVZz4BQ5FFBusz\x0cZl5\x0bt\tbdOUhAAAKyA6Jwl 7OjzojiRHGD6dl ncsgndsKURhFv4\tV5d\n73iPzbT\t8v6IrJtnq\nJuFl7A\x0b\rVnnsjTW0Y4QB1BgCy3B\x0cma7\tpPt5jmcJH7v5J\tYKEXh UqRChBFY5nbFbmXjJYxevPYJmSHC\rDQ4j9de\rTMZ\rtWaPAzkJjH\x0c\nyrEuf9WaMM\trFlKo9r9w\r\nQkQqIEu8Gfr\t aRzvN\r2oZhCyB4fa\np37\tXQi4Wa\no7gHUDQLoRvkK1dy2K3ydrI0O6\rFTGS7oHA\x0bajFOd\rcS5W25tFGhocwxM0\nuugNGDLjBQ\tWGdJV0\x0c\r7bNLs\x0cr deAWt35A4co\x0bPCuYmQ ExxtK\rvpckCyJxLrK5xULK\x0cvqtiGyovHQW8aDjV3rhXhR\nmQvmK\x0czLx\x0cECSYSF5jP35zN VkaRzQ lZ4 l06X4HHpsVn 8y8fGbIP\tRWFUAeFI24\rqN\x0cBW7u7WPMv36BmkgzQ\x0c2\x0cyLf\tYo8iRjE7zMsceym4ZnWg7EsOedh2cES\rz2n\x0cJi52uIPfSkAPzW\rEekjgWdb8y 285F4xae8\n8AiIkT4l3AOy\rT4yeXgaRMCI4t3PkHeFZ\rEb6R4FNCE \nbVil\x0c6qxSVPnU\nh\ttFMNE4\x0c\rwF\t\x0cW5vebbRWG\x0biVZLP\x0ct\x0c5gQ4CJ9KJl\x0cwyIfSIYaCvi4m1r\tJbYqmI0NVO36A\t8BSPNlaKbR73l9mxZxoqD4yca\n5h\r7a0z\tVm34aTy\tnLj5nSrh8er5lN0J7hcjmUk2DL\nyWEVNXTF8RWfC\x0bpcgBQXOQzidyYO\x0bh76UyUPAjELmNoECgGq06hiFGDI LiPZcofhcm\r62fEixIoyG\tmI\x0cYLQvBCbCluGgbm\x0c7GI6\n19il8PdPqss2uQqA5KgkHMIb hh211YuqV9kdmVnwyD63pz3p t58q6kHX\r\teYBrg6eDh\x0bx8\x0cI1SOV3Gt5qubmixHR\rApbgkTQJQ\tX0t11IP55hys2d\x0bF dh7j7G0Ac\x0bQMNvkSU9AV\x0b8mcIPHy9d\x0cyINf5qu\x0cdiBFrhiNRmCZ4r\tSx4N5VOm6KCp2T8bOVEjOR6otPAN5e\n\x0csyJ3giBjkgg 9dYQKq5P75AG5\x0bfD6zZO4DxQ44uX7Kz50dv4ncXQA\rqgHT\rLRcsRl\rW\t7We\tpAEJHMChxwVK\x0cprVvINvolf7hj\tUrob\rW3pXlqKIEQT8t7\x0bGODJanb328OiQCxE\rPfW4j\rl3p\x0cRXDB55u0MN7isBL\ty3UvE1 7I\nfuoZVPzk7az1\rMzA2FROXu0k\rFq pby6pHMqfTQT7iTw izlk0CUpyoUaq5w3UPFK7\rMOPw2cZ6FsVITbCoPhT\rIvuImCFGqmYpE hNevWkPCtwwnx2sX\x0c7oKzBExp32ZpdY\tstuDjSzfalsO1M\x0bNMUegnBDr3Liv3Lv\x0b\n37VZT2LEJ9fNYDi9r\t\x0bYC\rHSt0oJbk\x0b\x0bUdS8eB\nMXBPDEppZjHR7vGZYqX7yFm t1i682AXWf VPTzYTvm6mhOre8\x0bk0spJNYuI\tk\tC1B1N0 AYYDWH\t\tX1TjinXdkXcbFTlIiBLzx\rmUoyx9b7paJSVMX\tfLo8hU1Dmuluyk8R8\x0c4\x0cBe\nCrIMlyek4i\x0bFwuE9\nXUqpVxikH0PZspopUwPM9Kcue\rBh2Mf\rme3h4qelC\x0bEH\x0bkkxi6U\x0cE\x0ctqBgN93 V4ovmocLrK6\ngCQlf\x0cshRVvrPq\x0cOjgbjhSEK8PIx8OYqjjDDkJ0AgLhfbdGw2\nLMv2M0E08PGXnqUyVsjN\t C 4\n80 Fia g\x0b5dEFvyl5Y80U6sMAdHgk2nzC5ElDBhgcBprXC\x0bIMKXyt\x0ce5SkYcRartfblLqD1 A5\nre\x0bj67lJYCs\t8b50xA69eMHqGDLLP8sJceN19kkonjLj\t\rS\tk9sMOeewQHbT \x0cp53aMX9\x0bDYCZWAtdA6h\rAFHDEYFBE1MzdOxMO\x0cvDE7QfLb3jq4s\tI3aVTmDDOQAnuvWb2AGUUP\rf2HinUAiF13LKEfpqcD06S8aQC0Kyl729L7a6CbuoB0GRlJx tD yuTVqD62HuXpfKrDsbejEdp3\rxjc\x0bn4lLNaViizec\rWR\x0cTT5aZ\ny9\rO1qB1XGQPnES\nUhJtU Ll7t3Zglj1IAEx 8Rh3V\x0bfmUSC4\x0bVR9l33LS3bPAJpLbH3Q2\nv2fqMeIt3nGR\x0cgCixM4qzVSx7Yb192a1HWx8nnuWQIEK7QHL6p\x0cD3d0Y1FoZqsmY2U\rspvt3gwKOHR6RaZlmhX\n3bmIEF6\x0b\x0bMXJKOnXPgjkdhun4aGDBw\x0cOEW\repDYTcc48oZ4lg7PukNq7TU\tWP0ZJbzVKK\rxAMaZujwTqQXsXODiE2DdwnstAa6CMYfzj7J\x0c2Q\tY2764IYCy 3Fqm0\x0ckbe7VvfqWUh0\tUlubxZ\rX59MfNSfCfcH8GFZIGIRPt\rZVXfra1 H7VI2yJ\x0cspGDCi\rcgHfZa8528CP9tilUx0ifWPGqskLVDPLJP\nciNxodMQSrJXp\ro\r9aBFHCV\x0cR\rrp\x0bmMfxg5rG\tSuWonbJQlmHQ\ri34w8S\x0cN9Ezj2k2OmLH\x0cEcVUDjXNZIFCtlA843I44p GZyhlOctwpd7 OZnUxk4uacN\r8NihNGO\n9eXy5l6gQe5srySxxvuX5jtCzuJ35xvCfEXYa\x0b2lTDBOAaSYpnl v9L\x0cY8RLg2oE7xeCUbD\tSHKZgeXHZIzYAmA7bsmiZUfzmo5ZZUhtBh4F\x0bTx1\x0bz zQov5mYwfpWJTR2Q\x0bLRXMuBzj\x0bZC\x0b pFNPj8ixWJQggQlr9eNW6SHLJk731nc\x0cBn\x0ckQxg2BdRT\x0bp6lf7G\x0bnIMDeY8w6fUf\x0cjGE1Pfsekv7EYEIHsOAsZb3lBfBPO9\tXpHPBMRmRtzMc5WoX6C5cc\x0cBuTPtPOgXnap1Y3xq7pcMcgu55xblsXEAJKsojjR7aDB\tU84kUKRNEj\n8mcqEyOmvq1WA\na6bhzYf9VQv2aj9KLfByVqUKNFVIc4Mkha\x0c0aCPQSKe0GGwPlSfbtNXhdhxAb3RLf1J\x0cshJzjQe4DCmlRmjt\tlB0BwzBpkg2hTYM\r S\x0cux\x0bj6IcEZ\n\ngQ\rKKgg \rrv4sUMy5sfY1aatjK1MmUyXR\rRHk\x0cqq\x0cD1fy4C0\n\x0byd4SFKOyKJqx2mzI74vPxLLo\x0c0OamjXuUu\nWGkiA70nuf0PGRfwLEBPCMeyneJI1HcIXH\nCTFEIMiAq6fT\rmJgC hXEU\rriAhCm3OzgbcDgvQgDSyUw5jl\x0cTaLOPuFseq\x0cj2npTd57itktTdWBY7sqlOGKNSc\x0ctx2mUoHi31EF3l5lvYPDeG6bIPFwIn7\tG6G \x0bgNkSn89flvqcvI73RA";
+    let mutated = "VCiYFr0HKsEIK6r\r1hJLYnOr90Aji\rDWAjQA3LVAzrluN484327FuSPrRpm\n \tV\x0cx\nSCc5sX\nTB\x0c6Of7ns\t2HDwQCduKTqG8gG\x0beszazwljW01H60HMOLziOKwQwEYV7CbrLWQiLeCWKVxX\rvag\nAAEOhjER7gURuGXw\nMyY\t8mSw\x0b\x0bK0Z9G0Pt\x0bJZItAIqAq FxeaoOeLqWVFvxtDFfko0YVYt1I\rNmSXZ4lnOoiBCLbu6TLb80lClhY\tPN7Lp36F786I\nglwRK2oD45EtN SWW IF6uqKdf\x0czAcVycf\x0cBzHYnn1HAkU2Jluos0qwMGJ2m74z\nLd3\x0cIUVZmnRmHHWQGd1u2xmsZR\x0bfnml10ur6J\x0ba8xOZatiY 15Aq3KOGWdD3xQwqo\r5SKnnxH5tqU\rO\rZpJ\n7t7UUgfE\niWFgqWDpMeOG 1248M I\ro5B9Yed\r2aq2\tXxLn31s3hCV WEfQd60DKp6eFhUeUSeXDq6qjgTnWigoCZQERf\rXp7s2L37 iOEMl3\r41\nBShOjLfD8Kj0\rbu0ENreRjP\nY77jsrsaYgOsUrEzw\x0bw3OLi\n8fkddcaOvJeutTy B\rsDMkK\x0cnx2S0N\x0cDaY\x0c9iyo6p4IL\tOC1qgNlWP4VLg\tWmPG46ZMCirth5h4FwkS\nD2WsiEA2Z\n0xbLd7Uww hUQC6 3V\r1SsWem4UcQxG\rfuVvWl\nD9\nDpZQFFgiqhQiq1I0LMAR\r\rKBmj4iurrxaoMHTl9oj\x0b0N3AfD17gyqZiJ67bgizvecsRGeB1f\x0c\nYRvieJqIVHDKOOR\ruhqnVZz4BQ5FFBusz\x0cZl5\x0bt\tbdOUhAAAKyA6Jwl 7OjzojiRHGD6dl ncsgndsKURhFv4\tV5d\n73iPzbT\t8v6IrJtnq\nJuFl7A\x0b\rVnnsjTW0Y4QB1BgCy3B\x0cma7\tpPt5jmcJH7v5J\tYKEXh UqRChBFY5nbFbmXjJYxevPYJmSHC\rDQ4j9de\rTMZ\rtWaPAzkJjH\x0c\nyrEuf9WaMM\trFlKo9r9w\r\nQkQqIEu8Gfr\t aRzvN\r2oZhCyB4fa\np37\tXQi4Wa\no7gHUDQLoRvkK1dy2K3ydrI0O6\rFTGS7oHA\x0bajFOd\rcS5W25tFGhocwxM0\nuugNGDLjBQ\tWGdJV0\x0c\r7bNLs\x0cr deAWt35A4co\x0bPCuYmQ ExxtK\rvpckCyJxLrK5xULK\x0cvqtiGyovHQW8aDjV3rhXhR\nmQvmK\x0czLx\x0cECSYSF5jP35zN VkaRzQ lZ4 l06X4HHpsVn 8y8fGbIP\tRWFUAeFI24\rqN\x0cBW7u7WPMv36BmkgzQ\x0c2\x0cyLf\tYo8iRjE7zMsceym4ZnWg7EsOedh2cES\rz2n\x0cJi52uIPfSkAPzW\rEekjgWdb8y 285F4xae8\n8AiIkT4l3AOy\rT4yeXgaRMCI4t3PkHeFZ\rEb6R4FNCE \nbVil\x0c6qxSVPnU\nh\ttFMNE4\x0c\rwF\t\x0cW5vebbRWG\x0biVZLP\x0ct\x0c5gQ4CJ9KJl\x0cwyIfSIYaCvi4m1r\tJbYqmI0NVO36A\t8BSPNlaKbR73l9mxZxoqD4yca\n5h\r7a0z\tVm34aTy\tnLj5nSrh8er5lN0J7hcjmUk2DL\nyWEVNXTF8RWfC\x0bpcgBQXOQzidyYO\x0bh76UyUPAjELmNoECgGq06hiFGDI LiPZcofhcm\r62fEixIoyG\tmI\x0cYLQvBCbCluGgbm\x0c7GI6\n19il8PdPqss2uQqA5KgkHMIb hh211YuqV9kdmVnwyD63pz3p t58q6kHX\r\teYBrg6eDh\x0bx8\x0cI1SOV3Gt5qubmixHR\rApbgkTQJQ\tX0t11IP55hys2d\x0bF dh7j7G0Ac\x0bQMNvkSU9AV\x0b8mcIPHy9d\x0cyINf5qu\x0cdiBFrhiNRmCZ4r\tSx4N5VOm6KCp2T8bOVEjOR6otPAN5e\n\x0csyJ3giBjkgg 9dYQKq5P75AG5\x0bfD6zZO4DxQ44uX7Kz50dv4ncXQA\rqgHT\rLRcsRl\rW\t7We\tpAEJHMChxwVK\x0cprVvINvolf7hj\tUrob\rW3pXlqKIEQT8t7\x0bGODJanb328OiQCxE\rPfW4j\rl3p\x0cRXDB55u0MN7isBL\ty3UvE1 7I\nfuoZVPzk7az1\rMzA2FROXu0k\rFq pby6pHMqfTQT7iTw izlk0CUpyoUaq5w3UPFK7\rMOPw2cZ6FsVITbCoPhT\rIvuImCFGqmYpE hNevWkPCtwwnx2sX\x0c7oKzBExp32ZpdY\tstuDjSzfalsO1M\x0bNMUegnBDr3Liv3Lv\x0b\n37VZT2LEJ9fNYDi9r\t\x0bYC\rHSt0oJbk\x0b\x0bUdS8eB\nMXBPDEppZjHR7vGZYqX7yFm t1i682AXWf VPTzYTvm6mhOre8\x0bk0spJNYuI\tk\tC1B1N0 AYYDWH\t\tX1TjinXdkXcbFTlIiBLzx\rmUoyx9b7paJSVMX\tfLo8hU1Dmuluyk8R8\x0c4\x0cBe\nCrIMlyek4i\x0bFwuE9\nXUqpVxikH0PZspopUwPM9Kcue\rBh2Mf\rme3h4qelC\x0bEH\x0bkkxi6U\x0cE\x0ctqBgN93 V4ovmocLrK6\ngCQlf\x0cshRVvrPq\x0cOjgbjhSEK8PIx8OYqjjDDkJ0AgLhfbdGw2\nLMv2M0E08PGXnqUyVsjN\t C 4\n80 Fia g\x0b5dEFvyl5Y80U6sMAdHgk2nzC5ElDBhgcBprXC\x0bIMKXyt\x0ce5SkYcRartfblLqD1 A5\nre\x0bj67lJYCs\t8b50xA69eMHqGDLLP8sJceN19kkonjLj\t\rS\tk9sMOeewQHbT \x0cp53aMX9\x0bDYCZWAtdA6h\rAFHDEYFBE1MzdOxMO\x0cvDE7QfLb3jq4s\tI3aVTmDDOQAnuvWb2AGUUP\rf2HinUAiF13LKEfpqcD06S8aQC0Kyl729L7a6CbuoB0GRlJx tD yuTVqD62HuXpfKrDsbejEdp3\rxjc\x0bn4lLNaViizec\rWR\x0cTT5aZ\ny9\rO1qB1XGQPnES\nUhJtU Ll7t3Zglj1IAEx 8Rh3V\x0bfmUSC4\x0bVR9l33LS3bPAJpLbH3Q2\nv2fqMeIt3nGR\x0cgCixM4qzVSx7Yb192a1HWx8nnuWQIEK7QHL6p\x0cD3d0Y1FoZqsmY2U\rspvt3gwKOHR6RaZlmhX\n3bmIEF6\x0b\x0bMXJKOnXPgjkdhun4aGDBw\x0cOEW\repDYTcc48oZ4lg7PukNq7TU\tWP0ZJbzVKK\rxAMaZujwTqQXsXODiE2DdwnstAa6CMYfzj7J\x0c2Q\tY2764IYCy 3Fqm0\x0ckbe7VvfqWUh0\tUlubxZ\rX59MfNSfCfcH8GFZIGIRPt\rZVXfra1 H7VI2yJ\x0cspGDCi\rcgHfZa8528CP9tilUx0ifWPGqskLVDPLJP\nciNxodMQSrJXp\ro\r9aBFHCV\x0cR\rrp\x0bmMfxg5rG\tSuWonbJQlmHQ\ri34w8S\x0cN9Ezj2k2OmLH\x0cEcVUDjXNZIFCtlA843I44p GZyhlOctwpd7 OZnUxk4uacN\r8NihNGO\n9eXy5l6gQe5srySxxvuX5jtCzuJ35xvCfEXYa\x0b2lTDBOAaSYpnl v9L\x0cY8RLg2oE7xeCUbD\tSHKZgeXHZIzYAmA7bsmiZUfzmo5ZZUhtBh4F\x0bTx1\x0bz zQov5mYwfpWJTR2Q\x0bLRXMuBzj\x0bZC\x0b pFNPj8ixWJQggQlr9eNW6SHLJk731nc\x0cBn\x0ckQxg2BdRT\x0bp6lf7G\x0bnIMDeY8w6fUf\x0cjGE1Pfsekv7EYEIHsOAsZb3lBfBPO9\tXpHPBMRmRtzMc5WoX6C5cc\x0cBuTPtPOgXnap1Y3xq7pcMcgu55xblsXEAJKsojjR7aDB\tU84kUKRNEj\n8mcqEyOmvq1WA\na6bhzYf9VQv2aj9KLfByVqUKNFVIc4Mkha\x0c0aCPQSKe0GGwPlSfbtNXhdhxAb3RLf1J\x0cshJzjQe4DCmlRmjt\tlB0BwzBpkg2hTYM\r S\x0cux\x0bj6IcEZ\n\ngQ\rKKgg \rrv4sUMy5sfY1aatjK1MmUyXR\rRHk\x0cqq\x0cD1fy4C0\n\x0byd4SFKOyKJqx2mzI74vPxLLo\x0c0OamjXuUu\nWGkiA70nuf0PGRfwLEBPCMeyneJI1HcIXH\nCTFEIMiAq6fT\rmJgC hXEU\rriAhCm3OzgbcDgvQgDSyUw5jl\x0cTaimauFseq\x0cj2npTd57itktTdWBY7sqlOGKNSc\x0ctx2mUoHi31EF3l5lvYPDeG6bIPFwIn7\tG6G \x0bgNkSn89flvqcvI73RA";
+
+    let canary = srv.mock(|when, then| {
+        when.method(GET).path("/canary");
+        then.status(200).body(content);
+    });
+
+    // not similar, should see results in output
+    let not_similar = srv.mock(|when, then| {
+        when.method(GET).path("/not-similar");
+        then.status(302).body("this is a test");
+    });
+
+    // similar, should not see results
+    let similar = srv.mock(|when, then| {
+        when.method(GET).path("/similar");
+        then.status(200).body(mutated);
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--filter-similar-to")
+        .arg(srv.url("/canary"))
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICfdafdsafdsafadsENSE")
+            .and(predicate::str::contains("302"))
+            .and(predicate::str::contains("14c"))
+            .and(predicate::str::contains("/similar"))
+            .not()
+            .and(predicate::str::contains("4100c"))
+            .not(),
+    );
+
+    assert_eq!(canary.hits(), 1);
+    assert_eq!(similar.hits(), 1);
+    assert_eq!(not_similar.hits(), 1);
     teardown_tmp_directory(tmp_dir);
 }

tests/test_heuristics.rs

@@ -2,7 +2,7 @@ mod utils;
 use assert_cmd::prelude::*;
 use assert_cmd::Command;
 use httpmock::Method::GET;
-use httpmock::{Mock, MockServer, Regex};
+use httpmock::{MockServer, Regex};
 use predicates::prelude::*;
 use utils::{setup_tmp_directory, teardown_tmp_directory};
 
@@ -65,12 +65,10 @@ fn test_one_good_and_one_bad_target_scan_succeeds() -> Result<(), Box<dyn std::e
     let urls = vec![not_real, srv.url("/"), String::from("LICENSE")];
     let (tmp_dir, file) = setup_tmp_directory(&urls, "wordlist")?;
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/LICENSE")
-        .return_status(200)
-        .return_body("this is a test")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
 
     let mut cmd = Command::cargo_bin("feroxbuster").unwrap();
 
@@ -86,7 +84,51 @@ fn test_one_good_and_one_bad_target_scan_succeeds() -> Result<(), Box<dyn std::e
                 .and(predicate::str::contains("200"))
                 .and(predicate::str::contains("14")),
         );
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
+
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// test pipes two good targets to the scanner, expected result is that both targets
+/// are scanned successfully and no error is reported (result of issue #169)
+fn test_two_good_targets_scan_succeeds() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let srv2 = MockServer::start();
+
+    let urls = vec![srv.url("/"), srv2.url("/"), String::from("LICENSE")];
+    let (tmp_dir, file) = setup_tmp_directory(&urls, "wordlist")?;
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
+
+    let mock2 = srv2.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(403).body("this also is a test");
+    });
+
+    let mut cmd = Command::cargo_bin("feroxbuster").unwrap();
+
+    cmd.arg("--stdin")
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .pipe_stdin(file)
+        .unwrap()
+        .assert()
+        .success()
+        .stdout(
+            predicate::str::contains("/LICENSE")
+                .and(predicate::str::contains("200"))
+                .and(predicate::str::contains("403"))
+                .and(predicate::str::contains("14c"))
+                .and(predicate::str::contains("19c")),
+        );
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock2.hits(), 1);
 
     teardown_tmp_directory(tmp_dir);
     Ok(())
@@ -98,12 +140,11 @@ fn test_static_wildcard_request_found() -> Result<(), Box<dyn std::error::Error>
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap())
-        .return_status(200)
-        .return_body("this is a test")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap());
+        then.status(200).body("this is a test");
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -123,30 +164,28 @@ fn test_static_wildcard_request_found() -> Result<(), Box<dyn std::error::Error>
             .and(predicate::str::contains("(url length: 32)")),
     );
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
     Ok(())
 }
 
 #[test]
 /// test finds a dynamic wildcard and reports as much to stdout and a file
-fn test_dynamic_wildcard_request_found() -> Result<(), Box<dyn std::error::Error>> {
+fn test_dynamic_wildcard_request_found() {
     let srv = MockServer::start();
-    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
     let outfile = tmp_dir.path().join("outfile");
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap())
-        .return_status(200)
-        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap());
+        then.status(200)
+            .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+    });
 
-    let mock2 = Mock::new()
-        .expect_method(GET)
-        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap())
-        .return_status(200)
-        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
-        .create_on(&srv);
+    let mock2 = srv.mock(|when, then| {
+        when.method(GET).path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap());
+        then.status(200).body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -166,31 +205,19 @@ fn test_dynamic_wildcard_request_found() -> Result<(), Box<dyn std::error::Error
     assert_eq!(contents.contains("WLD"), true);
     assert_eq!(contents.contains("Got"), true);
     assert_eq!(contents.contains("200"), true);
-    assert_eq!(contents.contains("auto-filtering"), true);
     assert_eq!(contents.contains("(url length: 32)"), true);
     assert_eq!(contents.contains("(url length: 96)"), true);
-    assert_eq!(contents.contains("Wildcard response is dynamic"), true);
-    assert_eq!(
-        contents.contains("(14 + url length) responses; toggle this behavior by using"),
-        true
-    );
 
     cmd.assert().success().stdout(
         predicate::str::contains("WLD")
             .and(predicate::str::contains("Got"))
             .and(predicate::str::contains("200"))
             .and(predicate::str::contains("(url length: 32)"))
-            .and(predicate::str::contains("(url length: 96)"))
-            .and(predicate::str::contains("Wildcard response is dynamic;"))
-            .and(predicate::str::contains("auto-filtering"))
-            .and(predicate::str::contains(
-                "(14 + url length) responses; toggle this behavior by using",
-            )),
+            .and(predicate::str::contains("(url length: 96)")),
     );
 
-    assert_eq!(mock.times_called(), 1);
-    assert_eq!(mock2.times_called(), 1);
-    Ok(())
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock2.hits(), 1);
 }
 
 #[test]
@@ -199,12 +226,11 @@ fn heuristics_static_wildcard_request_with_dont_filter() -> Result<(), Box<dyn s
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap())
-        .return_status(200)
-        .return_body("this is a test")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap());
+        then.status(200).body("this is a test");
+    });
 
     Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -217,29 +243,29 @@ fn heuristics_static_wildcard_request_with_dont_filter() -> Result<(), Box<dyn s
 
     teardown_tmp_directory(tmp_dir);
 
-    assert_eq!(mock.times_called(), 0);
+    assert_eq!(mock.hits(), 0);
     Ok(())
 }
 
 #[test]
 /// test finds a static wildcard and reports as much to stdout
-fn heuristics_wildcard_test_with_two_static_wildcards() -> Result<(), Box<dyn std::error::Error>> {
+fn heuristics_wildcard_test_with_two_static_wildcards() {
     let srv = MockServer::start();
-    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap())
-        .return_status(200)
-        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap());
+        then.status(200)
+            .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+    });
 
-    let mock2 = Mock::new()
-        .expect_method(GET)
-        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap())
-        .return_status(200)
-        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
-        .create_on(&srv);
+    let mock2 = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap());
+        then.status(200)
+            .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -263,9 +289,8 @@ fn heuristics_wildcard_test_with_two_static_wildcards() -> Result<(), Box<dyn st
             )),
     );
 
-    assert_eq!(mock.times_called(), 1);
-    assert_eq!(mock2.times_called(), 1);
-    Ok(())
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock2.hits(), 1);
 }
 
 #[test]
@@ -275,19 +300,19 @@ fn heuristics_wildcard_test_with_two_static_wildcards_with_quiet_enabled(
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap())
-        .return_status(200)
-        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap());
+        then.status(200)
+            .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+    });
 
-    let mock2 = Mock::new()
-        .expect_method(GET)
-        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap())
-        .return_status(200)
-        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
-        .create_on(&srv);
+    let mock2 = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap());
+        then.status(200)
+            .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -303,32 +328,31 @@ fn heuristics_wildcard_test_with_two_static_wildcards_with_quiet_enabled(
 
     cmd.assert().success().stdout(predicate::str::is_empty());
 
-    assert_eq!(mock.times_called(), 1);
-    assert_eq!(mock2.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock2.hits(), 1);
     Ok(())
 }
 
 #[test]
 /// test finds a static wildcard and reports as much to stdout and a file
-fn heuristics_wildcard_test_with_two_static_wildcards_and_output_to_file(
-) -> Result<(), Box<dyn std::error::Error>> {
+fn heuristics_wildcard_test_with_two_static_wildcards_and_output_to_file() {
     let srv = MockServer::start();
-    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
     let outfile = tmp_dir.path().join("outfile");
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap())
-        .return_status(200)
-        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap());
+        then.status(200)
+            .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+    });
 
-    let mock2 = Mock::new()
-        .expect_method(GET)
-        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap())
-        .return_status(200)
-        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
-        .create_on(&srv);
+    let mock2 = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap());
+        then.status(200)
+            .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -350,10 +374,6 @@ fn heuristics_wildcard_test_with_two_static_wildcards_and_output_to_file(
     assert_eq!(contents.contains("200"), true);
     assert_eq!(contents.contains("(url length: 32)"), true);
     assert_eq!(contents.contains("(url length: 96)"), true);
-    assert_eq!(
-        contents.contains("Wildcard response is static; auto-filtering 46"),
-        true
-    );
 
     cmd.assert().success().stdout(
         predicate::str::contains("WLD")
@@ -366,10 +386,8 @@ fn heuristics_wildcard_test_with_two_static_wildcards_and_output_to_file(
             )),
     );
 
-    assert_eq!(mock.times_called(), 1);
-    assert_eq!(mock2.times_called(), 1);
-
-    Ok(())
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock2.hits(), 1);
 }
 
 #[test]
@@ -381,20 +399,20 @@ fn heuristics_wildcard_test_with_redirect_as_response_code(
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
     let outfile = tmp_dir.path().join("outfile");
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap())
-        .return_status(301)
-        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap());
+        then.status(301)
+            .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+    });
 
-    let mock2 = Mock::new()
-        .expect_method(GET)
-        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap())
-        .return_status(301)
-        .return_header("Location", &srv.url("/some-redirect"))
-        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
-        .create_on(&srv);
+    let mock2 = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap());
+        then.status(301)
+            .header("Location", &srv.url("/some-redirect"))
+            .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -427,7 +445,7 @@ fn heuristics_wildcard_test_with_redirect_as_response_code(
             .and(predicate::str::contains("WLD")),
     );
 
-    assert_eq!(mock.times_called(), 1);
-    assert_eq!(mock2.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock2.hits(), 1);
     Ok(())
 }

tests/test_main.rs

@@ -1,7 +1,7 @@
 mod utils;
 use assert_cmd::Command;
 use httpmock::Method::GET;
-use httpmock::{Mock, MockServer};
+use httpmock::MockServer;
 use predicates::prelude::*;
 use utils::{setup_tmp_directory, teardown_tmp_directory};
 
@@ -10,12 +10,10 @@ use utils::{setup_tmp_directory, teardown_tmp_directory};
 fn main_use_root_owned_file_as_wordlist() -> Result<(), Box<dyn std::error::Error>> {
     let srv = MockServer::start();
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/")
-        .return_status(200)
-        .return_body("this is a test")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/");
+        then.status(200).body("this is a test");
+    });
 
     Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -29,7 +27,7 @@ fn main_use_root_owned_file_as_wordlist() -> Result<(), Box<dyn std::error::Erro
         .stdout(predicate::str::contains("Permission denied (os error 13)"));
 
     // connectivity test hits it once
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
     Ok(())
 }
 
@@ -39,12 +37,10 @@ fn main_use_empty_wordlist() -> Result<(), Box<dyn std::error::Error>> {
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&[], "wordlist")?;
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/")
-        .return_status(200)
-        .return_body("this is a test")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/");
+        then.status(200).body("this is a test");
+    });
 
     Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -57,7 +53,7 @@ fn main_use_empty_wordlist() -> Result<(), Box<dyn std::error::Error>> {
         .failure()
         .stdout(predicate::str::contains("Did not find any words in"));
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
 
     teardown_tmp_directory(tmp_dir);
     Ok(())

tests/test_scan_manager.rs

@@ -0,0 +1,130 @@
+mod utils;
+use assert_cmd::Command;
+use httpmock::Method::GET;
+use httpmock::MockServer;
+use predicates::prelude::*;
+use std::fs::{read_to_string, write};
+use std::path::Path;
+use std::time;
+use utils::{setup_tmp_directory, teardown_tmp_directory};
+
+#[test]
+/// pass a known serialized scan with 1 scan complete and 1 not. expect the incomplete scan to
+/// start and the complete to not start. expect the responses, scans, and configuration structures
+/// to be populated based off the contents of the given state file
+fn resume_scan_works() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) =
+        setup_tmp_directory(&["css".to_string(), "stuff".to_string()], "wordlist").unwrap();
+
+    // localhost:PORT/ <- complete
+    // localhost:PORT/js <- will get scanned with /css and /stuff
+    let complete_scan = format!(
+        r#"{{"id":"057016a14769414aac9a7a62707598cb","url":"{}","scan_type":"Directory","status":"Complete"}}"#,
+        srv.url("/")
+    );
+    let incomplete_scan = format!(
+        r#"{{"id":"400b2323a16f43468a04ffcbbeba34c6","url":"{}","scan_type":"Directory","status":"NotStarted"}}"#,
+        srv.url("/js")
+    );
+    let scans = format!(r#""scans":[{},{}]"#, complete_scan, incomplete_scan);
+
+    let config = format!(
+        r#""config": {{"type":"configuration","wordlist":"{}","config":"","proxy":"","replay_proxy":"","target_url":"{}","status_codes":[200,204,301,302,307,308,401,403,405],"replay_codes":[200,204,301,302,307,308,401,403,405],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"quiet":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/1.9.0","redirects":false,"insecure":false,"extensions":[],"headers":{{}},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":false,"depth":2,"scan_limit":1,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false}}"#,
+        file.to_string_lossy(),
+        srv.url("/")
+    );
+
+    // // localhost:PORT/js/css has already been seen, expect not to be scanned
+    let response = format!(
+        r#"{{"type":"response","url":"{}","path":"/js/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{{"server":"nginx/1.16.1"}}}}"#,
+        srv.url("/js/css")
+    );
+    let responses = format!(r#""responses":[{}]"#, response);
+
+    // not scanned because /js is not complete, and /js/stuff response is not known
+    let not_scanned_yet = srv.mock(|when, then| {
+        when.method(GET).path("/js/stuff");
+        then.status(200).body("i expect to be scanned");
+    });
+
+    // will get scanned because /js is not complete, but because response of /js/css is known, the
+    // response will not be in stdout
+    let already_scanned = srv.mock(|when, then| {
+        when.method(GET).path("/js/css");
+        then.status(200);
+    });
+
+    // already scanned because scan on / is complete
+    let also_already_scanned = srv.mock(|when, then| {
+        when.method(GET).path("/css");
+        then.status(200).body("two words");
+    });
+
+    let state_file_contents = format!("{{{},{},{}}}", scans, config, responses);
+    let (tmp_dir2, state_file) = setup_tmp_directory(&[state_file_contents], "state-file").unwrap();
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--resume-from")
+        .arg(state_file.as_os_str())
+        .assert()
+        .success()
+        .stdout(
+            predicate::str::contains("/js/stuff")
+                .and(predicate::str::contains("22c"))
+                .and(predicate::str::contains("5w"))
+                .and(predicate::str::contains("/js/css"))
+                .not()
+                .and(predicate::str::contains("2w"))
+                .not()
+                .and(predicate::str::contains("9c"))
+                .not(),
+        );
+
+    teardown_tmp_directory(tmp_dir);
+    teardown_tmp_directory(tmp_dir2);
+
+    assert_eq!(already_scanned.hits(), 1);
+    assert_eq!(also_already_scanned.hits(), 0);
+    assert_eq!(not_scanned_yet.hits(), 1);
+}
+
+#[test]
+/// kick off scan with a time limit;  
+fn time_limit_enforced_when_specified() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) =
+        setup_tmp_directory(&["css".to_string(), "stuff".to_string()], "wordlist").unwrap();
+
+    // ensure the command will run long enough by adding crap to the wordlist
+    let more_words = read_to_string(Path::new("tests/extra-words")).unwrap();
+    write(&file, more_words).unwrap();
+
+    assert!(file.metadata().unwrap().len() > 100); // sanity check on wordlist size
+
+    let now = time::Instant::now();
+    let lower_bound = time::Duration::new(5, 0);
+    let upper_bound = time::Duration::new(6, 0);
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--time-limit")
+        .arg("5s")
+        .assert()
+        .failure();
+
+    // expected run time is somewhere in the 30 seconds ballpark (real    0m37.376s)
+    // so if the cmd returns in a significantly shorter amount of time, the test will have
+    // succeeded
+
+    // --time-limit is 5 seconds, so elapsed should be in a window that is greater than 5
+    // but significantly less than 30ish
+    assert!(now.elapsed() > lower_bound && now.elapsed() < upper_bound);
+
+    teardown_tmp_directory(tmp_dir);
+}

tests/test_scanner.rs

@@ -1,7 +1,7 @@
 mod utils;
 use assert_cmd::prelude::*;
 use httpmock::Method::GET;
-use httpmock::{Mock, MockServer};
+use httpmock::MockServer;
 use predicates::prelude::*;
 use std::process::Command;
 use utils::{setup_tmp_directory, teardown_tmp_directory};
@@ -12,12 +12,10 @@ fn scanner_single_request_scan() -> Result<(), Box<dyn std::error::Error>> {
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/LICENSE")
-        .return_status(200)
-        .return_body("this is a test")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -34,7 +32,7 @@ fn scanner_single_request_scan() -> Result<(), Box<dyn std::error::Error>> {
             .and(predicate::str::contains("14")),
     );
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
     teardown_tmp_directory(tmp_dir);
     Ok(())
 }
@@ -51,33 +49,26 @@ fn scanner_recursive_request_scan() -> Result<(), Box<dyn std::error::Error>> {
     ];
     let (tmp_dir, file) = setup_tmp_directory(&urls, "wordlist")?;
 
-    let js_mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/js")
-        .return_status(301)
-        .return_header("Location", &srv.url("/js/"))
-        .create_on(&srv);
+    let js_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js");
+        then.status(301).header("Location", &srv.url("/js/"));
+    });
 
-    let js_prod_mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/js/prod")
-        .return_status(301)
-        .return_header("Location", &srv.url("/js/prod/"))
-        .create_on(&srv);
+    let js_prod_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js/prod");
+        then.status(301).header("Location", &srv.url("/js/prod/"));
+    });
 
-    let js_dev_mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/js/dev")
-        .return_status(301)
-        .return_header("Location", &srv.url("/js/dev/"))
-        .create_on(&srv);
+    let js_dev_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js/dev");
+        then.status(301).header("Location", &srv.url("/js/dev/"));
+    });
 
-    let js_dev_file_mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/js/dev/file.js")
-        .return_status(200)
-        .return_body("this is a test and is more bytes than other ones")
-        .create_on(&srv);
+    let js_dev_file_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js/dev/file.js");
+        then.status(200)
+            .body("this is a test and is more bytes than other ones");
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -98,10 +89,10 @@ fn scanner_recursive_request_scan() -> Result<(), Box<dyn std::error::Error>> {
             .and(predicate::str::is_match("200.*js/dev/file.js").unwrap()),
     );
 
-    assert_eq!(js_mock.times_called(), 1);
-    assert_eq!(js_prod_mock.times_called(), 1);
-    assert_eq!(js_dev_mock.times_called(), 1);
-    assert_eq!(js_dev_file_mock.times_called(), 1);
+    assert_eq!(js_mock.hits(), 1);
+    assert_eq!(js_prod_mock.hits(), 1);
+    assert_eq!(js_dev_mock.hits(), 1);
+    assert_eq!(js_dev_file_mock.hits(), 1);
 
     teardown_tmp_directory(tmp_dir);
 
@@ -121,33 +112,26 @@ fn scanner_recursive_request_scan_using_only_success_responses(
     ];
     let (tmp_dir, file) = setup_tmp_directory(&urls, "wordlist")?;
 
-    let js_mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/js/")
-        .return_status(200)
-        .return_header("Location", &srv.url("/js/"))
-        .create_on(&srv);
+    let js_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js/");
+        then.status(200).header("Location", &srv.url("/js/"));
+    });
 
-    let js_prod_mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/js/prod/")
-        .return_status(200)
-        .return_header("Location", &srv.url("/js/prod/"))
-        .create_on(&srv);
+    let js_prod_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js/prod/");
+        then.status(200).header("Location", &srv.url("/js/prod/"));
+    });
 
-    let js_dev_mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/js/dev/")
-        .return_status(200)
-        .return_header("Location", &srv.url("/js/dev/"))
-        .create_on(&srv);
+    let js_dev_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js/dev/");
+        then.status(200).header("Location", &srv.url("/js/dev/"));
+    });
 
-    let js_dev_file_mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/js/dev/file.js")
-        .return_status(200)
-        .return_body("this is a test and is more bytes than other ones")
-        .create_on(&srv);
+    let js_dev_file_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js/dev/file.js");
+        then.status(200)
+            .body("this is a test and is more bytes than other ones");
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -169,10 +153,10 @@ fn scanner_recursive_request_scan_using_only_success_responses(
             .and(predicate::str::is_match("200.*js/dev/file.js").unwrap()),
     );
 
-    assert_eq!(js_mock.times_called(), 1);
-    assert_eq!(js_prod_mock.times_called(), 1);
-    assert_eq!(js_dev_mock.times_called(), 1);
-    assert_eq!(js_dev_file_mock.times_called(), 1);
+    assert_eq!(js_mock.hits(), 1);
+    assert_eq!(js_prod_mock.hits(), 1);
+    assert_eq!(js_dev_mock.hits(), 1);
+    assert_eq!(js_dev_file_mock.hits(), 1);
 
     teardown_tmp_directory(tmp_dir);
 
@@ -185,12 +169,10 @@ fn scanner_single_request_scan_with_file_output() -> Result<(), Box<dyn std::err
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/LICENSE")
-        .return_status(200)
-        .return_body("this is a test")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
 
     let outfile = tmp_dir.path().join("output");
 
@@ -211,7 +193,7 @@ fn scanner_single_request_scan_with_file_output() -> Result<(), Box<dyn std::err
     assert!(contents.contains("200"));
     assert!(contents.contains("14"));
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
     teardown_tmp_directory(tmp_dir);
     Ok(())
 }
@@ -223,12 +205,10 @@ fn scanner_single_request_scan_with_file_output_and_tack_q(
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/LICENSE")
-        .return_status(200)
-        .return_body("this is a test")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
 
     let outfile = tmp_dir.path().join("output");
 
@@ -249,7 +229,7 @@ fn scanner_single_request_scan_with_file_output_and_tack_q(
     let url = srv.url("/LICENSE");
     assert!(contents.contains(&url));
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
     teardown_tmp_directory(tmp_dir);
     Ok(())
 }
@@ -261,12 +241,10 @@ fn scanner_single_request_scan_with_invalid_file_output() -> Result<(), Box<dyn
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/LICENSE")
-        .return_status(200)
-        .return_body("this is a test")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
 
     let outfile = tmp_dir.path(); // outfile is a directory
 
@@ -285,7 +263,7 @@ fn scanner_single_request_scan_with_invalid_file_output() -> Result<(), Box<dyn
     let contents = std::fs::read_to_string(outfile);
     assert!(contents.is_err());
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
     teardown_tmp_directory(tmp_dir);
     Ok(())
 }
@@ -296,12 +274,10 @@ fn scanner_single_request_quiet_scan() -> Result<(), Box<dyn std::error::Error>>
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/LICENSE")
-        .return_status(200)
-        .return_body("this is a test")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -321,7 +297,7 @@ fn scanner_single_request_quiet_scan() -> Result<(), Box<dyn std::error::Error>>
             .not(),
     );
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
     teardown_tmp_directory(tmp_dir);
     Ok(())
 }
@@ -334,12 +310,10 @@ fn scanner_single_request_returns_301_without_location_header(
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/LICENSE")
-        .return_body("this is a test")
-        .return_status(301)
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(301).body("this is a test");
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -359,7 +333,52 @@ fn scanner_single_request_returns_301_without_location_header(
             .and(predicate::str::contains("14")),
     );
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// send a single valid request, expect a 200 response that then gets routed to the replay
+/// proxy
+fn scanner_single_request_replayed_to_proxy() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let proxy = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
+
+    let mock_two = proxy.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--replay-proxy")
+        .arg(format!("http://{}", proxy.address().to_string()))
+        .arg("--replay-codes")
+        .arg("200")
+        .unwrap();
+
+    cmd.assert()
+        .success()
+        .stdout(
+            predicate::str::contains("/LICENSE")
+                .and(predicate::str::contains("200"))
+                .and(predicate::str::contains("14c")),
+        )
+        .stderr(predicate::str::contains("Replay Proxy Codes"));
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_two.hits(), 1);
     teardown_tmp_directory(tmp_dir);
     Ok(())
 }
@@ -371,19 +390,15 @@ fn scanner_single_request_scan_with_filtered_result() -> Result<(), Box<dyn std:
     let (tmp_dir, file) =
         setup_tmp_directory(&["LICENSE".to_string(), "ignored".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/LICENSE")
-        .return_status(200)
-        .return_body("this is a not a test")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a not a test");
+    });
 
-    let filtered_mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/ignored")
-        .return_status(200)
-        .return_body("this is a test")
-        .create_on(&srv);
+    let filtered_mock = srv.mock(|when, then| {
+        when.method(GET).path("/ignored");
+        then.status(200).body("this is a test");
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -406,33 +421,106 @@ fn scanner_single_request_scan_with_filtered_result() -> Result<(), Box<dyn std:
             .not(),
     );
 
-    assert_eq!(mock.times_called(), 1);
-    assert_eq!(filtered_mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(filtered_mock.hits(), 1);
     teardown_tmp_directory(tmp_dir);
     Ok(())
 }
 
 #[test]
-/// send a single valid request, expect a 200 response that then gets routed to the replay
-/// proxy
-fn scanner_single_request_replayed_to_proxy() -> Result<(), Box<dyn std::error::Error>> {
+/// send a single valid request, get a response, and write the logging messages to disk
+fn scanner_single_request_scan_with_debug_logging() {
     let srv = MockServer::start();
-    let proxy = MockServer::start();
-    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
 
-    let mock = Mock::new()
-        .expect_method(GET)
-        .expect_path("/LICENSE")
-        .return_status(200)
-        .return_body("this is a test")
-        .create_on(&srv);
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
 
-    let mock_two = Mock::new()
-        .expect_method(GET)
-        .expect_path("/LICENSE")
-        .return_status(200)
-        .return_body("this is a test")
-        .create_on(&proxy);
+    let outfile = tmp_dir.path().join("debug.log");
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-vvvv")
+        .arg("--debug-log")
+        .arg(outfile.as_os_str())
+        .unwrap();
+
+    let contents = std::fs::read_to_string(outfile).unwrap();
+    println!("{}", contents);
+    assert!(contents.starts_with("Configuration {"));
+    assert!(contents.contains("TRC"));
+    assert!(contents.contains("DBG"));
+    assert!(contents.contains("INF"));
+    assert!(contents.contains("feroxbuster All scans complete!"));
+    assert!(contents.contains("feroxbuster exit: terminal_input_handler"));
+
+    assert_eq!(mock.hits(), 1);
+    teardown_tmp_directory(tmp_dir);
+}
+
+#[test]
+/// send a single valid request, get a response, and write the logging messages to disk as NDJSON
+fn scanner_single_request_scan_with_debug_logging_as_json() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
+
+    let outfile = tmp_dir.path().join("debug.log");
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-vvvv")
+        .arg("--debug-log")
+        .arg(outfile.as_os_str())
+        .arg("--json")
+        .unwrap();
+
+    let contents = std::fs::read_to_string(outfile).unwrap();
+    println!("{}", contents);
+    assert!(contents.starts_with("{\"type\":\"configuration\""));
+    assert!(contents.contains("\"level\":\"TRACE\""));
+    assert!(contents.contains("\"level\":\"DEBUG\""));
+    assert!(contents.contains("\"level\":\"INFO\""));
+    assert!(contents.contains("time_offset"));
+    assert!(contents.contains("\"module\":\"feroxbuster::scanner\""));
+    assert!(contents.contains("All scans complete!"));
+    assert!(contents.contains("exit: terminal_input_handler"));
+
+    assert_eq!(mock.hits(), 1);
+    teardown_tmp_directory(tmp_dir);
+}
+
+#[test]
+/// send a single valid request, filter the response by regex, expect one out of 2 urls
+fn scanner_single_request_scan_with_regex_filtered_result() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) =
+        setup_tmp_directory(&["LICENSE".to_string(), "ignored".to_string()], "wordlist").unwrap();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
+
+    let filtered_mock = srv.mock(|when, then| {
+        when.method(GET).path("/ignored");
+        then.status(200)
+            .body("this is a test\nThat rug really tied the room together");
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -440,23 +528,71 @@ fn scanner_single_request_replayed_to_proxy() -> Result<(), Box<dyn std::error::
         .arg(srv.url("/"))
         .arg("--wordlist")
         .arg(file.as_os_str())
-        .arg("--replay-proxy")
-        .arg(format!("http://{}", proxy.address().to_string()))
-        .arg("--replay-codes")
-        .arg("200")
+        .arg("--filter-regex")
+        .arg("'That rug.*together$'")
         .unwrap();
 
-    cmd.assert()
-        .success()
-        .stdout(
-            predicate::str::contains("/LICENSE")
-                .and(predicate::str::contains("200"))
-                .and(predicate::str::contains("14")),
-        )
-        .stderr(predicate::str::contains("Replay Proxy Codes"));
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .and(predicate::str::contains("200"))
+            .and(predicate::str::contains("20"))
+            .and(predicate::str::contains("ignored"))
+            .not()
+            .and(predicate::str::contains(" 14 "))
+            .not(),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(filtered_mock.hits(), 1);
+    teardown_tmp_directory(tmp_dir);
+}
+
+#[test]
+/// send a request to a 403 directory, expect recursion to work into the 403
+fn scanner_recursion_works_with_403_directories() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) =
+        setup_tmp_directory(&["LICENSE".to_string(), "ignored/".to_string()], "wordlist").unwrap();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
+
+    let forbidden_dir = srv.mock(|when, then| {
+        when.method(GET).path("/ignored/");
+        then.status(403);
+    });
+
+    let found_anyway = srv.mock(|when, then| {
+        when.method(GET).path("/ignored/LICENSE");
+        then.status(200)
+            .body("this is a test\nThat rug really tied the room together");
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .count(2)
+            .and(predicate::str::contains("200").count(2))
+            .and(predicate::str::contains("403"))
+            .and(predicate::str::contains("53c"))
+            .and(predicate::str::contains("14c"))
+            .and(predicate::str::contains("0c"))
+            .and(predicate::str::contains("ignored").count(2))
+            .and(predicate::str::contains("/ignored/LICENSE")),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(found_anyway.hits(), 1);
+    assert_eq!(forbidden_dir.hits(), 1);
 
-    assert_eq!(mock.times_called(), 1);
-    assert_eq!(mock_two.times_called(), 1);
     teardown_tmp_directory(tmp_dir);
-    Ok(())
 }