updated status code defaults to include 500

Log to separate files when using --parallel

.cargo/config

@@ -0,0 +1,5 @@
+[target.armv7-unknown-linux-gnueabihf]
+linker = "arm-linux-gnueabihf-gcc"
+
+[target.aarch64-unknown-linux-gnu]
+linker = "aarch64-linux-gnu-gcc"

.github/FUNDING.yml

@@ -0,0 +1,4 @@
+# These are supported funding model platforms
+
+github: [epi052]
+ko_fi: epi052

.github/pull_request_template.md

@@ -7,11 +7,11 @@ Long form explanations of most of the items below can be found in the [CONTRIBUT
 - [ ] Your PR description references the associated issue (i.e. fixes #123456)
 - [ ] Code is in its own branch
 - [ ] Branch name is related to the PR contents
-- [ ] PR targets master
+- [ ] PR targets main
 
 ## Static analysis checks
 - [ ] All rust files are formatted using `cargo fmt`
-- [ ] All `clippy` checks pass when running `cargo clippy --all-targets --all-features -- -D warnings -A clippy::deref_addrof`
+- [ ] All `clippy` checks pass when running `cargo clippy --all-targets --all-features -- -D warnings -A clippy::mutex-atomic`
 - [ ] All existing tests pass
 
 ## Documentation

.github/workflows/build.yml

@@ -4,11 +4,13 @@ on: [push]
 
 jobs:
   build-nix:
+    env:
+      IN_PIPELINE: true
     runs-on: ${{ matrix.os }}
     if: github.ref == 'refs/heads/main'
     strategy:
       matrix:
-        type: [ubuntu-x64, ubuntu-x86]
+        type: [ubuntu-x64, ubuntu-x86, armv7, aarch64]
         include:
           - type: ubuntu-x64
             os: ubuntu-latest
@@ -22,12 +24,25 @@ jobs:
             name: x86-linux-feroxbuster
             path: target/i686-unknown-linux-musl/release/feroxbuster
             pkg_config_path: /usr/lib/i686-linux-gnu/pkgconfig
+          - type: armv7
+            os: ubuntu-latest
+            target: armv7-unknown-linux-gnueabihf
+            name: armv7-feroxbuster
+            path: target/armv7-unknown-linux-gnueabihf/release/feroxbuster
+            pkg_config_path: /usr/lib/x86_64-linux-gnu/pkgconfig
+          - type: aarch64
+            os: ubuntu-latest
+            target: aarch64-unknown-linux-gnu
+            name: aarch64-feroxbuster
+            path: target/aarch64-unknown-linux-gnu/release/feroxbuster
+            pkg_config_path: /usr/lib/x86_64-linux-gnu/pkgconfig
     steps:
       - uses: actions/checkout@v2
       - name: Install System Dependencies
         run: |
+          env
           sudo apt-get update
-          sudo apt-get install -y --no-install-recommends libssl-dev pkg-config
+          sudo apt-get install -y --no-install-recommends libssl-dev pkg-config gcc-arm-linux-gnueabihf gcc-aarch64-linux-gnu
       - uses: actions-rs/toolchain@v1
         with:
           toolchain: stable
@@ -43,7 +58,7 @@ jobs:
           args: --release --target=${{ matrix.target }}
       - name: Strip symbols from binary
         run: |
-          strip -s ${{ matrix.path }}
+          strip -s ${{ matrix.path }} || arm-linux-gnueabihf-strip -s ${{ matrix.path }} || aarch64-linux-gnu-strip -s ${{ matrix.path }}
       - name: Build tar.gz for homebrew installs
         if: matrix.type == 'ubuntu-x64'
         run: |
@@ -72,6 +87,8 @@ jobs:
           path: ./target/x86_64-unknown-linux-musl/debian/*
 
   build-macos:
+    env:
+      IN_PIPELINE: true
     runs-on: macos-latest
     if: github.ref == 'refs/heads/main'
     steps:
@@ -102,6 +119,8 @@ jobs:
           path: x86_64-macos-feroxbuster.tar.gz
 
   build-windows:
+    env:
+      IN_PIPELINE: true
     runs-on: ${{ matrix.os }}
     if: github.ref == 'refs/heads/main'
     strategy:
@@ -134,4 +153,3 @@ jobs:
         with:
           name: ${{ matrix.name }}
           path: ${{ matrix.path }}
-

.github/workflows/check.yml

@@ -61,4 +61,4 @@ jobs:
       - uses: actions-rs/cargo@v1
         with:
           command: clippy
-          args: --all-targets --all-features -- -D warnings -A clippy::deref_addrof
+          args: --all-targets --all-features -- -D warnings -A clippy::deref_addrof -A clippy::mutex-atomic

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "feroxbuster"
-version = "2.0.2"
+version = "2.3.2"
 authors = ["Ben 'epi' Risher <epibar052@gmail.com>"]
 license = "MIT"
 edition = "2018"
@@ -19,13 +19,14 @@ maintenance = { status = "actively-developed" }
 clap = "2.33"
 regex = "1"
 lazy_static = "1.4"
+dirs = "3.0"
 
 [dependencies]
 futures = { version = "0.3"}
-tokio = { version = "1.0", features = ["full"] }
-tokio-util = {version = "0.6.3", features = ["codec"]}
+tokio = { version = "1.9", features = ["full"] }
+tokio-util = {version = "0.6.6", features = ["codec"]}
 log = "0.4"
-env_logger = "0.8"
+env_logger = "0.9"
 reqwest = { version = "0.11", features = ["socks"] }
 clap = "2.33"
 lazy_static = "1.4"
@@ -38,18 +39,18 @@ console = "0.14"
 openssl = { version = "0.10", features = ["vendored"] }
 dirs = "3.0"
 regex = "1"
-crossterm = "0.19"
-rlimit = "0.5"
-ctrlc = "3.1"
+crossterm = "0.20"
+rlimit = "0.6"
+ctrlc = "3.1.9"
 fuzzyhash = "0.2.1"
 anyhow = "1.0"
 leaky-bucket = "0.10.0"
 
 [dev-dependencies]
 tempfile = "3.1"
-httpmock = "0.5.2"
-assert_cmd = "1.0.3"
-predicates = "1.0.7"
+httpmock = "0.5.8"
+assert_cmd = "1.0"
+predicates = "2.0"
 
 [profile.release]
 lto = true
@@ -63,4 +64,7 @@ conf-files = ["/etc/feroxbuster/ferox-config.toml"]
 assets = [
     ["target/release/feroxbuster", "/usr/bin/", "755"],
     ["ferox-config.toml.example", "/etc/feroxbuster/ferox-config.toml", "644"],
+    ["shell_completions/feroxbuster.bash", "/usr/share/bash-completion/completions/feroxbuster.bash", "644"],
+    ["shell_completions/feroxbuster.fish", "/usr/share/fish/completions/feroxbuster.fish", "644"],
+    ["shell_completions/_feroxbuster", "/usr/share/zsh/vendor-completions/_feroxbuster", "644"],
 ]

Makefile

@@ -6,12 +6,16 @@ datarootdir = $(prefix)/share
 datadir = $(datarootdir)
 example_config = ferox-config.toml.example
 config_file = ferox-config.toml
+completion_dir = shell_completions
+completion_prefix = $(completion_dir)/$(BIN)
 
+BIN=feroxbuster
 SHR_SOURCES = $(shell find src -type f -wholename '*src/*.rs') Cargo.toml Cargo.lock
 
 RELEASE = debug
 DEBUG ?= 0
-ifeq (0,$(DEBUG))
+
+ifeq (0, $(DEBUG))
 	ARGS = --release
 	RELEASE = release
 endif
@@ -23,54 +27,52 @@ endif
 
 TARGET = target/$(RELEASE)
 
-.PHONY: all clean distclean install uninstall update
-
-BIN=feroxbuster
-DESKTOP=$(APPID).desktop
+.PHONY: all clean install uninstall test update
 
 all: cli
-
 cli: $(TARGET)/$(BIN) $(TARGET)/$(BIN).1.gz $(SHR_SOURCES)
+install: all install-cli
+
+verify:
+	cargo fmt
+	cargo clippy --all-targets --all-features -- -D warnings -A clippy::mutex-atomic
+	cargo test
 
 clean:
 	cargo clean
 
-distclean: clean
-	rm -rf .cargo vendor Cargo.lock vendor.tar
-
 vendor: vendor.tar
 
 vendor.tar:
-	mkdir -p .cargo
-	cargo vendor | head -n -1 > .cargo/config
-	echo 'directory = "vendor"' >> .cargo/config
+	cargo vendor
 	tar pcf vendor.tar vendor
 	rm -rf vendor
 
 install-cli: cli
-	install -Dm 0755 "$(TARGET)/$(BIN)" "$(DESTDIR)$(bindir)/$(BIN)"
+	install -Dm 0644 "$(completion_prefix).bash" "$(DESTDIR)/usr/share/bash-completion/completions/$(BIN).bash"
+	install -Dm 0644 "$(completion_prefix).fish" "$(DESTDIR)/usr/share/fish/completions/$(BIN).fish"
+	install -Dm 0644 "$(completion_dir)/_$(BIN)" "$(DESTDIR)/usr/share/zsh/vendor-completions/_$(BIN)"
+	install -sDm 0755 "$(TARGET)/$(BIN)" "$(DESTDIR)$(bindir)/$(BIN)"
 	install -Dm 0644 "$(TARGET)/$(BIN).1.gz" "$(DESTDIR)$(datadir)/man/man1/$(BIN).1.gz"
-	install -Dm 0644 "$(example_config)" "/etc/$(BIN)/$(config_File)"
+	install -Dm 0644 "$(example_config)" "$(DESTDIR)/etc/$(BIN)/$(config_file)"
 
-install: all install-cli
-
-uninstall-cli:
+uninstall:
 	rm -f "$(DESTDIR)$(bindir)/$(BIN)"
 	rm -f "$(DESTDIR)$(datadir)/man/man1/$(BIN).1.gz"
-	rm -rf "/etc/$(BIN)/"
-
-uninstall: uninstall-cli
-
-update:
-	cargo update
+	rm -rf "$(DESTDIR)/etc/$(BIN)/"
+	rm -f "$(DESTDIR)/usr/share/bash-completion/completions/$(BIN).bash"
+	rm -f "$(DESTDIR)/usr/share/zsh/vendor-completions/_$(BIN)"
+	rm -f "$(DESTDIR)/usr/share/fish/completions/$(BIN).fish"
 
 extract:
-ifeq ($(VENDORED),1)
+ifeq (1, $(VENDORED))
 	tar pxf vendor.tar
 endif
 
 $(TARGET)/$(BIN): extract
-	cargo build --manifest-path Cargo.toml $(ARGS)
+	mkdir -p .cargo
+	cp debian/cargo.config .cargo/config.toml
+	cargo build $(ARGS)
 
 $(TARGET)/$(BIN).1.gz: $(TARGET)/$(BIN)
 	help2man --no-info $< | gzip -c > $@.partial

README.md

@@ -8,7 +8,7 @@
 
 <p align="center">
   <a href="https://github.com/epi052/feroxbuster/actions?query=workflow%3A%22CI+Pipeline%22">
-    <img src="https://img.shields.io/github/workflow/status/epi052/feroxbuster/CI%20Pipeline/master?logo=github">
+    <img src="https://img.shields.io/github/workflow/status/epi052/feroxbuster/CI%20Pipeline/main?logo=github">
   </a>
 
   <a href="https://github.com/epi052/feroxbuster/releases">
@@ -70,6 +70,7 @@ Enumeration.
     - [Snap Install](#snap-install)
     - [Homebrew on MacOS and Linux](#homebrew-on-macos-and-linux)
     - [Cargo Install](#cargo-install)
+    - [Kali Install](#kali-install)
     - [apt Install](#apt-install)
     - [AUR Install](#aur-install)
     - [Docker Install](#docker-install)
@@ -104,6 +105,9 @@ Enumeration.
     - [Cancel a Recursive Scan Interactively (new in `v1.12.0`)](#cancel-a-recursive-scan-interactively-new-in-v1120)
     - [Limit Number of Requests per Second (Rate Limiting) (new in `v2.0.0`)](#limit-number-of-requests-per-second-rate-limiting-new-in-v200)
     - [Silence all Output or Be Kinda Quiet (new in `v2.0.0`)](#silence-all-output-or-be-kinda-quiet-new-in-v200)
+    - [Auto-tune or Auto-bail from Scans (new in `v2.1.0`)](#auto-tune-or-auto-bail-from-scans-new-in-v210)
+    - [Run Scans in Parallel (new in `v2.2.0`)](#run-scans-in-parallel-new-in-v220)
+    - [Prevent Specific Domain/Directory Scans aka a Deny List (new in `v2.3.0`)](#prevent-specific-domaindirectory-scans-aka-a-deny-list-new-in-v230)
 - [Comparison w/ Similar Tools](#-comparison-w-similar-tools)
 - [Common Problems/Issues (FAQ)](#-common-problemsissues-faq)
     - [No file descriptors available](#no-file-descriptors-available)
@@ -116,8 +120,9 @@ Enumeration.
 
 ### Download a Release
 
-Releases for multiple architectures can be found in the [Releases](https://github.com/epi052/feroxbuster/releases)
-section. The latest release for each of the following systems can be downloaded and executed as shown below.
+Releases for `armv7`, `aarch64`, and an `x86_64 .deb` can be found in the [Releases](https://github.com/epi052/feroxbuster/releases) section.
+
+All other OS/architecture combinations can be installed dynamically using one of the methods shown below.
 
 #### Linux (32 and 64-bit) & MacOS
 
@@ -192,6 +197,16 @@ brew install feroxbuster
 cargo install feroxbuster
 ```
 
+### Kali Install
+
+🥳 `feroxbuster` was recently added to the official Kali Linux repos 🥳 
+
+If you're using kali, this is the preferred install method. Installing from the repos adds a [**ferox-config.toml**](#ferox-config.toml) in `/etc/feroxbuster/`, adds command completion for bash, fish, and zsh, includes a man page entry, and installs `feroxbuster` itself. 
+
+```
+sudo apt update && sudo apt install -y feroxbuster
+```
+
 ### apt Install
 
 Download `feroxbuster_amd64.deb` from the [Releases](https://github.com/epi052/feroxbuster/releases) section. After
@@ -211,6 +226,14 @@ Install `feroxbuster-git` on Arch Linux with your AUR helper of choice:
 yay -S feroxbuster-git
 ```
 
+### BlackArch install
+
+Install `feroxbuster` on BlackArch Linux:
+
+```
+pacman -S feroxbuster
+```
+
 ### Docker Install
 
 > The following steps assume you have docker installed / setup
@@ -281,7 +304,7 @@ Configuration begins with with the following built-in default values baked into
 - verbosity: `0` (no logging enabled)
 - scan_limit: `0` (no limit imposed on concurrent scans)
 - rate_limit: `0` (no limit imposed on requests per second)
-- status_codes: `200 204 301 302 307 308 401 403 405`
+- status_codes: `200 204 301 302 307 308 401 403 405 500`
 - user_agent: `feroxbuster/VERSION`
 - recursion depth: `4`
 - auto-filter wildcards - `true`
@@ -369,7 +392,10 @@ A pre-made configuration file with examples of all available settings can be fou
 # status_codes = [200, 500]
 # filter_status = [301]
 # threads = 1
+# parallel = 2
 # timeout = 5
+# auto_tune = true
+# auto_bail = true
 # proxy = "http://127.0.0.1:8080"
 # replay_proxy = "http://127.0.0.1:8081"
 # replay_codes = [200, 302]
@@ -391,6 +417,7 @@ A pre-made configuration file with examples of all available settings can be fou
 # dont_filter = true
 # extract_links = true
 # depth = 1
+# url_denylist = ["https://dont-scan-me.com/"]
 # filter_size = [5174]
 # filter_regex = ["^ignore me$"]
 # filter_similar = ["https://somesite.com/soft404"]
@@ -424,44 +451,95 @@ USAGE:
     feroxbuster [FLAGS] [OPTIONS] --url <URL>...
 
 FLAGS:
-    -f, --add-slash        Append / to each request
-    -D, --dont-filter      Don't auto-filter wildcard responses
-    -e, --extract-links    Extract links from response body (html, javascript, etc...); make new requests based on
-                           findings (default: false)
-    -h, --help             Prints help information
-    -k, --insecure         Disables TLS certificate validation
-        --json             Emit JSON logs to --output and --debug-log instead of normal text
-    -n, --no-recursion     Do not scan recursively
-    -q, --quiet            Hide progress bars and banner (good for tmux windows w/ notifications)
-    -r, --redirects        Follow redirects
-        --silent           Only print URLs + turn off logging (good for piping a list of urls to other commands)
-        --stdin            Read url(s) from STDIN
-    -V, --version          Prints version information
-    -v, --verbosity        Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v's is probably
-                           too much)
+    -f, --add-slash        
+            Append / to each request
+
+        --auto-bail        
+            Automatically stop scanning when an excessive amount of errors are encountered
+
+        --auto-tune        
+            Automatically lower scan rate when an excessive amount of errors are encountered
+
+    -D, --dont-filter      
+            Don't auto-filter wildcard responses
+
+    -e, --extract-links    
+            Extract links from response body (html, javascript, etc...); make new requests based on findings (default:
+            false)
+    -h, --help             
+            Prints help information
+
+    -k, --insecure         
+            Disables TLS certificate validation
+
+        --json             
+            Emit JSON logs to --output and --debug-log instead of normal text
+
+    -n, --no-recursion     
+            Do not scan recursively
+
+    -q, --quiet            
+            Hide progress bars and banner (good for tmux windows w/ notifications)
+
+    -r, --redirects        
+            Follow redirects
+
+        --silent           
+            Only print URLs + turn off logging (good for piping a list of urls to other commands)
+
+        --stdin            
+            Read url(s) from STDIN
+
+    -V, --version          
+            Prints version information
+
+    -v, --verbosity        
+            Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v's is probably too much)
+
 
 OPTIONS:
-        --debug-log <FILE>                        Output file to write log entries (use w/ --json for JSON entries)
+        --debug-log <FILE>                        
+            Output file to write log entries (use w/ --json for JSON entries)
+
     -d, --depth <RECURSION_DEPTH>
             Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)
 
-    -x, --extensions <FILE_EXTENSION>...          File extension(s) to search for (ex: -x php -x pdf js)
-    -N, --filter-lines <LINES>...                 Filter out messages of a particular line count (ex: -N 20 -N 31,30)
+    -x, --extensions <FILE_EXTENSION>...          
+            File extension(s) to search for (ex: -x php -x pdf js)
+
+    -N, --filter-lines <LINES>...                 
+            Filter out messages of a particular line count (ex: -N 20 -N 31,30)
+
     -X, --filter-regex <REGEX>...
             Filter out messages via regular expression matching on the response's body (ex: -X '^ignore me$')
 
         --filter-similar-to <UNWANTED_PAGE>...
             Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)
 
-    -S, --filter-size <SIZE>...                   Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)
-    -C, --filter-status <STATUS_CODE>...          Filter out status codes (deny list) (ex: -C 200 -C 401)
-    -W, --filter-words <WORDS>...                 Filter out messages of a particular word count (ex: -W 312 -W 91,82)
-    -H, --headers <HEADER>...                     Specify HTTP headers (ex: -H Header:val 'stuff: things')
-    -o, --output <FILE>                           Output file to write results to (use w/ --json for JSON entries)
+    -S, --filter-size <SIZE>...                   
+            Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)
+
+    -C, --filter-status <STATUS_CODE>...          
+            Filter out status codes (deny list) (ex: -C 200 -C 401)
+
+    -W, --filter-words <WORDS>...                 
+            Filter out messages of a particular word count (ex: -W 312 -W 91,82)
+
+    -H, --headers <HEADER>...                     
+            Specify HTTP headers (ex: -H Header:val 'stuff: things')
+
+    -o, --output <FILE>                           
+            Output file to write results to (use w/ --json for JSON entries)
+
+        --parallel <PARALLEL_SCANS>
+            Run parallel feroxbuster instances (one child process per url passed via stdin)
+
     -p, --proxy <PROXY>
             Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)
 
-    -Q, --query <QUERY>...                        Specify URL query parameters (ex: -Q token=stuff -Q secret=key)
+    -Q, --query <QUERY>...                        
+            Specify URL query parameters (ex: -Q token=stuff -Q secret=key)
+
         --rate-limit <RATE_LIMIT>
             Limit number of requests per second (per directory) (default: 0, i.e. no limit)
 
@@ -474,17 +552,32 @@ OPTIONS:
         --resume-from <STATE_FILE>
             State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)
 
-    -L, --scan-limit <SCAN_LIMIT>                 Limit total number of concurrent scans (default: 0, i.e. no limit)
+    -L, --scan-limit <SCAN_LIMIT>                 
+            Limit total number of concurrent scans (default: 0, i.e. no limit)
+
     -s, --status-codes <STATUS_CODE>...
-            Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)
+            Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405 500)
 
-    -t, --threads <THREADS>                       Number of concurrent threads (default: 50)
-        --time-limit <TIME_SPEC>                  Limit total run time of all scans (ex: --time-limit 10m)
-    -T, --timeout <SECONDS>                       Number of seconds before a request times out (default: 7)
-    -u, --url <URL>...                            The target URL(s) (required, unless --stdin used)
-    -a, --user-agent <USER_AGENT>                 Sets the User-Agent (default: feroxbuster/VERSION)
-    -w, --wordlist <FILE>                         Path to the wordlist
+    -t, --threads <THREADS>                       
+            Number of concurrent threads (default: 50)
 
+        --time-limit <TIME_SPEC>                  
+            Limit total run time of all scans (ex: --time-limit 10m)
+
+    -T, --timeout <SECONDS>                       
+            Number of seconds before a request times out (default: 7)
+
+    -u, --url <URL>...                            
+            The target URL(s) (required, unless --stdin used)
+
+        --dont-scan <URL>...                      
+            URL(s) to exclude from recursion/scans
+
+    -a, --user-agent <USER_AGENT>                 
+            Sets the User-Agent (default: feroxbuster/VERSION)
+
+    -w, --wordlist <FILE>                         
+            Path to the wordlist
 ```
 
 ## 📊 Scan's Display Explained
@@ -807,10 +900,11 @@ Below is an example of the Scan Cancel Menu™.
 Using the menu is pretty simple:
 - Press `ENTER` to view the menu
 - Choose a scan to cancel by entering its scan index (`1`)
-  - more than one scan can be selected by using a comma-separated list (`1,2,3` ... etc)
+  - more than one scan can be selected by using a comma-separated list of indexes and/or ranges (`1-4,8,9-13` ... etc)
 - Confirm selections, after which all non-cancelled scans will resume
+  - To skip confirmation, simply add a `-f` somewhere in your input (`3-5 -f`)
 
-Here is a short demonstration of cancelling two in-progress scans found via recursion.
+Here is a short demonstration of force cancelling a range of scans followed by a single scan with interactive prompt.
 
 ![cancel-scan](img/cancel-scan.gif)
 
@@ -871,6 +965,95 @@ Scanning: https://localhost.com/homepage
 Scanning: https://localhost.com/api
 ```
 
+### Auto-tune or Auto-bail from scans (new in `v2.1.0`)
+
+Version 2.1.0 introduces the `--auto-tune` and `--auto-bail` flags. You can think of these flags as Policies. Both actions (tuning and bailing) are triggered by the same criteria (below).  Policies are only enforced after at least 50 requests have been made (or # of threads, if that's > 50).
+
+Policy Enforcement Criteria:
+  - number of general errors (timeouts, etc) is higher than half the number of threads (or at least 25 if threads are lower) (per directory scanned)
+  - 90% of responses are `403|Forbidden` (per directory scanned)
+  - 30% of requests are `429|Too Many Requests` (per directory scanned)
+
+> both demo gifs below use --timeout to overload a single-threaded python web server and elicit timeouts
+
+#### --auto-tune
+
+The AutoTune policy enforces a rate limit on individual directory scans when one of the criteria above is met.  The rate limit self-adjusts every (`timeout / 2`) seconds. If the number of errors have increased during that time, the allowed rate of requests is lowered.  On the other hand, if the number of errors hasn't moved, the allowed rate of requests is increased.  If no additional errors are found after a certain number of checks, the rate limit will be removed completely. 
+
+![auto-tune](img/auto-tune-demo.gif)
+
+#### --auto-bail
+
+The AutoBail policy aborts individual directory scans when one of the criteria above is met.  They just stop getting scanned, no muss, no fuss. 
+
+![auto-bail](img/auto-bail-demo.gif)
+
+### Run Scans in Parallel (new in `v2.2.0`)
+
+Version 2.2.0 introduces the `--parallel` option.  If you're one of those people who use `feroxbuster` to scan 100s of hosts at a time, this is the option for you! `--parallel` spawns a child process per target passed in over stdin (recursive directories are still async within each child).
+
+The number of parallel scans is limited to whatever you pass to `--parallel`. When one child finishes its scan, the next child will be spawned.  
+
+Unfortunately, using `--parallel` limits terminal output such that only discovered URLs are shown. No amount of `-v`'s will help you here. I imagine this isn't too big of a deal, as folks that need `--parallel` probably aren't sitting there watching the output... 🙃
+
+Example Command:
+```
+cat large-target-list | ./feroxbuster --stdin --parallel 10 --extract-links --auto-bail
+```
+
+Resulting Process List (illustrative):
+```
+feroxbuster --stdin --parallel 10
+ \_ feroxbuster --silent --extract-links --auto-bail -u https://target-one
+ \_ feroxbuster --silent --extract-links --auto-bail -u https://target-two
+ \_ feroxbuster --silent --extract-links --auto-bail -u https://target-three
+ \_ ...
+ \_ feroxbuster --silent --extract-links --auto-bail -u https://target-ten
+```
+
+As of `v2.3.2`, logging while using `--parallel` uses the value of `-o`|`--output` as a seed to create a directory named `OUTPUT_VALUE-TIMESTAMP.logs/`. Within the directory, an individual log file is created for each target passed over stdin.
+
+Example Command:
+```
+cat large-target-list | ./feroxbuster --stdin --parallel 10 --output super-cool-mega-scan
+```
+
+Resulting directory structure (illustrative):
+```
+super-cool-mega-scan-1627865696.logs/
+├── ferox-https_target_one_com-1627865696.log
+├── ...
+└── ferox-https_target_two_net-1627865696.log
+```
+
+### Prevent Specific Domain/Directory Scans aka a Deny List (new in `v2.3.0`)
+
+> This action is taken BEFORE a request is sent to the target, which differs from the filter-* options that are applied to responses
+
+Version 2.3.0 introduces the `--dont-scan` option. The values passed to `--dont-scan` act as a deny-list.  The values 
+can be an entire domain (`http://some.domain`), a specific folder (`http://some.domain/js`), or a specific file 
+(`http://some.domain/some-application/stupid-page.php`)  If a folder/domain is used any sub-folder/sub-file of the 
+url passed to `--dont-scan` will be blocked before it can be requested.
+
+For example, given the command 
+
+```
+./feroxbuster -u http://some.domain --dont-scan http://some.domain/js
+```
+
+`http://some.domain` will be scanned recursively, but any url path that begins with `/js/` will not be requested at all.
+
+A caveat to the sub-folder/sub-file rule is when the value passed to `--dont-scan` is a parent of the scan you want to 
+perform. When denying at a hierarchical level higher than your scan, only sub-files/sub-folders of your `-u|--stdin` 
+value(s) will be processed.
+
+```
+./feroxbuster -u http://some.domain/some-application --dont-scan http://some.domain/
+```
+
+In the command above, only `http://some.domain/some-application` and children of that directory found via recursion will
+be scanned. Anything 'outside' of `/some-application` will not be scanned.
+
 ## 🧐 Comparison w/ Similar Tools
 
 There are quite a few similar tools for forced browsing/content discovery. Burp Suite Pro, Dirb, Dirbuster, etc...
@@ -894,7 +1077,7 @@ few of the use-cases in which feroxbuster may be a better fit:
 | fast                                                                         | ✔ | ✔ | ✔ |
 | allows recursion                                                             | ✔ |   | ✔ |
 | can specify query parameters                                                 | ✔ |   | ✔ |
-| SOCKS proxy support                                                          | ✔ |   |   |
+| SOCKS proxy support                                                          | ✔ |   | ✔ |
 | multiple target scan (via stdin or multiple -u)                              | ✔ |   | ✔ |
 | configuration file for default value override                                | ✔ |   | ✔ |
 | can accept urls via STDIN as part of a pipeline                              | ✔ |   | ✔ |
@@ -918,6 +1101,10 @@ few of the use-cases in which feroxbuster may be a better fit:
 | cancel a recursive scan interactively (`v1.12.0`)                            | ✔ |   |   |
 | limit number of requests per second (`v2.0.0`)                               | ✔ | ✔ | ✔ |
 | hide progress bars or be silent (or some variation) (`v2.0.0`)               | ✔ | ✔ | ✔ |
+| automatically tune scans based on errors/403s/429s  (`v2.1.0`)               | ✔ |   |   |
+| automatically stop scans based on errors/403s/429s  (`v2.1.0`)               | ✔ |   | ✔ |
+| run scans in parallel (1 process per target) (`v2.2.0`)                      | ✔ |   |   |
+| prevent requests to given domain/folder/file (`v2.3.0`)                      | ✔ |   |   |
 | **huge** number of other options                                             |   |   | ✔ |
 
 Of note, there's another written-in-rust content discovery tool, [rustbuster](https://github.com/phra/rustbuster). I

build.rs

@@ -1,4 +1,7 @@
+use std::fs::{copy, create_dir_all, OpenOptions};
+use std::io::{Read, Seek, SeekFrom, Write};
 extern crate clap;
+extern crate dirs;
 
 use clap::Shell;
 
@@ -20,4 +23,64 @@ fn main() {
     for shell in &shells {
         app.gen_completions("feroxbuster", *shell, outdir);
     }
+
+    // 0xdf pointed out an oddity when tab-completing options that expect file paths, the fix we
+    // landed on was to add -o plusdirs to the bash completion script. The following code aims to
+    // automate that fix and have it present in all future builds
+    let mut contents = String::new();
+
+    let mut bash_file = OpenOptions::new()
+        .read(true)
+        .write(true)
+        .open(format!("{}/feroxbuster.bash", outdir))
+        .expect("Couldn't open bash completion script");
+
+    bash_file
+        .read_to_string(&mut contents)
+        .expect("Couldn't read bash completion script");
+
+    contents = contents.replace("default feroxbuster", "default -o plusdirs feroxbuster");
+
+    bash_file
+        .seek(SeekFrom::Start(0))
+        .expect("Couldn't seek to position 0 in bash completion script");
+
+    bash_file
+        .write_all(contents.as_bytes())
+        .expect("Couldn't write updated bash completion script to disk");
+
+    // hunter0x8 let me know that when installing via cargo, it would be nice if we dropped a
+    // config file during the build process. The following code will place an example config in
+    // the user's configuration directory
+    //   - linux: $XDG_CONFIG_HOME or $HOME/.config
+    //   - macOS: $HOME/Library/Application Support
+    //   - windows: {FOLDERID_RoamingAppData}
+    let mut config_dir = dirs::config_dir().expect("Couldn't resolve user's config directory");
+    config_dir = config_dir.join("feroxbuster"); // $HOME/.config/feroxbuster
+
+    if !config_dir.exists() {
+        // recursively create the feroxbuster directory and all of its parent components if
+        // they are missing
+        if !config_dir.exists() {
+            // recursively create the feroxbuster directory and all of its parent components if
+            // they are missing
+            if create_dir_all(&config_dir).is_err() {
+                // only copy the config file when we're not running in the CI/CD pipeline
+                // which fails with permission denied
+                eprintln!("Couldn't create one or more directories needed to copy the config file");
+                return;
+            }
+        }
+    }
+
+    // hard-coding config name here to not rely on the crate we're building, if DEFAULT_CONFIG_NAME
+    // ever changes, this will need to be updated
+    let config_file = config_dir.join("ferox-config.toml");
+
+    if !config_file.exists() {
+        // config file doesn't exist, add it to the config directory
+        if copy("ferox-config.toml.example", config_file).is_err() {
+            eprintln!("Couldn't copy example config into config directory");
+        }
+    }
 }

ferox-config.toml.example

@@ -16,10 +16,13 @@
 # replay_proxy = "http://127.0.0.1:8081"
 # replay_codes = [200, 302]
 # verbosity = 1
+# parallel = 8
 # scan_limit = 6
 # rate_limit = 250
 # quiet = true
 # silent = true
+# auto_tune = true
+# auto_bail = true
 # json = true
 # output = "/targets/ellingson_mineral_company/gibson.txt"
 # debug_log = "/var/log/find-the-derp.log"
@@ -27,6 +30,7 @@
 # redirects = true
 # insecure = true
 # extensions = ["php", "html"]
+# url_denylist = ["http://dont-scan.me", "https://also-not.me"]
 # no_recursion = true
 # add_slash = true
 # stdin = true

install-nix.sh

@@ -3,59 +3,63 @@
 BASE_URL=https://github.com/epi052/feroxbuster/releases/latest/download
 
 MAC_ZIP=x86_64-macos-feroxbuster.zip
-MAC_URL="${BASE_URL}/${MAC_ZIP}"
+MAC_URL="$BASE_URL/$MAC_ZIP"
 
 LIN32_ZIP=x86-linux-feroxbuster.zip
-LIN32_URL="${BASE_URL}/${LIN32_ZIP}"
+LIN32_URL="$BASE_URL/$LIN32_ZIP"
 
 LIN64_ZIP=x86_64-linux-feroxbuster.zip
-LIN64_URL="${BASE_URL}/${LIN64_ZIP}"
+LIN64_URL="$BASE_URL/$LIN64_ZIP"
 
 EMOJI_URL=https://gist.github.com/epi052/8196b550ea51d0907ad4b93751b1b57d/raw/6112c9f32ae07922983fdc549c54fd3fb9a38e4c/NotoColorEmoji.ttf
 
 echo "[+] Installing feroxbuster!"
 
-if [[ "$(uname)" == "Darwin" ]]; then
-    echo "[=] Found MacOS, downloading from ${MAC_URL}"
-
-    curl -sLO "${MAC_URL}"
-    unzip -o "${MAC_ZIP}" > /dev/null
-    rm "${MAC_ZIP}"
-elif [[ "$(expr substr $(uname -s) 1 5)" == "Linux" ]]; then
-    if [[ $(getconf LONG_BIT) == 32 ]]; then
-        echo "[=] Found 32-bit Linux, downloading from ${LIN32_URL}"
-
-        curl -sLO "${LIN32_URL}"
-        unzip -o "${LIN32_ZIP}" > /dev/null
-        rm "${LIN32_ZIP}"
-    else
-        echo "[=] Found 64-bit Linux, downloading from ${LIN64_URL}"
-
-        curl -sLO "${LIN64_URL}"
-        unzip -o "${LIN64_ZIP}" > /dev/null
-        rm "${LIN64_ZIP}"
-    fi
-
-    if [[ -e ~/.fonts/NotoColorEmoji.ttf ]]; then
-       echo "[=] Found Noto Emoji Font, skipping install" 
-    else 
-        echo "[=] Installing Noto Emoji Font"
-        mkdir -p ~/.fonts
-        pushd ~/.fonts 2>&1 >/dev/null
-
-        curl -sLO "${EMOJI_URL}"
-
-        fc-cache -f -v >/dev/null
-
-        popd 2>&1 >/dev/null
-        echo "[+] Noto Emoji Font installed"
-    fi
+which unzip &>/dev/null
+if [ "$?" = "0" ]; then
+  echo "[+] unzip found"
+else
+  echo "[ ] unzip not found, exiting. "
+  exit -1
 fi
 
+if [[ "$(uname)" == "Darwin" ]]; then
+  echo "[=] Found MacOS, downloading from $MAC_URL"
+
+  curl -sLO "$MAC_URL"
+  unzip -o "$MAC_ZIP" >/dev/null
+  rm "$MAC_ZIP"
+elif [[ "$(expr substr $(uname -s) 1 5)" == "Linux" ]]; then
+  if [[ $(getconf LONG_BIT) == 32 ]]; then
+    echo "[=] Found 32-bit Linux, downloading from $LIN32_URL"
+
+    curl -sLO "$LIN32_URL"
+    unzip -o "$LIN32_ZIP" >/dev/null
+    rm "$LIN32_ZIP"
+  else
+    echo "[=] Found 64-bit Linux, downloading from $LIN64_URL"
+
+    curl -sLO "$LIN64_URL"
+    unzip -o "$LIN64_ZIP" >/dev/null
+    rm "$LIN64_ZIP"
+  fi
+
+  if [[ -e ~/.fonts/NotoColorEmoji.ttf ]]; then
+    echo "[=] Found Noto Emoji Font, skipping install"
+  else
+    echo "[=] Installing Noto Emoji Font"
+    mkdir -p ~/.fonts
+    pushd ~/.fonts 2>&1 >/dev/null
+
+    curl -sLO "$EMOJI_URL"
+
+    fc-cache -f -v >/dev/null
+
+    popd 2>&1 >/dev/null
+    echo "[+] Noto Emoji Font installed"
+  fi
+fi
 
 chmod +x ./feroxbuster
 
 echo "[+] Installed feroxbuster version $(./feroxbuster -V)"
-
-
-

shell_completions/_feroxbuster

@@ -41,6 +41,7 @@ _feroxbuster() {
 '--user-agent=[Sets the User-Agent (default: feroxbuster/VERSION)]' \
 '*-x+[File extension(s) to search for (ex: -x php -x pdf js)]' \
 '*--extensions=[File extension(s) to search for (ex: -x php -x pdf js)]' \
+'*--dont-scan=[URL(s) to exclude from recursion/scans]' \
 '*-H+[Specify HTTP headers (ex: -H Header:val '\''stuff: things'\'')]' \
 '*--headers=[Specify HTTP headers (ex: -H Header:val '\''stuff: things'\'')]' \
 '*-Q+[Specify URL query parameters (ex: -Q token=stuff -Q secret=key)]' \
@@ -58,13 +59,16 @@ _feroxbuster() {
 '*--filter-similar-to=[Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)]' \
 '-L+[Limit total number of concurrent scans (default: 0, i.e. no limit)]' \
 '--scan-limit=[Limit total number of concurrent scans (default: 0, i.e. no limit)]' \
-'--rate-limit=[Limit number of requests per second (per directory) (default: 0, i.e. no limit)]' \
+'--parallel=[Run parallel feroxbuster instances (one child process per url passed via stdin)]' \
+'(--auto-tune)--rate-limit=[Limit number of requests per second (per directory) (default: 0, i.e. no limit)]' \
 '--time-limit=[Limit total run time of all scans (ex: --time-limit 10m)]' \
 '(--silent)*-v[Increase verbosity level (use -vv or more for greater effect. \[CAUTION\] 4 -v'\''s is probably too much)]' \
 '(--silent)*--verbosity[Increase verbosity level (use -vv or more for greater effect. \[CAUTION\] 4 -v'\''s is probably too much)]' \
 '(-q --quiet)--silent[Only print URLs + turn off logging (good for piping a list of urls to other commands)]' \
 '-q[Hide progress bars and banner (good for tmux windows w/ notifications)]' \
 '--quiet[Hide progress bars and banner (good for tmux windows w/ notifications)]' \
+'(--auto-bail)--auto-tune[Automatically lower scan rate when an excessive amount of errors are encountered]' \
+'--auto-bail[Automatically stop scanning when an excessive amount of errors are encountered]' \
 '--json[Emit JSON logs to --output and --debug-log instead of normal text]' \
 '-D[Don'\''t auto-filter wildcard responses]' \
 '--dont-filter[Don'\''t auto-filter wildcard responses]' \

shell_completions/_feroxbuster.ps1

@@ -46,6 +46,7 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
             [CompletionResult]::new('--user-agent', 'user-agent', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/VERSION)')
             [CompletionResult]::new('-x', 'x', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js)')
             [CompletionResult]::new('--extensions', 'extensions', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js)')
+            [CompletionResult]::new('--dont-scan', 'dont-scan', [CompletionResultType]::ParameterName, 'URL(s) to exclude from recursion/scans')
             [CompletionResult]::new('-H', 'H', [CompletionResultType]::ParameterName, 'Specify HTTP headers (ex: -H Header:val ''stuff: things'')')
             [CompletionResult]::new('--headers', 'headers', [CompletionResultType]::ParameterName, 'Specify HTTP headers (ex: -H Header:val ''stuff: things'')')
             [CompletionResult]::new('-Q', 'Q', [CompletionResultType]::ParameterName, 'Specify URL query parameters (ex: -Q token=stuff -Q secret=key)')
@@ -63,6 +64,7 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
             [CompletionResult]::new('--filter-similar-to', 'filter-similar-to', [CompletionResultType]::ParameterName, 'Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)')
             [CompletionResult]::new('-L', 'L', [CompletionResultType]::ParameterName, 'Limit total number of concurrent scans (default: 0, i.e. no limit)')
             [CompletionResult]::new('--scan-limit', 'scan-limit', [CompletionResultType]::ParameterName, 'Limit total number of concurrent scans (default: 0, i.e. no limit)')
+            [CompletionResult]::new('--parallel', 'parallel', [CompletionResultType]::ParameterName, 'Run parallel feroxbuster instances (one child process per url passed via stdin)')
             [CompletionResult]::new('--rate-limit', 'rate-limit', [CompletionResultType]::ParameterName, 'Limit number of requests per second (per directory) (default: 0, i.e. no limit)')
             [CompletionResult]::new('--time-limit', 'time-limit', [CompletionResultType]::ParameterName, 'Limit total run time of all scans (ex: --time-limit 10m)')
             [CompletionResult]::new('-v', 'v', [CompletionResultType]::ParameterName, 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v''s is probably too much)')
@@ -70,6 +72,8 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
             [CompletionResult]::new('--silent', 'silent', [CompletionResultType]::ParameterName, 'Only print URLs + turn off logging (good for piping a list of urls to other commands)')
             [CompletionResult]::new('-q', 'q', [CompletionResultType]::ParameterName, 'Hide progress bars and banner (good for tmux windows w/ notifications)')
             [CompletionResult]::new('--quiet', 'quiet', [CompletionResultType]::ParameterName, 'Hide progress bars and banner (good for tmux windows w/ notifications)')
+            [CompletionResult]::new('--auto-tune', 'auto-tune', [CompletionResultType]::ParameterName, 'Automatically lower scan rate when an excessive amount of errors are encountered')
+            [CompletionResult]::new('--auto-bail', 'auto-bail', [CompletionResultType]::ParameterName, 'Automatically stop scanning when an excessive amount of errors are encountered')
             [CompletionResult]::new('--json', 'json', [CompletionResultType]::ParameterName, 'Emit JSON logs to --output and --debug-log instead of normal text')
             [CompletionResult]::new('-D', 'D', [CompletionResultType]::ParameterName, 'Don''t auto-filter wildcard responses')
             [CompletionResult]::new('--dont-filter', 'dont-filter', [CompletionResultType]::ParameterName, 'Don''t auto-filter wildcard responses')

shell_completions/feroxbuster.bash

@@ -20,7 +20,7 @@ _feroxbuster() {
 
     case "${cmd}" in
         feroxbuster)
-            opts=" -v -q -D -r -k -n -f -e -h -V -w -u -t -d -T -p -P -R -s -o -a -x -H -Q -S -X -W -N -C -L  --verbosity --silent --quiet --json --dont-filter --redirects --insecure --no-recursion --add-slash --stdin --extract-links --help --version --wordlist --url --threads --depth --timeout --proxy --replay-proxy --replay-codes --status-codes --output --resume-from --debug-log --user-agent --extensions --headers --query --filter-size --filter-regex --filter-words --filter-lines --filter-status --filter-similar-to --scan-limit --rate-limit --time-limit  "
+            opts=" -v -q -D -r -k -n -f -e -h -V -w -u -t -d -T -p -P -R -s -o -a -x -H -Q -S -X -W -N -C -L  --verbosity --silent --quiet --auto-tune --auto-bail --json --dont-filter --redirects --insecure --no-recursion --add-slash --stdin --extract-links --help --version --wordlist --url --threads --depth --timeout --proxy --replay-proxy --replay-codes --status-codes --output --resume-from --debug-log --user-agent --extensions --dont-scan --headers --query --filter-size --filter-regex --filter-words --filter-lines --filter-status --filter-similar-to --scan-limit --parallel --rate-limit --time-limit  "
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 1 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -131,6 +131,10 @@ _feroxbuster() {
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
                     ;;
+                --dont-scan)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
                 --headers)
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
@@ -199,6 +203,10 @@ _feroxbuster() {
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
                     ;;
+                --parallel)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
                 --rate-limit)
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
@@ -218,4 +226,4 @@ _feroxbuster() {
     esac
 }
 
-complete -F _feroxbuster -o bashdefault -o default feroxbuster
+complete -F _feroxbuster -o bashdefault -o default -o plusdirs feroxbuster

shell_completions/feroxbuster.fish

@@ -12,6 +12,7 @@ complete -c feroxbuster -n "__fish_use_subcommand" -l resume-from -d 'State file
 complete -c feroxbuster -n "__fish_use_subcommand" -l debug-log -d 'Output file to write log entries (use w/ --json for JSON entries)'
 complete -c feroxbuster -n "__fish_use_subcommand" -s a -l user-agent -d 'Sets the User-Agent (default: feroxbuster/VERSION)'
 complete -c feroxbuster -n "__fish_use_subcommand" -s x -l extensions -d 'File extension(s) to search for (ex: -x php -x pdf js)'
+complete -c feroxbuster -n "__fish_use_subcommand" -l dont-scan -d 'URL(s) to exclude from recursion/scans'
 complete -c feroxbuster -n "__fish_use_subcommand" -s H -l headers -d 'Specify HTTP headers (ex: -H Header:val \'stuff: things\')'
 complete -c feroxbuster -n "__fish_use_subcommand" -s Q -l query -d 'Specify URL query parameters (ex: -Q token=stuff -Q secret=key)'
 complete -c feroxbuster -n "__fish_use_subcommand" -s S -l filter-size -d 'Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)'
@@ -21,11 +22,14 @@ complete -c feroxbuster -n "__fish_use_subcommand" -s N -l filter-lines -d 'Filt
 complete -c feroxbuster -n "__fish_use_subcommand" -s C -l filter-status -d 'Filter out status codes (deny list) (ex: -C 200 -C 401)'
 complete -c feroxbuster -n "__fish_use_subcommand" -l filter-similar-to -d 'Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)'
 complete -c feroxbuster -n "__fish_use_subcommand" -s L -l scan-limit -d 'Limit total number of concurrent scans (default: 0, i.e. no limit)'
+complete -c feroxbuster -n "__fish_use_subcommand" -l parallel -d 'Run parallel feroxbuster instances (one child process per url passed via stdin)'
 complete -c feroxbuster -n "__fish_use_subcommand" -l rate-limit -d 'Limit number of requests per second (per directory) (default: 0, i.e. no limit)'
 complete -c feroxbuster -n "__fish_use_subcommand" -l time-limit -d 'Limit total run time of all scans (ex: --time-limit 10m)'
 complete -c feroxbuster -n "__fish_use_subcommand" -s v -l verbosity -d 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v\'s is probably too much)'
 complete -c feroxbuster -n "__fish_use_subcommand" -l silent -d 'Only print URLs + turn off logging (good for piping a list of urls to other commands)'
 complete -c feroxbuster -n "__fish_use_subcommand" -s q -l quiet -d 'Hide progress bars and banner (good for tmux windows w/ notifications)'
+complete -c feroxbuster -n "__fish_use_subcommand" -l auto-tune -d 'Automatically lower scan rate when an excessive amount of errors are encountered'
+complete -c feroxbuster -n "__fish_use_subcommand" -l auto-bail -d 'Automatically stop scanning when an excessive amount of errors are encountered'
 complete -c feroxbuster -n "__fish_use_subcommand" -l json -d 'Emit JSON logs to --output and --debug-log instead of normal text'
 complete -c feroxbuster -n "__fish_use_subcommand" -s D -l dont-filter -d 'Don\'t auto-filter wildcard responses'
 complete -c feroxbuster -n "__fish_use_subcommand" -s r -l redirects -d 'Follow redirects'

src/banner/container.rs

@@ -1,8 +1,8 @@
 use super::entry::BannerEntry;
-use crate::event_handlers::Handles;
 use crate::{
     config::Configuration,
-    utils::{make_request, status_colorizer},
+    event_handlers::Handles,
+    utils::{logged_request, status_colorizer},
     VERSION,
 };
 use anyhow::{bail, Result};
@@ -125,6 +125,18 @@ pub struct Banner {
     /// represents Configuration.rate_limit
     rate_limit: BannerEntry,
 
+    /// represents Configuration.parallel
+    parallel: BannerEntry,
+
+    /// represents Configuration.auto_tune
+    auto_tune: BannerEntry,
+
+    /// represents Configuration.auto_bail
+    auto_bail: BannerEntry,
+
+    /// represents Configuration.url_denylist
+    url_denylist: Vec<BannerEntry>,
+
     /// current version of feroxbuster
     pub(super) version: String,
 
@@ -137,6 +149,7 @@ impl Banner {
     /// Create a new Banner from a Configuration and live targets
     pub fn new(tgts: &[String], config: &Configuration) -> Self {
         let mut targets = Vec::new();
+        let mut url_denylist = Vec::new();
         let mut code_filters = Vec::new();
         let mut replay_codes = Vec::new();
         let mut headers = Vec::new();
@@ -151,6 +164,10 @@ impl Banner {
             targets.push(BannerEntry::new("🎯", "Target Url", target));
         }
 
+        for denied_url in &config.url_denylist {
+            url_denylist.push(BannerEntry::new("🚫", "Don't Scan", denied_url));
+        }
+
         let mut codes = vec![];
         for code in &config.status_codes {
             codes.push(status_colorizer(&code.to_string()))
@@ -162,7 +179,7 @@ impl Banner {
             code_filters.push(status_colorizer(&code.to_string()))
         }
         let filter_status = BannerEntry::new(
-            "🗑",
+            "💢",
             "Status Code Filters",
             &format!("[{}]", code_filters.join(", ")),
         );
@@ -251,6 +268,8 @@ impl Banner {
         );
 
         let replay_proxy = BannerEntry::new("🎥", "Replay Proxy", &config.replay_proxy);
+        let auto_tune = BannerEntry::new("🎶", "Auto Tune", &config.auto_tune.to_string());
+        let auto_bail = BannerEntry::new("🪣", "Auto Bail", &config.auto_bail.to_string());
         let cfg = BannerEntry::new("💉", "Config File", &config.config);
         let proxy = BannerEntry::new("💎", "Proxy", &config.proxy);
         let threads = BannerEntry::new("🚀", "Threads", &config.threads.to_string());
@@ -273,6 +292,7 @@ impl Banner {
             BannerEntry::new("🤪", "Filter Wildcards", &(!config.dont_filter).to_string());
         let add_slash = BannerEntry::new("🪓", "Add Slash", &config.add_slash.to_string());
         let time_limit = BannerEntry::new("🕖", "Time Limit", &config.time_limit);
+        let parallel = BannerEntry::new("🛤", "Parallel Scans", &config.parallel.to_string());
         let rate_limit =
             BannerEntry::new("🚧", "Requests per Second", &config.rate_limit.to_string());
 
@@ -284,6 +304,8 @@ impl Banner {
             filter_status,
             timeout,
             user_agent,
+            auto_bail,
+            auto_tune,
             proxy,
             replay_codes,
             replay_proxy,
@@ -294,6 +316,7 @@ impl Banner {
             filter_line_count,
             filter_regex,
             extract_links,
+            parallel,
             json,
             queries,
             output,
@@ -308,6 +331,7 @@ impl Banner {
             rate_limit,
             scan_limit,
             time_limit,
+            url_denylist,
             config: cfg,
             version: VERSION.to_string(),
             update_status: UpdateStatus::Unknown,
@@ -354,15 +378,8 @@ by Ben "epi" Risher {}                 ver: {}"#,
 
         let api_url = Url::parse(url)?;
 
-        let response = make_request(
-            &handles.config.client,
-            &api_url,
-            handles.config.output_level,
-            handles.stats.tx.clone(),
-        )
-        .await?;
-
-        let body = response.text().await?;
+        let result = logged_request(&api_url, handles.clone()).await?;
+        let body = result.text().await?;
 
         let json_response: Value = serde_json::from_str(&body)?;
 
@@ -405,6 +422,10 @@ by Ben "epi" Risher {}                 ver: {}"#,
             writeln!(&mut writer, "{}", target)?;
         }
 
+        for denied_url in &self.url_denylist {
+            writeln!(&mut writer, "{}", denied_url)?;
+        }
+
         writeln!(&mut writer, "{}", self.threads)?;
         writeln!(&mut writer, "{}", self.wordlist)?;
         writeln!(&mut writer, "{}", self.status_codes)?;
@@ -486,6 +507,13 @@ by Ben "epi" Risher {}                 ver: {}"#,
             writeln!(&mut writer, "{}", self.insecure)?;
         }
 
+        if config.auto_bail {
+            writeln!(&mut writer, "{}", self.auto_bail)?;
+        }
+        if config.auto_tune {
+            writeln!(&mut writer, "{}", self.auto_tune)?;
+        }
+
         if config.redirects {
             writeln!(&mut writer, "{}", self.redirects)?;
         }
@@ -508,6 +536,10 @@ by Ben "epi" Risher {}                 ver: {}"#,
             writeln!(&mut writer, "{}", self.scan_limit)?;
         }
 
+        if config.parallel > 0 {
+            writeln!(&mut writer, "{}", self.parallel)?;
+        }
+
         if config.rate_limit > 0 {
             writeln!(&mut writer, "{}", self.rate_limit)?;
         }

src/banner/tests.rs

@@ -1,6 +1,6 @@
 use super::container::UpdateStatus;
 use super::*;
-use crate::{config::Configuration, event_handlers::Handles};
+use crate::{config::Configuration, event_handlers::Handles, scan_manager::FeroxScans};
 use httpmock::Method::GET;
 use httpmock::MockServer;
 use std::{io::stderr, sync::Arc, time::Duration};
@@ -73,8 +73,9 @@ async fn banner_needs_update_returns_up_to_date() {
         when.method(GET).path("/latest");
         then.status(200).body("{\"tag_name\":\"v1.1.0\"}");
     });
+    let scans = Arc::new(FeroxScans::default());
 
-    let handles = Arc::new(Handles::for_testing(None, None).0);
+    let handles = Arc::new(Handles::for_testing(Some(scans), None).0);
 
     let mut banner = Banner::new(&[srv.url("")], &Configuration::new().unwrap());
     banner.version = String::from("1.1.0");
@@ -95,7 +96,9 @@ async fn banner_needs_update_returns_out_of_date() {
         then.status(200).body("{\"tag_name\":\"v1.1.0\"}");
     });
 
-    let handles = Arc::new(Handles::for_testing(None, None).0);
+    let scans = Arc::new(FeroxScans::default());
+
+    let handles = Arc::new(Handles::for_testing(Some(scans), None).0);
 
     let mut banner = Banner::new(&[srv.url("")], &Configuration::new().unwrap());
     banner.version = String::from("1.0.1");

src/config/container.rs

@@ -1,8 +1,9 @@
 use super::utils::{
     depth, report_and_exit, save_state, serialized_type, status_codes, threads, timeout,
-    user_agent, wordlist, OutputLevel,
+    user_agent, wordlist, OutputLevel, RequesterPolicy,
 };
 use crate::config::determine_output_level;
+use crate::config::utils::determine_requester_policy;
 use crate::{
     client, parser, scan_manager::resume_scan, traits::FeroxSerialize, utils::fmt_err,
     DEFAULT_CONFIG_NAME,
@@ -124,6 +125,18 @@ pub struct Configuration {
     #[serde(skip)]
     pub output_level: OutputLevel,
 
+    /// automatically bail at certain error thresholds
+    #[serde(default)]
+    pub auto_bail: bool,
+
+    /// automatically try to lower request rate in order to reduce errors
+    #[serde(default)]
+    pub auto_tune: bool,
+
+    /// more easily differentiate between the three requester policies
+    #[serde(skip)]
+    pub requester_policy: RequesterPolicy,
+
     /// Store log output as NDJSON
     #[serde(default)]
     pub json: bool,
@@ -185,6 +198,10 @@ pub struct Configuration {
     #[serde(default)]
     pub scan_limit: usize,
 
+    /// Number of parallel scans permitted; a limit of 0 means no limit is imposed
+    #[serde(default)]
+    pub parallel: usize,
+
     /// Number of requests per second permitted (per directory); a limit of 0 means no limit is imposed
     #[serde(default)]
     pub rate_limit: usize,
@@ -231,6 +248,10 @@ pub struct Configuration {
     /// Filter out response bodies that meet a certain threshold of similarity
     #[serde(default)]
     pub filter_similar: Vec<String>,
+
+    /// URLs that should never be scanned/recursed into
+    #[serde(default)]
+    pub url_denylist: Vec<String>,
 }
 
 impl Default for Configuration {
@@ -245,6 +266,7 @@ impl Default for Configuration {
         let replay_codes = status_codes.clone();
         let kind = serialized_type();
         let output_level = OutputLevel::Default;
+        let requester_policy = RequesterPolicy::Default;
 
         Configuration {
             kind,
@@ -254,7 +276,10 @@ impl Default for Configuration {
             replay_codes,
             status_codes,
             replay_client,
+            requester_policy,
             dont_filter: false,
+            auto_bail: false,
+            auto_tune: false,
             silent: false,
             quiet: false,
             output_level,
@@ -263,6 +288,7 @@ impl Default for Configuration {
             json: false,
             verbosity: 0,
             scan_limit: 0,
+            parallel: 0,
             rate_limit: 0,
             add_slash: false,
             insecure: false,
@@ -282,6 +308,7 @@ impl Default for Configuration {
             extensions: Vec::new(),
             filter_size: Vec::new(),
             filter_regex: Vec::new(),
+            url_denylist: Vec::new(),
             filter_line_count: Vec::new(),
             filter_word_count: Vec::new(),
             filter_status: Vec::new(),
@@ -313,10 +340,13 @@ impl Configuration {
     /// - **debug_log**: `None`
     /// - **quiet**: `false`
     /// - **silent**: `false`
+    /// - **auto_tune**: `false`
+    /// - **auto_bail**: `false`
     /// - **save_state**: `true`
     /// - **user_agent**: `feroxbuster/VERSION`
     /// - **insecure**: `false` (don't be insecure, i.e. don't allow invalid certs)
     /// - **extensions**: `None`
+    /// - **url_denylist**: `None`
     /// - **filter_size**: `None`
     /// - **filter_similar**: `None`
     /// - **filter_regex**: `None`
@@ -331,7 +361,8 @@ impl Configuration {
     /// - **dont_filter**: `false` (auto filter wildcard responses)
     /// - **depth**: `4` (maximum recursion depth)
     /// - **scan_limit**: `0` (no limit on concurrent scans imposed)
-    /// - **rate_limit**: `0` (no limit on concurrent scans imposed)
+    /// - **parallel**: `0` (no limit on parallel scans imposed)
+    /// - **rate_limit**: `0` (no limit on requests per second imposed)
     /// - **time_limit**: `None` (no limit on length of scan imposed)
     /// - **replay_proxy**: `None` (no limit on concurrent scans imposed)
     /// - **replay_codes**: [`DEFAULT_RESPONSE_CODES`](constant.DEFAULT_RESPONSE_CODES.html)
@@ -467,6 +498,7 @@ impl Configuration {
         update_config_if_present!(&mut config.threads, args, "threads", usize);
         update_config_if_present!(&mut config.depth, args, "depth", usize);
         update_config_if_present!(&mut config.scan_limit, args, "scan_limit", usize);
+        update_config_if_present!(&mut config.parallel, args, "parallel", usize);
         update_config_if_present!(&mut config.rate_limit, args, "rate_limit", usize);
         update_config_if_present!(&mut config.wordlist, args, "wordlist", String);
         update_config_if_present!(&mut config.output, args, "output", String);
@@ -512,6 +544,10 @@ impl Configuration {
             config.extensions = arg.map(|val| val.to_string()).collect();
         }
 
+        if let Some(arg) = args.values_of("url_denylist") {
+            config.url_denylist = arg.map(|val| val.to_string()).collect();
+        }
+
         if let Some(arg) = args.values_of("filter_regex") {
             config.filter_regex = arg.map(|val| val.to_string()).collect();
         }
@@ -561,6 +597,16 @@ impl Configuration {
             config.output_level = OutputLevel::Quiet;
         }
 
+        if args.is_present("auto_tune") {
+            config.auto_tune = true;
+            config.requester_policy = RequesterPolicy::AutoTune;
+        }
+
+        if args.is_present("auto_bail") {
+            config.auto_bail = true;
+            config.requester_policy = RequesterPolicy::AutoBail;
+        }
+
         if args.is_present("dont_filter") {
             config.dont_filter = true;
         }
@@ -721,13 +767,21 @@ impl Configuration {
         update_if_not_default!(&mut conf.verbosity, new.verbosity, 0);
         update_if_not_default!(&mut conf.silent, new.silent, false);
         update_if_not_default!(&mut conf.quiet, new.quiet, false);
-        // use updated quiet/silent values to determin output level
+        update_if_not_default!(&mut conf.auto_bail, new.auto_bail, false);
+        update_if_not_default!(&mut conf.auto_tune, new.auto_tune, false);
+        // use updated quiet/silent values to determine output level; same for requester policy
         conf.output_level = determine_output_level(conf.quiet, conf.silent);
+        conf.requester_policy = determine_requester_policy(conf.auto_tune, conf.auto_bail);
         update_if_not_default!(&mut conf.output, new.output, "");
         update_if_not_default!(&mut conf.redirects, new.redirects, false);
         update_if_not_default!(&mut conf.insecure, new.insecure, false);
         update_if_not_default!(&mut conf.extract_links, new.extract_links, false);
         update_if_not_default!(&mut conf.extensions, new.extensions, Vec::<String>::new());
+        update_if_not_default!(
+            &mut conf.url_denylist,
+            new.url_denylist,
+            Vec::<String>::new()
+        );
         update_if_not_default!(&mut conf.headers, new.headers, HashMap::new());
         update_if_not_default!(&mut conf.queries, new.queries, Vec::new());
         update_if_not_default!(&mut conf.no_recursion, new.no_recursion, false);
@@ -761,6 +815,7 @@ impl Configuration {
         );
         update_if_not_default!(&mut conf.dont_filter, new.dont_filter, false);
         update_if_not_default!(&mut conf.scan_limit, new.scan_limit, 0);
+        update_if_not_default!(&mut conf.parallel, new.parallel, 0);
         update_if_not_default!(&mut conf.rate_limit, new.rate_limit, 0);
         update_if_not_default!(&mut conf.replay_proxy, new.replay_proxy, "");
         update_if_not_default!(&mut conf.debug_log, new.debug_log, "");

src/config/mod.rs

@@ -6,4 +6,4 @@ mod utils;
 mod tests;
 
 pub use self::container::Configuration;
-pub use self::utils::{determine_output_level, OutputLevel};
+pub use self::utils::{determine_output_level, OutputLevel, RequesterPolicy};

src/config/tests.rs

@@ -16,8 +16,11 @@ fn setup_config_test() -> Configuration {
             replay_proxy = "http://127.0.0.1:8081"
             quiet = true
             silent = true
+            auto_tune = true
+            auto_bail = true
             verbosity = 1
             scan_limit = 6
+            parallel = 14
             rate_limit = 250
             time_limit = "10m"
             output = "/some/otherpath"
@@ -26,6 +29,7 @@ fn setup_config_test() -> Configuration {
             redirects = true
             insecure = true
             extensions = ["html", "php", "js"]
+            url_denylist = ["http://dont-scan.me", "https://also-not.me"]
             headers = {stuff = "things", mostuff = "mothings"}
             queries = [["name","value"], ["rick", "astley"]]
             no_recursion = true
@@ -69,20 +73,25 @@ fn default_configuration() {
     assert_eq!(config.timeout, timeout());
     assert_eq!(config.verbosity, 0);
     assert_eq!(config.scan_limit, 0);
-    assert_eq!(config.silent, false);
-    assert_eq!(config.quiet, false);
-    assert_eq!(config.dont_filter, false);
-    assert_eq!(config.no_recursion, false);
-    assert_eq!(config.json, false);
-    assert_eq!(config.save_state, true);
-    assert_eq!(config.stdin, false);
-    assert_eq!(config.add_slash, false);
-    assert_eq!(config.redirects, false);
-    assert_eq!(config.extract_links, false);
-    assert_eq!(config.insecure, false);
+    assert!(!config.silent);
+    assert!(!config.quiet);
+    assert_eq!(config.output_level, OutputLevel::Default);
+    assert!(!config.dont_filter);
+    assert!(!config.auto_tune);
+    assert!(!config.auto_bail);
+    assert_eq!(config.requester_policy, RequesterPolicy::Default);
+    assert!(!config.no_recursion);
+    assert!(!config.json);
+    assert!(config.save_state);
+    assert!(!config.stdin);
+    assert!(!config.add_slash);
+    assert!(!config.redirects);
+    assert!(!config.extract_links);
+    assert!(!config.insecure);
     assert_eq!(config.queries, Vec::new());
-    assert_eq!(config.extensions, Vec::<String>::new());
     assert_eq!(config.filter_size, Vec::<u64>::new());
+    assert_eq!(config.extensions, Vec::<String>::new());
+    assert_eq!(config.url_denylist, Vec::<String>::new());
     assert_eq!(config.filter_regex, Vec::<String>::new());
     assert_eq!(config.filter_similar, Vec::<String>::new());
     assert_eq!(config.filter_word_count, Vec::<usize>::new());
@@ -140,6 +149,13 @@ fn config_reads_scan_limit() {
     assert_eq!(config.scan_limit, 6);
 }
 
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_parallel() {
+    let config = setup_config_test();
+    assert_eq!(config.parallel, 14);
+}
+
 #[test]
 /// parse the test config and see that the value parsed is correct
 fn config_reads_rate_limit() {
@@ -172,21 +188,35 @@ fn config_reads_replay_proxy() {
 /// parse the test config and see that the value parsed is correct
 fn config_reads_silent() {
     let config = setup_config_test();
-    assert_eq!(config.silent, true);
+    assert!(config.silent);
 }
 
 #[test]
 /// parse the test config and see that the value parsed is correct
 fn config_reads_quiet() {
     let config = setup_config_test();
-    assert_eq!(config.quiet, true);
+    assert!(config.quiet);
 }
 
 #[test]
 /// parse the test config and see that the value parsed is correct
 fn config_reads_json() {
     let config = setup_config_test();
-    assert_eq!(config.json, true);
+    assert!(config.json);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_auto_bail() {
+    let config = setup_config_test();
+    assert!(config.auto_bail);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_auto_tune() {
+    let config = setup_config_test();
+    assert!(config.auto_tune);
 }
 
 #[test]
@@ -207,49 +237,49 @@ fn config_reads_output() {
 /// parse the test config and see that the value parsed is correct
 fn config_reads_redirects() {
     let config = setup_config_test();
-    assert_eq!(config.redirects, true);
+    assert!(config.redirects);
 }
 
 #[test]
 /// parse the test config and see that the value parsed is correct
 fn config_reads_insecure() {
     let config = setup_config_test();
-    assert_eq!(config.insecure, true);
+    assert!(config.insecure);
 }
 
 #[test]
 /// parse the test config and see that the value parsed is correct
 fn config_reads_no_recursion() {
     let config = setup_config_test();
-    assert_eq!(config.no_recursion, true);
+    assert!(config.no_recursion);
 }
 
 #[test]
 /// parse the test config and see that the value parsed is correct
 fn config_reads_stdin() {
     let config = setup_config_test();
-    assert_eq!(config.stdin, true);
+    assert!(config.stdin);
 }
 
 #[test]
 /// parse the test config and see that the value parsed is correct
 fn config_reads_dont_filter() {
     let config = setup_config_test();
-    assert_eq!(config.dont_filter, true);
+    assert!(config.dont_filter);
 }
 
 #[test]
 /// parse the test config and see that the value parsed is correct
 fn config_reads_add_slash() {
     let config = setup_config_test();
-    assert_eq!(config.add_slash, true);
+    assert!(config.add_slash);
 }
 
 #[test]
 /// parse the test config and see that the value parsed is correct
 fn config_reads_extract_links() {
     let config = setup_config_test();
-    assert_eq!(config.extract_links, true);
+    assert!(config.extract_links);
 }
 
 #[test]
@@ -259,6 +289,16 @@ fn config_reads_extensions() {
     assert_eq!(config.extensions, vec!["html", "php", "js"]);
 }
 
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_url_denylist() {
+    let config = setup_config_test();
+    assert_eq!(
+        config.url_denylist,
+        vec!["http://dont-scan.me", "https://also-not.me"]
+    );
+}
+
 #[test]
 /// parse the test config and see that the value parsed is correct
 fn config_reads_filter_regex() {
@@ -305,7 +345,7 @@ fn config_reads_filter_status() {
 /// parse the test config and see that the value parsed is correct
 fn config_reads_save_state() {
     let config = setup_config_test();
-    assert_eq!(config.save_state, false);
+    assert!(!config.save_state);
 }
 
 #[test]
@@ -336,9 +376,10 @@ fn config_reads_headers() {
 /// parse the test config and see that the values parsed are correct
 fn config_reads_queries() {
     let config = setup_config_test();
-    let mut queries = vec![];
-    queries.push(("name".to_string(), "value".to_string()));
-    queries.push(("rick".to_string(), "astley".to_string()));
+    let queries = vec![
+        ("name".to_string(), "value".to_string()),
+        ("rick".to_string(), "astley".to_string()),
+    ];
     assert_eq!(config.queries, queries);
 }
 

src/config/utils.rs

@@ -102,6 +102,41 @@ pub fn determine_output_level(quiet: bool, silent: bool) -> OutputLevel {
     }
 }
 
+/// represents actions the Requester should take in certain situations
+#[derive(Debug, PartialEq, Copy, Clone)]
+pub enum RequesterPolicy {
+    /// automatically try to lower request rate in order to reduce errors
+    AutoTune,
+
+    /// automatically bail at certain error thresholds
+    AutoBail,
+
+    /// just let that junk run super natural
+    Default,
+}
+
+/// default implementation for RequesterPolicy
+impl Default for RequesterPolicy {
+    /// Default as default
+    fn default() -> Self {
+        Self::Default
+    }
+}
+
+/// given the current settings for quiet and silent, determine output_level (DRY helper)
+pub fn determine_requester_policy(auto_tune: bool, auto_bail: bool) -> RequesterPolicy {
+    if auto_tune && auto_bail {
+        // user COULD have both as true in config file, take the more aggressive of the two
+        RequesterPolicy::AutoBail
+    } else if auto_tune {
+        RequesterPolicy::AutoTune
+    } else if auto_bail {
+        RequesterPolicy::AutoBail
+    } else {
+        RequesterPolicy::Default
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -122,6 +157,22 @@ mod tests {
         assert_eq!(level, OutputLevel::Quiet);
     }
 
+    #[test]
+    /// test determine_requester_policy returns higher of the two levels if both given values are true
+    fn determine_requester_policy_returns_correct_results() {
+        let mut level = determine_requester_policy(true, true);
+        assert_eq!(level, RequesterPolicy::AutoBail);
+
+        level = determine_requester_policy(false, true);
+        assert_eq!(level, RequesterPolicy::AutoBail);
+
+        level = determine_requester_policy(false, false);
+        assert_eq!(level, RequesterPolicy::Default);
+
+        level = determine_requester_policy(true, false);
+        assert_eq!(level, RequesterPolicy::AutoTune);
+    }
+
     #[test]
     #[should_panic]
     /// report_and_exit should panic/exit when called

src/event_handlers/command.rs

@@ -1,4 +1,3 @@
-use std::collections::HashSet;
 use std::sync::Arc;
 
 use reqwest::StatusCode;
@@ -25,11 +24,14 @@ pub enum Command {
     /// Create the progress bar (`BarType::Total`) that is updated from the stats thread
     CreateBar,
 
-    /// Update a `Stats` field that corresponds to the given `StatField` by the given `usize` value
-    UpdateUsizeField(StatField, usize),
+    /// Add to a `Stats` field that corresponds to the given `StatField` by the given `usize` value
+    AddToUsizeField(StatField, usize),
+
+    /// Subtract from a `Stats` field that corresponds to the given `StatField` by the given `usize` value
+    SubtractFromUsizeField(StatField, usize),
 
     /// Update a `Stats` field that corresponds to the given `StatField` by the given `f64` value
-    UpdateF64Field(StatField, f64),
+    AddToF64Field(StatField, f64),
 
     /// Save a `Stats` object to disk using `reporter::get_cached_file_handle`
     Save,
@@ -50,7 +52,7 @@ pub enum Command {
     TryRecursion(Box<FeroxResponse>),
 
     /// Send a pointer to the wordlist to the recursion handler
-    UpdateWordlist(Arc<HashSet<String>>),
+    UpdateWordlist(Arc<Vec<String>>),
 
     /// Instruct the ScanHandler to join on all known scans, use sender to notify main when done
     JoinTasks(Sender<bool>),

src/event_handlers/inputs.rs

@@ -4,6 +4,7 @@ use crate::{
     scan_manager::{FeroxState, PAUSE_SCAN},
     scanner::RESPONSES,
     statistics::StatError,
+    utils::slugify_filename,
     utils::{open_file, write_to},
     SLEEP_DURATION,
 };
@@ -17,7 +18,6 @@ use std::{
     },
     thread::sleep,
     time::Duration,
-    time::{SystemTime, UNIX_EPOCH},
 };
 
 /// Atomic boolean flag, used to determine whether or not the terminal input handler should exit
@@ -77,22 +77,14 @@ impl TermInputHandler {
     pub fn sigint_handler(handles: Arc<Handles>) -> Result<()> {
         log::trace!("enter: sigint_handler({:?})", handles);
 
-        let ts = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs();
-
-        let slug = if !handles.config.target_url.is_empty() {
+        let filename = if !handles.config.target_url.is_empty() {
             // target url populated
-            handles
-                .config
-                .target_url
-                .replace("://", "_")
-                .replace("/", "_")
-                .replace(".", "_")
+            slugify_filename(&handles.config.target_url, "ferox", "state")
         } else {
             // stdin used
-            "stdin".to_string()
+            slugify_filename("stdin", "ferox", "state")
         };
 
-        let filename = format!("ferox-{}-{}.state", slug, ts);
         let warning = format!(
             "🚨 Caught {} 🚨 saving scan state to {} ...",
             style("ctrl+c").yellow(),

src/event_handlers/outputs.rs

@@ -1,4 +1,4 @@
-use super::Command::UpdateUsizeField;
+use super::Command::AddToUsizeField;
 use super::*;
 
 use anyhow::{Context, Result};
@@ -139,8 +139,8 @@ impl TermOutHandler {
         Self {
             receiver,
             tx_file,
-            config,
             file_task,
+            config,
         }
     }
 
@@ -195,7 +195,7 @@ impl TermOutHandler {
                         // print to stdout
                         ferox_print(&resp.as_str(), &PROGRESS_PRINTER);
 
-                        send_command!(tx_stats, UpdateUsizeField(ResourcesDiscovered, 1));
+                        send_command!(tx_stats, AddToUsizeField(ResourcesDiscovered, 1));
 
                         if self.file_task.is_some() {
                             // -o used, need to send the report to be written out to disk
@@ -210,10 +210,10 @@ impl TermOutHandler {
 
                     if self.config.replay_client.is_some() && should_process_response {
                         // replay proxy specified/client created and this response's status code is one that
-                        // should be replayed
+                        // should be replayed; not using logged_request due to replay proxy client
                         make_request(
                             self.config.replay_client.as_ref().unwrap(),
-                            &resp.url(),
+                            resp.url(),
                             self.config.output_level,
                             tx_stats.clone(),
                         )

src/event_handlers/scans.rs

@@ -1,20 +1,22 @@
-use std::collections::HashSet;
 use std::sync::Arc;
 
 use anyhow::{bail, Result};
 use tokio::sync::{mpsc, Semaphore};
 
-use crate::response::FeroxResponse;
-use crate::url::FeroxUrl;
 use crate::{
+    response::FeroxResponse,
     scan_manager::{FeroxScan, FeroxScans, ScanOrder},
     scanner::FeroxScanner,
     statistics::StatField::TotalScans,
-    CommandReceiver, CommandSender, FeroxChannel, Joiner,
+    url::FeroxUrl,
+    utils::should_deny_url,
+    CommandReceiver, CommandSender, FeroxChannel, Joiner, SLEEP_DURATION,
 };
 
-use super::command::Command::UpdateUsizeField;
+use super::command::Command::AddToUsizeField;
 use super::*;
+use reqwest::Url;
+use tokio::time::Duration;
 
 #[derive(Debug)]
 /// Container for recursion transmitter and FeroxScans object
@@ -53,7 +55,7 @@ pub struct ScanHandler {
     receiver: CommandReceiver,
 
     /// wordlist (re)used for each scan
-    wordlist: std::sync::Mutex<Option<Arc<HashSet<String>>>>,
+    wordlist: std::sync::Mutex<Option<Arc<Vec<String>>>>,
 
     /// group of scans that need to be joined
     tasks: Vec<Arc<FeroxScan>>,
@@ -104,7 +106,7 @@ impl ScanHandler {
     }
 
     /// Set the wordlist
-    fn wordlist(&self, wordlist: Arc<HashSet<String>>) {
+    fn wordlist(&self, wordlist: Arc<Vec<String>>) {
         if let Ok(mut guard) = self.wordlist.lock() {
             if guard.is_none() {
                 let _ = std::mem::replace(&mut *guard, Some(wordlist));
@@ -153,9 +155,7 @@ impl ScanHandler {
 
                     tokio::spawn(async move {
                         while ferox_scans.has_active_scans() {
-                            for scan in ferox_scans.get_active_scans() {
-                                scan.join().await;
-                            }
+                            tokio::time::sleep(Duration::from_millis(SLEEP_DURATION + 250)).await;
                         }
                         limiter_clone.close();
                         sender.send(true).expect("oneshot channel failed");
@@ -176,7 +176,7 @@ impl ScanHandler {
     }
 
     /// Helper to easily get the (locked) underlying wordlist
-    pub fn get_wordlist(&self) -> Result<Arc<HashSet<String>>> {
+    pub fn get_wordlist(&self) -> Result<Arc<Vec<String>>> {
         if let Ok(guard) = self.wordlist.lock().as_ref() {
             if let Some(list) = guard.as_ref() {
                 return Ok(list.clone());
@@ -189,6 +189,7 @@ impl ScanHandler {
     /// wrapper around scanning a url to stay DRY
     async fn ordered_scan_url(&mut self, targets: Vec<String>, order: ScanOrder) -> Result<()> {
         log::trace!("enter: ordered_scan_url({:?}, {:?})", targets, order);
+        let should_test_deny = !self.handles.config.url_denylist.is_empty();
 
         for target in targets {
             if self.data.contains(&target) && matches!(order, ScanOrder::Latest) {
@@ -205,6 +206,13 @@ impl ScanHandler {
                 self.data.add_directory_scan(&target, order).1 // add the new target; return FeroxScan
             };
 
+            if should_test_deny && should_deny_url(&Url::parse(&target)?, self.handles.clone())? {
+                // response was caught by a user-provided deny list
+                // checking this last, since it's most susceptible to longer runtimes due to what
+                // input is received
+                continue;
+            }
+
             let list = self.get_wordlist()?;
 
             log::info!("scan handler received {} - beginning scan", target);
@@ -231,7 +239,7 @@ impl ScanHandler {
                 }
             });
 
-            self.handles.stats.send(UpdateUsizeField(TotalScans, 1))?;
+            self.handles.stats.send(AddToUsizeField(TotalScans, 1))?;
 
             scan.set_task(task).await?;
 
@@ -245,6 +253,11 @@ impl ScanHandler {
     async fn try_recursion(&mut self, response: Box<FeroxResponse>) -> Result<()> {
         log::trace!("enter: try_recursion({:?})", response,);
 
+        if !response.is_directory() {
+            // not a directory, quick exit
+            return Ok(());
+        }
+
         let mut base_depth = 1_usize;
 
         for (base_url, base_url_depth) in &self.depths {
@@ -258,11 +271,6 @@ impl ScanHandler {
             return Ok(());
         }
 
-        if !response.is_directory() {
-            // not a directory
-            return Ok(());
-        }
-
         let targets = vec![response.url().to_string()];
         self.ordered_scan_url(targets, ScanOrder::Latest).await?;
 

src/event_handlers/statistics.rs

@@ -89,6 +89,7 @@ impl StatsHandler {
                 }
                 Command::AddStatus(status) => {
                     self.stats.add_status_code(status);
+
                     self.increment_bar();
                 }
                 Command::AddRequest => {
@@ -99,14 +100,21 @@ impl StatsHandler {
                     self.stats
                         .save(start.elapsed().as_secs_f64(), output_file)?;
                 }
-                Command::UpdateUsizeField(field, value) => {
+                Command::AddToUsizeField(field, value) => {
                     self.stats.update_usize_field(field, value);
 
                     if matches!(field, StatField::TotalScans) {
                         self.bar.set_length(self.stats.total_expected() as u64);
                     }
                 }
-                Command::UpdateF64Field(field, value) => self.stats.update_f64_field(field, value),
+                Command::SubtractFromUsizeField(field, value) => {
+                    self.stats.subtract_from_usize_field(field, value);
+
+                    if matches!(field, StatField::TotalExpected) {
+                        self.bar.set_length(self.stats.total_expected() as u64);
+                    }
+                }
+                Command::AddToF64Field(field, value) => self.stats.update_f64_field(field, value),
                 Command::CreateBar => {
                     self.bar = add_bar("", self.stats.total_expected() as u64, BarType::Total);
                 }

src/extractor/container.rs

@@ -1,9 +1,10 @@
 use super::*;
+use crate::utils::should_deny_url;
 use crate::{
     client,
     event_handlers::{
         Command,
-        Command::{AddError, UpdateUsizeField},
+        Command::{AddError, AddToUsizeField},
         Handles,
     },
     scan_manager::ScanOrder,
@@ -12,7 +13,7 @@ use crate::{
         StatField::{LinksExtracted, TotalExpected},
     },
     url::FeroxUrl,
-    utils::make_request,
+    utils::{logged_request, make_request},
 };
 use anyhow::{bail, Context, Result};
 use reqwest::{StatusCode, Url};
@@ -53,13 +54,19 @@ pub struct Extractor<'a> {
 
 /// Extractor implementation
 impl<'a> Extractor<'a> {
-    /// business logic that handles getting links from a normal http body response
-    pub async fn extract(&self) -> Result<()> {
-        let links = match self.target {
-            ExtractionTarget::ResponseBody => self.extract_from_body().await?,
-            ExtractionTarget::RobotsTxt => self.extract_from_robots().await?,
-        };
+    /// perform extraction from the given target and return any links found
+    pub async fn extract(&self) -> Result<HashSet<String>> {
+        log::trace!("enter: extract (this fn has associated trace exit msg)");
+        match self.target {
+            ExtractionTarget::ResponseBody => Ok(self.extract_from_body().await?),
+            ExtractionTarget::RobotsTxt => Ok(self.extract_from_robots().await?),
+        }
+    }
 
+    /// given a set of links from a normal http body response, task the request handler to make
+    /// the requests
+    pub async fn request_links(&self, links: HashSet<String>) -> Result<()> {
+        log::trace!("enter: request_links({:?})", links);
         let recursive = if self.handles.config.no_recursion {
             RecursionStatus::NotRecursive
         } else {
@@ -121,6 +128,7 @@ impl<'a> Extractor<'a> {
                 rx.await?;
             }
         }
+        log::trace!("exit: request_links");
         Ok(())
     }
 
@@ -141,7 +149,7 @@ impl<'a> Extractor<'a> {
 
         let body = self.response.unwrap().text();
 
-        for capture in self.links_regex.captures_iter(&body) {
+        for capture in self.links_regex.captures_iter(body) {
             // remove single & double quotes from both ends of the capture
             // capture[0] is the entire match, additional capture groups start at [1]
             let link = capture[0].trim_matches(|c| c == '\'' || c == '"');
@@ -267,7 +275,7 @@ impl<'a> Extractor<'a> {
         };
 
         let new_url = old_url
-            .join(&link)
+            .join(link)
             .with_context(|| format!("Could not join {} with {}", old_url, link))?;
 
         links.insert(new_url.to_string());
@@ -289,10 +297,10 @@ impl<'a> Extractor<'a> {
     pub(super) async fn request_link(&self, url: &str) -> Result<FeroxResponse> {
         log::trace!("enter: request_link({})", url);
 
-        let ferox_url = FeroxUrl::from_string(&url, self.handles.clone());
+        let ferox_url = FeroxUrl::from_string(url, self.handles.clone());
 
         // create a url based on the given command line options
-        let new_url = ferox_url.format(&"", None)?;
+        let new_url = ferox_url.format("", None)?;
 
         let scanned_urls = self.handles.ferox_scans()?;
 
@@ -302,14 +310,19 @@ impl<'a> Extractor<'a> {
             bail!("previously seen url");
         }
 
+        if !self.handles.config.url_denylist.is_empty()
+            && should_deny_url(&new_url, self.handles.clone())?
+        {
+            // can't allow a denied url to be requested
+            bail!(
+                "prevented request to {} due to {:?}",
+                url,
+                self.handles.config.url_denylist
+            );
+        }
+
         // make the request and store the response
-        let new_response = make_request(
-            &self.handles.config.client,
-            &new_url,
-            self.handles.config.output_level,
-            self.handles.stats.tx.clone(),
-        )
-        .await?;
+        let new_response = logged_request(&new_url, self.handles.clone()).await?;
 
         let new_ferox_response =
             FeroxResponse::from(new_response, true, self.handles.config.output_level).await;
@@ -338,7 +351,7 @@ impl<'a> Extractor<'a> {
             if let Some(new_path) = capture.name("url_path") {
                 let mut new_url = Url::parse(&self.url)?;
                 new_url.set_path(new_path.as_str());
-                if self.add_all_sub_paths(&new_url.path(), &mut links).is_err() {
+                if self.add_all_sub_paths(new_url.path(), &mut links).is_err() {
                     log::warn!("could not add sub-paths from {} to {:?}", new_url, links);
                 }
             }
@@ -384,6 +397,7 @@ impl<'a> Extractor<'a> {
         let mut url = Url::parse(&self.url)?;
         url.set_path("/robots.txt"); // overwrite existing path with /robots.txt
 
+        // purposefully not using logged_request here due to using the special client
         let response = make_request(
             &client,
             &url,
@@ -391,11 +405,12 @@ impl<'a> Extractor<'a> {
             self.handles.stats.tx.clone(),
         )
         .await?;
+
         let ferox_response =
             FeroxResponse::from(response, true, self.handles.config.output_level).await;
 
         log::trace!("exit: get_robots_file -> {}", ferox_response);
-        return Ok(ferox_response);
+        Ok(ferox_response)
     }
 
     /// update total number of links extracted and expected responses
@@ -404,10 +419,10 @@ impl<'a> Extractor<'a> {
 
         self.handles
             .stats
-            .send(UpdateUsizeField(LinksExtracted, num_links))?;
+            .send(AddToUsizeField(LinksExtracted, num_links))?;
         self.handles
             .stats
-            .send(UpdateUsizeField(TotalExpected, num_links * multiplier))?;
+            .send(AddToUsizeField(TotalExpected, num_links * multiplier))?;
 
         Ok(())
     }

src/extractor/tests.rs

@@ -54,8 +54,8 @@ fn setup_extractor(target: ExtractionTarget, scanned_urls: Arc<FeroxScans>) -> E
 /// in the expected array
 fn extractor_get_sub_paths_from_path_with_multiple_paths() {
     let path = "homepage/assets/img/icons/handshake.svg";
-    let r_paths = ROBOTS_EXT.get_sub_paths_from_path(&path);
-    let b_paths = BODY_EXT.get_sub_paths_from_path(&path);
+    let r_paths = ROBOTS_EXT.get_sub_paths_from_path(path);
+    let b_paths = BODY_EXT.get_sub_paths_from_path(path);
     let expected = vec![
         "homepage/",
         "homepage/assets/",
@@ -67,8 +67,8 @@ fn extractor_get_sub_paths_from_path_with_multiple_paths() {
     assert_eq!(r_paths.len(), expected.len());
     assert_eq!(b_paths.len(), expected.len());
     for expected_path in expected {
-        assert_eq!(r_paths.contains(&expected_path.to_string()), true);
-        assert_eq!(b_paths.contains(&expected_path.to_string()), true);
+        assert!(r_paths.contains(&expected_path.to_string()));
+        assert!(b_paths.contains(&expected_path.to_string()));
     }
 }
 
@@ -78,15 +78,15 @@ fn extractor_get_sub_paths_from_path_with_multiple_paths() {
 /// returned
 fn extractor_get_sub_paths_from_path_with_enclosing_slashes() {
     let path = "/homepage/assets/";
-    let r_paths = ROBOTS_EXT.get_sub_paths_from_path(&path);
-    let b_paths = BODY_EXT.get_sub_paths_from_path(&path);
+    let r_paths = ROBOTS_EXT.get_sub_paths_from_path(path);
+    let b_paths = BODY_EXT.get_sub_paths_from_path(path);
     let expected = vec!["homepage/", "homepage/assets"];
 
     assert_eq!(r_paths.len(), expected.len());
     assert_eq!(b_paths.len(), expected.len());
     for expected_path in expected {
-        assert_eq!(r_paths.contains(&expected_path.to_string()), true);
-        assert_eq!(b_paths.contains(&expected_path.to_string()), true);
+        assert!(r_paths.contains(&expected_path.to_string()));
+        assert!(b_paths.contains(&expected_path.to_string()));
     }
 }
 
@@ -95,15 +95,15 @@ fn extractor_get_sub_paths_from_path_with_enclosing_slashes() {
 /// included
 fn extractor_get_sub_paths_from_path_with_only_a_word() {
     let path = "homepage";
-    let r_paths = ROBOTS_EXT.get_sub_paths_from_path(&path);
-    let b_paths = BODY_EXT.get_sub_paths_from_path(&path);
+    let r_paths = ROBOTS_EXT.get_sub_paths_from_path(path);
+    let b_paths = BODY_EXT.get_sub_paths_from_path(path);
     let expected = vec!["homepage"];
 
     assert_eq!(r_paths.len(), expected.len());
     assert_eq!(b_paths.len(), expected.len());
     for expected_path in expected {
-        assert_eq!(r_paths.contains(&expected_path.to_string()), true);
-        assert_eq!(b_paths.contains(&expected_path.to_string()), true);
+        assert!(r_paths.contains(&expected_path.to_string()));
+        assert!(b_paths.contains(&expected_path.to_string()));
     }
 }
 
@@ -111,15 +111,15 @@ fn extractor_get_sub_paths_from_path_with_only_a_word() {
 /// extract sub paths from the given url fragment; expect 1 sub path, forward slash removed
 fn extractor_get_sub_paths_from_path_with_an_absolute_word() {
     let path = "/homepage";
-    let r_paths = ROBOTS_EXT.get_sub_paths_from_path(&path);
-    let b_paths = BODY_EXT.get_sub_paths_from_path(&path);
+    let r_paths = ROBOTS_EXT.get_sub_paths_from_path(path);
+    let b_paths = BODY_EXT.get_sub_paths_from_path(path);
     let expected = vec!["homepage"];
 
     assert_eq!(r_paths.len(), expected.len());
     assert_eq!(b_paths.len(), expected.len());
     for expected_path in expected {
-        assert_eq!(r_paths.contains(&expected_path.to_string()), true);
-        assert_eq!(b_paths.contains(&expected_path.to_string()), true);
+        assert!(r_paths.contains(&expected_path.to_string()));
+        assert!(b_paths.contains(&expected_path.to_string()));
     }
 }
 

src/filters/container.rs

@@ -4,7 +4,7 @@ use anyhow::Result;
 
 use crate::response::FeroxResponse;
 use crate::{
-    event_handlers::Command::UpdateUsizeField, statistics::StatField::WildcardsFiltered,
+    event_handlers::Command::AddToUsizeField, statistics::StatField::WildcardsFiltered,
     CommandSender,
 };
 
@@ -41,10 +41,10 @@ impl FeroxFilters {
         if let Ok(filters) = self.filters.lock() {
             for filter in filters.iter() {
                 // wildcard.should_filter goes here
-                if filter.should_filter_response(&response) {
+                if filter.should_filter_response(response) {
                     if filter.as_any().downcast_ref::<WildcardFilter>().is_some() {
                         tx_stats
-                            .send(UpdateUsizeField(WildcardsFiltered, 1))
+                            .send(AddToUsizeField(WildcardsFiltered, 1))
                             .unwrap_or_default();
                     }
                     return true;

src/filters/init.rs

@@ -5,7 +5,7 @@ use crate::{
     event_handlers::Handles,
     response::FeroxResponse,
     skip_fail,
-    utils::{fmt_err, make_request},
+    utils::{fmt_err, logged_request},
     Command::AddFilter,
     SIMILARITY_THRESHOLD,
 };
@@ -56,7 +56,7 @@ pub async fn initialize(handles: Arc<Handles>) -> Result<()> {
     // add any regex filters to filters handler's FeroxFilters  (-X|--filter-regex)
     for regex_filter in &handles.config.filter_regex {
         let raw = regex_filter;
-        let compiled = skip_fail!(Regex::new(&raw));
+        let compiled = skip_fail!(Regex::new(raw));
 
         let filter = RegexFilter {
             raw_string: raw.to_owned(),
@@ -69,18 +69,10 @@ pub async fn initialize(handles: Arc<Handles>) -> Result<()> {
     // add any similarity filters to filters handler's FeroxFilters  (--filter-similar-to)
     for similarity_filter in &handles.config.filter_similar {
         // url as-is based on input, ignores user-specified url manipulation options (add-slash etc)
-        let url = skip_fail!(Url::parse(&similarity_filter));
+        let url = skip_fail!(Url::parse(similarity_filter));
 
         // attempt to request the given url
-        let resp = skip_fail!(
-            make_request(
-                &handles.config.client,
-                &url,
-                handles.config.output_level,
-                handles.stats.tx.clone()
-            )
-            .await
-        );
+        let resp = skip_fail!(logged_request(&url, handles.clone()).await);
 
         // if successful, create a filter based on the response's body
         let fr = FeroxResponse::from(resp, true, handles.config.output_level).await;

src/filters/wildcard.rs

@@ -76,7 +76,7 @@ impl FeroxFilter for WildcardFilter {
             // except that I don't want an empty string taking up the last index in the
             // event that the url ends with a forward slash.  It's ugly enough to be split
             // into its own function for readability.
-            let url_len = FeroxUrl::path_length_of_url(&response.url());
+            let url_len = FeroxUrl::path_length_of_url(response.url());
 
             if url_len + self.dynamic == response.content_length() {
                 log::debug!("dynamic wildcard: filtered out {}", response.url());

src/heuristics.rs

@@ -12,7 +12,7 @@ use crate::{
     response::FeroxResponse,
     skip_fail,
     url::FeroxUrl,
-    utils::{ferox_print, fmt_err, make_request, status_colorizer},
+    utils::{ferox_print, fmt_err, logged_request, status_colorizer},
 };
 
 /// length of a standard UUID, used when determining wildcard responses
@@ -158,13 +158,7 @@ impl HeuristicTests {
         let unique_str = self.unique_string(length);
         let nonexistent_url = target.format(&unique_str, None)?;
 
-        let response = make_request(
-            &self.handles.config.client,
-            &nonexistent_url.to_owned(),
-            self.handles.config.output_level,
-            self.handles.stats.tx.clone(),
-        )
-        .await?;
+        let response = logged_request(&nonexistent_url.to_owned(), self.handles.clone()).await?;
 
         if self
             .handles
@@ -213,15 +207,10 @@ impl HeuristicTests {
         let mut good_urls = vec![];
 
         for target_url in target_urls {
-            let url = FeroxUrl::from_string(&target_url, self.handles.clone());
+            let url = FeroxUrl::from_string(target_url, self.handles.clone());
             let request = skip_fail!(url.format("", None));
-            let result = make_request(
-                &self.handles.config.client,
-                &request,
-                self.handles.config.output_level,
-                self.handles.stats.tx.clone(),
-            )
-            .await;
+
+            let result = logged_request(&request, self.handles.clone()).await;
 
             match result {
                 Ok(_) => {

src/lib.rs

@@ -43,7 +43,7 @@ pub(crate) type FeroxChannel<T> = (UnboundedSender<T>, UnboundedReceiver<T>);
 pub(crate) const VERSION: &str = env!("CARGO_PKG_VERSION");
 
 /// Maximum number of file descriptors that can be opened during a scan
-pub const DEFAULT_OPEN_FILE_LIMIT: usize = 8192;
+pub const DEFAULT_OPEN_FILE_LIMIT: u64 = 8192;
 
 /// Default value used to determine near-duplicate web pages (equivalent to 95%)
 pub const SIMILARITY_THRESHOLD: u32 = 95;
@@ -57,7 +57,10 @@ pub const DEFAULT_WORDLIST: &str =
     "/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt";
 
 /// Number of milliseconds to wait between polls of `PAUSE_SCAN` when user pauses a scan
-pub(crate) static SLEEP_DURATION: u64 = 500;
+pub(crate) const SLEEP_DURATION: u64 = 500;
+
+/// The percentage of requests as errors it takes to be deemed too high
+pub const HIGH_ERROR_RATIO: f64 = 0.90;
 
 /// Default list of status codes to report
 ///
@@ -70,7 +73,8 @@ pub(crate) static SLEEP_DURATION: u64 = 500;
 /// * 401 Unauthorized
 /// * 403 Forbidden
 /// * 405 Method Not Allowed
-pub const DEFAULT_STATUS_CODES: [StatusCode; 9] = [
+/// * 500 Internal Server Error
+pub const DEFAULT_STATUS_CODES: [StatusCode; 10] = [
     StatusCode::OK,
     StatusCode::NO_CONTENT,
     StatusCode::MOVED_PERMANENTLY,
@@ -80,6 +84,7 @@ pub const DEFAULT_STATUS_CODES: [StatusCode; 9] = [
     StatusCode::UNAUTHORIZED,
     StatusCode::FORBIDDEN,
     StatusCode::METHOD_NOT_ALLOWED,
+    StatusCode::INTERNAL_SERVER_ERROR,
 ];
 
 /// Default filename for config file settings

src/main.rs

@@ -1,13 +1,19 @@
 use std::{
-    collections::HashSet,
-    fs::File,
+    env::args,
+    fs::{create_dir, remove_file, File},
     io::{stderr, BufRead, BufReader},
+    ops::Index,
+    path::Path,
+    process::Command,
     sync::{atomic::Ordering, Arc},
 };
 
 use anyhow::{bail, Context, Result};
 use futures::StreamExt;
-use tokio::{io, sync::oneshot};
+use tokio::{
+    io,
+    sync::{oneshot, Semaphore},
+};
 use tokio_util::codec::{FramedRead, LinesCodec};
 
 use feroxbuster::{
@@ -22,20 +28,27 @@ use feroxbuster::{
     progress::{PROGRESS_BAR, PROGRESS_PRINTER},
     scan_manager::{self},
     scanner,
-    utils::fmt_err,
+    utils::{fmt_err, slugify_filename},
 };
 #[cfg(not(target_os = "windows"))]
 use feroxbuster::{utils::set_open_file_limit, DEFAULT_OPEN_FILE_LIMIT};
+use lazy_static::lazy_static;
+use regex::Regex;
+
+lazy_static! {
+    /// Limits the number of parallel scans active at any given time when using --parallel
+    static ref PARALLEL_LIMITER: Semaphore = Semaphore::new(0);
+}
 
 /// Create a HashSet of Strings from the given wordlist then stores it inside an Arc
-fn get_unique_words_from_wordlist(path: &str) -> Result<Arc<HashSet<String>>> {
+fn get_unique_words_from_wordlist(path: &str) -> Result<Arc<Vec<String>>> {
     log::trace!("enter: get_unique_words_from_wordlist({})", path);
 
     let file = File::open(&path).with_context(|| format!("Could not open {}", path))?;
 
     let reader = BufReader::new(file);
 
-    let mut words = HashSet::new();
+    let mut words = Vec::new();
 
     for line in reader.lines() {
         let result = match line {
@@ -47,7 +60,7 @@ fn get_unique_words_from_wordlist(path: &str) -> Result<Arc<HashSet<String>>> {
             continue;
         }
 
-        words.insert(result);
+        words.push(result);
     }
 
     log::trace!(
@@ -65,11 +78,7 @@ async fn scan(targets: Vec<String>, handles: Arc<Handles>) -> Result<()> {
     // so that will allow for cheap/safe sharing of a single wordlist across multi-target scans
     // as well as additional directories found as part of recursion
 
-    let words = {
-        let words_handles = handles.clone();
-        tokio::spawn(async move { get_unique_words_from_wordlist(&words_handles.config.wordlist) })
-            .await??
-    };
+    let words = get_unique_words_from_wordlist(&handles.config.wordlist)?;
 
     if words.len() == 0 {
         bail!("Did not find any words in {}", handles.config.wordlist);
@@ -226,6 +235,132 @@ async fn wrapped_main(config: Arc<Configuration>) -> Result<()> {
         }
     };
 
+    // --parallel branch
+    if config.parallel > 0 {
+        log::trace!("enter: parallel branch");
+
+        PARALLEL_LIMITER.add_permits(config.parallel);
+
+        let invocation = args();
+
+        let para_regex =
+            Regex::new("--stdin|-q|--quiet|--silent|--verbosity|-v|-vv|-vvv|-vvvv").unwrap();
+
+        // remove stdin since only the original process will process targets
+        // remove quiet and silent so we can force silent later to normalize output
+        let mut original = invocation
+            .filter(|s| !para_regex.is_match(s))
+            .collect::<Vec<String>>();
+
+        original.push("--silent".to_string()); // only output modifier allowed
+
+        // we need remove --parallel from command line so we don't hit this branch over and over
+        // but we must remove --parallel N manually; the filter above never sees --parallel and the
+        // value passed to it at the same time, so can't filter them out in one pass
+
+        // unwrap is fine, as it has to be in the args for us to be in this code branch
+        let parallel_index = original.iter().position(|s| *s == "--parallel").unwrap();
+
+        // remove --parallel
+        original.remove(parallel_index);
+
+        // remove N passed to --parallel (it's the same index again since everything shifts
+        // from removing --parallel)
+        original.remove(parallel_index);
+
+        // to log unique files to a shared folder, we need to first check for the presence
+        // of -o|--output.
+        let out_dir = if !config.output.is_empty() {
+            // -o|--output was used, so we'll attempt to create a directory to store the files
+            let output_path = Path::new(&handles.config.output);
+
+            // this only returns None if the path terminates in `..`. Since I don't want to
+            // hand-hold to that degree, we'll unwrap and fail if the output path ends in `..`
+            let base_name = output_path.file_name().unwrap();
+
+            let new_folder = slugify_filename(&base_name.to_string_lossy(), "", "logs");
+
+            let final_path = output_path.with_file_name(&new_folder);
+
+            // create the directory or fail silently, assuming the reason for failure is that
+            // the path exists already
+            create_dir(&final_path).unwrap_or(());
+
+            final_path.to_string_lossy().to_string()
+        } else {
+            String::new()
+        };
+
+        // unvalidated targets fresh from stdin, just spawn children and let them do all checks
+        for target in targets {
+            // add the current target to the provided command
+            let mut cloned = original.clone();
+
+            if !out_dir.is_empty() {
+                // output directory value is not empty, need to join output directory with
+                // unique scan filename
+
+                // unwrap is ok, we already know -o was used
+                let out_idx = original
+                    .iter()
+                    .position(|s| *s == "--output" || *s == "-o")
+                    .unwrap();
+
+                let filename = slugify_filename(&target, "ferox", "log");
+
+                let full_path = Path::new(&out_dir)
+                    .join(filename)
+                    .to_string_lossy()
+                    .to_string();
+
+                // a +1 to the index is fine here, as clap has already validated that
+                // -o|--output has a value associated with it
+                cloned[out_idx + 1] = full_path;
+            }
+
+            cloned.push("-u".to_string());
+            cloned.push(target);
+
+            let bin = cloned.index(0).to_owned(); // user's path to feroxbuster
+            let args = cloned.index(1..).to_vec(); // and args
+
+            let permit = PARALLEL_LIMITER.acquire().await?;
+
+            log::debug!("parallel exec: {} {}", bin, args.join(" "));
+
+            tokio::task::spawn_blocking(move || {
+                let result = Command::new(bin)
+                    .args(&args)
+                    .spawn()
+                    .expect("failed to spawn a child process")
+                    .wait()
+                    .expect("child process errored during execution");
+
+                drop(permit);
+                result
+            });
+        }
+
+        // the output handler creates an empty file to which it will try to write, because
+        // this happens before we enter the --parallel branch, we need to remove that file
+        // if it's empty
+        let output = handles.config.output.to_owned();
+
+        clean_up(handles, tasks).await?;
+
+        let file = Path::new(&output);
+        if file.exists() {
+            // expectation is that this is always true for the first ferox process
+            if file.metadata()?.len() == 0 {
+                // empty file, attempt to remove it
+                remove_file(file)?;
+            }
+        }
+
+        log::trace!("exit: parallel branch && wrapped main");
+        return Ok(());
+    }
+
     if matches!(config.output_level, OutputLevel::Default) {
         // only print banner if output level is default (no banner on --quiet|--silent)
         let std_stderr = stderr(); // std::io::stderr
@@ -246,7 +381,7 @@ async fn wrapped_main(config: Arc<Configuration>) -> Result<()> {
 
         // The TermOutHandler spawns a FileOutHandler, so errors in the FileOutHandler never bubble
         // up due to the TermOutHandler never awaiting the result of FileOutHandler::start (that's
-        // done later here in main). Ping checks that the tx/rx connection to the file handler works
+        // done later here in main). sync checks that the tx/rx connection to the file handler works
         if send_to_file && handles.output.sync(send_to_file).await.is_err() {
             // output file specified and file handler could not initialize
             clean_up(handles, tasks).await?;

src/message.rs

@@ -1,9 +1,9 @@
 use anyhow::Context;
 use console::{style, Color};
+use serde::{Deserialize, Serialize};
 
 use crate::traits::FeroxSerialize;
 use crate::utils::fmt_err;
-use serde::{Deserialize, Serialize};
 
 #[derive(Serialize, Deserialize, Default)]
 /// Representation of a log entry, can be represented as a human readable string or JSON
@@ -118,4 +118,31 @@ mod tests {
         assert_eq!(json.level, message.level);
         assert_eq!(json.kind, message.kind);
     }
+
+    #[test]
+    /// test defaults for coverage
+    fn message_defaults() {
+        let msg = FeroxMessage::default();
+        assert_eq!(msg.level, String::new());
+        assert_eq!(msg.kind, String::new());
+        assert_eq!(msg.message, String::new());
+        assert_eq!(msg.module, String::new());
+        assert!(msg.time_offset < 0.1);
+    }
+
+    #[test]
+    /// ensure WILDCARD messages serialize to WLD and anything not known to UNK
+    fn message_as_str_edges() {
+        let mut msg = FeroxMessage {
+            message: "message".to_string(),
+            module: "utils".to_string(),
+            time_offset: 1.0,
+            level: "WILDCARD".to_string(),
+            kind: "log".to_string(),
+        };
+        assert!(console::strip_ansi_codes(&msg.as_str()).starts_with("WLD"));
+
+        msg.level = "UNKNOWN".to_string();
+        assert!(console::strip_ansi_codes(&msg.as_str()).starts_with("UNK"));
+    }
 }

src/parser.rs

@@ -1,6 +1,8 @@
 use clap::{App, Arg, ArgGroup};
 use lazy_static::lazy_static;
 use regex::Regex;
+use std::env;
+use std::process;
 
 lazy_static! {
     /// Regex used to validate values passed to --time-limit
@@ -16,7 +18,7 @@ lazy_static! {
 
 /// Create and return an instance of [clap::App](https://docs.rs/clap/latest/clap/struct.App.html), i.e. the Command Line Interface's configuration
 pub fn initialize() -> App<'static, 'static> {
-    App::new("feroxbuster")
+    let mut app = App::new("feroxbuster")
         .version(env!("CARGO_PKG_VERSION"))
         .author("Ben 'epi' Risher (@epi052)")
         .about("A fast, simple, recursive content discovery tool written in Rust")
@@ -130,6 +132,19 @@ pub fn initialize() -> App<'static, 'static> {
                 .takes_value(false)
                 .help("Hide progress bars and banner (good for tmux windows w/ notifications)")
         )
+        .arg(
+            Arg::with_name("auto_tune")
+                .long("auto-tune")
+                .takes_value(false)
+                .conflicts_with("auto_bail")
+                .help("Automatically lower scan rate when an excessive amount of errors are encountered")
+        )
+        .arg(
+            Arg::with_name("auto_bail")
+                .long("auto-bail")
+                .takes_value(false)
+                .help("Automatically stop scanning when an excessive amount of errors are encountered")
+        )
         .arg(
             Arg::with_name("json")
                 .long("json")
@@ -203,6 +218,17 @@ pub fn initialize() -> App<'static, 'static> {
                     "File extension(s) to search for (ex: -x php -x pdf js)",
                 ),
         )
+        .arg(
+            Arg::with_name("url_denylist")
+                .long("dont-scan")
+                .value_name("URL")
+                .takes_value(true)
+                .multiple(true)
+                .use_delimiter(true)
+                .help(
+                    "URL(s) to exclude from recursion/scans",
+                ),
+        )
         .arg(
             Arg::with_name("headers")
                 .short("H")
@@ -335,11 +361,20 @@ pub fn initialize() -> App<'static, 'static> {
                 .takes_value(true)
                 .help("Limit total number of concurrent scans (default: 0, i.e. no limit)")
         )
+        .arg(
+            Arg::with_name("parallel")
+                .long("parallel")
+                .value_name("PARALLEL_SCANS")
+                .takes_value(true)
+                .requires("stdin")
+                .help("Run parallel feroxbuster instances (one child process per url passed via stdin)")
+        )
         .arg(
             Arg::with_name("rate_limit")
                 .long("rate-limit")
                 .value_name("RATE_LIMIT")
                 .takes_value(true)
+                .conflicts_with("auto_tune")
                 .help("Limit number of requests per second (per directory) (default: 0, i.e. no limit)")
         )
         .arg(
@@ -388,7 +423,20 @@ EXAMPLES:
 
     Ludicrous speed... go!
         ./feroxbuster -u http://127.1 -t 200
-    "#)
+    "#);
+
+    for arg in env::args() {
+        // secure-77 noticed that when an incorrect flag/option is used, the short help message is printed
+        // which is fine, but if you add -h|--help, it still errors out on the bad flag/option,
+        // never showing the full help message. This code addresses that behavior
+        if arg == "--help" || arg == "-h" {
+            app.print_long_help().unwrap();
+            println!(); // just a newline to mirror original --help output
+            process::exit(0);
+        }
+    }
+
+    app
 }
 
 /// Validate that a string is formatted as a number followed by s, m, h, or d (10d, 30s, etc...)

src/progress.rs

@@ -51,7 +51,7 @@ pub fn add_bar(prefix: &str, length: u64, bar_type: BarType) -> ProgressBar {
 
     progress_bar.set_style(style);
 
-    progress_bar.set_prefix(&prefix);
+    progress_bar.set_prefix(prefix);
 
     progress_bar
 }

src/response.rs

@@ -121,7 +121,7 @@ impl FeroxResponse {
 
     /// Set `FeroxResponse`'s `url` attribute, has no affect if an error occurs
     pub fn set_url(&mut self, url: &str) {
-        match Url::parse(&url) {
+        match Url::parse(url) {
             Ok(url) => {
                 self.url = url;
             }
@@ -339,7 +339,7 @@ impl FeroxSerialize for FeroxResponse {
                 lines,
                 words,
                 chars,
-                status_colorizer(&status),
+                status_colorizer(status),
                 self.url(),
                 FeroxUrl::path_length_of_url(&self.url)
             );

src/scan_manager/menu.rs

@@ -31,9 +31,10 @@ impl Menu {
         let separator = "─".to_string();
 
         let instructions = format!(
-            "Enter a {} list of indexes to {} (ex: 2,3)",
+            "Enter a {} list of indexes/ranges to {} ({}: 1-4,8,9-13)",
             style("comma-separated").yellow(),
             style("cancel").red(),
+            style("ex").cyan(),
         );
 
         let name = format!(
@@ -43,14 +44,22 @@ impl Menu {
             "💀"
         );
 
+        let force_msg = format!(
+            "Add {} to {} confirmation ({}: 3-5 -f)",
+            style("-f").yellow(),
+            style("skip").yellow(),
+            style("ex").cyan(),
+        );
+
         let longest = measure_text_width(&instructions).max(measure_text_width(&name));
 
         let border = separator.repeat(longest);
 
         let padded_name = pad_str(&name, longest, Alignment::Center, None);
+        let padded_force = pad_str(&force_msg, longest, Alignment::Center, None);
 
         let header = format!("{}\n{}\n{}", border, padded_name, border);
-        let footer = format!("{}\n{}\n{}", border, instructions, border);
+        let footer = format!("{}\n{}\n{}\n{}", border, instructions, padded_force, border);
 
         Self {
             separator,
@@ -93,23 +102,71 @@ impl Menu {
         self.term.write_line(msg).unwrap_or_default();
     }
 
-    /// split a string into vec of usizes
-    pub(super) fn split_to_nums(&self, line: &str) -> Vec<usize> {
-        line.split(',')
-            .map(|s| {
-                s.trim().to_string().parse::<usize>().unwrap_or_else(|e| {
-                    self.println(&format!("Found non-numeric input: {}", e));
-                    0
-                })
+    /// Helper for parsing a usize from a str
+    fn str_to_usize(&self, value: &str) -> usize {
+        if value.is_empty() {
+            return 0;
+        }
+
+        value
+            .trim()
+            .to_string()
+            .parse::<usize>()
+            .unwrap_or_else(|e| {
+                self.println(&format!("Found non-numeric input: {}: {:?}", e, value));
+                0
             })
-            .filter(|m| *m != 0)
-            .collect()
+    }
+
+    /// split a comma delimited string into vec of usizes
+    pub(super) fn split_to_nums(&self, line: &str) -> Vec<usize> {
+        let mut nums = Vec::new();
+        let values = line.split(',');
+
+        for mut value in values {
+            value = value.trim();
+
+            if value.contains('-') {
+                // range of two values, needs further processing
+
+                let range: Vec<usize> = value
+                    .split('-')
+                    .map(|s| self.str_to_usize(s))
+                    .filter(|m| *m != 0)
+                    .collect();
+
+                if range.len() != 2 {
+                    // expecting [1, 4] or similar, if a 0 was used, we'd be left with a vec of size 1
+                    self.println(&format!("Found invalid range of scans: {}", value));
+                    continue;
+                }
+
+                (range[0]..=range[1]).for_each(|n| {
+                    // iterate from lower to upper bound and add all interim values, skipping
+                    // any already known
+                    if !nums.contains(&n) {
+                        nums.push(n)
+                    }
+                });
+            } else {
+                let value = self.str_to_usize(value);
+
+                if value != 0 && !nums.contains(&value) {
+                    // the zeroth scan is always skipped, skip already known values
+                    nums.push(value);
+                }
+            }
+        }
+
+        nums
     }
 
     /// get comma-separated list of scan indexes from the user
-    pub(super) fn get_scans_from_user(&self) -> Option<Vec<usize>> {
+    pub(super) fn get_scans_from_user(&self) -> Option<(Vec<usize>, bool)> {
         if let Ok(line) = self.term.read_line() {
-            Some(self.split_to_nums(&line))
+            let force = line.contains("-f");
+            let line = line.replace("-f", "");
+            Some((self.split_to_nums(&line), force))
         } else {
             None
         }

src/scan_manager/scan.rs

@@ -2,6 +2,7 @@ use super::*;
 use crate::{
     config::OutputLevel,
     progress::{add_bar, BarType},
+    scanner::PolicyTrigger,
 };
 use anyhow::Result;
 use console::style;
@@ -12,8 +13,10 @@ use std::{
     collections::HashMap,
     fmt,
     sync::{Arc, Mutex},
+    time::Instant,
 };
 
+use std::sync::atomic::{AtomicUsize, Ordering};
 use tokio::{sync, task::JoinHandle};
 use uuid::Uuid;
 
@@ -33,7 +36,7 @@ pub struct FeroxScan {
     pub(super) scan_type: ScanType,
 
     /// The order in which the scan was received
-    pub(super) scan_order: ScanOrder,
+    pub(crate) scan_order: ScanOrder,
 
     /// Number of requests to populate the progress bar with
     pub(super) num_requests: u64,
@@ -49,6 +52,18 @@ pub struct FeroxScan {
 
     /// whether or not the user passed --silent|--quiet on the command line
     pub(super) output_level: OutputLevel,
+
+    /// tracker for overall number of 403s seen by the FeroxScan instance
+    pub(super) status_403s: AtomicUsize,
+
+    /// tracker for overall number of 429s seen by the FeroxScan instance
+    pub(super) status_429s: AtomicUsize,
+
+    /// tracker for total number of errors encountered by the FeroxScan instance
+    pub(super) errors: AtomicUsize,
+
+    /// tracker for the time at which this scan was started
+    pub(super) start_time: Instant,
 }
 
 /// Default implementation for FeroxScan
@@ -67,6 +82,10 @@ impl Default for FeroxScan {
             progress_bar: Mutex::new(None),
             scan_type: ScanType::File,
             output_level: Default::default(),
+            errors: Default::default(),
+            status_429s: Default::default(),
+            status_403s: Default::default(),
+            start_time: Instant::now(),
         }
     }
 }
@@ -75,16 +94,22 @@ impl Default for FeroxScan {
 impl FeroxScan {
     /// Stop a currently running scan
     pub async fn abort(&self) -> Result<()> {
-        let mut guard = self.task.lock().await;
+        log::trace!("enter: abort");
 
-        if guard.is_some() {
-            if let Some(task) = std::mem::replace(&mut *guard, None) {
-                task.abort();
-                self.set_status(ScanStatus::Cancelled)?;
-                self.stop_progress_bar();
+        match self.task.try_lock() {
+            Ok(mut guard) => {
+                if let Some(task) = std::mem::replace(&mut *guard, None) {
+                    log::trace!("aborting {:?}", self);
+                    task.abort();
+                    self.set_status(ScanStatus::Cancelled)?;
+                    self.stop_progress_bar();
+                }
+            }
+            Err(e) => {
+                log::warn!("Could not acquire lock to abort scan (we're already waiting for its results): {:?} {}", self, e);
             }
         }
-
+        log::trace!("exit: abort");
         Ok(())
     }
 
@@ -134,6 +159,7 @@ impl FeroxScan {
                     pb.reset_elapsed();
 
                     let _ = std::mem::replace(&mut *guard, Some(pb.clone()));
+
                     pb
                 }
             }
@@ -217,6 +243,61 @@ impl FeroxScan {
 
         log::trace!("exit join({:?})", self);
     }
+    /// increment the value in question by 1
+    pub(crate) fn add_403(&self) {
+        self.status_403s.fetch_add(1, Ordering::Relaxed);
+    }
+
+    /// increment the value in question by 1
+    pub(crate) fn add_429(&self) {
+        self.status_429s.fetch_add(1, Ordering::Relaxed);
+    }
+
+    /// increment the value in question by 1
+    pub(crate) fn add_error(&self) {
+        self.errors.fetch_add(1, Ordering::Relaxed);
+    }
+
+    /// simple wrapper to call the appropriate getter based on the given PolicyTrigger
+    pub fn num_errors(&self, trigger: PolicyTrigger) -> usize {
+        match trigger {
+            PolicyTrigger::Status403 => self.status_403s(),
+            PolicyTrigger::Status429 => self.status_429s(),
+            PolicyTrigger::Errors => self.errors(),
+        }
+    }
+
+    /// return the number of errors seen by this scan
+    fn errors(&self) -> usize {
+        self.errors.load(Ordering::Relaxed)
+    }
+
+    /// return the number of 403s seen by this scan
+    fn status_403s(&self) -> usize {
+        self.status_403s.load(Ordering::Relaxed)
+    }
+
+    /// return the number of 429s seen by this scan
+    fn status_429s(&self) -> usize {
+        self.status_429s.load(Ordering::Relaxed)
+    }
+
+    /// return the number of requests per second performed by this scan's scanner
+    pub fn requests_per_second(&self) -> u64 {
+        if !self.is_active() {
+            return 0;
+        }
+
+        let reqs = self.requests();
+        let seconds = self.start_time.elapsed().as_secs();
+
+        reqs.checked_div(seconds).unwrap_or(0)
+    }
+
+    /// return the number of requests performed by this scan's scanner
+    pub fn requests(&self) -> u64 {
+        self.progress_bar().position()
+    }
 }
 
 /// Display implementation
@@ -360,3 +441,68 @@ impl Default for ScanStatus {
         Self::NotStarted
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::thread::sleep;
+    use tokio::time::Duration;
+
+    #[test]
+    /// ensure that num_errors returns the correct values for the given PolicyTrigger
+    ///
+    /// covers tests for add_[403,429,error] and the related getters in addition to num_errors
+    fn num_errors_returns_correct_values() {
+        let scan = FeroxScan::new(
+            "http://localhost",
+            ScanType::Directory,
+            ScanOrder::Latest,
+            1000,
+            OutputLevel::Default,
+            None,
+        );
+
+        scan.add_error();
+        scan.add_403();
+        scan.add_403();
+        scan.add_429();
+        scan.add_429();
+        scan.add_429();
+
+        assert_eq!(scan.num_errors(PolicyTrigger::Errors), 1);
+        assert_eq!(scan.num_errors(PolicyTrigger::Status403), 2);
+        assert_eq!(scan.num_errors(PolicyTrigger::Status429), 3);
+    }
+
+    #[test]
+    /// ensure that requests_per_second returns the correct values
+    fn requests_per_second_returns_correct_values() {
+        let scan = FeroxScan {
+            id: "".to_string(),
+            url: "".to_string(),
+            scan_type: ScanType::Directory,
+            scan_order: ScanOrder::Initial,
+            num_requests: 0,
+            status: Mutex::new(ScanStatus::Running),
+            task: Default::default(),
+            progress_bar: Mutex::new(None),
+            output_level: Default::default(),
+            status_403s: Default::default(),
+            status_429s: Default::default(),
+            errors: Default::default(),
+            start_time: Instant::now(),
+        };
+
+        let pb = scan.progress_bar();
+        pb.set_position(100);
+
+        sleep(Duration::new(1, 0));
+
+        let req_sec = scan.requests_per_second();
+
+        assert_eq!(req_sec, 100);
+
+        scan.finish().unwrap();
+        assert_eq!(scan.requests_per_second(), 0);
+    }
+}

src/scan_manager/scan_container.rs

@@ -9,6 +9,7 @@ use crate::{
     SLEEP_DURATION,
 };
 use anyhow::Result;
+use reqwest::StatusCode;
 use serde::{ser::SerializeSeq, Serialize, Serializer};
 use std::{
     convert::TryInto,
@@ -161,6 +162,63 @@ impl FeroxScans {
         None
     }
 
+    pub(super) fn get_base_scan_by_url(&self, url: &str) -> Option<Arc<FeroxScan>> {
+        log::trace!("enter: get_sub_paths_from_path({})", url);
+
+        // rmatch_indices returns tuples in index, match form, i.e. (10, "/")
+        // with the furthest-right match in the first position in the vector
+        let matches: Vec<_> = url.rmatch_indices('/').collect();
+
+        // iterate from the furthest right matching index and check the given url from the
+        // start to the furthest-right '/' character. compare that slice to the urls associated
+        // with directory scans and return the first match, since it should be the 'deepest'
+        // match.
+        // Example:
+        //   url: http://shmocalhost/src/release/examples/stuff.php
+        //   scans:
+        //      http://shmocalhost/src/statistics
+        //      http://shmocalhost/src/banner
+        //      http://shmocalhost/src/release
+        //      http://shmocalhost/src/release/examples
+        //
+        //  returns: http://shmocalhost/src/release/examples
+        if let Ok(guard) = self.scans.read() {
+            for (idx, _) in &matches {
+                for scan in guard.iter() {
+                    let slice = url.index(0..*idx);
+                    if slice == scan.url || format!("{}/", slice).as_str() == scan.url {
+                        log::trace!("enter: get_sub_paths_from_path -> {}", scan);
+                        return Some(scan.clone());
+                    }
+                }
+            }
+        }
+
+        log::trace!("enter: get_sub_paths_from_path -> None");
+        None
+    }
+    /// add one to either 403 or 429 tracker in the scan related to the given url
+    pub fn increment_status_code(&self, url: &str, code: StatusCode) {
+        if let Some(scan) = self.get_base_scan_by_url(url) {
+            match code {
+                StatusCode::TOO_MANY_REQUESTS => {
+                    scan.add_429();
+                }
+                StatusCode::FORBIDDEN => {
+                    scan.add_403();
+                }
+                _ => {}
+            }
+        }
+    }
+
+    /// add one to either 403 or 429 tracker in the scan related to the given url
+    pub fn increment_error(&self, url: &str) {
+        if let Some(scan) = self.get_base_scan_by_url(url) {
+            scan.add_error();
+        }
+    }
+
     /// Print all FeroxScans of type Directory
     ///
     /// Example:
@@ -194,9 +252,11 @@ impl FeroxScans {
     }
 
     /// Given a list of indexes, cancel their associated FeroxScans
-    async fn cancel_scans(&self, indexes: Vec<usize>) {
+    async fn cancel_scans(&self, indexes: Vec<usize>, force: bool) -> usize {
         let menu_pause_duration = Duration::from_millis(SLEEP_DURATION);
 
+        let mut num_cancelled = 0_usize;
+
         for num in indexes {
             let selected = match self.scans.read() {
                 Ok(u_scans) => {
@@ -213,36 +273,50 @@ impl FeroxScans {
                 Err(..) => continue,
             };
 
-            let input = self.menu.confirm_cancellation(&selected.url);
+            let input = if force {
+                'y'
+            } else {
+                self.menu.confirm_cancellation(&selected.url)
+            };
 
             if input == 'y' || input == '\n' {
                 self.menu.println(&format!("Stopping {}...", selected.url));
+
                 selected
                     .abort()
                     .await
                     .unwrap_or_else(|e| log::warn!("Could not cancel task: {}", e));
+
+                let pb = selected.progress_bar();
+                num_cancelled += pb.length() as usize - pb.position() as usize
             } else {
                 self.menu.println("Ok, doing nothing...");
             }
 
             sleep(menu_pause_duration);
         }
+
+        num_cancelled
     }
 
     /// CLI menu that allows for interactive cancellation of recursed-into directories
-    async fn interactive_menu(&self) {
+    async fn interactive_menu(&self) -> usize {
         self.menu.hide_progress_bars();
         self.menu.clear_screen();
         self.menu.print_header();
         self.display_scans().await;
         self.menu.print_footer();
 
-        if let Some(input) = self.menu.get_scans_from_user() {
-            self.cancel_scans(input).await
+        let mut num_cancelled = 0_usize;
+
+        if let Some((input, force)) = self.menu.get_scans_from_user() {
+            num_cancelled += self.cancel_scans(input, force).await;
         };
 
         self.menu.clear_screen();
         self.menu.show_progress_bars();
+
+        num_cancelled
     }
 
     /// prints all known responses that the scanner has already seen
@@ -290,18 +364,19 @@ impl FeroxScans {
     ///
     /// When the value stored in `PAUSE_SCAN` becomes `false`, the function returns, exiting the busy
     /// loop
-    pub async fn pause(&self, get_user_input: bool) {
+    pub async fn pause(&self, get_user_input: bool) -> usize {
         // function uses tokio::time, not std
 
         // local testing showed a pretty slow increase (less than linear) in CPU usage as # of
         // concurrent scans rose when SLEEP_DURATION was set to 500, using that as the default for now
         let mut interval = time::interval(time::Duration::from_millis(SLEEP_DURATION));
+        let mut num_cancelled = 0_usize;
 
         if INTERACTIVE_BARRIER.load(Ordering::Relaxed) == 0 {
             INTERACTIVE_BARRIER.fetch_add(1, Ordering::Relaxed);
 
             if get_user_input {
-                self.interactive_menu().await;
+                num_cancelled += self.interactive_menu().await;
                 PAUSE_SCAN.store(false, Ordering::Relaxed);
                 self.print_known_responses();
             }
@@ -318,8 +393,8 @@ impl FeroxScans {
                     INTERACTIVE_BARRIER.fetch_sub(1, Ordering::Relaxed);
                 }
 
-                log::trace!("exit: pause_scan");
-                return;
+                log::trace!("exit: pause_scan -> {}", num_cancelled);
+                return num_cancelled;
             }
         }
     }
@@ -356,7 +431,7 @@ impl FeroxScans {
                     OutputLevel::Silent => BarType::Hidden,
                 };
 
-                let progress_bar = add_bar(&url, bar_length, bar_type);
+                let progress_bar = add_bar(url, bar_length, bar_type);
 
                 progress_bar.reset_elapsed();
 
@@ -366,7 +441,7 @@ impl FeroxScans {
         };
 
         let ferox_scan = FeroxScan::new(
-            &url,
+            url,
             scan_type,
             scan_order,
             bar_length,
@@ -387,7 +462,7 @@ impl FeroxScans {
     ///
     /// Also return a reference to the new `FeroxScan`
     pub fn add_directory_scan(&self, url: &str, scan_order: ScanOrder) -> (bool, Arc<FeroxScan>) {
-        self.add_scan(&url, ScanType::Directory, scan_order)
+        self.add_scan(url, ScanType::Directory, scan_order)
     }
 
     /// Given a url, create a new `FeroxScan` and add it to `FeroxScans` as a File Scan
@@ -396,7 +471,7 @@ impl FeroxScans {
     ///
     /// Also return a reference to the new `FeroxScan`
     pub fn add_file_scan(&self, url: &str, scan_order: ScanOrder) -> (bool, Arc<FeroxScan>) {
-        self.add_scan(&url, ScanType::File, scan_order)
+        self.add_scan(url, ScanType::File, scan_order)
     }
 
     /// small helper to determine whether any scans are active or not

src/scan_manager/state.rs

@@ -47,7 +47,7 @@ impl FeroxSerialize for FeroxState {
 
     /// Simple call to produce a JSON string using the given FeroxState
     fn as_json(&self) -> Result<String> {
-        Ok(serde_json::to_string(&self)
-            .with_context(|| fmt_err("Could not convert scan's running state to JSON"))?)
+        serde_json::to_string(&self)
+            .with_context(|| fmt_err("Could not convert scan's running state to JSON"))
     }
 }

src/scan_manager/tests.rs

@@ -12,6 +12,7 @@ use indicatif::ProgressBar;
 use predicates::prelude::*;
 use std::sync::{atomic::Ordering, Arc};
 use std::thread::sleep;
+use std::time::Instant;
 use tokio::time::{self, Duration};
 
 #[test]
@@ -51,7 +52,7 @@ fn add_url_to_list_of_scanned_urls_with_unknown_url() {
     let urls = FeroxScans::default();
     let url = "http://unknown_url";
     let (result, _scan) = urls.add_scan(url, ScanType::Directory, ScanOrder::Latest);
-    assert_eq!(result, true);
+    assert!(result);
 }
 
 #[test]
@@ -70,11 +71,11 @@ fn add_url_to_list_of_scanned_urls_with_known_url() {
         Some(pb),
     );
 
-    assert_eq!(urls.insert(scan), true);
+    assert!(urls.insert(scan));
 
     let (result, _scan) = urls.add_scan(url, ScanType::Directory, ScanOrder::Latest);
 
-    assert_eq!(result, false);
+    assert!(!result);
 }
 
 #[test]
@@ -92,27 +93,23 @@ fn stop_progress_bar_stops_bar() {
         Some(pb),
     );
 
-    assert_eq!(
-        scan.progress_bar
-            .lock()
-            .unwrap()
-            .as_ref()
-            .unwrap()
-            .is_finished(),
-        false
-    );
+    assert!(!scan
+        .progress_bar
+        .lock()
+        .unwrap()
+        .as_ref()
+        .unwrap()
+        .is_finished());
 
     scan.stop_progress_bar();
 
-    assert_eq!(
-        scan.progress_bar
-            .lock()
-            .unwrap()
-            .as_ref()
-            .unwrap()
-            .is_finished(),
-        true
-    );
+    assert!(scan
+        .progress_bar
+        .lock()
+        .unwrap()
+        .as_ref()
+        .unwrap()
+        .is_finished());
 }
 
 #[test]
@@ -130,11 +127,11 @@ fn add_url_to_list_of_scanned_urls_with_known_url_without_slash() {
         None,
     );
 
-    assert_eq!(urls.insert(scan), true);
+    assert!(urls.insert(scan));
 
     let (result, _scan) = urls.add_scan(url, ScanType::File, ScanOrder::Latest);
 
-    assert_eq!(result, false);
+    assert!(!result);
 }
 
 #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
@@ -170,8 +167,8 @@ async fn call_display_scans() {
         .await
         .unwrap();
 
-    assert_eq!(urls.insert(scan), true);
-    assert_eq!(urls.insert(scan_two), true);
+    assert!(urls.insert(scan));
+    assert!(urls.insert(scan_two));
 
     urls.display_scans().await;
 }
@@ -329,7 +326,7 @@ fn ferox_response_serialize_and_deserialize() {
 
     assert_eq!(response.url().as_str(), "https://nerdcore.com/css");
     assert_eq!(response.url().path(), "/css");
-    assert_eq!(response.wildcard(), true);
+    assert!(response.wildcard());
     assert_eq!(response.status().as_u16(), 301);
     assert_eq!(response.content_length(), 173);
     assert_eq!(response.line_count(), 10);
@@ -382,7 +379,7 @@ fn feroxstates_feroxserialize_implementation() {
 
     let json_state = ferox_state.as_json().unwrap();
     let expected = format!(
-        r#"{{"scans":[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","status":"NotStarted","num_requests":0}}],"config":{{"type":"configuration","wordlist":"/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt","config":"","proxy":"","replay_proxy":"","target_url":"","status_codes":[200,204,301,302,307,308,401,403,405],"replay_codes":[200,204,301,302,307,308,401,403,405],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"silent":false,"quiet":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/{}","redirects":false,"insecure":false,"extensions":[],"headers":{{}},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":false,"depth":4,"scan_limit":0,"rate_limit":0,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false,"resumed":false,"resume_from":"","save_state":false,"time_limit":"","filter_similar":[]}},"responses":[{{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{{"server":"nginx/1.16.1"}}}}]"#,
+        r#"{{"scans":[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","status":"NotStarted","num_requests":0}}],"config":{{"type":"configuration","wordlist":"/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt","config":"","proxy":"","replay_proxy":"","target_url":"","status_codes":[200,204,301,302,307,308,401,403,405,500],"replay_codes":[200,204,301,302,307,308,401,403,405,500],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"silent":false,"quiet":false,"auto_bail":false,"auto_tune":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/{}","redirects":false,"insecure":false,"extensions":[],"headers":{{}},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":false,"depth":4,"scan_limit":0,"parallel":0,"rate_limit":0,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false,"resumed":false,"resume_from":"","save_state":false,"time_limit":"","filter_similar":[],"url_denylist":[]}},"responses":[{{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{{"server":"nginx/1.16.1"}}}}]"#,
         saved_id, VERSION
     );
     println!("{}\n{}", expected, json_state);
@@ -437,10 +434,14 @@ fn feroxscan_display() {
         scan_order: ScanOrder::Latest,
         scan_type: Default::default(),
         num_requests: 0,
+        start_time: Instant::now(),
         output_level: OutputLevel::Default,
+        status_403s: Default::default(),
+        status_429s: Default::default(),
         status: Default::default(),
         task: tokio::sync::Mutex::new(None),
         progress_bar: std::sync::Mutex::new(None),
+        errors: Default::default(),
     };
 
     let not_started = format!("{}", scan);
@@ -477,12 +478,16 @@ async fn ferox_scan_abort() {
         scan_order: ScanOrder::Latest,
         scan_type: Default::default(),
         num_requests: 0,
+        start_time: Instant::now(),
         output_level: OutputLevel::Default,
+        status_403s: Default::default(),
+        status_429s: Default::default(),
         status: std::sync::Mutex::new(ScanStatus::Running),
         task: tokio::sync::Mutex::new(Some(tokio::spawn(async move {
             sleep(Duration::from_millis(SLEEP_DURATION * 2));
         }))),
         progress_bar: std::sync::Mutex::new(None),
+        errors: Default::default(),
     };
 
     scan.abort().await.unwrap();
@@ -512,7 +517,78 @@ fn menu_print_header_and_footer() {
 fn split_to_nums_is_correct() {
     let menu = Menu::new();
 
-    let nums = menu.split_to_nums("1, 3,      4");
+    let nums = menu.split_to_nums("1, 3,      4, 7 -     12, 10-10, 10-11, 9-12, 12-6, -1, 4-");
 
-    assert_eq!(nums, vec![1, 3, 4]);
+    assert_eq!(nums, vec![1, 3, 4, 7, 8, 9, 10, 11, 12]);
+    assert_eq!(menu.split_to_nums("9-12"), vec![9, 10, 11, 12]);
+    assert!(menu.split_to_nums("-12").is_empty());
+    assert!(menu.split_to_nums("12-").is_empty());
+    assert!(menu.split_to_nums("\n").is_empty());
+}
+
+#[test]
+/// given a deep url, find the correct scan
+fn get_base_scan_by_url_finds_correct_scan() {
+    let urls = FeroxScans::default();
+    let url = "http://localhost";
+    let url1 = "http://localhost/stuff";
+    let url2 = "http://shlocalhost/stuff/things";
+    let url3 = "http://shlocalhost/stuff/things/mostuff";
+    let (_, scan) = urls.add_scan(url, ScanType::Directory, ScanOrder::Latest);
+    let (_, scan1) = urls.add_scan(url1, ScanType::Directory, ScanOrder::Latest);
+    let (_, scan2) = urls.add_scan(url2, ScanType::Directory, ScanOrder::Latest);
+    let (_, scan3) = urls.add_scan(url3, ScanType::Directory, ScanOrder::Latest);
+
+    assert_eq!(
+        urls.get_base_scan_by_url("http://localhost/things.php")
+            .unwrap()
+            .id,
+        scan.id
+    );
+    assert_eq!(
+        urls.get_base_scan_by_url("http://localhost/stuff/things.php")
+            .unwrap()
+            .id,
+        scan1.id
+    );
+    assert_eq!(
+        urls.get_base_scan_by_url("http://shlocalhost/stuff/things/mostuff.php")
+            .unwrap()
+            .id,
+        scan2.id
+    );
+    assert_eq!(
+        urls.get_base_scan_by_url("http://shlocalhost/stuff/things/mostuff/mothings.php")
+            .unwrap()
+            .id,
+        scan3.id
+    );
+}
+
+#[test]
+/// given a shallow url without a trailing slash, find the correct scan
+fn get_base_scan_by_url_finds_correct_scan_without_trailing_slash() {
+    let urls = FeroxScans::default();
+    let url = "http://localhost";
+    let (_, scan) = urls.add_scan(url, ScanType::Directory, ScanOrder::Latest);
+    assert_eq!(
+        urls.get_base_scan_by_url("http://localhost/BKPMiherrortBPKcw")
+            .unwrap()
+            .id,
+        scan.id
+    );
+}
+
+#[test]
+/// given a shallow url with a trailing slash, find the correct scan
+fn get_base_scan_by_url_finds_correct_scan_with_trailing_slash() {
+    let urls = FeroxScans::default();
+    let url = "http://127.0.0.1:41971/";
+    let (_, scan) = urls.add_scan(url, ScanType::Directory, ScanOrder::Latest);
+    assert_eq!(
+        urls.get_base_scan_by_url("http://127.0.0.1:41971/BKPMiherrortBPKcw")
+            .unwrap()
+            .id,
+        scan.id
+    );
 }

src/scan_manager/utils.rs

@@ -41,7 +41,7 @@ pub async fn start_max_time_thread(handles: Arc<Handles>) {
         log::trace!("exit: start_max_time_thread");
 
         #[cfg(test)]
-        panic!(handles);
+        panic!("{:?}", handles);
         #[cfg(not(test))]
         let _ = TermInputHandler::sigint_handler(handles.clone());
     }

src/scanner.rs

@@ -1,352 +0,0 @@
-use std::{
-    cmp::max, collections::HashSet, convert::TryInto, ops::Deref, sync::atomic::Ordering,
-    sync::Arc, time::Instant,
-};
-
-use anyhow::{bail, Result};
-use futures::{stream, StreamExt};
-use lazy_static::lazy_static;
-use leaky_bucket::LeakyBucket;
-use tokio::sync::{oneshot, Semaphore};
-
-use crate::{
-    event_handlers::{
-        Command::{self, AddError, UpdateF64Field, UpdateUsizeField},
-        Handles,
-    },
-    extractor::{
-        ExtractionTarget::{ResponseBody, RobotsTxt},
-        ExtractorBuilder,
-    },
-    heuristics,
-    response::FeroxResponse,
-    scan_manager::{FeroxResponses, ScanOrder, ScanStatus, PAUSE_SCAN},
-    statistics::{
-        StatError::Other,
-        StatField::{DirScanTimes, ExpectedPerScan},
-    },
-    url::FeroxUrl,
-    utils::{fmt_err, make_request},
-};
-use tokio::time::Duration;
-
-lazy_static! {
-    /// Vector of FeroxResponse objects
-    pub static ref RESPONSES: FeroxResponses = FeroxResponses::default();
-    // todo consider removing this
-}
-
-/// Makes multiple requests based on the presence of extensions
-struct Requester {
-    /// handles to handlers and config
-    handles: Arc<Handles>,
-
-    /// url that will be scanned
-    target_url: String,
-
-    /// limits requests per second if present
-    rate_limiter: Option<LeakyBucket>,
-}
-
-/// Requester implementation
-impl Requester {
-    /// given a FeroxScanner, create a Requester
-    pub fn from(scanner: &FeroxScanner) -> Result<Self> {
-        let limit = scanner.handles.config.rate_limit;
-        let refill = max(limit / 10, 1); // minimum of 1 per second
-        let tokens = max(limit / 2, 1);
-        let interval = if refill == 1 { 1000 } else { 100 }; // 1 second if refill is 1
-
-        let rate_limiter = if limit > 0 {
-            let bucket = LeakyBucket::builder()
-                .refill_interval(Duration::from_millis(interval)) // add tokens every 0.1s
-                .refill_amount(refill) // ex: 100 req/s -> 10 tokens per 0.1s
-                .tokens(tokens) // reduce initial burst, 2 is arbitrary, but felt good
-                .max(limit)
-                .build()?;
-            Some(bucket)
-        } else {
-            None
-        };
-
-        Ok(Self {
-            rate_limiter,
-            handles: scanner.handles.clone(),
-            target_url: scanner.target_url.to_owned(),
-        })
-    }
-
-    /// limit the number of requests per second
-    pub async fn limit(&self) -> Result<()> {
-        self.rate_limiter.as_ref().unwrap().acquire_one().await?;
-        Ok(())
-    }
-
-    /// Wrapper for [make_request](fn.make_request.html)
-    ///
-    /// Attempts recursion when appropriate and sends Responses to the output handler for processing
-    async fn request(&self, word: &str) -> Result<()> {
-        log::trace!("enter: request({})", word);
-
-        let urls =
-            FeroxUrl::from_string(&self.target_url, self.handles.clone()).formatted_urls(word)?;
-
-        for url in urls {
-            if self.rate_limiter.is_some() {
-                // found a rate limiter, limit that junk!
-                if let Err(e) = self.limit().await {
-                    log::warn!("Could not rate limit scan: {}", e);
-                    self.handles.stats.send(AddError(Other)).unwrap_or_default();
-                }
-            }
-
-            let response = make_request(
-                &self.handles.config.client,
-                &url,
-                self.handles.config.output_level,
-                self.handles.stats.tx.clone(),
-            )
-            .await?;
-
-            // response came back without error, convert it to FeroxResponse
-            let ferox_response =
-                FeroxResponse::from(response, true, self.handles.config.output_level).await;
-
-            // do recursion if appropriate
-            if !self.handles.config.no_recursion {
-                self.handles
-                    .send_scan_command(Command::TryRecursion(Box::new(ferox_response.clone())))?;
-                let (tx, rx) = oneshot::channel::<bool>();
-                self.handles.send_scan_command(Command::Sync(tx))?;
-                rx.await?;
-            }
-
-            // purposefully doing recursion before filtering. the thought process is that
-            // even though this particular url is filtered, subsequent urls may not
-            if self
-                .handles
-                .filters
-                .data
-                .should_filter_response(&ferox_response, self.handles.stats.tx.clone())
-            {
-                continue;
-            }
-
-            if self.handles.config.extract_links && !ferox_response.status().is_redirection() {
-                let extractor = ExtractorBuilder::default()
-                    .target(ResponseBody)
-                    .response(&ferox_response)
-                    .handles(self.handles.clone())
-                    .build()?;
-
-                extractor.extract().await?;
-            }
-
-            // everything else should be reported
-            if let Err(e) = ferox_response.send_report(self.handles.output.tx.clone()) {
-                log::warn!("Could not send FeroxResponse to output handler: {}", e);
-            }
-        }
-
-        log::trace!("exit: request");
-        Ok(())
-    }
-}
-
-/// handles the main muscle movement of scanning a url
-pub struct FeroxScanner {
-    /// handles to handlers and config
-    handles: Arc<Handles>,
-
-    /// url that will be scanned
-    target_url: String,
-
-    /// whether or not this scanner is targeting an initial target specified by the user or one
-    /// found via recursion
-    order: ScanOrder,
-
-    /// wordlist that's already been read from disk
-    wordlist: Arc<HashSet<String>>,
-
-    /// limiter that restricts the number of active FeroxScanners  
-    scan_limiter: Arc<Semaphore>,
-}
-
-/// FeroxScanner implementation
-impl FeroxScanner {
-    /// create a new FeroxScanner
-    pub fn new(
-        target_url: &str,
-        order: ScanOrder,
-        wordlist: Arc<HashSet<String>>,
-        scan_limiter: Arc<Semaphore>,
-        handles: Arc<Handles>,
-    ) -> Self {
-        Self {
-            order,
-            handles,
-            wordlist,
-            scan_limiter,
-            target_url: target_url.to_string(),
-        }
-    }
-
-    /// Scan a given url using a given wordlist
-    ///
-    /// This is the primary entrypoint for the scanner
-    pub async fn scan_url(&self) -> Result<()> {
-        log::trace!("enter: scan_url");
-        log::info!("Starting scan against: {}", self.target_url);
-
-        let scan_timer = Instant::now();
-
-        if matches!(self.order, ScanOrder::Initial) && self.handles.config.extract_links {
-            // only grab robots.txt on the initial scan_url calls. all fresh dirs will be passed
-            // to try_recursion
-            let extractor = ExtractorBuilder::default()
-                .url(&self.target_url)
-                .handles(self.handles.clone())
-                .target(RobotsTxt)
-                .build()?;
-
-            let _ = extractor.extract().await;
-        }
-
-        let scanned_urls = self.handles.ferox_scans()?;
-
-        let ferox_scan = match scanned_urls.get_scan_by_url(&self.target_url) {
-            Some(scan) => {
-                scan.set_status(ScanStatus::Running)?;
-                scan
-            }
-            None => {
-                let msg = format!(
-                    "Could not find FeroxScan associated with {}; this shouldn't happen... exiting",
-                    self.target_url
-                );
-                bail!(fmt_err(&msg))
-            }
-        };
-
-        let progress_bar = ferox_scan.progress_bar();
-
-        // When acquire is called and the semaphore has remaining permits, the function immediately
-        // returns a permit. However, if no remaining permits are available, acquire (asynchronously)
-        // waits until an outstanding permit is dropped, at which point, the freed permit is assigned
-        // to the caller.
-        let _permit = self.scan_limiter.acquire().await;
-
-        // Arc clones to be passed around to the various scans
-        let looping_words = self.wordlist.clone();
-
-        {
-            let test = heuristics::HeuristicTests::new(self.handles.clone());
-            if let Ok(num_reqs) = test.wildcard(&self.target_url).await {
-                progress_bar.inc(num_reqs);
-            }
-        }
-
-        let requester = Arc::new(Requester::from(self)?);
-        let increment_len = (self.handles.config.extensions.len() + 1) as u64;
-
-        // producer tasks (mp of mpsc); responsible for making requests
-        let producers = stream::iter(looping_words.deref().to_owned())
-            .map(|word| {
-                let pb = progress_bar.clone(); // progress bar is an Arc around internal state
-                let scanned_urls_clone = scanned_urls.clone();
-                let requester_clone = requester.clone();
-                (
-                    tokio::spawn(async move {
-                        if PAUSE_SCAN.load(Ordering::Acquire) {
-                            // for every word in the wordlist, check to see if PAUSE_SCAN is set to true
-                            // when true; enter a busy loop that only exits by setting PAUSE_SCAN back
-                            // to false
-                            scanned_urls_clone.pause(true).await;
-                        }
-                        requester_clone.request(&word).await
-                    }),
-                    pb,
-                )
-            })
-            .for_each_concurrent(self.handles.config.threads, |(resp, bar)| async move {
-                match resp.await {
-                    Ok(_) => {
-                        bar.inc(increment_len);
-                    }
-                    Err(e) => {
-                        log::warn!("error awaiting a response: {}", e);
-                        self.handles.stats.send(AddError(Other)).unwrap_or_default();
-                    }
-                }
-            });
-
-        // await tx tasks
-        log::trace!("awaiting scan producers");
-        producers.await;
-        log::trace!("done awaiting scan producers");
-
-        self.handles.stats.send(UpdateF64Field(
-            DirScanTimes,
-            scan_timer.elapsed().as_secs_f64(),
-        ))?;
-
-        ferox_scan.finish()?;
-
-        log::trace!("exit: scan_url");
-
-        Ok(())
-    }
-}
-
-/// Perform steps necessary to run scans that only need to be performed once (warming up the
-/// engine, as it were)
-pub async fn initialize(num_words: usize, handles: Arc<Handles>) -> Result<()> {
-    log::trace!("enter: initialize({}, {:?})", num_words, handles);
-
-    // number of requests only needs to be calculated once, and then can be reused
-    let num_reqs_expected: u64 = if handles.config.extensions.is_empty() {
-        num_words.try_into()?
-    } else {
-        let total = num_words * (handles.config.extensions.len() + 1);
-        total.try_into()?
-    };
-
-    {
-        // no real reason to keep the arc around beyond this call
-        let scans = handles.ferox_scans()?;
-        scans.set_bar_length(num_reqs_expected);
-    }
-
-    // tell Stats object about the number of expected requests
-    handles.stats.send(UpdateUsizeField(
-        ExpectedPerScan,
-        num_reqs_expected as usize,
-    ))?;
-
-    log::trace!("exit: initialize");
-    Ok(())
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::config::OutputLevel;
-    use crate::scan_manager::FeroxScans;
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
-    #[should_panic]
-    /// try to hit struct field coverage of FileOutHandler
-    async fn get_scan_by_url_bails_on_unfound_url() {
-        let sem = Semaphore::new(10);
-        let urls = FeroxScans::new(OutputLevel::Default);
-
-        let scanner = FeroxScanner::new(
-            "http://localhost",
-            ScanOrder::Initial,
-            Arc::new(Default::default()),
-            Arc::new(sem),
-            Arc::new(Handles::for_testing(Some(Arc::new(urls)), None).0),
-        );
-        scanner.scan_url().await.unwrap();
-    }
-}

src/scanner/ferox_scanner.rs

@@ -0,0 +1,186 @@
+use std::{ops::Deref, sync::atomic::Ordering, sync::Arc, time::Instant};
+
+use anyhow::{bail, Result};
+use futures::{stream, StreamExt};
+use lazy_static::lazy_static;
+use tokio::sync::Semaphore;
+
+use crate::{
+    event_handlers::{
+        Command::{AddError, AddToF64Field, SubtractFromUsizeField},
+        Handles,
+    },
+    extractor::{ExtractionTarget::RobotsTxt, ExtractorBuilder},
+    heuristics,
+    scan_manager::{FeroxResponses, ScanOrder, ScanStatus, PAUSE_SCAN},
+    statistics::{
+        StatError::Other,
+        StatField::{DirScanTimes, TotalExpected},
+    },
+    utils::fmt_err,
+};
+
+use super::requester::Requester;
+
+lazy_static! {
+    /// Vector of FeroxResponse objects
+    pub static ref RESPONSES: FeroxResponses = FeroxResponses::default();
+    // todo consider removing this
+}
+/// handles the main muscle movement of scanning a url
+pub struct FeroxScanner {
+    /// handles to handlers and config
+    pub(super) handles: Arc<Handles>,
+
+    /// url that will be scanned
+    pub(super) target_url: String,
+
+    /// whether or not this scanner is targeting an initial target specified by the user or one
+    /// found via recursion
+    order: ScanOrder,
+
+    /// wordlist that's already been read from disk
+    wordlist: Arc<Vec<String>>,
+
+    /// limiter that restricts the number of active FeroxScanners  
+    scan_limiter: Arc<Semaphore>,
+}
+
+/// FeroxScanner implementation
+impl FeroxScanner {
+    /// create a new FeroxScanner
+    pub fn new(
+        target_url: &str,
+        order: ScanOrder,
+        wordlist: Arc<Vec<String>>,
+        scan_limiter: Arc<Semaphore>,
+        handles: Arc<Handles>,
+    ) -> Self {
+        Self {
+            order,
+            handles,
+            wordlist,
+            scan_limiter,
+            target_url: target_url.to_string(),
+        }
+    }
+
+    /// Scan a given url using a given wordlist
+    ///
+    /// This is the primary entrypoint for the scanner
+    pub async fn scan_url(&self) -> Result<()> {
+        log::trace!("enter: scan_url");
+        log::info!("Starting scan against: {}", self.target_url);
+
+        let scan_timer = Instant::now();
+
+        if matches!(self.order, ScanOrder::Initial) && self.handles.config.extract_links {
+            // only grab robots.txt on the initial scan_url calls. all fresh dirs will be passed
+            // to try_recursion
+            let extractor = ExtractorBuilder::default()
+                .url(&self.target_url)
+                .handles(self.handles.clone())
+                .target(RobotsTxt)
+                .build()?;
+
+            let links = extractor.extract().await?;
+            extractor.request_links(links).await?;
+        }
+
+        let scanned_urls = self.handles.ferox_scans()?;
+
+        let ferox_scan = match scanned_urls.get_scan_by_url(&self.target_url) {
+            Some(scan) => {
+                scan.set_status(ScanStatus::Running)?;
+                scan
+            }
+            None => {
+                let msg = format!(
+                    "Could not find FeroxScan associated with {}; this shouldn't happen... exiting",
+                    self.target_url
+                );
+                bail!(fmt_err(&msg))
+            }
+        };
+
+        let progress_bar = ferox_scan.progress_bar();
+
+        // When acquire is called and the semaphore has remaining permits, the function immediately
+        // returns a permit. However, if no remaining permits are available, acquire (asynchronously)
+        // waits until an outstanding permit is dropped, at which point, the freed permit is assigned
+        // to the caller.
+        let _permit = self.scan_limiter.acquire().await;
+
+        // Arc clones to be passed around to the various scans
+        let looping_words = self.wordlist.clone();
+
+        {
+            let test = heuristics::HeuristicTests::new(self.handles.clone());
+            if let Ok(num_reqs) = test.wildcard(&self.target_url).await {
+                progress_bar.inc(num_reqs);
+            }
+        }
+
+        let requester = Arc::new(Requester::from(self, ferox_scan.clone())?);
+        let increment_len = (self.handles.config.extensions.len() + 1) as u64;
+
+        // producer tasks (mp of mpsc); responsible for making requests
+        let producers = stream::iter(looping_words.deref().to_owned())
+            .map(|word| {
+                let pb = progress_bar.clone(); // progress bar is an Arc around internal state
+                let scanned_urls_clone = scanned_urls.clone();
+                let requester_clone = requester.clone();
+                let handles_clone = self.handles.clone();
+                (
+                    tokio::spawn(async move {
+                        if PAUSE_SCAN.load(Ordering::Acquire) {
+                            // for every word in the wordlist, check to see if PAUSE_SCAN is set to true
+                            // when true; enter a busy loop that only exits by setting PAUSE_SCAN back
+                            // to false
+                            let num_cancelled = scanned_urls_clone.pause(true).await;
+                            if num_cancelled > 0 {
+                                handles_clone
+                                    .stats
+                                    .send(SubtractFromUsizeField(TotalExpected, num_cancelled))
+                                    .unwrap_or_else(|e| {
+                                        log::warn!("Could not update overall scan bar: {}", e)
+                                    });
+                            }
+                        }
+                        requester_clone
+                            .request(&word)
+                            .await
+                            .unwrap_or_else(|e| log::warn!("Requester encountered an error: {}", e))
+                    }),
+                    pb,
+                )
+            })
+            .for_each_concurrent(self.handles.config.threads, |(resp, bar)| async move {
+                match resp.await {
+                    Ok(_) => {
+                        bar.inc(increment_len);
+                    }
+                    Err(e) => {
+                        log::warn!("error awaiting a response: {}", e);
+                        self.handles.stats.send(AddError(Other)).unwrap_or_default();
+                    }
+                }
+            });
+
+        // await tx tasks
+        log::trace!("awaiting scan producers");
+        producers.await;
+        log::trace!("done awaiting scan producers");
+
+        self.handles.stats.send(AddToF64Field(
+            DirScanTimes,
+            scan_timer.elapsed().as_secs_f64(),
+        ))?;
+
+        ferox_scan.finish()?;
+
+        log::trace!("exit: scan_url");
+
+        Ok(())
+    }
+}

src/scanner/init.rs

@@ -0,0 +1,34 @@
+use crate::{
+    event_handlers::{Command::AddToUsizeField, Handles},
+    statistics::StatField::ExpectedPerScan,
+};
+use anyhow::Result;
+use std::{convert::TryInto, sync::Arc};
+
+/// Perform steps necessary to run scans that only need to be performed once (warming up the
+/// engine, as it were)
+pub async fn initialize(num_words: usize, handles: Arc<Handles>) -> Result<()> {
+    log::trace!("enter: initialize({}, {:?})", num_words, handles);
+
+    // number of requests only needs to be calculated once, and then can be reused
+    let num_reqs_expected: u64 = if handles.config.extensions.is_empty() {
+        num_words.try_into()?
+    } else {
+        let total = num_words * (handles.config.extensions.len() + 1);
+        total.try_into()?
+    };
+
+    {
+        // no real reason to keep the arc around beyond this call
+        let scans = handles.ferox_scans()?;
+        scans.set_bar_length(num_reqs_expected);
+    }
+
+    // tell Stats object about the number of expected requests
+    handles
+        .stats
+        .send(AddToUsizeField(ExpectedPerScan, num_reqs_expected as usize))?;
+
+    log::trace!("exit: initialize");
+    Ok(())
+}

src/scanner/limit_heap.rs

@@ -0,0 +1,171 @@
+use std::fmt::{Debug, Formatter, Result};
+
+/// bespoke variation on an array-backed max-heap
+///
+/// 255 possible values generated from the initial requests/second
+///
+/// when no additional errors are encountered, the left child is taken (increasing req/sec)
+/// if errors have increased since the last interval, the right child is taken (decreasing req/sec)
+///
+/// formula for each child:
+/// - left: (|parent - current|) / 2 + current
+/// - right: current - ((|parent - current|) / 2)
+pub(super) struct LimitHeap {
+    /// backing array, 255 nodes == height of 7 ( 2^(h+1) -1 nodes )
+    pub(super) inner: [i32; 255],
+
+    /// original # of requests / second
+    pub(super) original: i32,
+
+    /// current position w/in the backing array
+    pub(super) current: usize,
+}
+
+/// default implementation of a LimitHeap
+impl Default for LimitHeap {
+    /// zero-initialize the backing array
+    fn default() -> Self {
+        Self {
+            inner: [0; 255],
+            original: 0,
+            current: 0,
+        }
+    }
+}
+
+/// Debug implementation of a LimitHeap
+impl Debug for LimitHeap {
+    /// return debug representation that conforms to <32 elements in array
+    fn fmt(&self, f: &mut Formatter<'_>) -> Result {
+        let msg = format!(
+            "LimitHeap {{ original: {}, current: {}, inner: [{}...] }}",
+            self.original, self.current, self.inner[0]
+        );
+        write!(f, "{}", msg)
+    }
+}
+
+/// implementation of a LimitHeap
+impl LimitHeap {
+    /// move to right child, return node's index from which the move was requested
+    pub(super) fn move_right(&mut self) -> usize {
+        if self.has_children() {
+            let tmp = self.current;
+            self.current = self.current * 2 + 2;
+            return tmp;
+        }
+        self.current
+    }
+
+    /// move to left child, return node's index from which the move was requested
+    pub(super) fn move_left(&mut self) -> usize {
+        if self.has_children() {
+            let tmp = self.current;
+            self.current = self.current * 2 + 1;
+            return tmp;
+        }
+        self.current
+    }
+
+    /// move to parent, return node's index from which the move was requested
+    pub(super) fn move_up(&mut self) -> usize {
+        if self.has_parent() {
+            let tmp = self.current;
+            self.current = (self.current - 1) / 2;
+            return tmp;
+        }
+        self.current
+    }
+
+    /// move directly to the given index
+    pub(super) fn move_to(&mut self, index: usize) {
+        self.current = index;
+    }
+
+    /// get the current node's value
+    pub(super) fn value(&self) -> i32 {
+        self.inner[self.current]
+    }
+
+    /// set the current node's value
+    pub(super) fn set_value(&mut self, value: i32) {
+        self.inner[self.current] = value;
+    }
+
+    /// check that this node has a parent (true for all except root)
+    pub(super) fn has_parent(&self) -> bool {
+        self.current > 0
+    }
+
+    /// get node's parent's value or self.original if at the root
+    pub(super) fn parent_value(&mut self) -> i32 {
+        if self.has_parent() {
+            let current = self.move_up();
+            let val = self.value();
+            self.move_to(current);
+            return val;
+        }
+        self.original
+    }
+
+    /// check if the current node has children
+    pub(super) fn has_children(&self) -> bool {
+        // inner structure is a complete tree, just check for the right child
+        self.current * 2 + 2 <= self.inner.len()
+    }
+
+    /// get current node's right child's value
+    fn right_child_value(&mut self) -> i32 {
+        let tmp = self.move_right();
+        let val = self.value();
+        self.move_to(tmp);
+        val
+    }
+
+    /// set current node's left child's value
+    fn set_left_child(&mut self) {
+        let parent = self.parent_value();
+        let current = self.value();
+        let value = ((parent - current).abs() / 2) + current;
+
+        self.move_left();
+        self.set_value(value);
+        self.move_up();
+    }
+
+    /// set current node's right child's value
+    fn set_right_child(&mut self) {
+        let parent = self.parent_value();
+        let current = self.value();
+        let value = current - ((parent - current).abs() / 2);
+
+        self.move_right();
+        self.set_value(value);
+        self.move_up();
+    }
+
+    /// iterate over the backing array, filling in each child's value based on the original value
+    pub(super) fn build(&mut self) {
+        // ex: original is 400
+        // arr[0] == 200
+        // arr[1] (left child) == 300
+        // arr[2] (right child) == 100
+        let root = self.original / 2;
+
+        self.inner[0] = root; // set root node to half of the original value
+        self.inner[1] = ((self.original - root).abs() / 2) + root;
+        self.inner[2] = root - ((self.original - root).abs() / 2);
+
+        // start with index 1 and fill in each child below that node
+        for i in 1..self.inner.len() {
+            self.move_to(i);
+
+            if self.has_children() && self.right_child_value() == 0 {
+                // this node has an unset child since the rchild is 0
+                self.set_left_child();
+                self.set_right_child();
+            }
+        }
+        self.move_to(0); // reset current index to the root of the tree
+    }
+}

src/scanner/mod.rs

@@ -0,0 +1,12 @@
+mod ferox_scanner;
+mod utils;
+mod init;
+#[cfg(test)]
+mod tests;
+mod limit_heap;
+mod policy_data;
+mod requester;
+
+pub use self::ferox_scanner::{FeroxScanner, RESPONSES};
+pub use self::init::initialize;
+pub use self::utils::PolicyTrigger;

src/scanner/policy_data.rs

@@ -0,0 +1,306 @@
+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+
+use crate::{atomic_load, atomic_store, config::RequesterPolicy};
+
+use super::limit_heap::LimitHeap;
+
+/// data regarding policy and metadata about last enforced trigger etc...
+#[derive(Default, Debug)]
+pub struct PolicyData {
+    /// how to handle exceptional cases such as too many errors / 403s / 429s etc
+    pub(super) policy: RequesterPolicy,
+
+    /// whether or not we're in the middle of a cooldown period
+    pub(super) cooling_down: AtomicBool,
+
+    /// length of time to pause tuning after making an adjustment
+    pub(super) wait_time: u64,
+
+    /// rate limit (at last interval)
+    limit: AtomicUsize,
+
+    /// number of errors (at last interval)
+    pub(super) errors: AtomicUsize,
+
+    /// whether or not the owning Requester should remove the rate_limiter, happens when a scan
+    /// has been limited and moves back up to the point of its original scan speed
+    pub(super) remove_limit: AtomicBool,
+
+    /// heap of values used for adjusting # of requests/second
+    pub(super) heap: std::sync::RwLock<LimitHeap>,
+}
+
+/// implementation of PolicyData
+impl PolicyData {
+    /// given a RequesterPolicy, create a new PolicyData
+    pub fn new(policy: RequesterPolicy, timeout: u64) -> Self {
+        // can use this as a tweak for how aggressively adjustments should be made when tuning
+        let wait_time = ((timeout as f64 / 2.0) * 1000.0) as u64;
+
+        Self {
+            policy,
+            wait_time,
+            ..Default::default()
+        }
+    }
+
+    /// setter for requests / second; populates the underlying heap with values from req/sec seed
+    pub(super) fn set_reqs_sec(&self, reqs_sec: usize) {
+        if let Ok(mut guard) = self.heap.write() {
+            guard.original = reqs_sec as i32;
+            guard.build();
+            self.set_limit(guard.inner[0] as usize); // set limit to 1/2 of current request rate
+        }
+    }
+
+    /// setter for errors
+    pub(super) fn set_errors(&self, errors: usize) {
+        atomic_store!(self.errors, errors);
+    }
+
+    /// setter for limit
+    fn set_limit(&self, limit: usize) {
+        atomic_store!(self.limit, limit);
+    }
+
+    /// getter for limit
+    pub(super) fn get_limit(&self) -> usize {
+        atomic_load!(self.limit)
+    }
+
+    /// adjust the rate of requests per second up (increase rate)
+    pub(super) fn adjust_up(&self, streak_counter: &usize) {
+        if let Ok(mut heap) = self.heap.try_write() {
+            if *streak_counter > 2 {
+                // streak of 3 upward moves in a row, traverse the tree upward instead of to a
+                // higher-valued branch lower in the tree
+                let current = heap.value();
+                heap.move_up();
+                heap.move_up();
+                if current > heap.value() {
+                    // the tree's structure makes it so that sometimes 2 moves up results in a
+                    // value greater than the current node's and other times we need to move 3 up
+                    // to arrive at a greater value
+                    if heap.has_parent() && heap.parent_value() > current {
+                        // all nodes except 0th node (root)
+                        heap.move_up();
+                    } else if !heap.has_parent() {
+                        // been here enough that we can try resuming the scan to its original
+                        // speed (no limiting at all)
+                        atomic_store!(self.remove_limit, true);
+                    }
+                }
+            } else if heap.has_children() {
+                // streak not at 3, just check that we can move down, and do so
+                heap.move_left();
+            } else {
+                // tree bottomed out, need to move back up the tree a bit
+                let current = heap.value();
+                heap.move_up();
+                heap.move_up();
+
+                if current > heap.value() {
+                    heap.move_up();
+                }
+            }
+            self.set_limit(heap.value() as usize);
+        }
+    }
+
+    /// adjust the rate of requests per second down (decrease rate)
+    pub(super) fn adjust_down(&self) {
+        if let Ok(mut heap) = self.heap.try_write() {
+            if heap.has_children() {
+                heap.move_right();
+                self.set_limit(heap.value() as usize);
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    /// PolicyData builds and sets correct values for the inner heap when set_reqs_sec is called
+    fn set_reqs_sec_builds_heap_and_sets_initial_value() {
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        assert_eq!(pd.wait_time, 3500);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+        assert_eq!(pd.heap.read().unwrap().original, 400);
+        assert_eq!(pd.heap.read().unwrap().current, 0);
+        assert_eq!(pd.heap.read().unwrap().inner[0], 200);
+        assert_eq!(pd.heap.read().unwrap().inner[1], 300);
+        assert_eq!(pd.heap.read().unwrap().inner[2], 100);
+    }
+
+    #[test]
+    /// PolicyData setters/getters tests for code coverage / sanity
+    fn policy_data_getters_and_setters() {
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_errors(20);
+        assert_eq!(pd.errors.load(Ordering::Relaxed), 20);
+        pd.set_limit(200);
+        assert_eq!(pd.get_limit(), 200);
+    }
+
+    #[test]
+    /// PolicyData adjust_down sets the limit to the correct value
+    fn policy_data_adjust_down_simple() {
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+        pd.adjust_down();
+        assert_eq!(pd.get_limit(), 100);
+    }
+
+    #[test]
+    /// PolicyData adjust_down sets the limit to the correct value when no child nodes are present
+    fn policy_data_adjust_down_no_children() {
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+        let mut guard = pd.heap.write().unwrap();
+        guard.move_to(250);
+        guard.set_value(27);
+        pd.set_limit(guard.value() as usize);
+        drop(guard);
+
+        pd.adjust_down();
+        assert_eq!(pd.get_limit(), 27);
+    }
+
+    #[test]
+    /// PolicyData adjust_up sets the limit to the correct value
+    fn policy_data_adjust_up_simple() {
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+        pd.adjust_up(&0);
+        assert_eq!(pd.get_limit(), 300);
+    }
+
+    #[test]
+    /// PolicyData adjust_up sets the limit to the correct value
+    fn policy_data_adjust_up_with_streak_and_2_moves() {
+        // original: 400
+        // [200, 300, 100, 350, 250, 150, 50, 375, 325, 275, 225, 175, 125, 75, 25, ...]
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+
+        // 2 moves
+        pd.heap.write().unwrap().move_to(9);
+        assert_eq!(pd.heap.read().unwrap().value(), 275);
+        pd.adjust_up(&3);
+        assert_eq!(pd.heap.read().unwrap().value(), 300);
+        assert_eq!(pd.limit.load(Ordering::Relaxed), 300);
+        assert!(!pd.remove_limit.load(Ordering::Relaxed));
+    }
+
+    #[test]
+    /// PolicyData adjust_up sets the limit to the correct value
+    fn policy_data_adjust_up_with_streak_and_2_moves_to_arrive_at_root() {
+        // original: 400
+        // [200, 300, 100, 350, 250, 150, 50, 375, 325, 275, 225, 175, 125, 75, 25, ...]
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+
+        pd.heap.write().unwrap().move_to(4);
+        assert_eq!(pd.heap.read().unwrap().value(), 250);
+        pd.adjust_up(&3);
+        assert_eq!(pd.heap.read().unwrap().value(), 200);
+        assert_eq!(pd.limit.load(Ordering::Relaxed), 200);
+        assert!(pd.remove_limit.load(Ordering::Relaxed));
+    }
+
+    #[test]
+    /// PolicyData adjust_up sets the limit to the correct value
+    fn policy_data_adjust_up_with_streak_and_2_moves_to_find_less_than_current() {
+        // original: 400
+        // [200, 300, 100, 350, 250, 150, 50, 375, 325, 275, 225, 175, 125, 75, 25, ...]
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+
+        pd.heap.write().unwrap().move_to(15);
+        assert_eq!(pd.heap.read().unwrap().value(), 387);
+        pd.adjust_up(&3);
+        assert_eq!(pd.heap.read().unwrap().value(), 350);
+        assert_eq!(pd.limit.load(Ordering::Relaxed), 350);
+        assert!(!pd.remove_limit.load(Ordering::Relaxed));
+    }
+
+    #[test]
+    /// PolicyData adjust_up sets the limit to the correct value
+    fn policy_data_adjust_up_with_streak_and_3_moves() {
+        // original: 400
+        // [200, 300, 100, 350, 250, 150, 50, 375, 325, 275, 225, 175, 125, 75, 25, ...]
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+
+        pd.heap.write().unwrap().move_to(19);
+        assert_eq!(pd.heap.read().unwrap().value(), 287);
+        pd.adjust_up(&3);
+        assert_eq!(pd.heap.read().unwrap().value(), 300);
+        assert_eq!(pd.limit.load(Ordering::Relaxed), 300);
+        assert!(!pd.remove_limit.load(Ordering::Relaxed));
+    }
+
+    #[test]
+    /// PolicyData adjust_up sets the limit to the correct value
+    fn policy_data_adjust_up_with_no_children_2_moves() {
+        // original: 400
+        // [200, 300, 100, 350, 250, 150, 50, 375, 325, 275, 225, 175, 125, 75, 25, ...]
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+
+        pd.heap.write().unwrap().move_to(241);
+
+        assert_eq!(pd.heap.read().unwrap().value(), 41);
+        pd.adjust_up(&0);
+        assert_eq!(pd.heap.read().unwrap().value(), 43);
+        assert_eq!(pd.limit.load(Ordering::Relaxed), 43);
+        assert!(!pd.remove_limit.load(Ordering::Relaxed));
+    }
+
+    #[test]
+    /// PolicyData adjust_up sets the limit to the correct value
+    fn policy_data_adjust_up_with_no_children_3_moves() {
+        // original: 400
+        // [200, 300, 100, 350, 250, 150, 50, 375, 325, 275, 225, 175, 125, 75, 25, ...]
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+
+        pd.heap.write().unwrap().move_to(240);
+
+        assert_eq!(pd.heap.read().unwrap().value(), 45);
+        pd.adjust_up(&0);
+        assert_eq!(pd.heap.read().unwrap().value(), 37);
+        assert_eq!(pd.limit.load(Ordering::Relaxed), 37);
+        assert!(!pd.remove_limit.load(Ordering::Relaxed));
+    }
+
+    #[test]
+    /// hit some of the out of the way corners of limitheap for coverage
+    fn increase_limit_heap_coverage_by_hitting_edge_cases() {
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+
+        println!("{:?}", pd.heap.read().unwrap()); // debug derivation
+
+        pd.heap.write().unwrap().move_to(240);
+        assert_eq!(pd.heap.write().unwrap().move_right(), 240);
+        assert_eq!(pd.heap.write().unwrap().move_left(), 240);
+
+        pd.heap.write().unwrap().move_to(0);
+        assert_eq!(pd.heap.write().unwrap().move_up(), 0);
+        assert_eq!(pd.heap.write().unwrap().parent_value(), 400);
+    }
+}

src/scanner/tests.rs

@@ -0,0 +1,28 @@
+use std::sync::Arc;
+
+use tokio::sync::Semaphore;
+
+use crate::{
+    config::OutputLevel,
+    event_handlers::Handles,
+    scan_manager::{FeroxScans, ScanOrder},
+};
+
+use super::*;
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+#[should_panic]
+/// try to hit struct field coverage of FileOutHandler
+async fn get_scan_by_url_bails_on_unfound_url() {
+    let sem = Semaphore::new(10);
+    let urls = FeroxScans::new(OutputLevel::Default);
+
+    let scanner = FeroxScanner::new(
+        "http://localhost",
+        ScanOrder::Initial,
+        Arc::new(Default::default()),
+        Arc::new(sem),
+        Arc::new(Handles::for_testing(Some(Arc::new(urls)), None).0),
+    );
+    scanner.scan_url().await.unwrap();
+}

src/scanner/utils.rs

@@ -0,0 +1,12 @@
+#[derive(Copy, Clone, PartialEq, Debug)]
+/// represents different situations where different criteria can trigger auto-tune/bail behavior
+pub enum PolicyTrigger {
+    /// excessive 403 trigger
+    Status403,
+
+    /// excessive 429 trigger
+    Status429,
+
+    /// excessive general errors
+    Errors,
+}

src/statistics/container.rs

@@ -1,4 +1,6 @@
 use std::{
+    collections::HashMap,
+    convert::TryFrom,
     fs::File,
     io::BufReader,
     sync::{
@@ -9,7 +11,8 @@ use std::{
 
 use anyhow::{Context, Result};
 use reqwest::StatusCode;
-use serde::{Deserialize, Serialize};
+use serde::{ser::SerializeStruct, Deserialize, Deserializer, Serialize, Serializer};
+use serde_json::Value;
 
 use crate::{
     traits::FeroxSerialize,
@@ -19,9 +22,8 @@ use crate::{
 use super::{error::StatError, field::StatField};
 
 /// Data collection of statistics related to a scan
-#[derive(Default, Deserialize, Debug, Serialize)]
+#[derive(Default, Debug)]
 pub struct Stats {
-    #[serde(rename = "type")]
     /// Name of this type of struct, used for serialization, i.e. `{"type":"statistics"}`
     kind: String,
 
@@ -29,7 +31,7 @@ pub struct Stats {
     timeouts: AtomicUsize,
 
     /// tracker for total number of requests sent by the client
-    requests: AtomicUsize,
+    pub(crate) requests: AtomicUsize,
 
     /// tracker for total number of requests expected to send if the scan runs to completion
     ///
@@ -42,7 +44,7 @@ pub struct Stats {
     total_expected: AtomicUsize,
 
     /// tracker for total number of errors encountered by the client
-    errors: AtomicUsize,
+    pub(crate) errors: AtomicUsize,
 
     /// tracker for overall number of 2xx status codes seen by the client
     successes: AtomicUsize,
@@ -58,7 +60,7 @@ pub struct Stats {
 
     /// tracker for number of scans performed, this directly equates to number of directories
     /// recursed into and affects the total number of expected requests
-    total_scans: AtomicUsize,
+    pub(crate) total_scans: AtomicUsize,
 
     /// tracker for initial number of requested targets
     initial_targets: AtomicUsize,
@@ -80,10 +82,10 @@ pub struct Stats {
     status_401s: AtomicUsize,
 
     /// tracker for overall number of 403s seen by the client
-    status_403s: AtomicUsize,
+    pub(crate) status_403s: AtomicUsize,
 
     /// tracker for overall number of 429s seen by the client
-    status_429s: AtomicUsize,
+    pub(crate) status_429s: AtomicUsize,
 
     /// tracker for overall number of 500s seen by the client
     status_500s: AtomicUsize,
@@ -125,11 +127,9 @@ pub struct Stats {
     total_runtime: Mutex<Vec<f64>>,
 
     /// tracker for the number of extensions the user specified
-    #[serde(skip)]
     num_extensions: usize,
 
     /// tracker for whether to use json during serialization or not
-    #[serde(skip)]
     json: bool,
 }
 
@@ -147,6 +147,301 @@ impl FeroxSerialize for Stats {
     }
 }
 
+/// Serialize implementation for Stats
+impl Serialize for Stats {
+    /// Function that handles serialization of Stats
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let mut state = serializer.serialize_struct("Stats", 32)?;
+
+        state.serialize_field("type", &self.kind)?;
+        state.serialize_field("timeouts", &atomic_load!(self.timeouts))?;
+        state.serialize_field("requests", &atomic_load!(self.requests))?;
+        state.serialize_field("expected_per_scan", &atomic_load!(self.expected_per_scan))?;
+        state.serialize_field("total_expected", &atomic_load!(self.total_expected))?;
+        state.serialize_field("errors", &atomic_load!(self.errors))?;
+        state.serialize_field("successes", &atomic_load!(self.successes))?;
+        state.serialize_field("redirects", &atomic_load!(self.redirects))?;
+        state.serialize_field("client_errors", &atomic_load!(self.client_errors))?;
+        state.serialize_field("server_errors", &atomic_load!(self.server_errors))?;
+        state.serialize_field("total_scans", &atomic_load!(self.total_scans))?;
+        state.serialize_field("initial_targets", &atomic_load!(self.initial_targets))?;
+        state.serialize_field("links_extracted", &atomic_load!(self.links_extracted))?;
+        state.serialize_field("status_200s", &atomic_load!(self.status_200s))?;
+        state.serialize_field("status_301s", &atomic_load!(self.status_301s))?;
+        state.serialize_field("status_302s", &atomic_load!(self.status_302s))?;
+        state.serialize_field("status_401s", &atomic_load!(self.status_401s))?;
+        state.serialize_field("status_403s", &atomic_load!(self.status_403s))?;
+        state.serialize_field("status_429s", &atomic_load!(self.status_429s))?;
+        state.serialize_field("status_500s", &atomic_load!(self.status_500s))?;
+        state.serialize_field("status_503s", &atomic_load!(self.status_503s))?;
+        state.serialize_field("status_504s", &atomic_load!(self.status_504s))?;
+        state.serialize_field("status_508s", &atomic_load!(self.status_508s))?;
+        state.serialize_field("wildcards_filtered", &atomic_load!(self.wildcards_filtered))?;
+        state.serialize_field("responses_filtered", &atomic_load!(self.responses_filtered))?;
+        state.serialize_field(
+            "resources_discovered",
+            &atomic_load!(self.resources_discovered),
+        )?;
+        state.serialize_field("url_format_errors", &atomic_load!(self.url_format_errors))?;
+        state.serialize_field("redirection_errors", &atomic_load!(self.redirection_errors))?;
+        state.serialize_field("connection_errors", &atomic_load!(self.connection_errors))?;
+        state.serialize_field("request_errors", &atomic_load!(self.request_errors))?;
+        state.serialize_field("directory_scan_times", &self.directory_scan_times)?;
+        state.serialize_field("total_runtime", &self.total_runtime)?;
+
+        state.end()
+    }
+}
+
+/// Deserialize implementation for Stats
+impl<'a> Deserialize<'a> for Stats {
+    /// Deserialize a Stats object from a serde_json::Value
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'a>,
+    {
+        let stats = Self::new(0, false);
+
+        let map: HashMap<String, Value> = HashMap::deserialize(deserializer)?;
+
+        for (key, value) in &map {
+            match key.as_str() {
+                "timeouts" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.timeouts, parsed);
+                        }
+                    }
+                }
+                "requests" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.requests, parsed);
+                        }
+                    }
+                }
+                "expected_per_scan" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.expected_per_scan, parsed);
+                        }
+                    }
+                }
+                "total_expected" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.total_expected, parsed);
+                        }
+                    }
+                }
+                "errors" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.errors, parsed);
+                        }
+                    }
+                }
+                "successes" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.successes, parsed);
+                        }
+                    }
+                }
+                "redirects" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.redirects, parsed);
+                        }
+                    }
+                }
+                "client_errors" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.client_errors, parsed);
+                        }
+                    }
+                }
+                "server_errors" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.server_errors, parsed);
+                        }
+                    }
+                }
+                "total_scans" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.total_scans, parsed);
+                        }
+                    }
+                }
+                "initial_targets" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.initial_targets, parsed);
+                        }
+                    }
+                }
+                "links_extracted" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.links_extracted, parsed);
+                        }
+                    }
+                }
+                "status_200s" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.status_200s, parsed);
+                        }
+                    }
+                }
+                "status_301s" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.status_301s, parsed);
+                        }
+                    }
+                }
+                "status_302s" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.status_302s, parsed);
+                        }
+                    }
+                }
+                "status_401s" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.status_401s, parsed);
+                        }
+                    }
+                }
+                "status_403s" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.status_403s, parsed);
+                        }
+                    }
+                }
+                "status_429s" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.status_429s, parsed);
+                        }
+                    }
+                }
+                "status_500s" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.status_500s, parsed);
+                        }
+                    }
+                }
+                "status_503s" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.status_503s, parsed);
+                        }
+                    }
+                }
+                "status_504s" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.status_504s, parsed);
+                        }
+                    }
+                }
+                "status_508s" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.status_508s, parsed);
+                        }
+                    }
+                }
+                "wildcards_filtered" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.wildcards_filtered, parsed);
+                        }
+                    }
+                }
+                "responses_filtered" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.responses_filtered, parsed);
+                        }
+                    }
+                }
+                "resources_discovered" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.resources_discovered, parsed);
+                        }
+                    }
+                }
+                "url_format_errors" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.url_format_errors, parsed);
+                        }
+                    }
+                }
+                "redirection_errors" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.redirection_errors, parsed);
+                        }
+                    }
+                }
+                "connection_errors" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.connection_errors, parsed);
+                        }
+                    }
+                }
+                "request_errors" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.request_errors, parsed);
+                        }
+                    }
+                }
+                "directory_scan_times" => {
+                    if let Some(arr) = value.as_array() {
+                        for val in arr {
+                            if let Some(parsed) = val.as_f64() {
+                                if let Ok(mut guard) = stats.directory_scan_times.lock() {
+                                    guard.push(parsed)
+                                }
+                            }
+                        }
+                    }
+                }
+                "total_runtime" => {
+                    if let Some(arr) = value.as_array() {
+                        for val in arr {
+                            if let Some(parsed) = val.as_f64() {
+                                if let Ok(mut guard) = stats.total_runtime.lock() {
+                                    guard.push(parsed)
+                                }
+                            }
+                        }
+                    }
+                }
+                _ => {}
+            }
+        }
+
+        Ok(stats)
+    }
+}
+
 /// implementation of statistics data collection struct
 impl Stats {
     /// Small wrapper for default to set `kind` to "statistics" and `total_runtime` to have at least
@@ -176,6 +471,16 @@ impl Stats {
         atomic_load!(self.errors)
     }
 
+    /// public getter for status_403s
+    pub fn status_403s(&self) -> usize {
+        atomic_load!(self.status_403s)
+    }
+
+    /// public getter for status_429s
+    pub fn status_429s(&self) -> usize {
+        atomic_load!(self.status_429s)
+    }
+
     /// public getter for total_expected
     pub fn total_expected(&self) -> usize {
         atomic_load!(self.total_expected)
@@ -222,10 +527,6 @@ impl Stats {
             StatError::Timeout => {
                 atomic_increment!(self.timeouts);
             }
-            StatError::Status403 => {
-                atomic_increment!(self.status_403s);
-                atomic_increment!(self.client_errors);
-            }
             StatError::UrlFormat => {
                 atomic_increment!(self.url_format_errors);
             }
@@ -238,9 +539,7 @@ impl Stats {
             StatError::Request => {
                 atomic_increment!(self.request_errors);
             }
-            StatError::Other => {
-                atomic_increment!(self.errors);
-            }
+            _ => {} // no need to hit Other as we always increment self.errors anyway
         }
     }
 
@@ -248,7 +547,7 @@ impl Stats {
     ///
     /// Implies incrementing:
     ///     - requests
-    ///     - status_403s (when code is 403)
+    ///     - appropriate status_* codes
     ///     - errors (when code is [45]xx)
     pub fn add_status_code(&self, status: StatusCode) {
         self.add_request();
@@ -264,9 +563,6 @@ impl Stats {
         }
 
         match status {
-            StatusCode::FORBIDDEN => {
-                atomic_increment!(self.status_403s);
-            }
             StatusCode::OK => {
                 atomic_increment!(self.status_200s);
             }
@@ -279,6 +575,9 @@ impl Stats {
             StatusCode::UNAUTHORIZED => {
                 atomic_increment!(self.status_401s);
             }
+            StatusCode::FORBIDDEN => {
+                atomic_increment!(self.status_403s);
+            }
             StatusCode::TOO_MANY_REQUESTS => {
                 atomic_increment!(self.status_429s);
             }
@@ -307,6 +606,13 @@ impl Stats {
         }
     }
 
+    /// subtract a value from the given field
+    pub fn subtract_from_usize_field(&self, field: StatField, value: usize) {
+        if let StatField::TotalExpected = field {
+            self.total_expected.fetch_sub(value, Ordering::Relaxed);
+        }
+    }
+
     /// Update a `Stats` field of type usize
     pub fn update_usize_field(&self, field: StatField, value: usize) {
         match field {
@@ -435,30 +741,6 @@ mod tests {
         Ok(())
     }
 
-    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
-    /// when sent StatCommand::AddRequest, stats object should reflect the change
-    ///
-    /// incrementing a 403 (tracked in status_403s) should also increment:
-    ///     - errors
-    ///     - requests
-    ///     - client_errors
-    async fn statistics_handler_increments_403() {
-        let (task, handle) = setup_stats_test();
-
-        let err = Command::AddError(StatError::Status403);
-        let err2 = Command::AddError(StatError::Status403);
-
-        handle.tx.send(err).unwrap_or_default();
-        handle.tx.send(err2).unwrap_or_default();
-
-        teardown_stats_test(handle.tx.clone(), task).await;
-
-        assert_eq!(handle.data.errors.load(Ordering::Relaxed), 2);
-        assert_eq!(handle.data.requests.load(Ordering::Relaxed), 2);
-        assert_eq!(handle.data.status_403s.load(Ordering::Relaxed), 2);
-        assert_eq!(handle.data.client_errors.load(Ordering::Relaxed), 2);
-    }
-
     #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// when sent StatCommand::AddRequest, stats object should reflect the change
     ///
@@ -567,7 +849,7 @@ mod tests {
 
         stats.merge_from(tfile.path().to_str().unwrap()).unwrap();
 
-        // as of 1.11.1; all Stats fields are accounted for whether they're updated in merge_from
+        // as of 2.1.0; all Stats fields are accounted for whether they're updated in merge_from
         // or not
         assert_eq!(atomic_load!(stats.timeouts), 1);
         assert_eq!(atomic_load!(stats.requests), 9207);
@@ -617,4 +899,22 @@ mod tests {
         stats.update_runtime(20.2);
         assert!((stats.total_runtime.lock().unwrap()[0] - 20.2).abs() < f64::EPSILON);
     }
+
+    #[test]
+    /// ensure status_403s returns the correct value
+    fn status_403s_returns_correct_value() {
+        let config = Configuration::new().unwrap();
+        let stats = Stats::new(config.extensions.len(), config.json);
+        stats.status_403s.store(12, Ordering::Relaxed);
+        assert_eq!(stats.status_403s(), 12);
+    }
+
+    #[test]
+    /// ensure status_403s returns the correct value
+    fn status_429s_returns_correct_value() {
+        let config = Configuration::new().unwrap();
+        let stats = Stats::new(config.extensions.len(), config.json);
+        stats.status_429s.store(141, Ordering::Relaxed);
+        assert_eq!(stats.status_429s(), 141);
+    }
 }

src/statistics/error.rs

@@ -1,9 +1,6 @@
 #[derive(Debug, Copy, Clone)]
 /// Enum variants used to inform the `StatCommand` protocol what `Stats` fields should be updated
 pub enum StatError {
-    /// Represents a 403 response code
-    Status403,
-
     /// Represents a timeout error
     Timeout,
 

src/statistics/macros.rs

@@ -20,4 +20,18 @@ macro_rules! atomic_load {
     ($metric:expr) => {
         $metric.load(Ordering::Relaxed);
     };
+    ($metric:expr, $ordering:expr) => {
+        $metric.load($ordering);
+    };
+}
+
+/// Wrapper around `Atomic*.store` to save me from writing Ordering::Relaxed a bajillion times
+#[macro_export]
+macro_rules! atomic_store {
+    ($metric:expr, $value:expr) => {
+        $metric.store($value, Ordering::Relaxed);
+    };
+    ($metric:expr, $value:expr, $ordering:expr) => {
+        $metric.store($value, $ordering);
+    };
 }

src/statistics/tests.rs

@@ -55,10 +55,7 @@ fn save_writes_stats_object_to_disk() {
     stats.add_status_code(StatusCode::OK);
     stats.add_status_code(StatusCode::OK);
     let outfile = NamedTempFile::new().unwrap();
-    if stats
-        .save(174.33, &outfile.path().to_str().unwrap())
-        .is_ok()
-    {}
+    if stats.save(174.33, outfile.path().to_str().unwrap()).is_ok() {}
 
     assert!(stats.as_json().unwrap().contains("statistics"));
     assert!(stats.as_json().unwrap().contains("11")); // requests made

src/url.rs

@@ -66,7 +66,7 @@ impl FeroxUrl {
     pub fn format(&self, word: &str, extension: Option<&str>) -> Result<Url> {
         log::trace!("enter: format({}, {:?})", word, extension);
 
-        if Url::parse(&word).is_ok() {
+        if Url::parse(word).is_ok() {
             // when a full url is passed in as a word to be joined to a base url using
             // reqwest::Url::join, the result is that the word (url) completely overwrites the base
             // url, potentially resulting in requests to places that aren't actually the target

src/utils.rs

@@ -1,18 +1,24 @@
 use anyhow::{bail, Context, Result};
 use console::{strip_ansi_codes, style, user_attended};
 use indicatif::ProgressBar;
-use reqwest::{Client, Response, Url};
+use reqwest::{Client, Response, StatusCode, Url};
 #[cfg(not(target_os = "windows"))]
-use rlimit::{getrlimit, setrlimit, Resource, Rlim};
+use rlimit::{getrlimit, setrlimit, Resource};
 use std::{
     fs,
     io::{self, BufWriter, Write},
+    sync::Arc,
+    time::Duration,
+    time::{SystemTime, UNIX_EPOCH},
 };
 use tokio::sync::mpsc::UnboundedSender;
 
 use crate::{
     config::OutputLevel,
-    event_handlers::Command::{self, AddError, AddStatus},
+    event_handlers::{
+        Command::{self, AddError, AddStatus},
+        Handles,
+    },
     progress::PROGRESS_PRINTER,
     send_command,
     statistics::StatError::{Connection, Other, Redirection, Request, Timeout},
@@ -81,6 +87,35 @@ pub fn ferox_print(msg: &str, bar: &ProgressBar) {
     }
 }
 
+/// wrapper for make_request used to pass error/response codes to FeroxScans for per-scan stats
+/// tracking of information related to auto-tune/bail
+pub async fn logged_request(url: &Url, handles: Arc<Handles>) -> Result<Response> {
+    let client = &handles.config.client;
+    let level = handles.config.output_level;
+    let tx_stats = handles.stats.tx.clone();
+
+    let response = make_request(client, url, level, tx_stats).await;
+
+    let scans = handles.ferox_scans()?;
+
+    match response {
+        Ok(resp) => {
+            match resp.status() {
+                StatusCode::TOO_MANY_REQUESTS | StatusCode::FORBIDDEN => {
+                    scans.increment_status_code(url.as_str(), resp.status());
+                }
+                _ => {}
+            }
+            Ok(resp)
+        }
+        Err(e) => {
+            log::warn!("err: {:?}", e);
+            scans.increment_error(url.as_str());
+            bail!(e)
+        }
+    }
+}
+
 /// Initiate request to the given `Url` using `Client`
 pub async fn make_request(
     client: &Client,
@@ -184,16 +219,15 @@ pub fn create_report_string(
 /// as the adjustment made here is only valid for the scan itself (and any child processes, of which
 /// there are none).
 #[cfg(not(target_os = "windows"))]
-pub fn set_open_file_limit(limit: usize) -> bool {
+pub fn set_open_file_limit(limit: u64) -> bool {
     log::trace!("enter: set_open_file_limit");
 
     if let Ok((soft, hard)) = getrlimit(Resource::NOFILE) {
-        if hard.as_usize() > limit {
+        if hard > limit {
             // our default open file limit is less than the current hard limit, this means we can
             // set the soft limit to our default
-            let new_soft_limit = Rlim::from_usize(limit);
 
-            if setrlimit(Resource::NOFILE, new_soft_limit, hard).is_ok() {
+            if setrlimit(Resource::NOFILE, limit, hard).is_ok() {
                 log::debug!("set open file descriptor limit to {}", limit);
 
                 log::trace!("exit: set_open_file_limit -> {}", true);
@@ -254,15 +288,141 @@ where
     Ok(())
 }
 
+/// determines whether or not a given url should be denied based on the user-supplied --dont-scan
+/// flag
+pub fn should_deny_url(url: &Url, handles: Arc<Handles>) -> Result<bool> {
+    log::trace!(
+        "enter: should_deny_url({}, {:?}, {:?})",
+        url.as_str(),
+        handles.config.url_denylist,
+        handles.ferox_scans()?
+    );
+    // normalization for comparison is to remove the trailing / if one exists, this is done for
+    // the given url and any url to which it's compared
+    let normed_url = Url::parse(url.to_string().trim_end_matches('/'))?;
+
+    for deny_url in &handles.config.url_denylist {
+        // parse the denying url for easier comparison
+        let denier = Url::parse(deny_url.trim_end_matches('/'))
+            .with_context(|| format!("Could not parse {} as a url", deny_url))?;
+
+        // simplest case is an exact match, check for it first
+        if normed_url == denier {
+            log::trace!("exit: should_deny_url -> true");
+            return Ok(true);
+        }
+
+        match (normed_url.host(), denier.host()) {
+            // .host() will return an enum with ipv4|6 or domain and is comparable
+            // whereas .domain() returns None for ip addresses
+            (Some(normed_host), Some(denier_host)) => {
+                if normed_host != denier_host {
+                    // domains don't even match, keep on keepin' on...
+                    continue;
+                }
+            }
+            _ => {
+                // one or the other couldn't determine the host value, which probably means
+                // it's not suitable for further comparison
+                continue;
+            }
+        }
+
+        let normed_host = normed_url.host().unwrap(); // match above will catch errors
+
+        // at this point, we have a matching set of ips or domain names. now we can process the
+        // url path. The goal is to determine whether the given url's path is a subpath of any
+        // url in the deny list, for example
+        //    GIVEN URL                        URL DENY LIST               USER-SPECIFIED URLS TO SCAN
+        //    http://some.domain/stuff/things, [http://some.domain/stuff], [http://some.domain] => true
+        //    http://some.domain/stuff/things, [http://some.domain/stuff/things], [http://some.domain] => true
+        //    http://some.domain/stuff/things, [http://some.domain/api], [http://some.domain] => false
+        // the examples above are all pretty obvious, the kicker comes when the blocking url's
+        // path is a parent to a scanned url
+        //    http://some.domain/stuff/things, [http://some.domain/], [http://some.domain/stuff] => false
+        //    http://some.domain/api, [http://some.domain/], [http://some.domain/stuff] => true
+        // we want to deny all children of the parent, unless that child is a child of a scan
+        // we specified through -u(s) or --stdin
+
+        let deny_path = denier.path();
+        let norm_path = normed_url.path();
+
+        if norm_path.starts_with(deny_path) {
+            // at this point, we know that the given normalized path is a sub-path of the
+            // current deny-url, now we just need to check to see if this deny-url is a parent
+            // to a scanned url that is also a parent of the given url
+            for ferox_scan in handles.ferox_scans()?.get_active_scans() {
+                let scanner = Url::parse(ferox_scan.url().trim_end_matches('/'))
+                    .with_context(|| format!("Could not parse {} as a url", ferox_scan))?;
+
+                if let Some(scan_host) = scanner.host() {
+                    // same domain/ip check we perform on the denier above
+                    if normed_host != scan_host {
+                        // domains don't even match, keep on keepin' on...
+                        continue;
+                    }
+                } else {
+                    // couldn't process .host from scanner
+                    continue;
+                };
+
+                let scan_path = scanner.path();
+
+                if scan_path.starts_with(deny_path) && norm_path.starts_with(scan_path) {
+                    // user-specified scan url is a sub-path of the deny-urls's path AND the
+                    // url to check is a sub-path of the user-specified scan url
+                    //
+                    // the assumption is the user knew what they wanted and we're going to give
+                    // the scanned url precedence, even though it's a sub-path
+                    log::trace!("exit: should_deny_url -> false");
+                    return Ok(false);
+                }
+            }
+            log::trace!("exit: should_deny_url -> true");
+            return Ok(true);
+        }
+    }
+    log::trace!("exit: should_deny_url -> false");
+    Ok(false)
+}
+
+/// given a url and filename-suffix, return a unique filename comprised of the slugified url,
+/// current unix timestamp and suffix
+///
+/// ex: ferox-http_telsa_com-1606947491.state
+pub fn slugify_filename(url: &str, prefix: &str, suffix: &str) -> String {
+    log::trace!("enter: slugify({:?}, {:?}, {:?})", url, prefix, suffix);
+
+    let ts = SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .unwrap_or_else(|_| Duration::from_secs(0))
+        .as_secs();
+
+    let altered_prefix = if !prefix.is_empty() {
+        format!("{}-", prefix)
+    } else {
+        String::new()
+    };
+
+    let slug = url.replace("://", "_").replace("/", "_").replace(".", "_");
+
+    let filename = format!("{}{}-{}.{}", altered_prefix, slug, ts, suffix);
+
+    log::trace!("exit: slugify -> {}", filename);
+    filename
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::config::Configuration;
+    use crate::scan_manager::{FeroxScans, ScanOrder};
 
     #[test]
     /// set_open_file_limit with a low requested limit succeeds
     fn utils_set_open_file_limit_with_low_requested_limit() {
         let (_, hard) = getrlimit(Resource::NOFILE).unwrap();
-        let lower_limit = hard.as_usize() - 1;
+        let lower_limit = hard - 1;
         assert!(set_open_file_limit(lower_limit));
     }
 
@@ -270,9 +430,9 @@ mod tests {
     /// set_open_file_limit with a high requested limit succeeds
     fn utils_set_open_file_limit_with_high_requested_limit() {
         let (_, hard) = getrlimit(Resource::NOFILE).unwrap();
-        let higher_limit = hard.as_usize() + 1;
+        let higher_limit = hard + 1;
         // calculate a new soft to ensure soft != hard and hit that logic branch
-        let new_soft = Rlim::from_usize(hard.as_usize() - 1);
+        let new_soft = hard - 1;
         setrlimit(Resource::NOFILE, new_soft, hard).unwrap();
         assert!(set_open_file_limit(higher_limit));
     }
@@ -283,7 +443,7 @@ mod tests {
         let (_, hard) = getrlimit(Resource::NOFILE).unwrap();
         // calculate a new soft to ensure soft == hard and hit the failure logic branch
         setrlimit(Resource::NOFILE, hard, hard).unwrap();
-        assert!(!set_open_file_limit(hard.as_usize())); // returns false
+        assert!(!set_open_file_limit(hard)); // returns false
     }
 
     #[test]
@@ -333,4 +493,180 @@ mod tests {
     fn status_colorizer_returns_as_is() {
         assert_eq!(status_colorizer("farfignewton"), "farfignewton".to_string());
     }
+
+    #[test]
+    /// provide a url that should be blocked where the denier is an exact match for the tested url
+    /// expect true
+    fn should_deny_url_blocks_when_denier_is_exact_match() {
+        let scan_url = "https://testdomain.com/";
+        let deny_url = "https://testdomain.com/denied";
+        let tested_url = Url::parse("https://testdomain.com/denied/").unwrap();
+
+        let scans = Arc::new(FeroxScans::default());
+        scans.add_directory_scan(scan_url, ScanOrder::Initial);
+
+        let mut config = Configuration::new().unwrap();
+        config.url_denylist = vec![String::from(deny_url)];
+        let config = Arc::new(config);
+
+        let handles = Arc::new(Handles::for_testing(Some(scans), Some(config)).0);
+
+        assert!(should_deny_url(&tested_url, handles).unwrap());
+    }
+
+    #[test]
+    /// provide a url that has a different host than the denier but the same path, expect false
+    fn should_deny_url_doesnt_compare_mismatched_domains() {
+        let scan_url = "https://testdomain.com/";
+        let deny_url = "https://dev.testdomain.com/denied";
+        let tested_url = Url::parse("https://testdomain.com/denied/").unwrap();
+
+        let scans = Arc::new(FeroxScans::default());
+        scans.add_directory_scan(scan_url, ScanOrder::Initial);
+
+        let mut config = Configuration::new().unwrap();
+        config.url_denylist = vec![String::from(deny_url)];
+        let config = Arc::new(config);
+
+        let handles = Arc::new(Handles::for_testing(Some(scans), Some(config)).0);
+
+        assert!(!should_deny_url(&tested_url, handles).unwrap());
+    }
+
+    #[test]
+    /// provide a denier from which we can't check a host, which results in no comparison, expect false
+    fn should_deny_url_doesnt_compare_non_domains() {
+        let scan_url = "https://testdomain.com/";
+        let deny_url = "unix:/run/foo.socket";
+        let tested_url = Url::parse("https://testdomain.com/denied/").unwrap();
+
+        let scans = Arc::new(FeroxScans::default());
+        scans.add_directory_scan(scan_url, ScanOrder::Initial);
+
+        let mut config = Configuration::new().unwrap();
+        config.url_denylist = vec![String::from(deny_url)];
+        let config = Arc::new(config);
+
+        let handles = Arc::new(Handles::for_testing(Some(scans), Some(config)).0);
+
+        assert!(!should_deny_url(&tested_url, handles).unwrap());
+    }
+
+    #[test]
+    /// provide a url that has a different host than the denier but the same path, expect false
+    /// because the denier is a parent to the tested, even tho the scanned doesn't compare, it
+    /// still returns true
+    fn should_deny_url_doesnt_compare_mismatched_domains_in_scanned() {
+        let deny_url = "https://testdomain.com/";
+        let scan_url = "https://dev.testdomain.com/denied";
+        let tested_url = Url::parse("https://testdomain.com/denied/").unwrap();
+
+        let scans = Arc::new(FeroxScans::default());
+        scans.add_directory_scan(scan_url, ScanOrder::Initial);
+
+        let mut config = Configuration::new().unwrap();
+        config.url_denylist = vec![String::from(deny_url)];
+        let config = Arc::new(config);
+
+        let handles = Arc::new(Handles::for_testing(Some(scans), Some(config)).0);
+
+        assert!(should_deny_url(&tested_url, handles).unwrap());
+    }
+
+    #[test]
+    /// provide a denier from which we can't check a host, which results in no comparison, expect false
+    /// because the denier is a parent to the tested, even tho the scanned doesn't compare, it
+    /// still returns true
+    fn should_deny_url_doesnt_compare_non_domains_in_scanned() {
+        let deny_url = "https://testdomain.com/";
+        let scan_url = "unix:/run/foo.socket";
+        let tested_url = Url::parse("https://testdomain.com/denied/").unwrap();
+
+        let scans = Arc::new(FeroxScans::default());
+        scans.add_directory_scan(scan_url, ScanOrder::Initial);
+
+        let mut config = Configuration::new().unwrap();
+        config.url_denylist = vec![String::from(deny_url)];
+        let config = Arc::new(config);
+
+        let handles = Arc::new(Handles::for_testing(Some(scans), Some(config)).0);
+
+        assert!(should_deny_url(&tested_url, handles).unwrap());
+    }
+
+    #[test]
+    /// provide a denier where the tested url is a sub-path and the scanned url is not, expect true
+    fn should_deny_url_blocks_child() {
+        let scan_url = "https://testdomain.com/";
+        let deny_url = "https://testdomain.com/api";
+        let tested_url = Url::parse("https://testdomain.com/api/denied/").unwrap();
+
+        let scans = Arc::new(FeroxScans::default());
+        scans.add_directory_scan(scan_url, ScanOrder::Initial);
+
+        let mut config = Configuration::new().unwrap();
+        config.url_denylist = vec![String::from(deny_url)];
+        let config = Arc::new(config);
+
+        let handles = Arc::new(Handles::for_testing(Some(scans), Some(config)).0);
+
+        assert!(should_deny_url(&tested_url, handles).unwrap());
+    }
+
+    #[test]
+    /// provide a denier where the tested url is not a sub-path and the scanned url is not, expect false
+    fn should_deny_url_doesnt_block_non_child() {
+        let scan_url = "https://testdomain.com/";
+        let deny_url = "https://testdomain.com/api";
+        let tested_url = Url::parse("https://testdomain.com/not-denied/").unwrap();
+
+        let scans = Arc::new(FeroxScans::default());
+        scans.add_directory_scan(scan_url, ScanOrder::Initial);
+
+        let mut config = Configuration::new().unwrap();
+        config.url_denylist = vec![String::from(deny_url)];
+        let config = Arc::new(config);
+
+        let handles = Arc::new(Handles::for_testing(Some(scans), Some(config)).0);
+
+        assert!(!should_deny_url(&tested_url, handles).unwrap());
+    }
+
+    #[test]
+    /// provide a denier where the tested url is a sub-path and the scanned url is not, expect true
+    fn should_deny_url_blocks_child_when_scan_url_isnt_parent() {
+        let scan_url = "https://testdomain.com/api";
+        let deny_url = "https://testdomain.com/";
+        let tested_url = Url::parse("https://testdomain.com/stuff/").unwrap();
+
+        let scans = Arc::new(FeroxScans::default());
+        scans.add_directory_scan(scan_url, ScanOrder::Initial);
+
+        let mut config = Configuration::new().unwrap();
+        config.url_denylist = vec![String::from(deny_url)];
+        let config = Arc::new(config);
+
+        let handles = Arc::new(Handles::for_testing(Some(scans), Some(config)).0);
+
+        assert!(should_deny_url(&tested_url, handles).unwrap());
+    }
+
+    #[test]
+    /// provide a denier where the tested url is not a sub-path and the scanned url is not, expect false
+    fn should_deny_url_doesnt_block_child_when_scan_url_is_parent() {
+        let scan_url = "https://testdomain.com/api";
+        let deny_url = "https://testdomain.com/";
+        let tested_url = Url::parse("https://testdomain.com/api/not-denied/").unwrap();
+
+        let scans = Arc::new(FeroxScans::default());
+        scans.add_directory_scan(scan_url, ScanOrder::Initial);
+
+        let mut config = Configuration::new().unwrap();
+        config.url_denylist = vec![String::from(deny_url)];
+        let config = Arc::new(config);
+
+        let handles = Arc::new(Handles::for_testing(Some(scans), Some(config)).0);
+
+        assert!(!should_deny_url(&tested_url, handles).unwrap());
+    }
 }

tests/test_banner.rs

@@ -113,6 +113,36 @@ fn banner_prints_headers() {
         );
 }
 
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + multiple dont scan entries
+fn banner_prints_denied_urls() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--dont-scan")
+        .arg("http://dont-scan.me")
+        .arg("--dont-scan")
+        .arg("https://also-not.me")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Don't Scan"))
+                .and(predicate::str::contains("http://dont-scan.me"))
+                .and(predicate::str::contains("https://also-not.me"))
+                .and(predicate::str::contains("─┴─")),
+        );
+}
+
 #[test]
 /// test allows non-existent wordlist to trigger the banner printing to stderr
 /// expect to see all mandatory prints + multiple size filters
@@ -849,6 +879,58 @@ fn banner_prints_rate_limit() {
         );
 }
 
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + auto tune
+fn banner_prints_auto_tune() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--auto-tune")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Auto Tune"))
+                .and(predicate::str::contains("│ true"))
+                .and(predicate::str::contains("─┴─")),
+        );
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + auto bail
+fn banner_prints_auto_bail() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--auto-bail")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Auto Bail"))
+                .and(predicate::str::contains("│ true"))
+                .and(predicate::str::contains("─┴─")),
+        );
+}
+
 #[test]
 /// test allows non-existent wordlist to trigger the banner printing to stderr
 /// expect to see no banner output
@@ -896,3 +978,27 @@ fn banner_doesnt_print_when_quiet() {
                 .and(predicate::str::contains("User-Agent").not()),
         );
 }
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see nothing as --parallel forces --silent to be true
+fn banner_prints_parallel() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--stdin")
+        .arg("--parallel")
+        .arg("4316")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .not()
+                .and(predicate::str::contains("Target Url").not())
+                .and(predicate::str::contains("Parallel Scans").not())
+                .and(predicate::str::contains("Threads").not())
+                .and(predicate::str::contains("Wordlist").not())
+                .and(predicate::str::contains("Status Codes").not())
+                .and(predicate::str::contains("Timeout (secs)").not())
+                .and(predicate::str::contains("User-Agent").not()),
+        );
+}

tests/test_deny_list.rs

@@ -0,0 +1,212 @@
+mod utils;
+use assert_cmd::prelude::*;
+use assert_cmd::Command;
+use httpmock::Method::GET;
+use httpmock::MockServer;
+use predicates::prelude::*;
+use utils::{setup_tmp_directory, teardown_tmp_directory};
+
+#[test]
+/// test that the deny list prevents a request if the requested url is a match
+fn deny_list_works_during_with_a_normal_scan() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--dont-scan")
+        .arg(srv.url("/LICENSE"))
+        .unwrap();
+
+    teardown_tmp_directory(tmp_dir);
+
+    cmd.assert()
+        .success()
+        .stdout(predicate::str::contains(srv.url("/LICENSE")).not());
+
+    assert_eq!(mock.hits(), 0);
+}
+
+#[test]
+/// test that the deny list prevents requests of urls found during extraction
+fn deny_list_works_during_extraction() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200)
+            .body(&srv.url("'/homepage/assets/img/icons/handshake.svg'"));
+    });
+
+    let mock_two = srv.mock(|when, then| {
+        when.method(GET)
+            .path("/homepage/assets/img/icons/handshake.svg");
+        then.status(200);
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--extract-links")
+        .arg("--dont-scan")
+        .arg(srv.url("/homepage/"))
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .and(predicate::str::contains("200"))
+            .and(predicate::str::contains("/homepage/assets/img/icons/handshake.svg").not()),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_two.hits(), 0);
+    teardown_tmp_directory(tmp_dir);
+}
+
+#[test]
+/// test that the deny list prevents requests of urls found during recursion
+fn deny_list_works_during_recursion() {
+    let srv = MockServer::start();
+    let urls = [
+        "js".to_string(),
+        "prod".to_string(),
+        "dev".to_string(),
+        "file.js".to_string(),
+    ];
+    let (tmp_dir, file) = setup_tmp_directory(&urls, "wordlist").unwrap();
+
+    let js_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js");
+        then.status(301).header("Location", &srv.url("/js/"));
+    });
+
+    let js_prod_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js/prod");
+        then.status(301).header("Location", &srv.url("/js/prod/"));
+    });
+
+    let js_dev_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js/dev");
+        then.status(301).header("Location", &srv.url("/js/dev/"));
+    });
+
+    let js_dev_file_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js/dev/file.js");
+        then.status(200)
+            .body("this is a test and is more bytes than other ones");
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-t")
+        .arg("1")
+        .arg("--dont-scan")
+        .arg(srv.url("/js/dev"))
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::is_match("301.*js")
+            .unwrap()
+            .and(predicate::str::is_match("301.*js/prod").unwrap())
+            .and(predicate::str::is_match("301.*js/dev").unwrap())
+            .not()
+            .and(predicate::str::is_match("200.*js/dev/file.js").unwrap())
+            .not(),
+    );
+
+    assert_eq!(js_mock.hits(), 1);
+    assert_eq!(js_prod_mock.hits(), 1);
+    assert_eq!(js_dev_mock.hits(), 0);
+    assert_eq!(js_dev_file_mock.hits(), 0);
+
+    teardown_tmp_directory(tmp_dir);
+}
+
+#[test]
+/// test that the deny list prevents requests of urls found during recursion when the denier is a
+/// parent of a user-specified scan
+fn deny_list_works_during_recursion_with_inverted_parents() {
+    let srv = MockServer::start();
+    let urls = [
+        "js".to_string(),
+        "prod".to_string(),
+        "dev".to_string(),
+        "api".to_string(),
+        "file.js".to_string(),
+    ];
+    let (tmp_dir, file) = setup_tmp_directory(&urls, "wordlist").unwrap();
+
+    let js_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js");
+        then.status(301).header("Location", &srv.url("/js/"));
+    });
+
+    let api_mock = srv.mock(|when, then| {
+        when.method(GET).path("/api");
+        then.status(200);
+    });
+
+    let js_prod_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js/prod");
+        then.status(301).header("Location", &srv.url("/js/prod/"));
+    });
+
+    let js_dev_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js/dev");
+        then.status(301).header("Location", &srv.url("/js/dev/"));
+    });
+
+    let js_dev_file_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js/dev/file.js");
+        then.status(200)
+            .body("this is a test and is more bytes than other ones");
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/js"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-t")
+        .arg("1")
+        .arg("-vvvv")
+        .arg("--dont-scan")
+        .arg(srv.url("/"))
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::is_match("301.*js")
+            .unwrap()
+            .and(predicate::str::is_match("301.*js/prod").unwrap())
+            .and(predicate::str::is_match("301.*js/dev").unwrap())
+            .and(predicate::str::is_match("200.*js/dev/file.js").unwrap())
+            .and(predicate::str::is_match("200.*api").unwrap())
+            .not(),
+    );
+
+    assert_eq!(js_mock.hits(), 1);
+    assert_eq!(js_prod_mock.hits(), 1);
+    assert_eq!(js_dev_mock.hits(), 1);
+    assert_eq!(js_dev_file_mock.hits(), 1);
+    assert_eq!(api_mock.hits(), 0);
+
+    teardown_tmp_directory(tmp_dir);
+}

tests/test_heuristics.rs

@@ -224,11 +224,11 @@ fn test_dynamic_wildcard_request_found() {
 
     teardown_tmp_directory(tmp_dir);
 
-    assert_eq!(contents.contains("WLD"), true);
-    assert_eq!(contents.contains("Got"), true);
-    assert_eq!(contents.contains("200"), true);
-    assert_eq!(contents.contains("(url length: 32)"), true);
-    assert_eq!(contents.contains("(url length: 96)"), true);
+    assert!(contents.contains("WLD"));
+    assert!(contents.contains("Got"));
+    assert!(contents.contains("200"));
+    assert!(contents.contains("(url length: 32)"));
+    assert!(contents.contains("(url length: 96)"));
 
     cmd.assert().success().stdout(
         predicate::str::contains("WLD")
@@ -391,11 +391,11 @@ fn heuristics_wildcard_test_with_two_static_wildcards_and_output_to_file() {
 
     teardown_tmp_directory(tmp_dir);
 
-    assert_eq!(contents.contains("WLD"), true);
-    assert_eq!(contents.contains("Got"), true);
-    assert_eq!(contents.contains("200"), true);
-    assert_eq!(contents.contains("(url length: 32)"), true);
-    assert_eq!(contents.contains("(url length: 96)"), true);
+    assert!(contents.contains("WLD"));
+    assert!(contents.contains("Got"));
+    assert!(contents.contains("200"));
+    assert!(contents.contains("(url length: 32)"));
+    assert!(contents.contains("(url length: 96)"));
 
     cmd.assert().success().stdout(
         predicate::str::contains("WLD")
@@ -451,12 +451,12 @@ fn heuristics_wildcard_test_with_redirect_as_response_code(
 
     teardown_tmp_directory(tmp_dir);
 
-    assert_eq!(contents.contains("WLD"), true);
-    assert_eq!(contents.contains("301"), true);
-    assert_eq!(contents.contains("/some-redirect"), true);
-    assert_eq!(contents.contains("redirects to => "), true);
-    assert_eq!(contents.contains(&srv.url("/")), true);
-    assert_eq!(contents.contains("(url length: 32)"), true);
+    assert!(contents.contains("WLD"));
+    assert!(contents.contains("301"));
+    assert!(contents.contains("/some-redirect"));
+    assert!(contents.contains("redirects to => "));
+    assert!(contents.contains(&srv.url("/")));
+    assert!(contents.contains("(url length: 32)"));
 
     cmd.assert().success().stdout(
         predicate::str::contains("redirects to => ")

tests/test_main.rs

@@ -1,8 +1,9 @@
 mod utils;
 use assert_cmd::Command;
 use httpmock::Method::GET;
-use httpmock::MockServer;
+use httpmock::{MockServer, Regex};
 use predicates::prelude::*;
+use std::fs::{read_dir, read_to_string};
 use utils::{setup_tmp_directory, teardown_tmp_directory};
 
 #[test]
@@ -89,3 +90,136 @@ fn main_use_empty_stdin_targets() -> Result<(), Box<dyn std::error::Error>> {
 
     Ok(())
 }
+
+#[test]
+/// send three targets over stdin, expect parallel to spawn children and each child config to show
+/// up in the output file
+fn main_parallel_spawns_children() -> Result<(), Box<dyn std::error::Error>> {
+    let t1 = MockServer::start();
+    let t2 = MockServer::start();
+    let t3 = MockServer::start();
+
+    let words = [
+        String::from("LICENSE"),
+        String::from("stuff"),
+        String::from("things"),
+        String::from("mostuff"),
+        String::from("mothings"),
+    ];
+    let (word_tmp_dir, wordlist) = setup_tmp_directory(&words, "wordlist")?;
+    let (output_dir, outfile) = setup_tmp_directory(&[], "output-file")?;
+    let (tgt_tmp_dir, targets) =
+        setup_tmp_directory(&[t1.url("/"), t2.url("/"), t3.url("/")], "targets")?;
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--stdin")
+        .arg("--parallel")
+        .arg("2")
+        .arg("-vvvv")
+        .arg("--debug-log")
+        .arg(outfile.as_os_str())
+        .arg("--wordlist")
+        .arg(wordlist.as_os_str())
+        .pipe_stdin(targets)
+        .unwrap()
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("Could not connect to any target provided")
+                .and(predicate::str::contains("Target Url"))
+                .not(), // no target url found
+        );
+
+    let contents = read_to_string(outfile).unwrap();
+    println!("contents: {}", contents);
+
+    assert!(contents.contains("parallel branch && wrapped main")); // exits parallel branch
+
+    // DBG      0.007 feroxbuster parallel exec: target/debug/feroxbuster
+    //   --debug-log /tmp/.tmpAjRts6/output-file --wordlist /tmp/.tmpS4CKKq/wordlist
+    //   --silent -u http://127.0.0.1:41979/
+    let r1 = Regex::new(&format!("parallel exec:.*-u {}", t1.url("/"))).unwrap();
+    let r2 = Regex::new(&format!("parallel exec:.*-u {}", t2.url("/"))).unwrap();
+    let r3 = Regex::new(&format!("parallel exec:.*-u {}", t3.url("/"))).unwrap();
+
+    assert!(r1.is_match(&contents)); // all 3 were spawned
+    assert!(r2.is_match(&contents));
+    assert!(r3.is_match(&contents));
+
+    teardown_tmp_directory(word_tmp_dir);
+    teardown_tmp_directory(tgt_tmp_dir);
+    teardown_tmp_directory(output_dir);
+
+    Ok(())
+}
+
+#[test]
+/// send three targets over stdin with --output enabled, expect parallel to create a new directory
+/// and the log files therein
+fn main_parallel_creates_output_directory() -> Result<(), Box<dyn std::error::Error>> {
+    let t1 = MockServer::start();
+    let t2 = MockServer::start();
+    let t3 = MockServer::start();
+
+    let words = [
+        String::from("LICENSE"),
+        String::from("stuff"),
+        String::from("things"),
+        String::from("mostuff"),
+        String::from("mothings"),
+    ];
+    let (word_tmp_dir, wordlist) = setup_tmp_directory(&words, "wordlist")?;
+    let (output_dir, outfile) = setup_tmp_directory(&[], "output-file")?;
+    let (tgt_tmp_dir, targets) =
+        setup_tmp_directory(&[t1.url("/"), t2.url("/"), t3.url("/")], "targets")?;
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--stdin")
+        .arg("--parallel")
+        .arg("2")
+        .arg("--output")
+        .arg(outfile.as_os_str())
+        .arg("--wordlist")
+        .arg(wordlist.as_os_str())
+        .pipe_stdin(targets)
+        .unwrap()
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("Could not connect to any target provided")
+                .and(predicate::str::contains("Target Url"))
+                .not(), // no target url found
+        );
+
+    // output_dir should return something similar to output-file-1627845244.logs with the
+    // line below. if it ever fails, can use the regex below to filter out the right directory
+    let sub_dir = read_dir(&output_dir)?.next().unwrap()?.file_name();
+
+    let mut num_logs = 0;
+    let file_regex = Regex::new("ferox-[a-zA-Z_:0-9]+-[0-9]+.log").unwrap();
+    let dir_regex = Regex::new("output-file-[0-9]+.logs").unwrap();
+
+    let sub_dir = output_dir.as_ref().join(&sub_dir);
+
+    // created directory like output-file-1627845741.logs/
+    assert!(dir_regex.is_match(&sub_dir.to_string_lossy().to_string()));
+
+    for entry in sub_dir.read_dir()? {
+        let entry = entry?;
+        // created each file like ferox-https_localhost-1627845741.log
+        println!("name: {:?}", entry.file_name().to_string_lossy());
+        assert!(file_regex.is_match(&entry.file_name().to_string_lossy()));
+        num_logs += 1;
+    }
+
+    // should be 3 log files total
+    assert_eq!(num_logs, 3);
+
+    teardown_tmp_directory(word_tmp_dir);
+    teardown_tmp_directory(tgt_tmp_dir);
+    teardown_tmp_directory(output_dir);
+
+    Ok(())
+}

tests/test_parser.rs

@@ -0,0 +1,46 @@
+use assert_cmd::Command;
+use predicates::prelude::*;
+
+#[test]
+/// specify an incorrect param (-fc) with --help after it on the command line
+/// old behavior printed
+/// error: Found argument '-c' which wasn't expected, or isn't valid in this context
+///
+/// USAGE:
+///     feroxbuster --add-slash --url <URL>...
+///
+/// For more information try --help
+///
+/// the new behavior we expect to see is to print the long form help message, of which
+/// Ludicrous speed... go! is near the bottom of that output, so we can test for that
+fn parser_incorrect_param_with_tack_tack_help() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("-fc")
+        .arg("--help")
+        .assert()
+        .success()
+        .stdout(predicate::str::contains("Ludicrous speed... go!"));
+}
+
+#[test]
+/// specify an incorrect param (-fc) with --help after it on the command line
+/// old behavior printed
+/// error: Found argument '-c' which wasn't expected, or isn't valid in this context
+///
+/// USAGE:
+///     feroxbuster --add-slash --url <URL>...
+///
+/// For more information try --help
+///
+/// the new behavior we expect to see is to print the long form help message, of which
+/// Ludicrous speed... go! is near the bottom of that output, so we can test for that
+fn parser_incorrect_param_with_tack_h() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("-fc")
+        .arg("-h")
+        .assert()
+        .success()
+        .stdout(predicate::str::contains("Ludicrous speed... go!"));
+}

tests/test_policies.rs

@@ -0,0 +1,432 @@
+mod utils;
+use assert_cmd::prelude::*;
+use httpmock::Method::GET;
+use httpmock::MockServer;
+use regex::Regex;
+use std::fs::{read_to_string, write};
+use std::path::Path;
+use std::process::Command;
+use std::time::Instant;
+use tokio::time::Duration;
+use utils::{setup_tmp_directory, teardown_tmp_directory};
+
+// tests/policy-test-error-words is a wordlist with the following attributes:
+// - 60 errors per error category (error, 403, 429)
+// - 1000 words tagged as normal for noise/padding
+// - each error string is 6_RANDOM_ASCII{error,status403,status429,normal}6_RANDOM_ASCII
+// examples:
+// - BKPMiherrortBPKcw
+// - lTjbLpstatus403fZQaFD
+// - ZhGBHGstatus429SIUZvI
+// - ufzEXWnormalOLhbLM
+// these words will be used along with pattern matching to trigger different policies
+
+#[test]
+/// --auto-bail should cancel a scan with spurious errors  
+fn auto_bail_cancels_scan_with_timeouts() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["ignored".to_string()], "wordlist").unwrap();
+    let (log_dir, logfile) = setup_tmp_directory(&[], "debug-log").unwrap();
+
+    let policy_words = read_to_string(Path::new("tests/policy-test-words.shuffled")).unwrap();
+
+    write(&file, policy_words).unwrap();
+
+    assert_eq!(file.metadata().unwrap().len(), 117720); // sanity check on wordlist size
+
+    let error_mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z]{6}error[a-zA-Z]{6}").unwrap());
+        then.delay(Duration::new(3, 0))
+            .status(200)
+            .body("verboten, nerd");
+    });
+
+    let other_errors_mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z]{6}(status429|status403)[a-zA-Z]{6}").unwrap());
+        then.status(200).body("other errors are a 200");
+    });
+
+    let normal_reqs_mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z]{6}normal[a-zA-Z]{6}").unwrap());
+        then.status(200).body("any normal request is a 200");
+    });
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--auto-bail")
+        .arg("--dont-filter")
+        .arg("--timeout")
+        .arg("2")
+        .arg("--threads")
+        .arg("4")
+        .arg("--debug-log")
+        .arg(logfile.as_os_str())
+        .arg("-vvvv")
+        .arg("--json")
+        .assert()
+        .success();
+
+    let debug_log = read_to_string(logfile).unwrap();
+
+    // read debug log to get the number of errors enforced
+    for line in debug_log.lines() {
+        let log: serde_json::Value = serde_json::from_str(line).unwrap_or_default();
+        if let Some(message) = log.get("message") {
+            let str_msg = message.as_str().unwrap_or_default().to_string();
+
+            if str_msg.starts_with("Stats") {
+                let re = Regex::new("total_expected: ([0-9]+),").unwrap();
+                assert!(re.is_match(&str_msg));
+                let total_expected = re
+                    .captures(&str_msg)
+                    .unwrap()
+                    .get(1)
+                    .map_or("", |m| m.as_str())
+                    .parse::<usize>()
+                    .unwrap();
+
+                println!("expected: {}", total_expected);
+                // without bailing, should be 6180; after bail decreases significantly
+                assert!(total_expected < 5000);
+            }
+        }
+    }
+
+    teardown_tmp_directory(tmp_dir);
+    teardown_tmp_directory(log_dir);
+
+    assert!(normal_reqs_mock.hits() < 6000); // not all requests should make it
+    assert!(error_mock.hits() >= 25); // need at least 25 to trigger the policy
+    assert!(other_errors_mock.hits() <= 120); // may or may not see all other error requests
+}
+
+#[test]
+/// --auto-bail should cancel a scan with spurious 403s
+fn auto_bail_cancels_scan_with_403s() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["ignored".to_string()], "wordlist").unwrap();
+    let (log_dir, logfile) = setup_tmp_directory(&[], "debug-log").unwrap();
+
+    let policy_words = read_to_string(Path::new("tests/policy-test-words.shuffled")).unwrap();
+
+    write(&file, policy_words).unwrap();
+
+    assert_eq!(file.metadata().unwrap().len(), 117720); // sanity check on wordlist size
+
+    let error_mock = srv.mock(|when, then| {
+        when.method(GET).path_matches(
+            Regex::new("/[a-zA-Z]{6}(error|status429|status403)[a-zA-Z]{6}").unwrap(),
+        );
+        then.status(200).body("other errors are still a 200");
+    });
+
+    let normal_reqs_mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z]{6}normal[a-zA-Z]{6}").unwrap());
+        then.status(403)
+            .body("these guys need to be 403 in order to trigger 90% threshold");
+    });
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--auto-bail")
+        .arg("--dont-filter")
+        .arg("--threads")
+        .arg("4")
+        .arg("--debug-log")
+        .arg(logfile.as_os_str())
+        .arg("-vvvv")
+        .arg("--json")
+        .assert()
+        .success();
+
+    println!("log filesize: {}", logfile.metadata().unwrap().len());
+    let debug_log = read_to_string(logfile).unwrap();
+
+    // read debug log to get the number of errors enforced
+    for line in debug_log.lines() {
+        let log: serde_json::Value = serde_json::from_str(line).unwrap_or_default();
+        if let Some(message) = log.get("message") {
+            let str_msg = message.as_str().unwrap_or_default().to_string();
+
+            if str_msg.starts_with("Stats") {
+                println!("{}", str_msg);
+                let re = Regex::new("total_expected: ([0-9]+),").unwrap();
+                assert!(re.is_match(&str_msg));
+                let total_expected = re
+                    .captures(&str_msg)
+                    .unwrap()
+                    .get(1)
+                    .map_or("", |m| m.as_str())
+                    .parse::<usize>()
+                    .unwrap();
+                println!("total_expected: {}", total_expected);
+                assert!(total_expected < 5000);
+            }
+        }
+    }
+
+    teardown_tmp_directory(tmp_dir);
+    teardown_tmp_directory(log_dir);
+
+    assert!(normal_reqs_mock.hits() + error_mock.hits() > 25); // must have at least 50 reqs fly
+
+    // expect much less in the way of requests for this one, 90% is measured against requests made,
+    // not requests expected, so 90% can be reached very quickly. for the same reason, the
+    // num_enforced can be less than 50
+    assert!(normal_reqs_mock.hits() < 500);
+    assert!(error_mock.hits() <= 180); // may or may not see all other error requests
+}
+
+#[test]
+/// --auto-bail should cancel a scan with spurious 429s
+fn auto_bail_cancels_scan_with_429s() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["ignored".to_string()], "wordlist").unwrap();
+    let (log_dir, logfile) = setup_tmp_directory(&[], "debug-log").unwrap();
+
+    let policy_words = read_to_string(Path::new("tests/policy-test-words.shuffled")).unwrap();
+
+    write(&file, policy_words).unwrap();
+
+    assert_eq!(file.metadata().unwrap().len(), 117720); // sanity check on wordlist size
+
+    let error_mock = srv.mock(|when, then| {
+        when.method(GET).path_matches(
+            Regex::new("/[a-zA-Z]{6}(error|status429|status403)[a-zA-Z]{6}").unwrap(),
+        );
+        then.status(200).body("other errors are still a 200");
+    });
+
+    let normal_reqs_mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z]{6}normal[a-zA-Z]{6}").unwrap());
+        then.status(429)
+            .body("these guys need to be 403 in order to trigger 90% threshold");
+    });
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--auto-bail")
+        .arg("--dont-filter")
+        .arg("--threads")
+        .arg("4")
+        .arg("--debug-log")
+        .arg(logfile.as_os_str())
+        .arg("-vvvv")
+        .arg("--json")
+        .assert()
+        .success();
+
+    println!("log filesize: {}", logfile.metadata().unwrap().len());
+    let debug_log = read_to_string(logfile).unwrap();
+
+    // read debug log to get the number of errors enforced
+    for line in debug_log.lines() {
+        let log: serde_json::Value = serde_json::from_str(line).unwrap_or_default();
+        if let Some(message) = log.get("message") {
+            let str_msg = message.as_str().unwrap_or_default().to_string();
+
+            if str_msg.starts_with("Stats") {
+                println!("{}", str_msg);
+                let re = Regex::new("total_expected: ([0-9]+),").unwrap();
+                assert!(re.is_match(&str_msg));
+                let total_expected = re
+                    .captures(&str_msg)
+                    .unwrap()
+                    .get(1)
+                    .map_or("", |m| m.as_str())
+                    .parse::<usize>()
+                    .unwrap();
+                println!("total_expected: {}", total_expected);
+                assert!(total_expected < 5000);
+            }
+        }
+    }
+
+    teardown_tmp_directory(tmp_dir);
+    teardown_tmp_directory(log_dir);
+
+    assert!(normal_reqs_mock.hits() + error_mock.hits() > 25); // must have at least 50 reqs fly
+
+    // expect much less in the way of requests for this one, 90% is measured against requests made,
+    // not requests expected, so 90% can be reached very quickly. for the same reason, the
+    // num_enforced can be less than 50
+    assert!(normal_reqs_mock.hits() < 500);
+    assert!(error_mock.hits() <= 180); // may or may not see all other error requests
+}
+
+#[test]
+/// --auto-tune should slow a scan with spurious 429s
+fn auto_tune_slows_scan_with_429s() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["ignored".to_string()], "wordlist").unwrap();
+
+    let policy_words = read_to_string(Path::new("tests/policy-test-words.shuffled")).unwrap();
+
+    write(&file, policy_words).unwrap();
+
+    assert_eq!(file.metadata().unwrap().len(), 117720); // sanity check on wordlist size
+
+    let error_mock = srv.mock(|when, then| {
+        when.method(GET).path_matches(
+            Regex::new("/[a-zA-Z]{6}(error|status429|status403)[a-zA-Z]{6}").unwrap(),
+        );
+        then.status(200).body("other errors are still a 200");
+    });
+
+    let normal_reqs_mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z]{6}normal[a-zA-Z]{6}").unwrap());
+        then.status(429)
+            .body("these guys need to be 429 in order to trigger 30% threshold");
+    });
+
+    let start = Instant::now();
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--auto-tune")
+        .arg("--dont-filter")
+        .arg("--time-limit")
+        .arg("7s")
+        .arg("--threads")
+        .arg("4")
+        .assert()
+        .failure();
+
+    teardown_tmp_directory(tmp_dir);
+
+    assert!(normal_reqs_mock.hits() + error_mock.hits() > 25); // must have at least 50 reqs fly
+
+    println!("elapsed: {}", start.elapsed().as_millis()); // 3523ms without tuning
+    assert!(normal_reqs_mock.hits() < 500);
+    assert!(error_mock.hits() <= 180); // may or may not see all other error requests
+    assert!(start.elapsed().as_millis() >= 7000); // scan should hit time limit due to limiting
+}
+
+#[test]
+/// --auto-tune should slow a scan with spurious 403s
+fn auto_tune_slows_scan_with_403s() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["ignored".to_string()], "wordlist").unwrap();
+
+    let policy_words = read_to_string(Path::new("tests/policy-test-words.shuffled")).unwrap();
+
+    write(&file, policy_words).unwrap();
+
+    assert_eq!(file.metadata().unwrap().len(), 117720); // sanity check on wordlist size
+
+    let error_mock = srv.mock(|when, then| {
+        when.method(GET).path_matches(
+            Regex::new("/[a-zA-Z]{6}(error|status429|status403)[a-zA-Z]{6}").unwrap(),
+        );
+        then.status(200).body("other errors are still a 200");
+    });
+
+    let normal_reqs_mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z]{6}normal[a-zA-Z]{6}").unwrap());
+        then.status(403)
+            .body("these guys need to be 403 in order to trigger 90% threshold");
+    });
+
+    let start = Instant::now();
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--auto-tune")
+        .arg("--dont-filter")
+        .arg("--time-limit")
+        .arg("7s")
+        .arg("--threads")
+        .arg("4")
+        .assert()
+        .failure();
+
+    teardown_tmp_directory(tmp_dir);
+
+    assert!(normal_reqs_mock.hits() + error_mock.hits() > 25); // must have at least 50 reqs fly
+
+    println!("elapsed: {}", start.elapsed().as_millis()); // 3523ms without tuning
+    assert!(normal_reqs_mock.hits() < 500);
+    assert!(error_mock.hits() <= 180); // may or may not see all other error requests
+    assert!(start.elapsed().as_millis() >= 7000); // scan should hit time limit due to limiting
+}
+
+#[test]
+/// --auto-tune should slow a scan with spurious errors
+fn auto_tune_slows_scan_with_general_errors() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["ignored".to_string()], "wordlist").unwrap();
+
+    let policy_words = read_to_string(Path::new("tests/policy-test-words.shuffled")).unwrap();
+
+    write(&file, policy_words).unwrap();
+
+    assert_eq!(file.metadata().unwrap().len(), 117720); // sanity check on wordlist size
+
+    let error_mock = srv.mock(|when, then| {
+        when.method(GET).path_matches(
+            Regex::new("/[a-zA-Z]{6}(error|status429|status403)[a-zA-Z]{6}").unwrap(),
+        );
+        then.status(200).body("other errors are still a 200");
+    });
+
+    let normal_reqs_mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z]{6}normal[a-zA-Z]{6}").unwrap());
+        then.status(200)
+            .body("these guys need to be 429 in order to trigger 30% threshold")
+            .delay(Duration::new(3, 0));
+    });
+
+    let start = Instant::now();
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--auto-tune")
+        .arg("--dont-filter")
+        .arg("--time-limit")
+        .arg("7s")
+        .arg("--threads")
+        .arg("4")
+        .arg("--timeout")
+        .arg("2")
+        .assert()
+        .failure();
+
+    teardown_tmp_directory(tmp_dir);
+
+    println!("elapsed: {}", start.elapsed().as_millis()); // 3523ms without tuning
+    assert!(normal_reqs_mock.hits() < 500);
+    assert!(error_mock.hits() <= 180); // may or may not see all other error requests
+    assert!(start.elapsed().as_millis() >= 7000); // scan should hit time limit due to limiting
+}

tests/test_scan_manager.rs

@@ -93,9 +93,13 @@ fn resume_scan_works() {
 #[test]
 /// kick off scan with a time limit;  
 fn time_limit_enforced_when_specified() {
-    let srv = MockServer::start();
+    let t1 = MockServer::start();
+    let t2 = MockServer::start();
+
     let (tmp_dir, file) =
         setup_tmp_directory(&["css".to_string(), "stuff".to_string()], "wordlist").unwrap();
+    let (tgt_tmp_dir, targets) =
+        setup_tmp_directory(&[t1.url("/"), t2.url("/")], "targets").unwrap();
 
     // ensure the command will run long enough by adding crap to the wordlist
     let more_words = read_to_string(Path::new("tests/extra-words")).unwrap();
@@ -109,12 +113,13 @@ fn time_limit_enforced_when_specified() {
 
     Command::cargo_bin("feroxbuster")
         .unwrap()
-        .arg("--url")
-        .arg(srv.url("/"))
+        .arg("--stdin")
         .arg("--wordlist")
         .arg(file.as_os_str())
         .arg("--time-limit")
         .arg("5s")
+        .pipe_stdin(targets)
+        .unwrap()
         .assert()
         .failure();
 
@@ -127,4 +132,5 @@ fn time_limit_enforced_when_specified() {
     assert!(now.elapsed() > lower_bound && now.elapsed() < upper_bound);
 
     teardown_tmp_directory(tmp_dir);
+    teardown_tmp_directory(tgt_tmp_dir);
 }

tests/test_scanner.rs

@@ -496,7 +496,10 @@ fn scanner_single_request_scan_with_debug_logging_as_json() {
     assert!(contents.contains("\"level\":\"DEBUG\""));
     assert!(contents.contains("\"level\":\"INFO\""));
     assert!(contents.contains("time_offset"));
-    assert!(contents.contains("\"module\":\"feroxbuster::scanner\""));
+    assert!(contents.contains("exit: main"));
+    assert!(contents.contains(&srv.url("/LICENSE")));
+    assert!(contents.contains("\"module\":\"feroxbuster::response\""));
+    assert!(contents.contains("\"module\":\"feroxbuster::url\""));
     assert!(contents.contains("\"module\":\"feroxbuster::event_handlers::inputs\""));
     assert!(contents.contains("exit: start_enter_handler"));
     assert!(contents.contains("All scans complete!"));