Merge pull request #360 from epi052/329-dont-scan-enhancements

329 dont scan enhancements

.all-contributorsrc

@@ -0,0 +1,283 @@
+{
+  "files": [
+    "README.md"
+  ],
+  "imageSize": 100,
+  "commit": false,
+  "contributors": [
+    {
+      "login": "joohoi",
+      "name": "Joona Hoikkala",
+      "avatar_url": "https://avatars.githubusercontent.com/u/5235109?v=4",
+      "profile": "https://io.fi",
+      "contributions": [
+        "doc"
+      ]
+    },
+    {
+      "login": "jsav0",
+      "name": "J Savage",
+      "avatar_url": "https://avatars.githubusercontent.com/u/20546041?v=4",
+      "profile": "https://github.com/jsav0",
+      "contributions": [
+        "infra",
+        "doc"
+      ]
+    },
+    {
+      "login": "TGotwig",
+      "name": "Thomas Gotwig",
+      "avatar_url": "https://avatars.githubusercontent.com/u/30773779?v=4",
+      "profile": "http://www.tgotwig.dev",
+      "contributions": [
+        "infra",
+        "doc"
+      ]
+    },
+    {
+      "login": "spikecodes",
+      "name": "Spike",
+      "avatar_url": "https://avatars.githubusercontent.com/u/19519553?v=4",
+      "profile": "https://github.com/spikecodes",
+      "contributions": [
+        "infra",
+        "doc"
+      ]
+    },
+    {
+      "login": "evanrichter",
+      "name": "Evan Richter",
+      "avatar_url": "https://avatars.githubusercontent.com/u/330292?v=4",
+      "profile": "https://github.com/evanrichter",
+      "contributions": [
+        "code",
+        "doc"
+      ]
+    },
+    {
+      "login": "mzpqnxow",
+      "name": "AG",
+      "avatar_url": "https://avatars.githubusercontent.com/u/8016228?v=4",
+      "profile": "https://github.com/mzpqnxow",
+      "contributions": [
+        "ideas",
+        "doc"
+      ]
+    },
+    {
+      "login": "n-thumann",
+      "name": "Nicolas Thumann",
+      "avatar_url": "https://avatars.githubusercontent.com/u/46975855?v=4",
+      "profile": "https://n-thumann.de/",
+      "contributions": [
+        "code",
+        "doc"
+      ]
+    },
+    {
+      "login": "tomtastic",
+      "name": "Tom Matthews",
+      "avatar_url": "https://avatars.githubusercontent.com/u/302127?v=4",
+      "profile": "https://github.com/tomtastic",
+      "contributions": [
+        "doc"
+      ]
+    },
+    {
+      "login": "bsysop",
+      "name": "bsysop",
+      "avatar_url": "https://avatars.githubusercontent.com/u/9998303?v=4",
+      "profile": "https://github.com/bsysop",
+      "contributions": [
+        "doc"
+      ]
+    },
+    {
+      "login": "bpsizemore",
+      "name": "Brian Sizemore",
+      "avatar_url": "https://avatars.githubusercontent.com/u/11645898?v=4",
+      "profile": "http://bpsizemore.me",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "noraj",
+      "name": "Alexandre ZANNI",
+      "avatar_url": "https://avatars.githubusercontent.com/u/16578570?v=4",
+      "profile": "https://pwn.by/noraj",
+      "contributions": [
+        "infra",
+        "doc"
+      ]
+    },
+    {
+      "login": "craig",
+      "name": "Craig",
+      "avatar_url": "https://avatars.githubusercontent.com/u/99729?v=4",
+      "profile": "https://github.com/craig",
+      "contributions": [
+        "infra"
+      ]
+    },
+    {
+      "login": "EONRaider",
+      "name": "EONRaider",
+      "avatar_url": "https://avatars.githubusercontent.com/u/15611424?v=4",
+      "profile": "https://www.reddit.com/u/EONRaider",
+      "contributions": [
+        "infra"
+      ]
+    },
+    {
+      "login": "wtwver",
+      "name": "wtwver",
+      "avatar_url": "https://avatars.githubusercontent.com/u/53866088?v=4",
+      "profile": "https://github.com/wtwver",
+      "contributions": [
+        "infra"
+      ]
+    },
+    {
+      "login": "Tib3rius",
+      "name": "Tib3rius",
+      "avatar_url": "https://avatars.githubusercontent.com/u/48113936?v=4",
+      "profile": "https://tib3rius.com",
+      "contributions": [
+        "bug"
+      ]
+    },
+    {
+      "login": "0xdf",
+      "name": "0xdf",
+      "avatar_url": "https://avatars.githubusercontent.com/u/1489045?v=4",
+      "profile": "https://github.com/0xdf",
+      "contributions": [
+        "bug"
+      ]
+    },
+    {
+      "login": "secure-77",
+      "name": "secure-77",
+      "avatar_url": "https://avatars.githubusercontent.com/u/31564517?v=4",
+      "profile": "http://secure77.de",
+      "contributions": [
+        "bug"
+      ]
+    },
+    {
+      "login": "sbrun",
+      "name": "Sophie Brun",
+      "avatar_url": "https://avatars.githubusercontent.com/u/7712154?v=4",
+      "profile": "https://github.com/sbrun",
+      "contributions": [
+        "infra"
+      ]
+    },
+    {
+      "login": "black-A",
+      "name": "black-A",
+      "avatar_url": "https://avatars.githubusercontent.com/u/30686803?v=4",
+      "profile": "https://github.com/black-A",
+      "contributions": [
+        "ideas"
+      ]
+    },
+    {
+      "login": "dinosn",
+      "name": "Nicolas Krassas",
+      "avatar_url": "https://avatars.githubusercontent.com/u/3851678?v=4",
+      "profile": "https://github.com/dinosn",
+      "contributions": [
+        "ideas"
+      ]
+    },
+    {
+      "login": "N0ur5",
+      "name": "N0ur5",
+      "avatar_url": "https://avatars.githubusercontent.com/u/24260009?v=4",
+      "profile": "https://github.com/N0ur5",
+      "contributions": [
+        "ideas"
+      ]
+    },
+    {
+      "login": "moscowchill",
+      "name": "mchill",
+      "avatar_url": "https://avatars.githubusercontent.com/u/72578879?v=4",
+      "profile": "https://github.com/moscowchill",
+      "contributions": [
+        "bug"
+      ]
+    },
+    {
+      "login": "BitThr3at",
+      "name": "Naman",
+      "avatar_url": "https://avatars.githubusercontent.com/u/45028933?v=4",
+      "profile": "http://BitThr3at.github.io",
+      "contributions": [
+        "bug"
+      ]
+    },
+    {
+      "login": "sicks3c",
+      "name": "Ayoub Elaich",
+      "avatar_url": "https://avatars.githubusercontent.com/u/32225186?v=4",
+      "profile": "https://github.com/Sicks3c",
+      "contributions": [
+        "bug"
+      ]
+    },
+    {
+      "login": "HenryHoggard",
+      "name": "Henry",
+      "avatar_url": "https://avatars.githubusercontent.com/u/1208121?v=4",
+      "profile": "https://github.com/HenryHoggard",
+      "contributions": [
+        "bug"
+      ]
+    },
+    {
+      "login": "SleepiPanda",
+      "name": "SleepiPanda",
+      "avatar_url": "https://avatars.githubusercontent.com/u/6428561?v=4",
+      "profile": "https://github.com/SleepiPanda",
+      "contributions": [
+        "bug"
+      ]
+    },
+    {
+      "login": "uBadRequest",
+      "name": "Bad Requests",
+      "avatar_url": "https://avatars.githubusercontent.com/u/47282747?v=4",
+      "profile": "https://github.com/uBadRequest",
+      "contributions": [
+        "bug"
+      ]
+    },
+    {
+      "login": "dnaka91",
+      "name": "Dominik Nakamura",
+      "avatar_url": "https://avatars.githubusercontent.com/u/36804488?v=4",
+      "profile": "https://home.dnaka91.rocks",
+      "contributions": [
+        "infra"
+      ]
+    },
+    {
+      "login": "hunter0x8",
+      "name": "Muhammad Ahsan",
+      "avatar_url": "https://avatars.githubusercontent.com/u/46222314?v=4",
+      "profile": "https://github.com/hunter0x8",
+      "contributions": [
+        "bug"
+      ]
+    }
+  ],
+  "contributorsPerLine": 7,
+  "projectName": "feroxbuster",
+  "projectOwner": "epi052",
+  "repoType": "github",
+  "repoHost": "https://github.com",
+  "skipCi": true
+}

.cargo/config

@@ -0,0 +1,5 @@
+[target.armv7-unknown-linux-gnueabihf]
+linker = "arm-linux-gnueabihf-gcc"
+
+[target.aarch64-unknown-linux-gnu]
+linker = "aarch64-linux-gnu-gcc"

.github/FUNDING.yml

@@ -0,0 +1,4 @@
+# These are supported funding model platforms
+
+github: [epi052]
+ko_fi: epi052

.github/pull_request_template.md

@@ -7,11 +7,11 @@ Long form explanations of most of the items below can be found in the [CONTRIBUT
 - [ ] Your PR description references the associated issue (i.e. fixes #123456)
 - [ ] Code is in its own branch
 - [ ] Branch name is related to the PR contents
-- [ ] PR targets master
+- [ ] PR targets main
 
 ## Static analysis checks
 - [ ] All rust files are formatted using `cargo fmt`
-- [ ] All `clippy` checks pass when running `cargo clippy --all-targets --all-features -- -D warnings -A clippy::deref_addrof`
+- [ ] All `clippy` checks pass when running `cargo clippy --all-targets --all-features -- -D warnings -A clippy::mutex-atomic`
 - [ ] All existing tests pass
 
 ## Documentation

.github/stale.yml

@@ -6,6 +6,7 @@ daysUntilClose: 7
 exemptLabels:
   - pinned
   - security
+  - confirmed
 # Label to use when marking an issue as stale
 staleLabel: stale
 # Comment to post when marking an issue as stale. Set to `false` to disable

.github/workflows/build.yml

@@ -4,11 +4,13 @@ on: [push]
 
 jobs:
   build-nix:
+    env:
+      IN_PIPELINE: true
     runs-on: ${{ matrix.os }}
     if: github.ref == 'refs/heads/main'
     strategy:
       matrix:
-        type: [ubuntu-x64, ubuntu-x86]
+        type: [ubuntu-x64, ubuntu-x86, armv7, aarch64]
         include:
           - type: ubuntu-x64
             os: ubuntu-latest
@@ -22,12 +24,25 @@ jobs:
             name: x86-linux-feroxbuster
             path: target/i686-unknown-linux-musl/release/feroxbuster
             pkg_config_path: /usr/lib/i686-linux-gnu/pkgconfig
+          - type: armv7
+            os: ubuntu-latest
+            target: armv7-unknown-linux-gnueabihf
+            name: armv7-feroxbuster
+            path: target/armv7-unknown-linux-gnueabihf/release/feroxbuster
+            pkg_config_path: /usr/lib/x86_64-linux-gnu/pkgconfig
+          - type: aarch64
+            os: ubuntu-latest
+            target: aarch64-unknown-linux-gnu
+            name: aarch64-feroxbuster
+            path: target/aarch64-unknown-linux-gnu/release/feroxbuster
+            pkg_config_path: /usr/lib/x86_64-linux-gnu/pkgconfig
     steps:
       - uses: actions/checkout@v2
       - name: Install System Dependencies
         run: |
+          env
           sudo apt-get update
-          sudo apt-get install -y --no-install-recommends libssl-dev pkg-config
+          sudo apt-get install -y --no-install-recommends libssl-dev pkg-config gcc-arm-linux-gnueabihf gcc-aarch64-linux-gnu
       - uses: actions-rs/toolchain@v1
         with:
           toolchain: stable
@@ -43,7 +58,7 @@ jobs:
           args: --release --target=${{ matrix.target }}
       - name: Strip symbols from binary
         run: |
-          strip -s ${{ matrix.path }}
+          strip -s ${{ matrix.path }} || arm-linux-gnueabihf-strip -s ${{ matrix.path }} || aarch64-linux-gnu-strip -s ${{ matrix.path }}
       - name: Build tar.gz for homebrew installs
         if: matrix.type == 'ubuntu-x64'
         run: |
@@ -72,6 +87,8 @@ jobs:
           path: ./target/x86_64-unknown-linux-musl/debian/*
 
   build-macos:
+    env:
+      IN_PIPELINE: true
     runs-on: macos-latest
     if: github.ref == 'refs/heads/main'
     steps:
@@ -102,6 +119,8 @@ jobs:
           path: x86_64-macos-feroxbuster.tar.gz
 
   build-windows:
+    env:
+      IN_PIPELINE: true
     runs-on: ${{ matrix.os }}
     if: github.ref == 'refs/heads/main'
     strategy:
@@ -134,4 +153,3 @@ jobs:
         with:
           name: ${{ matrix.name }}
           path: ${{ matrix.path }}
-

.github/workflows/check.yml

@@ -61,4 +61,4 @@ jobs:
       - uses: actions-rs/cargo@v1
         with:
           command: clippy
-          args: --all-targets --all-features -- -D warnings -A clippy::deref_addrof
+          args: --all-targets --all-features -- -D warnings -A clippy::deref_addrof -A clippy::mutex-atomic

.github/workflows/cicd-to-dockerhub.yml

@@ -0,0 +1,34 @@
+name: ci-to-dockerhub
+
+on:
+  push:
+    branches: [ main ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build and push
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          context: ./
+          file: ./Dockerfile
+          push: true
+          tags: ${{ secrets.DOCKER_HUB_USERNAME }}/feroxbuster:latest
+
+      - name: Image digest
+        run: echo ${{ steps.docker_build.outputs.digest }}

.gitignore

@@ -23,7 +23,7 @@ lcov_cobertura.py
 .dockerignore
 
 # state file created during tests
-ferox-http*
+ferox-*.state
 
 # python stuff cuz reasons
 Pipfile*

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "feroxbuster"
-version = "2.0.2"
+version = "2.4.0"
 authors = ["Ben 'epi' Risher <epibar052@gmail.com>"]
 license = "MIT"
 edition = "2018"
@@ -19,14 +19,17 @@ maintenance = { status = "actively-developed" }
 clap = "2.33"
 regex = "1"
 lazy_static = "1.4"
+dirs = "4.0"
 
 [dependencies]
 futures = { version = "0.3"}
-tokio = { version = "1.0", features = ["full"] }
-tokio-util = {version = "0.6.3", features = ["codec"]}
+tokio = { version = "1.11", features = ["full"] }
+tokio-util = {version = "0.6", features = ["codec"]}
 log = "0.4"
-env_logger = "0.8"
+env_logger = "0.9"
 reqwest = { version = "0.11", features = ["socks"] }
+url = { version = "2.2", features = ["serde"]}  # uses feature unification to add 'serde' to reqwest::Url
+serde_regex = "1.1"
 clap = "2.33"
 lazy_static = "1.4"
 toml = "0.5"
@@ -34,22 +37,22 @@ serde = { version = "1.0", features = ["derive", "rc"] }
 serde_json = "1.0"
 uuid = { version = "0.8", features = ["v4"] }
 indicatif = "0.15"
-console = "0.14"
+console = "0.15"
 openssl = { version = "0.10", features = ["vendored"] }
-dirs = "3.0"
+dirs = "4.0"
 regex = "1"
-crossterm = "0.19"
-rlimit = "0.5"
-ctrlc = "3.1"
+crossterm = "0.20"
+rlimit = "0.6"
+ctrlc = "3.2"
 fuzzyhash = "0.2.1"
 anyhow = "1.0"
 leaky-bucket = "0.10.0"
 
 [dev-dependencies]
 tempfile = "3.1"
-httpmock = "0.5.2"
-assert_cmd = "1.0.3"
-predicates = "1.0.7"
+httpmock = "0.6.2"
+assert_cmd = "2.0"
+predicates = "2.0"
 
 [profile.release]
 lto = true
@@ -63,4 +66,7 @@ conf-files = ["/etc/feroxbuster/ferox-config.toml"]
 assets = [
     ["target/release/feroxbuster", "/usr/bin/", "755"],
     ["ferox-config.toml.example", "/etc/feroxbuster/ferox-config.toml", "644"],
+    ["shell_completions/feroxbuster.bash", "/usr/share/bash-completion/completions/feroxbuster.bash", "644"],
+    ["shell_completions/feroxbuster.fish", "/usr/share/fish/completions/feroxbuster.fish", "644"],
+    ["shell_completions/_feroxbuster", "/usr/share/zsh/vendor-completions/_feroxbuster", "644"],
 ]

Dockerfile

@@ -1,14 +1,28 @@
-FROM alpine:latest
+# Image: alpine:3.14.2
+FROM alpine@sha256:69704ef328d05a9f806b6b8502915e6a0a4faa4d72018dc42343f511490daf8a as build
 LABEL maintainer="wfnintr@null.net"
 
-RUN sed -i -e 's/v[[:digit:]]\..*\//edge\//g' /etc/apk/repositories && apk upgrade --update-cache --available
+RUN sed -i -e 's/v[[:digit:]]\..*\//edge\//g' /etc/apk/repositories \
+    && apk upgrade --update-cache --available && apk add --update openssl
 
-# download default wordlists 
-RUN apk add --no-cache --virtual .depends subversion font-noto-emoji && \
-	svn export https://github.com/danielmiessler/SecLists/trunk/Discovery/Web-Content /usr/share/seclists/Discovery/Web-Content && \
-	apk del .depends
 
-# install latest release
-RUN wget https://github.com/epi052/feroxbuster/releases/latest/download/x86_64-linux-feroxbuster.zip -qO feroxbuster.zip && unzip -d /usr/local/bin/ feroxbuster.zip feroxbuster && rm feroxbuster.zip && chmod +x /usr/local/bin/feroxbuster
+# Download latest release
+RUN wget https://github.com/epi052/feroxbuster/releases/latest/download/x86_64-linux-feroxbuster.zip -qO feroxbuster.zip \
+    && unzip -d /tmp/ feroxbuster.zip feroxbuster \
+    && chmod +x /tmp/feroxbuster \
+    && wget https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/Web-Content/raft-medium-directories.txt -O /tmp/raft-medium-directories.txt
+
+# Image: alpine:3.14.2
+FROM alpine@sha256:69704ef328d05a9f806b6b8502915e6a0a4faa4d72018dc42343f511490daf8a as release
+
+COPY --from=build /tmp/raft-medium-directories.txt /usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt
+COPY --from=build /tmp/feroxbuster /usr/local/bin/feroxbuster
+
+RUN adduser \
+    --gecos "" \
+    --disabled-password \
+    feroxbuster
+
+USER feroxbuster
 
 ENTRYPOINT ["feroxbuster"]

Makefile

@@ -6,12 +6,16 @@ datarootdir = $(prefix)/share
 datadir = $(datarootdir)
 example_config = ferox-config.toml.example
 config_file = ferox-config.toml
+completion_dir = shell_completions
+completion_prefix = $(completion_dir)/$(BIN)
 
+BIN=feroxbuster
 SHR_SOURCES = $(shell find src -type f -wholename '*src/*.rs') Cargo.toml Cargo.lock
 
 RELEASE = debug
 DEBUG ?= 0
-ifeq (0,$(DEBUG))
+
+ifeq (0, $(DEBUG))
 	ARGS = --release
 	RELEASE = release
 endif
@@ -23,54 +27,52 @@ endif
 
 TARGET = target/$(RELEASE)
 
-.PHONY: all clean distclean install uninstall update
-
-BIN=feroxbuster
-DESKTOP=$(APPID).desktop
+.PHONY: all clean install uninstall test update
 
 all: cli
-
 cli: $(TARGET)/$(BIN) $(TARGET)/$(BIN).1.gz $(SHR_SOURCES)
+install: all install-cli
+
+verify:
+	cargo fmt
+	cargo clippy --all-targets --all-features -- -D warnings -A clippy::mutex-atomic
+	cargo test
 
 clean:
 	cargo clean
 
-distclean: clean
-	rm -rf .cargo vendor Cargo.lock vendor.tar
-
 vendor: vendor.tar
 
 vendor.tar:
-	mkdir -p .cargo
-	cargo vendor | head -n -1 > .cargo/config
-	echo 'directory = "vendor"' >> .cargo/config
+	cargo vendor
 	tar pcf vendor.tar vendor
 	rm -rf vendor
 
 install-cli: cli
-	install -Dm 0755 "$(TARGET)/$(BIN)" "$(DESTDIR)$(bindir)/$(BIN)"
+	install -Dm 0644 "$(completion_prefix).bash" "$(DESTDIR)/usr/share/bash-completion/completions/$(BIN).bash"
+	install -Dm 0644 "$(completion_prefix).fish" "$(DESTDIR)/usr/share/fish/completions/$(BIN).fish"
+	install -Dm 0644 "$(completion_dir)/_$(BIN)" "$(DESTDIR)/usr/share/zsh/vendor-completions/_$(BIN)"
+	install -sDm 0755 "$(TARGET)/$(BIN)" "$(DESTDIR)$(bindir)/$(BIN)"
 	install -Dm 0644 "$(TARGET)/$(BIN).1.gz" "$(DESTDIR)$(datadir)/man/man1/$(BIN).1.gz"
-	install -Dm 0644 "$(example_config)" "/etc/$(BIN)/$(config_File)"
+	install -Dm 0644 "$(example_config)" "$(DESTDIR)/etc/$(BIN)/$(config_file)"
 
-install: all install-cli
-
-uninstall-cli:
+uninstall:
 	rm -f "$(DESTDIR)$(bindir)/$(BIN)"
 	rm -f "$(DESTDIR)$(datadir)/man/man1/$(BIN).1.gz"
-	rm -rf "/etc/$(BIN)/"
-
-uninstall: uninstall-cli
-
-update:
-	cargo update
+	rm -rf "$(DESTDIR)/etc/$(BIN)/"
+	rm -f "$(DESTDIR)/usr/share/bash-completion/completions/$(BIN).bash"
+	rm -f "$(DESTDIR)/usr/share/zsh/vendor-completions/_$(BIN)"
+	rm -f "$(DESTDIR)/usr/share/fish/completions/$(BIN).fish"
 
 extract:
-ifeq ($(VENDORED),1)
+ifeq (1, $(VENDORED))
 	tar pxf vendor.tar
 endif
 
 $(TARGET)/$(BIN): extract
-	cargo build --manifest-path Cargo.toml $(ARGS)
+	mkdir -p .cargo
+	cp debian/cargo.config .cargo/config.toml
+	cargo build $(ARGS)
 
 $(TARGET)/$(BIN).1.gz: $(TARGET)/$(BIN)
 	help2man --no-info $< | gzip -c > $@.partial

build.rs

@@ -1,4 +1,7 @@
+use std::fs::{copy, create_dir_all, OpenOptions};
+use std::io::{Read, Seek, SeekFrom, Write};
 extern crate clap;
+extern crate dirs;
 
 use clap::Shell;
 
@@ -20,4 +23,64 @@ fn main() {
     for shell in &shells {
         app.gen_completions("feroxbuster", *shell, outdir);
     }
+
+    // 0xdf pointed out an oddity when tab-completing options that expect file paths, the fix we
+    // landed on was to add -o plusdirs to the bash completion script. The following code aims to
+    // automate that fix and have it present in all future builds
+    let mut contents = String::new();
+
+    let mut bash_file = OpenOptions::new()
+        .read(true)
+        .write(true)
+        .open(format!("{}/feroxbuster.bash", outdir))
+        .expect("Couldn't open bash completion script");
+
+    bash_file
+        .read_to_string(&mut contents)
+        .expect("Couldn't read bash completion script");
+
+    contents = contents.replace("default feroxbuster", "default -o plusdirs feroxbuster");
+
+    bash_file
+        .seek(SeekFrom::Start(0))
+        .expect("Couldn't seek to position 0 in bash completion script");
+
+    bash_file
+        .write_all(contents.as_bytes())
+        .expect("Couldn't write updated bash completion script to disk");
+
+    // hunter0x8 let me know that when installing via cargo, it would be nice if we dropped a
+    // config file during the build process. The following code will place an example config in
+    // the user's configuration directory
+    //   - linux: $XDG_CONFIG_HOME or $HOME/.config
+    //   - macOS: $HOME/Library/Application Support
+    //   - windows: {FOLDERID_RoamingAppData}
+    let mut config_dir = dirs::config_dir().expect("Couldn't resolve user's config directory");
+    config_dir = config_dir.join("feroxbuster"); // $HOME/.config/feroxbuster
+
+    if !config_dir.exists() {
+        // recursively create the feroxbuster directory and all of its parent components if
+        // they are missing
+        if !config_dir.exists() {
+            // recursively create the feroxbuster directory and all of its parent components if
+            // they are missing
+            if create_dir_all(&config_dir).is_err() {
+                // only copy the config file when we're not running in the CI/CD pipeline
+                // which fails with permission denied
+                eprintln!("Couldn't create one or more directories needed to copy the config file");
+                return;
+            }
+        }
+    }
+
+    // hard-coding config name here to not rely on the crate we're building, if DEFAULT_CONFIG_NAME
+    // ever changes, this will need to be updated
+    let config_file = config_dir.join("ferox-config.toml");
+
+    if !config_file.exists() {
+        // config file doesn't exist, add it to the config directory
+        if copy("ferox-config.toml.example", config_file).is_err() {
+            eprintln!("Couldn't copy example config into config directory");
+        }
+    }
 }

docs/index.html

@@ -0,0 +1,11 @@
+<html>
+<head>
+   <meta http-equiv="refresh"
+   content="0; url=https://epi052.github.io/feroxbuster-docs/">
+</head>
+<body>
+   <p>The page has moved to:
+   <a href="https://epi052.github.io/feroxbuster-docs/">feroxbuster-docs</a></p>
+</body>
+</html>
+

ferox-config.toml.example

@@ -16,17 +16,23 @@
 # replay_proxy = "http://127.0.0.1:8081"
 # replay_codes = [200, 302]
 # verbosity = 1
+# parallel = 8
 # scan_limit = 6
 # rate_limit = 250
 # quiet = true
 # silent = true
+# auto_tune = true
+# auto_bail = true
 # json = true
 # output = "/targets/ellingson_mineral_company/gibson.txt"
 # debug_log = "/var/log/find-the-derp.log"
 # user_agent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0"
+# random_agent = false
 # redirects = true
 # insecure = true
 # extensions = ["php", "html"]
+# url_denylist = ["http://dont-scan.me", "https://also-not.me"]
+# regex_denylist = ["/deny.*"]
 # no_recursion = true
 # add_slash = true
 # stdin = true

install-nix.sh

@@ -3,59 +3,63 @@
 BASE_URL=https://github.com/epi052/feroxbuster/releases/latest/download
 
 MAC_ZIP=x86_64-macos-feroxbuster.zip
-MAC_URL="${BASE_URL}/${MAC_ZIP}"
+MAC_URL="$BASE_URL/$MAC_ZIP"
 
 LIN32_ZIP=x86-linux-feroxbuster.zip
-LIN32_URL="${BASE_URL}/${LIN32_ZIP}"
+LIN32_URL="$BASE_URL/$LIN32_ZIP"
 
 LIN64_ZIP=x86_64-linux-feroxbuster.zip
-LIN64_URL="${BASE_URL}/${LIN64_ZIP}"
+LIN64_URL="$BASE_URL/$LIN64_ZIP"
 
 EMOJI_URL=https://gist.github.com/epi052/8196b550ea51d0907ad4b93751b1b57d/raw/6112c9f32ae07922983fdc549c54fd3fb9a38e4c/NotoColorEmoji.ttf
 
 echo "[+] Installing feroxbuster!"
 
-if [[ "$(uname)" == "Darwin" ]]; then
-    echo "[=] Found MacOS, downloading from ${MAC_URL}"
-
-    curl -sLO "${MAC_URL}"
-    unzip -o "${MAC_ZIP}" > /dev/null
-    rm "${MAC_ZIP}"
-elif [[ "$(expr substr $(uname -s) 1 5)" == "Linux" ]]; then
-    if [[ $(getconf LONG_BIT) == 32 ]]; then
-        echo "[=] Found 32-bit Linux, downloading from ${LIN32_URL}"
-
-        curl -sLO "${LIN32_URL}"
-        unzip -o "${LIN32_ZIP}" > /dev/null
-        rm "${LIN32_ZIP}"
-    else
-        echo "[=] Found 64-bit Linux, downloading from ${LIN64_URL}"
-
-        curl -sLO "${LIN64_URL}"
-        unzip -o "${LIN64_ZIP}" > /dev/null
-        rm "${LIN64_ZIP}"
-    fi
-
-    if [[ -e ~/.fonts/NotoColorEmoji.ttf ]]; then
-       echo "[=] Found Noto Emoji Font, skipping install" 
-    else 
-        echo "[=] Installing Noto Emoji Font"
-        mkdir -p ~/.fonts
-        pushd ~/.fonts 2>&1 >/dev/null
-
-        curl -sLO "${EMOJI_URL}"
-
-        fc-cache -f -v >/dev/null
-
-        popd 2>&1 >/dev/null
-        echo "[+] Noto Emoji Font installed"
-    fi
+which unzip &>/dev/null
+if [ "$?" = "0" ]; then
+  echo "[+] unzip found"
+else
+  echo "[ ] unzip not found, exiting. "
+  exit -1
 fi
 
+if [[ "$(uname)" == "Darwin" ]]; then
+  echo "[=] Found MacOS, downloading from $MAC_URL"
+
+  curl -sLO "$MAC_URL"
+  unzip -o "$MAC_ZIP" >/dev/null
+  rm "$MAC_ZIP"
+elif [[ "$(expr substr $(uname -s) 1 5)" == "Linux" ]]; then
+  if [[ $(getconf LONG_BIT) == 32 ]]; then
+    echo "[=] Found 32-bit Linux, downloading from $LIN32_URL"
+
+    curl -sLO "$LIN32_URL"
+    unzip -o "$LIN32_ZIP" >/dev/null
+    rm "$LIN32_ZIP"
+  else
+    echo "[=] Found 64-bit Linux, downloading from $LIN64_URL"
+
+    curl -sLO "$LIN64_URL"
+    unzip -o "$LIN64_ZIP" >/dev/null
+    rm "$LIN64_ZIP"
+  fi
+
+  if [[ -e ~/.fonts/NotoColorEmoji.ttf ]]; then
+    echo "[=] Found Noto Emoji Font, skipping install"
+  else
+    echo "[=] Installing Noto Emoji Font"
+    mkdir -p ~/.fonts
+    pushd ~/.fonts 2>&1 >/dev/null
+
+    curl -sLO "$EMOJI_URL"
+
+    fc-cache -f -v >/dev/null
+
+    popd 2>&1 >/dev/null
+    echo "[+] Noto Emoji Font installed"
+  fi
+fi
 
 chmod +x ./feroxbuster
 
 echo "[+] Installed feroxbuster version $(./feroxbuster -V)"
-
-
-

shell_completions/_feroxbuster

@@ -41,6 +41,7 @@ _feroxbuster() {
 '--user-agent=[Sets the User-Agent (default: feroxbuster/VERSION)]' \
 '*-x+[File extension(s) to search for (ex: -x php -x pdf js)]' \
 '*--extensions=[File extension(s) to search for (ex: -x php -x pdf js)]' \
+'*--dont-scan=[URL(s) or Regex Pattern(s) to exclude from recursion/scans]' \
 '*-H+[Specify HTTP headers (ex: -H Header:val '\''stuff: things'\'')]' \
 '*--headers=[Specify HTTP headers (ex: -H Header:val '\''stuff: things'\'')]' \
 '*-Q+[Specify URL query parameters (ex: -Q token=stuff -Q secret=key)]' \
@@ -58,24 +59,29 @@ _feroxbuster() {
 '*--filter-similar-to=[Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)]' \
 '-L+[Limit total number of concurrent scans (default: 0, i.e. no limit)]' \
 '--scan-limit=[Limit total number of concurrent scans (default: 0, i.e. no limit)]' \
-'--rate-limit=[Limit number of requests per second (per directory) (default: 0, i.e. no limit)]' \
+'--parallel=[Run parallel feroxbuster instances (one child process per url passed via stdin)]' \
+'(--auto-tune)--rate-limit=[Limit number of requests per second (per directory) (default: 0, i.e. no limit)]' \
 '--time-limit=[Limit total run time of all scans (ex: --time-limit 10m)]' \
 '(--silent)*-v[Increase verbosity level (use -vv or more for greater effect. \[CAUTION\] 4 -v'\''s is probably too much)]' \
 '(--silent)*--verbosity[Increase verbosity level (use -vv or more for greater effect. \[CAUTION\] 4 -v'\''s is probably too much)]' \
 '(-q --quiet)--silent[Only print URLs + turn off logging (good for piping a list of urls to other commands)]' \
 '-q[Hide progress bars and banner (good for tmux windows w/ notifications)]' \
 '--quiet[Hide progress bars and banner (good for tmux windows w/ notifications)]' \
+'(--auto-bail)--auto-tune[Automatically lower scan rate when an excessive amount of errors are encountered]' \
+'--auto-bail[Automatically stop scanning when an excessive amount of errors are encountered]' \
 '--json[Emit JSON logs to --output and --debug-log instead of normal text]' \
 '-D[Don'\''t auto-filter wildcard responses]' \
 '--dont-filter[Don'\''t auto-filter wildcard responses]' \
+'-A[Use a random User-Agent]' \
+'--random-agent[Use a random User-Agent]' \
 '-r[Follow redirects]' \
 '--redirects[Follow redirects]' \
 '-k[Disables TLS certificate validation]' \
 '--insecure[Disables TLS certificate validation]' \
 '-n[Do not scan recursively]' \
 '--no-recursion[Do not scan recursively]' \
-'(-x --extensions)-f[Append / to each request]' \
-'(-x --extensions)--add-slash[Append / to each request]' \
+'-f[Append / to each request]' \
+'--add-slash[Append / to each request]' \
 '(-u --url)--stdin[Read url(s) from STDIN]' \
 '-e[Extract links from response body (html, javascript, etc...); make new requests based on findings (default: false)]' \
 '--extract-links[Extract links from response body (html, javascript, etc...); make new requests based on findings (default: false)]' \

shell_completions/_feroxbuster.ps1

@@ -46,6 +46,7 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
             [CompletionResult]::new('--user-agent', 'user-agent', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/VERSION)')
             [CompletionResult]::new('-x', 'x', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js)')
             [CompletionResult]::new('--extensions', 'extensions', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js)')
+            [CompletionResult]::new('--dont-scan', 'dont-scan', [CompletionResultType]::ParameterName, 'URL(s) or Regex Pattern(s) to exclude from recursion/scans')
             [CompletionResult]::new('-H', 'H', [CompletionResultType]::ParameterName, 'Specify HTTP headers (ex: -H Header:val ''stuff: things'')')
             [CompletionResult]::new('--headers', 'headers', [CompletionResultType]::ParameterName, 'Specify HTTP headers (ex: -H Header:val ''stuff: things'')')
             [CompletionResult]::new('-Q', 'Q', [CompletionResultType]::ParameterName, 'Specify URL query parameters (ex: -Q token=stuff -Q secret=key)')
@@ -63,6 +64,7 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
             [CompletionResult]::new('--filter-similar-to', 'filter-similar-to', [CompletionResultType]::ParameterName, 'Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)')
             [CompletionResult]::new('-L', 'L', [CompletionResultType]::ParameterName, 'Limit total number of concurrent scans (default: 0, i.e. no limit)')
             [CompletionResult]::new('--scan-limit', 'scan-limit', [CompletionResultType]::ParameterName, 'Limit total number of concurrent scans (default: 0, i.e. no limit)')
+            [CompletionResult]::new('--parallel', 'parallel', [CompletionResultType]::ParameterName, 'Run parallel feroxbuster instances (one child process per url passed via stdin)')
             [CompletionResult]::new('--rate-limit', 'rate-limit', [CompletionResultType]::ParameterName, 'Limit number of requests per second (per directory) (default: 0, i.e. no limit)')
             [CompletionResult]::new('--time-limit', 'time-limit', [CompletionResultType]::ParameterName, 'Limit total run time of all scans (ex: --time-limit 10m)')
             [CompletionResult]::new('-v', 'v', [CompletionResultType]::ParameterName, 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v''s is probably too much)')
@@ -70,9 +72,13 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
             [CompletionResult]::new('--silent', 'silent', [CompletionResultType]::ParameterName, 'Only print URLs + turn off logging (good for piping a list of urls to other commands)')
             [CompletionResult]::new('-q', 'q', [CompletionResultType]::ParameterName, 'Hide progress bars and banner (good for tmux windows w/ notifications)')
             [CompletionResult]::new('--quiet', 'quiet', [CompletionResultType]::ParameterName, 'Hide progress bars and banner (good for tmux windows w/ notifications)')
+            [CompletionResult]::new('--auto-tune', 'auto-tune', [CompletionResultType]::ParameterName, 'Automatically lower scan rate when an excessive amount of errors are encountered')
+            [CompletionResult]::new('--auto-bail', 'auto-bail', [CompletionResultType]::ParameterName, 'Automatically stop scanning when an excessive amount of errors are encountered')
             [CompletionResult]::new('--json', 'json', [CompletionResultType]::ParameterName, 'Emit JSON logs to --output and --debug-log instead of normal text')
             [CompletionResult]::new('-D', 'D', [CompletionResultType]::ParameterName, 'Don''t auto-filter wildcard responses')
             [CompletionResult]::new('--dont-filter', 'dont-filter', [CompletionResultType]::ParameterName, 'Don''t auto-filter wildcard responses')
+            [CompletionResult]::new('-A', 'A', [CompletionResultType]::ParameterName, 'Use a random User-Agent')
+            [CompletionResult]::new('--random-agent', 'random-agent', [CompletionResultType]::ParameterName, 'Use a random User-Agent')
             [CompletionResult]::new('-r', 'r', [CompletionResultType]::ParameterName, 'Follow redirects')
             [CompletionResult]::new('--redirects', 'redirects', [CompletionResultType]::ParameterName, 'Follow redirects')
             [CompletionResult]::new('-k', 'k', [CompletionResultType]::ParameterName, 'Disables TLS certificate validation')

shell_completions/feroxbuster.bash

@@ -20,7 +20,7 @@ _feroxbuster() {
 
     case "${cmd}" in
         feroxbuster)
-            opts=" -v -q -D -r -k -n -f -e -h -V -w -u -t -d -T -p -P -R -s -o -a -x -H -Q -S -X -W -N -C -L  --verbosity --silent --quiet --json --dont-filter --redirects --insecure --no-recursion --add-slash --stdin --extract-links --help --version --wordlist --url --threads --depth --timeout --proxy --replay-proxy --replay-codes --status-codes --output --resume-from --debug-log --user-agent --extensions --headers --query --filter-size --filter-regex --filter-words --filter-lines --filter-status --filter-similar-to --scan-limit --rate-limit --time-limit  "
+            opts=" -v -q -D -A -r -k -n -f -e -h -V -w -u -t -d -T -p -P -R -s -o -a -x -H -Q -S -X -W -N -C -L  --verbosity --silent --quiet --auto-tune --auto-bail --json --dont-filter --random-agent --redirects --insecure --no-recursion --add-slash --stdin --extract-links --help --version --wordlist --url --threads --depth --timeout --proxy --replay-proxy --replay-codes --status-codes --output --resume-from --debug-log --user-agent --extensions --dont-scan --headers --query --filter-size --filter-regex --filter-words --filter-lines --filter-status --filter-similar-to --scan-limit --parallel --rate-limit --time-limit  "
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 1 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -131,6 +131,10 @@ _feroxbuster() {
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
                     ;;
+                --dont-scan)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
                 --headers)
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
@@ -199,6 +203,10 @@ _feroxbuster() {
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
                     ;;
+                --parallel)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
                 --rate-limit)
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
@@ -218,4 +226,4 @@ _feroxbuster() {
     esac
 }
 
-complete -F _feroxbuster -o bashdefault -o default feroxbuster
+complete -F _feroxbuster -o bashdefault -o default -o plusdirs feroxbuster

shell_completions/feroxbuster.fish

@@ -12,6 +12,7 @@ complete -c feroxbuster -n "__fish_use_subcommand" -l resume-from -d 'State file
 complete -c feroxbuster -n "__fish_use_subcommand" -l debug-log -d 'Output file to write log entries (use w/ --json for JSON entries)'
 complete -c feroxbuster -n "__fish_use_subcommand" -s a -l user-agent -d 'Sets the User-Agent (default: feroxbuster/VERSION)'
 complete -c feroxbuster -n "__fish_use_subcommand" -s x -l extensions -d 'File extension(s) to search for (ex: -x php -x pdf js)'
+complete -c feroxbuster -n "__fish_use_subcommand" -l dont-scan -d 'URL(s) or Regex Pattern(s) to exclude from recursion/scans'
 complete -c feroxbuster -n "__fish_use_subcommand" -s H -l headers -d 'Specify HTTP headers (ex: -H Header:val \'stuff: things\')'
 complete -c feroxbuster -n "__fish_use_subcommand" -s Q -l query -d 'Specify URL query parameters (ex: -Q token=stuff -Q secret=key)'
 complete -c feroxbuster -n "__fish_use_subcommand" -s S -l filter-size -d 'Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)'
@@ -21,13 +22,17 @@ complete -c feroxbuster -n "__fish_use_subcommand" -s N -l filter-lines -d 'Filt
 complete -c feroxbuster -n "__fish_use_subcommand" -s C -l filter-status -d 'Filter out status codes (deny list) (ex: -C 200 -C 401)'
 complete -c feroxbuster -n "__fish_use_subcommand" -l filter-similar-to -d 'Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)'
 complete -c feroxbuster -n "__fish_use_subcommand" -s L -l scan-limit -d 'Limit total number of concurrent scans (default: 0, i.e. no limit)'
+complete -c feroxbuster -n "__fish_use_subcommand" -l parallel -d 'Run parallel feroxbuster instances (one child process per url passed via stdin)'
 complete -c feroxbuster -n "__fish_use_subcommand" -l rate-limit -d 'Limit number of requests per second (per directory) (default: 0, i.e. no limit)'
 complete -c feroxbuster -n "__fish_use_subcommand" -l time-limit -d 'Limit total run time of all scans (ex: --time-limit 10m)'
 complete -c feroxbuster -n "__fish_use_subcommand" -s v -l verbosity -d 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v\'s is probably too much)'
 complete -c feroxbuster -n "__fish_use_subcommand" -l silent -d 'Only print URLs + turn off logging (good for piping a list of urls to other commands)'
 complete -c feroxbuster -n "__fish_use_subcommand" -s q -l quiet -d 'Hide progress bars and banner (good for tmux windows w/ notifications)'
+complete -c feroxbuster -n "__fish_use_subcommand" -l auto-tune -d 'Automatically lower scan rate when an excessive amount of errors are encountered'
+complete -c feroxbuster -n "__fish_use_subcommand" -l auto-bail -d 'Automatically stop scanning when an excessive amount of errors are encountered'
 complete -c feroxbuster -n "__fish_use_subcommand" -l json -d 'Emit JSON logs to --output and --debug-log instead of normal text'
 complete -c feroxbuster -n "__fish_use_subcommand" -s D -l dont-filter -d 'Don\'t auto-filter wildcard responses'
+complete -c feroxbuster -n "__fish_use_subcommand" -s A -l random-agent -d 'Use a random User-Agent'
 complete -c feroxbuster -n "__fish_use_subcommand" -s r -l redirects -d 'Follow redirects'
 complete -c feroxbuster -n "__fish_use_subcommand" -s k -l insecure -d 'Disables TLS certificate validation'
 complete -c feroxbuster -n "__fish_use_subcommand" -s n -l no-recursion -d 'Do not scan recursively'

src/banner/container.rs

@@ -1,8 +1,8 @@
 use super::entry::BannerEntry;
-use crate::event_handlers::Handles;
 use crate::{
     config::Configuration,
-    utils::{make_request, status_colorizer},
+    event_handlers::Handles,
+    utils::{logged_request, status_colorizer},
     VERSION,
 };
 use anyhow::{bail, Result};
@@ -50,6 +50,9 @@ pub struct Banner {
     /// represents Configuration.user_agent
     user_agent: BannerEntry,
 
+    /// represents Configuration.random_agent
+    random_agent: BannerEntry,
+
     /// represents Configuration.config
     config: BannerEntry,
 
@@ -125,6 +128,18 @@ pub struct Banner {
     /// represents Configuration.rate_limit
     rate_limit: BannerEntry,
 
+    /// represents Configuration.parallel
+    parallel: BannerEntry,
+
+    /// represents Configuration.auto_tune
+    auto_tune: BannerEntry,
+
+    /// represents Configuration.auto_bail
+    auto_bail: BannerEntry,
+
+    /// represents Configuration.url_denylist
+    url_denylist: Vec<BannerEntry>,
+
     /// current version of feroxbuster
     pub(super) version: String,
 
@@ -137,6 +152,7 @@ impl Banner {
     /// Create a new Banner from a Configuration and live targets
     pub fn new(tgts: &[String], config: &Configuration) -> Self {
         let mut targets = Vec::new();
+        let mut url_denylist = Vec::new();
         let mut code_filters = Vec::new();
         let mut replay_codes = Vec::new();
         let mut headers = Vec::new();
@@ -151,6 +167,22 @@ impl Banner {
             targets.push(BannerEntry::new("🎯", "Target Url", target));
         }
 
+        for denied_url in &config.url_denylist {
+            url_denylist.push(BannerEntry::new(
+                "🚫",
+                "Don't Scan Url",
+                denied_url.as_str(),
+            ));
+        }
+
+        for denied_regex in &config.regex_denylist {
+            url_denylist.push(BannerEntry::new(
+                "🚫",
+                "Don't Scan Regex",
+                denied_regex.as_str(),
+            ));
+        }
+
         let mut codes = vec![];
         for code in &config.status_codes {
             codes.push(status_colorizer(&code.to_string()))
@@ -162,7 +194,7 @@ impl Banner {
             code_filters.push(status_colorizer(&code.to_string()))
         }
         let filter_status = BannerEntry::new(
-            "🗑",
+            "💢",
             "Status Code Filters",
             &format!("[{}]", code_filters.join(", ")),
         );
@@ -251,12 +283,15 @@ impl Banner {
         );
 
         let replay_proxy = BannerEntry::new("🎥", "Replay Proxy", &config.replay_proxy);
+        let auto_tune = BannerEntry::new("🎶", "Auto Tune", &config.auto_tune.to_string());
+        let auto_bail = BannerEntry::new("🪣", "Auto Bail", &config.auto_bail.to_string());
         let cfg = BannerEntry::new("💉", "Config File", &config.config);
         let proxy = BannerEntry::new("💎", "Proxy", &config.proxy);
         let threads = BannerEntry::new("🚀", "Threads", &config.threads.to_string());
         let wordlist = BannerEntry::new("📖", "Wordlist", &config.wordlist);
         let timeout = BannerEntry::new("💥", "Timeout (secs)", &config.timeout.to_string());
         let user_agent = BannerEntry::new("🦡", "User-Agent", &config.user_agent);
+        let random_agent = BannerEntry::new("🦡", "User-Agent", "Random");
         let extract_links =
             BannerEntry::new("🔎", "Extract Links", &config.extract_links.to_string());
         let json = BannerEntry::new("🧔", "JSON Output", &config.json.to_string());
@@ -273,6 +308,7 @@ impl Banner {
             BannerEntry::new("🤪", "Filter Wildcards", &(!config.dont_filter).to_string());
         let add_slash = BannerEntry::new("🪓", "Add Slash", &config.add_slash.to_string());
         let time_limit = BannerEntry::new("🕖", "Time Limit", &config.time_limit);
+        let parallel = BannerEntry::new("🛤", "Parallel Scans", &config.parallel.to_string());
         let rate_limit =
             BannerEntry::new("🚧", "Requests per Second", &config.rate_limit.to_string());
 
@@ -284,6 +320,9 @@ impl Banner {
             filter_status,
             timeout,
             user_agent,
+            random_agent,
+            auto_bail,
+            auto_tune,
             proxy,
             replay_codes,
             replay_proxy,
@@ -294,6 +333,7 @@ impl Banner {
             filter_line_count,
             filter_regex,
             extract_links,
+            parallel,
             json,
             queries,
             output,
@@ -308,6 +348,7 @@ impl Banner {
             rate_limit,
             scan_limit,
             time_limit,
+            url_denylist,
             config: cfg,
             version: VERSION.to_string(),
             update_status: UpdateStatus::Unknown,
@@ -354,15 +395,8 @@ by Ben "epi" Risher {}                 ver: {}"#,
 
         let api_url = Url::parse(url)?;
 
-        let response = make_request(
-            &handles.config.client,
-            &api_url,
-            handles.config.output_level,
-            handles.stats.tx.clone(),
-        )
-        .await?;
-
-        let body = response.text().await?;
+        let result = logged_request(&api_url, handles.clone()).await?;
+        let body = result.text().await?;
 
         let json_response: Value = serde_json::from_str(&body)?;
 
@@ -405,6 +439,10 @@ by Ben "epi" Risher {}                 ver: {}"#,
             writeln!(&mut writer, "{}", target)?;
         }
 
+        for denied_url in &self.url_denylist {
+            writeln!(&mut writer, "{}", denied_url)?;
+        }
+
         writeln!(&mut writer, "{}", self.threads)?;
         writeln!(&mut writer, "{}", self.wordlist)?;
         writeln!(&mut writer, "{}", self.status_codes)?;
@@ -416,7 +454,12 @@ by Ben "epi" Risher {}                 ver: {}"#,
         }
 
         writeln!(&mut writer, "{}", self.timeout)?;
-        writeln!(&mut writer, "{}", self.user_agent)?;
+
+        if config.random_agent {
+            writeln!(&mut writer, "{}", self.random_agent)?;
+        } else {
+            writeln!(&mut writer, "{}", self.user_agent)?;
+        }
 
         // followed by the maybe printed or variably displayed values
         if !config.config.is_empty() {
@@ -486,6 +529,13 @@ by Ben "epi" Risher {}                 ver: {}"#,
             writeln!(&mut writer, "{}", self.insecure)?;
         }
 
+        if config.auto_bail {
+            writeln!(&mut writer, "{}", self.auto_bail)?;
+        }
+        if config.auto_tune {
+            writeln!(&mut writer, "{}", self.auto_tune)?;
+        }
+
         if config.redirects {
             writeln!(&mut writer, "{}", self.redirects)?;
         }
@@ -508,6 +558,10 @@ by Ben "epi" Risher {}                 ver: {}"#,
             writeln!(&mut writer, "{}", self.scan_limit)?;
         }
 
+        if config.parallel > 0 {
+            writeln!(&mut writer, "{}", self.parallel)?;
+        }
+
         if config.rate_limit > 0 {
             writeln!(&mut writer, "{}", self.rate_limit)?;
         }

src/banner/tests.rs

@@ -1,6 +1,6 @@
 use super::container::UpdateStatus;
 use super::*;
-use crate::{config::Configuration, event_handlers::Handles};
+use crate::{config::Configuration, event_handlers::Handles, scan_manager::FeroxScans};
 use httpmock::Method::GET;
 use httpmock::MockServer;
 use std::{io::stderr, sync::Arc, time::Duration};
@@ -73,8 +73,9 @@ async fn banner_needs_update_returns_up_to_date() {
         when.method(GET).path("/latest");
         then.status(200).body("{\"tag_name\":\"v1.1.0\"}");
     });
+    let scans = Arc::new(FeroxScans::default());
 
-    let handles = Arc::new(Handles::for_testing(None, None).0);
+    let handles = Arc::new(Handles::for_testing(Some(scans), None).0);
 
     let mut banner = Banner::new(&[srv.url("")], &Configuration::new().unwrap());
     banner.version = String::from("1.1.0");
@@ -95,7 +96,9 @@ async fn banner_needs_update_returns_out_of_date() {
         then.status(200).body("{\"tag_name\":\"v1.1.0\"}");
     });
 
-    let handles = Arc::new(Handles::for_testing(None, None).0);
+    let scans = Arc::new(FeroxScans::default());
+
+    let handles = Arc::new(Handles::for_testing(Some(scans), None).0);
 
     let mut banner = Banner::new(&[srv.url("")], &Configuration::new().unwrap());
     banner.version = String::from("1.0.1");

src/config/container.rs

@@ -1,15 +1,17 @@
 use super::utils::{
     depth, report_and_exit, save_state, serialized_type, status_codes, threads, timeout,
-    user_agent, wordlist, OutputLevel,
+    user_agent, wordlist, OutputLevel, RequesterPolicy,
 };
 use crate::config::determine_output_level;
+use crate::config::utils::determine_requester_policy;
 use crate::{
     client, parser, scan_manager::resume_scan, traits::FeroxSerialize, utils::fmt_err,
     DEFAULT_CONFIG_NAME,
 };
 use anyhow::{anyhow, Context, Result};
 use clap::{value_t, ArgMatches};
-use reqwest::{Client, StatusCode};
+use regex::Regex;
+use reqwest::{Client, StatusCode, Url};
 use serde::{Deserialize, Serialize};
 use std::{
     collections::HashMap,
@@ -124,6 +126,18 @@ pub struct Configuration {
     #[serde(skip)]
     pub output_level: OutputLevel,
 
+    /// automatically bail at certain error thresholds
+    #[serde(default)]
+    pub auto_bail: bool,
+
+    /// automatically try to lower request rate in order to reduce errors
+    #[serde(default)]
+    pub auto_tune: bool,
+
+    /// more easily differentiate between the three requester policies
+    #[serde(skip)]
+    pub requester_policy: RequesterPolicy,
+
     /// Store log output as NDJSON
     #[serde(default)]
     pub json: bool,
@@ -141,6 +155,10 @@ pub struct Configuration {
     #[serde(default = "user_agent")]
     pub user_agent: String,
 
+    /// Use random User-Agent
+    #[serde(default)]
+    pub random_agent: bool,
+
     /// Follow redirects
     #[serde(default)]
     pub redirects: bool,
@@ -185,6 +203,10 @@ pub struct Configuration {
     #[serde(default)]
     pub scan_limit: usize,
 
+    /// Number of parallel scans permitted; a limit of 0 means no limit is imposed
+    #[serde(default)]
+    pub parallel: usize,
+
     /// Number of requests per second permitted (per directory); a limit of 0 means no limit is imposed
     #[serde(default)]
     pub rate_limit: usize,
@@ -231,6 +253,13 @@ pub struct Configuration {
     /// Filter out response bodies that meet a certain threshold of similarity
     #[serde(default)]
     pub filter_similar: Vec<String>,
+
+    /// URLs that should never be scanned/recursed into
+    #[serde(default)]
+    pub url_denylist: Vec<Url>,
+
+    #[serde(with = "serde_regex", default)]
+    pub regex_denylist: Vec<Regex>,
 }
 
 impl Default for Configuration {
@@ -245,6 +274,7 @@ impl Default for Configuration {
         let replay_codes = status_codes.clone();
         let kind = serialized_type();
         let output_level = OutputLevel::Default;
+        let requester_policy = RequesterPolicy::Default;
 
         Configuration {
             kind,
@@ -254,7 +284,10 @@ impl Default for Configuration {
             replay_codes,
             status_codes,
             replay_client,
+            requester_policy,
             dont_filter: false,
+            auto_bail: false,
+            auto_tune: false,
             silent: false,
             quiet: false,
             output_level,
@@ -263,12 +296,14 @@ impl Default for Configuration {
             json: false,
             verbosity: 0,
             scan_limit: 0,
+            parallel: 0,
             rate_limit: 0,
             add_slash: false,
             insecure: false,
             redirects: false,
             no_recursion: false,
             extract_links: false,
+            random_agent: false,
             save_state: true,
             proxy: String::new(),
             config: String::new(),
@@ -282,6 +317,8 @@ impl Default for Configuration {
             extensions: Vec::new(),
             filter_size: Vec::new(),
             filter_regex: Vec::new(),
+            url_denylist: Vec::new(),
+            regex_denylist: Vec::new(),
             filter_line_count: Vec::new(),
             filter_word_count: Vec::new(),
             filter_status: Vec::new(),
@@ -313,10 +350,15 @@ impl Configuration {
     /// - **debug_log**: `None`
     /// - **quiet**: `false`
     /// - **silent**: `false`
+    /// - **auto_tune**: `false`
+    /// - **auto_bail**: `false`
     /// - **save_state**: `true`
     /// - **user_agent**: `feroxbuster/VERSION`
+    /// - **random_agent**: `false`
     /// - **insecure**: `false` (don't be insecure, i.e. don't allow invalid certs)
     /// - **extensions**: `None`
+    /// - **url_denylist**: `None`
+    /// - **regex_denylist**: `None`
     /// - **filter_size**: `None`
     /// - **filter_similar**: `None`
     /// - **filter_regex**: `None`
@@ -331,7 +373,8 @@ impl Configuration {
     /// - **dont_filter**: `false` (auto filter wildcard responses)
     /// - **depth**: `4` (maximum recursion depth)
     /// - **scan_limit**: `0` (no limit on concurrent scans imposed)
-    /// - **rate_limit**: `0` (no limit on concurrent scans imposed)
+    /// - **parallel**: `0` (no limit on parallel scans imposed)
+    /// - **rate_limit**: `0` (no limit on requests per second imposed)
     /// - **time_limit**: `None` (no limit on length of scan imposed)
     /// - **replay_proxy**: `None` (no limit on concurrent scans imposed)
     /// - **replay_codes**: [`DEFAULT_RESPONSE_CODES`](constant.DEFAULT_RESPONSE_CODES.html)
@@ -467,6 +510,7 @@ impl Configuration {
         update_config_if_present!(&mut config.threads, args, "threads", usize);
         update_config_if_present!(&mut config.depth, args, "depth", usize);
         update_config_if_present!(&mut config.scan_limit, args, "scan_limit", usize);
+        update_config_if_present!(&mut config.parallel, args, "parallel", usize);
         update_config_if_present!(&mut config.rate_limit, args, "rate_limit", usize);
         update_config_if_present!(&mut config.wordlist, args, "wordlist", String);
         update_config_if_present!(&mut config.output, args, "output", String);
@@ -512,6 +556,56 @@ impl Configuration {
             config.extensions = arg.map(|val| val.to_string()).collect();
         }
 
+        if args.is_present("stdin") {
+            config.stdin = true;
+        } else if let Some(url) = args.value_of("url") {
+            config.target_url = String::from(url);
+        }
+
+        if let Some(arg) = args.values_of("url_denylist") {
+            // compile all regular expressions and absolute urls used for --dont-scan
+            //
+            // when --dont-scan is used, the should_deny_url function is called at least once per
+            // url to be scanned. With the addition of regex support, I want to move parsing
+            // out of should_deny_url and into here, so it's performed once instead of thousands
+            // of times
+            for denier in arg.into_iter() {
+                // could be an absolute url or a regex, need to determine which and populate the
+                // appropriate vector
+                match Url::parse(denier.trim_end_matches('/')) {
+                    Ok(absolute) => {
+                        // denier is an absolute url and can be parsed as such
+                        config.url_denylist.push(absolute);
+                    }
+                    Err(err) => {
+                        // there are some expected errors that happen when we try to parse a url
+                        //     ex: Url::parse("/login") -> Err("relative URL without a base")
+                        //     ex: Url::parse("http:") -> Err("empty host")
+                        //
+                        // these are known errors and are used to determine a valid value to
+                        // --dont-scan, when it's not an absolute url
+                        //
+                        // when expected errors are encountered, we're going to assume
+                        // that the input is a regular expression to be parsed. The possibility
+                        // exists that the user rolled their face across the keyboard and we're
+                        // dealing with the results, in which case we'll report it as an error and
+                        // give up
+                        if err.to_string().contains("relative URL without a base")
+                            || err.to_string().contains("empty host")
+                        {
+                            let regex = Regex::new(denier)
+                                .unwrap_or_else(|e| report_and_exit(&e.to_string()));
+
+                            config.regex_denylist.push(regex);
+                        } else {
+                            // unexpected error has occurred; bail
+                            report_and_exit(&err.to_string());
+                        }
+                    }
+                }
+            }
+        }
+
         if let Some(arg) = args.values_of("filter_regex") {
             config.filter_regex = arg.map(|val| val.to_string()).collect();
         }
@@ -561,6 +655,16 @@ impl Configuration {
             config.output_level = OutputLevel::Quiet;
         }
 
+        if args.is_present("auto_tune") {
+            config.auto_tune = true;
+            config.requester_policy = RequesterPolicy::AutoTune;
+        }
+
+        if args.is_present("auto_bail") {
+            config.auto_bail = true;
+            config.requester_policy = RequesterPolicy::AutoBail;
+        }
+
         if args.is_present("dont_filter") {
             config.dont_filter = true;
         }
@@ -587,12 +691,6 @@ impl Configuration {
             config.json = true;
         }
 
-        if args.is_present("stdin") {
-            config.stdin = true;
-        } else if let Some(url) = args.value_of("url") {
-            config.target_url = String::from(url);
-        }
-
         ////
         // organizational breakpoint; all options below alter the Client configuration
         ////
@@ -601,6 +699,10 @@ impl Configuration {
         update_config_if_present!(&mut config.user_agent, args, "user_agent", String);
         update_config_if_present!(&mut config.timeout, args, "timeout", u64);
 
+        if args.is_present("random_agent") {
+            config.random_agent = true;
+        }
+
         if args.is_present("redirects") {
             config.redirects = true;
         }
@@ -721,13 +823,25 @@ impl Configuration {
         update_if_not_default!(&mut conf.verbosity, new.verbosity, 0);
         update_if_not_default!(&mut conf.silent, new.silent, false);
         update_if_not_default!(&mut conf.quiet, new.quiet, false);
-        // use updated quiet/silent values to determin output level
+        update_if_not_default!(&mut conf.auto_bail, new.auto_bail, false);
+        update_if_not_default!(&mut conf.auto_tune, new.auto_tune, false);
+        // use updated quiet/silent values to determine output level; same for requester policy
         conf.output_level = determine_output_level(conf.quiet, conf.silent);
+        conf.requester_policy = determine_requester_policy(conf.auto_tune, conf.auto_bail);
         update_if_not_default!(&mut conf.output, new.output, "");
         update_if_not_default!(&mut conf.redirects, new.redirects, false);
         update_if_not_default!(&mut conf.insecure, new.insecure, false);
         update_if_not_default!(&mut conf.extract_links, new.extract_links, false);
         update_if_not_default!(&mut conf.extensions, new.extensions, Vec::<String>::new());
+        update_if_not_default!(&mut conf.url_denylist, new.url_denylist, Vec::<Url>::new());
+        if !new.regex_denylist.is_empty() {
+            // cant use the update_if_not_default macro due to the following error
+            //
+            //    binary operation `!=` cannot be applied to type `Vec<regex::Regex>`
+            //
+            // if we get a non-empty list of regex in the new config, override the old
+            conf.regex_denylist = new.regex_denylist;
+        }
         update_if_not_default!(&mut conf.headers, new.headers, HashMap::new());
         update_if_not_default!(&mut conf.queries, new.queries, Vec::new());
         update_if_not_default!(&mut conf.no_recursion, new.no_recursion, false);
@@ -761,6 +875,7 @@ impl Configuration {
         );
         update_if_not_default!(&mut conf.dont_filter, new.dont_filter, false);
         update_if_not_default!(&mut conf.scan_limit, new.scan_limit, 0);
+        update_if_not_default!(&mut conf.parallel, new.parallel, 0);
         update_if_not_default!(&mut conf.rate_limit, new.rate_limit, 0);
         update_if_not_default!(&mut conf.replay_proxy, new.replay_proxy, "");
         update_if_not_default!(&mut conf.debug_log, new.debug_log, "");
@@ -769,6 +884,7 @@ impl Configuration {
 
         update_if_not_default!(&mut conf.timeout, new.timeout, timeout());
         update_if_not_default!(&mut conf.user_agent, new.user_agent, user_agent());
+        update_if_not_default!(&mut conf.random_agent, new.random_agent, false);
         update_if_not_default!(&mut conf.threads, new.threads, threads());
         update_if_not_default!(&mut conf.depth, new.depth, depth());
         update_if_not_default!(&mut conf.wordlist, new.wordlist, wordlist());

src/config/mod.rs

@@ -6,4 +6,4 @@ mod utils;
 mod tests;
 
 pub use self::container::Configuration;
-pub use self::utils::{determine_output_level, OutputLevel};
+pub use self::utils::{determine_output_level, OutputLevel, RequesterPolicy};

src/config/tests.rs

@@ -1,6 +1,8 @@
 use super::utils::*;
 use super::*;
 use crate::{traits::FeroxSerialize, DEFAULT_CONFIG_NAME};
+use regex::Regex;
+use reqwest::Url;
 use std::{collections::HashMap, fs::write};
 use tempfile::TempDir;
 
@@ -16,8 +18,11 @@ fn setup_config_test() -> Configuration {
             replay_proxy = "http://127.0.0.1:8081"
             quiet = true
             silent = true
+            auto_tune = true
+            auto_bail = true
             verbosity = 1
             scan_limit = 6
+            parallel = 14
             rate_limit = 250
             time_limit = "10m"
             output = "/some/otherpath"
@@ -26,6 +31,8 @@ fn setup_config_test() -> Configuration {
             redirects = true
             insecure = true
             extensions = ["html", "php", "js"]
+            url_denylist = ["http://dont-scan.me", "https://also-not.me"]
+            regex_denylist = ["/deny.*"]
             headers = {stuff = "things", mostuff = "mothings"}
             queries = [["name","value"], ["rick", "astley"]]
             no_recursion = true
@@ -69,20 +76,27 @@ fn default_configuration() {
     assert_eq!(config.timeout, timeout());
     assert_eq!(config.verbosity, 0);
     assert_eq!(config.scan_limit, 0);
-    assert_eq!(config.silent, false);
-    assert_eq!(config.quiet, false);
-    assert_eq!(config.dont_filter, false);
-    assert_eq!(config.no_recursion, false);
-    assert_eq!(config.json, false);
-    assert_eq!(config.save_state, true);
-    assert_eq!(config.stdin, false);
-    assert_eq!(config.add_slash, false);
-    assert_eq!(config.redirects, false);
-    assert_eq!(config.extract_links, false);
-    assert_eq!(config.insecure, false);
+    assert!(!config.silent);
+    assert!(!config.quiet);
+    assert_eq!(config.output_level, OutputLevel::Default);
+    assert!(!config.dont_filter);
+    assert!(!config.auto_tune);
+    assert!(!config.auto_bail);
+    assert_eq!(config.requester_policy, RequesterPolicy::Default);
+    assert!(!config.no_recursion);
+    assert!(!config.random_agent);
+    assert!(!config.json);
+    assert!(config.save_state);
+    assert!(!config.stdin);
+    assert!(!config.add_slash);
+    assert!(!config.redirects);
+    assert!(!config.extract_links);
+    assert!(!config.insecure);
+    assert!(config.regex_denylist.is_empty());
     assert_eq!(config.queries, Vec::new());
-    assert_eq!(config.extensions, Vec::<String>::new());
     assert_eq!(config.filter_size, Vec::<u64>::new());
+    assert_eq!(config.extensions, Vec::<String>::new());
+    assert_eq!(config.url_denylist, Vec::<Url>::new());
     assert_eq!(config.filter_regex, Vec::<String>::new());
     assert_eq!(config.filter_similar, Vec::<String>::new());
     assert_eq!(config.filter_word_count, Vec::<usize>::new());
@@ -140,6 +154,13 @@ fn config_reads_scan_limit() {
     assert_eq!(config.scan_limit, 6);
 }
 
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_parallel() {
+    let config = setup_config_test();
+    assert_eq!(config.parallel, 14);
+}
+
 #[test]
 /// parse the test config and see that the value parsed is correct
 fn config_reads_rate_limit() {
@@ -172,21 +193,35 @@ fn config_reads_replay_proxy() {
 /// parse the test config and see that the value parsed is correct
 fn config_reads_silent() {
     let config = setup_config_test();
-    assert_eq!(config.silent, true);
+    assert!(config.silent);
 }
 
 #[test]
 /// parse the test config and see that the value parsed is correct
 fn config_reads_quiet() {
     let config = setup_config_test();
-    assert_eq!(config.quiet, true);
+    assert!(config.quiet);
 }
 
 #[test]
 /// parse the test config and see that the value parsed is correct
 fn config_reads_json() {
     let config = setup_config_test();
-    assert_eq!(config.json, true);
+    assert!(config.json);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_auto_bail() {
+    let config = setup_config_test();
+    assert!(config.auto_bail);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_auto_tune() {
+    let config = setup_config_test();
+    assert!(config.auto_tune);
 }
 
 #[test]
@@ -207,49 +242,49 @@ fn config_reads_output() {
 /// parse the test config and see that the value parsed is correct
 fn config_reads_redirects() {
     let config = setup_config_test();
-    assert_eq!(config.redirects, true);
+    assert!(config.redirects);
 }
 
 #[test]
 /// parse the test config and see that the value parsed is correct
 fn config_reads_insecure() {
     let config = setup_config_test();
-    assert_eq!(config.insecure, true);
+    assert!(config.insecure);
 }
 
 #[test]
 /// parse the test config and see that the value parsed is correct
 fn config_reads_no_recursion() {
     let config = setup_config_test();
-    assert_eq!(config.no_recursion, true);
+    assert!(config.no_recursion);
 }
 
 #[test]
 /// parse the test config and see that the value parsed is correct
 fn config_reads_stdin() {
     let config = setup_config_test();
-    assert_eq!(config.stdin, true);
+    assert!(config.stdin);
 }
 
 #[test]
 /// parse the test config and see that the value parsed is correct
 fn config_reads_dont_filter() {
     let config = setup_config_test();
-    assert_eq!(config.dont_filter, true);
+    assert!(config.dont_filter);
 }
 
 #[test]
 /// parse the test config and see that the value parsed is correct
 fn config_reads_add_slash() {
     let config = setup_config_test();
-    assert_eq!(config.add_slash, true);
+    assert!(config.add_slash);
 }
 
 #[test]
 /// parse the test config and see that the value parsed is correct
 fn config_reads_extract_links() {
     let config = setup_config_test();
-    assert_eq!(config.extract_links, true);
+    assert!(config.extract_links);
 }
 
 #[test]
@@ -259,6 +294,29 @@ fn config_reads_extensions() {
     assert_eq!(config.extensions, vec!["html", "php", "js"]);
 }
 
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_regex_denylist() {
+    let config = setup_config_test();
+    assert_eq!(
+        config.regex_denylist[0].as_str(),
+        Regex::new("/deny.*").unwrap().as_str()
+    );
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_url_denylist() {
+    let config = setup_config_test();
+    assert_eq!(
+        config.url_denylist,
+        vec![
+            Url::parse("http://dont-scan.me").unwrap(),
+            Url::parse("https://also-not.me").unwrap(),
+        ]
+    );
+}
+
 #[test]
 /// parse the test config and see that the value parsed is correct
 fn config_reads_filter_regex() {
@@ -305,7 +363,7 @@ fn config_reads_filter_status() {
 /// parse the test config and see that the value parsed is correct
 fn config_reads_save_state() {
     let config = setup_config_test();
-    assert_eq!(config.save_state, false);
+    assert!(!config.save_state);
 }
 
 #[test]
@@ -336,12 +394,19 @@ fn config_reads_headers() {
 /// parse the test config and see that the values parsed are correct
 fn config_reads_queries() {
     let config = setup_config_test();
-    let mut queries = vec![];
-    queries.push(("name".to_string(), "value".to_string()));
-    queries.push(("rick".to_string(), "astley".to_string()));
+    let queries = vec![
+        ("name".to_string(), "value".to_string()),
+        ("rick".to_string(), "astley".to_string()),
+    ];
     assert_eq!(config.queries, queries);
 }
 
+#[test]
+fn config_default_not_random_agent() {
+    let config = setup_config_test();
+    assert!(!config.random_agent);
+}
+
 #[test]
 #[should_panic]
 /// test that an error message is printed and panic is called when report_and_exit is called

src/config/utils.rs

@@ -102,6 +102,41 @@ pub fn determine_output_level(quiet: bool, silent: bool) -> OutputLevel {
     }
 }
 
+/// represents actions the Requester should take in certain situations
+#[derive(Debug, PartialEq, Copy, Clone)]
+pub enum RequesterPolicy {
+    /// automatically try to lower request rate in order to reduce errors
+    AutoTune,
+
+    /// automatically bail at certain error thresholds
+    AutoBail,
+
+    /// just let that junk run super natural
+    Default,
+}
+
+/// default implementation for RequesterPolicy
+impl Default for RequesterPolicy {
+    /// Default as default
+    fn default() -> Self {
+        Self::Default
+    }
+}
+
+/// given the current settings for quiet and silent, determine output_level (DRY helper)
+pub fn determine_requester_policy(auto_tune: bool, auto_bail: bool) -> RequesterPolicy {
+    if auto_tune && auto_bail {
+        // user COULD have both as true in config file, take the more aggressive of the two
+        RequesterPolicy::AutoBail
+    } else if auto_tune {
+        RequesterPolicy::AutoTune
+    } else if auto_bail {
+        RequesterPolicy::AutoBail
+    } else {
+        RequesterPolicy::Default
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -122,6 +157,22 @@ mod tests {
         assert_eq!(level, OutputLevel::Quiet);
     }
 
+    #[test]
+    /// test determine_requester_policy returns higher of the two levels if both given values are true
+    fn determine_requester_policy_returns_correct_results() {
+        let mut level = determine_requester_policy(true, true);
+        assert_eq!(level, RequesterPolicy::AutoBail);
+
+        level = determine_requester_policy(false, true);
+        assert_eq!(level, RequesterPolicy::AutoBail);
+
+        level = determine_requester_policy(false, false);
+        assert_eq!(level, RequesterPolicy::Default);
+
+        level = determine_requester_policy(true, false);
+        assert_eq!(level, RequesterPolicy::AutoTune);
+    }
+
     #[test]
     #[should_panic]
     /// report_and_exit should panic/exit when called

src/event_handlers/command.rs

@@ -1,4 +1,3 @@
-use std::collections::HashSet;
 use std::sync::Arc;
 
 use reqwest::StatusCode;
@@ -25,11 +24,14 @@ pub enum Command {
     /// Create the progress bar (`BarType::Total`) that is updated from the stats thread
     CreateBar,
 
-    /// Update a `Stats` field that corresponds to the given `StatField` by the given `usize` value
-    UpdateUsizeField(StatField, usize),
+    /// Add to a `Stats` field that corresponds to the given `StatField` by the given `usize` value
+    AddToUsizeField(StatField, usize),
+
+    /// Subtract from a `Stats` field that corresponds to the given `StatField` by the given `usize` value
+    SubtractFromUsizeField(StatField, usize),
 
     /// Update a `Stats` field that corresponds to the given `StatField` by the given `f64` value
-    UpdateF64Field(StatField, f64),
+    AddToF64Field(StatField, f64),
 
     /// Save a `Stats` object to disk using `reporter::get_cached_file_handle`
     Save,
@@ -50,7 +52,7 @@ pub enum Command {
     TryRecursion(Box<FeroxResponse>),
 
     /// Send a pointer to the wordlist to the recursion handler
-    UpdateWordlist(Arc<HashSet<String>>),
+    UpdateWordlist(Arc<Vec<String>>),
 
     /// Instruct the ScanHandler to join on all known scans, use sender to notify main when done
     JoinTasks(Sender<bool>),

src/event_handlers/inputs.rs

@@ -4,6 +4,7 @@ use crate::{
     scan_manager::{FeroxState, PAUSE_SCAN},
     scanner::RESPONSES,
     statistics::StatError,
+    utils::slugify_filename,
     utils::{open_file, write_to},
     SLEEP_DURATION,
 };
@@ -17,7 +18,6 @@ use std::{
     },
     thread::sleep,
     time::Duration,
-    time::{SystemTime, UNIX_EPOCH},
 };
 
 /// Atomic boolean flag, used to determine whether or not the terminal input handler should exit
@@ -77,22 +77,14 @@ impl TermInputHandler {
     pub fn sigint_handler(handles: Arc<Handles>) -> Result<()> {
         log::trace!("enter: sigint_handler({:?})", handles);
 
-        let ts = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs();
-
-        let slug = if !handles.config.target_url.is_empty() {
+        let filename = if !handles.config.target_url.is_empty() {
             // target url populated
-            handles
-                .config
-                .target_url
-                .replace("://", "_")
-                .replace("/", "_")
-                .replace(".", "_")
+            slugify_filename(&handles.config.target_url, "ferox", "state")
         } else {
             // stdin used
-            "stdin".to_string()
+            slugify_filename("stdin", "ferox", "state")
         };
 
-        let filename = format!("ferox-{}-{}.state", slug, ts);
         let warning = format!(
             "🚨 Caught {} 🚨 saving scan state to {} ...",
             style("ctrl+c").yellow(),

src/event_handlers/outputs.rs

@@ -1,4 +1,4 @@
-use super::Command::UpdateUsizeField;
+use super::Command::AddToUsizeField;
 use super::*;
 
 use anyhow::{Context, Result};
@@ -139,8 +139,8 @@ impl TermOutHandler {
         Self {
             receiver,
             tx_file,
-            config,
             file_task,
+            config,
         }
     }
 
@@ -195,7 +195,7 @@ impl TermOutHandler {
                         // print to stdout
                         ferox_print(&resp.as_str(), &PROGRESS_PRINTER);
 
-                        send_command!(tx_stats, UpdateUsizeField(ResourcesDiscovered, 1));
+                        send_command!(tx_stats, AddToUsizeField(ResourcesDiscovered, 1));
 
                         if self.file_task.is_some() {
                             // -o used, need to send the report to be written out to disk
@@ -210,11 +210,12 @@ impl TermOutHandler {
 
                     if self.config.replay_client.is_some() && should_process_response {
                         // replay proxy specified/client created and this response's status code is one that
-                        // should be replayed
+                        // should be replayed; not using logged_request due to replay proxy client
                         make_request(
                             self.config.replay_client.as_ref().unwrap(),
-                            &resp.url(),
+                            resp.url(),
                             self.config.output_level,
+                            &self.config,
                             tx_stats.clone(),
                         )
                         .await

src/event_handlers/scans.rs

@@ -1,20 +1,22 @@
-use std::collections::HashSet;
 use std::sync::Arc;
 
 use anyhow::{bail, Result};
 use tokio::sync::{mpsc, Semaphore};
 
-use crate::response::FeroxResponse;
-use crate::url::FeroxUrl;
 use crate::{
+    response::FeroxResponse,
     scan_manager::{FeroxScan, FeroxScans, ScanOrder},
     scanner::FeroxScanner,
     statistics::StatField::TotalScans,
-    CommandReceiver, CommandSender, FeroxChannel, Joiner,
+    url::FeroxUrl,
+    utils::should_deny_url,
+    CommandReceiver, CommandSender, FeroxChannel, Joiner, SLEEP_DURATION,
 };
 
-use super::command::Command::UpdateUsizeField;
+use super::command::Command::AddToUsizeField;
 use super::*;
+use reqwest::Url;
+use tokio::time::Duration;
 
 #[derive(Debug)]
 /// Container for recursion transmitter and FeroxScans object
@@ -53,7 +55,7 @@ pub struct ScanHandler {
     receiver: CommandReceiver,
 
     /// wordlist (re)used for each scan
-    wordlist: std::sync::Mutex<Option<Arc<HashSet<String>>>>,
+    wordlist: std::sync::Mutex<Option<Arc<Vec<String>>>>,
 
     /// group of scans that need to be joined
     tasks: Vec<Arc<FeroxScan>>,
@@ -104,7 +106,7 @@ impl ScanHandler {
     }
 
     /// Set the wordlist
-    fn wordlist(&self, wordlist: Arc<HashSet<String>>) {
+    fn wordlist(&self, wordlist: Arc<Vec<String>>) {
         if let Ok(mut guard) = self.wordlist.lock() {
             if guard.is_none() {
                 let _ = std::mem::replace(&mut *guard, Some(wordlist));
@@ -153,9 +155,7 @@ impl ScanHandler {
 
                     tokio::spawn(async move {
                         while ferox_scans.has_active_scans() {
-                            for scan in ferox_scans.get_active_scans() {
-                                scan.join().await;
-                            }
+                            tokio::time::sleep(Duration::from_millis(SLEEP_DURATION + 250)).await;
                         }
                         limiter_clone.close();
                         sender.send(true).expect("oneshot channel failed");
@@ -176,7 +176,7 @@ impl ScanHandler {
     }
 
     /// Helper to easily get the (locked) underlying wordlist
-    pub fn get_wordlist(&self) -> Result<Arc<HashSet<String>>> {
+    pub fn get_wordlist(&self) -> Result<Arc<Vec<String>>> {
         if let Ok(guard) = self.wordlist.lock().as_ref() {
             if let Some(list) = guard.as_ref() {
                 return Ok(list.clone());
@@ -189,6 +189,8 @@ impl ScanHandler {
     /// wrapper around scanning a url to stay DRY
     async fn ordered_scan_url(&mut self, targets: Vec<String>, order: ScanOrder) -> Result<()> {
         log::trace!("enter: ordered_scan_url({:?}, {:?})", targets, order);
+        let should_test_deny = !self.handles.config.url_denylist.is_empty()
+            || !self.handles.config.regex_denylist.is_empty();
 
         for target in targets {
             if self.data.contains(&target) && matches!(order, ScanOrder::Latest) {
@@ -205,6 +207,13 @@ impl ScanHandler {
                 self.data.add_directory_scan(&target, order).1 // add the new target; return FeroxScan
             };
 
+            if should_test_deny && should_deny_url(&Url::parse(&target)?, self.handles.clone())? {
+                // response was caught by a user-provided deny list
+                // checking this last, since it's most susceptible to longer runtimes due to what
+                // input is received
+                continue;
+            }
+
             let list = self.get_wordlist()?;
 
             log::info!("scan handler received {} - beginning scan", target);
@@ -231,7 +240,7 @@ impl ScanHandler {
                 }
             });
 
-            self.handles.stats.send(UpdateUsizeField(TotalScans, 1))?;
+            self.handles.stats.send(AddToUsizeField(TotalScans, 1))?;
 
             scan.set_task(task).await?;
 
@@ -245,6 +254,11 @@ impl ScanHandler {
     async fn try_recursion(&mut self, response: Box<FeroxResponse>) -> Result<()> {
         log::trace!("enter: try_recursion({:?})", response,);
 
+        if !response.is_directory() {
+            // not a directory, quick exit
+            return Ok(());
+        }
+
         let mut base_depth = 1_usize;
 
         for (base_url, base_url_depth) in &self.depths {
@@ -258,11 +272,6 @@ impl ScanHandler {
             return Ok(());
         }
 
-        if !response.is_directory() {
-            // not a directory
-            return Ok(());
-        }
-
         let targets = vec![response.url().to_string()];
         self.ordered_scan_url(targets, ScanOrder::Latest).await?;
 

src/event_handlers/statistics.rs

@@ -89,6 +89,7 @@ impl StatsHandler {
                 }
                 Command::AddStatus(status) => {
                     self.stats.add_status_code(status);
+
                     self.increment_bar();
                 }
                 Command::AddRequest => {
@@ -99,14 +100,21 @@ impl StatsHandler {
                     self.stats
                         .save(start.elapsed().as_secs_f64(), output_file)?;
                 }
-                Command::UpdateUsizeField(field, value) => {
+                Command::AddToUsizeField(field, value) => {
                     self.stats.update_usize_field(field, value);
 
                     if matches!(field, StatField::TotalScans) {
                         self.bar.set_length(self.stats.total_expected() as u64);
                     }
                 }
-                Command::UpdateF64Field(field, value) => self.stats.update_f64_field(field, value),
+                Command::SubtractFromUsizeField(field, value) => {
+                    self.stats.subtract_from_usize_field(field, value);
+
+                    if matches!(field, StatField::TotalExpected) {
+                        self.bar.set_length(self.stats.total_expected() as u64);
+                    }
+                }
+                Command::AddToF64Field(field, value) => self.stats.update_f64_field(field, value),
                 Command::CreateBar => {
                     self.bar = add_bar("", self.stats.total_expected() as u64, BarType::Total);
                 }

src/extractor/container.rs

@@ -1,9 +1,10 @@
 use super::*;
+use crate::utils::should_deny_url;
 use crate::{
     client,
     event_handlers::{
         Command,
-        Command::{AddError, UpdateUsizeField},
+        Command::{AddError, AddToUsizeField},
         Handles,
     },
     scan_manager::ScanOrder,
@@ -12,7 +13,7 @@ use crate::{
         StatField::{LinksExtracted, TotalExpected},
     },
     url::FeroxUrl,
-    utils::make_request,
+    utils::{logged_request, make_request},
 };
 use anyhow::{bail, Context, Result};
 use reqwest::{StatusCode, Url};
@@ -53,13 +54,19 @@ pub struct Extractor<'a> {
 
 /// Extractor implementation
 impl<'a> Extractor<'a> {
-    /// business logic that handles getting links from a normal http body response
-    pub async fn extract(&self) -> Result<()> {
-        let links = match self.target {
-            ExtractionTarget::ResponseBody => self.extract_from_body().await?,
-            ExtractionTarget::RobotsTxt => self.extract_from_robots().await?,
-        };
+    /// perform extraction from the given target and return any links found
+    pub async fn extract(&self) -> Result<HashSet<String>> {
+        log::trace!("enter: extract (this fn has associated trace exit msg)");
+        match self.target {
+            ExtractionTarget::ResponseBody => Ok(self.extract_from_body().await?),
+            ExtractionTarget::RobotsTxt => Ok(self.extract_from_robots().await?),
+        }
+    }
 
+    /// given a set of links from a normal http body response, task the request handler to make
+    /// the requests
+    pub async fn request_links(&self, links: HashSet<String>) -> Result<()> {
+        log::trace!("enter: request_links({:?})", links);
         let recursive = if self.handles.config.no_recursion {
             RecursionStatus::NotRecursive
         } else {
@@ -121,6 +128,7 @@ impl<'a> Extractor<'a> {
                 rx.await?;
             }
         }
+        log::trace!("exit: request_links");
         Ok(())
     }
 
@@ -141,7 +149,7 @@ impl<'a> Extractor<'a> {
 
         let body = self.response.unwrap().text();
 
-        for capture in self.links_regex.captures_iter(&body) {
+        for capture in self.links_regex.captures_iter(body) {
             // remove single & double quotes from both ends of the capture
             // capture[0] is the entire match, additional capture groups start at [1]
             let link = capture[0].trim_matches(|c| c == '\'' || c == '"');
@@ -267,7 +275,7 @@ impl<'a> Extractor<'a> {
         };
 
         let new_url = old_url
-            .join(&link)
+            .join(link)
             .with_context(|| format!("Could not join {} with {}", old_url, link))?;
 
         links.insert(new_url.to_string());
@@ -289,10 +297,10 @@ impl<'a> Extractor<'a> {
     pub(super) async fn request_link(&self, url: &str) -> Result<FeroxResponse> {
         log::trace!("enter: request_link({})", url);
 
-        let ferox_url = FeroxUrl::from_string(&url, self.handles.clone());
+        let ferox_url = FeroxUrl::from_string(url, self.handles.clone());
 
         // create a url based on the given command line options
-        let new_url = ferox_url.format(&"", None)?;
+        let new_url = ferox_url.format("", None)?;
 
         let scanned_urls = self.handles.ferox_scans()?;
 
@@ -302,14 +310,21 @@ impl<'a> Extractor<'a> {
             bail!("previously seen url");
         }
 
+        if (!self.handles.config.url_denylist.is_empty()
+            || !self.handles.config.regex_denylist.is_empty())
+            && should_deny_url(&new_url, self.handles.clone())?
+        {
+            // can't allow a denied url to be requested
+            bail!(
+                "prevented request to {} due to {:?} || {:?}",
+                url,
+                self.handles.config.url_denylist,
+                self.handles.config.regex_denylist,
+            );
+        }
+
         // make the request and store the response
-        let new_response = make_request(
-            &self.handles.config.client,
-            &new_url,
-            self.handles.config.output_level,
-            self.handles.stats.tx.clone(),
-        )
-        .await?;
+        let new_response = logged_request(&new_url, self.handles.clone()).await?;
 
         let new_ferox_response =
             FeroxResponse::from(new_response, true, self.handles.config.output_level).await;
@@ -338,7 +353,7 @@ impl<'a> Extractor<'a> {
             if let Some(new_path) = capture.name("url_path") {
                 let mut new_url = Url::parse(&self.url)?;
                 new_url.set_path(new_path.as_str());
-                if self.add_all_sub_paths(&new_url.path(), &mut links).is_err() {
+                if self.add_all_sub_paths(new_url.path(), &mut links).is_err() {
                     log::warn!("could not add sub-paths from {} to {:?}", new_url, links);
                 }
             }
@@ -384,18 +399,21 @@ impl<'a> Extractor<'a> {
         let mut url = Url::parse(&self.url)?;
         url.set_path("/robots.txt"); // overwrite existing path with /robots.txt
 
+        // purposefully not using logged_request here due to using the special client
         let response = make_request(
             &client,
             &url,
             self.handles.config.output_level,
+            &self.handles.config,
             self.handles.stats.tx.clone(),
         )
         .await?;
+
         let ferox_response =
             FeroxResponse::from(response, true, self.handles.config.output_level).await;
 
         log::trace!("exit: get_robots_file -> {}", ferox_response);
-        return Ok(ferox_response);
+        Ok(ferox_response)
     }
 
     /// update total number of links extracted and expected responses
@@ -404,10 +422,10 @@ impl<'a> Extractor<'a> {
 
         self.handles
             .stats
-            .send(UpdateUsizeField(LinksExtracted, num_links))?;
+            .send(AddToUsizeField(LinksExtracted, num_links))?;
         self.handles
             .stats
-            .send(UpdateUsizeField(TotalExpected, num_links * multiplier))?;
+            .send(AddToUsizeField(TotalExpected, num_links * multiplier))?;
 
         Ok(())
     }

src/extractor/tests.rs

@@ -54,8 +54,8 @@ fn setup_extractor(target: ExtractionTarget, scanned_urls: Arc<FeroxScans>) -> E
 /// in the expected array
 fn extractor_get_sub_paths_from_path_with_multiple_paths() {
     let path = "homepage/assets/img/icons/handshake.svg";
-    let r_paths = ROBOTS_EXT.get_sub_paths_from_path(&path);
-    let b_paths = BODY_EXT.get_sub_paths_from_path(&path);
+    let r_paths = ROBOTS_EXT.get_sub_paths_from_path(path);
+    let b_paths = BODY_EXT.get_sub_paths_from_path(path);
     let expected = vec![
         "homepage/",
         "homepage/assets/",
@@ -67,8 +67,8 @@ fn extractor_get_sub_paths_from_path_with_multiple_paths() {
     assert_eq!(r_paths.len(), expected.len());
     assert_eq!(b_paths.len(), expected.len());
     for expected_path in expected {
-        assert_eq!(r_paths.contains(&expected_path.to_string()), true);
-        assert_eq!(b_paths.contains(&expected_path.to_string()), true);
+        assert!(r_paths.contains(&expected_path.to_string()));
+        assert!(b_paths.contains(&expected_path.to_string()));
     }
 }
 
@@ -78,15 +78,15 @@ fn extractor_get_sub_paths_from_path_with_multiple_paths() {
 /// returned
 fn extractor_get_sub_paths_from_path_with_enclosing_slashes() {
     let path = "/homepage/assets/";
-    let r_paths = ROBOTS_EXT.get_sub_paths_from_path(&path);
-    let b_paths = BODY_EXT.get_sub_paths_from_path(&path);
+    let r_paths = ROBOTS_EXT.get_sub_paths_from_path(path);
+    let b_paths = BODY_EXT.get_sub_paths_from_path(path);
     let expected = vec!["homepage/", "homepage/assets"];
 
     assert_eq!(r_paths.len(), expected.len());
     assert_eq!(b_paths.len(), expected.len());
     for expected_path in expected {
-        assert_eq!(r_paths.contains(&expected_path.to_string()), true);
-        assert_eq!(b_paths.contains(&expected_path.to_string()), true);
+        assert!(r_paths.contains(&expected_path.to_string()));
+        assert!(b_paths.contains(&expected_path.to_string()));
     }
 }
 
@@ -95,15 +95,15 @@ fn extractor_get_sub_paths_from_path_with_enclosing_slashes() {
 /// included
 fn extractor_get_sub_paths_from_path_with_only_a_word() {
     let path = "homepage";
-    let r_paths = ROBOTS_EXT.get_sub_paths_from_path(&path);
-    let b_paths = BODY_EXT.get_sub_paths_from_path(&path);
+    let r_paths = ROBOTS_EXT.get_sub_paths_from_path(path);
+    let b_paths = BODY_EXT.get_sub_paths_from_path(path);
     let expected = vec!["homepage"];
 
     assert_eq!(r_paths.len(), expected.len());
     assert_eq!(b_paths.len(), expected.len());
     for expected_path in expected {
-        assert_eq!(r_paths.contains(&expected_path.to_string()), true);
-        assert_eq!(b_paths.contains(&expected_path.to_string()), true);
+        assert!(r_paths.contains(&expected_path.to_string()));
+        assert!(b_paths.contains(&expected_path.to_string()));
     }
 }
 
@@ -111,15 +111,15 @@ fn extractor_get_sub_paths_from_path_with_only_a_word() {
 /// extract sub paths from the given url fragment; expect 1 sub path, forward slash removed
 fn extractor_get_sub_paths_from_path_with_an_absolute_word() {
     let path = "/homepage";
-    let r_paths = ROBOTS_EXT.get_sub_paths_from_path(&path);
-    let b_paths = BODY_EXT.get_sub_paths_from_path(&path);
+    let r_paths = ROBOTS_EXT.get_sub_paths_from_path(path);
+    let b_paths = BODY_EXT.get_sub_paths_from_path(path);
     let expected = vec!["homepage"];
 
     assert_eq!(r_paths.len(), expected.len());
     assert_eq!(b_paths.len(), expected.len());
     for expected_path in expected {
-        assert_eq!(r_paths.contains(&expected_path.to_string()), true);
-        assert_eq!(b_paths.contains(&expected_path.to_string()), true);
+        assert!(r_paths.contains(&expected_path.to_string()));
+        assert!(b_paths.contains(&expected_path.to_string()));
     }
 }
 
@@ -211,16 +211,23 @@ async fn extractor_get_links_with_absolute_url_that_differs_from_target_domain()
     let mock = srv.mock(|when, then| {
         when.method(GET).path("/some-path");
         then.status(200).body(
-            "\"http://defintely.not.a.thing.probably.com/homepage/assets/img/icons/handshake.svg\"",
+            "\"http://definitely.not.a.thing.probably.com/homepage/assets/img/icons/handshake.svg\"",
         );
     });
 
     let client = Client::new();
     let url = Url::parse(&srv.url("/some-path")).unwrap();
+    let config = Configuration::new().unwrap();
 
-    let response = make_request(&client, &url, OutputLevel::Default, tx_stats.clone())
-        .await
-        .unwrap();
+    let response = make_request(
+        &client,
+        &url,
+        OutputLevel::Default,
+        &config,
+        tx_stats.clone(),
+    )
+    .await
+    .unwrap();
     let (handles, _rx) = Handles::for_testing(None, None);
 
     let handles = Arc::new(handles);

src/filters/container.rs

@@ -4,7 +4,7 @@ use anyhow::Result;
 
 use crate::response::FeroxResponse;
 use crate::{
-    event_handlers::Command::UpdateUsizeField, statistics::StatField::WildcardsFiltered,
+    event_handlers::Command::AddToUsizeField, statistics::StatField::WildcardsFiltered,
     CommandSender,
 };
 
@@ -41,10 +41,10 @@ impl FeroxFilters {
         if let Ok(filters) = self.filters.lock() {
             for filter in filters.iter() {
                 // wildcard.should_filter goes here
-                if filter.should_filter_response(&response) {
+                if filter.should_filter_response(response) {
                     if filter.as_any().downcast_ref::<WildcardFilter>().is_some() {
                         tx_stats
-                            .send(UpdateUsizeField(WildcardsFiltered, 1))
+                            .send(AddToUsizeField(WildcardsFiltered, 1))
                             .unwrap_or_default();
                     }
                     return true;

src/filters/init.rs

@@ -5,7 +5,7 @@ use crate::{
     event_handlers::Handles,
     response::FeroxResponse,
     skip_fail,
-    utils::{fmt_err, make_request},
+    utils::{fmt_err, logged_request},
     Command::AddFilter,
     SIMILARITY_THRESHOLD,
 };
@@ -56,7 +56,7 @@ pub async fn initialize(handles: Arc<Handles>) -> Result<()> {
     // add any regex filters to filters handler's FeroxFilters  (-X|--filter-regex)
     for regex_filter in &handles.config.filter_regex {
         let raw = regex_filter;
-        let compiled = skip_fail!(Regex::new(&raw));
+        let compiled = skip_fail!(Regex::new(raw));
 
         let filter = RegexFilter {
             raw_string: raw.to_owned(),
@@ -69,18 +69,10 @@ pub async fn initialize(handles: Arc<Handles>) -> Result<()> {
     // add any similarity filters to filters handler's FeroxFilters  (--filter-similar-to)
     for similarity_filter in &handles.config.filter_similar {
         // url as-is based on input, ignores user-specified url manipulation options (add-slash etc)
-        let url = skip_fail!(Url::parse(&similarity_filter));
+        let url = skip_fail!(Url::parse(similarity_filter));
 
         // attempt to request the given url
-        let resp = skip_fail!(
-            make_request(
-                &handles.config.client,
-                &url,
-                handles.config.output_level,
-                handles.stats.tx.clone()
-            )
-            .await
-        );
+        let resp = skip_fail!(logged_request(&url, handles.clone()).await);
 
         // if successful, create a filter based on the response's body
         let fr = FeroxResponse::from(resp, true, handles.config.output_level).await;

src/filters/tests.rs

@@ -127,6 +127,19 @@ fn wildcard_should_filter_when_static_wildcard_found() {
     assert!(filter.should_filter_response(&resp));
 }
 
+#[test]
+/// test should_filter on WilcardFilter where static logic matches but response length is 0
+fn wildcard_should_filter_when_static_wildcard_len_is_zero() {
+    let mut resp = FeroxResponse::default();
+    resp.set_wildcard(true);
+    resp.set_url("http://localhost");
+
+    // default WildcardFilter is used in the code that executes when response.content_length() == 0
+    let filter = WildcardFilter::new(false);
+
+    assert!(filter.should_filter_response(&resp));
+}
+
 #[test]
 /// test should_filter on WilcardFilter where dynamic logic matches
 fn wildcard_should_filter_when_dynamic_wildcard_found() {

src/filters/wildcard.rs

@@ -68,6 +68,14 @@ impl FeroxFilter for WildcardFilter {
             return true;
         }
 
+        if self.size == u64::MAX && response.content_length() == 0 {
+            // static wildcard size found during testing
+            // but response length was zero; pointed out by @Tib3rius
+            log::debug!("static wildcard: filtered out {}", response.url());
+            log::trace!("exit: should_filter_response -> true");
+            return true;
+        }
+
         if self.dynamic != u64::MAX {
             // dynamic wildcard offset found during testing
 
@@ -76,7 +84,7 @@ impl FeroxFilter for WildcardFilter {
             // except that I don't want an empty string taking up the last index in the
             // event that the url ends with a forward slash.  It's ugly enough to be split
             // into its own function for readability.
-            let url_len = FeroxUrl::path_length_of_url(&response.url());
+            let url_len = FeroxUrl::path_length_of_url(response.url());
 
             if url_len + self.dynamic == response.content_length() {
                 log::debug!("dynamic wildcard: filtered out {}", response.url());

src/heuristics.rs

@@ -12,7 +12,7 @@ use crate::{
     response::FeroxResponse,
     skip_fail,
     url::FeroxUrl,
-    utils::{ferox_print, fmt_err, make_request, status_colorizer},
+    utils::{ferox_print, fmt_err, logged_request, status_colorizer},
 };
 
 /// length of a standard UUID, used when determining wildcard responses
@@ -156,15 +156,17 @@ impl HeuristicTests {
         log::trace!("enter: make_wildcard_request({}, {})", target, length);
 
         let unique_str = self.unique_string(length);
-        let nonexistent_url = target.format(&unique_str, None)?;
 
-        let response = make_request(
-            &self.handles.config.client,
-            &nonexistent_url.to_owned(),
-            self.handles.config.output_level,
-            self.handles.stats.tx.clone(),
-        )
-        .await?;
+        // To take care of slash when needed
+        let slash = if self.handles.config.add_slash {
+            Some("/")
+        } else {
+            None
+        };
+
+        let nonexistent_url = target.format(&unique_str, slash)?;
+
+        let response = logged_request(&nonexistent_url.to_owned(), self.handles.clone()).await?;
 
         if self
             .handles
@@ -213,15 +215,10 @@ impl HeuristicTests {
         let mut good_urls = vec![];
 
         for target_url in target_urls {
-            let url = FeroxUrl::from_string(&target_url, self.handles.clone());
+            let url = FeroxUrl::from_string(target_url, self.handles.clone());
             let request = skip_fail!(url.format("", None));
-            let result = make_request(
-                &self.handles.config.client,
-                &request,
-                self.handles.config.output_level,
-                self.handles.stats.tx.clone(),
-            )
-            .await;
+
+            let result = logged_request(&request, self.handles.clone()).await;
 
             match result {
                 Ok(_) => {

src/lib.rs

@@ -43,7 +43,7 @@ pub(crate) type FeroxChannel<T> = (UnboundedSender<T>, UnboundedReceiver<T>);
 pub(crate) const VERSION: &str = env!("CARGO_PKG_VERSION");
 
 /// Maximum number of file descriptors that can be opened during a scan
-pub const DEFAULT_OPEN_FILE_LIMIT: usize = 8192;
+pub const DEFAULT_OPEN_FILE_LIMIT: u64 = 8192;
 
 /// Default value used to determine near-duplicate web pages (equivalent to 95%)
 pub const SIMILARITY_THRESHOLD: u32 = 95;
@@ -57,7 +57,10 @@ pub const DEFAULT_WORDLIST: &str =
     "/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt";
 
 /// Number of milliseconds to wait between polls of `PAUSE_SCAN` when user pauses a scan
-pub(crate) static SLEEP_DURATION: u64 = 500;
+pub(crate) const SLEEP_DURATION: u64 = 500;
+
+/// The percentage of requests as errors it takes to be deemed too high
+pub const HIGH_ERROR_RATIO: f64 = 0.90;
 
 /// Default list of status codes to report
 ///
@@ -70,7 +73,8 @@ pub(crate) static SLEEP_DURATION: u64 = 500;
 /// * 401 Unauthorized
 /// * 403 Forbidden
 /// * 405 Method Not Allowed
-pub const DEFAULT_STATUS_CODES: [StatusCode; 9] = [
+/// * 500 Internal Server Error
+pub const DEFAULT_STATUS_CODES: [StatusCode; 10] = [
     StatusCode::OK,
     StatusCode::NO_CONTENT,
     StatusCode::MOVED_PERMANENTLY,
@@ -80,12 +84,28 @@ pub const DEFAULT_STATUS_CODES: [StatusCode; 9] = [
     StatusCode::UNAUTHORIZED,
     StatusCode::FORBIDDEN,
     StatusCode::METHOD_NOT_ALLOWED,
+    StatusCode::INTERNAL_SERVER_ERROR,
 ];
 
 /// Default filename for config file settings
 ///
 /// Expected location is in the same directory as the feroxbuster binary.
 pub const DEFAULT_CONFIG_NAME: &str = "ferox-config.toml";
+/// User agents to select from when random agent is being used
+pub const USER_AGENTS: [&str; 12] = [
+    "Mozilla/5.0 (Linux; Android 8.0.0; SM-G960F Build/R16NW) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.84 Mobile Safari/537.36",
+    "Mozilla/5.0 (iPhone; CPU iPhone OS 12_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1",
+    "Mozilla/5.0 (Windows Phone 10.0; Android 6.0.1; Microsoft; RM-1152) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Mobile Safari/537.36 Edge/15.15254",
+    "Mozilla/5.0 (Linux; Android 7.0; Pixel C Build/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/52.0.2743.98 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246",
+    "Mozilla/5.0 (X11; CrOS x86_64 8172.45.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.64 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/601.3.9 (KHTML, like Gecko) Version/9.0.2 Safari/601.3.9",
+    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.111 Safari/537.36",
+    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 Firefox/15.0.1",
+    "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
+    "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)",
+    "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)",
+];
 
 #[cfg(test)]
 mod tests {

src/main.rs

@@ -1,13 +1,19 @@
 use std::{
-    collections::HashSet,
-    fs::File,
+    env::args,
+    fs::{create_dir, remove_file, File},
     io::{stderr, BufRead, BufReader},
+    ops::Index,
+    path::Path,
+    process::Command,
     sync::{atomic::Ordering, Arc},
 };
 
 use anyhow::{bail, Context, Result};
 use futures::StreamExt;
-use tokio::{io, sync::oneshot};
+use tokio::{
+    io,
+    sync::{oneshot, Semaphore},
+};
 use tokio_util::codec::{FramedRead, LinesCodec};
 
 use feroxbuster::{
@@ -22,20 +28,27 @@ use feroxbuster::{
     progress::{PROGRESS_BAR, PROGRESS_PRINTER},
     scan_manager::{self},
     scanner,
-    utils::fmt_err,
+    utils::{fmt_err, slugify_filename},
 };
 #[cfg(not(target_os = "windows"))]
 use feroxbuster::{utils::set_open_file_limit, DEFAULT_OPEN_FILE_LIMIT};
+use lazy_static::lazy_static;
+use regex::Regex;
+
+lazy_static! {
+    /// Limits the number of parallel scans active at any given time when using --parallel
+    static ref PARALLEL_LIMITER: Semaphore = Semaphore::new(0);
+}
 
 /// Create a HashSet of Strings from the given wordlist then stores it inside an Arc
-fn get_unique_words_from_wordlist(path: &str) -> Result<Arc<HashSet<String>>> {
+fn get_unique_words_from_wordlist(path: &str) -> Result<Arc<Vec<String>>> {
     log::trace!("enter: get_unique_words_from_wordlist({})", path);
 
     let file = File::open(&path).with_context(|| format!("Could not open {}", path))?;
 
     let reader = BufReader::new(file);
 
-    let mut words = HashSet::new();
+    let mut words = Vec::new();
 
     for line in reader.lines() {
         let result = match line {
@@ -47,7 +60,7 @@ fn get_unique_words_from_wordlist(path: &str) -> Result<Arc<HashSet<String>>> {
             continue;
         }
 
-        words.insert(result);
+        words.push(result);
     }
 
     log::trace!(
@@ -65,11 +78,7 @@ async fn scan(targets: Vec<String>, handles: Arc<Handles>) -> Result<()> {
     // so that will allow for cheap/safe sharing of a single wordlist across multi-target scans
     // as well as additional directories found as part of recursion
 
-    let words = {
-        let words_handles = handles.clone();
-        tokio::spawn(async move { get_unique_words_from_wordlist(&words_handles.config.wordlist) })
-            .await??
-    };
+    let words = get_unique_words_from_wordlist(&handles.config.wordlist)?;
 
     if words.len() == 0 {
         bail!("Did not find any words in {}", handles.config.wordlist);
@@ -145,6 +154,28 @@ async fn get_targets(handles: Arc<Handles>) -> Result<Vec<String>> {
         targets.push(handles.config.target_url.clone());
     }
 
+    // remove footgun that arises if a --dont-scan value matches on a base url
+    for target in &targets {
+        for denier in &handles.config.regex_denylist {
+            if denier.is_match(target) {
+                bail!(
+                    "The regex '{}' matches {}; the scan will never start",
+                    denier,
+                    target
+                );
+            }
+        }
+        for denier in &handles.config.url_denylist {
+            if denier.as_str().trim_end_matches('/') == target.trim_end_matches('/') {
+                bail!(
+                    "The url '{}' matches {}; the scan will never start",
+                    denier,
+                    target
+                );
+            }
+        }
+    }
+
     log::trace!("exit: get_targets -> {:?}", targets);
 
     Ok(targets)
@@ -222,10 +253,136 @@ async fn wrapped_main(config: Arc<Configuration>) -> Result<()> {
         Err(e) => {
             // should only happen in the event that there was an error reading from stdin
             clean_up(handles, tasks).await?;
-            bail!("Could not get determine initial targets: {}", e);
+            bail!("Could not determine initial targets: {}", e);
         }
     };
 
+    // --parallel branch
+    if config.parallel > 0 {
+        log::trace!("enter: parallel branch");
+
+        PARALLEL_LIMITER.add_permits(config.parallel);
+
+        let invocation = args();
+
+        let para_regex =
+            Regex::new("--stdin|-q|--quiet|--silent|--verbosity|-v|-vv|-vvv|-vvvv").unwrap();
+
+        // remove stdin since only the original process will process targets
+        // remove quiet and silent so we can force silent later to normalize output
+        let mut original = invocation
+            .filter(|s| !para_regex.is_match(s))
+            .collect::<Vec<String>>();
+
+        original.push("--silent".to_string()); // only output modifier allowed
+
+        // we need remove --parallel from command line so we don't hit this branch over and over
+        // but we must remove --parallel N manually; the filter above never sees --parallel and the
+        // value passed to it at the same time, so can't filter them out in one pass
+
+        // unwrap is fine, as it has to be in the args for us to be in this code branch
+        let parallel_index = original.iter().position(|s| *s == "--parallel").unwrap();
+
+        // remove --parallel
+        original.remove(parallel_index);
+
+        // remove N passed to --parallel (it's the same index again since everything shifts
+        // from removing --parallel)
+        original.remove(parallel_index);
+
+        // to log unique files to a shared folder, we need to first check for the presence
+        // of -o|--output.
+        let out_dir = if !config.output.is_empty() {
+            // -o|--output was used, so we'll attempt to create a directory to store the files
+            let output_path = Path::new(&handles.config.output);
+
+            // this only returns None if the path terminates in `..`. Since I don't want to
+            // hand-hold to that degree, we'll unwrap and fail if the output path ends in `..`
+            let base_name = output_path.file_name().unwrap();
+
+            let new_folder = slugify_filename(&base_name.to_string_lossy(), "", "logs");
+
+            let final_path = output_path.with_file_name(&new_folder);
+
+            // create the directory or fail silently, assuming the reason for failure is that
+            // the path exists already
+            create_dir(&final_path).unwrap_or(());
+
+            final_path.to_string_lossy().to_string()
+        } else {
+            String::new()
+        };
+
+        // unvalidated targets fresh from stdin, just spawn children and let them do all checks
+        for target in targets {
+            // add the current target to the provided command
+            let mut cloned = original.clone();
+
+            if !out_dir.is_empty() {
+                // output directory value is not empty, need to join output directory with
+                // unique scan filename
+
+                // unwrap is ok, we already know -o was used
+                let out_idx = original
+                    .iter()
+                    .position(|s| *s == "--output" || *s == "-o")
+                    .unwrap();
+
+                let filename = slugify_filename(&target, "ferox", "log");
+
+                let full_path = Path::new(&out_dir)
+                    .join(filename)
+                    .to_string_lossy()
+                    .to_string();
+
+                // a +1 to the index is fine here, as clap has already validated that
+                // -o|--output has a value associated with it
+                cloned[out_idx + 1] = full_path;
+            }
+
+            cloned.push("-u".to_string());
+            cloned.push(target);
+
+            let bin = cloned.index(0).to_owned(); // user's path to feroxbuster
+            let args = cloned.index(1..).to_vec(); // and args
+
+            let permit = PARALLEL_LIMITER.acquire().await?;
+
+            log::debug!("parallel exec: {} {}", bin, args.join(" "));
+
+            tokio::task::spawn_blocking(move || {
+                let result = Command::new(bin)
+                    .args(&args)
+                    .spawn()
+                    .expect("failed to spawn a child process")
+                    .wait()
+                    .expect("child process errored during execution");
+
+                drop(permit);
+                result
+            });
+        }
+
+        // the output handler creates an empty file to which it will try to write, because
+        // this happens before we enter the --parallel branch, we need to remove that file
+        // if it's empty
+        let output = handles.config.output.to_owned();
+
+        clean_up(handles, tasks).await?;
+
+        let file = Path::new(&output);
+        if file.exists() {
+            // expectation is that this is always true for the first ferox process
+            if file.metadata()?.len() == 0 {
+                // empty file, attempt to remove it
+                remove_file(file)?;
+            }
+        }
+
+        log::trace!("exit: parallel branch && wrapped main");
+        return Ok(());
+    }
+
     if matches!(config.output_level, OutputLevel::Default) {
         // only print banner if output level is default (no banner on --quiet|--silent)
         let std_stderr = stderr(); // std::io::stderr
@@ -246,7 +403,7 @@ async fn wrapped_main(config: Arc<Configuration>) -> Result<()> {
 
         // The TermOutHandler spawns a FileOutHandler, so errors in the FileOutHandler never bubble
         // up due to the TermOutHandler never awaiting the result of FileOutHandler::start (that's
-        // done later here in main). Ping checks that the tx/rx connection to the file handler works
+        // done later here in main). sync checks that the tx/rx connection to the file handler works
         if send_to_file && handles.output.sync(send_to_file).await.is_err() {
             // output file specified and file handler could not initialize
             clean_up(handles, tasks).await?;

src/message.rs

@@ -1,9 +1,9 @@
 use anyhow::Context;
 use console::{style, Color};
+use serde::{Deserialize, Serialize};
 
 use crate::traits::FeroxSerialize;
 use crate::utils::fmt_err;
-use serde::{Deserialize, Serialize};
 
 #[derive(Serialize, Deserialize, Default)]
 /// Representation of a log entry, can be represented as a human readable string or JSON
@@ -118,4 +118,31 @@ mod tests {
         assert_eq!(json.level, message.level);
         assert_eq!(json.kind, message.kind);
     }
+
+    #[test]
+    /// test defaults for coverage
+    fn message_defaults() {
+        let msg = FeroxMessage::default();
+        assert_eq!(msg.level, String::new());
+        assert_eq!(msg.kind, String::new());
+        assert_eq!(msg.message, String::new());
+        assert_eq!(msg.module, String::new());
+        assert!(msg.time_offset < 0.1);
+    }
+
+    #[test]
+    /// ensure WILDCARD messages serialize to WLD and anything not known to UNK
+    fn message_as_str_edges() {
+        let mut msg = FeroxMessage {
+            message: "message".to_string(),
+            module: "utils".to_string(),
+            time_offset: 1.0,
+            level: "WILDCARD".to_string(),
+            kind: "log".to_string(),
+        };
+        assert!(console::strip_ansi_codes(&msg.as_str()).starts_with("WLD"));
+
+        msg.level = "UNKNOWN".to_string();
+        assert!(console::strip_ansi_codes(&msg.as_str()).starts_with("UNK"));
+    }
 }

src/parser.rs

@@ -1,6 +1,8 @@
 use clap::{App, Arg, ArgGroup};
 use lazy_static::lazy_static;
 use regex::Regex;
+use std::env;
+use std::process;
 
 lazy_static! {
     /// Regex used to validate values passed to --time-limit
@@ -16,7 +18,7 @@ lazy_static! {
 
 /// Create and return an instance of [clap::App](https://docs.rs/clap/latest/clap/struct.App.html), i.e. the Command Line Interface's configuration
 pub fn initialize() -> App<'static, 'static> {
-    App::new("feroxbuster")
+    let mut app = App::new("feroxbuster")
         .version(env!("CARGO_PKG_VERSION"))
         .author("Ben 'epi' Risher (@epi052)")
         .about("A fast, simple, recursive content discovery tool written in Rust")
@@ -130,6 +132,19 @@ pub fn initialize() -> App<'static, 'static> {
                 .takes_value(false)
                 .help("Hide progress bars and banner (good for tmux windows w/ notifications)")
         )
+        .arg(
+            Arg::with_name("auto_tune")
+                .long("auto-tune")
+                .takes_value(false)
+                .conflicts_with("auto_bail")
+                .help("Automatically lower scan rate when an excessive amount of errors are encountered")
+        )
+        .arg(
+            Arg::with_name("auto_bail")
+                .long("auto-bail")
+                .takes_value(false)
+                .help("Automatically stop scanning when an excessive amount of errors are encountered")
+        )
         .arg(
             Arg::with_name("json")
                 .long("json")
@@ -177,6 +192,15 @@ pub fn initialize() -> App<'static, 'static> {
                     "Sets the User-Agent (default: feroxbuster/VERSION)"
                 ),
         )
+        .arg(
+            Arg::with_name("random_agent")
+                .short("A")
+                .long("random-agent")
+                .takes_value(false)
+                .help(
+                    "Use a random User-Agent"
+                ),
+        )
         .arg(
             Arg::with_name("redirects")
                 .short("r")
@@ -203,6 +227,17 @@ pub fn initialize() -> App<'static, 'static> {
                     "File extension(s) to search for (ex: -x php -x pdf js)",
                 ),
         )
+        .arg(
+            Arg::with_name("url_denylist")
+                .long("dont-scan")
+                .value_name("URL")
+                .takes_value(true)
+                .multiple(true)
+                .use_delimiter(true)
+                .help(
+                    "URL(s) or Regex Pattern(s) to exclude from recursion/scans",
+                ),
+        )
         .arg(
             Arg::with_name("headers")
                 .short("H")
@@ -239,7 +274,6 @@ pub fn initialize() -> App<'static, 'static> {
                 .short("f")
                 .long("add-slash")
                 .takes_value(false)
-                .conflicts_with("extensions")
                 .help("Append / to each request")
         )
         .arg(
@@ -335,11 +369,20 @@ pub fn initialize() -> App<'static, 'static> {
                 .takes_value(true)
                 .help("Limit total number of concurrent scans (default: 0, i.e. no limit)")
         )
+        .arg(
+            Arg::with_name("parallel")
+                .long("parallel")
+                .value_name("PARALLEL_SCANS")
+                .takes_value(true)
+                .requires("stdin")
+                .help("Run parallel feroxbuster instances (one child process per url passed via stdin)")
+        )
         .arg(
             Arg::with_name("rate_limit")
                 .long("rate-limit")
                 .value_name("RATE_LIMIT")
                 .takes_value(true)
+                .conflicts_with("auto_tune")
                 .help("Limit number of requests per second (per directory) (default: 0, i.e. no limit)")
         )
         .arg(
@@ -388,7 +431,20 @@ EXAMPLES:
 
     Ludicrous speed... go!
         ./feroxbuster -u http://127.1 -t 200
-    "#)
+    "#);
+
+    for arg in env::args() {
+        // secure-77 noticed that when an incorrect flag/option is used, the short help message is printed
+        // which is fine, but if you add -h|--help, it still errors out on the bad flag/option,
+        // never showing the full help message. This code addresses that behavior
+        if arg == "--help" || arg == "-h" {
+            app.print_long_help().unwrap();
+            println!(); // just a newline to mirror original --help output
+            process::exit(0);
+        }
+    }
+
+    app
 }
 
 /// Validate that a string is formatted as a number followed by s, m, h, or d (10d, 30s, etc...)
@@ -410,7 +466,7 @@ mod tests {
     use super::*;
 
     #[test]
-    /// initalize parser, expect a clap::App returned
+    /// initialize parser, expect a clap::App returned
     fn parser_initialize_gives_defaults() {
         let app = initialize();
         assert_eq!(app.get_name(), "feroxbuster");

src/progress.rs

@@ -51,7 +51,7 @@ pub fn add_bar(prefix: &str, length: u64, bar_type: BarType) -> ProgressBar {
 
     progress_bar.set_style(style);
 
-    progress_bar.set_prefix(&prefix);
+    progress_bar.set_prefix(prefix);
 
     progress_bar
 }

src/response.rs

@@ -121,7 +121,7 @@ impl FeroxResponse {
 
     /// Set `FeroxResponse`'s `url` attribute, has no affect if an error occurs
     pub fn set_url(&mut self, url: &str) {
-        match Url::parse(&url) {
+        match Url::parse(url) {
             Ok(url) => {
                 self.url = url;
             }
@@ -339,7 +339,7 @@ impl FeroxSerialize for FeroxResponse {
                 lines,
                 words,
                 chars,
-                status_colorizer(&status),
+                status_colorizer(status),
                 self.url(),
                 FeroxUrl::path_length_of_url(&self.url)
             );

src/scan_manager/menu.rs

@@ -31,9 +31,10 @@ impl Menu {
         let separator = "─".to_string();
 
         let instructions = format!(
-            "Enter a {} list of indexes to {} (ex: 2,3)",
+            "Enter a {} list of indexes/ranges to {} ({}: 1-4,8,9-13)",
             style("comma-separated").yellow(),
             style("cancel").red(),
+            style("ex").cyan(),
         );
 
         let name = format!(
@@ -43,14 +44,22 @@ impl Menu {
             "💀"
         );
 
+        let force_msg = format!(
+            "Add {} to {} confirmation ({}: 3-5 -f)",
+            style("-f").yellow(),
+            style("skip").yellow(),
+            style("ex").cyan(),
+        );
+
         let longest = measure_text_width(&instructions).max(measure_text_width(&name));
 
         let border = separator.repeat(longest);
 
         let padded_name = pad_str(&name, longest, Alignment::Center, None);
+        let padded_force = pad_str(&force_msg, longest, Alignment::Center, None);
 
         let header = format!("{}\n{}\n{}", border, padded_name, border);
-        let footer = format!("{}\n{}\n{}", border, instructions, border);
+        let footer = format!("{}\n{}\n{}\n{}", border, instructions, padded_force, border);
 
         Self {
             separator,
@@ -93,23 +102,71 @@ impl Menu {
         self.term.write_line(msg).unwrap_or_default();
     }
 
-    /// split a string into vec of usizes
-    pub(super) fn split_to_nums(&self, line: &str) -> Vec<usize> {
-        line.split(',')
-            .map(|s| {
-                s.trim().to_string().parse::<usize>().unwrap_or_else(|e| {
-                    self.println(&format!("Found non-numeric input: {}", e));
-                    0
-                })
+    /// Helper for parsing a usize from a str
+    fn str_to_usize(&self, value: &str) -> usize {
+        if value.is_empty() {
+            return 0;
+        }
+
+        value
+            .trim()
+            .to_string()
+            .parse::<usize>()
+            .unwrap_or_else(|e| {
+                self.println(&format!("Found non-numeric input: {}: {:?}", e, value));
+                0
             })
-            .filter(|m| *m != 0)
-            .collect()
+    }
+
+    /// split a comma delimited string into vec of usizes
+    pub(super) fn split_to_nums(&self, line: &str) -> Vec<usize> {
+        let mut nums = Vec::new();
+        let values = line.split(',');
+
+        for mut value in values {
+            value = value.trim();
+
+            if value.contains('-') {
+                // range of two values, needs further processing
+
+                let range: Vec<usize> = value
+                    .split('-')
+                    .map(|s| self.str_to_usize(s))
+                    .filter(|m| *m != 0)
+                    .collect();
+
+                if range.len() != 2 {
+                    // expecting [1, 4] or similar, if a 0 was used, we'd be left with a vec of size 1
+                    self.println(&format!("Found invalid range of scans: {}", value));
+                    continue;
+                }
+
+                (range[0]..=range[1]).for_each(|n| {
+                    // iterate from lower to upper bound and add all interim values, skipping
+                    // any already known
+                    if !nums.contains(&n) {
+                        nums.push(n)
+                    }
+                });
+            } else {
+                let value = self.str_to_usize(value);
+
+                if value != 0 && !nums.contains(&value) {
+                    // the zeroth scan is always skipped, skip already known values
+                    nums.push(value);
+                }
+            }
+        }
+
+        nums
     }
 
     /// get comma-separated list of scan indexes from the user
-    pub(super) fn get_scans_from_user(&self) -> Option<Vec<usize>> {
+    pub(super) fn get_scans_from_user(&self) -> Option<(Vec<usize>, bool)> {
         if let Ok(line) = self.term.read_line() {
-            Some(self.split_to_nums(&line))
+            let force = line.contains("-f");
+            let line = line.replace("-f", "");
+            Some((self.split_to_nums(&line), force))
         } else {
             None
         }

src/scan_manager/scan.rs

@@ -2,6 +2,7 @@ use super::*;
 use crate::{
     config::OutputLevel,
     progress::{add_bar, BarType},
+    scanner::PolicyTrigger,
 };
 use anyhow::Result;
 use console::style;
@@ -12,8 +13,10 @@ use std::{
     collections::HashMap,
     fmt,
     sync::{Arc, Mutex},
+    time::Instant,
 };
 
+use std::sync::atomic::{AtomicUsize, Ordering};
 use tokio::{sync, task::JoinHandle};
 use uuid::Uuid;
 
@@ -33,7 +36,7 @@ pub struct FeroxScan {
     pub(super) scan_type: ScanType,
 
     /// The order in which the scan was received
-    pub(super) scan_order: ScanOrder,
+    pub(crate) scan_order: ScanOrder,
 
     /// Number of requests to populate the progress bar with
     pub(super) num_requests: u64,
@@ -49,6 +52,18 @@ pub struct FeroxScan {
 
     /// whether or not the user passed --silent|--quiet on the command line
     pub(super) output_level: OutputLevel,
+
+    /// tracker for overall number of 403s seen by the FeroxScan instance
+    pub(super) status_403s: AtomicUsize,
+
+    /// tracker for overall number of 429s seen by the FeroxScan instance
+    pub(super) status_429s: AtomicUsize,
+
+    /// tracker for total number of errors encountered by the FeroxScan instance
+    pub(super) errors: AtomicUsize,
+
+    /// tracker for the time at which this scan was started
+    pub(super) start_time: Instant,
 }
 
 /// Default implementation for FeroxScan
@@ -67,6 +82,10 @@ impl Default for FeroxScan {
             progress_bar: Mutex::new(None),
             scan_type: ScanType::File,
             output_level: Default::default(),
+            errors: Default::default(),
+            status_429s: Default::default(),
+            status_403s: Default::default(),
+            start_time: Instant::now(),
         }
     }
 }
@@ -75,16 +94,22 @@ impl Default for FeroxScan {
 impl FeroxScan {
     /// Stop a currently running scan
     pub async fn abort(&self) -> Result<()> {
-        let mut guard = self.task.lock().await;
+        log::trace!("enter: abort");
 
-        if guard.is_some() {
-            if let Some(task) = std::mem::replace(&mut *guard, None) {
-                task.abort();
-                self.set_status(ScanStatus::Cancelled)?;
-                self.stop_progress_bar();
+        match self.task.try_lock() {
+            Ok(mut guard) => {
+                if let Some(task) = std::mem::replace(&mut *guard, None) {
+                    log::trace!("aborting {:?}", self);
+                    task.abort();
+                    self.set_status(ScanStatus::Cancelled)?;
+                    self.stop_progress_bar();
+                }
+            }
+            Err(e) => {
+                log::warn!("Could not acquire lock to abort scan (we're already waiting for its results): {:?} {}", self, e);
             }
         }
-
+        log::trace!("exit: abort");
         Ok(())
     }
 
@@ -134,6 +159,7 @@ impl FeroxScan {
                     pb.reset_elapsed();
 
                     let _ = std::mem::replace(&mut *guard, Some(pb.clone()));
+
                     pb
                 }
             }
@@ -217,6 +243,61 @@ impl FeroxScan {
 
         log::trace!("exit join({:?})", self);
     }
+    /// increment the value in question by 1
+    pub(crate) fn add_403(&self) {
+        self.status_403s.fetch_add(1, Ordering::Relaxed);
+    }
+
+    /// increment the value in question by 1
+    pub(crate) fn add_429(&self) {
+        self.status_429s.fetch_add(1, Ordering::Relaxed);
+    }
+
+    /// increment the value in question by 1
+    pub(crate) fn add_error(&self) {
+        self.errors.fetch_add(1, Ordering::Relaxed);
+    }
+
+    /// simple wrapper to call the appropriate getter based on the given PolicyTrigger
+    pub fn num_errors(&self, trigger: PolicyTrigger) -> usize {
+        match trigger {
+            PolicyTrigger::Status403 => self.status_403s(),
+            PolicyTrigger::Status429 => self.status_429s(),
+            PolicyTrigger::Errors => self.errors(),
+        }
+    }
+
+    /// return the number of errors seen by this scan
+    fn errors(&self) -> usize {
+        self.errors.load(Ordering::Relaxed)
+    }
+
+    /// return the number of 403s seen by this scan
+    fn status_403s(&self) -> usize {
+        self.status_403s.load(Ordering::Relaxed)
+    }
+
+    /// return the number of 429s seen by this scan
+    fn status_429s(&self) -> usize {
+        self.status_429s.load(Ordering::Relaxed)
+    }
+
+    /// return the number of requests per second performed by this scan's scanner
+    pub fn requests_per_second(&self) -> u64 {
+        if !self.is_active() {
+            return 0;
+        }
+
+        let reqs = self.requests();
+        let seconds = self.start_time.elapsed().as_secs();
+
+        reqs.checked_div(seconds).unwrap_or(0)
+    }
+
+    /// return the number of requests performed by this scan's scanner
+    pub fn requests(&self) -> u64 {
+        self.progress_bar().position()
+    }
 }
 
 /// Display implementation
@@ -360,3 +441,68 @@ impl Default for ScanStatus {
         Self::NotStarted
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::thread::sleep;
+    use tokio::time::Duration;
+
+    #[test]
+    /// ensure that num_errors returns the correct values for the given PolicyTrigger
+    ///
+    /// covers tests for add_[403,429,error] and the related getters in addition to num_errors
+    fn num_errors_returns_correct_values() {
+        let scan = FeroxScan::new(
+            "http://localhost",
+            ScanType::Directory,
+            ScanOrder::Latest,
+            1000,
+            OutputLevel::Default,
+            None,
+        );
+
+        scan.add_error();
+        scan.add_403();
+        scan.add_403();
+        scan.add_429();
+        scan.add_429();
+        scan.add_429();
+
+        assert_eq!(scan.num_errors(PolicyTrigger::Errors), 1);
+        assert_eq!(scan.num_errors(PolicyTrigger::Status403), 2);
+        assert_eq!(scan.num_errors(PolicyTrigger::Status429), 3);
+    }
+
+    #[test]
+    /// ensure that requests_per_second returns the correct values
+    fn requests_per_second_returns_correct_values() {
+        let scan = FeroxScan {
+            id: "".to_string(),
+            url: "".to_string(),
+            scan_type: ScanType::Directory,
+            scan_order: ScanOrder::Initial,
+            num_requests: 0,
+            status: Mutex::new(ScanStatus::Running),
+            task: Default::default(),
+            progress_bar: Mutex::new(None),
+            output_level: Default::default(),
+            status_403s: Default::default(),
+            status_429s: Default::default(),
+            errors: Default::default(),
+            start_time: Instant::now(),
+        };
+
+        let pb = scan.progress_bar();
+        pb.set_position(100);
+
+        sleep(Duration::new(1, 0));
+
+        let req_sec = scan.requests_per_second();
+
+        assert_eq!(req_sec, 100);
+
+        scan.finish().unwrap();
+        assert_eq!(scan.requests_per_second(), 0);
+    }
+}

src/scan_manager/scan_container.rs

@@ -9,6 +9,7 @@ use crate::{
     SLEEP_DURATION,
 };
 use anyhow::Result;
+use reqwest::StatusCode;
 use serde::{ser::SerializeSeq, Serialize, Serializer};
 use std::{
     convert::TryInto,
@@ -161,6 +162,63 @@ impl FeroxScans {
         None
     }
 
+    pub(super) fn get_base_scan_by_url(&self, url: &str) -> Option<Arc<FeroxScan>> {
+        log::trace!("enter: get_sub_paths_from_path({})", url);
+
+        // rmatch_indices returns tuples in index, match form, i.e. (10, "/")
+        // with the furthest-right match in the first position in the vector
+        let matches: Vec<_> = url.rmatch_indices('/').collect();
+
+        // iterate from the furthest right matching index and check the given url from the
+        // start to the furthest-right '/' character. compare that slice to the urls associated
+        // with directory scans and return the first match, since it should be the 'deepest'
+        // match.
+        // Example:
+        //   url: http://shmocalhost/src/release/examples/stuff.php
+        //   scans:
+        //      http://shmocalhost/src/statistics
+        //      http://shmocalhost/src/banner
+        //      http://shmocalhost/src/release
+        //      http://shmocalhost/src/release/examples
+        //
+        //  returns: http://shmocalhost/src/release/examples
+        if let Ok(guard) = self.scans.read() {
+            for (idx, _) in &matches {
+                for scan in guard.iter() {
+                    let slice = url.index(0..*idx);
+                    if slice == scan.url || format!("{}/", slice).as_str() == scan.url {
+                        log::trace!("enter: get_sub_paths_from_path -> {}", scan);
+                        return Some(scan.clone());
+                    }
+                }
+            }
+        }
+
+        log::trace!("enter: get_sub_paths_from_path -> None");
+        None
+    }
+    /// add one to either 403 or 429 tracker in the scan related to the given url
+    pub fn increment_status_code(&self, url: &str, code: StatusCode) {
+        if let Some(scan) = self.get_base_scan_by_url(url) {
+            match code {
+                StatusCode::TOO_MANY_REQUESTS => {
+                    scan.add_429();
+                }
+                StatusCode::FORBIDDEN => {
+                    scan.add_403();
+                }
+                _ => {}
+            }
+        }
+    }
+
+    /// add one to either 403 or 429 tracker in the scan related to the given url
+    pub fn increment_error(&self, url: &str) {
+        if let Some(scan) = self.get_base_scan_by_url(url) {
+            scan.add_error();
+        }
+    }
+
     /// Print all FeroxScans of type Directory
     ///
     /// Example:
@@ -194,9 +252,11 @@ impl FeroxScans {
     }
 
     /// Given a list of indexes, cancel their associated FeroxScans
-    async fn cancel_scans(&self, indexes: Vec<usize>) {
+    async fn cancel_scans(&self, indexes: Vec<usize>, force: bool) -> usize {
         let menu_pause_duration = Duration::from_millis(SLEEP_DURATION);
 
+        let mut num_cancelled = 0_usize;
+
         for num in indexes {
             let selected = match self.scans.read() {
                 Ok(u_scans) => {
@@ -213,36 +273,50 @@ impl FeroxScans {
                 Err(..) => continue,
             };
 
-            let input = self.menu.confirm_cancellation(&selected.url);
+            let input = if force {
+                'y'
+            } else {
+                self.menu.confirm_cancellation(&selected.url)
+            };
 
             if input == 'y' || input == '\n' {
                 self.menu.println(&format!("Stopping {}...", selected.url));
+
                 selected
                     .abort()
                     .await
                     .unwrap_or_else(|e| log::warn!("Could not cancel task: {}", e));
+
+                let pb = selected.progress_bar();
+                num_cancelled += pb.length() as usize - pb.position() as usize
             } else {
                 self.menu.println("Ok, doing nothing...");
             }
 
             sleep(menu_pause_duration);
         }
+
+        num_cancelled
     }
 
     /// CLI menu that allows for interactive cancellation of recursed-into directories
-    async fn interactive_menu(&self) {
+    async fn interactive_menu(&self) -> usize {
         self.menu.hide_progress_bars();
         self.menu.clear_screen();
         self.menu.print_header();
         self.display_scans().await;
         self.menu.print_footer();
 
-        if let Some(input) = self.menu.get_scans_from_user() {
-            self.cancel_scans(input).await
+        let mut num_cancelled = 0_usize;
+
+        if let Some((input, force)) = self.menu.get_scans_from_user() {
+            num_cancelled += self.cancel_scans(input, force).await;
         };
 
         self.menu.clear_screen();
         self.menu.show_progress_bars();
+
+        num_cancelled
     }
 
     /// prints all known responses that the scanner has already seen
@@ -290,18 +364,19 @@ impl FeroxScans {
     ///
     /// When the value stored in `PAUSE_SCAN` becomes `false`, the function returns, exiting the busy
     /// loop
-    pub async fn pause(&self, get_user_input: bool) {
+    pub async fn pause(&self, get_user_input: bool) -> usize {
         // function uses tokio::time, not std
 
         // local testing showed a pretty slow increase (less than linear) in CPU usage as # of
         // concurrent scans rose when SLEEP_DURATION was set to 500, using that as the default for now
         let mut interval = time::interval(time::Duration::from_millis(SLEEP_DURATION));
+        let mut num_cancelled = 0_usize;
 
         if INTERACTIVE_BARRIER.load(Ordering::Relaxed) == 0 {
             INTERACTIVE_BARRIER.fetch_add(1, Ordering::Relaxed);
 
             if get_user_input {
-                self.interactive_menu().await;
+                num_cancelled += self.interactive_menu().await;
                 PAUSE_SCAN.store(false, Ordering::Relaxed);
                 self.print_known_responses();
             }
@@ -318,8 +393,8 @@ impl FeroxScans {
                     INTERACTIVE_BARRIER.fetch_sub(1, Ordering::Relaxed);
                 }
 
-                log::trace!("exit: pause_scan");
-                return;
+                log::trace!("exit: pause_scan -> {}", num_cancelled);
+                return num_cancelled;
             }
         }
     }
@@ -356,7 +431,7 @@ impl FeroxScans {
                     OutputLevel::Silent => BarType::Hidden,
                 };
 
-                let progress_bar = add_bar(&url, bar_length, bar_type);
+                let progress_bar = add_bar(url, bar_length, bar_type);
 
                 progress_bar.reset_elapsed();
 
@@ -366,7 +441,7 @@ impl FeroxScans {
         };
 
         let ferox_scan = FeroxScan::new(
-            &url,
+            url,
             scan_type,
             scan_order,
             bar_length,
@@ -387,7 +462,7 @@ impl FeroxScans {
     ///
     /// Also return a reference to the new `FeroxScan`
     pub fn add_directory_scan(&self, url: &str, scan_order: ScanOrder) -> (bool, Arc<FeroxScan>) {
-        self.add_scan(&url, ScanType::Directory, scan_order)
+        self.add_scan(url, ScanType::Directory, scan_order)
     }
 
     /// Given a url, create a new `FeroxScan` and add it to `FeroxScans` as a File Scan
@@ -396,7 +471,7 @@ impl FeroxScans {
     ///
     /// Also return a reference to the new `FeroxScan`
     pub fn add_file_scan(&self, url: &str, scan_order: ScanOrder) -> (bool, Arc<FeroxScan>) {
-        self.add_scan(&url, ScanType::File, scan_order)
+        self.add_scan(url, ScanType::File, scan_order)
     }
 
     /// small helper to determine whether any scans are active or not

src/scan_manager/state.rs

@@ -47,7 +47,7 @@ impl FeroxSerialize for FeroxState {
 
     /// Simple call to produce a JSON string using the given FeroxState
     fn as_json(&self) -> Result<String> {
-        Ok(serde_json::to_string(&self)
-            .with_context(|| fmt_err("Could not convert scan's running state to JSON"))?)
+        serde_json::to_string(&self)
+            .with_context(|| fmt_err("Could not convert scan's running state to JSON"))
     }
 }

src/scan_manager/tests.rs

@@ -12,6 +12,7 @@ use indicatif::ProgressBar;
 use predicates::prelude::*;
 use std::sync::{atomic::Ordering, Arc};
 use std::thread::sleep;
+use std::time::Instant;
 use tokio::time::{self, Duration};
 
 #[test]
@@ -51,7 +52,7 @@ fn add_url_to_list_of_scanned_urls_with_unknown_url() {
     let urls = FeroxScans::default();
     let url = "http://unknown_url";
     let (result, _scan) = urls.add_scan(url, ScanType::Directory, ScanOrder::Latest);
-    assert_eq!(result, true);
+    assert!(result);
 }
 
 #[test]
@@ -70,11 +71,11 @@ fn add_url_to_list_of_scanned_urls_with_known_url() {
         Some(pb),
     );
 
-    assert_eq!(urls.insert(scan), true);
+    assert!(urls.insert(scan));
 
     let (result, _scan) = urls.add_scan(url, ScanType::Directory, ScanOrder::Latest);
 
-    assert_eq!(result, false);
+    assert!(!result);
 }
 
 #[test]
@@ -92,27 +93,23 @@ fn stop_progress_bar_stops_bar() {
         Some(pb),
     );
 
-    assert_eq!(
-        scan.progress_bar
-            .lock()
-            .unwrap()
-            .as_ref()
-            .unwrap()
-            .is_finished(),
-        false
-    );
+    assert!(!scan
+        .progress_bar
+        .lock()
+        .unwrap()
+        .as_ref()
+        .unwrap()
+        .is_finished());
 
     scan.stop_progress_bar();
 
-    assert_eq!(
-        scan.progress_bar
-            .lock()
-            .unwrap()
-            .as_ref()
-            .unwrap()
-            .is_finished(),
-        true
-    );
+    assert!(scan
+        .progress_bar
+        .lock()
+        .unwrap()
+        .as_ref()
+        .unwrap()
+        .is_finished());
 }
 
 #[test]
@@ -130,11 +127,11 @@ fn add_url_to_list_of_scanned_urls_with_known_url_without_slash() {
         None,
     );
 
-    assert_eq!(urls.insert(scan), true);
+    assert!(urls.insert(scan));
 
     let (result, _scan) = urls.add_scan(url, ScanType::File, ScanOrder::Latest);
 
-    assert_eq!(result, false);
+    assert!(!result);
 }
 
 #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
@@ -170,8 +167,8 @@ async fn call_display_scans() {
         .await
         .unwrap();
 
-    assert_eq!(urls.insert(scan), true);
-    assert_eq!(urls.insert(scan_two), true);
+    assert!(urls.insert(scan));
+    assert!(urls.insert(scan_two));
 
     urls.display_scans().await;
 }
@@ -329,7 +326,7 @@ fn ferox_response_serialize_and_deserialize() {
 
     assert_eq!(response.url().as_str(), "https://nerdcore.com/css");
     assert_eq!(response.url().path(), "/css");
-    assert_eq!(response.wildcard(), true);
+    assert!(response.wildcard());
     assert_eq!(response.status().as_u16(), 301);
     assert_eq!(response.content_length(), 173);
     assert_eq!(response.line_count(), 10);
@@ -381,12 +378,79 @@ fn feroxstates_feroxserialize_implementation() {
     assert!(expected_strs.eval(&ferox_state.as_str()));
 
     let json_state = ferox_state.as_json().unwrap();
-    let expected = format!(
-        r#"{{"scans":[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","status":"NotStarted","num_requests":0}}],"config":{{"type":"configuration","wordlist":"/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt","config":"","proxy":"","replay_proxy":"","target_url":"","status_codes":[200,204,301,302,307,308,401,403,405],"replay_codes":[200,204,301,302,307,308,401,403,405],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"silent":false,"quiet":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/{}","redirects":false,"insecure":false,"extensions":[],"headers":{{}},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":false,"depth":4,"scan_limit":0,"rate_limit":0,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false,"resumed":false,"resume_from":"","save_state":false,"time_limit":"","filter_similar":[]}},"responses":[{{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{{"server":"nginx/1.16.1"}}}}]"#,
-        saved_id, VERSION
-    );
-    println!("{}\n{}", expected, json_state);
-    assert!(predicates::str::contains(expected).eval(&json_state));
+    for expected in [
+        r#""scans""#,
+        &format!(r#""id":"{}""#, saved_id),
+        r#""url":"https://spiritanimal.com""#,
+        r#""scan_type":"Directory""#,
+        r#""status":"NotStarted""#,
+        r#""num_requests":0"#,
+        r#""config""#,
+        r#""type":"configuration""#,
+        r#""wordlist":"/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt""#,
+        r#""config""#,
+        r#""proxy":"""#,
+        r#""replay_proxy":"""#,
+        r#""target_url":"""#,
+        r#""status_codes":[200,204,301,302,307,308,401,403,405,500]"#,
+        r#""replay_codes":[200,204,301,302,307,308,401,403,405,500]"#,
+        r#""filter_status":[]"#,
+        r#""threads":50"#,
+        r#""timeout":7"#,
+        r#""verbosity":0"#,
+        r#""silent":false"#,
+        r#""quiet":false"#,
+        r#""auto_bail":false"#,
+        r#""auto_tune":false"#,
+        r#""json":false"#,
+        r#""output":"""#,
+        r#""debug_log":"""#,
+        &format!(r#""user_agent":"feroxbuster/{}""#, VERSION),
+        r#""random_agent":false"#,
+        r#""redirects":false"#,
+        r#""insecure":false"#,
+        r#""extensions":[]"#,
+        r#""headers""#,
+        r#""queries":[]"#,
+        r#""no_recursion":false"#,
+        r#""extract_links":false"#,
+        r#""add_slash":false"#,
+        r#""stdin":false"#,
+        r#""depth":4"#,
+        r#""scan_limit":0"#,
+        r#""parallel":0"#,
+        r#""rate_limit":0"#,
+        r#""filter_size":[]"#,
+        r#""filter_line_count":[]"#,
+        r#""filter_word_count":[]"#,
+        r#""filter_regex":[]"#,
+        r#""dont_filter":false"#,
+        r#""resumed":false"#,
+        r#""resume_from":"""#,
+        r#""save_state":false"#,
+        r#""time_limit":"""#,
+        r#""filter_similar":[]"#,
+        r#""url_denylist":[]"#,
+        r#""responses""#,
+        r#""type":"response""#,
+        r#""url":"https://nerdcore.com/css""#,
+        r#""path":"/css""#,
+        r#""wildcard":true"#,
+        r#""status":301"#,
+        r#""content_length":173"#,
+        r#""line_count":10"#,
+        r#""word_count":16"#,
+        r#""headers""#,
+        r#""server":"nginx/1.16.1"#,
+    ]
+    .iter()
+    {
+        assert!(
+            predicates::str::contains(*expected).eval(&json_state),
+            "{}",
+            expected
+        )
+    }
 }
 
 #[should_panic]
@@ -437,10 +501,14 @@ fn feroxscan_display() {
         scan_order: ScanOrder::Latest,
         scan_type: Default::default(),
         num_requests: 0,
+        start_time: Instant::now(),
         output_level: OutputLevel::Default,
+        status_403s: Default::default(),
+        status_429s: Default::default(),
         status: Default::default(),
         task: tokio::sync::Mutex::new(None),
         progress_bar: std::sync::Mutex::new(None),
+        errors: Default::default(),
     };
 
     let not_started = format!("{}", scan);
@@ -477,12 +545,16 @@ async fn ferox_scan_abort() {
         scan_order: ScanOrder::Latest,
         scan_type: Default::default(),
         num_requests: 0,
+        start_time: Instant::now(),
         output_level: OutputLevel::Default,
+        status_403s: Default::default(),
+        status_429s: Default::default(),
         status: std::sync::Mutex::new(ScanStatus::Running),
         task: tokio::sync::Mutex::new(Some(tokio::spawn(async move {
             sleep(Duration::from_millis(SLEEP_DURATION * 2));
         }))),
         progress_bar: std::sync::Mutex::new(None),
+        errors: Default::default(),
     };
 
     scan.abort().await.unwrap();
@@ -512,7 +584,78 @@ fn menu_print_header_and_footer() {
 fn split_to_nums_is_correct() {
     let menu = Menu::new();
 
-    let nums = menu.split_to_nums("1, 3,      4");
+    let nums = menu.split_to_nums("1, 3,      4, 7 -     12, 10-10, 10-11, 9-12, 12-6, -1, 4-");
 
-    assert_eq!(nums, vec![1, 3, 4]);
+    assert_eq!(nums, vec![1, 3, 4, 7, 8, 9, 10, 11, 12]);
+    assert_eq!(menu.split_to_nums("9-12"), vec![9, 10, 11, 12]);
+    assert!(menu.split_to_nums("-12").is_empty());
+    assert!(menu.split_to_nums("12-").is_empty());
+    assert!(menu.split_to_nums("\n").is_empty());
+}
+
+#[test]
+/// given a deep url, find the correct scan
+fn get_base_scan_by_url_finds_correct_scan() {
+    let urls = FeroxScans::default();
+    let url = "http://localhost";
+    let url1 = "http://localhost/stuff";
+    let url2 = "http://shlocalhost/stuff/things";
+    let url3 = "http://shlocalhost/stuff/things/mostuff";
+    let (_, scan) = urls.add_scan(url, ScanType::Directory, ScanOrder::Latest);
+    let (_, scan1) = urls.add_scan(url1, ScanType::Directory, ScanOrder::Latest);
+    let (_, scan2) = urls.add_scan(url2, ScanType::Directory, ScanOrder::Latest);
+    let (_, scan3) = urls.add_scan(url3, ScanType::Directory, ScanOrder::Latest);
+
+    assert_eq!(
+        urls.get_base_scan_by_url("http://localhost/things.php")
+            .unwrap()
+            .id,
+        scan.id
+    );
+    assert_eq!(
+        urls.get_base_scan_by_url("http://localhost/stuff/things.php")
+            .unwrap()
+            .id,
+        scan1.id
+    );
+    assert_eq!(
+        urls.get_base_scan_by_url("http://shlocalhost/stuff/things/mostuff.php")
+            .unwrap()
+            .id,
+        scan2.id
+    );
+    assert_eq!(
+        urls.get_base_scan_by_url("http://shlocalhost/stuff/things/mostuff/mothings.php")
+            .unwrap()
+            .id,
+        scan3.id
+    );
+}
+
+#[test]
+/// given a shallow url without a trailing slash, find the correct scan
+fn get_base_scan_by_url_finds_correct_scan_without_trailing_slash() {
+    let urls = FeroxScans::default();
+    let url = "http://localhost";
+    let (_, scan) = urls.add_scan(url, ScanType::Directory, ScanOrder::Latest);
+    assert_eq!(
+        urls.get_base_scan_by_url("http://localhost/BKPMiherrortBPKcw")
+            .unwrap()
+            .id,
+        scan.id
+    );
+}
+
+#[test]
+/// given a shallow url with a trailing slash, find the correct scan
+fn get_base_scan_by_url_finds_correct_scan_with_trailing_slash() {
+    let urls = FeroxScans::default();
+    let url = "http://127.0.0.1:41971/";
+    let (_, scan) = urls.add_scan(url, ScanType::Directory, ScanOrder::Latest);
+    assert_eq!(
+        urls.get_base_scan_by_url("http://127.0.0.1:41971/BKPMiherrortBPKcw")
+            .unwrap()
+            .id,
+        scan.id
+    );
 }

src/scan_manager/utils.rs

@@ -41,7 +41,7 @@ pub async fn start_max_time_thread(handles: Arc<Handles>) {
         log::trace!("exit: start_max_time_thread");
 
         #[cfg(test)]
-        panic!(handles);
+        panic!("{:?}", handles);
         #[cfg(not(test))]
         let _ = TermInputHandler::sigint_handler(handles.clone());
     }

src/scanner.rs

@@ -1,352 +0,0 @@
-use std::{
-    cmp::max, collections::HashSet, convert::TryInto, ops::Deref, sync::atomic::Ordering,
-    sync::Arc, time::Instant,
-};
-
-use anyhow::{bail, Result};
-use futures::{stream, StreamExt};
-use lazy_static::lazy_static;
-use leaky_bucket::LeakyBucket;
-use tokio::sync::{oneshot, Semaphore};
-
-use crate::{
-    event_handlers::{
-        Command::{self, AddError, UpdateF64Field, UpdateUsizeField},
-        Handles,
-    },
-    extractor::{
-        ExtractionTarget::{ResponseBody, RobotsTxt},
-        ExtractorBuilder,
-    },
-    heuristics,
-    response::FeroxResponse,
-    scan_manager::{FeroxResponses, ScanOrder, ScanStatus, PAUSE_SCAN},
-    statistics::{
-        StatError::Other,
-        StatField::{DirScanTimes, ExpectedPerScan},
-    },
-    url::FeroxUrl,
-    utils::{fmt_err, make_request},
-};
-use tokio::time::Duration;
-
-lazy_static! {
-    /// Vector of FeroxResponse objects
-    pub static ref RESPONSES: FeroxResponses = FeroxResponses::default();
-    // todo consider removing this
-}
-
-/// Makes multiple requests based on the presence of extensions
-struct Requester {
-    /// handles to handlers and config
-    handles: Arc<Handles>,
-
-    /// url that will be scanned
-    target_url: String,
-
-    /// limits requests per second if present
-    rate_limiter: Option<LeakyBucket>,
-}
-
-/// Requester implementation
-impl Requester {
-    /// given a FeroxScanner, create a Requester
-    pub fn from(scanner: &FeroxScanner) -> Result<Self> {
-        let limit = scanner.handles.config.rate_limit;
-        let refill = max(limit / 10, 1); // minimum of 1 per second
-        let tokens = max(limit / 2, 1);
-        let interval = if refill == 1 { 1000 } else { 100 }; // 1 second if refill is 1
-
-        let rate_limiter = if limit > 0 {
-            let bucket = LeakyBucket::builder()
-                .refill_interval(Duration::from_millis(interval)) // add tokens every 0.1s
-                .refill_amount(refill) // ex: 100 req/s -> 10 tokens per 0.1s
-                .tokens(tokens) // reduce initial burst, 2 is arbitrary, but felt good
-                .max(limit)
-                .build()?;
-            Some(bucket)
-        } else {
-            None
-        };
-
-        Ok(Self {
-            rate_limiter,
-            handles: scanner.handles.clone(),
-            target_url: scanner.target_url.to_owned(),
-        })
-    }
-
-    /// limit the number of requests per second
-    pub async fn limit(&self) -> Result<()> {
-        self.rate_limiter.as_ref().unwrap().acquire_one().await?;
-        Ok(())
-    }
-
-    /// Wrapper for [make_request](fn.make_request.html)
-    ///
-    /// Attempts recursion when appropriate and sends Responses to the output handler for processing
-    async fn request(&self, word: &str) -> Result<()> {
-        log::trace!("enter: request({})", word);
-
-        let urls =
-            FeroxUrl::from_string(&self.target_url, self.handles.clone()).formatted_urls(word)?;
-
-        for url in urls {
-            if self.rate_limiter.is_some() {
-                // found a rate limiter, limit that junk!
-                if let Err(e) = self.limit().await {
-                    log::warn!("Could not rate limit scan: {}", e);
-                    self.handles.stats.send(AddError(Other)).unwrap_or_default();
-                }
-            }
-
-            let response = make_request(
-                &self.handles.config.client,
-                &url,
-                self.handles.config.output_level,
-                self.handles.stats.tx.clone(),
-            )
-            .await?;
-
-            // response came back without error, convert it to FeroxResponse
-            let ferox_response =
-                FeroxResponse::from(response, true, self.handles.config.output_level).await;
-
-            // do recursion if appropriate
-            if !self.handles.config.no_recursion {
-                self.handles
-                    .send_scan_command(Command::TryRecursion(Box::new(ferox_response.clone())))?;
-                let (tx, rx) = oneshot::channel::<bool>();
-                self.handles.send_scan_command(Command::Sync(tx))?;
-                rx.await?;
-            }
-
-            // purposefully doing recursion before filtering. the thought process is that
-            // even though this particular url is filtered, subsequent urls may not
-            if self
-                .handles
-                .filters
-                .data
-                .should_filter_response(&ferox_response, self.handles.stats.tx.clone())
-            {
-                continue;
-            }
-
-            if self.handles.config.extract_links && !ferox_response.status().is_redirection() {
-                let extractor = ExtractorBuilder::default()
-                    .target(ResponseBody)
-                    .response(&ferox_response)
-                    .handles(self.handles.clone())
-                    .build()?;
-
-                extractor.extract().await?;
-            }
-
-            // everything else should be reported
-            if let Err(e) = ferox_response.send_report(self.handles.output.tx.clone()) {
-                log::warn!("Could not send FeroxResponse to output handler: {}", e);
-            }
-        }
-
-        log::trace!("exit: request");
-        Ok(())
-    }
-}
-
-/// handles the main muscle movement of scanning a url
-pub struct FeroxScanner {
-    /// handles to handlers and config
-    handles: Arc<Handles>,
-
-    /// url that will be scanned
-    target_url: String,
-
-    /// whether or not this scanner is targeting an initial target specified by the user or one
-    /// found via recursion
-    order: ScanOrder,
-
-    /// wordlist that's already been read from disk
-    wordlist: Arc<HashSet<String>>,
-
-    /// limiter that restricts the number of active FeroxScanners  
-    scan_limiter: Arc<Semaphore>,
-}
-
-/// FeroxScanner implementation
-impl FeroxScanner {
-    /// create a new FeroxScanner
-    pub fn new(
-        target_url: &str,
-        order: ScanOrder,
-        wordlist: Arc<HashSet<String>>,
-        scan_limiter: Arc<Semaphore>,
-        handles: Arc<Handles>,
-    ) -> Self {
-        Self {
-            order,
-            handles,
-            wordlist,
-            scan_limiter,
-            target_url: target_url.to_string(),
-        }
-    }
-
-    /// Scan a given url using a given wordlist
-    ///
-    /// This is the primary entrypoint for the scanner
-    pub async fn scan_url(&self) -> Result<()> {
-        log::trace!("enter: scan_url");
-        log::info!("Starting scan against: {}", self.target_url);
-
-        let scan_timer = Instant::now();
-
-        if matches!(self.order, ScanOrder::Initial) && self.handles.config.extract_links {
-            // only grab robots.txt on the initial scan_url calls. all fresh dirs will be passed
-            // to try_recursion
-            let extractor = ExtractorBuilder::default()
-                .url(&self.target_url)
-                .handles(self.handles.clone())
-                .target(RobotsTxt)
-                .build()?;
-
-            let _ = extractor.extract().await;
-        }
-
-        let scanned_urls = self.handles.ferox_scans()?;
-
-        let ferox_scan = match scanned_urls.get_scan_by_url(&self.target_url) {
-            Some(scan) => {
-                scan.set_status(ScanStatus::Running)?;
-                scan
-            }
-            None => {
-                let msg = format!(
-                    "Could not find FeroxScan associated with {}; this shouldn't happen... exiting",
-                    self.target_url
-                );
-                bail!(fmt_err(&msg))
-            }
-        };
-
-        let progress_bar = ferox_scan.progress_bar();
-
-        // When acquire is called and the semaphore has remaining permits, the function immediately
-        // returns a permit. However, if no remaining permits are available, acquire (asynchronously)
-        // waits until an outstanding permit is dropped, at which point, the freed permit is assigned
-        // to the caller.
-        let _permit = self.scan_limiter.acquire().await;
-
-        // Arc clones to be passed around to the various scans
-        let looping_words = self.wordlist.clone();
-
-        {
-            let test = heuristics::HeuristicTests::new(self.handles.clone());
-            if let Ok(num_reqs) = test.wildcard(&self.target_url).await {
-                progress_bar.inc(num_reqs);
-            }
-        }
-
-        let requester = Arc::new(Requester::from(self)?);
-        let increment_len = (self.handles.config.extensions.len() + 1) as u64;
-
-        // producer tasks (mp of mpsc); responsible for making requests
-        let producers = stream::iter(looping_words.deref().to_owned())
-            .map(|word| {
-                let pb = progress_bar.clone(); // progress bar is an Arc around internal state
-                let scanned_urls_clone = scanned_urls.clone();
-                let requester_clone = requester.clone();
-                (
-                    tokio::spawn(async move {
-                        if PAUSE_SCAN.load(Ordering::Acquire) {
-                            // for every word in the wordlist, check to see if PAUSE_SCAN is set to true
-                            // when true; enter a busy loop that only exits by setting PAUSE_SCAN back
-                            // to false
-                            scanned_urls_clone.pause(true).await;
-                        }
-                        requester_clone.request(&word).await
-                    }),
-                    pb,
-                )
-            })
-            .for_each_concurrent(self.handles.config.threads, |(resp, bar)| async move {
-                match resp.await {
-                    Ok(_) => {
-                        bar.inc(increment_len);
-                    }
-                    Err(e) => {
-                        log::warn!("error awaiting a response: {}", e);
-                        self.handles.stats.send(AddError(Other)).unwrap_or_default();
-                    }
-                }
-            });
-
-        // await tx tasks
-        log::trace!("awaiting scan producers");
-        producers.await;
-        log::trace!("done awaiting scan producers");
-
-        self.handles.stats.send(UpdateF64Field(
-            DirScanTimes,
-            scan_timer.elapsed().as_secs_f64(),
-        ))?;
-
-        ferox_scan.finish()?;
-
-        log::trace!("exit: scan_url");
-
-        Ok(())
-    }
-}
-
-/// Perform steps necessary to run scans that only need to be performed once (warming up the
-/// engine, as it were)
-pub async fn initialize(num_words: usize, handles: Arc<Handles>) -> Result<()> {
-    log::trace!("enter: initialize({}, {:?})", num_words, handles);
-
-    // number of requests only needs to be calculated once, and then can be reused
-    let num_reqs_expected: u64 = if handles.config.extensions.is_empty() {
-        num_words.try_into()?
-    } else {
-        let total = num_words * (handles.config.extensions.len() + 1);
-        total.try_into()?
-    };
-
-    {
-        // no real reason to keep the arc around beyond this call
-        let scans = handles.ferox_scans()?;
-        scans.set_bar_length(num_reqs_expected);
-    }
-
-    // tell Stats object about the number of expected requests
-    handles.stats.send(UpdateUsizeField(
-        ExpectedPerScan,
-        num_reqs_expected as usize,
-    ))?;
-
-    log::trace!("exit: initialize");
-    Ok(())
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::config::OutputLevel;
-    use crate::scan_manager::FeroxScans;
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
-    #[should_panic]
-    /// try to hit struct field coverage of FileOutHandler
-    async fn get_scan_by_url_bails_on_unfound_url() {
-        let sem = Semaphore::new(10);
-        let urls = FeroxScans::new(OutputLevel::Default);
-
-        let scanner = FeroxScanner::new(
-            "http://localhost",
-            ScanOrder::Initial,
-            Arc::new(Default::default()),
-            Arc::new(sem),
-            Arc::new(Handles::for_testing(Some(Arc::new(urls)), None).0),
-        );
-        scanner.scan_url().await.unwrap();
-    }
-}

src/scanner/ferox_scanner.rs

@@ -0,0 +1,186 @@
+use std::{ops::Deref, sync::atomic::Ordering, sync::Arc, time::Instant};
+
+use anyhow::{bail, Result};
+use futures::{stream, StreamExt};
+use lazy_static::lazy_static;
+use tokio::sync::Semaphore;
+
+use crate::{
+    event_handlers::{
+        Command::{AddError, AddToF64Field, SubtractFromUsizeField},
+        Handles,
+    },
+    extractor::{ExtractionTarget::RobotsTxt, ExtractorBuilder},
+    heuristics,
+    scan_manager::{FeroxResponses, ScanOrder, ScanStatus, PAUSE_SCAN},
+    statistics::{
+        StatError::Other,
+        StatField::{DirScanTimes, TotalExpected},
+    },
+    utils::fmt_err,
+};
+
+use super::requester::Requester;
+
+lazy_static! {
+    /// Vector of FeroxResponse objects
+    pub static ref RESPONSES: FeroxResponses = FeroxResponses::default();
+    // todo consider removing this
+}
+/// handles the main muscle movement of scanning a url
+pub struct FeroxScanner {
+    /// handles to handlers and config
+    pub(super) handles: Arc<Handles>,
+
+    /// url that will be scanned
+    pub(super) target_url: String,
+
+    /// whether or not this scanner is targeting an initial target specified by the user or one
+    /// found via recursion
+    order: ScanOrder,
+
+    /// wordlist that's already been read from disk
+    wordlist: Arc<Vec<String>>,
+
+    /// limiter that restricts the number of active FeroxScanners  
+    scan_limiter: Arc<Semaphore>,
+}
+
+/// FeroxScanner implementation
+impl FeroxScanner {
+    /// create a new FeroxScanner
+    pub fn new(
+        target_url: &str,
+        order: ScanOrder,
+        wordlist: Arc<Vec<String>>,
+        scan_limiter: Arc<Semaphore>,
+        handles: Arc<Handles>,
+    ) -> Self {
+        Self {
+            order,
+            handles,
+            wordlist,
+            scan_limiter,
+            target_url: target_url.to_string(),
+        }
+    }
+
+    /// Scan a given url using a given wordlist
+    ///
+    /// This is the primary entrypoint for the scanner
+    pub async fn scan_url(&self) -> Result<()> {
+        log::trace!("enter: scan_url");
+        log::info!("Starting scan against: {}", self.target_url);
+
+        let scan_timer = Instant::now();
+
+        if matches!(self.order, ScanOrder::Initial) && self.handles.config.extract_links {
+            // only grab robots.txt on the initial scan_url calls. all fresh dirs will be passed
+            // to try_recursion
+            let extractor = ExtractorBuilder::default()
+                .url(&self.target_url)
+                .handles(self.handles.clone())
+                .target(RobotsTxt)
+                .build()?;
+
+            let links = extractor.extract().await?;
+            extractor.request_links(links).await?;
+        }
+
+        let scanned_urls = self.handles.ferox_scans()?;
+
+        let ferox_scan = match scanned_urls.get_scan_by_url(&self.target_url) {
+            Some(scan) => {
+                scan.set_status(ScanStatus::Running)?;
+                scan
+            }
+            None => {
+                let msg = format!(
+                    "Could not find FeroxScan associated with {}; this shouldn't happen... exiting",
+                    self.target_url
+                );
+                bail!(fmt_err(&msg))
+            }
+        };
+
+        let progress_bar = ferox_scan.progress_bar();
+
+        // When acquire is called and the semaphore has remaining permits, the function immediately
+        // returns a permit. However, if no remaining permits are available, acquire (asynchronously)
+        // waits until an outstanding permit is dropped, at which point, the freed permit is assigned
+        // to the caller.
+        let _permit = self.scan_limiter.acquire().await;
+
+        // Arc clones to be passed around to the various scans
+        let looping_words = self.wordlist.clone();
+
+        {
+            let test = heuristics::HeuristicTests::new(self.handles.clone());
+            if let Ok(num_reqs) = test.wildcard(&self.target_url).await {
+                progress_bar.inc(num_reqs);
+            }
+        }
+
+        let requester = Arc::new(Requester::from(self, ferox_scan.clone())?);
+        let increment_len = (self.handles.config.extensions.len() + 1) as u64;
+
+        // producer tasks (mp of mpsc); responsible for making requests
+        let producers = stream::iter(looping_words.deref().to_owned())
+            .map(|word| {
+                let pb = progress_bar.clone(); // progress bar is an Arc around internal state
+                let scanned_urls_clone = scanned_urls.clone();
+                let requester_clone = requester.clone();
+                let handles_clone = self.handles.clone();
+                (
+                    tokio::spawn(async move {
+                        if PAUSE_SCAN.load(Ordering::Acquire) {
+                            // for every word in the wordlist, check to see if PAUSE_SCAN is set to true
+                            // when true; enter a busy loop that only exits by setting PAUSE_SCAN back
+                            // to false
+                            let num_cancelled = scanned_urls_clone.pause(true).await;
+                            if num_cancelled > 0 {
+                                handles_clone
+                                    .stats
+                                    .send(SubtractFromUsizeField(TotalExpected, num_cancelled))
+                                    .unwrap_or_else(|e| {
+                                        log::warn!("Could not update overall scan bar: {}", e)
+                                    });
+                            }
+                        }
+                        requester_clone
+                            .request(&word)
+                            .await
+                            .unwrap_or_else(|e| log::warn!("Requester encountered an error: {}", e))
+                    }),
+                    pb,
+                )
+            })
+            .for_each_concurrent(self.handles.config.threads, |(resp, bar)| async move {
+                match resp.await {
+                    Ok(_) => {
+                        bar.inc(increment_len);
+                    }
+                    Err(e) => {
+                        log::warn!("error awaiting a response: {}", e);
+                        self.handles.stats.send(AddError(Other)).unwrap_or_default();
+                    }
+                }
+            });
+
+        // await tx tasks
+        log::trace!("awaiting scan producers");
+        producers.await;
+        log::trace!("done awaiting scan producers");
+
+        self.handles.stats.send(AddToF64Field(
+            DirScanTimes,
+            scan_timer.elapsed().as_secs_f64(),
+        ))?;
+
+        ferox_scan.finish()?;
+
+        log::trace!("exit: scan_url");
+
+        Ok(())
+    }
+}

src/scanner/init.rs

@@ -0,0 +1,34 @@
+use crate::{
+    event_handlers::{Command::AddToUsizeField, Handles},
+    statistics::StatField::ExpectedPerScan,
+};
+use anyhow::Result;
+use std::{convert::TryInto, sync::Arc};
+
+/// Perform steps necessary to run scans that only need to be performed once (warming up the
+/// engine, as it were)
+pub async fn initialize(num_words: usize, handles: Arc<Handles>) -> Result<()> {
+    log::trace!("enter: initialize({}, {:?})", num_words, handles);
+
+    // number of requests only needs to be calculated once, and then can be reused
+    let num_reqs_expected: u64 = if handles.config.extensions.is_empty() {
+        num_words.try_into()?
+    } else {
+        let total = num_words * (handles.config.extensions.len() + 1);
+        total.try_into()?
+    };
+
+    {
+        // no real reason to keep the arc around beyond this call
+        let scans = handles.ferox_scans()?;
+        scans.set_bar_length(num_reqs_expected);
+    }
+
+    // tell Stats object about the number of expected requests
+    handles
+        .stats
+        .send(AddToUsizeField(ExpectedPerScan, num_reqs_expected as usize))?;
+
+    log::trace!("exit: initialize");
+    Ok(())
+}

src/scanner/limit_heap.rs

@@ -0,0 +1,171 @@
+use std::fmt::{Debug, Formatter, Result};
+
+/// bespoke variation on an array-backed max-heap
+///
+/// 255 possible values generated from the initial requests/second
+///
+/// when no additional errors are encountered, the left child is taken (increasing req/sec)
+/// if errors have increased since the last interval, the right child is taken (decreasing req/sec)
+///
+/// formula for each child:
+/// - left: (|parent - current|) / 2 + current
+/// - right: current - ((|parent - current|) / 2)
+pub(super) struct LimitHeap {
+    /// backing array, 255 nodes == height of 7 ( 2^(h+1) -1 nodes )
+    pub(super) inner: [i32; 255],
+
+    /// original # of requests / second
+    pub(super) original: i32,
+
+    /// current position w/in the backing array
+    pub(super) current: usize,
+}
+
+/// default implementation of a LimitHeap
+impl Default for LimitHeap {
+    /// zero-initialize the backing array
+    fn default() -> Self {
+        Self {
+            inner: [0; 255],
+            original: 0,
+            current: 0,
+        }
+    }
+}
+
+/// Debug implementation of a LimitHeap
+impl Debug for LimitHeap {
+    /// return debug representation that conforms to <32 elements in array
+    fn fmt(&self, f: &mut Formatter<'_>) -> Result {
+        let msg = format!(
+            "LimitHeap {{ original: {}, current: {}, inner: [{}...] }}",
+            self.original, self.current, self.inner[0]
+        );
+        write!(f, "{}", msg)
+    }
+}
+
+/// implementation of a LimitHeap
+impl LimitHeap {
+    /// move to right child, return node's index from which the move was requested
+    pub(super) fn move_right(&mut self) -> usize {
+        if self.has_children() {
+            let tmp = self.current;
+            self.current = self.current * 2 + 2;
+            return tmp;
+        }
+        self.current
+    }
+
+    /// move to left child, return node's index from which the move was requested
+    pub(super) fn move_left(&mut self) -> usize {
+        if self.has_children() {
+            let tmp = self.current;
+            self.current = self.current * 2 + 1;
+            return tmp;
+        }
+        self.current
+    }
+
+    /// move to parent, return node's index from which the move was requested
+    pub(super) fn move_up(&mut self) -> usize {
+        if self.has_parent() {
+            let tmp = self.current;
+            self.current = (self.current - 1) / 2;
+            return tmp;
+        }
+        self.current
+    }
+
+    /// move directly to the given index
+    pub(super) fn move_to(&mut self, index: usize) {
+        self.current = index;
+    }
+
+    /// get the current node's value
+    pub(super) fn value(&self) -> i32 {
+        self.inner[self.current]
+    }
+
+    /// set the current node's value
+    pub(super) fn set_value(&mut self, value: i32) {
+        self.inner[self.current] = value;
+    }
+
+    /// check that this node has a parent (true for all except root)
+    pub(super) fn has_parent(&self) -> bool {
+        self.current > 0
+    }
+
+    /// get node's parent's value or self.original if at the root
+    pub(super) fn parent_value(&mut self) -> i32 {
+        if self.has_parent() {
+            let current = self.move_up();
+            let val = self.value();
+            self.move_to(current);
+            return val;
+        }
+        self.original
+    }
+
+    /// check if the current node has children
+    pub(super) fn has_children(&self) -> bool {
+        // inner structure is a complete tree, just check for the right child
+        self.current * 2 + 2 <= self.inner.len()
+    }
+
+    /// get current node's right child's value
+    fn right_child_value(&mut self) -> i32 {
+        let tmp = self.move_right();
+        let val = self.value();
+        self.move_to(tmp);
+        val
+    }
+
+    /// set current node's left child's value
+    fn set_left_child(&mut self) {
+        let parent = self.parent_value();
+        let current = self.value();
+        let value = ((parent - current).abs() / 2) + current;
+
+        self.move_left();
+        self.set_value(value);
+        self.move_up();
+    }
+
+    /// set current node's right child's value
+    fn set_right_child(&mut self) {
+        let parent = self.parent_value();
+        let current = self.value();
+        let value = current - ((parent - current).abs() / 2);
+
+        self.move_right();
+        self.set_value(value);
+        self.move_up();
+    }
+
+    /// iterate over the backing array, filling in each child's value based on the original value
+    pub(super) fn build(&mut self) {
+        // ex: original is 400
+        // arr[0] == 200
+        // arr[1] (left child) == 300
+        // arr[2] (right child) == 100
+        let root = self.original / 2;
+
+        self.inner[0] = root; // set root node to half of the original value
+        self.inner[1] = ((self.original - root).abs() / 2) + root;
+        self.inner[2] = root - ((self.original - root).abs() / 2);
+
+        // start with index 1 and fill in each child below that node
+        for i in 1..self.inner.len() {
+            self.move_to(i);
+
+            if self.has_children() && self.right_child_value() == 0 {
+                // this node has an unset child since the rchild is 0
+                self.set_left_child();
+                self.set_right_child();
+            }
+        }
+        self.move_to(0); // reset current index to the root of the tree
+    }
+}

src/scanner/mod.rs

@@ -0,0 +1,12 @@
+mod ferox_scanner;
+mod utils;
+mod init;
+#[cfg(test)]
+mod tests;
+mod limit_heap;
+mod policy_data;
+mod requester;
+
+pub use self::ferox_scanner::{FeroxScanner, RESPONSES};
+pub use self::init::initialize;
+pub use self::utils::PolicyTrigger;

src/scanner/policy_data.rs

@@ -0,0 +1,306 @@
+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+
+use crate::{atomic_load, atomic_store, config::RequesterPolicy};
+
+use super::limit_heap::LimitHeap;
+
+/// data regarding policy and metadata about last enforced trigger etc...
+#[derive(Default, Debug)]
+pub struct PolicyData {
+    /// how to handle exceptional cases such as too many errors / 403s / 429s etc
+    pub(super) policy: RequesterPolicy,
+
+    /// whether or not we're in the middle of a cooldown period
+    pub(super) cooling_down: AtomicBool,
+
+    /// length of time to pause tuning after making an adjustment
+    pub(super) wait_time: u64,
+
+    /// rate limit (at last interval)
+    limit: AtomicUsize,
+
+    /// number of errors (at last interval)
+    pub(super) errors: AtomicUsize,
+
+    /// whether or not the owning Requester should remove the rate_limiter, happens when a scan
+    /// has been limited and moves back up to the point of its original scan speed
+    pub(super) remove_limit: AtomicBool,
+
+    /// heap of values used for adjusting # of requests/second
+    pub(super) heap: std::sync::RwLock<LimitHeap>,
+}
+
+/// implementation of PolicyData
+impl PolicyData {
+    /// given a RequesterPolicy, create a new PolicyData
+    pub fn new(policy: RequesterPolicy, timeout: u64) -> Self {
+        // can use this as a tweak for how aggressively adjustments should be made when tuning
+        let wait_time = ((timeout as f64 / 2.0) * 1000.0) as u64;
+
+        Self {
+            policy,
+            wait_time,
+            ..Default::default()
+        }
+    }
+
+    /// setter for requests / second; populates the underlying heap with values from req/sec seed
+    pub(super) fn set_reqs_sec(&self, reqs_sec: usize) {
+        if let Ok(mut guard) = self.heap.write() {
+            guard.original = reqs_sec as i32;
+            guard.build();
+            self.set_limit(guard.inner[0] as usize); // set limit to 1/2 of current request rate
+        }
+    }
+
+    /// setter for errors
+    pub(super) fn set_errors(&self, errors: usize) {
+        atomic_store!(self.errors, errors);
+    }
+
+    /// setter for limit
+    fn set_limit(&self, limit: usize) {
+        atomic_store!(self.limit, limit);
+    }
+
+    /// getter for limit
+    pub(super) fn get_limit(&self) -> usize {
+        atomic_load!(self.limit)
+    }
+
+    /// adjust the rate of requests per second up (increase rate)
+    pub(super) fn adjust_up(&self, streak_counter: &usize) {
+        if let Ok(mut heap) = self.heap.try_write() {
+            if *streak_counter > 2 {
+                // streak of 3 upward moves in a row, traverse the tree upward instead of to a
+                // higher-valued branch lower in the tree
+                let current = heap.value();
+                heap.move_up();
+                heap.move_up();
+                if current > heap.value() {
+                    // the tree's structure makes it so that sometimes 2 moves up results in a
+                    // value greater than the current node's and other times we need to move 3 up
+                    // to arrive at a greater value
+                    if heap.has_parent() && heap.parent_value() > current {
+                        // all nodes except 0th node (root)
+                        heap.move_up();
+                    } else if !heap.has_parent() {
+                        // been here enough that we can try resuming the scan to its original
+                        // speed (no limiting at all)
+                        atomic_store!(self.remove_limit, true);
+                    }
+                }
+            } else if heap.has_children() {
+                // streak not at 3, just check that we can move down, and do so
+                heap.move_left();
+            } else {
+                // tree bottomed out, need to move back up the tree a bit
+                let current = heap.value();
+                heap.move_up();
+                heap.move_up();
+
+                if current > heap.value() {
+                    heap.move_up();
+                }
+            }
+            self.set_limit(heap.value() as usize);
+        }
+    }
+
+    /// adjust the rate of requests per second down (decrease rate)
+    pub(super) fn adjust_down(&self) {
+        if let Ok(mut heap) = self.heap.try_write() {
+            if heap.has_children() {
+                heap.move_right();
+                self.set_limit(heap.value() as usize);
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    /// PolicyData builds and sets correct values for the inner heap when set_reqs_sec is called
+    fn set_reqs_sec_builds_heap_and_sets_initial_value() {
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        assert_eq!(pd.wait_time, 3500);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+        assert_eq!(pd.heap.read().unwrap().original, 400);
+        assert_eq!(pd.heap.read().unwrap().current, 0);
+        assert_eq!(pd.heap.read().unwrap().inner[0], 200);
+        assert_eq!(pd.heap.read().unwrap().inner[1], 300);
+        assert_eq!(pd.heap.read().unwrap().inner[2], 100);
+    }
+
+    #[test]
+    /// PolicyData setters/getters tests for code coverage / sanity
+    fn policy_data_getters_and_setters() {
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_errors(20);
+        assert_eq!(pd.errors.load(Ordering::Relaxed), 20);
+        pd.set_limit(200);
+        assert_eq!(pd.get_limit(), 200);
+    }
+
+    #[test]
+    /// PolicyData adjust_down sets the limit to the correct value
+    fn policy_data_adjust_down_simple() {
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+        pd.adjust_down();
+        assert_eq!(pd.get_limit(), 100);
+    }
+
+    #[test]
+    /// PolicyData adjust_down sets the limit to the correct value when no child nodes are present
+    fn policy_data_adjust_down_no_children() {
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+        let mut guard = pd.heap.write().unwrap();
+        guard.move_to(250);
+        guard.set_value(27);
+        pd.set_limit(guard.value() as usize);
+        drop(guard);
+
+        pd.adjust_down();
+        assert_eq!(pd.get_limit(), 27);
+    }
+
+    #[test]
+    /// PolicyData adjust_up sets the limit to the correct value
+    fn policy_data_adjust_up_simple() {
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+        pd.adjust_up(&0);
+        assert_eq!(pd.get_limit(), 300);
+    }
+
+    #[test]
+    /// PolicyData adjust_up sets the limit to the correct value
+    fn policy_data_adjust_up_with_streak_and_2_moves() {
+        // original: 400
+        // [200, 300, 100, 350, 250, 150, 50, 375, 325, 275, 225, 175, 125, 75, 25, ...]
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+
+        // 2 moves
+        pd.heap.write().unwrap().move_to(9);
+        assert_eq!(pd.heap.read().unwrap().value(), 275);
+        pd.adjust_up(&3);
+        assert_eq!(pd.heap.read().unwrap().value(), 300);
+        assert_eq!(pd.limit.load(Ordering::Relaxed), 300);
+        assert!(!pd.remove_limit.load(Ordering::Relaxed));
+    }
+
+    #[test]
+    /// PolicyData adjust_up sets the limit to the correct value
+    fn policy_data_adjust_up_with_streak_and_2_moves_to_arrive_at_root() {
+        // original: 400
+        // [200, 300, 100, 350, 250, 150, 50, 375, 325, 275, 225, 175, 125, 75, 25, ...]
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+
+        pd.heap.write().unwrap().move_to(4);
+        assert_eq!(pd.heap.read().unwrap().value(), 250);
+        pd.adjust_up(&3);
+        assert_eq!(pd.heap.read().unwrap().value(), 200);
+        assert_eq!(pd.limit.load(Ordering::Relaxed), 200);
+        assert!(pd.remove_limit.load(Ordering::Relaxed));
+    }
+
+    #[test]
+    /// PolicyData adjust_up sets the limit to the correct value
+    fn policy_data_adjust_up_with_streak_and_2_moves_to_find_less_than_current() {
+        // original: 400
+        // [200, 300, 100, 350, 250, 150, 50, 375, 325, 275, 225, 175, 125, 75, 25, ...]
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+
+        pd.heap.write().unwrap().move_to(15);
+        assert_eq!(pd.heap.read().unwrap().value(), 387);
+        pd.adjust_up(&3);
+        assert_eq!(pd.heap.read().unwrap().value(), 350);
+        assert_eq!(pd.limit.load(Ordering::Relaxed), 350);
+        assert!(!pd.remove_limit.load(Ordering::Relaxed));
+    }
+
+    #[test]
+    /// PolicyData adjust_up sets the limit to the correct value
+    fn policy_data_adjust_up_with_streak_and_3_moves() {
+        // original: 400
+        // [200, 300, 100, 350, 250, 150, 50, 375, 325, 275, 225, 175, 125, 75, 25, ...]
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+
+        pd.heap.write().unwrap().move_to(19);
+        assert_eq!(pd.heap.read().unwrap().value(), 287);
+        pd.adjust_up(&3);
+        assert_eq!(pd.heap.read().unwrap().value(), 300);
+        assert_eq!(pd.limit.load(Ordering::Relaxed), 300);
+        assert!(!pd.remove_limit.load(Ordering::Relaxed));
+    }
+
+    #[test]
+    /// PolicyData adjust_up sets the limit to the correct value
+    fn policy_data_adjust_up_with_no_children_2_moves() {
+        // original: 400
+        // [200, 300, 100, 350, 250, 150, 50, 375, 325, 275, 225, 175, 125, 75, 25, ...]
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+
+        pd.heap.write().unwrap().move_to(241);
+
+        assert_eq!(pd.heap.read().unwrap().value(), 41);
+        pd.adjust_up(&0);
+        assert_eq!(pd.heap.read().unwrap().value(), 43);
+        assert_eq!(pd.limit.load(Ordering::Relaxed), 43);
+        assert!(!pd.remove_limit.load(Ordering::Relaxed));
+    }
+
+    #[test]
+    /// PolicyData adjust_up sets the limit to the correct value
+    fn policy_data_adjust_up_with_no_children_3_moves() {
+        // original: 400
+        // [200, 300, 100, 350, 250, 150, 50, 375, 325, 275, 225, 175, 125, 75, 25, ...]
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+        assert_eq!(pd.get_limit(), 200);
+
+        pd.heap.write().unwrap().move_to(240);
+
+        assert_eq!(pd.heap.read().unwrap().value(), 45);
+        pd.adjust_up(&0);
+        assert_eq!(pd.heap.read().unwrap().value(), 37);
+        assert_eq!(pd.limit.load(Ordering::Relaxed), 37);
+        assert!(!pd.remove_limit.load(Ordering::Relaxed));
+    }
+
+    #[test]
+    /// hit some of the out of the way corners of limitheap for coverage
+    fn increase_limit_heap_coverage_by_hitting_edge_cases() {
+        let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+        pd.set_reqs_sec(400);
+
+        println!("{:?}", pd.heap.read().unwrap()); // debug derivation
+
+        pd.heap.write().unwrap().move_to(240);
+        assert_eq!(pd.heap.write().unwrap().move_right(), 240);
+        assert_eq!(pd.heap.write().unwrap().move_left(), 240);
+
+        pd.heap.write().unwrap().move_to(0);
+        assert_eq!(pd.heap.write().unwrap().move_up(), 0);
+        assert_eq!(pd.heap.write().unwrap().parent_value(), 400);
+    }
+}

src/scanner/tests.rs

@@ -0,0 +1,28 @@
+use std::sync::Arc;
+
+use tokio::sync::Semaphore;
+
+use crate::{
+    config::OutputLevel,
+    event_handlers::Handles,
+    scan_manager::{FeroxScans, ScanOrder},
+};
+
+use super::*;
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+#[should_panic]
+/// try to hit struct field coverage of FileOutHandler
+async fn get_scan_by_url_bails_on_unfound_url() {
+    let sem = Semaphore::new(10);
+    let urls = FeroxScans::new(OutputLevel::Default);
+
+    let scanner = FeroxScanner::new(
+        "http://localhost",
+        ScanOrder::Initial,
+        Arc::new(Default::default()),
+        Arc::new(sem),
+        Arc::new(Handles::for_testing(Some(Arc::new(urls)), None).0),
+    );
+    scanner.scan_url().await.unwrap();
+}

src/scanner/utils.rs

@@ -0,0 +1,12 @@
+#[derive(Copy, Clone, PartialEq, Debug)]
+/// represents different situations where different criteria can trigger auto-tune/bail behavior
+pub enum PolicyTrigger {
+    /// excessive 403 trigger
+    Status403,
+
+    /// excessive 429 trigger
+    Status429,
+
+    /// excessive general errors
+    Errors,
+}

src/statistics/container.rs

@@ -1,4 +1,6 @@
 use std::{
+    collections::HashMap,
+    convert::TryFrom,
     fs::File,
     io::BufReader,
     sync::{
@@ -9,7 +11,8 @@ use std::{
 
 use anyhow::{Context, Result};
 use reqwest::StatusCode;
-use serde::{Deserialize, Serialize};
+use serde::{ser::SerializeStruct, Deserialize, Deserializer, Serialize, Serializer};
+use serde_json::Value;
 
 use crate::{
     traits::FeroxSerialize,
@@ -19,9 +22,8 @@ use crate::{
 use super::{error::StatError, field::StatField};
 
 /// Data collection of statistics related to a scan
-#[derive(Default, Deserialize, Debug, Serialize)]
+#[derive(Default, Debug)]
 pub struct Stats {
-    #[serde(rename = "type")]
     /// Name of this type of struct, used for serialization, i.e. `{"type":"statistics"}`
     kind: String,
 
@@ -29,7 +31,7 @@ pub struct Stats {
     timeouts: AtomicUsize,
 
     /// tracker for total number of requests sent by the client
-    requests: AtomicUsize,
+    pub(crate) requests: AtomicUsize,
 
     /// tracker for total number of requests expected to send if the scan runs to completion
     ///
@@ -42,7 +44,7 @@ pub struct Stats {
     total_expected: AtomicUsize,
 
     /// tracker for total number of errors encountered by the client
-    errors: AtomicUsize,
+    pub(crate) errors: AtomicUsize,
 
     /// tracker for overall number of 2xx status codes seen by the client
     successes: AtomicUsize,
@@ -58,7 +60,7 @@ pub struct Stats {
 
     /// tracker for number of scans performed, this directly equates to number of directories
     /// recursed into and affects the total number of expected requests
-    total_scans: AtomicUsize,
+    pub(crate) total_scans: AtomicUsize,
 
     /// tracker for initial number of requested targets
     initial_targets: AtomicUsize,
@@ -80,10 +82,10 @@ pub struct Stats {
     status_401s: AtomicUsize,
 
     /// tracker for overall number of 403s seen by the client
-    status_403s: AtomicUsize,
+    pub(crate) status_403s: AtomicUsize,
 
     /// tracker for overall number of 429s seen by the client
-    status_429s: AtomicUsize,
+    pub(crate) status_429s: AtomicUsize,
 
     /// tracker for overall number of 500s seen by the client
     status_500s: AtomicUsize,
@@ -125,11 +127,9 @@ pub struct Stats {
     total_runtime: Mutex<Vec<f64>>,
 
     /// tracker for the number of extensions the user specified
-    #[serde(skip)]
     num_extensions: usize,
 
     /// tracker for whether to use json during serialization or not
-    #[serde(skip)]
     json: bool,
 }
 
@@ -147,6 +147,301 @@ impl FeroxSerialize for Stats {
     }
 }
 
+/// Serialize implementation for Stats
+impl Serialize for Stats {
+    /// Function that handles serialization of Stats
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let mut state = serializer.serialize_struct("Stats", 32)?;
+
+        state.serialize_field("type", &self.kind)?;
+        state.serialize_field("timeouts", &atomic_load!(self.timeouts))?;
+        state.serialize_field("requests", &atomic_load!(self.requests))?;
+        state.serialize_field("expected_per_scan", &atomic_load!(self.expected_per_scan))?;
+        state.serialize_field("total_expected", &atomic_load!(self.total_expected))?;
+        state.serialize_field("errors", &atomic_load!(self.errors))?;
+        state.serialize_field("successes", &atomic_load!(self.successes))?;
+        state.serialize_field("redirects", &atomic_load!(self.redirects))?;
+        state.serialize_field("client_errors", &atomic_load!(self.client_errors))?;
+        state.serialize_field("server_errors", &atomic_load!(self.server_errors))?;
+        state.serialize_field("total_scans", &atomic_load!(self.total_scans))?;
+        state.serialize_field("initial_targets", &atomic_load!(self.initial_targets))?;
+        state.serialize_field("links_extracted", &atomic_load!(self.links_extracted))?;
+        state.serialize_field("status_200s", &atomic_load!(self.status_200s))?;
+        state.serialize_field("status_301s", &atomic_load!(self.status_301s))?;
+        state.serialize_field("status_302s", &atomic_load!(self.status_302s))?;
+        state.serialize_field("status_401s", &atomic_load!(self.status_401s))?;
+        state.serialize_field("status_403s", &atomic_load!(self.status_403s))?;
+        state.serialize_field("status_429s", &atomic_load!(self.status_429s))?;
+        state.serialize_field("status_500s", &atomic_load!(self.status_500s))?;
+        state.serialize_field("status_503s", &atomic_load!(self.status_503s))?;
+        state.serialize_field("status_504s", &atomic_load!(self.status_504s))?;
+        state.serialize_field("status_508s", &atomic_load!(self.status_508s))?;
+        state.serialize_field("wildcards_filtered", &atomic_load!(self.wildcards_filtered))?;
+        state.serialize_field("responses_filtered", &atomic_load!(self.responses_filtered))?;
+        state.serialize_field(
+            "resources_discovered",
+            &atomic_load!(self.resources_discovered),
+        )?;
+        state.serialize_field("url_format_errors", &atomic_load!(self.url_format_errors))?;
+        state.serialize_field("redirection_errors", &atomic_load!(self.redirection_errors))?;
+        state.serialize_field("connection_errors", &atomic_load!(self.connection_errors))?;
+        state.serialize_field("request_errors", &atomic_load!(self.request_errors))?;
+        state.serialize_field("directory_scan_times", &self.directory_scan_times)?;
+        state.serialize_field("total_runtime", &self.total_runtime)?;
+
+        state.end()
+    }
+}
+
+/// Deserialize implementation for Stats
+impl<'a> Deserialize<'a> for Stats {
+    /// Deserialize a Stats object from a serde_json::Value
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'a>,
+    {
+        let stats = Self::new(0, false);
+
+        let map: HashMap<String, Value> = HashMap::deserialize(deserializer)?;
+
+        for (key, value) in &map {
+            match key.as_str() {
+                "timeouts" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.timeouts, parsed);
+                        }
+                    }
+                }
+                "requests" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.requests, parsed);
+                        }
+                    }
+                }
+                "expected_per_scan" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.expected_per_scan, parsed);
+                        }
+                    }
+                }
+                "total_expected" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.total_expected, parsed);
+                        }
+                    }
+                }
+                "errors" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.errors, parsed);
+                        }
+                    }
+                }
+                "successes" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.successes, parsed);
+                        }
+                    }
+                }
+                "redirects" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.redirects, parsed);
+                        }
+                    }
+                }
+                "client_errors" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.client_errors, parsed);
+                        }
+                    }
+                }
+                "server_errors" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.server_errors, parsed);
+                        }
+                    }
+                }
+                "total_scans" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.total_scans, parsed);
+                        }
+                    }
+                }
+                "initial_targets" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.initial_targets, parsed);
+                        }
+                    }
+                }
+                "links_extracted" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.links_extracted, parsed);
+                        }
+                    }
+                }
+                "status_200s" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.status_200s, parsed);
+                        }
+                    }
+                }
+                "status_301s" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.status_301s, parsed);
+                        }
+                    }
+                }
+                "status_302s" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.status_302s, parsed);
+                        }
+                    }
+                }
+                "status_401s" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.status_401s, parsed);
+                        }
+                    }
+                }
+                "status_403s" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.status_403s, parsed);
+                        }
+                    }
+                }
+                "status_429s" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.status_429s, parsed);
+                        }
+                    }
+                }
+                "status_500s" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.status_500s, parsed);
+                        }
+                    }
+                }
+                "status_503s" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.status_503s, parsed);
+                        }
+                    }
+                }
+                "status_504s" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.status_504s, parsed);
+                        }
+                    }
+                }
+                "status_508s" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.status_508s, parsed);
+                        }
+                    }
+                }
+                "wildcards_filtered" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.wildcards_filtered, parsed);
+                        }
+                    }
+                }
+                "responses_filtered" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.responses_filtered, parsed);
+                        }
+                    }
+                }
+                "resources_discovered" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.resources_discovered, parsed);
+                        }
+                    }
+                }
+                "url_format_errors" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.url_format_errors, parsed);
+                        }
+                    }
+                }
+                "redirection_errors" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.redirection_errors, parsed);
+                        }
+                    }
+                }
+                "connection_errors" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.connection_errors, parsed);
+                        }
+                    }
+                }
+                "request_errors" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.request_errors, parsed);
+                        }
+                    }
+                }
+                "directory_scan_times" => {
+                    if let Some(arr) = value.as_array() {
+                        for val in arr {
+                            if let Some(parsed) = val.as_f64() {
+                                if let Ok(mut guard) = stats.directory_scan_times.lock() {
+                                    guard.push(parsed)
+                                }
+                            }
+                        }
+                    }
+                }
+                "total_runtime" => {
+                    if let Some(arr) = value.as_array() {
+                        for val in arr {
+                            if let Some(parsed) = val.as_f64() {
+                                if let Ok(mut guard) = stats.total_runtime.lock() {
+                                    guard.push(parsed)
+                                }
+                            }
+                        }
+                    }
+                }
+                _ => {}
+            }
+        }
+
+        Ok(stats)
+    }
+}
+
 /// implementation of statistics data collection struct
 impl Stats {
     /// Small wrapper for default to set `kind` to "statistics" and `total_runtime` to have at least
@@ -176,6 +471,16 @@ impl Stats {
         atomic_load!(self.errors)
     }
 
+    /// public getter for status_403s
+    pub fn status_403s(&self) -> usize {
+        atomic_load!(self.status_403s)
+    }
+
+    /// public getter for status_429s
+    pub fn status_429s(&self) -> usize {
+        atomic_load!(self.status_429s)
+    }
+
     /// public getter for total_expected
     pub fn total_expected(&self) -> usize {
         atomic_load!(self.total_expected)
@@ -222,10 +527,6 @@ impl Stats {
             StatError::Timeout => {
                 atomic_increment!(self.timeouts);
             }
-            StatError::Status403 => {
-                atomic_increment!(self.status_403s);
-                atomic_increment!(self.client_errors);
-            }
             StatError::UrlFormat => {
                 atomic_increment!(self.url_format_errors);
             }
@@ -238,9 +539,7 @@ impl Stats {
             StatError::Request => {
                 atomic_increment!(self.request_errors);
             }
-            StatError::Other => {
-                atomic_increment!(self.errors);
-            }
+            _ => {} // no need to hit Other as we always increment self.errors anyway
         }
     }
 
@@ -248,7 +547,7 @@ impl Stats {
     ///
     /// Implies incrementing:
     ///     - requests
-    ///     - status_403s (when code is 403)
+    ///     - appropriate status_* codes
     ///     - errors (when code is [45]xx)
     pub fn add_status_code(&self, status: StatusCode) {
         self.add_request();
@@ -264,9 +563,6 @@ impl Stats {
         }
 
         match status {
-            StatusCode::FORBIDDEN => {
-                atomic_increment!(self.status_403s);
-            }
             StatusCode::OK => {
                 atomic_increment!(self.status_200s);
             }
@@ -279,6 +575,9 @@ impl Stats {
             StatusCode::UNAUTHORIZED => {
                 atomic_increment!(self.status_401s);
             }
+            StatusCode::FORBIDDEN => {
+                atomic_increment!(self.status_403s);
+            }
             StatusCode::TOO_MANY_REQUESTS => {
                 atomic_increment!(self.status_429s);
             }
@@ -307,6 +606,13 @@ impl Stats {
         }
     }
 
+    /// subtract a value from the given field
+    pub fn subtract_from_usize_field(&self, field: StatField, value: usize) {
+        if let StatField::TotalExpected = field {
+            self.total_expected.fetch_sub(value, Ordering::Relaxed);
+        }
+    }
+
     /// Update a `Stats` field of type usize
     pub fn update_usize_field(&self, field: StatField, value: usize) {
         match field {
@@ -314,12 +620,10 @@ impl Stats {
                 atomic_increment!(self.expected_per_scan, value);
             }
             StatField::TotalScans => {
-                let multiplier = self.num_extensions.max(1);
-
                 atomic_increment!(self.total_scans, value);
                 atomic_increment!(
                     self.total_expected,
-                    value * self.expected_per_scan.load(Ordering::Relaxed) * multiplier
+                    value * self.expected_per_scan.load(Ordering::Relaxed)
                 );
             }
             StatField::TotalExpected => {
@@ -435,30 +739,6 @@ mod tests {
         Ok(())
     }
 
-    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
-    /// when sent StatCommand::AddRequest, stats object should reflect the change
-    ///
-    /// incrementing a 403 (tracked in status_403s) should also increment:
-    ///     - errors
-    ///     - requests
-    ///     - client_errors
-    async fn statistics_handler_increments_403() {
-        let (task, handle) = setup_stats_test();
-
-        let err = Command::AddError(StatError::Status403);
-        let err2 = Command::AddError(StatError::Status403);
-
-        handle.tx.send(err).unwrap_or_default();
-        handle.tx.send(err2).unwrap_or_default();
-
-        teardown_stats_test(handle.tx.clone(), task).await;
-
-        assert_eq!(handle.data.errors.load(Ordering::Relaxed), 2);
-        assert_eq!(handle.data.requests.load(Ordering::Relaxed), 2);
-        assert_eq!(handle.data.status_403s.load(Ordering::Relaxed), 2);
-        assert_eq!(handle.data.client_errors.load(Ordering::Relaxed), 2);
-    }
-
     #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
     /// when sent StatCommand::AddRequest, stats object should reflect the change
     ///
@@ -567,7 +847,7 @@ mod tests {
 
         stats.merge_from(tfile.path().to_str().unwrap()).unwrap();
 
-        // as of 1.11.1; all Stats fields are accounted for whether they're updated in merge_from
+        // as of 2.1.0; all Stats fields are accounted for whether they're updated in merge_from
         // or not
         assert_eq!(atomic_load!(stats.timeouts), 1);
         assert_eq!(atomic_load!(stats.requests), 9207);
@@ -617,4 +897,22 @@ mod tests {
         stats.update_runtime(20.2);
         assert!((stats.total_runtime.lock().unwrap()[0] - 20.2).abs() < f64::EPSILON);
     }
+
+    #[test]
+    /// ensure status_403s returns the correct value
+    fn status_403s_returns_correct_value() {
+        let config = Configuration::new().unwrap();
+        let stats = Stats::new(config.extensions.len(), config.json);
+        stats.status_403s.store(12, Ordering::Relaxed);
+        assert_eq!(stats.status_403s(), 12);
+    }
+
+    #[test]
+    /// ensure status_403s returns the correct value
+    fn status_429s_returns_correct_value() {
+        let config = Configuration::new().unwrap();
+        let stats = Stats::new(config.extensions.len(), config.json);
+        stats.status_429s.store(141, Ordering::Relaxed);
+        assert_eq!(stats.status_429s(), 141);
+    }
 }

src/statistics/error.rs

@@ -1,9 +1,6 @@
 #[derive(Debug, Copy, Clone)]
 /// Enum variants used to inform the `StatCommand` protocol what `Stats` fields should be updated
 pub enum StatError {
-    /// Represents a 403 response code
-    Status403,
-
     /// Represents a timeout error
     Timeout,
 

src/statistics/macros.rs

@@ -20,4 +20,18 @@ macro_rules! atomic_load {
     ($metric:expr) => {
         $metric.load(Ordering::Relaxed);
     };
+    ($metric:expr, $ordering:expr) => {
+        $metric.load($ordering);
+    };
+}
+
+/// Wrapper around `Atomic*.store` to save me from writing Ordering::Relaxed a bajillion times
+#[macro_export]
+macro_rules! atomic_store {
+    ($metric:expr, $value:expr) => {
+        $metric.store($value, Ordering::Relaxed);
+    };
+    ($metric:expr, $value:expr, $ordering:expr) => {
+        $metric.store($value, $ordering);
+    };
 }

src/statistics/tests.rs

@@ -55,10 +55,7 @@ fn save_writes_stats_object_to_disk() {
     stats.add_status_code(StatusCode::OK);
     stats.add_status_code(StatusCode::OK);
     let outfile = NamedTempFile::new().unwrap();
-    if stats
-        .save(174.33, &outfile.path().to_str().unwrap())
-        .is_ok()
-    {}
+    if stats.save(174.33, outfile.path().to_str().unwrap()).is_ok() {}
 
     assert!(stats.as_json().unwrap().contains("statistics"));
     assert!(stats.as_json().unwrap().contains("11")); // requests made

src/url.rs

@@ -42,7 +42,13 @@ impl FeroxUrl {
 
         let mut urls = vec![];
 
-        match self.format(word, None) {
+        let slash = if self.handles.config.add_slash {
+            Some("/")
+        } else {
+            None
+        };
+
+        match self.format(word, slash) {
             // default request, i.e. no extension
             Ok(url) => urls.push(url),
             Err(_) => self.handles.stats.send(AddError(UrlFormat))?,
@@ -55,7 +61,6 @@ impl FeroxUrl {
                 Err(_) => self.handles.stats.send(AddError(UrlFormat))?,
             }
         }
-
         log::trace!("exit: formatted_urls -> {:?}", urls);
         Ok(urls)
     }
@@ -66,7 +71,7 @@ impl FeroxUrl {
     pub fn format(&self, word: &str, extension: Option<&str>) -> Result<Url> {
         log::trace!("enter: format({}, {:?})", word, extension);
 
-        if Url::parse(&word).is_ok() {
+        if Url::parse(word).is_ok() {
             // when a full url is passed in as a word to be joined to a base url using
             // reqwest::Url::join, the result is that the word (url) completely overwrites the base
             // url, potentially resulting in requests to places that aren't actually the target
@@ -97,13 +102,25 @@ impl FeroxUrl {
             self.target.to_string()
         };
 
-        // extensions and slashes are mutually exclusive cases
-        let word = if extension.is_some() {
-            format!("{}.{}", word, extension.unwrap())
-        } else if self.handles.config.add_slash && !word.ends_with('/') {
-            // -f used, and word doesn't already end with a /
-            format!("{}/", word)
-        } else if word.starts_with("//") {
+        // As of version 2.3.4, extensions and trailing slashes are no longer mutually exclusive.
+        // Trailing slashes are now treated as just another extension, which is pretty clever.
+        //
+        // In addition to the change above, @cortantief ID'd a bug here that incorrectly handled
+        // 2 leading forward slashes when extensions were used. This block addresses the bugfix.
+        let mut word = if let Some(ext) = extension {
+            // We handle the special case of forward slash
+            // That allow us to treat it as an extension with a particular format
+            if ext == "/" {
+                format!("{}/", word)
+            } else {
+                format!("{}.{}", word, ext)
+            }
+        } else {
+            String::from(word)
+        };
+
+        // We check separately if the current word begins with 2 forward slashes
+        if word.starts_with("//") {
             // bug ID'd by @Sicks3c, when a wordlist contains words that begin with 2 forward slashes
             // i.e. //1_40_0/static/js, it gets joined onto the base url in a surprising way
             // ex: https://localhost/ + //1_40_0/static/js -> https://1_40_0/static/js
@@ -111,9 +128,7 @@ impl FeroxUrl {
             // and simply removes prefixed forward slashes if there are two of them. Additionally,
             // trim_start_matches will trim the pattern until it's gone, so even if there are more than
             // 2 /'s, they'll still be trimmed
-            word.trim_start_matches('/').to_string()
-        } else {
-            String::from(word)
+            word = word.trim_start_matches('/').to_string();
         };
 
         let base_url = Url::parse(&url)?;
@@ -451,6 +466,20 @@ mod tests {
         );
     }
 
+    #[test]
+    /// word with two prepended slashes and extensions doesn't discard the entire domain
+    fn format_url_word_with_two_prepended_slashes_and_extensions() {
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+        let url = FeroxUrl::from_string("http://localhost", handles);
+        for ext in ["rocks", "fun"] {
+            let to_check = format!("http://localhost/upload/ferox.{}", ext);
+            assert_eq!(
+                url.format("//upload/ferox", Some(ext)).unwrap(),
+                reqwest::Url::parse(&to_check[..]).unwrap()
+            );
+        }
+    }
+
     #[test]
     /// word that is a fully formed url, should return an error
     fn format_url_word_that_is_a_url() {
@@ -460,4 +489,33 @@ mod tests {
 
         assert!(formatted.is_err());
     }
+
+    #[test]
+    /// sending url + word with both an extension and add-slash should get back
+    /// two urls, one with '/' appended to the word, and the other with the extension
+    /// appended
+    fn formatted_urls_with_postslash_and_extensions() {
+        let config = Configuration {
+            add_slash: true,
+            extensions: vec!["rocks".to_string(), "fun".to_string()],
+            ..Default::default()
+        };
+        let handles = Arc::new(Handles::for_testing(None, Some(Arc::new(config))).0);
+        let url = FeroxUrl::from_string("http://localhost", handles);
+        match url.formatted_urls("ferox") {
+            Ok(urls) => {
+                // 3 = One for the main word + slash and for the two extensions
+                assert_eq!(urls.len(), 3);
+                assert_eq!(
+                    urls,
+                    [
+                        Url::parse("http://localhost/ferox/").unwrap(),
+                        Url::parse("http://localhost/ferox.rocks").unwrap(),
+                        Url::parse("http://localhost/ferox.fun").unwrap(),
+                    ]
+                )
+            }
+            Err(err) => panic!("{}", err.to_string()),
+        }
+    }
 }

src/utils.rs

@@ -1,24 +1,36 @@
 use anyhow::{bail, Context, Result};
 use console::{strip_ansi_codes, style, user_attended};
 use indicatif::ProgressBar;
-use reqwest::{Client, Response, Url};
+use regex::Regex;
+use reqwest::{Client, Response, StatusCode, Url};
 #[cfg(not(target_os = "windows"))]
-use rlimit::{getrlimit, setrlimit, Resource, Rlim};
+use rlimit::{getrlimit, setrlimit, Resource};
 use std::{
     fs,
     io::{self, BufWriter, Write},
+    sync::Arc,
+    time::Duration,
+    time::{SystemTime, UNIX_EPOCH},
 };
 use tokio::sync::mpsc::UnboundedSender;
 
+use crate::config::Configuration;
 use crate::{
     config::OutputLevel,
-    event_handlers::Command::{self, AddError, AddStatus},
+    event_handlers::{
+        Command::{self, AddError, AddStatus},
+        Handles,
+    },
     progress::PROGRESS_PRINTER,
     send_command,
     statistics::StatError::{Connection, Other, Redirection, Request, Timeout},
     traits::FeroxSerialize,
+    USER_AGENTS,
 };
 
+/// simple counter for grabbing 'random' user agents
+static mut USER_AGENT_CTR: usize = 0;
+
 /// Given the path to a file, open the file in append mode (create it if it doesn't exist) and
 /// return a reference to the buffered file
 pub fn open_file(filename: &str) -> Result<BufWriter<fs::File>> {
@@ -81,11 +93,41 @@ pub fn ferox_print(msg: &str, bar: &ProgressBar) {
     }
 }
 
+/// wrapper for make_request used to pass error/response codes to FeroxScans for per-scan stats
+/// tracking of information related to auto-tune/bail
+pub async fn logged_request(url: &Url, handles: Arc<Handles>) -> Result<Response> {
+    let client = &handles.config.client;
+    let level = handles.config.output_level;
+    let tx_stats = handles.stats.tx.clone();
+
+    let response = make_request(client, url, level, &handles.config, tx_stats).await;
+
+    let scans = handles.ferox_scans()?;
+
+    match response {
+        Ok(resp) => {
+            match resp.status() {
+                StatusCode::TOO_MANY_REQUESTS | StatusCode::FORBIDDEN => {
+                    scans.increment_status_code(url.as_str(), resp.status());
+                }
+                _ => {}
+            }
+            Ok(resp)
+        }
+        Err(e) => {
+            log::warn!("err: {:?}", e);
+            scans.increment_error(url.as_str());
+            bail!(e)
+        }
+    }
+}
+
 /// Initiate request to the given `Url` using `Client`
 pub async fn make_request(
     client: &Client,
     url: &Url,
     output_level: OutputLevel,
+    config: &Configuration,
     tx_stats: UnboundedSender<Command>,
 ) -> Result<Response> {
     log::trace!(
@@ -95,7 +137,20 @@ pub async fn make_request(
         tx_stats
     );
 
-    match client.get(url.to_owned()).send().await {
+    let mut request = client.get(url.to_owned());
+
+    if config.random_agent {
+        let index = unsafe {
+            USER_AGENT_CTR += 1;
+            USER_AGENT_CTR % USER_AGENTS.len()
+        };
+
+        let user_agent = USER_AGENTS[index];
+
+        request = request.header("User-Agent", user_agent);
+    }
+
+    match request.send().await {
         Err(e) => {
             log::trace!("exit: make_request -> {}", e);
 
@@ -184,16 +239,15 @@ pub fn create_report_string(
 /// as the adjustment made here is only valid for the scan itself (and any child processes, of which
 /// there are none).
 #[cfg(not(target_os = "windows"))]
-pub fn set_open_file_limit(limit: usize) -> bool {
+pub fn set_open_file_limit(limit: u64) -> bool {
     log::trace!("enter: set_open_file_limit");
 
     if let Ok((soft, hard)) = getrlimit(Resource::NOFILE) {
-        if hard.as_usize() > limit {
+        if hard > limit {
             // our default open file limit is less than the current hard limit, this means we can
             // set the soft limit to our default
-            let new_soft_limit = Rlim::from_usize(limit);
 
-            if setrlimit(Resource::NOFILE, new_soft_limit, hard).is_ok() {
+            if setrlimit(Resource::NOFILE, limit, hard).is_ok() {
                 log::debug!("set open file descriptor limit to {}", limit);
 
                 log::trace!("exit: set_open_file_limit -> {}", true);
@@ -254,15 +308,183 @@ where
     Ok(())
 }
 
+/// determine if a url should be denied based on the given absolute url
+fn should_deny_absolute(url_to_test: &Url, denier: &Url, handles: Arc<Handles>) -> Result<bool> {
+    log::trace!(
+        "enter: should_deny_absolute({}, {:?})",
+        url_to_test.as_str(),
+        denier.as_str(),
+    );
+
+    // simplest case is an exact match, check for it first
+    if url_to_test == denier {
+        log::trace!("exit: should_deny_absolute -> true");
+        return Ok(true);
+    }
+
+    match (url_to_test.host(), denier.host()) {
+        // .host() will return an enum with ipv4|6 or domain and is comparable
+        // whereas .domain() returns None for ip addresses
+        (Some(normed_host), Some(denier_host)) => {
+            if normed_host != denier_host {
+                // domains don't even match
+                return Ok(false);
+            }
+        }
+        _ => {
+            // one or the other couldn't determine the host value, which probably means
+            // it's not suitable for further comparison
+            return Ok(false);
+        }
+    }
+
+    let tested_host = url_to_test.host().unwrap(); // match above will catch errors
+
+    // at this point, we have a matching set of ips or domain names. now we can process the
+    // url path. The goal is to determine whether the given url's path is a subpath of any
+    // url in the deny list, for example
+    //    GIVEN URL                        URL DENY LIST               USER-SPECIFIED URLS TO SCAN
+    //    http://some.domain/stuff/things, [http://some.domain/stuff], [http://some.domain] => true
+    //    http://some.domain/stuff/things, [http://some.domain/stuff/things], [http://some.domain] => true
+    //    http://some.domain/stuff/things, [http://some.domain/api], [http://some.domain] => false
+    // the examples above are all pretty obvious, the kicker comes when the blocking url's
+    // path is a parent to a scanned url
+    //    http://some.domain/stuff/things, [http://some.domain/], [http://some.domain/stuff] => false
+    //    http://some.domain/api, [http://some.domain/], [http://some.domain/stuff] => true
+    // we want to deny all children of the parent, unless that child is a child of a scan
+    // we specified through -u(s) or --stdin
+
+    let deny_path = denier.path();
+    let tested_path = url_to_test.path();
+
+    if tested_path.starts_with(deny_path) {
+        // at this point, we know that the given normalized path is a sub-path of the
+        // current deny-url, now we just need to check to see if this deny-url is a parent
+        // to a scanned url that is also a parent of the given url
+        for ferox_scan in handles.ferox_scans()?.get_active_scans() {
+            let scanner = Url::parse(ferox_scan.url().trim_end_matches('/'))
+                .with_context(|| format!("Could not parse {} as a url", ferox_scan))?;
+
+            if let Some(scan_host) = scanner.host() {
+                // same domain/ip check we perform on the denier above
+                if tested_host != scan_host {
+                    // domains don't even match, keep on keepin' on...
+                    continue;
+                }
+            } else {
+                // couldn't process .host from scanner
+                continue;
+            };
+
+            let scan_path = scanner.path();
+
+            if scan_path.starts_with(deny_path) && tested_path.starts_with(scan_path) {
+                // user-specified scan url is a sub-path of the deny-urls's path AND the
+                // url to check is a sub-path of the user-specified scan url
+                //
+                // the assumption is the user knew what they wanted and we're going to give
+                // the scanned url precedence, even though it's a sub-path
+                log::trace!("exit: should_deny_absolute -> false");
+                return Ok(false);
+            }
+        }
+        log::trace!("exit: should_deny_absolute -> true");
+        return Ok(true);
+    }
+
+    log::trace!("exit: should_deny_absolute -> false");
+    Ok(false)
+}
+
+/// determine if a url should be denied based on the given regular expression
+///
+/// the regex ONLY matches against the PATH of the url (not the scheme, host, port, etc)
+fn should_deny_regex(url_to_test: &Url, denier: &Regex) -> bool {
+    log::trace!(
+        "enter: should_deny_regex({}, {})",
+        url_to_test.as_str(),
+        denier,
+    );
+
+    let result = denier.is_match(url_to_test.as_str());
+
+    log::trace!("exit: should_deny_regex -> {}", result);
+    result
+}
+
+/// determines whether or not a given url should be denied based on the user-supplied --dont-scan
+/// flag
+pub fn should_deny_url(url: &Url, handles: Arc<Handles>) -> Result<bool> {
+    log::trace!(
+        "enter: should_deny_url({}, {:?}, {:?})",
+        url.as_str(),
+        handles.config.url_denylist,
+        handles.ferox_scans()?
+    );
+
+    // normalization for comparison is to remove the trailing / if one exists, this is done for
+    // the given url and any url to which it's compared
+    let normed_url = Url::parse(url.to_string().trim_end_matches('/'))?;
+
+    for denier in &handles.config.url_denylist {
+        // note to self: it may seem as though we can use regex only for --dont-scan, however, in
+        // doing so, we lose the ability to block a parent directory while scanning a child
+        if let Ok(should_deny) = should_deny_absolute(&normed_url, denier, handles.clone()) {
+            if should_deny {
+                return Ok(true);
+            }
+        }
+    }
+
+    for denier in &handles.config.regex_denylist {
+        if should_deny_regex(&normed_url, denier) {
+            return Ok(true);
+        }
+    }
+
+    // made it to the end of the deny lists unscathed, return false, indicating we should not deny
+    // this particular url
+    log::trace!("exit: should_deny_url -> false");
+    Ok(false)
+}
+
+/// given a url and filename-suffix, return a unique filename comprised of the slugified url,
+/// current unix timestamp and suffix
+///
+/// ex: ferox-http_telsa_com-1606947491.state
+pub fn slugify_filename(url: &str, prefix: &str, suffix: &str) -> String {
+    log::trace!("enter: slugify({:?}, {:?}, {:?})", url, prefix, suffix);
+
+    let ts = SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .unwrap_or_else(|_| Duration::from_secs(0))
+        .as_secs();
+
+    let altered_prefix = if !prefix.is_empty() {
+        format!("{}-", prefix)
+    } else {
+        String::new()
+    };
+
+    let slug = url.replace("://", "_").replace("/", "_").replace(".", "_");
+
+    let filename = format!("{}{}-{}.{}", altered_prefix, slug, ts, suffix);
+
+    log::trace!("exit: slugify -> {}", filename);
+    filename
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::config::Configuration;
+    use crate::scan_manager::{FeroxScans, ScanOrder};
 
     #[test]
     /// set_open_file_limit with a low requested limit succeeds
     fn utils_set_open_file_limit_with_low_requested_limit() {
         let (_, hard) = getrlimit(Resource::NOFILE).unwrap();
-        let lower_limit = hard.as_usize() - 1;
+        let lower_limit = hard - 1;
         assert!(set_open_file_limit(lower_limit));
     }
 
@@ -270,9 +492,9 @@ mod tests {
     /// set_open_file_limit with a high requested limit succeeds
     fn utils_set_open_file_limit_with_high_requested_limit() {
         let (_, hard) = getrlimit(Resource::NOFILE).unwrap();
-        let higher_limit = hard.as_usize() + 1;
+        let higher_limit = hard + 1;
         // calculate a new soft to ensure soft != hard and hit that logic branch
-        let new_soft = Rlim::from_usize(hard.as_usize() - 1);
+        let new_soft = hard - 1;
         setrlimit(Resource::NOFILE, new_soft, hard).unwrap();
         assert!(set_open_file_limit(higher_limit));
     }
@@ -283,7 +505,7 @@ mod tests {
         let (_, hard) = getrlimit(Resource::NOFILE).unwrap();
         // calculate a new soft to ensure soft == hard and hit the failure logic branch
         setrlimit(Resource::NOFILE, hard, hard).unwrap();
-        assert!(!set_open_file_limit(hard.as_usize())); // returns false
+        assert!(!set_open_file_limit(hard)); // returns false
     }
 
     #[test]
@@ -333,4 +555,222 @@ mod tests {
     fn status_colorizer_returns_as_is() {
         assert_eq!(status_colorizer("farfignewton"), "farfignewton".to_string());
     }
+
+    #[test]
+    /// provide a url that should be blocked where the denier is an exact match for the tested url
+    /// expect true
+    fn should_deny_url_blocks_when_denier_is_exact_match() {
+        let scan_url = "https://testdomain.com/";
+        let deny_url = "https://testdomain.com/denied";
+        let tested_url = Url::parse("https://testdomain.com/denied/").unwrap();
+
+        let scans = Arc::new(FeroxScans::default());
+        scans.add_directory_scan(scan_url, ScanOrder::Initial);
+
+        let mut config = Configuration::new().unwrap();
+        config.url_denylist = vec![Url::parse(deny_url).unwrap()];
+        let config = Arc::new(config);
+
+        let handles = Arc::new(Handles::for_testing(Some(scans), Some(config)).0);
+
+        assert!(should_deny_url(&tested_url, handles).unwrap());
+    }
+
+    #[test]
+    /// provide a url that has a different host than the denier but the same path, expect false
+    fn should_deny_url_doesnt_compare_mismatched_domains() {
+        let scan_url = "https://testdomain.com/";
+        let deny_url = "https://dev.testdomain.com/denied";
+        let tested_url = Url::parse("https://testdomain.com/denied/").unwrap();
+
+        let scans = Arc::new(FeroxScans::default());
+        scans.add_directory_scan(scan_url, ScanOrder::Initial);
+
+        let mut config = Configuration::new().unwrap();
+        config.url_denylist = vec![Url::parse(deny_url).unwrap()];
+        let config = Arc::new(config);
+
+        let handles = Arc::new(Handles::for_testing(Some(scans), Some(config)).0);
+
+        assert!(!should_deny_url(&tested_url, handles).unwrap());
+    }
+
+    #[test]
+    /// provide a denier from which we can't check a host, which results in no comparison, expect false
+    fn should_deny_url_doesnt_compare_non_domains() {
+        let scan_url = "https://testdomain.com/";
+        let deny_url = "unix:/run/foo.socket";
+        let tested_url = Url::parse("https://testdomain.com/denied/").unwrap();
+
+        let scans = Arc::new(FeroxScans::default());
+        scans.add_directory_scan(scan_url, ScanOrder::Initial);
+
+        let mut config = Configuration::new().unwrap();
+        config.url_denylist = vec![Url::parse(deny_url).unwrap()];
+        let config = Arc::new(config);
+
+        let handles = Arc::new(Handles::for_testing(Some(scans), Some(config)).0);
+
+        assert!(!should_deny_url(&tested_url, handles).unwrap());
+    }
+
+    #[test]
+    /// provide a url that has a different host than the denier but the same path, expect false
+    /// because the denier is a parent to the tested, even tho the scanned doesn't compare, it
+    /// still returns true
+    fn should_deny_url_doesnt_compare_mismatched_domains_in_scanned() {
+        let deny_url = "https://testdomain.com/";
+        let scan_url = "https://dev.testdomain.com/denied";
+        let tested_url = Url::parse("https://testdomain.com/denied/").unwrap();
+
+        let scans = Arc::new(FeroxScans::default());
+        scans.add_directory_scan(scan_url, ScanOrder::Initial);
+
+        let mut config = Configuration::new().unwrap();
+        config.url_denylist = vec![Url::parse(deny_url).unwrap()];
+        let config = Arc::new(config);
+
+        let handles = Arc::new(Handles::for_testing(Some(scans), Some(config)).0);
+
+        assert!(should_deny_url(&tested_url, handles).unwrap());
+    }
+
+    #[test]
+    /// provide a denier from which we can't check a host, which results in no comparison, expect false
+    /// because the denier is a parent to the tested, even tho the scanned doesn't compare, it
+    /// still returns true
+    fn should_deny_url_doesnt_compare_non_domains_in_scanned() {
+        let deny_url = "https://testdomain.com/";
+        let scan_url = "unix:/run/foo.socket";
+        let tested_url = Url::parse("https://testdomain.com/denied/").unwrap();
+
+        let scans = Arc::new(FeroxScans::default());
+        scans.add_directory_scan(scan_url, ScanOrder::Initial);
+
+        let mut config = Configuration::new().unwrap();
+        config.url_denylist = vec![Url::parse(deny_url).unwrap()];
+        let config = Arc::new(config);
+
+        let handles = Arc::new(Handles::for_testing(Some(scans), Some(config)).0);
+
+        assert!(should_deny_url(&tested_url, handles).unwrap());
+    }
+
+    #[test]
+    /// provide a denier where the tested url is a sub-path and the scanned url is not, expect true
+    fn should_deny_url_blocks_child() {
+        let scan_url = "https://testdomain.com/";
+        let deny_url = "https://testdomain.com/api";
+        let tested_url = Url::parse("https://testdomain.com/api/denied/").unwrap();
+
+        let scans = Arc::new(FeroxScans::default());
+        scans.add_directory_scan(scan_url, ScanOrder::Initial);
+
+        let mut config = Configuration::new().unwrap();
+        config.url_denylist = vec![Url::parse(deny_url).unwrap()];
+        let config = Arc::new(config);
+
+        let handles = Arc::new(Handles::for_testing(Some(scans), Some(config)).0);
+
+        assert!(should_deny_url(&tested_url, handles).unwrap());
+    }
+
+    #[test]
+    /// provide a denier where the tested url is not a sub-path and the scanned url is not, expect false
+    fn should_deny_url_doesnt_block_non_child() {
+        let scan_url = "https://testdomain.com/";
+        let deny_url = "https://testdomain.com/api";
+        let tested_url = Url::parse("https://testdomain.com/not-denied/").unwrap();
+
+        let scans = Arc::new(FeroxScans::default());
+        scans.add_directory_scan(scan_url, ScanOrder::Initial);
+
+        let mut config = Configuration::new().unwrap();
+        config.url_denylist = vec![Url::parse(deny_url).unwrap()];
+        let config = Arc::new(config);
+
+        let handles = Arc::new(Handles::for_testing(Some(scans), Some(config)).0);
+
+        assert!(!should_deny_url(&tested_url, handles).unwrap());
+    }
+
+    #[test]
+    /// provide a denier where the tested url is a sub-path and the scanned url is not, expect true
+    fn should_deny_url_blocks_child_when_scan_url_isnt_parent() {
+        let scan_url = "https://testdomain.com/api";
+        let deny_url = "https://testdomain.com/";
+        let tested_url = Url::parse("https://testdomain.com/stuff/").unwrap();
+
+        let scans = Arc::new(FeroxScans::default());
+        scans.add_directory_scan(scan_url, ScanOrder::Initial);
+
+        let mut config = Configuration::new().unwrap();
+        config.url_denylist = vec![Url::parse(deny_url).unwrap()];
+        let config = Arc::new(config);
+
+        let handles = Arc::new(Handles::for_testing(Some(scans), Some(config)).0);
+
+        assert!(should_deny_url(&tested_url, handles).unwrap());
+    }
+
+    #[test]
+    /// provide a denier where the tested url is not a sub-path and the scanned url is not, expect false
+    fn should_deny_url_doesnt_block_child_when_scan_url_is_parent() {
+        let scan_url = "https://testdomain.com/api";
+        let deny_url = "https://testdomain.com/";
+        let tested_url = Url::parse("https://testdomain.com/api/not-denied/").unwrap();
+
+        let scans = Arc::new(FeroxScans::default());
+        scans.add_directory_scan(scan_url, ScanOrder::Initial);
+
+        let mut config = Configuration::new().unwrap();
+        config.url_denylist = vec![Url::parse(deny_url).unwrap()];
+        let config = Arc::new(config);
+
+        let handles = Arc::new(Handles::for_testing(Some(scans), Some(config)).0);
+
+        assert!(!should_deny_url(&tested_url, handles).unwrap());
+    }
+
+    #[test]
+    /// provide a denier where the tested url is matched against a regular expression in the path
+    /// of the url
+    fn should_deny_url_blocks_urls_based_on_regex_in_path() {
+        let scan_url = "https://testdomain.com/";
+        let deny_pattern = "/deni.*";
+        let tested_url = Url::parse("https://testdomain.com/denied/").unwrap();
+
+        let scans = Arc::new(FeroxScans::default());
+        scans.add_directory_scan(scan_url, ScanOrder::Initial);
+
+        let mut config = Configuration::new().unwrap();
+        config.regex_denylist = vec![Regex::new(deny_pattern).unwrap()];
+        let config = Arc::new(config);
+
+        let handles = Arc::new(Handles::for_testing(Some(scans), Some(config)).0);
+
+        assert!(should_deny_url(&tested_url, handles).unwrap());
+    }
+
+    #[test]
+    /// provide a denier where the tested url is matched against a regular expression in the scheme
+    /// of the url
+    fn should_deny_url_blocks_urls_based_on_regex_in_scheme() {
+        let scan_url = "https://testdomain.com/";
+        let deny_pattern = "http:";
+        let tested_http_url = Url::parse("http://testdomain.com/denied/").unwrap();
+        let tested_https_url = Url::parse("https://testdomain.com/denied/").unwrap();
+
+        let scans = Arc::new(FeroxScans::default());
+        scans.add_directory_scan(scan_url, ScanOrder::Initial);
+
+        let mut config = Configuration::new().unwrap();
+        config.regex_denylist = vec![Regex::new(deny_pattern).unwrap()];
+        let config = Arc::new(config);
+
+        let handles = Arc::new(Handles::for_testing(Some(scans), Some(config)).0);
+
+        assert!(!should_deny_url(&tested_https_url, handles.clone()).unwrap());
+        assert!(should_deny_url(&tested_http_url, handles).unwrap());
+    }
 }

tests/test_banner.rs

@@ -113,6 +113,65 @@ fn banner_prints_headers() {
         );
 }
 
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + multiple dont scan url & regex entries
+fn banner_prints_denied_urls() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--dont-scan")
+        .arg("http://dont-scan.me")
+        .arg("https://also-not.me")
+        .arg("https:")
+        .arg("/deny.*")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Don't Scan Url"))
+                .and(predicate::str::contains("Don't Scan Regex"))
+                .and(predicate::str::contains("http://dont-scan.me"))
+                .and(predicate::str::contains("https://also-not.me"))
+                .and(predicate::str::contains("https:"))
+                .and(predicate::str::contains("/deny.*"))
+                .and(predicate::str::contains("─┴─")),
+        );
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + multiple headers
+fn banner_prints_random_agent() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--random-agent")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Random"))
+                .and(predicate::str::contains("─┴─")),
+        );
+}
+
 #[test]
 /// test allows non-existent wordlist to trigger the banner printing to stderr
 /// expect to see all mandatory prints + multiple size filters
@@ -849,6 +908,58 @@ fn banner_prints_rate_limit() {
         );
 }
 
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + auto tune
+fn banner_prints_auto_tune() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--auto-tune")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Auto Tune"))
+                .and(predicate::str::contains("│ true"))
+                .and(predicate::str::contains("─┴─")),
+        );
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + auto bail
+fn banner_prints_auto_bail() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--auto-bail")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Auto Bail"))
+                .and(predicate::str::contains("│ true"))
+                .and(predicate::str::contains("─┴─")),
+        );
+}
+
 #[test]
 /// test allows non-existent wordlist to trigger the banner printing to stderr
 /// expect to see no banner output
@@ -896,3 +1007,27 @@ fn banner_doesnt_print_when_quiet() {
                 .and(predicate::str::contains("User-Agent").not()),
         );
 }
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see nothing as --parallel forces --silent to be true
+fn banner_prints_parallel() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--stdin")
+        .arg("--parallel")
+        .arg("4316")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .not()
+                .and(predicate::str::contains("Target Url").not())
+                .and(predicate::str::contains("Parallel Scans").not())
+                .and(predicate::str::contains("Threads").not())
+                .and(predicate::str::contains("Wordlist").not())
+                .and(predicate::str::contains("Status Codes").not())
+                .and(predicate::str::contains("Timeout (secs)").not())
+                .and(predicate::str::contains("User-Agent").not()),
+        );
+}

tests/test_deny_list.rs

@@ -0,0 +1,282 @@
+mod utils;
+use assert_cmd::prelude::*;
+use assert_cmd::Command;
+use httpmock::Method::GET;
+use httpmock::MockServer;
+use predicates::prelude::*;
+use utils::{setup_tmp_directory, teardown_tmp_directory};
+
+#[test]
+/// test that the deny list prevents a request if the requested url is a match
+fn deny_list_works_during_with_a_normal_scan() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--dont-scan")
+        .arg(srv.url("/LICENSE"))
+        .unwrap();
+
+    teardown_tmp_directory(tmp_dir);
+
+    cmd.assert()
+        .success()
+        .stdout(predicate::str::contains(srv.url("/LICENSE")).not());
+
+    assert_eq!(mock.hits(), 0);
+}
+
+#[test]
+/// test that the deny list prevents requests of urls found during extraction
+fn deny_list_works_during_extraction() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200)
+            .body(&srv.url("'/homepage/assets/img/icons/handshake.svg'"));
+    });
+
+    let mock_two = srv.mock(|when, then| {
+        when.method(GET)
+            .path("/homepage/assets/img/icons/handshake.svg");
+        then.status(200);
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--extract-links")
+        .arg("--dont-scan")
+        .arg(srv.url("/homepage/"))
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .and(predicate::str::contains("200"))
+            .and(predicate::str::contains("/homepage/assets/img/icons/handshake.svg").not()),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_two.hits(), 0);
+    teardown_tmp_directory(tmp_dir);
+}
+
+#[test]
+/// test that the deny list prevents requests of urls found during recursion
+fn deny_list_works_during_recursion() {
+    let srv = MockServer::start();
+    let urls = [
+        "js".to_string(),
+        "prod".to_string(),
+        "dev".to_string(),
+        "file.js".to_string(),
+    ];
+    let (tmp_dir, file) = setup_tmp_directory(&urls, "wordlist").unwrap();
+
+    let js_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js");
+        then.status(301).header("Location", &srv.url("/js/"));
+    });
+
+    let js_prod_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js/prod");
+        then.status(301).header("Location", &srv.url("/js/prod/"));
+    });
+
+    let js_dev_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js/dev");
+        then.status(301).header("Location", &srv.url("/js/dev/"));
+    });
+
+    let js_dev_file_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js/dev/file.js");
+        then.status(200)
+            .body("this is a test and is more bytes than other ones");
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-t")
+        .arg("1")
+        .arg("--dont-scan")
+        .arg(srv.url("/js/dev"))
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::is_match("301.*js")
+            .unwrap()
+            .and(predicate::str::is_match("301.*js/prod").unwrap())
+            .and(predicate::str::is_match("301.*js/dev").unwrap())
+            .not()
+            .and(predicate::str::is_match("200.*js/dev/file.js").unwrap())
+            .not(),
+    );
+
+    assert_eq!(js_mock.hits(), 1);
+    assert_eq!(js_prod_mock.hits(), 1);
+    assert_eq!(js_dev_mock.hits(), 0);
+    assert_eq!(js_dev_file_mock.hits(), 0);
+
+    teardown_tmp_directory(tmp_dir);
+}
+
+#[test]
+/// test that the deny list prevents requests of urls found during recursion when the denier is a
+/// parent of a user-specified scan
+fn deny_list_works_during_recursion_with_inverted_parents() {
+    let srv = MockServer::start();
+    let urls = [
+        "js".to_string(),
+        "prod".to_string(),
+        "dev".to_string(),
+        "api".to_string(),
+        "file.js".to_string(),
+    ];
+    let (tmp_dir, file) = setup_tmp_directory(&urls, "wordlist").unwrap();
+
+    let js_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js");
+        then.status(301).header("Location", &srv.url("/js/"));
+    });
+
+    let api_mock = srv.mock(|when, then| {
+        when.method(GET).path("/api");
+        then.status(200);
+    });
+
+    let js_prod_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js/prod");
+        then.status(301).header("Location", &srv.url("/js/prod/"));
+    });
+
+    let js_dev_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js/dev");
+        then.status(301).header("Location", &srv.url("/js/dev/"));
+    });
+
+    let js_dev_file_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js/dev/file.js");
+        then.status(200)
+            .body("this is a test and is more bytes than other ones");
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/js"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-t")
+        .arg("1")
+        .arg("-vvvv")
+        .arg("--dont-scan")
+        .arg(srv.url("/"))
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::is_match("301.*js")
+            .unwrap()
+            .and(predicate::str::is_match("301.*js/prod").unwrap())
+            .and(predicate::str::is_match("301.*js/dev").unwrap())
+            .and(predicate::str::is_match("200.*js/dev/file.js").unwrap())
+            .and(predicate::str::is_match("200.*api").unwrap())
+            .not(),
+    );
+
+    assert_eq!(js_mock.hits(), 1);
+    assert_eq!(js_prod_mock.hits(), 1);
+    assert_eq!(js_dev_mock.hits(), 1);
+    assert_eq!(js_dev_file_mock.hits(), 1);
+    assert_eq!(api_mock.hits(), 0);
+
+    teardown_tmp_directory(tmp_dir);
+}
+
+#[test]
+/// test that a regex that prevents the base url from being scanned results in an early exit
+fn deny_list_prevents_regex_that_denies_base_url() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--dont-scan")
+        .arg("/")
+        .unwrap();
+
+    teardown_tmp_directory(tmp_dir);
+
+    let err_msg = format!(
+        "Could not determine initial targets: The regex '/' matches {}/; the scan will never start",
+        srv.base_url()
+    );
+    cmd.assert()
+        .success()
+        .stderr(predicate::str::contains(err_msg));
+
+    assert_eq!(mock.hits(), 0);
+}
+
+#[test]
+/// test that a url that prevents the base url from being scanned results in an early exit
+fn deny_list_prevents_url_that_denies_base_url() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--dont-scan")
+        .arg(srv.base_url())
+        .unwrap();
+
+    teardown_tmp_directory(tmp_dir);
+
+    let err_msg = format!(
+        "Could not determine initial targets: The url '{}/' matches {}/; the scan will never start",
+        srv.base_url(),
+        srv.base_url()
+    );
+
+    cmd.assert()
+        .success()
+        .stderr(predicate::str::contains(err_msg));
+
+    assert_eq!(mock.hits(), 0);
+}

tests/test_heuristics.rs

@@ -224,11 +224,11 @@ fn test_dynamic_wildcard_request_found() {
 
     teardown_tmp_directory(tmp_dir);
 
-    assert_eq!(contents.contains("WLD"), true);
-    assert_eq!(contents.contains("Got"), true);
-    assert_eq!(contents.contains("200"), true);
-    assert_eq!(contents.contains("(url length: 32)"), true);
-    assert_eq!(contents.contains("(url length: 96)"), true);
+    assert!(contents.contains("WLD"));
+    assert!(contents.contains("Got"));
+    assert!(contents.contains("200"));
+    assert!(contents.contains("(url length: 32)"));
+    assert!(contents.contains("(url length: 96)"));
 
     cmd.assert().success().stdout(
         predicate::str::contains("WLD")
@@ -391,11 +391,11 @@ fn heuristics_wildcard_test_with_two_static_wildcards_and_output_to_file() {
 
     teardown_tmp_directory(tmp_dir);
 
-    assert_eq!(contents.contains("WLD"), true);
-    assert_eq!(contents.contains("Got"), true);
-    assert_eq!(contents.contains("200"), true);
-    assert_eq!(contents.contains("(url length: 32)"), true);
-    assert_eq!(contents.contains("(url length: 96)"), true);
+    assert!(contents.contains("WLD"));
+    assert!(contents.contains("Got"));
+    assert!(contents.contains("200"));
+    assert!(contents.contains("(url length: 32)"));
+    assert!(contents.contains("(url length: 96)"));
 
     cmd.assert().success().stdout(
         predicate::str::contains("WLD")
@@ -451,12 +451,12 @@ fn heuristics_wildcard_test_with_redirect_as_response_code(
 
     teardown_tmp_directory(tmp_dir);
 
-    assert_eq!(contents.contains("WLD"), true);
-    assert_eq!(contents.contains("301"), true);
-    assert_eq!(contents.contains("/some-redirect"), true);
-    assert_eq!(contents.contains("redirects to => "), true);
-    assert_eq!(contents.contains(&srv.url("/")), true);
-    assert_eq!(contents.contains("(url length: 32)"), true);
+    assert!(contents.contains("WLD"));
+    assert!(contents.contains("301"));
+    assert!(contents.contains("/some-redirect"));
+    assert!(contents.contains("redirects to => "));
+    assert!(contents.contains(&srv.url("/")));
+    assert!(contents.contains("(url length: 32)"));
 
     cmd.assert().success().stdout(
         predicate::str::contains("redirects to => ")

tests/test_main.rs

@@ -1,8 +1,9 @@
 mod utils;
 use assert_cmd::Command;
 use httpmock::Method::GET;
-use httpmock::MockServer;
+use httpmock::{MockServer, Regex};
 use predicates::prelude::*;
+use std::fs::{read_dir, read_to_string};
 use utils::{setup_tmp_directory, teardown_tmp_directory};
 
 #[test]
@@ -89,3 +90,136 @@ fn main_use_empty_stdin_targets() -> Result<(), Box<dyn std::error::Error>> {
 
     Ok(())
 }
+
+#[test]
+/// send three targets over stdin, expect parallel to spawn children and each child config to show
+/// up in the output file
+fn main_parallel_spawns_children() -> Result<(), Box<dyn std::error::Error>> {
+    let t1 = MockServer::start();
+    let t2 = MockServer::start();
+    let t3 = MockServer::start();
+
+    let words = [
+        String::from("LICENSE"),
+        String::from("stuff"),
+        String::from("things"),
+        String::from("mostuff"),
+        String::from("mothings"),
+    ];
+    let (word_tmp_dir, wordlist) = setup_tmp_directory(&words, "wordlist")?;
+    let (output_dir, outfile) = setup_tmp_directory(&[], "output-file")?;
+    let (tgt_tmp_dir, targets) =
+        setup_tmp_directory(&[t1.url("/"), t2.url("/"), t3.url("/")], "targets")?;
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--stdin")
+        .arg("--parallel")
+        .arg("2")
+        .arg("-vvvv")
+        .arg("--debug-log")
+        .arg(outfile.as_os_str())
+        .arg("--wordlist")
+        .arg(wordlist.as_os_str())
+        .pipe_stdin(targets)
+        .unwrap()
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("Could not connect to any target provided")
+                .and(predicate::str::contains("Target Url"))
+                .not(), // no target url found
+        );
+
+    let contents = read_to_string(outfile).unwrap();
+    println!("contents: {}", contents);
+
+    assert!(contents.contains("parallel branch && wrapped main")); // exits parallel branch
+
+    // DBG      0.007 feroxbuster parallel exec: target/debug/feroxbuster
+    //   --debug-log /tmp/.tmpAjRts6/output-file --wordlist /tmp/.tmpS4CKKq/wordlist
+    //   --silent -u http://127.0.0.1:41979/
+    let r1 = Regex::new(&format!("parallel exec:.*-u {}", t1.url("/"))).unwrap();
+    let r2 = Regex::new(&format!("parallel exec:.*-u {}", t2.url("/"))).unwrap();
+    let r3 = Regex::new(&format!("parallel exec:.*-u {}", t3.url("/"))).unwrap();
+
+    assert!(r1.is_match(&contents)); // all 3 were spawned
+    assert!(r2.is_match(&contents));
+    assert!(r3.is_match(&contents));
+
+    teardown_tmp_directory(word_tmp_dir);
+    teardown_tmp_directory(tgt_tmp_dir);
+    teardown_tmp_directory(output_dir);
+
+    Ok(())
+}
+
+#[test]
+/// send three targets over stdin with --output enabled, expect parallel to create a new directory
+/// and the log files therein
+fn main_parallel_creates_output_directory() -> Result<(), Box<dyn std::error::Error>> {
+    let t1 = MockServer::start();
+    let t2 = MockServer::start();
+    let t3 = MockServer::start();
+
+    let words = [
+        String::from("LICENSE"),
+        String::from("stuff"),
+        String::from("things"),
+        String::from("mostuff"),
+        String::from("mothings"),
+    ];
+    let (word_tmp_dir, wordlist) = setup_tmp_directory(&words, "wordlist")?;
+    let (output_dir, outfile) = setup_tmp_directory(&[], "output-file")?;
+    let (tgt_tmp_dir, targets) =
+        setup_tmp_directory(&[t1.url("/"), t2.url("/"), t3.url("/")], "targets")?;
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--stdin")
+        .arg("--parallel")
+        .arg("2")
+        .arg("--output")
+        .arg(outfile.as_os_str())
+        .arg("--wordlist")
+        .arg(wordlist.as_os_str())
+        .pipe_stdin(targets)
+        .unwrap()
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("Could not connect to any target provided")
+                .and(predicate::str::contains("Target Url"))
+                .not(), // no target url found
+        );
+
+    // output_dir should return something similar to output-file-1627845244.logs with the
+    // line below. if it ever fails, can use the regex below to filter out the right directory
+    let sub_dir = read_dir(&output_dir)?.next().unwrap()?.file_name();
+
+    let mut num_logs = 0;
+    let file_regex = Regex::new("ferox-[a-zA-Z_:0-9]+-[0-9]+.log").unwrap();
+    let dir_regex = Regex::new("output-file-[0-9]+.logs").unwrap();
+
+    let sub_dir = output_dir.as_ref().join(&sub_dir);
+
+    // created directory like output-file-1627845741.logs/
+    assert!(dir_regex.is_match(&sub_dir.to_string_lossy().to_string()));
+
+    for entry in sub_dir.read_dir()? {
+        let entry = entry?;
+        // created each file like ferox-https_localhost-1627845741.log
+        println!("name: {:?}", entry.file_name().to_string_lossy());
+        assert!(file_regex.is_match(&entry.file_name().to_string_lossy()));
+        num_logs += 1;
+    }
+
+    // should be 3 log files total
+    assert_eq!(num_logs, 3);
+
+    teardown_tmp_directory(word_tmp_dir);
+    teardown_tmp_directory(tgt_tmp_dir);
+    teardown_tmp_directory(output_dir);
+
+    Ok(())
+}

tests/test_parser.rs

@@ -0,0 +1,46 @@
+use assert_cmd::Command;
+use predicates::prelude::*;
+
+#[test]
+/// specify an incorrect param (-fc) with --help after it on the command line
+/// old behavior printed
+/// error: Found argument '-c' which wasn't expected, or isn't valid in this context
+///
+/// USAGE:
+///     feroxbuster --add-slash --url <URL>...
+///
+/// For more information try --help
+///
+/// the new behavior we expect to see is to print the long form help message, of which
+/// Ludicrous speed... go! is near the bottom of that output, so we can test for that
+fn parser_incorrect_param_with_tack_tack_help() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("-fc")
+        .arg("--help")
+        .assert()
+        .success()
+        .stdout(predicate::str::contains("Ludicrous speed... go!"));
+}
+
+#[test]
+/// specify an incorrect param (-fc) with --help after it on the command line
+/// old behavior printed
+/// error: Found argument '-c' which wasn't expected, or isn't valid in this context
+///
+/// USAGE:
+///     feroxbuster --add-slash --url <URL>...
+///
+/// For more information try --help
+///
+/// the new behavior we expect to see is to print the long form help message, of which
+/// Ludicrous speed... go! is near the bottom of that output, so we can test for that
+fn parser_incorrect_param_with_tack_h() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("-fc")
+        .arg("-h")
+        .assert()
+        .success()
+        .stdout(predicate::str::contains("Ludicrous speed... go!"));
+}

tests/test_policies.rs

@@ -0,0 +1,432 @@
+mod utils;
+use assert_cmd::prelude::*;
+use httpmock::Method::GET;
+use httpmock::MockServer;
+use regex::Regex;
+use std::fs::{read_to_string, write};
+use std::path::Path;
+use std::process::Command;
+use std::time::Instant;
+use tokio::time::Duration;
+use utils::{setup_tmp_directory, teardown_tmp_directory};
+
+// tests/policy-test-error-words is a wordlist with the following attributes:
+// - 60 errors per error category (error, 403, 429)
+// - 1000 words tagged as normal for noise/padding
+// - each error string is 6_RANDOM_ASCII{error,status403,status429,normal}6_RANDOM_ASCII
+// examples:
+// - BKPMiherrortBPKcw
+// - lTjbLpstatus403fZQaFD
+// - ZhGBHGstatus429SIUZvI
+// - ufzEXWnormalOLhbLM
+// these words will be used along with pattern matching to trigger different policies
+
+#[test]
+/// --auto-bail should cancel a scan with spurious errors  
+fn auto_bail_cancels_scan_with_timeouts() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["ignored".to_string()], "wordlist").unwrap();
+    let (log_dir, logfile) = setup_tmp_directory(&[], "debug-log").unwrap();
+
+    let policy_words = read_to_string(Path::new("tests/policy-test-words.shuffled")).unwrap();
+
+    write(&file, policy_words).unwrap();
+
+    assert_eq!(file.metadata().unwrap().len(), 117720); // sanity check on wordlist size
+
+    let error_mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z]{6}error[a-zA-Z]{6}").unwrap());
+        then.delay(Duration::new(3, 0))
+            .status(200)
+            .body("verboten, nerd");
+    });
+
+    let other_errors_mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z]{6}(status429|status403)[a-zA-Z]{6}").unwrap());
+        then.status(200).body("other errors are a 200");
+    });
+
+    let normal_reqs_mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z]{6}normal[a-zA-Z]{6}").unwrap());
+        then.status(200).body("any normal request is a 200");
+    });
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--auto-bail")
+        .arg("--dont-filter")
+        .arg("--timeout")
+        .arg("2")
+        .arg("--threads")
+        .arg("4")
+        .arg("--debug-log")
+        .arg(logfile.as_os_str())
+        .arg("-vvvv")
+        .arg("--json")
+        .assert()
+        .success();
+
+    let debug_log = read_to_string(logfile).unwrap();
+
+    // read debug log to get the number of errors enforced
+    for line in debug_log.lines() {
+        let log: serde_json::Value = serde_json::from_str(line).unwrap_or_default();
+        if let Some(message) = log.get("message") {
+            let str_msg = message.as_str().unwrap_or_default().to_string();
+
+            if str_msg.starts_with("Stats") {
+                let re = Regex::new("total_expected: ([0-9]+),").unwrap();
+                assert!(re.is_match(&str_msg));
+                let total_expected = re
+                    .captures(&str_msg)
+                    .unwrap()
+                    .get(1)
+                    .map_or("", |m| m.as_str())
+                    .parse::<usize>()
+                    .unwrap();
+
+                println!("expected: {}", total_expected);
+                // without bailing, should be 6180; after bail decreases significantly
+                assert!(total_expected < 5000);
+            }
+        }
+    }
+
+    teardown_tmp_directory(tmp_dir);
+    teardown_tmp_directory(log_dir);
+
+    assert!(normal_reqs_mock.hits() < 6000); // not all requests should make it
+    assert!(error_mock.hits() >= 25); // need at least 25 to trigger the policy
+    assert!(other_errors_mock.hits() <= 120); // may or may not see all other error requests
+}
+
+#[test]
+/// --auto-bail should cancel a scan with spurious 403s
+fn auto_bail_cancels_scan_with_403s() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["ignored".to_string()], "wordlist").unwrap();
+    let (log_dir, logfile) = setup_tmp_directory(&[], "debug-log").unwrap();
+
+    let policy_words = read_to_string(Path::new("tests/policy-test-words.shuffled")).unwrap();
+
+    write(&file, policy_words).unwrap();
+
+    assert_eq!(file.metadata().unwrap().len(), 117720); // sanity check on wordlist size
+
+    let error_mock = srv.mock(|when, then| {
+        when.method(GET).path_matches(
+            Regex::new("/[a-zA-Z]{6}(error|status429|status403)[a-zA-Z]{6}").unwrap(),
+        );
+        then.status(200).body("other errors are still a 200");
+    });
+
+    let normal_reqs_mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z]{6}normal[a-zA-Z]{6}").unwrap());
+        then.status(403)
+            .body("these guys need to be 403 in order to trigger 90% threshold");
+    });
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--auto-bail")
+        .arg("--dont-filter")
+        .arg("--threads")
+        .arg("4")
+        .arg("--debug-log")
+        .arg(logfile.as_os_str())
+        .arg("-vvvv")
+        .arg("--json")
+        .assert()
+        .success();
+
+    println!("log filesize: {}", logfile.metadata().unwrap().len());
+    let debug_log = read_to_string(logfile).unwrap();
+
+    // read debug log to get the number of errors enforced
+    for line in debug_log.lines() {
+        let log: serde_json::Value = serde_json::from_str(line).unwrap_or_default();
+        if let Some(message) = log.get("message") {
+            let str_msg = message.as_str().unwrap_or_default().to_string();
+
+            if str_msg.starts_with("Stats") {
+                println!("{}", str_msg);
+                let re = Regex::new("total_expected: ([0-9]+),").unwrap();
+                assert!(re.is_match(&str_msg));
+                let total_expected = re
+                    .captures(&str_msg)
+                    .unwrap()
+                    .get(1)
+                    .map_or("", |m| m.as_str())
+                    .parse::<usize>()
+                    .unwrap();
+                println!("total_expected: {}", total_expected);
+                assert!(total_expected < 5000);
+            }
+        }
+    }
+
+    teardown_tmp_directory(tmp_dir);
+    teardown_tmp_directory(log_dir);
+
+    assert!(normal_reqs_mock.hits() + error_mock.hits() > 25); // must have at least 50 reqs fly
+
+    // expect much less in the way of requests for this one, 90% is measured against requests made,
+    // not requests expected, so 90% can be reached very quickly. for the same reason, the
+    // num_enforced can be less than 50
+    assert!(normal_reqs_mock.hits() < 500);
+    assert!(error_mock.hits() <= 180); // may or may not see all other error requests
+}
+
+#[test]
+/// --auto-bail should cancel a scan with spurious 429s
+fn auto_bail_cancels_scan_with_429s() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["ignored".to_string()], "wordlist").unwrap();
+    let (log_dir, logfile) = setup_tmp_directory(&[], "debug-log").unwrap();
+
+    let policy_words = read_to_string(Path::new("tests/policy-test-words.shuffled")).unwrap();
+
+    write(&file, policy_words).unwrap();
+
+    assert_eq!(file.metadata().unwrap().len(), 117720); // sanity check on wordlist size
+
+    let error_mock = srv.mock(|when, then| {
+        when.method(GET).path_matches(
+            Regex::new("/[a-zA-Z]{6}(error|status429|status403)[a-zA-Z]{6}").unwrap(),
+        );
+        then.status(200).body("other errors are still a 200");
+    });
+
+    let normal_reqs_mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z]{6}normal[a-zA-Z]{6}").unwrap());
+        then.status(429)
+            .body("these guys need to be 403 in order to trigger 90% threshold");
+    });
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--auto-bail")
+        .arg("--dont-filter")
+        .arg("--threads")
+        .arg("4")
+        .arg("--debug-log")
+        .arg(logfile.as_os_str())
+        .arg("-vvvv")
+        .arg("--json")
+        .assert()
+        .success();
+
+    println!("log filesize: {}", logfile.metadata().unwrap().len());
+    let debug_log = read_to_string(logfile).unwrap();
+
+    // read debug log to get the number of errors enforced
+    for line in debug_log.lines() {
+        let log: serde_json::Value = serde_json::from_str(line).unwrap_or_default();
+        if let Some(message) = log.get("message") {
+            let str_msg = message.as_str().unwrap_or_default().to_string();
+
+            if str_msg.starts_with("Stats") {
+                println!("{}", str_msg);
+                let re = Regex::new("total_expected: ([0-9]+),").unwrap();
+                assert!(re.is_match(&str_msg));
+                let total_expected = re
+                    .captures(&str_msg)
+                    .unwrap()
+                    .get(1)
+                    .map_or("", |m| m.as_str())
+                    .parse::<usize>()
+                    .unwrap();
+                println!("total_expected: {}", total_expected);
+                assert!(total_expected < 5000);
+            }
+        }
+    }
+
+    teardown_tmp_directory(tmp_dir);
+    teardown_tmp_directory(log_dir);
+
+    assert!(normal_reqs_mock.hits() + error_mock.hits() > 25); // must have at least 50 reqs fly
+
+    // expect much less in the way of requests for this one, 90% is measured against requests made,
+    // not requests expected, so 90% can be reached very quickly. for the same reason, the
+    // num_enforced can be less than 50
+    assert!(normal_reqs_mock.hits() < 500);
+    assert!(error_mock.hits() <= 180); // may or may not see all other error requests
+}
+
+#[test]
+/// --auto-tune should slow a scan with spurious 429s
+fn auto_tune_slows_scan_with_429s() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["ignored".to_string()], "wordlist").unwrap();
+
+    let policy_words = read_to_string(Path::new("tests/policy-test-words.shuffled")).unwrap();
+
+    write(&file, policy_words).unwrap();
+
+    assert_eq!(file.metadata().unwrap().len(), 117720); // sanity check on wordlist size
+
+    let error_mock = srv.mock(|when, then| {
+        when.method(GET).path_matches(
+            Regex::new("/[a-zA-Z]{6}(error|status429|status403)[a-zA-Z]{6}").unwrap(),
+        );
+        then.status(200).body("other errors are still a 200");
+    });
+
+    let normal_reqs_mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z]{6}normal[a-zA-Z]{6}").unwrap());
+        then.status(429)
+            .body("these guys need to be 429 in order to trigger 30% threshold");
+    });
+
+    let start = Instant::now();
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--auto-tune")
+        .arg("--dont-filter")
+        .arg("--time-limit")
+        .arg("7s")
+        .arg("--threads")
+        .arg("4")
+        .assert()
+        .failure();
+
+    teardown_tmp_directory(tmp_dir);
+
+    assert!(normal_reqs_mock.hits() + error_mock.hits() > 25); // must have at least 50 reqs fly
+
+    println!("elapsed: {}", start.elapsed().as_millis()); // 3523ms without tuning
+    assert!(normal_reqs_mock.hits() < 500);
+    assert!(error_mock.hits() <= 180); // may or may not see all other error requests
+    assert!(start.elapsed().as_millis() >= 7000); // scan should hit time limit due to limiting
+}
+
+#[test]
+/// --auto-tune should slow a scan with spurious 403s
+fn auto_tune_slows_scan_with_403s() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["ignored".to_string()], "wordlist").unwrap();
+
+    let policy_words = read_to_string(Path::new("tests/policy-test-words.shuffled")).unwrap();
+
+    write(&file, policy_words).unwrap();
+
+    assert_eq!(file.metadata().unwrap().len(), 117720); // sanity check on wordlist size
+
+    let error_mock = srv.mock(|when, then| {
+        when.method(GET).path_matches(
+            Regex::new("/[a-zA-Z]{6}(error|status429|status403)[a-zA-Z]{6}").unwrap(),
+        );
+        then.status(200).body("other errors are still a 200");
+    });
+
+    let normal_reqs_mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z]{6}normal[a-zA-Z]{6}").unwrap());
+        then.status(403)
+            .body("these guys need to be 403 in order to trigger 90% threshold");
+    });
+
+    let start = Instant::now();
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--auto-tune")
+        .arg("--dont-filter")
+        .arg("--time-limit")
+        .arg("7s")
+        .arg("--threads")
+        .arg("4")
+        .assert()
+        .failure();
+
+    teardown_tmp_directory(tmp_dir);
+
+    assert!(normal_reqs_mock.hits() + error_mock.hits() > 25); // must have at least 50 reqs fly
+
+    println!("elapsed: {}", start.elapsed().as_millis()); // 3523ms without tuning
+    assert!(normal_reqs_mock.hits() < 500);
+    assert!(error_mock.hits() <= 180); // may or may not see all other error requests
+    assert!(start.elapsed().as_millis() >= 7000); // scan should hit time limit due to limiting
+}
+
+#[test]
+/// --auto-tune should slow a scan with spurious errors
+fn auto_tune_slows_scan_with_general_errors() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["ignored".to_string()], "wordlist").unwrap();
+
+    let policy_words = read_to_string(Path::new("tests/policy-test-words.shuffled")).unwrap();
+
+    write(&file, policy_words).unwrap();
+
+    assert_eq!(file.metadata().unwrap().len(), 117720); // sanity check on wordlist size
+
+    let error_mock = srv.mock(|when, then| {
+        when.method(GET).path_matches(
+            Regex::new("/[a-zA-Z]{6}(error|status429|status403)[a-zA-Z]{6}").unwrap(),
+        );
+        then.status(200).body("other errors are still a 200");
+    });
+
+    let normal_reqs_mock = srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/[a-zA-Z]{6}normal[a-zA-Z]{6}").unwrap());
+        then.status(200)
+            .body("these guys need to be 429 in order to trigger 30% threshold")
+            .delay(Duration::new(3, 0));
+    });
+
+    let start = Instant::now();
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--auto-tune")
+        .arg("--dont-filter")
+        .arg("--time-limit")
+        .arg("7s")
+        .arg("--threads")
+        .arg("4")
+        .arg("--timeout")
+        .arg("2")
+        .assert()
+        .failure();
+
+    teardown_tmp_directory(tmp_dir);
+
+    println!("elapsed: {}", start.elapsed().as_millis()); // 3523ms without tuning
+    assert!(normal_reqs_mock.hits() < 500);
+    assert!(error_mock.hits() <= 180); // may or may not see all other error requests
+    assert!(start.elapsed().as_millis() >= 7000); // scan should hit time limit due to limiting
+}

tests/test_scan_manager.rs

@@ -93,9 +93,13 @@ fn resume_scan_works() {
 #[test]
 /// kick off scan with a time limit;  
 fn time_limit_enforced_when_specified() {
-    let srv = MockServer::start();
+    let t1 = MockServer::start();
+    let t2 = MockServer::start();
+
     let (tmp_dir, file) =
         setup_tmp_directory(&["css".to_string(), "stuff".to_string()], "wordlist").unwrap();
+    let (tgt_tmp_dir, targets) =
+        setup_tmp_directory(&[t1.url("/"), t2.url("/")], "targets").unwrap();
 
     // ensure the command will run long enough by adding crap to the wordlist
     let more_words = read_to_string(Path::new("tests/extra-words")).unwrap();
@@ -109,12 +113,13 @@ fn time_limit_enforced_when_specified() {
 
     Command::cargo_bin("feroxbuster")
         .unwrap()
-        .arg("--url")
-        .arg(srv.url("/"))
+        .arg("--stdin")
         .arg("--wordlist")
         .arg(file.as_os_str())
         .arg("--time-limit")
         .arg("5s")
+        .pipe_stdin(targets)
+        .unwrap()
         .assert()
         .failure();
 
@@ -127,4 +132,5 @@ fn time_limit_enforced_when_specified() {
     assert!(now.elapsed() > lower_bound && now.elapsed() < upper_bound);
 
     teardown_tmp_directory(tmp_dir);
+    teardown_tmp_directory(tgt_tmp_dir);
 }

tests/test_scanner.rs

@@ -496,7 +496,10 @@ fn scanner_single_request_scan_with_debug_logging_as_json() {
     assert!(contents.contains("\"level\":\"DEBUG\""));
     assert!(contents.contains("\"level\":\"INFO\""));
     assert!(contents.contains("time_offset"));
-    assert!(contents.contains("\"module\":\"feroxbuster::scanner\""));
+    assert!(contents.contains("exit: main"));
+    assert!(contents.contains(&srv.url("/LICENSE")));
+    assert!(contents.contains("\"module\":\"feroxbuster::response\""));
+    assert!(contents.contains("\"module\":\"feroxbuster::url\""));
     assert!(contents.contains("\"module\":\"feroxbuster::event_handlers::inputs\""));
     assert!(contents.contains("exit: start_enter_handler"));
     assert!(contents.contains("All scans complete!"));