updated status code defaults to include 500

updated deps
added 500 to status code defaults
2026-05-22 20:31:13 -03:00 · 2021-08-02 19:28:09 -05:00 · 2021-08-02 19:27:55 -05:00 · 2021-08-02 19:23:20 -05:00 · 2021-08-02 05:41:50 -05:00 · 2021-08-02 05:21:16 -05:00
55 changed files with 2141 additions and 849 deletions
--- a/.cargo/config
+++ b/.cargo/config
@@ -0,0 +1,5 @@
+[target.armv7-unknown-linux-gnueabihf]
+linker = "arm-linux-gnueabihf-gcc"
+
+[target.aarch64-unknown-linux-gnu]
+linker = "aarch64-linux-gnu-gcc"
--- a/.github/FUNDING.yml
+++ b/.github/FUNDING.yml
@@ -0,0 +1,4 @@
+# These are supported funding model platforms
+
+github: [epi052]
+ko_fi: epi052
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -11,7 +11,7 @@ Long form explanations of most of the items below can be found in the [CONTRIBUT

 ## Static analysis checks
 - [ ] All rust files are formatted using `cargo fmt`
- [ ] All `clippy` checks pass when running `cargo clippy --all-targets --all-features -- -D warnings -A clippy::deref_addrof -A clippy::mutex-atomic`
+- [ ] All `clippy` checks pass when running `cargo clippy --all-targets --all-features -- -D warnings -A clippy::mutex-atomic`
 - [ ] All existing tests pass

 ## Documentation
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -4,11 +4,13 @@ on: [push]

 jobs:
  build-nix:
+    env:
+      IN_PIPELINE: true
    runs-on: ${{ matrix.os }}
    if: github.ref == 'refs/heads/main'
    strategy:
      matrix:
-        type: [ubuntu-x64, ubuntu-x86]
+        type: [ubuntu-x64, ubuntu-x86, armv7, aarch64]
        include:
          - type: ubuntu-x64
            os: ubuntu-latest
@@ -22,12 +24,25 @@ jobs:
            name: x86-linux-feroxbuster
            path: target/i686-unknown-linux-musl/release/feroxbuster
            pkg_config_path: /usr/lib/i686-linux-gnu/pkgconfig
+          - type: armv7
+            os: ubuntu-latest
+            target: armv7-unknown-linux-gnueabihf
+            name: armv7-feroxbuster
+            path: target/armv7-unknown-linux-gnueabihf/release/feroxbuster
+            pkg_config_path: /usr/lib/x86_64-linux-gnu/pkgconfig
+          - type: aarch64
+            os: ubuntu-latest
+            target: aarch64-unknown-linux-gnu
+            name: aarch64-feroxbuster
+            path: target/aarch64-unknown-linux-gnu/release/feroxbuster
+            pkg_config_path: /usr/lib/x86_64-linux-gnu/pkgconfig
    steps:
      - uses: actions/checkout@v2
      - name: Install System Dependencies
        run: |
+          env
          sudo apt-get update
-          sudo apt-get install -y --no-install-recommends libssl-dev pkg-config
+          sudo apt-get install -y --no-install-recommends libssl-dev pkg-config gcc-arm-linux-gnueabihf gcc-aarch64-linux-gnu
      - uses: actions-rs/toolchain@v1
        with:
          toolchain: stable
@@ -43,7 +58,7 @@ jobs:
          args: --release --target=${{ matrix.target }}
      - name: Strip symbols from binary
        run: |
-          strip -s ${{ matrix.path }}
+          strip -s ${{ matrix.path }} || arm-linux-gnueabihf-strip -s ${{ matrix.path }} || aarch64-linux-gnu-strip -s ${{ matrix.path }}
      - name: Build tar.gz for homebrew installs
        if: matrix.type == 'ubuntu-x64'
        run: |
@@ -72,6 +87,8 @@ jobs:
          path: ./target/x86_64-unknown-linux-musl/debian/*

  build-macos:
+    env:
+      IN_PIPELINE: true
    runs-on: macos-latest
    if: github.ref == 'refs/heads/main'
    steps:
@@ -102,6 +119,8 @@ jobs:
          path: x86_64-macos-feroxbuster.tar.gz

  build-windows:
+    env:
+      IN_PIPELINE: true
    runs-on: ${{ matrix.os }}
    if: github.ref == 'refs/heads/main'
    strategy:
@@ -134,4 +153,3 @@ jobs:
        with:
          name: ${{ matrix.name }}
          path: ${{ matrix.path }}
-
--- a/Cargo.lock
+++ b/Cargo.lock
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "feroxbuster"
-version = "2.2.0"
+version = "2.3.2"
 authors = ["Ben 'epi' Risher <epibar052@gmail.com>"]
 license = "MIT"
 edition = "2018"
@@ -19,37 +19,38 @@ maintenance = { status = "actively-developed" }
 clap = "2.33"
 regex = "1"
 lazy_static = "1.4"
+dirs = "3.0"

 [dependencies]
 futures = { version = "0.3"}
-tokio = { version = "1.2.0", features = ["full"] }
-tokio-util = {version = "0.6.3", features = ["codec"]}
+tokio = { version = "1.9", features = ["full"] }
+tokio-util = {version = "0.6.6", features = ["codec"]}
 log = "0.4"
-env_logger = "0.8.3"
+env_logger = "0.9"
 reqwest = { version = "0.11", features = ["socks"] }
 clap = "2.33"
 lazy_static = "1.4"
 toml = "0.5"
 serde = { version = "1.0", features = ["derive", "rc"] }
-serde_json = "1.0.62"
+serde_json = "1.0"
 uuid = { version = "0.8", features = ["v4"] }
 indicatif = "0.15"
 console = "0.14"
 openssl = { version = "0.10", features = ["vendored"] }
 dirs = "3.0"
 regex = "1"
-crossterm = "0.19"
-rlimit = "0.5"
-ctrlc = "3.1"
+crossterm = "0.20"
+rlimit = "0.6"
+ctrlc = "3.1.9"
 fuzzyhash = "0.2.1"
 anyhow = "1.0"
 leaky-bucket = "0.10.0"

 [dev-dependencies]
 tempfile = "3.1"
-httpmock = "0.5.2"
-assert_cmd = "1.0.3"
-predicates = "1.0.7"
+httpmock = "0.5.8"
+assert_cmd = "1.0"
+predicates = "2.0"

 [profile.release]
 lto = true
@@ -63,4 +64,7 @@ conf-files = ["/etc/feroxbuster/ferox-config.toml"]
 assets = [
    ["target/release/feroxbuster", "/usr/bin/", "755"],
    ["ferox-config.toml.example", "/etc/feroxbuster/ferox-config.toml", "644"],
+    ["shell_completions/feroxbuster.bash", "/usr/share/bash-completion/completions/feroxbuster.bash", "644"],
+    ["shell_completions/feroxbuster.fish", "/usr/share/fish/completions/feroxbuster.fish", "644"],
+    ["shell_completions/_feroxbuster", "/usr/share/zsh/vendor-completions/_feroxbuster", "644"],
 ]
--- a/52
+++ b/52
@@ -6,12 +6,16 @@ datarootdir = $(prefix)/share
 datadir = $(datarootdir)
 example_config = ferox-config.toml.example
 config_file = ferox-config.toml
+completion_dir = shell_completions
+completion_prefix = $(completion_dir)/$(BIN)

+BIN=feroxbuster
 SHR_SOURCES = $(shell find src -type f -wholename '*src/*.rs') Cargo.toml Cargo.lock

 RELEASE = debug
 DEBUG ?= 0
-ifeq (0,$(DEBUG))
+
+ifeq (0, $(DEBUG))
 	ARGS = --release
 	RELEASE = release
 endif
@@ -23,54 +27,52 @@ endif

 TARGET = target/$(RELEASE)

-.PHONY: all clean distclean install uninstall update
-
-BIN=feroxbuster
-DESKTOP=$(APPID).desktop
+.PHONY: all clean install uninstall test update

 all: cli
-
 cli: $(TARGET)/$(BIN) $(TARGET)/$(BIN).1.gz $(SHR_SOURCES)
+install: all install-cli
+
+verify:
+	cargo fmt
+	cargo clippy --all-targets --all-features -- -D warnings -A clippy::mutex-atomic
+	cargo test

 clean:
 	cargo clean

-distclean: clean
-	rm -rf .cargo vendor Cargo.lock vendor.tar
-
 vendor: vendor.tar

 vendor.tar:
-	mkdir -p .cargo
-	cargo vendor | head -n -1 > .cargo/config
-	echo 'directory = "vendor"' >> .cargo/config
+	cargo vendor
 	tar pcf vendor.tar vendor
 	rm -rf vendor

 install-cli: cli
-	install -Dm 0755 "$(TARGET)/$(BIN)" "$(DESTDIR)$(bindir)/$(BIN)"
+	install -Dm 0644 "$(completion_prefix).bash" "$(DESTDIR)/usr/share/bash-completion/completions/$(BIN).bash"
+	install -Dm 0644 "$(completion_prefix).fish" "$(DESTDIR)/usr/share/fish/completions/$(BIN).fish"
+	install -Dm 0644 "$(completion_dir)/_$(BIN)" "$(DESTDIR)/usr/share/zsh/vendor-completions/_$(BIN)"
+	install -sDm 0755 "$(TARGET)/$(BIN)" "$(DESTDIR)$(bindir)/$(BIN)"
 	install -Dm 0644 "$(TARGET)/$(BIN).1.gz" "$(DESTDIR)$(datadir)/man/man1/$(BIN).1.gz"
-	install -Dm 0644 "$(example_config)" "/etc/$(BIN)/$(config_File)"
+	install -Dm 0644 "$(example_config)" "$(DESTDIR)/etc/$(BIN)/$(config_file)"

-install: all install-cli
-
-uninstall-cli:
+uninstall:
 	rm -f "$(DESTDIR)$(bindir)/$(BIN)"
 	rm -f "$(DESTDIR)$(datadir)/man/man1/$(BIN).1.gz"
-	rm -rf "/etc/$(BIN)/"
-
-uninstall: uninstall-cli
-
-update:
-	cargo update
+	rm -rf "$(DESTDIR)/etc/$(BIN)/"
+	rm -f "$(DESTDIR)/usr/share/bash-completion/completions/$(BIN).bash"
+	rm -f "$(DESTDIR)/usr/share/zsh/vendor-completions/_$(BIN)"
+	rm -f "$(DESTDIR)/usr/share/fish/completions/$(BIN).fish"

 extract:
-ifeq ($(VENDORED),1)
+ifeq (1, $(VENDORED))
 	tar pxf vendor.tar
 endif

 $(TARGET)/$(BIN): extract
-	cargo build --manifest-path Cargo.toml $(ARGS)
+	mkdir -p .cargo
+	cp debian/cargo.config .cargo/config.toml
+	cargo build $(ARGS)

 $(TARGET)/$(BIN).1.gz: $(TARGET)/$(BIN)
 	help2man --no-info $< | gzip -c > $@.partial
--- a/README.md
+++ b/README.md
@@ -70,6 +70,7 @@ Enumeration.
    - [Snap Install](#snap-install)
    - [Homebrew on MacOS and Linux](#homebrew-on-macos-and-linux)
    - [Cargo Install](#cargo-install)
+    - [Kali Install](#kali-install)
    - [apt Install](#apt-install)
    - [AUR Install](#aur-install)
    - [Docker Install](#docker-install)
@@ -106,6 +107,7 @@ Enumeration.
    - [Silence all Output or Be Kinda Quiet (new in `v2.0.0`)](#silence-all-output-or-be-kinda-quiet-new-in-v200)
    - [Auto-tune or Auto-bail from Scans (new in `v2.1.0`)](#auto-tune-or-auto-bail-from-scans-new-in-v210)
    - [Run Scans in Parallel (new in `v2.2.0`)](#run-scans-in-parallel-new-in-v220)
+    - [Prevent Specific Domain/Directory Scans aka a Deny List (new in `v2.3.0`)](#prevent-specific-domaindirectory-scans-aka-a-deny-list-new-in-v230)
 - [Comparison w/ Similar Tools](#-comparison-w-similar-tools)
 - [Common Problems/Issues (FAQ)](#-common-problemsissues-faq)
    - [No file descriptors available](#no-file-descriptors-available)
@@ -118,8 +120,9 @@ Enumeration.

 ### Download a Release

-Releases for multiple architectures can be found in the [Releases](https://github.com/epi052/feroxbuster/releases)
-section. The latest release for each of the following systems can be downloaded and executed as shown below.
+Releases for `armv7`, `aarch64`, and an `x86_64 .deb` can be found in the [Releases](https://github.com/epi052/feroxbuster/releases) section.
+
+All other OS/architecture combinations can be installed dynamically using one of the methods shown below.

 #### Linux (32 and 64-bit) & MacOS

@@ -194,6 +197,16 @@ brew install feroxbuster
 cargo install feroxbuster
 ```

+### Kali Install
+
+🥳 `feroxbuster` was recently added to the official Kali Linux repos 🥳 
+
+If you're using kali, this is the preferred install method. Installing from the repos adds a [**ferox-config.toml**](#ferox-config.toml) in `/etc/feroxbuster/`, adds command completion for bash, fish, and zsh, includes a man page entry, and installs `feroxbuster` itself. 
+
+```
+sudo apt update && sudo apt install -y feroxbuster
+```
+
 ### apt Install

 Download `feroxbuster_amd64.deb` from the [Releases](https://github.com/epi052/feroxbuster/releases) section. After
@@ -213,6 +226,14 @@ Install `feroxbuster-git` on Arch Linux with your AUR helper of choice:
 yay -S feroxbuster-git
 ```

+### BlackArch install
+
+Install `feroxbuster` on BlackArch Linux:
+
+```
+pacman -S feroxbuster
+```
+
 ### Docker Install

 > The following steps assume you have docker installed / setup
@@ -283,7 +304,7 @@ Configuration begins with with the following built-in default values baked into
 - verbosity: `0` (no logging enabled)
 - scan_limit: `0` (no limit imposed on concurrent scans)
 - rate_limit: `0` (no limit imposed on requests per second)
- status_codes: `200 204 301 302 307 308 401 403 405`
+- status_codes: `200 204 301 302 307 308 401 403 405 500`
 - user_agent: `feroxbuster/VERSION`
 - recursion depth: `4`
 - auto-filter wildcards - `true`
@@ -396,6 +417,7 @@ A pre-made configuration file with examples of all available settings can be fou
 # dont_filter = true
 # extract_links = true
 # depth = 1
+# url_denylist = ["https://dont-scan-me.com/"]
 # filter_size = [5174]
 # filter_regex = ["^ignore me$"]
 # filter_similar = ["https://somesite.com/soft404"]
@@ -429,49 +451,95 @@ USAGE:
    feroxbuster [FLAGS] [OPTIONS] --url <URL>...

 FLAGS:
-    -f, --add-slash        Append / to each request
-        --auto-bail        Automatically stop scanning when an excessive amount of errors are encountered
-        --auto-tune        Automatically lower scan rate when an excessive amount of errors are encountered
-    -D, --dont-filter      Don't auto-filter wildcard responses
-    -e, --extract-links    Extract links from response body (html, javascript, etc...); make new requests based on
-                           findings (default: false)
-    -h, --help             Prints help information
-    -k, --insecure         Disables TLS certificate validation
-        --json             Emit JSON logs to --output and --debug-log instead of normal text
-    -n, --no-recursion     Do not scan recursively
-    -q, --quiet            Hide progress bars and banner (good for tmux windows w/ notifications)
-    -r, --redirects        Follow redirects
-        --silent           Only print URLs + turn off logging (good for piping a list of urls to other commands)
-        --stdin            Read url(s) from STDIN
-    -V, --version          Prints version information
-    -v, --verbosity        Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v's is probably
-                           too much)
+    -f, --add-slash        
+            Append / to each request
+
+        --auto-bail        
+            Automatically stop scanning when an excessive amount of errors are encountered
+
+        --auto-tune        
+            Automatically lower scan rate when an excessive amount of errors are encountered
+
+    -D, --dont-filter      
+            Don't auto-filter wildcard responses
+
+    -e, --extract-links    
+            Extract links from response body (html, javascript, etc...); make new requests based on findings (default:
+            false)
+    -h, --help             
+            Prints help information
+
+    -k, --insecure         
+            Disables TLS certificate validation
+
+        --json             
+            Emit JSON logs to --output and --debug-log instead of normal text
+
+    -n, --no-recursion     
+            Do not scan recursively
+
+    -q, --quiet            
+            Hide progress bars and banner (good for tmux windows w/ notifications)
+
+    -r, --redirects        
+            Follow redirects
+
+        --silent           
+            Only print URLs + turn off logging (good for piping a list of urls to other commands)
+
+        --stdin            
+            Read url(s) from STDIN
+
+    -V, --version          
+            Prints version information
+
+    -v, --verbosity        
+            Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v's is probably too much)
+

 OPTIONS:
-        --debug-log <FILE>                        Output file to write log entries (use w/ --json for JSON entries)
+        --debug-log <FILE>                        
+            Output file to write log entries (use w/ --json for JSON entries)
+
    -d, --depth <RECURSION_DEPTH>
            Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)

-    -x, --extensions <FILE_EXTENSION>...          File extension(s) to search for (ex: -x php -x pdf js)
-    -N, --filter-lines <LINES>...                 Filter out messages of a particular line count (ex: -N 20 -N 31,30)
+    -x, --extensions <FILE_EXTENSION>...          
+            File extension(s) to search for (ex: -x php -x pdf js)
+
+    -N, --filter-lines <LINES>...                 
+            Filter out messages of a particular line count (ex: -N 20 -N 31,30)
+
    -X, --filter-regex <REGEX>...
            Filter out messages via regular expression matching on the response's body (ex: -X '^ignore me$')

        --filter-similar-to <UNWANTED_PAGE>...
            Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)

-    -S, --filter-size <SIZE>...                   Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)
-    -C, --filter-status <STATUS_CODE>...          Filter out status codes (deny list) (ex: -C 200 -C 401)
-    -W, --filter-words <WORDS>...                 Filter out messages of a particular word count (ex: -W 312 -W 91,82)
-    -H, --headers <HEADER>...                     Specify HTTP headers (ex: -H Header:val 'stuff: things')
-    -o, --output <FILE>                           Output file to write results to (use w/ --json for JSON entries)
+    -S, --filter-size <SIZE>...                   
+            Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)
+
+    -C, --filter-status <STATUS_CODE>...          
+            Filter out status codes (deny list) (ex: -C 200 -C 401)
+
+    -W, --filter-words <WORDS>...                 
+            Filter out messages of a particular word count (ex: -W 312 -W 91,82)
+
+    -H, --headers <HEADER>...                     
+            Specify HTTP headers (ex: -H Header:val 'stuff: things')
+
+    -o, --output <FILE>                           
+            Output file to write results to (use w/ --json for JSON entries)
+
        --parallel <PARALLEL_SCANS>
            Run parallel feroxbuster instances (one child process per url passed via stdin)

    -p, --proxy <PROXY>
            Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)

-    -Q, --query <QUERY>...                        Specify URL query parameters (ex: -Q token=stuff -Q secret=key)
+    -Q, --query <QUERY>...                        
+            Specify URL query parameters (ex: -Q token=stuff -Q secret=key)
+
        --rate-limit <RATE_LIMIT>
            Limit number of requests per second (per directory) (default: 0, i.e. no limit)

@@ -484,16 +552,32 @@ OPTIONS:
        --resume-from <STATE_FILE>
            State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)

-    -L, --scan-limit <SCAN_LIMIT>                 Limit total number of concurrent scans (default: 0, i.e. no limit)
-    -s, --status-codes <STATUS_CODE>...
-            Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)
+    -L, --scan-limit <SCAN_LIMIT>                 
+            Limit total number of concurrent scans (default: 0, i.e. no limit)

-    -t, --threads <THREADS>                       Number of concurrent threads (default: 50)
-        --time-limit <TIME_SPEC>                  Limit total run time of all scans (ex: --time-limit 10m)
-    -T, --timeout <SECONDS>                       Number of seconds before a request times out (default: 7)
-    -u, --url <URL>...                            The target URL(s) (required, unless --stdin used)
-    -a, --user-agent <USER_AGENT>                 Sets the User-Agent (default: feroxbuster/VERSION)
-    -w, --wordlist <FILE>                         Path to the wordlist
+    -s, --status-codes <STATUS_CODE>...
+            Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405 500)
+
+    -t, --threads <THREADS>                       
+            Number of concurrent threads (default: 50)
+
+        --time-limit <TIME_SPEC>                  
+            Limit total run time of all scans (ex: --time-limit 10m)
+
+    -T, --timeout <SECONDS>                       
+            Number of seconds before a request times out (default: 7)
+
+    -u, --url <URL>...                            
+            The target URL(s) (required, unless --stdin used)
+
+        --dont-scan <URL>...                      
+            URL(s) to exclude from recursion/scans
+
+    -a, --user-agent <USER_AGENT>                 
+            Sets the User-Agent (default: feroxbuster/VERSION)
+
+    -w, --wordlist <FILE>                         
+            Path to the wordlist
 ```

 ## 📊 Scan's Display Explained
@@ -816,10 +900,11 @@ Below is an example of the Scan Cancel Menu™.
 Using the menu is pretty simple:
 - Press `ENTER` to view the menu
 - Choose a scan to cancel by entering its scan index (`1`)
-  - more than one scan can be selected by using a comma-separated list (`1,2,3` ... etc)
+  - more than one scan can be selected by using a comma-separated list of indexes and/or ranges (`1-4,8,9-13` ... etc)
 - Confirm selections, after which all non-cancelled scans will resume
+  - To skip confirmation, simply add a `-f` somewhere in your input (`3-5 -f`)

-Here is a short demonstration of cancelling two in-progress scans found via recursion.
+Here is a short demonstration of force cancelling a range of scans followed by a single scan with interactive prompt.

 ![cancel-scan](img/cancel-scan.gif)

@@ -916,16 +1001,59 @@ Example Command:
 cat large-target-list | ./feroxbuster --stdin --parallel 10 --extract-links --auto-bail
 ```

-Resuling Process List (illustrative):
+Resulting Process List (illustrative):
 ```
-\_ target/debug/feroxbuster --stdin --parallel 10
-    \_ target/debug/feroxbuster --silent --extract-links --auto-bail -u https://target-one
-    \_ target/debug/feroxbuster --silent --extract-links --auto-bail -u https://target-two
-    \_ target/debug/feroxbuster --silent --extract-links --auto-bail -u https://target-three
-    \_ ...
-    \_ target/debug/feroxbuster --silent --extract-links --auto-bail -u https://target-ten
+feroxbuster --stdin --parallel 10
+ \_ feroxbuster --silent --extract-links --auto-bail -u https://target-one
+ \_ feroxbuster --silent --extract-links --auto-bail -u https://target-two
+ \_ feroxbuster --silent --extract-links --auto-bail -u https://target-three
+ \_ ...
+ \_ feroxbuster --silent --extract-links --auto-bail -u https://target-ten
 ```

+As of `v2.3.2`, logging while using `--parallel` uses the value of `-o`|`--output` as a seed to create a directory named `OUTPUT_VALUE-TIMESTAMP.logs/`. Within the directory, an individual log file is created for each target passed over stdin.
+
+Example Command:
+```
+cat large-target-list | ./feroxbuster --stdin --parallel 10 --output super-cool-mega-scan
+```
+
+Resulting directory structure (illustrative):
+```
+super-cool-mega-scan-1627865696.logs/
+├── ferox-https_target_one_com-1627865696.log
+├── ...
+└── ferox-https_target_two_net-1627865696.log
+```
+
+### Prevent Specific Domain/Directory Scans aka a Deny List (new in `v2.3.0`)
+
+> This action is taken BEFORE a request is sent to the target, which differs from the filter-* options that are applied to responses
+
+Version 2.3.0 introduces the `--dont-scan` option. The values passed to `--dont-scan` act as a deny-list.  The values 
+can be an entire domain (`http://some.domain`), a specific folder (`http://some.domain/js`), or a specific file 
+(`http://some.domain/some-application/stupid-page.php`)  If a folder/domain is used any sub-folder/sub-file of the 
+url passed to `--dont-scan` will be blocked before it can be requested.
+
+For example, given the command 
+
+```
+./feroxbuster -u http://some.domain --dont-scan http://some.domain/js
+```
+
+`http://some.domain` will be scanned recursively, but any url path that begins with `/js/` will not be requested at all.
+
+A caveat to the sub-folder/sub-file rule is when the value passed to `--dont-scan` is a parent of the scan you want to 
+perform. When denying at a hierarchical level higher than your scan, only sub-files/sub-folders of your `-u|--stdin` 
+value(s) will be processed.
+
+```
+./feroxbuster -u http://some.domain/some-application --dont-scan http://some.domain/
+```
+
+In the command above, only `http://some.domain/some-application` and children of that directory found via recursion will
+be scanned. Anything 'outside' of `/some-application` will not be scanned.
+
 ## 🧐 Comparison w/ Similar Tools

 There are quite a few similar tools for forced browsing/content discovery. Burp Suite Pro, Dirb, Dirbuster, etc...
@@ -949,7 +1077,7 @@ few of the use-cases in which feroxbuster may be a better fit:
 | fast                                                                         | ✔ | ✔ | ✔ |
 | allows recursion                                                             | ✔ |   | ✔ |
 | can specify query parameters                                                 | ✔ |   | ✔ |
-| SOCKS proxy support                                                          | ✔ |   |   |
+| SOCKS proxy support                                                          | ✔ |   | ✔ |
 | multiple target scan (via stdin or multiple -u)                              | ✔ |   | ✔ |
 | configuration file for default value override                                | ✔ |   | ✔ |
 | can accept urls via STDIN as part of a pipeline                              | ✔ |   | ✔ |
@@ -976,6 +1104,7 @@ few of the use-cases in which feroxbuster may be a better fit:
 | automatically tune scans based on errors/403s/429s  (`v2.1.0`)               | ✔ |   |   |
 | automatically stop scans based on errors/403s/429s  (`v2.1.0`)               | ✔ |   | ✔ |
 | run scans in parallel (1 process per target) (`v2.2.0`)                      | ✔ |   |   |
+| prevent requests to given domain/folder/file (`v2.3.0`)                      | ✔ |   |   |
 | **huge** number of other options                                             |   |   | ✔ |

 Of note, there's another written-in-rust content discovery tool, [rustbuster](https://github.com/phra/rustbuster). I
--- a/build.rs
+++ b/build.rs
@@ -1,4 +1,7 @@
+use std::fs::{copy, create_dir_all, OpenOptions};
+use std::io::{Read, Seek, SeekFrom, Write};
 extern crate clap;
+extern crate dirs;

 use clap::Shell;

@@ -20,4 +23,64 @@ fn main() {
    for shell in &shells {
        app.gen_completions("feroxbuster", *shell, outdir);
    }
+
+    // 0xdf pointed out an oddity when tab-completing options that expect file paths, the fix we
+    // landed on was to add -o plusdirs to the bash completion script. The following code aims to
+    // automate that fix and have it present in all future builds
+    let mut contents = String::new();
+
+    let mut bash_file = OpenOptions::new()
+        .read(true)
+        .write(true)
+        .open(format!("{}/feroxbuster.bash", outdir))
+        .expect("Couldn't open bash completion script");
+
+    bash_file
+        .read_to_string(&mut contents)
+        .expect("Couldn't read bash completion script");
+
+    contents = contents.replace("default feroxbuster", "default -o plusdirs feroxbuster");
+
+    bash_file
+        .seek(SeekFrom::Start(0))
+        .expect("Couldn't seek to position 0 in bash completion script");
+
+    bash_file
+        .write_all(contents.as_bytes())
+        .expect("Couldn't write updated bash completion script to disk");
+
+    // hunter0x8 let me know that when installing via cargo, it would be nice if we dropped a
+    // config file during the build process. The following code will place an example config in
+    // the user's configuration directory
+    //   - linux: $XDG_CONFIG_HOME or $HOME/.config
+    //   - macOS: $HOME/Library/Application Support
+    //   - windows: {FOLDERID_RoamingAppData}
+    let mut config_dir = dirs::config_dir().expect("Couldn't resolve user's config directory");
+    config_dir = config_dir.join("feroxbuster"); // $HOME/.config/feroxbuster
+
+    if !config_dir.exists() {
+        // recursively create the feroxbuster directory and all of its parent components if
+        // they are missing
+        if !config_dir.exists() {
+            // recursively create the feroxbuster directory and all of its parent components if
+            // they are missing
+            if create_dir_all(&config_dir).is_err() {
+                // only copy the config file when we're not running in the CI/CD pipeline
+                // which fails with permission denied
+                eprintln!("Couldn't create one or more directories needed to copy the config file");
+                return;
+            }
+        }
+    }
+
+    // hard-coding config name here to not rely on the crate we're building, if DEFAULT_CONFIG_NAME
+    // ever changes, this will need to be updated
+    let config_file = config_dir.join("ferox-config.toml");
+
+    if !config_file.exists() {
+        // config file doesn't exist, add it to the config directory
+        if copy("ferox-config.toml.example", config_file).is_err() {
+            eprintln!("Couldn't copy example config into config directory");
+        }
+    }
 }
--- a/ferox-config.toml.example
+++ b/ferox-config.toml.example
@@ -30,6 +30,7 @@
 # redirects = true
 # insecure = true
 # extensions = ["php", "html"]
+# url_denylist = ["http://dont-scan.me", "https://also-not.me"]
 # no_recursion = true
 # add_slash = true
 # stdin = true
--- a/img/cancel-menu.png
+++ b/img/cancel-menu.png
--- a/img/cancel-scan.gif
+++ b/img/cancel-scan.gif
--- a/install-nix.sh
+++ b/install-nix.sh
@@ -3,59 +3,63 @@
 BASE_URL=https://github.com/epi052/feroxbuster/releases/latest/download

 MAC_ZIP=x86_64-macos-feroxbuster.zip
-MAC_URL="${BASE_URL}/${MAC_ZIP}"
+MAC_URL="$BASE_URL/$MAC_ZIP"

 LIN32_ZIP=x86-linux-feroxbuster.zip
-LIN32_URL="${BASE_URL}/${LIN32_ZIP}"
+LIN32_URL="$BASE_URL/$LIN32_ZIP"

 LIN64_ZIP=x86_64-linux-feroxbuster.zip
-LIN64_URL="${BASE_URL}/${LIN64_ZIP}"
+LIN64_URL="$BASE_URL/$LIN64_ZIP"

 EMOJI_URL=https://gist.github.com/epi052/8196b550ea51d0907ad4b93751b1b57d/raw/6112c9f32ae07922983fdc549c54fd3fb9a38e4c/NotoColorEmoji.ttf

 echo "[+] Installing feroxbuster!"

-if [[ "$(uname)" == "Darwin" ]]; then
-    echo "[=] Found MacOS, downloading from ${MAC_URL}"
-
-    curl -sLO "${MAC_URL}"
-    unzip -o "${MAC_ZIP}" > /dev/null
-    rm "${MAC_ZIP}"
-elif [[ "$(expr substr $(uname -s) 1 5)" == "Linux" ]]; then
-    if [[ $(getconf LONG_BIT) == 32 ]]; then
-        echo "[=] Found 32-bit Linux, downloading from ${LIN32_URL}"
-
-        curl -sLO "${LIN32_URL}"
-        unzip -o "${LIN32_ZIP}" > /dev/null
-        rm "${LIN32_ZIP}"
-    else
-        echo "[=] Found 64-bit Linux, downloading from ${LIN64_URL}"
-
-        curl -sLO "${LIN64_URL}"
-        unzip -o "${LIN64_ZIP}" > /dev/null
-        rm "${LIN64_ZIP}"
-    fi
-
-    if [[ -e ~/.fonts/NotoColorEmoji.ttf ]]; then
-       echo "[=] Found Noto Emoji Font, skipping install" 
-    else 
-        echo "[=] Installing Noto Emoji Font"
-        mkdir -p ~/.fonts
-        pushd ~/.fonts 2>&1 >/dev/null
-
-        curl -sLO "${EMOJI_URL}"
-
-        fc-cache -f -v >/dev/null
-
-        popd 2>&1 >/dev/null
-        echo "[+] Noto Emoji Font installed"
-    fi
+which unzip &>/dev/null
+if [ "$?" = "0" ]; then
+  echo "[+] unzip found"
+else
+  echo "[ ] unzip not found, exiting. "
+  exit -1
 fi

+if [[ "$(uname)" == "Darwin" ]]; then
+  echo "[=] Found MacOS, downloading from $MAC_URL"
+
+  curl -sLO "$MAC_URL"
+  unzip -o "$MAC_ZIP" >/dev/null
+  rm "$MAC_ZIP"
+elif [[ "$(expr substr $(uname -s) 1 5)" == "Linux" ]]; then
+  if [[ $(getconf LONG_BIT) == 32 ]]; then
+    echo "[=] Found 32-bit Linux, downloading from $LIN32_URL"
+
+    curl -sLO "$LIN32_URL"
+    unzip -o "$LIN32_ZIP" >/dev/null
+    rm "$LIN32_ZIP"
+  else
+    echo "[=] Found 64-bit Linux, downloading from $LIN64_URL"
+
+    curl -sLO "$LIN64_URL"
+    unzip -o "$LIN64_ZIP" >/dev/null
+    rm "$LIN64_ZIP"
+  fi
+
+  if [[ -e ~/.fonts/NotoColorEmoji.ttf ]]; then
+    echo "[=] Found Noto Emoji Font, skipping install"
+  else
+    echo "[=] Installing Noto Emoji Font"
+    mkdir -p ~/.fonts
+    pushd ~/.fonts 2>&1 >/dev/null
+
+    curl -sLO "$EMOJI_URL"
+
+    fc-cache -f -v >/dev/null
+
+    popd 2>&1 >/dev/null
+    echo "[+] Noto Emoji Font installed"
+  fi
+fi

 chmod +x ./feroxbuster

 echo "[+] Installed feroxbuster version $(./feroxbuster -V)"
-
-
-
--- a/shell_completions/_feroxbuster
+++ b/shell_completions/_feroxbuster
@@ -41,6 +41,7 @@ _feroxbuster() {
 '--user-agent=[Sets the User-Agent (default: feroxbuster/VERSION)]' \
 '*-x+[File extension(s) to search for (ex: -x php -x pdf js)]' \
 '*--extensions=[File extension(s) to search for (ex: -x php -x pdf js)]' \
+'*--dont-scan=[URL(s) to exclude from recursion/scans]' \
 '*-H+[Specify HTTP headers (ex: -H Header:val '\''stuff: things'\'')]' \
 '*--headers=[Specify HTTP headers (ex: -H Header:val '\''stuff: things'\'')]' \
 '*-Q+[Specify URL query parameters (ex: -Q token=stuff -Q secret=key)]' \
--- a/shell_completions/_feroxbuster.ps1
+++ b/shell_completions/_feroxbuster.ps1
@@ -46,6 +46,7 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
            [CompletionResult]::new('--user-agent', 'user-agent', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/VERSION)')
            [CompletionResult]::new('-x', 'x', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js)')
            [CompletionResult]::new('--extensions', 'extensions', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js)')
+            [CompletionResult]::new('--dont-scan', 'dont-scan', [CompletionResultType]::ParameterName, 'URL(s) to exclude from recursion/scans')
            [CompletionResult]::new('-H', 'H', [CompletionResultType]::ParameterName, 'Specify HTTP headers (ex: -H Header:val ''stuff: things'')')
            [CompletionResult]::new('--headers', 'headers', [CompletionResultType]::ParameterName, 'Specify HTTP headers (ex: -H Header:val ''stuff: things'')')
            [CompletionResult]::new('-Q', 'Q', [CompletionResultType]::ParameterName, 'Specify URL query parameters (ex: -Q token=stuff -Q secret=key)')
--- a/shell_completions/feroxbuster.bash
+++ b/shell_completions/feroxbuster.bash
@@ -20,7 +20,7 @@ _feroxbuster() {

    case "${cmd}" in
        feroxbuster)
-            opts=" -v -q -D -r -k -n -f -e -h -V -w -u -t -d -T -p -P -R -s -o -a -x -H -Q -S -X -W -N -C -L  --verbosity --silent --quiet --auto-tune --auto-bail --json --dont-filter --redirects --insecure --no-recursion --add-slash --stdin --extract-links --help --version --wordlist --url --threads --depth --timeout --proxy --replay-proxy --replay-codes --status-codes --output --resume-from --debug-log --user-agent --extensions --headers --query --filter-size --filter-regex --filter-words --filter-lines --filter-status --filter-similar-to --scan-limit --parallel --rate-limit --time-limit  "
+            opts=" -v -q -D -r -k -n -f -e -h -V -w -u -t -d -T -p -P -R -s -o -a -x -H -Q -S -X -W -N -C -L  --verbosity --silent --quiet --auto-tune --auto-bail --json --dont-filter --redirects --insecure --no-recursion --add-slash --stdin --extract-links --help --version --wordlist --url --threads --depth --timeout --proxy --replay-proxy --replay-codes --status-codes --output --resume-from --debug-log --user-agent --extensions --dont-scan --headers --query --filter-size --filter-regex --filter-words --filter-lines --filter-status --filter-similar-to --scan-limit --parallel --rate-limit --time-limit  "
            if [[ ${cur} == -* || ${COMP_CWORD} -eq 1 ]] ; then
                COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                return 0
@@ -131,6 +131,10 @@ _feroxbuster() {
                    COMPREPLY=($(compgen -f "${cur}"))
                    return 0
                    ;;
+                --dont-scan)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
                --headers)
                    COMPREPLY=($(compgen -f "${cur}"))
                    return 0
@@ -222,4 +226,4 @@ _feroxbuster() {
    esac
 }

-complete -F _feroxbuster -o bashdefault -o default feroxbuster
+complete -F _feroxbuster -o bashdefault -o default -o plusdirs feroxbuster
--- a/shell_completions/feroxbuster.fish
+++ b/shell_completions/feroxbuster.fish
@@ -12,6 +12,7 @@ complete -c feroxbuster -n "__fish_use_subcommand" -l resume-from -d 'State file
 complete -c feroxbuster -n "__fish_use_subcommand" -l debug-log -d 'Output file to write log entries (use w/ --json for JSON entries)'
 complete -c feroxbuster -n "__fish_use_subcommand" -s a -l user-agent -d 'Sets the User-Agent (default: feroxbuster/VERSION)'
 complete -c feroxbuster -n "__fish_use_subcommand" -s x -l extensions -d 'File extension(s) to search for (ex: -x php -x pdf js)'
+complete -c feroxbuster -n "__fish_use_subcommand" -l dont-scan -d 'URL(s) to exclude from recursion/scans'
 complete -c feroxbuster -n "__fish_use_subcommand" -s H -l headers -d 'Specify HTTP headers (ex: -H Header:val \'stuff: things\')'
 complete -c feroxbuster -n "__fish_use_subcommand" -s Q -l query -d 'Specify URL query parameters (ex: -Q token=stuff -Q secret=key)'
 complete -c feroxbuster -n "__fish_use_subcommand" -s S -l filter-size -d 'Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)'
--- a/src/banner/container.rs
+++ b/src/banner/container.rs
@@ -134,6 +134,9 @@ pub struct Banner {
    /// represents Configuration.auto_bail
    auto_bail: BannerEntry,

+    /// represents Configuration.url_denylist
+    url_denylist: Vec<BannerEntry>,
+
    /// current version of feroxbuster
    pub(super) version: String,

@@ -146,6 +149,7 @@ impl Banner {
    /// Create a new Banner from a Configuration and live targets
    pub fn new(tgts: &[String], config: &Configuration) -> Self {
        let mut targets = Vec::new();
+        let mut url_denylist = Vec::new();
        let mut code_filters = Vec::new();
        let mut replay_codes = Vec::new();
        let mut headers = Vec::new();
@@ -160,6 +164,10 @@ impl Banner {
            targets.push(BannerEntry::new("🎯", "Target Url", target));
        }

+        for denied_url in &config.url_denylist {
+            url_denylist.push(BannerEntry::new("🚫", "Don't Scan", denied_url));
+        }
+
        let mut codes = vec![];
        for code in &config.status_codes {
            codes.push(status_colorizer(&code.to_string()))
@@ -171,7 +179,7 @@ impl Banner {
            code_filters.push(status_colorizer(&code.to_string()))
        }
        let filter_status = BannerEntry::new(
-            "🗑",
+            "💢",
            "Status Code Filters",
            &format!("[{}]", code_filters.join(", ")),
        );
@@ -323,6 +331,7 @@ impl Banner {
            rate_limit,
            scan_limit,
            time_limit,
+            url_denylist,
            config: cfg,
            version: VERSION.to_string(),
            update_status: UpdateStatus::Unknown,
@@ -413,6 +422,10 @@ by Ben "epi" Risher {}                 ver: {}"#,
            writeln!(&mut writer, "{}", target)?;
        }

+        for denied_url in &self.url_denylist {
+            writeln!(&mut writer, "{}", denied_url)?;
+        }
+
        writeln!(&mut writer, "{}", self.threads)?;
        writeln!(&mut writer, "{}", self.wordlist)?;
        writeln!(&mut writer, "{}", self.status_codes)?;
--- a/src/config/container.rs
+++ b/src/config/container.rs
@@ -248,6 +248,10 @@ pub struct Configuration {
    /// Filter out response bodies that meet a certain threshold of similarity
    #[serde(default)]
    pub filter_similar: Vec<String>,
+
+    /// URLs that should never be scanned/recursed into
+    #[serde(default)]
+    pub url_denylist: Vec<String>,
 }

 impl Default for Configuration {
@@ -304,6 +308,7 @@ impl Default for Configuration {
            extensions: Vec::new(),
            filter_size: Vec::new(),
            filter_regex: Vec::new(),
+            url_denylist: Vec::new(),
            filter_line_count: Vec::new(),
            filter_word_count: Vec::new(),
            filter_status: Vec::new(),
@@ -341,6 +346,7 @@ impl Configuration {
    /// - **user_agent**: `feroxbuster/VERSION`
    /// - **insecure**: `false` (don't be insecure, i.e. don't allow invalid certs)
    /// - **extensions**: `None`
+    /// - **url_denylist**: `None`
    /// - **filter_size**: `None`
    /// - **filter_similar**: `None`
    /// - **filter_regex**: `None`
@@ -538,6 +544,10 @@ impl Configuration {
            config.extensions = arg.map(|val| val.to_string()).collect();
        }

+        if let Some(arg) = args.values_of("url_denylist") {
+            config.url_denylist = arg.map(|val| val.to_string()).collect();
+        }
+
        if let Some(arg) = args.values_of("filter_regex") {
            config.filter_regex = arg.map(|val| val.to_string()).collect();
        }
@@ -767,6 +777,11 @@ impl Configuration {
        update_if_not_default!(&mut conf.insecure, new.insecure, false);
        update_if_not_default!(&mut conf.extract_links, new.extract_links, false);
        update_if_not_default!(&mut conf.extensions, new.extensions, Vec::<String>::new());
+        update_if_not_default!(
+            &mut conf.url_denylist,
+            new.url_denylist,
+            Vec::<String>::new()
+        );
        update_if_not_default!(&mut conf.headers, new.headers, HashMap::new());
        update_if_not_default!(&mut conf.queries, new.queries, Vec::new());
        update_if_not_default!(&mut conf.no_recursion, new.no_recursion, false);
--- a/src/config/tests.rs
+++ b/src/config/tests.rs
@@ -29,6 +29,7 @@ fn setup_config_test() -> Configuration {
            redirects = true
            insecure = true
            extensions = ["html", "php", "js"]
+            url_denylist = ["http://dont-scan.me", "https://also-not.me"]
            headers = {stuff = "things", mostuff = "mothings"}
            queries = [["name","value"], ["rick", "astley"]]
            no_recursion = true
@@ -72,24 +73,25 @@ fn default_configuration() {
    assert_eq!(config.timeout, timeout());
    assert_eq!(config.verbosity, 0);
    assert_eq!(config.scan_limit, 0);
-    assert_eq!(config.silent, false);
-    assert_eq!(config.quiet, false);
+    assert!(!config.silent);
+    assert!(!config.quiet);
    assert_eq!(config.output_level, OutputLevel::Default);
-    assert_eq!(config.dont_filter, false);
-    assert_eq!(config.auto_tune, false);
-    assert_eq!(config.auto_bail, false);
+    assert!(!config.dont_filter);
+    assert!(!config.auto_tune);
+    assert!(!config.auto_bail);
    assert_eq!(config.requester_policy, RequesterPolicy::Default);
-    assert_eq!(config.no_recursion, false);
-    assert_eq!(config.json, false);
-    assert_eq!(config.save_state, true);
-    assert_eq!(config.stdin, false);
-    assert_eq!(config.add_slash, false);
-    assert_eq!(config.redirects, false);
-    assert_eq!(config.extract_links, false);
-    assert_eq!(config.insecure, false);
+    assert!(!config.no_recursion);
+    assert!(!config.json);
+    assert!(config.save_state);
+    assert!(!config.stdin);
+    assert!(!config.add_slash);
+    assert!(!config.redirects);
+    assert!(!config.extract_links);
+    assert!(!config.insecure);
    assert_eq!(config.queries, Vec::new());
-    assert_eq!(config.extensions, Vec::<String>::new());
    assert_eq!(config.filter_size, Vec::<u64>::new());
+    assert_eq!(config.extensions, Vec::<String>::new());
+    assert_eq!(config.url_denylist, Vec::<String>::new());
    assert_eq!(config.filter_regex, Vec::<String>::new());
    assert_eq!(config.filter_similar, Vec::<String>::new());
    assert_eq!(config.filter_word_count, Vec::<usize>::new());
@@ -186,35 +188,35 @@ fn config_reads_replay_proxy() {
 /// parse the test config and see that the value parsed is correct
 fn config_reads_silent() {
    let config = setup_config_test();
-    assert_eq!(config.silent, true);
+    assert!(config.silent);
 }

 #[test]
 /// parse the test config and see that the value parsed is correct
 fn config_reads_quiet() {
    let config = setup_config_test();
-    assert_eq!(config.quiet, true);
+    assert!(config.quiet);
 }

 #[test]
 /// parse the test config and see that the value parsed is correct
 fn config_reads_json() {
    let config = setup_config_test();
-    assert_eq!(config.json, true);
+    assert!(config.json);
 }

 #[test]
 /// parse the test config and see that the value parsed is correct
 fn config_reads_auto_bail() {
    let config = setup_config_test();
-    assert_eq!(config.auto_bail, true);
+    assert!(config.auto_bail);
 }

 #[test]
 /// parse the test config and see that the value parsed is correct
 fn config_reads_auto_tune() {
    let config = setup_config_test();
-    assert_eq!(config.auto_tune, true);
+    assert!(config.auto_tune);
 }

 #[test]
@@ -235,49 +237,49 @@ fn config_reads_output() {
 /// parse the test config and see that the value parsed is correct
 fn config_reads_redirects() {
    let config = setup_config_test();
-    assert_eq!(config.redirects, true);
+    assert!(config.redirects);
 }

 #[test]
 /// parse the test config and see that the value parsed is correct
 fn config_reads_insecure() {
    let config = setup_config_test();
-    assert_eq!(config.insecure, true);
+    assert!(config.insecure);
 }

 #[test]
 /// parse the test config and see that the value parsed is correct
 fn config_reads_no_recursion() {
    let config = setup_config_test();
-    assert_eq!(config.no_recursion, true);
+    assert!(config.no_recursion);
 }

 #[test]
 /// parse the test config and see that the value parsed is correct
 fn config_reads_stdin() {
    let config = setup_config_test();
-    assert_eq!(config.stdin, true);
+    assert!(config.stdin);
 }

 #[test]
 /// parse the test config and see that the value parsed is correct
 fn config_reads_dont_filter() {
    let config = setup_config_test();
-    assert_eq!(config.dont_filter, true);
+    assert!(config.dont_filter);
 }

 #[test]
 /// parse the test config and see that the value parsed is correct
 fn config_reads_add_slash() {
    let config = setup_config_test();
-    assert_eq!(config.add_slash, true);
+    assert!(config.add_slash);
 }

 #[test]
 /// parse the test config and see that the value parsed is correct
 fn config_reads_extract_links() {
    let config = setup_config_test();
-    assert_eq!(config.extract_links, true);
+    assert!(config.extract_links);
 }

 #[test]
@@ -287,6 +289,16 @@ fn config_reads_extensions() {
    assert_eq!(config.extensions, vec!["html", "php", "js"]);
 }

+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_url_denylist() {
+    let config = setup_config_test();
+    assert_eq!(
+        config.url_denylist,
+        vec!["http://dont-scan.me", "https://also-not.me"]
+    );
+}
+
 #[test]
 /// parse the test config and see that the value parsed is correct
 fn config_reads_filter_regex() {
@@ -333,7 +345,7 @@ fn config_reads_filter_status() {
 /// parse the test config and see that the value parsed is correct
 fn config_reads_save_state() {
    let config = setup_config_test();
-    assert_eq!(config.save_state, false);
+    assert!(!config.save_state);
 }

 #[test]
@@ -364,9 +376,10 @@ fn config_reads_headers() {
 /// parse the test config and see that the values parsed are correct
 fn config_reads_queries() {
    let config = setup_config_test();
-    let mut queries = vec![];
-    queries.push(("name".to_string(), "value".to_string()));
-    queries.push(("rick".to_string(), "astley".to_string()));
+    let queries = vec![
+        ("name".to_string(), "value".to_string()),
+        ("rick".to_string(), "astley".to_string()),
+    ];
    assert_eq!(config.queries, queries);
 }

--- a/src/event_handlers/command.rs
+++ b/src/event_handlers/command.rs
@@ -1,4 +1,3 @@
-use std::collections::HashSet;
 use std::sync::Arc;

 use reqwest::StatusCode;
@@ -53,7 +52,7 @@ pub enum Command {
    TryRecursion(Box<FeroxResponse>),

    /// Send a pointer to the wordlist to the recursion handler
-    UpdateWordlist(Arc<HashSet<String>>),
+    UpdateWordlist(Arc<Vec<String>>),

    /// Instruct the ScanHandler to join on all known scans, use sender to notify main when done
    JoinTasks(Sender<bool>),
--- a/src/event_handlers/inputs.rs
+++ b/src/event_handlers/inputs.rs
@@ -4,6 +4,7 @@ use crate::{
    scan_manager::{FeroxState, PAUSE_SCAN},
    scanner::RESPONSES,
    statistics::StatError,
+    utils::slugify_filename,
    utils::{open_file, write_to},
    SLEEP_DURATION,
 };
@@ -17,7 +18,6 @@ use std::{
    },
    thread::sleep,
    time::Duration,
-    time::{SystemTime, UNIX_EPOCH},
 };

 /// Atomic boolean flag, used to determine whether or not the terminal input handler should exit
@@ -77,22 +77,14 @@ impl TermInputHandler {
    pub fn sigint_handler(handles: Arc<Handles>) -> Result<()> {
        log::trace!("enter: sigint_handler({:?})", handles);

-        let ts = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs();
-
-        let slug = if !handles.config.target_url.is_empty() {
+        let filename = if !handles.config.target_url.is_empty() {
            // target url populated
-            handles
-                .config
-                .target_url
-                .replace("://", "_")
-                .replace("/", "_")
-                .replace(".", "_")
+            slugify_filename(&handles.config.target_url, "ferox", "state")
        } else {
            // stdin used
-            "stdin".to_string()
+            slugify_filename("stdin", "ferox", "state")
        };

-        let filename = format!("ferox-{}-{}.state", slug, ts);
        let warning = format!(
            "🚨 Caught {} 🚨 saving scan state to {} ...",
            style("ctrl+c").yellow(),
--- a/src/event_handlers/outputs.rs
+++ b/src/event_handlers/outputs.rs
@@ -139,8 +139,8 @@ impl TermOutHandler {
        Self {
            receiver,
            tx_file,
-            config,
            file_task,
+            config,
        }
    }

@@ -213,7 +213,7 @@ impl TermOutHandler {
                        // should be replayed; not using logged_request due to replay proxy client
                        make_request(
                            self.config.replay_client.as_ref().unwrap(),
-                            &resp.url(),
+                            resp.url(),
                            self.config.output_level,
                            tx_stats.clone(),
                        )
--- a/src/event_handlers/scans.rs
+++ b/src/event_handlers/scans.rs
@@ -1,20 +1,21 @@
-use std::collections::HashSet;
 use std::sync::Arc;

 use anyhow::{bail, Result};
 use tokio::sync::{mpsc, Semaphore};

-use crate::response::FeroxResponse;
-use crate::url::FeroxUrl;
 use crate::{
+    response::FeroxResponse,
    scan_manager::{FeroxScan, FeroxScans, ScanOrder},
    scanner::FeroxScanner,
    statistics::StatField::TotalScans,
+    url::FeroxUrl,
+    utils::should_deny_url,
    CommandReceiver, CommandSender, FeroxChannel, Joiner, SLEEP_DURATION,
 };

 use super::command::Command::AddToUsizeField;
 use super::*;
+use reqwest::Url;
 use tokio::time::Duration;

 #[derive(Debug)]
@@ -54,7 +55,7 @@ pub struct ScanHandler {
    receiver: CommandReceiver,

    /// wordlist (re)used for each scan
-    wordlist: std::sync::Mutex<Option<Arc<HashSet<String>>>>,
+    wordlist: std::sync::Mutex<Option<Arc<Vec<String>>>>,

    /// group of scans that need to be joined
    tasks: Vec<Arc<FeroxScan>>,
@@ -105,7 +106,7 @@ impl ScanHandler {
    }

    /// Set the wordlist
-    fn wordlist(&self, wordlist: Arc<HashSet<String>>) {
+    fn wordlist(&self, wordlist: Arc<Vec<String>>) {
        if let Ok(mut guard) = self.wordlist.lock() {
            if guard.is_none() {
                let _ = std::mem::replace(&mut *guard, Some(wordlist));
@@ -175,7 +176,7 @@ impl ScanHandler {
    }

    /// Helper to easily get the (locked) underlying wordlist
-    pub fn get_wordlist(&self) -> Result<Arc<HashSet<String>>> {
+    pub fn get_wordlist(&self) -> Result<Arc<Vec<String>>> {
        if let Ok(guard) = self.wordlist.lock().as_ref() {
            if let Some(list) = guard.as_ref() {
                return Ok(list.clone());
@@ -188,6 +189,7 @@ impl ScanHandler {
    /// wrapper around scanning a url to stay DRY
    async fn ordered_scan_url(&mut self, targets: Vec<String>, order: ScanOrder) -> Result<()> {
        log::trace!("enter: ordered_scan_url({:?}, {:?})", targets, order);
+        let should_test_deny = !self.handles.config.url_denylist.is_empty();

        for target in targets {
            if self.data.contains(&target) && matches!(order, ScanOrder::Latest) {
@@ -204,6 +206,13 @@ impl ScanHandler {
                self.data.add_directory_scan(&target, order).1 // add the new target; return FeroxScan
            };

+            if should_test_deny && should_deny_url(&Url::parse(&target)?, self.handles.clone())? {
+                // response was caught by a user-provided deny list
+                // checking this last, since it's most susceptible to longer runtimes due to what
+                // input is received
+                continue;
+            }
+
            let list = self.get_wordlist()?;

            log::info!("scan handler received {} - beginning scan", target);
@@ -244,6 +253,11 @@ impl ScanHandler {
    async fn try_recursion(&mut self, response: Box<FeroxResponse>) -> Result<()> {
        log::trace!("enter: try_recursion({:?})", response,);

+        if !response.is_directory() {
+            // not a directory, quick exit
+            return Ok(());
+        }
+
        let mut base_depth = 1_usize;

        for (base_url, base_url_depth) in &self.depths {
@@ -257,11 +271,6 @@ impl ScanHandler {
            return Ok(());
        }

-        if !response.is_directory() {
-            // not a directory
-            return Ok(());
-        }
-
        let targets = vec![response.url().to_string()];
        self.ordered_scan_url(targets, ScanOrder::Latest).await?;

--- a/src/extractor/container.rs
+++ b/src/extractor/container.rs
@@ -1,4 +1,5 @@
 use super::*;
+use crate::utils::should_deny_url;
 use crate::{
    client,
    event_handlers::{
@@ -53,13 +54,19 @@ pub struct Extractor<'a> {

 /// Extractor implementation
 impl<'a> Extractor<'a> {
-    /// business logic that handles getting links from a normal http body response
-    pub async fn extract(&self) -> Result<()> {
-        let links = match self.target {
-            ExtractionTarget::ResponseBody => self.extract_from_body().await?,
-            ExtractionTarget::RobotsTxt => self.extract_from_robots().await?,
-        };
+    /// perform extraction from the given target and return any links found
+    pub async fn extract(&self) -> Result<HashSet<String>> {
+        log::trace!("enter: extract (this fn has associated trace exit msg)");
+        match self.target {
+            ExtractionTarget::ResponseBody => Ok(self.extract_from_body().await?),
+            ExtractionTarget::RobotsTxt => Ok(self.extract_from_robots().await?),
+        }
+    }

+    /// given a set of links from a normal http body response, task the request handler to make
+    /// the requests
+    pub async fn request_links(&self, links: HashSet<String>) -> Result<()> {
+        log::trace!("enter: request_links({:?})", links);
        let recursive = if self.handles.config.no_recursion {
            RecursionStatus::NotRecursive
        } else {
@@ -121,6 +128,7 @@ impl<'a> Extractor<'a> {
                rx.await?;
            }
        }
+        log::trace!("exit: request_links");
        Ok(())
    }

@@ -141,7 +149,7 @@ impl<'a> Extractor<'a> {

        let body = self.response.unwrap().text();

-        for capture in self.links_regex.captures_iter(&body) {
+        for capture in self.links_regex.captures_iter(body) {
            // remove single & double quotes from both ends of the capture
            // capture[0] is the entire match, additional capture groups start at [1]
            let link = capture[0].trim_matches(|c| c == '\'' || c == '"');
@@ -267,7 +275,7 @@ impl<'a> Extractor<'a> {
        };

        let new_url = old_url
-            .join(&link)
+            .join(link)
            .with_context(|| format!("Could not join {} with {}", old_url, link))?;

        links.insert(new_url.to_string());
@@ -289,10 +297,10 @@ impl<'a> Extractor<'a> {
    pub(super) async fn request_link(&self, url: &str) -> Result<FeroxResponse> {
        log::trace!("enter: request_link({})", url);

-        let ferox_url = FeroxUrl::from_string(&url, self.handles.clone());
+        let ferox_url = FeroxUrl::from_string(url, self.handles.clone());

        // create a url based on the given command line options
-        let new_url = ferox_url.format(&"", None)?;
+        let new_url = ferox_url.format("", None)?;

        let scanned_urls = self.handles.ferox_scans()?;

@@ -302,6 +310,17 @@ impl<'a> Extractor<'a> {
            bail!("previously seen url");
        }

+        if !self.handles.config.url_denylist.is_empty()
+            && should_deny_url(&new_url, self.handles.clone())?
+        {
+            // can't allow a denied url to be requested
+            bail!(
+                "prevented request to {} due to {:?}",
+                url,
+                self.handles.config.url_denylist
+            );
+        }
+
        // make the request and store the response
        let new_response = logged_request(&new_url, self.handles.clone()).await?;

@@ -332,7 +351,7 @@ impl<'a> Extractor<'a> {
            if let Some(new_path) = capture.name("url_path") {
                let mut new_url = Url::parse(&self.url)?;
                new_url.set_path(new_path.as_str());
-                if self.add_all_sub_paths(&new_url.path(), &mut links).is_err() {
+                if self.add_all_sub_paths(new_url.path(), &mut links).is_err() {
                    log::warn!("could not add sub-paths from {} to {:?}", new_url, links);
                }
            }
@@ -391,7 +410,7 @@ impl<'a> Extractor<'a> {
            FeroxResponse::from(response, true, self.handles.config.output_level).await;

        log::trace!("exit: get_robots_file -> {}", ferox_response);
-        return Ok(ferox_response);
+        Ok(ferox_response)
    }

    /// update total number of links extracted and expected responses
--- a/src/extractor/tests.rs
+++ b/src/extractor/tests.rs
@@ -54,8 +54,8 @@ fn setup_extractor(target: ExtractionTarget, scanned_urls: Arc<FeroxScans>) -> E
 /// in the expected array
 fn extractor_get_sub_paths_from_path_with_multiple_paths() {
    let path = "homepage/assets/img/icons/handshake.svg";
-    let r_paths = ROBOTS_EXT.get_sub_paths_from_path(&path);
-    let b_paths = BODY_EXT.get_sub_paths_from_path(&path);
+    let r_paths = ROBOTS_EXT.get_sub_paths_from_path(path);
+    let b_paths = BODY_EXT.get_sub_paths_from_path(path);
    let expected = vec![
        "homepage/",
        "homepage/assets/",
@@ -67,8 +67,8 @@ fn extractor_get_sub_paths_from_path_with_multiple_paths() {
    assert_eq!(r_paths.len(), expected.len());
    assert_eq!(b_paths.len(), expected.len());
    for expected_path in expected {
-        assert_eq!(r_paths.contains(&expected_path.to_string()), true);
-        assert_eq!(b_paths.contains(&expected_path.to_string()), true);
+        assert!(r_paths.contains(&expected_path.to_string()));
+        assert!(b_paths.contains(&expected_path.to_string()));
    }
 }

@@ -78,15 +78,15 @@ fn extractor_get_sub_paths_from_path_with_multiple_paths() {
 /// returned
 fn extractor_get_sub_paths_from_path_with_enclosing_slashes() {
    let path = "/homepage/assets/";
-    let r_paths = ROBOTS_EXT.get_sub_paths_from_path(&path);
-    let b_paths = BODY_EXT.get_sub_paths_from_path(&path);
+    let r_paths = ROBOTS_EXT.get_sub_paths_from_path(path);
+    let b_paths = BODY_EXT.get_sub_paths_from_path(path);
    let expected = vec!["homepage/", "homepage/assets"];

    assert_eq!(r_paths.len(), expected.len());
    assert_eq!(b_paths.len(), expected.len());
    for expected_path in expected {
-        assert_eq!(r_paths.contains(&expected_path.to_string()), true);
-        assert_eq!(b_paths.contains(&expected_path.to_string()), true);
+        assert!(r_paths.contains(&expected_path.to_string()));
+        assert!(b_paths.contains(&expected_path.to_string()));
    }
 }

@@ -95,15 +95,15 @@ fn extractor_get_sub_paths_from_path_with_enclosing_slashes() {
 /// included
 fn extractor_get_sub_paths_from_path_with_only_a_word() {
    let path = "homepage";
-    let r_paths = ROBOTS_EXT.get_sub_paths_from_path(&path);
-    let b_paths = BODY_EXT.get_sub_paths_from_path(&path);
+    let r_paths = ROBOTS_EXT.get_sub_paths_from_path(path);
+    let b_paths = BODY_EXT.get_sub_paths_from_path(path);
    let expected = vec!["homepage"];

    assert_eq!(r_paths.len(), expected.len());
    assert_eq!(b_paths.len(), expected.len());
    for expected_path in expected {
-        assert_eq!(r_paths.contains(&expected_path.to_string()), true);
-        assert_eq!(b_paths.contains(&expected_path.to_string()), true);
+        assert!(r_paths.contains(&expected_path.to_string()));
+        assert!(b_paths.contains(&expected_path.to_string()));
    }
 }

@@ -111,15 +111,15 @@ fn extractor_get_sub_paths_from_path_with_only_a_word() {
 /// extract sub paths from the given url fragment; expect 1 sub path, forward slash removed
 fn extractor_get_sub_paths_from_path_with_an_absolute_word() {
    let path = "/homepage";
-    let r_paths = ROBOTS_EXT.get_sub_paths_from_path(&path);
-    let b_paths = BODY_EXT.get_sub_paths_from_path(&path);
+    let r_paths = ROBOTS_EXT.get_sub_paths_from_path(path);
+    let b_paths = BODY_EXT.get_sub_paths_from_path(path);
    let expected = vec!["homepage"];

    assert_eq!(r_paths.len(), expected.len());
    assert_eq!(b_paths.len(), expected.len());
    for expected_path in expected {
-        assert_eq!(r_paths.contains(&expected_path.to_string()), true);
-        assert_eq!(b_paths.contains(&expected_path.to_string()), true);
+        assert!(r_paths.contains(&expected_path.to_string()));
+        assert!(b_paths.contains(&expected_path.to_string()));
    }
 }

--- a/src/filters/container.rs
+++ b/src/filters/container.rs
@@ -41,7 +41,7 @@ impl FeroxFilters {
        if let Ok(filters) = self.filters.lock() {
            for filter in filters.iter() {
                // wildcard.should_filter goes here
-                if filter.should_filter_response(&response) {
+                if filter.should_filter_response(response) {
                    if filter.as_any().downcast_ref::<WildcardFilter>().is_some() {
                        tx_stats
                            .send(AddToUsizeField(WildcardsFiltered, 1))
--- a/src/filters/init.rs
+++ b/src/filters/init.rs
@@ -56,7 +56,7 @@ pub async fn initialize(handles: Arc<Handles>) -> Result<()> {
    // add any regex filters to filters handler's FeroxFilters  (-X|--filter-regex)
    for regex_filter in &handles.config.filter_regex {
        let raw = regex_filter;
-        let compiled = skip_fail!(Regex::new(&raw));
+        let compiled = skip_fail!(Regex::new(raw));

        let filter = RegexFilter {
            raw_string: raw.to_owned(),
@@ -69,7 +69,7 @@ pub async fn initialize(handles: Arc<Handles>) -> Result<()> {
    // add any similarity filters to filters handler's FeroxFilters  (--filter-similar-to)
    for similarity_filter in &handles.config.filter_similar {
        // url as-is based on input, ignores user-specified url manipulation options (add-slash etc)
-        let url = skip_fail!(Url::parse(&similarity_filter));
+        let url = skip_fail!(Url::parse(similarity_filter));

        // attempt to request the given url
        let resp = skip_fail!(logged_request(&url, handles.clone()).await);
--- a/src/filters/wildcard.rs
+++ b/src/filters/wildcard.rs
@@ -76,7 +76,7 @@ impl FeroxFilter for WildcardFilter {
            // except that I don't want an empty string taking up the last index in the
            // event that the url ends with a forward slash.  It's ugly enough to be split
            // into its own function for readability.
-            let url_len = FeroxUrl::path_length_of_url(&response.url());
+            let url_len = FeroxUrl::path_length_of_url(response.url());

            if url_len + self.dynamic == response.content_length() {
                log::debug!("dynamic wildcard: filtered out {}", response.url());
--- a/src/heuristics.rs
+++ b/src/heuristics.rs
@@ -207,7 +207,7 @@ impl HeuristicTests {
        let mut good_urls = vec![];

        for target_url in target_urls {
-            let url = FeroxUrl::from_string(&target_url, self.handles.clone());
+            let url = FeroxUrl::from_string(target_url, self.handles.clone());
            let request = skip_fail!(url.format("", None));

            let result = logged_request(&request, self.handles.clone()).await;
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -43,7 +43,7 @@ pub(crate) type FeroxChannel<T> = (UnboundedSender<T>, UnboundedReceiver<T>);
 pub(crate) const VERSION: &str = env!("CARGO_PKG_VERSION");

 /// Maximum number of file descriptors that can be opened during a scan
-pub const DEFAULT_OPEN_FILE_LIMIT: usize = 8192;
+pub const DEFAULT_OPEN_FILE_LIMIT: u64 = 8192;

 /// Default value used to determine near-duplicate web pages (equivalent to 95%)
 pub const SIMILARITY_THRESHOLD: u32 = 95;
@@ -73,7 +73,8 @@ pub const HIGH_ERROR_RATIO: f64 = 0.90;
 /// * 401 Unauthorized
 /// * 403 Forbidden
 /// * 405 Method Not Allowed
-pub const DEFAULT_STATUS_CODES: [StatusCode; 9] = [
+/// * 500 Internal Server Error
+pub const DEFAULT_STATUS_CODES: [StatusCode; 10] = [
    StatusCode::OK,
    StatusCode::NO_CONTENT,
    StatusCode::MOVED_PERMANENTLY,
@@ -83,6 +84,7 @@ pub const DEFAULT_STATUS_CODES: [StatusCode; 9] = [
    StatusCode::UNAUTHORIZED,
    StatusCode::FORBIDDEN,
    StatusCode::METHOD_NOT_ALLOWED,
+    StatusCode::INTERNAL_SERVER_ERROR,
 ];

 /// Default filename for config file settings
--- a/src/main.rs
+++ b/src/main.rs
@@ -1,9 +1,9 @@
 use std::{
-    collections::HashSet,
    env::args,
-    fs::File,
+    fs::{create_dir, remove_file, File},
    io::{stderr, BufRead, BufReader},
    ops::Index,
+    path::Path,
    process::Command,
    sync::{atomic::Ordering, Arc},
 };
@@ -28,7 +28,7 @@ use feroxbuster::{
    progress::{PROGRESS_BAR, PROGRESS_PRINTER},
    scan_manager::{self},
    scanner,
-    utils::fmt_err,
+    utils::{fmt_err, slugify_filename},
 };
 #[cfg(not(target_os = "windows"))]
 use feroxbuster::{utils::set_open_file_limit, DEFAULT_OPEN_FILE_LIMIT};
@@ -41,14 +41,14 @@ lazy_static! {
 }

 /// Create a HashSet of Strings from the given wordlist then stores it inside an Arc
-fn get_unique_words_from_wordlist(path: &str) -> Result<Arc<HashSet<String>>> {
+fn get_unique_words_from_wordlist(path: &str) -> Result<Arc<Vec<String>>> {
    log::trace!("enter: get_unique_words_from_wordlist({})", path);

    let file = File::open(&path).with_context(|| format!("Could not open {}", path))?;

    let reader = BufReader::new(file);

-    let mut words = HashSet::new();
+    let mut words = Vec::new();

    for line in reader.lines() {
        let result = match line {
@@ -60,7 +60,7 @@ fn get_unique_words_from_wordlist(path: &str) -> Result<Arc<HashSet<String>>> {
            continue;
        }

-        words.insert(result);
+        words.push(result);
    }

    log::trace!(
@@ -78,11 +78,7 @@ async fn scan(targets: Vec<String>, handles: Arc<Handles>) -> Result<()> {
    // so that will allow for cheap/safe sharing of a single wordlist across multi-target scans
    // as well as additional directories found as part of recursion

-    let words = {
-        let words_handles = handles.clone();
-        tokio::spawn(async move { get_unique_words_from_wordlist(&words_handles.config.wordlist) })
-            .await??
-    };
+    let words = get_unique_words_from_wordlist(&handles.config.wordlist)?;

    if words.len() == 0 {
        bail!("Did not find any words in {}", handles.config.wordlist);
@@ -272,10 +268,56 @@ async fn wrapped_main(config: Arc<Configuration>) -> Result<()> {
        // from removing --parallel)
        original.remove(parallel_index);

+        // to log unique files to a shared folder, we need to first check for the presence
+        // of -o|--output.
+        let out_dir = if !config.output.is_empty() {
+            // -o|--output was used, so we'll attempt to create a directory to store the files
+            let output_path = Path::new(&handles.config.output);
+
+            // this only returns None if the path terminates in `..`. Since I don't want to
+            // hand-hold to that degree, we'll unwrap and fail if the output path ends in `..`
+            let base_name = output_path.file_name().unwrap();
+
+            let new_folder = slugify_filename(&base_name.to_string_lossy(), "", "logs");
+
+            let final_path = output_path.with_file_name(&new_folder);
+
+            // create the directory or fail silently, assuming the reason for failure is that
+            // the path exists already
+            create_dir(&final_path).unwrap_or(());
+
+            final_path.to_string_lossy().to_string()
+        } else {
+            String::new()
+        };
+
        // unvalidated targets fresh from stdin, just spawn children and let them do all checks
        for target in targets {
            // add the current target to the provided command
            let mut cloned = original.clone();
+
+            if !out_dir.is_empty() {
+                // output directory value is not empty, need to join output directory with
+                // unique scan filename
+
+                // unwrap is ok, we already know -o was used
+                let out_idx = original
+                    .iter()
+                    .position(|s| *s == "--output" || *s == "-o")
+                    .unwrap();
+
+                let filename = slugify_filename(&target, "ferox", "log");
+
+                let full_path = Path::new(&out_dir)
+                    .join(filename)
+                    .to_string_lossy()
+                    .to_string();
+
+                // a +1 to the index is fine here, as clap has already validated that
+                // -o|--output has a value associated with it
+                cloned[out_idx + 1] = full_path;
+            }
+
            cloned.push("-u".to_string());
            cloned.push(target);

@@ -299,8 +341,22 @@ async fn wrapped_main(config: Arc<Configuration>) -> Result<()> {
            });
        }

+        // the output handler creates an empty file to which it will try to write, because
+        // this happens before we enter the --parallel branch, we need to remove that file
+        // if it's empty
+        let output = handles.config.output.to_owned();
+
        clean_up(handles, tasks).await?;

+        let file = Path::new(&output);
+        if file.exists() {
+            // expectation is that this is always true for the first ferox process
+            if file.metadata()?.len() == 0 {
+                // empty file, attempt to remove it
+                remove_file(file)?;
+            }
+        }
+
        log::trace!("exit: parallel branch && wrapped main");
        return Ok(());
    }
--- a/src/parser.rs
+++ b/src/parser.rs
@@ -1,6 +1,8 @@
 use clap::{App, Arg, ArgGroup};
 use lazy_static::lazy_static;
 use regex::Regex;
+use std::env;
+use std::process;

 lazy_static! {
    /// Regex used to validate values passed to --time-limit
@@ -16,7 +18,7 @@ lazy_static! {

 /// Create and return an instance of [clap::App](https://docs.rs/clap/latest/clap/struct.App.html), i.e. the Command Line Interface's configuration
 pub fn initialize() -> App<'static, 'static> {
-    App::new("feroxbuster")
+    let mut app = App::new("feroxbuster")
        .version(env!("CARGO_PKG_VERSION"))
        .author("Ben 'epi' Risher (@epi052)")
        .about("A fast, simple, recursive content discovery tool written in Rust")
@@ -216,6 +218,17 @@ pub fn initialize() -> App<'static, 'static> {
                    "File extension(s) to search for (ex: -x php -x pdf js)",
                ),
        )
+        .arg(
+            Arg::with_name("url_denylist")
+                .long("dont-scan")
+                .value_name("URL")
+                .takes_value(true)
+                .multiple(true)
+                .use_delimiter(true)
+                .help(
+                    "URL(s) to exclude from recursion/scans",
+                ),
+        )
        .arg(
            Arg::with_name("headers")
                .short("H")
@@ -410,7 +423,20 @@ EXAMPLES:

    Ludicrous speed... go!
        ./feroxbuster -u http://127.1 -t 200
-    "#)
+    "#);
+
+    for arg in env::args() {
+        // secure-77 noticed that when an incorrect flag/option is used, the short help message is printed
+        // which is fine, but if you add -h|--help, it still errors out on the bad flag/option,
+        // never showing the full help message. This code addresses that behavior
+        if arg == "--help" || arg == "-h" {
+            app.print_long_help().unwrap();
+            println!(); // just a newline to mirror original --help output
+            process::exit(0);
+        }
+    }
+
+    app
 }

 /// Validate that a string is formatted as a number followed by s, m, h, or d (10d, 30s, etc...)
--- a/src/progress.rs
+++ b/src/progress.rs
@@ -51,7 +51,7 @@ pub fn add_bar(prefix: &str, length: u64, bar_type: BarType) -> ProgressBar {

    progress_bar.set_style(style);

-    progress_bar.set_prefix(&prefix);
+    progress_bar.set_prefix(prefix);

    progress_bar
 }
--- a/src/response.rs
+++ b/src/response.rs
@@ -121,7 +121,7 @@ impl FeroxResponse {

    /// Set `FeroxResponse`'s `url` attribute, has no affect if an error occurs
    pub fn set_url(&mut self, url: &str) {
-        match Url::parse(&url) {
+        match Url::parse(url) {
            Ok(url) => {
                self.url = url;
            }
@@ -339,7 +339,7 @@ impl FeroxSerialize for FeroxResponse {
                lines,
                words,
                chars,
-                status_colorizer(&status),
+                status_colorizer(status),
                self.url(),
                FeroxUrl::path_length_of_url(&self.url)
            );
--- a/src/scan_manager/menu.rs
+++ b/src/scan_manager/menu.rs
@@ -31,9 +31,10 @@ impl Menu {
        let separator = "─".to_string();

        let instructions = format!(
-            "Enter a {} list of indexes to {} (ex: 2,3)",
+            "Enter a {} list of indexes/ranges to {} ({}: 1-4,8,9-13)",
            style("comma-separated").yellow(),
            style("cancel").red(),
+            style("ex").cyan(),
        );

        let name = format!(
@@ -43,14 +44,22 @@ impl Menu {
            "💀"
        );

+        let force_msg = format!(
+            "Add {} to {} confirmation ({}: 3-5 -f)",
+            style("-f").yellow(),
+            style("skip").yellow(),
+            style("ex").cyan(),
+        );
+
        let longest = measure_text_width(&instructions).max(measure_text_width(&name));

        let border = separator.repeat(longest);

        let padded_name = pad_str(&name, longest, Alignment::Center, None);
+        let padded_force = pad_str(&force_msg, longest, Alignment::Center, None);

        let header = format!("{}\n{}\n{}", border, padded_name, border);
-        let footer = format!("{}\n{}\n{}", border, instructions, border);
+        let footer = format!("{}\n{}\n{}\n{}", border, instructions, padded_force, border);

        Self {
            separator,
@@ -93,23 +102,71 @@ impl Menu {
        self.term.write_line(msg).unwrap_or_default();
    }

-    /// split a string into vec of usizes
-    pub(super) fn split_to_nums(&self, line: &str) -> Vec<usize> {
-        line.split(',')
-            .map(|s| {
-                s.trim().to_string().parse::<usize>().unwrap_or_else(|e| {
-                    self.println(&format!("Found non-numeric input: {}", e));
-                    0
-                })
+    /// Helper for parsing a usize from a str
+    fn str_to_usize(&self, value: &str) -> usize {
+        if value.is_empty() {
+            return 0;
+        }
+
+        value
+            .trim()
+            .to_string()
+            .parse::<usize>()
+            .unwrap_or_else(|e| {
+                self.println(&format!("Found non-numeric input: {}: {:?}", e, value));
+                0
            })
-            .filter(|m| *m != 0)
-            .collect()
+    }
+
+    /// split a comma delimited string into vec of usizes
+    pub(super) fn split_to_nums(&self, line: &str) -> Vec<usize> {
+        let mut nums = Vec::new();
+        let values = line.split(',');
+
+        for mut value in values {
+            value = value.trim();
+
+            if value.contains('-') {
+                // range of two values, needs further processing
+
+                let range: Vec<usize> = value
+                    .split('-')
+                    .map(|s| self.str_to_usize(s))
+                    .filter(|m| *m != 0)
+                    .collect();
+
+                if range.len() != 2 {
+                    // expecting [1, 4] or similar, if a 0 was used, we'd be left with a vec of size 1
+                    self.println(&format!("Found invalid range of scans: {}", value));
+                    continue;
+                }
+
+                (range[0]..=range[1]).for_each(|n| {
+                    // iterate from lower to upper bound and add all interim values, skipping
+                    // any already known
+                    if !nums.contains(&n) {
+                        nums.push(n)
+                    }
+                });
+            } else {
+                let value = self.str_to_usize(value);
+
+                if value != 0 && !nums.contains(&value) {
+                    // the zeroth scan is always skipped, skip already known values
+                    nums.push(value);
+                }
+            }
+        }
+
+        nums
    }

    /// get comma-separated list of scan indexes from the user
-    pub(super) fn get_scans_from_user(&self) -> Option<Vec<usize>> {
+    pub(super) fn get_scans_from_user(&self) -> Option<(Vec<usize>, bool)> {
        if let Ok(line) = self.term.read_line() {
-            Some(self.split_to_nums(&line))
+            let force = line.contains("-f");
+            let line = line.replace("-f", "");
+            Some((self.split_to_nums(&line), force))
        } else {
            None
        }
--- a/src/scan_manager/scan.rs
+++ b/src/scan_manager/scan.rs
@@ -36,7 +36,7 @@ pub struct FeroxScan {
    pub(super) scan_type: ScanType,

    /// The order in which the scan was received
-    pub(super) scan_order: ScanOrder,
+    pub(crate) scan_order: ScanOrder,

    /// Number of requests to populate the progress bar with
    pub(super) num_requests: u64,
--- a/src/scan_manager/scan_container.rs
+++ b/src/scan_manager/scan_container.rs
@@ -252,7 +252,7 @@ impl FeroxScans {
    }

    /// Given a list of indexes, cancel their associated FeroxScans
-    async fn cancel_scans(&self, indexes: Vec<usize>) -> usize {
+    async fn cancel_scans(&self, indexes: Vec<usize>, force: bool) -> usize {
        let menu_pause_duration = Duration::from_millis(SLEEP_DURATION);

        let mut num_cancelled = 0_usize;
@@ -273,7 +273,11 @@ impl FeroxScans {
                Err(..) => continue,
            };

-            let input = self.menu.confirm_cancellation(&selected.url);
+            let input = if force {
+                'y'
+            } else {
+                self.menu.confirm_cancellation(&selected.url)
+            };

            if input == 'y' || input == '\n' {
                self.menu.println(&format!("Stopping {}...", selected.url));
@@ -305,8 +309,8 @@ impl FeroxScans {

        let mut num_cancelled = 0_usize;

-        if let Some(input) = self.menu.get_scans_from_user() {
-            num_cancelled += self.cancel_scans(input).await;
+        if let Some((input, force)) = self.menu.get_scans_from_user() {
+            num_cancelled += self.cancel_scans(input, force).await;
        };

        self.menu.clear_screen();
@@ -427,7 +431,7 @@ impl FeroxScans {
                    OutputLevel::Silent => BarType::Hidden,
                };

-                let progress_bar = add_bar(&url, bar_length, bar_type);
+                let progress_bar = add_bar(url, bar_length, bar_type);

                progress_bar.reset_elapsed();

@@ -437,7 +441,7 @@ impl FeroxScans {
        };

        let ferox_scan = FeroxScan::new(
-            &url,
+            url,
            scan_type,
            scan_order,
            bar_length,
@@ -458,7 +462,7 @@ impl FeroxScans {
    ///
    /// Also return a reference to the new `FeroxScan`
    pub fn add_directory_scan(&self, url: &str, scan_order: ScanOrder) -> (bool, Arc<FeroxScan>) {
-        self.add_scan(&url, ScanType::Directory, scan_order)
+        self.add_scan(url, ScanType::Directory, scan_order)
    }

    /// Given a url, create a new `FeroxScan` and add it to `FeroxScans` as a File Scan
@@ -467,7 +471,7 @@ impl FeroxScans {
    ///
    /// Also return a reference to the new `FeroxScan`
    pub fn add_file_scan(&self, url: &str, scan_order: ScanOrder) -> (bool, Arc<FeroxScan>) {
-        self.add_scan(&url, ScanType::File, scan_order)
+        self.add_scan(url, ScanType::File, scan_order)
    }

    /// small helper to determine whether any scans are active or not
--- a/src/scan_manager/state.rs
+++ b/src/scan_manager/state.rs
@@ -47,7 +47,7 @@ impl FeroxSerialize for FeroxState {

    /// Simple call to produce a JSON string using the given FeroxState
    fn as_json(&self) -> Result<String> {
-        Ok(serde_json::to_string(&self)
-            .with_context(|| fmt_err("Could not convert scan's running state to JSON"))?)
+        serde_json::to_string(&self)
+            .with_context(|| fmt_err("Could not convert scan's running state to JSON"))
    }
 }
--- a/src/scan_manager/tests.rs
+++ b/src/scan_manager/tests.rs
@@ -52,7 +52,7 @@ fn add_url_to_list_of_scanned_urls_with_unknown_url() {
    let urls = FeroxScans::default();
    let url = "http://unknown_url";
    let (result, _scan) = urls.add_scan(url, ScanType::Directory, ScanOrder::Latest);
-    assert_eq!(result, true);
+    assert!(result);
 }

 #[test]
@@ -71,11 +71,11 @@ fn add_url_to_list_of_scanned_urls_with_known_url() {
        Some(pb),
    );

-    assert_eq!(urls.insert(scan), true);
+    assert!(urls.insert(scan));

    let (result, _scan) = urls.add_scan(url, ScanType::Directory, ScanOrder::Latest);

-    assert_eq!(result, false);
+    assert!(!result);
 }

 #[test]
@@ -93,27 +93,23 @@ fn stop_progress_bar_stops_bar() {
        Some(pb),
    );

-    assert_eq!(
-        scan.progress_bar
-            .lock()
-            .unwrap()
-            .as_ref()
-            .unwrap()
-            .is_finished(),
-        false
-    );
+    assert!(!scan
+        .progress_bar
+        .lock()
+        .unwrap()
+        .as_ref()
+        .unwrap()
+        .is_finished());

    scan.stop_progress_bar();

-    assert_eq!(
-        scan.progress_bar
-            .lock()
-            .unwrap()
-            .as_ref()
-            .unwrap()
-            .is_finished(),
-        true
-    );
+    assert!(scan
+        .progress_bar
+        .lock()
+        .unwrap()
+        .as_ref()
+        .unwrap()
+        .is_finished());
 }

 #[test]
@@ -131,11 +127,11 @@ fn add_url_to_list_of_scanned_urls_with_known_url_without_slash() {
        None,
    );

-    assert_eq!(urls.insert(scan), true);
+    assert!(urls.insert(scan));

    let (result, _scan) = urls.add_scan(url, ScanType::File, ScanOrder::Latest);

-    assert_eq!(result, false);
+    assert!(!result);
 }

 #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
@@ -171,8 +167,8 @@ async fn call_display_scans() {
        .await
        .unwrap();

-    assert_eq!(urls.insert(scan), true);
-    assert_eq!(urls.insert(scan_two), true);
+    assert!(urls.insert(scan));
+    assert!(urls.insert(scan_two));

    urls.display_scans().await;
 }
@@ -330,7 +326,7 @@ fn ferox_response_serialize_and_deserialize() {

    assert_eq!(response.url().as_str(), "https://nerdcore.com/css");
    assert_eq!(response.url().path(), "/css");
-    assert_eq!(response.wildcard(), true);
+    assert!(response.wildcard());
    assert_eq!(response.status().as_u16(), 301);
    assert_eq!(response.content_length(), 173);
    assert_eq!(response.line_count(), 10);
@@ -383,7 +379,7 @@ fn feroxstates_feroxserialize_implementation() {

    let json_state = ferox_state.as_json().unwrap();
    let expected = format!(
-        r#"{{"scans":[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","status":"NotStarted","num_requests":0}}],"config":{{"type":"configuration","wordlist":"/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt","config":"","proxy":"","replay_proxy":"","target_url":"","status_codes":[200,204,301,302,307,308,401,403,405],"replay_codes":[200,204,301,302,307,308,401,403,405],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"silent":false,"quiet":false,"auto_bail":false,"auto_tune":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/{}","redirects":false,"insecure":false,"extensions":[],"headers":{{}},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":false,"depth":4,"scan_limit":0,"parallel":0,"rate_limit":0,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false,"resumed":false,"resume_from":"","save_state":false,"time_limit":"","filter_similar":[]}},"responses":[{{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{{"server":"nginx/1.16.1"}}}}]"#,
+        r#"{{"scans":[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","status":"NotStarted","num_requests":0}}],"config":{{"type":"configuration","wordlist":"/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt","config":"","proxy":"","replay_proxy":"","target_url":"","status_codes":[200,204,301,302,307,308,401,403,405,500],"replay_codes":[200,204,301,302,307,308,401,403,405,500],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"silent":false,"quiet":false,"auto_bail":false,"auto_tune":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/{}","redirects":false,"insecure":false,"extensions":[],"headers":{{}},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":false,"depth":4,"scan_limit":0,"parallel":0,"rate_limit":0,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false,"resumed":false,"resume_from":"","save_state":false,"time_limit":"","filter_similar":[],"url_denylist":[]}},"responses":[{{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{{"server":"nginx/1.16.1"}}}}]"#,
        saved_id, VERSION
    );
    println!("{}\n{}", expected, json_state);
@@ -521,9 +517,13 @@ fn menu_print_header_and_footer() {
 fn split_to_nums_is_correct() {
    let menu = Menu::new();

-    let nums = menu.split_to_nums("1, 3,      4");
+    let nums = menu.split_to_nums("1, 3,      4, 7 -     12, 10-10, 10-11, 9-12, 12-6, -1, 4-");

-    assert_eq!(nums, vec![1, 3, 4]);
+    assert_eq!(nums, vec![1, 3, 4, 7, 8, 9, 10, 11, 12]);
+    assert_eq!(menu.split_to_nums("9-12"), vec![9, 10, 11, 12]);
+    assert!(menu.split_to_nums("-12").is_empty());
+    assert!(menu.split_to_nums("12-").is_empty());
+    assert!(menu.split_to_nums("\n").is_empty());
 }

 #[test]
--- a/src/scan_manager/utils.rs
+++ b/src/scan_manager/utils.rs
@@ -41,7 +41,7 @@ pub async fn start_max_time_thread(handles: Arc<Handles>) {
        log::trace!("exit: start_max_time_thread");

        #[cfg(test)]
-        panic!(handles);
+        panic!("{:?}", handles);
        #[cfg(not(test))]
        let _ = TermInputHandler::sigint_handler(handles.clone());
    }
--- a/src/scanner/ferox_scanner.rs
+++ b/src/scanner/ferox_scanner.rs
@@ -1,4 +1,4 @@
-use std::{collections::HashSet, ops::Deref, sync::atomic::Ordering, sync::Arc, time::Instant};
+use std::{ops::Deref, sync::atomic::Ordering, sync::Arc, time::Instant};

 use anyhow::{bail, Result};
 use futures::{stream, StreamExt};
@@ -40,7 +40,7 @@ pub struct FeroxScanner {
    order: ScanOrder,

    /// wordlist that's already been read from disk
-    wordlist: Arc<HashSet<String>>,
+    wordlist: Arc<Vec<String>>,

    /// limiter that restricts the number of active FeroxScanners  
    scan_limiter: Arc<Semaphore>,
@@ -52,7 +52,7 @@ impl FeroxScanner {
    pub fn new(
        target_url: &str,
        order: ScanOrder,
-        wordlist: Arc<HashSet<String>>,
+        wordlist: Arc<Vec<String>>,
        scan_limiter: Arc<Semaphore>,
        handles: Arc<Handles>,
    ) -> Self {
@@ -83,7 +83,8 @@ impl FeroxScanner {
                .target(RobotsTxt)
                .build()?;

-            let _ = extractor.extract().await;
+            let links = extractor.extract().await?;
+            extractor.request_links(links).await?;
        }

        let scanned_urls = self.handles.ferox_scans()?;
--- a/src/scanner/policy_data.rs
+++ b/src/scanner/policy_data.rs
@@ -90,11 +90,9 @@ impl PolicyData {
                        atomic_store!(self.remove_limit, true);
                    }
                }
-                self.set_limit(heap.value() as usize);
            } else if heap.has_children() {
                // streak not at 3, just check that we can move down, and do so
                heap.move_left();
-                self.set_limit(heap.value() as usize);
            } else {
                // tree bottomed out, need to move back up the tree a bit
                let current = heap.value();
@@ -104,9 +102,8 @@ impl PolicyData {
                if current > heap.value() {
                    heap.move_up();
                }
-
-                self.set_limit(heap.value() as usize);
            }
+            self.set_limit(heap.value() as usize);
        }
    }

@@ -200,7 +197,7 @@ mod tests {
        pd.adjust_up(&3);
        assert_eq!(pd.heap.read().unwrap().value(), 300);
        assert_eq!(pd.limit.load(Ordering::Relaxed), 300);
-        assert_eq!(pd.remove_limit.load(Ordering::Relaxed), false);
+        assert!(!pd.remove_limit.load(Ordering::Relaxed));
    }

    #[test]
@@ -217,7 +214,7 @@ mod tests {
        pd.adjust_up(&3);
        assert_eq!(pd.heap.read().unwrap().value(), 200);
        assert_eq!(pd.limit.load(Ordering::Relaxed), 200);
-        assert_eq!(pd.remove_limit.load(Ordering::Relaxed), true);
+        assert!(pd.remove_limit.load(Ordering::Relaxed));
    }

    #[test]
@@ -234,7 +231,7 @@ mod tests {
        pd.adjust_up(&3);
        assert_eq!(pd.heap.read().unwrap().value(), 350);
        assert_eq!(pd.limit.load(Ordering::Relaxed), 350);
-        assert_eq!(pd.remove_limit.load(Ordering::Relaxed), false);
+        assert!(!pd.remove_limit.load(Ordering::Relaxed));
    }

    #[test]
@@ -251,7 +248,7 @@ mod tests {
        pd.adjust_up(&3);
        assert_eq!(pd.heap.read().unwrap().value(), 300);
        assert_eq!(pd.limit.load(Ordering::Relaxed), 300);
-        assert_eq!(pd.remove_limit.load(Ordering::Relaxed), false);
+        assert!(!pd.remove_limit.load(Ordering::Relaxed));
    }

    #[test]
@@ -269,7 +266,7 @@ mod tests {
        pd.adjust_up(&0);
        assert_eq!(pd.heap.read().unwrap().value(), 43);
        assert_eq!(pd.limit.load(Ordering::Relaxed), 43);
-        assert_eq!(pd.remove_limit.load(Ordering::Relaxed), false);
+        assert!(!pd.remove_limit.load(Ordering::Relaxed));
    }

    #[test]
@@ -287,7 +284,7 @@ mod tests {
        pd.adjust_up(&0);
        assert_eq!(pd.heap.read().unwrap().value(), 37);
        assert_eq!(pd.limit.load(Ordering::Relaxed), 37);
-        assert_eq!(pd.remove_limit.load(Ordering::Relaxed), false);
+        assert!(!pd.remove_limit.load(Ordering::Relaxed));
    }

    #[test]
--- a/src/scanner/requester.rs
+++ b/src/scanner/requester.rs
@@ -27,6 +27,8 @@ use crate::{
 };

 use super::{policy_data::PolicyData, FeroxScanner, PolicyTrigger};
+use crate::utils::should_deny_url;
+use std::collections::HashSet;

 /// Makes multiple requests based on the presence of extensions
 pub(super) struct Requester {
@@ -45,11 +47,16 @@ pub(super) struct Requester {
    /// FeroxScan associated with the creation of this Requester
    ferox_scan: Arc<FeroxScan>,

+    /// cache of previously seen links gotten via link extraction. since the requester is passed
+    /// around as an arc, and seen_links needs to be mutable, putting it behind a lock for
+    /// interior mutability, similar to the tuning_lock below
+    seen_links: RwLock<HashSet<String>>,
+
    /// simple lock to control access to tuning to a single thread (per-scan)
    ///
    /// need a usize to determine the number of consecutive non-error calls that a requester has
    /// seen; this will satisfy the non-mut self constraint (due to us being behind an Arc, and
-    /// the need for a counter
+    /// the need for a counter)
    tuning_lock: Mutex<usize>,
 }

@@ -73,6 +80,7 @@ impl Requester {
        Ok(Self {
            ferox_scan,
            policy_data,
+            seen_links: RwLock::new(HashSet::<String>::new()),
            rate_limiter: RwLock::new(rate_limiter),
            handles: scanner.handles.clone(),
            target_url: scanner.target_url.to_owned(),
@@ -298,6 +306,8 @@ impl Requester {
        let urls =
            FeroxUrl::from_string(&self.target_url, self.handles.clone()).formatted_urls(word)?;

+        let should_test_deny = !self.handles.config.url_denylist.is_empty();
+
        for url in urls {
            // auto_tune is true, or rate_limit was set (mutually exclusive to user)
            // and a rate_limiter has been created
@@ -313,6 +323,11 @@ impl Requester {
                }
            }

+            if should_test_deny && should_deny_url(&url, self.handles.clone())? {
+                // can't allow a denied url to be requested
+                continue;
+            }
+
            let response = logged_request(&url, self.handles.clone()).await?;

            if (should_tune || self.handles.config.auto_bail)
@@ -367,7 +382,26 @@ impl Requester {
                    .handles(self.handles.clone())
                    .build()?;

-                extractor.extract().await?;
+                let new_links: HashSet<_>;
+                let extracted = extractor.extract().await?;
+
+                {
+                    // gain and quickly drop the read lock on seen_links, using it while unlocked
+                    // to determine if there are any new links to process
+                    let read_links = self.seen_links.read().await;
+                    new_links = extracted.difference(&read_links).cloned().collect();
+                }
+
+                if !new_links.is_empty() {
+                    // using is_empty instead of direct iteration to acquire the write lock behind
+                    // some kind of less expensive gate (and not in a loop, obv)
+                    let mut write_links = self.seen_links.write().await;
+                    for new_link in &new_links {
+                        write_links.insert(new_link.to_owned());
+                    }
+                }
+
+                extractor.request_links(new_links).await?;
            }

            // everything else should be reported
@@ -536,6 +570,7 @@ mod tests {

        let requester = Requester {
            handles,
+            seen_links: RwLock::new(HashSet::<String>::new()),
            tuning_lock: Mutex::new(0),
            ferox_scan: Arc::new(FeroxScan::default()),
            target_url: "http://localhost".to_string(),
@@ -563,6 +598,7 @@ mod tests {

        let requester = Requester {
            handles,
+            seen_links: RwLock::new(HashSet::<String>::new()),
            tuning_lock: Mutex::new(0),
            ferox_scan: ferox_scan.clone(),
            target_url: "http://localhost".to_string(),
@@ -587,6 +623,7 @@ mod tests {

        let requester = Requester {
            handles,
+            seen_links: RwLock::new(HashSet::<String>::new()),
            tuning_lock: Mutex::new(0),
            ferox_scan: ferox_scan.clone(),
            target_url: "http://localhost".to_string(),
@@ -626,6 +663,7 @@ mod tests {

        let requester = Requester {
            handles,
+            seen_links: RwLock::new(HashSet::<String>::new()),
            tuning_lock: Mutex::new(0),
            ferox_scan: ferox_scan.clone(),
            target_url: "http://localhost".to_string(),
@@ -680,6 +718,7 @@ mod tests {
        let req_clone = scan_two.clone();
        let requester = Requester {
            handles,
+            seen_links: RwLock::new(HashSet::<String>::new()),
            tuning_lock: Mutex::new(0),
            ferox_scan: req_clone,
            target_url: "http://one/one/stuff.php".to_string(),
@@ -713,6 +752,7 @@ mod tests {

        let requester = Requester {
            handles,
+            seen_links: RwLock::new(HashSet::<String>::new()),
            tuning_lock: Mutex::new(0),
            ferox_scan: Arc::new(FeroxScan::default()),
            target_url: "http://one/one/stuff.php".to_string(),
@@ -734,6 +774,7 @@ mod tests {

        let requester = Requester {
            handles,
+            seen_links: RwLock::new(HashSet::<String>::new()),
            tuning_lock: Mutex::new(0),
            ferox_scan: Arc::new(FeroxScan::default()),
            target_url: "http://localhost".to_string(),
@@ -756,6 +797,7 @@ mod tests {

        let requester = Arc::new(Requester {
            handles,
+            seen_links: RwLock::new(HashSet::<String>::new()),
            tuning_lock: Mutex::new(0),
            ferox_scan: Arc::new(FeroxScan::default()),
            target_url: "http://localhost".to_string(),
@@ -772,7 +814,7 @@ mod tests {

        requester.cool_down().await;

-        assert_eq!(resp.await.unwrap(), true);
+        assert!(resp.await.unwrap());
        println!("{}", start.elapsed().as_millis());
        assert!(start.elapsed().as_millis() >= 3500);
    }
@@ -785,6 +827,7 @@ mod tests {

        let requester = Requester {
            handles,
+            seen_links: RwLock::new(HashSet::<String>::new()),
            tuning_lock: Mutex::new(0),
            ferox_scan: Arc::new(FeroxScan::default()),
            target_url: "http://localhost".to_string(),
@@ -822,6 +865,7 @@ mod tests {

        let requester = Requester {
            handles,
+            seen_links: RwLock::new(HashSet::<String>::new()),
            tuning_lock: Mutex::new(0),
            ferox_scan: Arc::new(scan),
            target_url: "http://localhost".to_string(),
@@ -857,6 +901,7 @@ mod tests {

        let requester = Requester {
            handles,
+            seen_links: RwLock::new(HashSet::<String>::new()),
            tuning_lock: Mutex::new(0),
            ferox_scan: Arc::new(scan),
            target_url: "http://localhost".to_string(),
@@ -884,6 +929,7 @@ mod tests {

        let mut requester = Requester {
            handles,
+            seen_links: RwLock::new(HashSet::<String>::new()),
            tuning_lock: Mutex::new(0),
            ferox_scan: Arc::new(FeroxScan::default()),
            target_url: "http://localhost".to_string(),
@@ -891,28 +937,16 @@ mod tests {
            policy_data: PolicyData::new(RequesterPolicy::AutoBail, 7),
        };

-        assert_eq!(
-            requester.too_many_status_errors(PolicyTrigger::Errors),
-            false
-        );
+        assert!(!requester.too_many_status_errors(PolicyTrigger::Errors));

-        assert_eq!(
-            requester.too_many_status_errors(PolicyTrigger::Status429),
-            false
-        );
+        assert!(!requester.too_many_status_errors(PolicyTrigger::Status429));
        requester.ferox_scan.progress_bar().set_position(10);
        requester.ferox_scan.add_429();
        requester.ferox_scan.add_429();
        requester.ferox_scan.add_429();
-        assert_eq!(
-            requester.too_many_status_errors(PolicyTrigger::Status429),
-            true
-        );
+        assert!(requester.too_many_status_errors(PolicyTrigger::Status429));

-        assert_eq!(
-            requester.too_many_status_errors(PolicyTrigger::Status403),
-            false
-        );
+        assert!(!requester.too_many_status_errors(PolicyTrigger::Status403));
        requester.ferox_scan = Arc::new(FeroxScan::default());
        requester.ferox_scan.progress_bar().set_position(10);
        requester.ferox_scan.add_403();
@@ -924,10 +958,7 @@ mod tests {
        requester.ferox_scan.add_403();
        requester.ferox_scan.add_403();
        requester.ferox_scan.add_403();
-        assert_eq!(
-            requester.too_many_status_errors(PolicyTrigger::Status403),
-            true
-        );
+        assert!(requester.too_many_status_errors(PolicyTrigger::Status403));
    }

    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
@@ -941,6 +972,7 @@ mod tests {

        let requester = Requester {
            handles,
+            seen_links: RwLock::new(HashSet::<String>::new()),
            tuning_lock: Mutex::new(0),
            ferox_scan: Arc::new(FeroxScan::default()),
            target_url: "http://localhost".to_string(),
@@ -983,6 +1015,7 @@ mod tests {

        let requester = Requester {
            handles,
+            seen_links: RwLock::new(HashSet::<String>::new()),
            tuning_lock: Mutex::new(0),
            ferox_scan: scan.clone(),
            target_url: "http://localhost".to_string(),
--- a/src/statistics/container.rs
+++ b/src/statistics/container.rs
@@ -1,4 +1,6 @@
 use std::{
+    collections::HashMap,
+    convert::TryFrom,
    fs::File,
    io::BufReader,
    sync::{
@@ -9,7 +11,8 @@ use std::{

 use anyhow::{Context, Result};
 use reqwest::StatusCode;
-use serde::{Deserialize, Serialize};
+use serde::{ser::SerializeStruct, Deserialize, Deserializer, Serialize, Serializer};
+use serde_json::Value;

 use crate::{
    traits::FeroxSerialize,
@@ -19,9 +22,8 @@ use crate::{
 use super::{error::StatError, field::StatField};

 /// Data collection of statistics related to a scan
-#[derive(Default, Deserialize, Debug, Serialize)]
+#[derive(Default, Debug)]
 pub struct Stats {
-    #[serde(rename = "type")]
    /// Name of this type of struct, used for serialization, i.e. `{"type":"statistics"}`
    kind: String,

@@ -125,11 +127,9 @@ pub struct Stats {
    total_runtime: Mutex<Vec<f64>>,

    /// tracker for the number of extensions the user specified
-    #[serde(skip)]
    num_extensions: usize,

    /// tracker for whether to use json during serialization or not
-    #[serde(skip)]
    json: bool,
 }

@@ -147,6 +147,301 @@ impl FeroxSerialize for Stats {
    }
 }

+/// Serialize implementation for Stats
+impl Serialize for Stats {
+    /// Function that handles serialization of Stats
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let mut state = serializer.serialize_struct("Stats", 32)?;
+
+        state.serialize_field("type", &self.kind)?;
+        state.serialize_field("timeouts", &atomic_load!(self.timeouts))?;
+        state.serialize_field("requests", &atomic_load!(self.requests))?;
+        state.serialize_field("expected_per_scan", &atomic_load!(self.expected_per_scan))?;
+        state.serialize_field("total_expected", &atomic_load!(self.total_expected))?;
+        state.serialize_field("errors", &atomic_load!(self.errors))?;
+        state.serialize_field("successes", &atomic_load!(self.successes))?;
+        state.serialize_field("redirects", &atomic_load!(self.redirects))?;
+        state.serialize_field("client_errors", &atomic_load!(self.client_errors))?;
+        state.serialize_field("server_errors", &atomic_load!(self.server_errors))?;
+        state.serialize_field("total_scans", &atomic_load!(self.total_scans))?;
+        state.serialize_field("initial_targets", &atomic_load!(self.initial_targets))?;
+        state.serialize_field("links_extracted", &atomic_load!(self.links_extracted))?;
+        state.serialize_field("status_200s", &atomic_load!(self.status_200s))?;
+        state.serialize_field("status_301s", &atomic_load!(self.status_301s))?;
+        state.serialize_field("status_302s", &atomic_load!(self.status_302s))?;
+        state.serialize_field("status_401s", &atomic_load!(self.status_401s))?;
+        state.serialize_field("status_403s", &atomic_load!(self.status_403s))?;
+        state.serialize_field("status_429s", &atomic_load!(self.status_429s))?;
+        state.serialize_field("status_500s", &atomic_load!(self.status_500s))?;
+        state.serialize_field("status_503s", &atomic_load!(self.status_503s))?;
+        state.serialize_field("status_504s", &atomic_load!(self.status_504s))?;
+        state.serialize_field("status_508s", &atomic_load!(self.status_508s))?;
+        state.serialize_field("wildcards_filtered", &atomic_load!(self.wildcards_filtered))?;
+        state.serialize_field("responses_filtered", &atomic_load!(self.responses_filtered))?;
+        state.serialize_field(
+            "resources_discovered",
+            &atomic_load!(self.resources_discovered),
+        )?;
+        state.serialize_field("url_format_errors", &atomic_load!(self.url_format_errors))?;
+        state.serialize_field("redirection_errors", &atomic_load!(self.redirection_errors))?;
+        state.serialize_field("connection_errors", &atomic_load!(self.connection_errors))?;
+        state.serialize_field("request_errors", &atomic_load!(self.request_errors))?;
+        state.serialize_field("directory_scan_times", &self.directory_scan_times)?;
+        state.serialize_field("total_runtime", &self.total_runtime)?;
+
+        state.end()
+    }
+}
+
+/// Deserialize implementation for Stats
+impl<'a> Deserialize<'a> for Stats {
+    /// Deserialize a Stats object from a serde_json::Value
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'a>,
+    {
+        let stats = Self::new(0, false);
+
+        let map: HashMap<String, Value> = HashMap::deserialize(deserializer)?;
+
+        for (key, value) in &map {
+            match key.as_str() {
+                "timeouts" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.timeouts, parsed);
+                        }
+                    }
+                }
+                "requests" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.requests, parsed);
+                        }
+                    }
+                }
+                "expected_per_scan" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.expected_per_scan, parsed);
+                        }
+                    }
+                }
+                "total_expected" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.total_expected, parsed);
+                        }
+                    }
+                }
+                "errors" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.errors, parsed);
+                        }
+                    }
+                }
+                "successes" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.successes, parsed);
+                        }
+                    }
+                }
+                "redirects" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.redirects, parsed);
+                        }
+                    }
+                }
+                "client_errors" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.client_errors, parsed);
+                        }
+                    }
+                }
+                "server_errors" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.server_errors, parsed);
+                        }
+                    }
+                }
+                "total_scans" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.total_scans, parsed);
+                        }
+                    }
+                }
+                "initial_targets" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.initial_targets, parsed);
+                        }
+                    }
+                }
+                "links_extracted" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.links_extracted, parsed);
+                        }
+                    }
+                }
+                "status_200s" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.status_200s, parsed);
+                        }
+                    }
+                }
+                "status_301s" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.status_301s, parsed);
+                        }
+                    }
+                }
+                "status_302s" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.status_302s, parsed);
+                        }
+                    }
+                }
+                "status_401s" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.status_401s, parsed);
+                        }
+                    }
+                }
+                "status_403s" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.status_403s, parsed);
+                        }
+                    }
+                }
+                "status_429s" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.status_429s, parsed);
+                        }
+                    }
+                }
+                "status_500s" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.status_500s, parsed);
+                        }
+                    }
+                }
+                "status_503s" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.status_503s, parsed);
+                        }
+                    }
+                }
+                "status_504s" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.status_504s, parsed);
+                        }
+                    }
+                }
+                "status_508s" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.status_508s, parsed);
+                        }
+                    }
+                }
+                "wildcards_filtered" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.wildcards_filtered, parsed);
+                        }
+                    }
+                }
+                "responses_filtered" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.responses_filtered, parsed);
+                        }
+                    }
+                }
+                "resources_discovered" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.resources_discovered, parsed);
+                        }
+                    }
+                }
+                "url_format_errors" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.url_format_errors, parsed);
+                        }
+                    }
+                }
+                "redirection_errors" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.redirection_errors, parsed);
+                        }
+                    }
+                }
+                "connection_errors" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.connection_errors, parsed);
+                        }
+                    }
+                }
+                "request_errors" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(parsed) = usize::try_from(num) {
+                            atomic_increment!(stats.request_errors, parsed);
+                        }
+                    }
+                }
+                "directory_scan_times" => {
+                    if let Some(arr) = value.as_array() {
+                        for val in arr {
+                            if let Some(parsed) = val.as_f64() {
+                                if let Ok(mut guard) = stats.directory_scan_times.lock() {
+                                    guard.push(parsed)
+                                }
+                            }
+                        }
+                    }
+                }
+                "total_runtime" => {
+                    if let Some(arr) = value.as_array() {
+                        for val in arr {
+                            if let Some(parsed) = val.as_f64() {
+                                if let Ok(mut guard) = stats.total_runtime.lock() {
+                                    guard.push(parsed)
+                                }
+                            }
+                        }
+                    }
+                }
+                _ => {}
+            }
+        }
+
+        Ok(stats)
+    }
+}
+
 /// implementation of statistics data collection struct
 impl Stats {
    /// Small wrapper for default to set `kind` to "statistics" and `total_runtime` to have at least
--- a/src/statistics/tests.rs
+++ b/src/statistics/tests.rs
@@ -55,10 +55,7 @@ fn save_writes_stats_object_to_disk() {
    stats.add_status_code(StatusCode::OK);
    stats.add_status_code(StatusCode::OK);
    let outfile = NamedTempFile::new().unwrap();
-    if stats
-        .save(174.33, &outfile.path().to_str().unwrap())
-        .is_ok()
-    {}
+    if stats.save(174.33, outfile.path().to_str().unwrap()).is_ok() {}

    assert!(stats.as_json().unwrap().contains("statistics"));
    assert!(stats.as_json().unwrap().contains("11")); // requests made
--- a/src/url.rs
+++ b/src/url.rs
@@ -66,7 +66,7 @@ impl FeroxUrl {
    pub fn format(&self, word: &str, extension: Option<&str>) -> Result<Url> {
        log::trace!("enter: format({}, {:?})", word, extension);

-        if Url::parse(&word).is_ok() {
+        if Url::parse(word).is_ok() {
            // when a full url is passed in as a word to be joined to a base url using
            // reqwest::Url::join, the result is that the word (url) completely overwrites the base
            // url, potentially resulting in requests to places that aren't actually the target
--- a/src/utils.rs
+++ b/src/utils.rs
@@ -3,11 +3,13 @@ use console::{strip_ansi_codes, style, user_attended};
 use indicatif::ProgressBar;
 use reqwest::{Client, Response, StatusCode, Url};
 #[cfg(not(target_os = "windows"))]
-use rlimit::{getrlimit, setrlimit, Resource, Rlim};
+use rlimit::{getrlimit, setrlimit, Resource};
 use std::{
    fs,
    io::{self, BufWriter, Write},
    sync::Arc,
+    time::Duration,
+    time::{SystemTime, UNIX_EPOCH},
 };
 use tokio::sync::mpsc::UnboundedSender;

@@ -217,16 +219,15 @@ pub fn create_report_string(
 /// as the adjustment made here is only valid for the scan itself (and any child processes, of which
 /// there are none).
 #[cfg(not(target_os = "windows"))]
-pub fn set_open_file_limit(limit: usize) -> bool {
+pub fn set_open_file_limit(limit: u64) -> bool {
    log::trace!("enter: set_open_file_limit");

    if let Ok((soft, hard)) = getrlimit(Resource::NOFILE) {
-        if hard.as_usize() > limit {
+        if hard > limit {
            // our default open file limit is less than the current hard limit, this means we can
            // set the soft limit to our default
-            let new_soft_limit = Rlim::from_usize(limit);

-            if setrlimit(Resource::NOFILE, new_soft_limit, hard).is_ok() {
+            if setrlimit(Resource::NOFILE, limit, hard).is_ok() {
                log::debug!("set open file descriptor limit to {}", limit);

                log::trace!("exit: set_open_file_limit -> {}", true);
@@ -287,15 +288,141 @@ where
    Ok(())
 }

+/// determines whether or not a given url should be denied based on the user-supplied --dont-scan
+/// flag
+pub fn should_deny_url(url: &Url, handles: Arc<Handles>) -> Result<bool> {
+    log::trace!(
+        "enter: should_deny_url({}, {:?}, {:?})",
+        url.as_str(),
+        handles.config.url_denylist,
+        handles.ferox_scans()?
+    );
+    // normalization for comparison is to remove the trailing / if one exists, this is done for
+    // the given url and any url to which it's compared
+    let normed_url = Url::parse(url.to_string().trim_end_matches('/'))?;
+
+    for deny_url in &handles.config.url_denylist {
+        // parse the denying url for easier comparison
+        let denier = Url::parse(deny_url.trim_end_matches('/'))
+            .with_context(|| format!("Could not parse {} as a url", deny_url))?;
+
+        // simplest case is an exact match, check for it first
+        if normed_url == denier {
+            log::trace!("exit: should_deny_url -> true");
+            return Ok(true);
+        }
+
+        match (normed_url.host(), denier.host()) {
+            // .host() will return an enum with ipv4|6 or domain and is comparable
+            // whereas .domain() returns None for ip addresses
+            (Some(normed_host), Some(denier_host)) => {
+                if normed_host != denier_host {
+                    // domains don't even match, keep on keepin' on...
+                    continue;
+                }
+            }
+            _ => {
+                // one or the other couldn't determine the host value, which probably means
+                // it's not suitable for further comparison
+                continue;
+            }
+        }
+
+        let normed_host = normed_url.host().unwrap(); // match above will catch errors
+
+        // at this point, we have a matching set of ips or domain names. now we can process the
+        // url path. The goal is to determine whether the given url's path is a subpath of any
+        // url in the deny list, for example
+        //    GIVEN URL                        URL DENY LIST               USER-SPECIFIED URLS TO SCAN
+        //    http://some.domain/stuff/things, [http://some.domain/stuff], [http://some.domain] => true
+        //    http://some.domain/stuff/things, [http://some.domain/stuff/things], [http://some.domain] => true
+        //    http://some.domain/stuff/things, [http://some.domain/api], [http://some.domain] => false
+        // the examples above are all pretty obvious, the kicker comes when the blocking url's
+        // path is a parent to a scanned url
+        //    http://some.domain/stuff/things, [http://some.domain/], [http://some.domain/stuff] => false
+        //    http://some.domain/api, [http://some.domain/], [http://some.domain/stuff] => true
+        // we want to deny all children of the parent, unless that child is a child of a scan
+        // we specified through -u(s) or --stdin
+
+        let deny_path = denier.path();
+        let norm_path = normed_url.path();
+
+        if norm_path.starts_with(deny_path) {
+            // at this point, we know that the given normalized path is a sub-path of the
+            // current deny-url, now we just need to check to see if this deny-url is a parent
+            // to a scanned url that is also a parent of the given url
+            for ferox_scan in handles.ferox_scans()?.get_active_scans() {
+                let scanner = Url::parse(ferox_scan.url().trim_end_matches('/'))
+                    .with_context(|| format!("Could not parse {} as a url", ferox_scan))?;
+
+                if let Some(scan_host) = scanner.host() {
+                    // same domain/ip check we perform on the denier above
+                    if normed_host != scan_host {
+                        // domains don't even match, keep on keepin' on...
+                        continue;
+                    }
+                } else {
+                    // couldn't process .host from scanner
+                    continue;
+                };
+
+                let scan_path = scanner.path();
+
+                if scan_path.starts_with(deny_path) && norm_path.starts_with(scan_path) {
+                    // user-specified scan url is a sub-path of the deny-urls's path AND the
+                    // url to check is a sub-path of the user-specified scan url
+                    //
+                    // the assumption is the user knew what they wanted and we're going to give
+                    // the scanned url precedence, even though it's a sub-path
+                    log::trace!("exit: should_deny_url -> false");
+                    return Ok(false);
+                }
+            }
+            log::trace!("exit: should_deny_url -> true");
+            return Ok(true);
+        }
+    }
+    log::trace!("exit: should_deny_url -> false");
+    Ok(false)
+}
+
+/// given a url and filename-suffix, return a unique filename comprised of the slugified url,
+/// current unix timestamp and suffix
+///
+/// ex: ferox-http_telsa_com-1606947491.state
+pub fn slugify_filename(url: &str, prefix: &str, suffix: &str) -> String {
+    log::trace!("enter: slugify({:?}, {:?}, {:?})", url, prefix, suffix);
+
+    let ts = SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .unwrap_or_else(|_| Duration::from_secs(0))
+        .as_secs();
+
+    let altered_prefix = if !prefix.is_empty() {
+        format!("{}-", prefix)
+    } else {
+        String::new()
+    };
+
+    let slug = url.replace("://", "_").replace("/", "_").replace(".", "_");
+
+    let filename = format!("{}{}-{}.{}", altered_prefix, slug, ts, suffix);
+
+    log::trace!("exit: slugify -> {}", filename);
+    filename
+}
+
 #[cfg(test)]
 mod tests {
    use super::*;
+    use crate::config::Configuration;
+    use crate::scan_manager::{FeroxScans, ScanOrder};

    #[test]
    /// set_open_file_limit with a low requested limit succeeds
    fn utils_set_open_file_limit_with_low_requested_limit() {
        let (_, hard) = getrlimit(Resource::NOFILE).unwrap();
-        let lower_limit = hard.as_usize() - 1;
+        let lower_limit = hard - 1;
        assert!(set_open_file_limit(lower_limit));
    }

@@ -303,9 +430,9 @@ mod tests {
    /// set_open_file_limit with a high requested limit succeeds
    fn utils_set_open_file_limit_with_high_requested_limit() {
        let (_, hard) = getrlimit(Resource::NOFILE).unwrap();
-        let higher_limit = hard.as_usize() + 1;
+        let higher_limit = hard + 1;
        // calculate a new soft to ensure soft != hard and hit that logic branch
-        let new_soft = Rlim::from_usize(hard.as_usize() - 1);
+        let new_soft = hard - 1;
        setrlimit(Resource::NOFILE, new_soft, hard).unwrap();
        assert!(set_open_file_limit(higher_limit));
    }
@@ -316,7 +443,7 @@ mod tests {
        let (_, hard) = getrlimit(Resource::NOFILE).unwrap();
        // calculate a new soft to ensure soft == hard and hit the failure logic branch
        setrlimit(Resource::NOFILE, hard, hard).unwrap();
-        assert!(!set_open_file_limit(hard.as_usize())); // returns false
+        assert!(!set_open_file_limit(hard)); // returns false
    }

    #[test]
@@ -366,4 +493,180 @@ mod tests {
    fn status_colorizer_returns_as_is() {
        assert_eq!(status_colorizer("farfignewton"), "farfignewton".to_string());
    }
+
+    #[test]
+    /// provide a url that should be blocked where the denier is an exact match for the tested url
+    /// expect true
+    fn should_deny_url_blocks_when_denier_is_exact_match() {
+        let scan_url = "https://testdomain.com/";
+        let deny_url = "https://testdomain.com/denied";
+        let tested_url = Url::parse("https://testdomain.com/denied/").unwrap();
+
+        let scans = Arc::new(FeroxScans::default());
+        scans.add_directory_scan(scan_url, ScanOrder::Initial);
+
+        let mut config = Configuration::new().unwrap();
+        config.url_denylist = vec![String::from(deny_url)];
+        let config = Arc::new(config);
+
+        let handles = Arc::new(Handles::for_testing(Some(scans), Some(config)).0);
+
+        assert!(should_deny_url(&tested_url, handles).unwrap());
+    }
+
+    #[test]
+    /// provide a url that has a different host than the denier but the same path, expect false
+    fn should_deny_url_doesnt_compare_mismatched_domains() {
+        let scan_url = "https://testdomain.com/";
+        let deny_url = "https://dev.testdomain.com/denied";
+        let tested_url = Url::parse("https://testdomain.com/denied/").unwrap();
+
+        let scans = Arc::new(FeroxScans::default());
+        scans.add_directory_scan(scan_url, ScanOrder::Initial);
+
+        let mut config = Configuration::new().unwrap();
+        config.url_denylist = vec![String::from(deny_url)];
+        let config = Arc::new(config);
+
+        let handles = Arc::new(Handles::for_testing(Some(scans), Some(config)).0);
+
+        assert!(!should_deny_url(&tested_url, handles).unwrap());
+    }
+
+    #[test]
+    /// provide a denier from which we can't check a host, which results in no comparison, expect false
+    fn should_deny_url_doesnt_compare_non_domains() {
+        let scan_url = "https://testdomain.com/";
+        let deny_url = "unix:/run/foo.socket";
+        let tested_url = Url::parse("https://testdomain.com/denied/").unwrap();
+
+        let scans = Arc::new(FeroxScans::default());
+        scans.add_directory_scan(scan_url, ScanOrder::Initial);
+
+        let mut config = Configuration::new().unwrap();
+        config.url_denylist = vec![String::from(deny_url)];
+        let config = Arc::new(config);
+
+        let handles = Arc::new(Handles::for_testing(Some(scans), Some(config)).0);
+
+        assert!(!should_deny_url(&tested_url, handles).unwrap());
+    }
+
+    #[test]
+    /// provide a url that has a different host than the denier but the same path, expect false
+    /// because the denier is a parent to the tested, even tho the scanned doesn't compare, it
+    /// still returns true
+    fn should_deny_url_doesnt_compare_mismatched_domains_in_scanned() {
+        let deny_url = "https://testdomain.com/";
+        let scan_url = "https://dev.testdomain.com/denied";
+        let tested_url = Url::parse("https://testdomain.com/denied/").unwrap();
+
+        let scans = Arc::new(FeroxScans::default());
+        scans.add_directory_scan(scan_url, ScanOrder::Initial);
+
+        let mut config = Configuration::new().unwrap();
+        config.url_denylist = vec![String::from(deny_url)];
+        let config = Arc::new(config);
+
+        let handles = Arc::new(Handles::for_testing(Some(scans), Some(config)).0);
+
+        assert!(should_deny_url(&tested_url, handles).unwrap());
+    }
+
+    #[test]
+    /// provide a denier from which we can't check a host, which results in no comparison, expect false
+    /// because the denier is a parent to the tested, even tho the scanned doesn't compare, it
+    /// still returns true
+    fn should_deny_url_doesnt_compare_non_domains_in_scanned() {
+        let deny_url = "https://testdomain.com/";
+        let scan_url = "unix:/run/foo.socket";
+        let tested_url = Url::parse("https://testdomain.com/denied/").unwrap();
+
+        let scans = Arc::new(FeroxScans::default());
+        scans.add_directory_scan(scan_url, ScanOrder::Initial);
+
+        let mut config = Configuration::new().unwrap();
+        config.url_denylist = vec![String::from(deny_url)];
+        let config = Arc::new(config);
+
+        let handles = Arc::new(Handles::for_testing(Some(scans), Some(config)).0);
+
+        assert!(should_deny_url(&tested_url, handles).unwrap());
+    }
+
+    #[test]
+    /// provide a denier where the tested url is a sub-path and the scanned url is not, expect true
+    fn should_deny_url_blocks_child() {
+        let scan_url = "https://testdomain.com/";
+        let deny_url = "https://testdomain.com/api";
+        let tested_url = Url::parse("https://testdomain.com/api/denied/").unwrap();
+
+        let scans = Arc::new(FeroxScans::default());
+        scans.add_directory_scan(scan_url, ScanOrder::Initial);
+
+        let mut config = Configuration::new().unwrap();
+        config.url_denylist = vec![String::from(deny_url)];
+        let config = Arc::new(config);
+
+        let handles = Arc::new(Handles::for_testing(Some(scans), Some(config)).0);
+
+        assert!(should_deny_url(&tested_url, handles).unwrap());
+    }
+
+    #[test]
+    /// provide a denier where the tested url is not a sub-path and the scanned url is not, expect false
+    fn should_deny_url_doesnt_block_non_child() {
+        let scan_url = "https://testdomain.com/";
+        let deny_url = "https://testdomain.com/api";
+        let tested_url = Url::parse("https://testdomain.com/not-denied/").unwrap();
+
+        let scans = Arc::new(FeroxScans::default());
+        scans.add_directory_scan(scan_url, ScanOrder::Initial);
+
+        let mut config = Configuration::new().unwrap();
+        config.url_denylist = vec![String::from(deny_url)];
+        let config = Arc::new(config);
+
+        let handles = Arc::new(Handles::for_testing(Some(scans), Some(config)).0);
+
+        assert!(!should_deny_url(&tested_url, handles).unwrap());
+    }
+
+    #[test]
+    /// provide a denier where the tested url is a sub-path and the scanned url is not, expect true
+    fn should_deny_url_blocks_child_when_scan_url_isnt_parent() {
+        let scan_url = "https://testdomain.com/api";
+        let deny_url = "https://testdomain.com/";
+        let tested_url = Url::parse("https://testdomain.com/stuff/").unwrap();
+
+        let scans = Arc::new(FeroxScans::default());
+        scans.add_directory_scan(scan_url, ScanOrder::Initial);
+
+        let mut config = Configuration::new().unwrap();
+        config.url_denylist = vec![String::from(deny_url)];
+        let config = Arc::new(config);
+
+        let handles = Arc::new(Handles::for_testing(Some(scans), Some(config)).0);
+
+        assert!(should_deny_url(&tested_url, handles).unwrap());
+    }
+
+    #[test]
+    /// provide a denier where the tested url is not a sub-path and the scanned url is not, expect false
+    fn should_deny_url_doesnt_block_child_when_scan_url_is_parent() {
+        let scan_url = "https://testdomain.com/api";
+        let deny_url = "https://testdomain.com/";
+        let tested_url = Url::parse("https://testdomain.com/api/not-denied/").unwrap();
+
+        let scans = Arc::new(FeroxScans::default());
+        scans.add_directory_scan(scan_url, ScanOrder::Initial);
+
+        let mut config = Configuration::new().unwrap();
+        config.url_denylist = vec![String::from(deny_url)];
+        let config = Arc::new(config);
+
+        let handles = Arc::new(Handles::for_testing(Some(scans), Some(config)).0);
+
+        assert!(!should_deny_url(&tested_url, handles).unwrap());
+    }
 }
--- a/tests/test_banner.rs
+++ b/tests/test_banner.rs
@@ -113,6 +113,36 @@ fn banner_prints_headers() {
        );
 }

+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + multiple dont scan entries
+fn banner_prints_denied_urls() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--dont-scan")
+        .arg("http://dont-scan.me")
+        .arg("--dont-scan")
+        .arg("https://also-not.me")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Don't Scan"))
+                .and(predicate::str::contains("http://dont-scan.me"))
+                .and(predicate::str::contains("https://also-not.me"))
+                .and(predicate::str::contains("─┴─")),
+        );
+}
+
 #[test]
 /// test allows non-existent wordlist to trigger the banner printing to stderr
 /// expect to see all mandatory prints + multiple size filters
--- a/tests/test_deny_list.rs
+++ b/tests/test_deny_list.rs
@@ -0,0 +1,212 @@
+mod utils;
+use assert_cmd::prelude::*;
+use assert_cmd::Command;
+use httpmock::Method::GET;
+use httpmock::MockServer;
+use predicates::prelude::*;
+use utils::{setup_tmp_directory, teardown_tmp_directory};
+
+#[test]
+/// test that the deny list prevents a request if the requested url is a match
+fn deny_list_works_during_with_a_normal_scan() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--dont-scan")
+        .arg(srv.url("/LICENSE"))
+        .unwrap();
+
+    teardown_tmp_directory(tmp_dir);
+
+    cmd.assert()
+        .success()
+        .stdout(predicate::str::contains(srv.url("/LICENSE")).not());
+
+    assert_eq!(mock.hits(), 0);
+}
+
+#[test]
+/// test that the deny list prevents requests of urls found during extraction
+fn deny_list_works_during_extraction() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200)
+            .body(&srv.url("'/homepage/assets/img/icons/handshake.svg'"));
+    });
+
+    let mock_two = srv.mock(|when, then| {
+        when.method(GET)
+            .path("/homepage/assets/img/icons/handshake.svg");
+        then.status(200);
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--extract-links")
+        .arg("--dont-scan")
+        .arg(srv.url("/homepage/"))
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .and(predicate::str::contains("200"))
+            .and(predicate::str::contains("/homepage/assets/img/icons/handshake.svg").not()),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_two.hits(), 0);
+    teardown_tmp_directory(tmp_dir);
+}
+
+#[test]
+/// test that the deny list prevents requests of urls found during recursion
+fn deny_list_works_during_recursion() {
+    let srv = MockServer::start();
+    let urls = [
+        "js".to_string(),
+        "prod".to_string(),
+        "dev".to_string(),
+        "file.js".to_string(),
+    ];
+    let (tmp_dir, file) = setup_tmp_directory(&urls, "wordlist").unwrap();
+
+    let js_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js");
+        then.status(301).header("Location", &srv.url("/js/"));
+    });
+
+    let js_prod_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js/prod");
+        then.status(301).header("Location", &srv.url("/js/prod/"));
+    });
+
+    let js_dev_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js/dev");
+        then.status(301).header("Location", &srv.url("/js/dev/"));
+    });
+
+    let js_dev_file_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js/dev/file.js");
+        then.status(200)
+            .body("this is a test and is more bytes than other ones");
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-t")
+        .arg("1")
+        .arg("--dont-scan")
+        .arg(srv.url("/js/dev"))
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::is_match("301.*js")
+            .unwrap()
+            .and(predicate::str::is_match("301.*js/prod").unwrap())
+            .and(predicate::str::is_match("301.*js/dev").unwrap())
+            .not()
+            .and(predicate::str::is_match("200.*js/dev/file.js").unwrap())
+            .not(),
+    );
+
+    assert_eq!(js_mock.hits(), 1);
+    assert_eq!(js_prod_mock.hits(), 1);
+    assert_eq!(js_dev_mock.hits(), 0);
+    assert_eq!(js_dev_file_mock.hits(), 0);
+
+    teardown_tmp_directory(tmp_dir);
+}
+
+#[test]
+/// test that the deny list prevents requests of urls found during recursion when the denier is a
+/// parent of a user-specified scan
+fn deny_list_works_during_recursion_with_inverted_parents() {
+    let srv = MockServer::start();
+    let urls = [
+        "js".to_string(),
+        "prod".to_string(),
+        "dev".to_string(),
+        "api".to_string(),
+        "file.js".to_string(),
+    ];
+    let (tmp_dir, file) = setup_tmp_directory(&urls, "wordlist").unwrap();
+
+    let js_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js");
+        then.status(301).header("Location", &srv.url("/js/"));
+    });
+
+    let api_mock = srv.mock(|when, then| {
+        when.method(GET).path("/api");
+        then.status(200);
+    });
+
+    let js_prod_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js/prod");
+        then.status(301).header("Location", &srv.url("/js/prod/"));
+    });
+
+    let js_dev_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js/dev");
+        then.status(301).header("Location", &srv.url("/js/dev/"));
+    });
+
+    let js_dev_file_mock = srv.mock(|when, then| {
+        when.method(GET).path("/js/dev/file.js");
+        then.status(200)
+            .body("this is a test and is more bytes than other ones");
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/js"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-t")
+        .arg("1")
+        .arg("-vvvv")
+        .arg("--dont-scan")
+        .arg(srv.url("/"))
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::is_match("301.*js")
+            .unwrap()
+            .and(predicate::str::is_match("301.*js/prod").unwrap())
+            .and(predicate::str::is_match("301.*js/dev").unwrap())
+            .and(predicate::str::is_match("200.*js/dev/file.js").unwrap())
+            .and(predicate::str::is_match("200.*api").unwrap())
+            .not(),
+    );
+
+    assert_eq!(js_mock.hits(), 1);
+    assert_eq!(js_prod_mock.hits(), 1);
+    assert_eq!(js_dev_mock.hits(), 1);
+    assert_eq!(js_dev_file_mock.hits(), 1);
+    assert_eq!(api_mock.hits(), 0);
+
+    teardown_tmp_directory(tmp_dir);
+}
--- a/tests/test_heuristics.rs
+++ b/tests/test_heuristics.rs
@@ -224,11 +224,11 @@ fn test_dynamic_wildcard_request_found() {

    teardown_tmp_directory(tmp_dir);

-    assert_eq!(contents.contains("WLD"), true);
-    assert_eq!(contents.contains("Got"), true);
-    assert_eq!(contents.contains("200"), true);
-    assert_eq!(contents.contains("(url length: 32)"), true);
-    assert_eq!(contents.contains("(url length: 96)"), true);
+    assert!(contents.contains("WLD"));
+    assert!(contents.contains("Got"));
+    assert!(contents.contains("200"));
+    assert!(contents.contains("(url length: 32)"));
+    assert!(contents.contains("(url length: 96)"));

    cmd.assert().success().stdout(
        predicate::str::contains("WLD")
@@ -391,11 +391,11 @@ fn heuristics_wildcard_test_with_two_static_wildcards_and_output_to_file() {

    teardown_tmp_directory(tmp_dir);

-    assert_eq!(contents.contains("WLD"), true);
-    assert_eq!(contents.contains("Got"), true);
-    assert_eq!(contents.contains("200"), true);
-    assert_eq!(contents.contains("(url length: 32)"), true);
-    assert_eq!(contents.contains("(url length: 96)"), true);
+    assert!(contents.contains("WLD"));
+    assert!(contents.contains("Got"));
+    assert!(contents.contains("200"));
+    assert!(contents.contains("(url length: 32)"));
+    assert!(contents.contains("(url length: 96)"));

    cmd.assert().success().stdout(
        predicate::str::contains("WLD")
@@ -451,12 +451,12 @@ fn heuristics_wildcard_test_with_redirect_as_response_code(

    teardown_tmp_directory(tmp_dir);

-    assert_eq!(contents.contains("WLD"), true);
-    assert_eq!(contents.contains("301"), true);
-    assert_eq!(contents.contains("/some-redirect"), true);
-    assert_eq!(contents.contains("redirects to => "), true);
-    assert_eq!(contents.contains(&srv.url("/")), true);
-    assert_eq!(contents.contains("(url length: 32)"), true);
+    assert!(contents.contains("WLD"));
+    assert!(contents.contains("301"));
+    assert!(contents.contains("/some-redirect"));
+    assert!(contents.contains("redirects to => "));
+    assert!(contents.contains(&srv.url("/")));
+    assert!(contents.contains("(url length: 32)"));

    cmd.assert().success().stdout(
        predicate::str::contains("redirects to => ")
--- a/tests/test_main.rs
+++ b/tests/test_main.rs
@@ -3,7 +3,7 @@ use assert_cmd::Command;
 use httpmock::Method::GET;
 use httpmock::{MockServer, Regex};
 use predicates::prelude::*;
-use std::fs::read_to_string;
+use std::fs::{read_dir, read_to_string};
 use utils::{setup_tmp_directory, teardown_tmp_directory};

 #[test]
@@ -153,3 +153,73 @@ fn main_parallel_spawns_children() -> Result<(), Box<dyn std::error::Error>> {

    Ok(())
 }
+
+#[test]
+/// send three targets over stdin with --output enabled, expect parallel to create a new directory
+/// and the log files therein
+fn main_parallel_creates_output_directory() -> Result<(), Box<dyn std::error::Error>> {
+    let t1 = MockServer::start();
+    let t2 = MockServer::start();
+    let t3 = MockServer::start();
+
+    let words = [
+        String::from("LICENSE"),
+        String::from("stuff"),
+        String::from("things"),
+        String::from("mostuff"),
+        String::from("mothings"),
+    ];
+    let (word_tmp_dir, wordlist) = setup_tmp_directory(&words, "wordlist")?;
+    let (output_dir, outfile) = setup_tmp_directory(&[], "output-file")?;
+    let (tgt_tmp_dir, targets) =
+        setup_tmp_directory(&[t1.url("/"), t2.url("/"), t3.url("/")], "targets")?;
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--stdin")
+        .arg("--parallel")
+        .arg("2")
+        .arg("--output")
+        .arg(outfile.as_os_str())
+        .arg("--wordlist")
+        .arg(wordlist.as_os_str())
+        .pipe_stdin(targets)
+        .unwrap()
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("Could not connect to any target provided")
+                .and(predicate::str::contains("Target Url"))
+                .not(), // no target url found
+        );
+
+    // output_dir should return something similar to output-file-1627845244.logs with the
+    // line below. if it ever fails, can use the regex below to filter out the right directory
+    let sub_dir = read_dir(&output_dir)?.next().unwrap()?.file_name();
+
+    let mut num_logs = 0;
+    let file_regex = Regex::new("ferox-[a-zA-Z_:0-9]+-[0-9]+.log").unwrap();
+    let dir_regex = Regex::new("output-file-[0-9]+.logs").unwrap();
+
+    let sub_dir = output_dir.as_ref().join(&sub_dir);
+
+    // created directory like output-file-1627845741.logs/
+    assert!(dir_regex.is_match(&sub_dir.to_string_lossy().to_string()));
+
+    for entry in sub_dir.read_dir()? {
+        let entry = entry?;
+        // created each file like ferox-https_localhost-1627845741.log
+        println!("name: {:?}", entry.file_name().to_string_lossy());
+        assert!(file_regex.is_match(&entry.file_name().to_string_lossy()));
+        num_logs += 1;
+    }
+
+    // should be 3 log files total
+    assert_eq!(num_logs, 3);
+
+    teardown_tmp_directory(word_tmp_dir);
+    teardown_tmp_directory(tgt_tmp_dir);
+    teardown_tmp_directory(output_dir);
+
+    Ok(())
+}
--- a/tests/test_parser.rs
+++ b/tests/test_parser.rs
@@ -0,0 +1,46 @@
+use assert_cmd::Command;
+use predicates::prelude::*;
+
+#[test]
+/// specify an incorrect param (-fc) with --help after it on the command line
+/// old behavior printed
+/// error: Found argument '-c' which wasn't expected, or isn't valid in this context
+///
+/// USAGE:
+///     feroxbuster --add-slash --url <URL>...
+///
+/// For more information try --help
+///
+/// the new behavior we expect to see is to print the long form help message, of which
+/// Ludicrous speed... go! is near the bottom of that output, so we can test for that
+fn parser_incorrect_param_with_tack_tack_help() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("-fc")
+        .arg("--help")
+        .assert()
+        .success()
+        .stdout(predicate::str::contains("Ludicrous speed... go!"));
+}
+
+#[test]
+/// specify an incorrect param (-fc) with --help after it on the command line
+/// old behavior printed
+/// error: Found argument '-c' which wasn't expected, or isn't valid in this context
+///
+/// USAGE:
+///     feroxbuster --add-slash --url <URL>...
+///
+/// For more information try --help
+///
+/// the new behavior we expect to see is to print the long form help message, of which
+/// Ludicrous speed... go! is near the bottom of that output, so we can test for that
+fn parser_incorrect_param_with_tack_h() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("-fc")
+        .arg("-h")
+        .assert()
+        .success()
+        .stdout(predicate::str::contains("Ludicrous speed... go!"));
+}
--- a/tests/test_policies.rs
+++ b/tests/test_policies.rs
@@ -77,7 +77,7 @@ fn auto_bail_cancels_scan_with_timeouts() {

    // read debug log to get the number of errors enforced
    for line in debug_log.lines() {
-        let log: serde_json::Value = serde_json::from_str(&line).unwrap_or_default();
+        let log: serde_json::Value = serde_json::from_str(line).unwrap_or_default();
        if let Some(message) = log.get("message") {
            let str_msg = message.as_str().unwrap_or_default().to_string();

@@ -156,7 +156,7 @@ fn auto_bail_cancels_scan_with_403s() {

    // read debug log to get the number of errors enforced
    for line in debug_log.lines() {
-        let log: serde_json::Value = serde_json::from_str(&line).unwrap_or_default();
+        let log: serde_json::Value = serde_json::from_str(line).unwrap_or_default();
        if let Some(message) = log.get("message") {
            let str_msg = message.as_str().unwrap_or_default().to_string();

@@ -238,7 +238,7 @@ fn auto_bail_cancels_scan_with_429s() {

    // read debug log to get the number of errors enforced
    for line in debug_log.lines() {
-        let log: serde_json::Value = serde_json::from_str(&line).unwrap_or_default();
+        let log: serde_json::Value = serde_json::from_str(line).unwrap_or_default();
        if let Some(message) = log.get("message") {
            let str_msg = message.as_str().unwrap_or_default().to_string();

--- a/tests/test_scan_manager.rs
+++ b/tests/test_scan_manager.rs
@@ -93,9 +93,13 @@ fn resume_scan_works() {
 #[test]
 /// kick off scan with a time limit;  
 fn time_limit_enforced_when_specified() {
-    let srv = MockServer::start();
+    let t1 = MockServer::start();
+    let t2 = MockServer::start();
+
    let (tmp_dir, file) =
        setup_tmp_directory(&["css".to_string(), "stuff".to_string()], "wordlist").unwrap();
+    let (tgt_tmp_dir, targets) =
+        setup_tmp_directory(&[t1.url("/"), t2.url("/")], "targets").unwrap();

    // ensure the command will run long enough by adding crap to the wordlist
    let more_words = read_to_string(Path::new("tests/extra-words")).unwrap();
@@ -109,12 +113,13 @@ fn time_limit_enforced_when_specified() {

    Command::cargo_bin("feroxbuster")
        .unwrap()
-        .arg("--url")
-        .arg(srv.url("/"))
+        .arg("--stdin")
        .arg("--wordlist")
        .arg(file.as_os_str())
        .arg("--time-limit")
        .arg("5s")
+        .pipe_stdin(targets)
+        .unwrap()
        .assert()
        .failure();

@@ -127,4 +132,5 @@ fn time_limit_enforced_when_specified() {
    assert!(now.elapsed() > lower_bound && now.elapsed() < upper_bound);

    teardown_tmp_directory(tmp_dir);
+    teardown_tmp_directory(tgt_tmp_dir);
 }
Author	SHA1	Message	Date
epi	15b4fd04e5	updated status code defaults to include 500	2021-08-02 19:28:09 -05:00
epi	fceba0b68b	updated deps	2021-08-02 19:27:55 -05:00
epi	eef4c9b5ed	added 500 to status code defaults	2021-08-02 19:23:20 -05:00
epi	24da4e017c	adjusted rlimit imports to ignore windows targets	2021-08-02 05:41:50 -05:00
epi	f3cedf01a5	Merge pull request #321 from epi052/319-separate-log-files-for-parallel Log to separate files when using --parallel	2021-08-02 05:21:16 -05:00
epi	08ee32595f	updated documentation for parallel logging change	2021-08-01 21:03:17 -05:00
epi	4c4d1a2a61	updated documentation for parallel logging change	2021-08-01 19:59:28 -05:00
epi	64b54a6308	fixed up a few todo items	2021-08-01 19:57:28 -05:00
epi	e27b3ee8da	added coverage for stdin slugifying	2021-08-01 15:17:59 -05:00
epi	129725cedd	added test for --parallel with -o	2021-08-01 14:32:14 -05:00
epi	17886da3df	handle dir/outfile case to -o	2021-08-01 14:31:48 -05:00
epi	c8a46b7e5a	added prefix param to slugify_filename	2021-08-01 09:23:42 -05:00
epi	f97d103fc6	unique file logging with parallel works	2021-08-01 08:07:12 -05:00
epi	aa2fecc5c1	bumped version to 2.3.2	2021-07-31 14:49:15 -05:00
epi	6f2244e1ff	put url slug logic into its own function	2021-07-31 14:48:35 -05:00
epi	a1dc90ba06	updated rlimit lib	2021-07-30 20:01:16 -05:00
epi	32f55ddfb7	Merge branch 'main' of github.com:epi052/feroxbuster	2021-07-30 16:14:32 -05:00
epi	9a65c7f1f5	fixed up code for new clippy checks	2021-07-30 16:14:25 -05:00
epi	0f6bc1c160	updated deps	2021-07-30 07:42:25 -05:00
epi	abef7a236b	Update README.md	2021-07-12 16:17:08 -05:00
epi	0cff62dbe2	return 0 when -h/--help is used	2021-07-05 06:33:40 -05:00
epi	a590188e44	Merge pull request #293 from epi052/286-url-blacklist add --dont-scan option for denying urls	2021-06-18 14:40:16 -07:00
epi	dc3aa11966	added tracing to new extractor fns	2021-06-18 16:32:04 -05:00
epi	57714d243a	fixed caching for extraction; much better performance now	2021-06-18 16:18:42 -05:00
epi	1d34a5e99f	updated readme	2021-06-18 11:05:57 -05:00
epi	9ab3e5515e	added short-circuit to deny check	2021-06-18 06:48:41 -05:00
epi	3abef25c8f	added integration tests	2021-06-17 20:34:21 -05:00
epi	454f3a4302	satisfied newest version of clippy	2021-06-17 16:13:18 -05:00
epi	acb9c19f4d	added should_deny_url function and unit tests	2021-06-17 14:44:33 -05:00
epi	98f06951bd	added banner test	2021-06-15 17:31:24 -05:00
epi	c9e1a7adbe	added deny list to banner	2021-06-15 16:59:36 -05:00
epi	c57cf82fce	added --dont-scan to options parser	2021-06-15 16:45:23 -05:00
epi	a3bcfaf95c	added url_denylist to config	2021-06-15 14:31:01 -05:00
epi	c99afec740	bumped version to 2.3.0	2021-06-15 13:52:45 -05:00
epi	fa9fd65c2f	bumped version to 2.3.0	2021-06-15 11:47:57 -05:00
epi	2af87971d5	fixed build script when on cd pipeline	2021-06-15 11:46:20 -05:00
epi	e6753d9474	fixed build script when on cd pipeline	2021-06-15 11:45:35 -05:00
epi	d23717dc6c	troubleshooting build script	2021-06-15 11:32:00 -05:00
epi	4debe68ed6	updated pipeline build	2021-06-15 11:28:46 -05:00
epi	e6b78e3986	Merge pull request #292 from epi052/287-always-parse-help-parameter added small check for help param to always print and exit	2021-06-15 09:22:34 -07:00
epi	7b268cf197	bumped version to 2.2.5	2021-06-15 11:22:14 -05:00
epi	34ff884d52	added tests for new help param catcher	2021-06-15 11:02:43 -05:00
epi	7fef23f888	added small check for help param to always print and exit	2021-06-15 10:28:20 -05:00
epi	7a8d6d0d52	fixed build when config copied on cd	2021-06-15 10:14:59 -05:00
epi	6d4f2a7ed9	Update build.yml	2021-06-15 10:14:16 -05:00
epi	329d04252f	config file is dropped to disk when installing via cargo	2021-06-15 08:00:21 -05:00
epi	9b4092ea8c	added plusdirs to bash completion script and regenerated it	2021-06-15 06:37:03 -05:00
epi	d942a7705a	bumped various lib versions	2021-06-14 20:09:22 -05:00
epi	e3365b42a2	Merge branch 'main' of github.com:epi052/feroxbuster	2021-06-14 20:08:11 -05:00
epi	41689bd742	added verify command	2021-06-14 20:07:03 -05:00
epi	bc487475f0	Update pull_request_template.md	2021-06-14 20:00:19 -05:00
epi	393e775285	satisfied clippy	2021-05-08 16:10:01 -05:00
epi	cf6c02307c	bumped version to 2.2.4	2021-05-08 16:01:23 -05:00
epi	88b9bc3a01	Merge pull request #270 from epi052/268-cancel-scan-by-range updated scan cancel input to support comma and range delimited values	2021-05-08 15:57:28 -05:00
epi	d1f90efb09	bumped lib versions	2021-05-05 06:12:50 -05:00
epi	df4fad07a9	Merge branch 'main' into 268-cancel-scan-by-range	2021-05-05 06:06:19 -05:00
epi	56d533117e	updated docs with new cancel scan info	2021-05-05 05:56:54 -05:00
epi	9549e27f19	updated cancel menu footer with description about -f	2021-04-20 11:57:56 -05:00
epi	1677b51c2d	reverted workflow file	2021-04-20 11:43:05 -05:00
epi	d4f9442d38	examining codecov env	2021-04-20 11:05:17 -05:00
epi	8191fa1a5e	updated scan cancel input to support comma and range delimd values	2021-04-20 07:39:11 -05:00
epi	4811b37aa4	Merge pull request #267 from epi052/restyled/pull-266 check for unzip before continuing	2021-04-15 05:15:59 -05:00
Restyled.io	941cad5844	Restyled by shfmt	2021-04-14 17:37:28 +00:00
Restyled.io	d59af94f62	Restyled by shellharden	2021-04-14 17:37:26 +00:00
Craig	cf403c4d4a	check for unzip before continuing	2021-04-14 19:35:24 +02:00
epi	57a2b1cbab	bumped ctrlc, tokio, reqwest, tokio-util, and futures	2021-04-14 05:59:35 -05:00
epi	ef195bd653	Merge branch 'main' of github.com:epi052/feroxbuster into main	2021-04-01 06:01:45 -05:00
epi	9b1a24bca3	updated libs; fixed new clippy errors	2021-04-01 06:00:44 -05:00
epi	c6aefbfa97	Merge pull request #249 from noraj/patch-1 add blackarch install description	2021-03-20 14:06:35 -05:00
Alexandre ZANNI	42bad85208	add blackarch install	2021-03-19 16:29:27 +01:00
epi	f5709739fa	Merge pull request #247 from epi052/dependabot/cargo/regex-1.4.5 Bump regex from 1.4.4 to 1.4.5	2021-03-17 05:59:12 -05:00
epi	248f56ed7a	Merge pull request #246 from epi052/dependabot/cargo/console-0.14.1 Bump console from 0.14.0 to 0.14.1	2021-03-17 05:58:55 -05:00
dependabot[bot]	3de6ed9696	Bump regex from 1.4.4 to 1.4.5 Bumps [regex](https://github.com/rust-lang/regex) from 1.4.4 to 1.4.5. - [Release notes](https://github.com/rust-lang/regex/releases) - [Changelog](https://github.com/rust-lang/regex/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-lang/regex/compare/1.4.4...1.4.5) Signed-off-by: dependabot[bot] <support@github.com>	2021-03-15 07:59:58 +00:00
dependabot[bot]	4bad39f4b9	Bump console from 0.14.0 to 0.14.1 Bumps [console](https://github.com/mitsuhiko/console) from 0.14.0 to 0.14.1. - [Release notes](https://github.com/mitsuhiko/console/releases) - [Changelog](https://github.com/mitsuhiko/console/blob/master/CHANGELOG.md) - [Commits](https://github.com/mitsuhiko/console/compare/0.14.0...0.14.1) Signed-off-by: dependabot[bot] <support@github.com>	2021-03-15 07:59:47 +00:00
epi	9b303d8b5a	update dependencies	2021-03-14 07:29:47 -05:00
epi	7e0b003216	Merge branch 'main' of github.com:epi052/feroxbuster into main	2021-03-14 07:22:30 -05:00
epi	dc36a7bf4d	updated deendencies	2021-03-14 07:22:16 -05:00
epi	d33632c421	update ko-fi	2021-03-13 07:05:01 -06:00
epi	7dc6a867a5	update ko-fi	2021-03-13 07:04:31 -06:00
epi	b937a0191e	Create FUNDING.yml	2021-03-13 06:24:03 -06:00
epi	d57a83956c	Merge pull request #237 from epi052/235-add-arm-build added support for arm builds	2021-03-05 11:10:09 -06:00
epi	71efd78f03	fixed docstrings for Stats	2021-03-05 11:04:00 -06:00
epi	139006d0a7	added aarch64 to the build matrix	2021-03-05 11:03:36 -06:00
epi	b5abb8b6e8	ci build for aarch64 works, restricting to main again	2021-03-05 10:49:13 -06:00
epi	a076a333df	updated README for arm installs	2021-03-05 10:31:56 -06:00
epi	461ed0a9ff	added aarch64 build to ci	2021-03-05 10:24:57 -06:00
epi	4381569a0f	ci build works, restricting to main again	2021-03-05 09:59:48 -06:00
epi	a52bd10340	added arm specific strip command	2021-03-05 09:34:23 -06:00
epi	56a1144865	testing ci build of armv7	2021-03-05 09:21:00 -06:00
epi	23ab009c08	added linker flag for arm builds	2021-03-05 09:14:43 -06:00
epi	fa4e3d5d88	added linker flag for arm builds	2021-03-05 09:13:10 -06:00
epi	ad7a1ffe44	added custom Stats::serialize to accomodated arm builds	2021-03-05 09:05:51 -06:00
epi	0e4f8893f8	added custom Stats::deserialize to accomodated arm builds	2021-03-05 08:25:33 -06:00
epi	8e0b801ec5	bumped version to 2.3.3	2021-03-05 06:55:00 -06:00
epi	97889f917d	Merge pull request #234 from epi052/233-ordered-wordlist changed wordlist read so that ordering is maintained	2021-03-01 16:48:54 -06:00
epi	cedb3ccc8d	wordlist order is now maintained	2021-03-01 16:38:59 -06:00
epi	d7cfd8ff60	bumped version to 2.2.2	2021-03-01 16:17:05 -06:00
epi	223e75923d	updated 3 libs, added command completion to .deb build	2021-03-01 06:18:47 -06:00
epi	dd9f2f72c0	added Kali install to readme	2021-02-27 07:01:46 -06:00
epi	8ffea2500d	Merge pull request #228 from epi052/fix-makefile-for-kali-repos Fix makefile for kali repos	2021-02-23 07:44:58 -06:00
epi	5ed890e3fd	bumped futures to 0.3.13	2021-02-23 07:34:44 -06:00
epi	8fe458263d	fixed bash completion location	2021-02-23 07:30:04 -06:00
epi	6de36585a9	Merge pull request #224 from bsysop/patch-1 Update README.md	2021-02-22 17:09:08 -06:00
epi	30538c366c	makefile works for kali	2021-02-22 16:48:42 -06:00
bsysop	89a0ac8aa4	Update README.md FFUF supports SOCKS, it's just not documented yet =] `ffuf -x socks5://127.0.0.1:1234`	2021-02-22 13:34:41 -03:00
epi	c9a93f2843	changed filter status emoji	2021-02-21 14:46:47 -06:00
epi	bfdb4abdce	changed filter status emoji	2021-02-21 14:37:04 -06:00
epi	eb17eeecd3	bumped reqwest version to 0.11.1	2021-02-21 11:49:34 -06:00
epi	c2819ef2e7	Update README.md	2021-02-18 13:24:40 -06:00