Merge pull request #147 from epi052/144-resume-scan

added 1.9 images to repo

.github/actions-rs/grcov.yml

@@ -0,0 +1,8 @@
+branch: false
+ignore-not-existing: true
+llvm: true
+output-type: lcov
+output-path: ./lcov.info
+# excl-br-line: "^\\s*((debug_)?assert(_eq|_ne)?!|#\\[derive\\(|log::)"
+ignore:
+  - "../*"

.github/pull_request_template.md

@@ -11,7 +11,7 @@ Long form explanations of most of the items below can be found in the [CONTRIBUT
 
 ## Static analysis checks
 - [ ] All rust files are formatted using `cargo fmt`
-- [ ] All `clippy` checks pass when running `cargo clippy --all-targets --all-features -- -D warnings -A clippy::unnecessary_unwrap`
+- [ ] All `clippy` checks pass when running `cargo clippy --all-targets --all-features -- -D warnings -A clippy::deref_addrof`
 - [ ] All existing tests pass
 
 ## Documentation

.github/workflows/build.yml

@@ -1,72 +1,11 @@
-name: CI Pipeline
+name: CD Pipeline
 
 on: [push]
 
 jobs:
-  check:
-    name: Check
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-      - uses: actions-rs/cargo@v1
-        with:
-          command: check
-
-  test:
-    name: Test Suite
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-      - uses: actions-rs/cargo@v1
-        with:
-          command: test
-
-  fmt:
-    name: Rust fmt
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-      - run: rustup component add rustfmt
-      - uses: actions-rs/cargo@v1
-        with:
-          command: fmt
-          args: --all -- --check
-
-  clippy:
-    name: Clippy
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-      - run: rustup component add clippy
-      - uses: actions-rs/cargo@v1
-        with:
-          command: clippy
-          args: --all-targets --all-features -- -D warnings -A clippy::unnecessary_unwrap
-
-
   build-nix:
     runs-on: ${{ matrix.os }}
-    if: github.ref == 'refs/heads/master-not-a-thing-revert-this-later-before-release'
+    if: github.ref == 'refs/heads/master'
     strategy:
       matrix:
         type: [ubuntu-x64, ubuntu-x86]
@@ -89,7 +28,6 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install -y --no-install-recommends libssl-dev pkg-config
-          sudo apt-get clean -y
       - uses: actions-rs/toolchain@v1
         with:
           toolchain: stable
@@ -103,23 +41,73 @@ jobs:
           use-cross: true
           command: build
           args: --release --target=${{ matrix.target }}
+      - name: Strip symbols from binary
+        run: |
+          strip -s ${{ matrix.path }}
+      - name: Build tar.gz for homebrew installs
+        if: matrix.type == 'ubuntu-x64'
+        run: |
+          tar czf ${{ matrix.name }}.tar.gz -C target/x86_64-unknown-linux-musl/release feroxbuster
       - uses: actions/upload-artifact@v2
         with:
           name: ${{ matrix.name }}
           path: ${{ matrix.path }}
+      - uses: actions/upload-artifact@v2
+        if: matrix.type == 'ubuntu-x64'
+        with:
+          name: ${{ matrix.name }}.tar.gz
+          path: ${{ matrix.name }}.tar.gz
 
-  build-rest:
+  build-deb:
+    needs: [build-nix]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@master
+      - name: Deb Build
+        uses: ebbflow-io/cargo-deb-amd64-ubuntu@1.0
+      - name: Upload Deb Artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: feroxbuster_amd64.deb
+          path: ./target/x86_64-unknown-linux-musl/debian/*
+
+  build-macos:
+    runs-on: macos-latest
+    if: github.ref == 'refs/heads/master'
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          target: x86_64-apple-darwin
+          override: true
+      - uses: actions-rs/cargo@v1
+        with:
+          use-cross: true
+          command: build
+          args: --release --target=x86_64-apple-darwin
+      - name: Strip symbols from binary
+        run: |
+          strip -u -r target/x86_64-apple-darwin/release/feroxbuster
+      - name: Build tar.gz for homebrew installs
+        run: |
+          tar czf x86_64-macos-feroxbuster.tar.gz -C target/x86_64-apple-darwin/release feroxbuster
+      - uses: actions/upload-artifact@v2
+        with:
+          name: x86_64-macos-feroxbuster
+          path: target/x86_64-apple-darwin/release/feroxbuster
+      - uses: actions/upload-artifact@v2
+        with:
+          name: x86_64-macos-feroxbuster.tar.gz
+          path: x86_64-macos-feroxbuster.tar.gz
+
+  build-windows:
     runs-on: ${{ matrix.os }}
-    if: github.ref == 'refs/heads/master-not-a-thing-revert-this-later-before-release'
+    if: github.ref == 'refs/heads/master'
     strategy:
       matrix:
-        type: [windows-x64, windows-x86, macos]
+        type: [windows-x64, windows-x86]
         include:
-          - type: macos
-            os: macos-latest
-            target: x86_64-apple-darwin
-            name: x86_64-macos-feroxbuster
-            path: target/x86_64-apple-darwin/release/feroxbuster
           - type: windows-x64
             os: windows-latest
             target: x86_64-pc-windows-msvc
@@ -146,3 +134,4 @@ jobs:
         with:
           name: ${{ matrix.name }}
           path: ${{ matrix.path }}
+

.github/workflows/check.yml

@@ -0,0 +1,64 @@
+name: CI Pipeline
+
+on: [push, pull_request]
+
+jobs:
+  check:
+    name: Check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+      - uses: actions-rs/cargo@v1
+        with:
+          command: check
+
+  test:
+    name: Test Suite
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+      - uses: actions-rs/cargo@v1
+        with:
+          command: test
+
+  fmt:
+    name: Rust fmt
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+      - run: rustup component add rustfmt
+      - uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --all -- --check
+
+  clippy:
+    name: Clippy
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+      - run: rustup component add clippy
+      - uses: actions-rs/cargo@v1
+        with:
+          command: clippy
+          args: --all-targets --all-features -- -D warnings -A clippy::deref_addrof

.github/workflows/coverage.yml

@@ -0,0 +1,36 @@
+on: [push]
+
+name: Code Coverage Pipeline
+
+jobs:
+  upload-coverage:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: nightly
+          override: true
+      - uses: actions-rs/cargo@v1
+        with:
+          command: clean
+      - uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --all-features --no-fail-fast
+        env:
+          CARGO_INCREMENTAL: '0'
+          RUSTFLAGS: '-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort'
+          RUSTDOCFLAGS: '-Cpanic=abort'
+      - uses: actions-rs/grcov@v0.1
+      - name: Convert lcov to xml
+        run: |
+          curl -O https://raw.githubusercontent.com/eriwen/lcov-to-cobertura-xml/master/lcov_cobertura/lcov_cobertura.py
+          chmod +x lcov_cobertura.py
+          ./lcov_cobertura.py ./lcov.info
+      - uses: codecov/codecov-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          file: ./coverage.xml
+          name: codecov-umbrella
+          fail_ci_if_error: true

.gitignore

@@ -15,3 +15,10 @@ Cargo.lock
 
 # personal feroxbuster config for testing
 ferox-config.toml
+
+# images for the README on github
+img/**
+
+# scripts to check code coverage using nightly compiler
+check-coverage.sh
+lcov_cobertura.py

Cargo.toml

@@ -1,24 +1,40 @@
 [package]
 name = "feroxbuster"
-version = "0.1.0"
-authors = ["epi <epibar052@gmail.com>"]
+version = "1.9.0"
+authors = ["Ben 'epi' Risher <epibar052@gmail.com>"]
+license = "MIT"
 edition = "2018"
+homepage = "https://github.com/epi052/feroxbuster"
+repository = "https://github.com/epi052/feroxbuster"
+description = "A fast, simple, recursive content discovery tool."
+categories = ["command-line-utilities"]
+keywords = ["pentest", "enumeration", "url-bruteforce", "content-discovery", "web"]
+exclude = [".github/*", "img/*", "check-coverage.sh"]
+
+[badges]
+maintenance = { status = "actively-developed" }
 
 [dependencies]
 futures = { version = "0.3"}
 tokio = { version = "0.2", features = ["full"] }
 tokio-util = {version = "0.3", features = ["codec"]}
 log = "0.4"
-env_logger = "0.7"
+env_logger = "0.8"
 reqwest = { version = "0.10", features = ["socks"] }
 clap = "2"
 lazy_static = "1.4"
 toml = "0.5"
 serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
 uuid = { version = "0.8", features = ["v4"] }
-ansi_term = "0.12"
 indicatif = "0.15"
 console = "0.12"
+openssl = { version = "0.10", features = ["vendored"] }
+dirs = "3.0"
+regex = "1"
+crossterm = "0.18"
+rlimit = "0.5"
+ctrlc = "3.1"
 
 [dev-dependencies]
 tempfile = "3.1"
@@ -27,7 +43,15 @@ assert_cmd = "1.0.1"
 predicates = "1.0.5"
 
 [profile.release]
-opt-level = 'z'  # optimize for size
 lto = true
 codegen-units = 1
 panic = 'abort'
+
+[package.metadata.deb]
+section = "utility"
+license-file = ["LICENSE", "4"]
+conf-files = ["/etc/feroxbuster/ferox-config.toml"]
+assets = [
+    ["target/release/feroxbuster", "/usr/bin/", "755"],
+    ["ferox-config.toml.example", "/etc/feroxbuster/ferox-config.toml", "644"],
+]

Dockerfile

@@ -0,0 +1,12 @@
+FROM alpine:latest
+LABEL maintainer="wfnintr@null.net"
+
+# download default wordlists 
+RUN apk add --no-cache --virtual .depends subversion && \
+	svn export https://github.com/danielmiessler/SecLists/trunk/Discovery/Web-Content /usr/share/seclists/Discovery/Web-Content && \
+	apk del .depends
+
+# install latest release
+RUN wget https://github.com/epi052/feroxbuster/releases/latest/download/x86_64-linux-feroxbuster.zip -qO feroxbuster.zip && unzip -d /usr/local/bin/ feroxbuster.zip feroxbuster && rm feroxbuster.zip && chmod +x /usr/local/bin/feroxbuster
+
+ENTRYPOINT ["feroxbuster"]

README.md

@@ -1,41 +1,249 @@
-# HOLUP / Hacktoberfest / Pre-release Version
+<h1 align="center">
+  <br>
+  <a href="https://github.com/epi052/feroxbuster"><img src="img/logo/default-cropped.png" alt="feroxbuster"></a>
+  <br>
+</h1>
 
-I'm making this project public earlier than I normally would for Hacktoberfest.  It is not done. I make no guarantees 
-about master even being in a state where the tool works. I'll remove this message once things stabilize, which should
-be relatively soon.
+<h4 align="center">A simple, fast, recursive content discovery tool written in Rust</h4>
 
-If you want to submit a PR as part of hacktoberfest, I'm mostly working off of the items in the 
-[Pre-release project](https://github.com/epi052/feroxbuster/projects/1).  It's very fluid as I've been working on it 
-myself up to this point.  I'll look at formalizing what's there into issues soon.
+<p align="center">
+  <a href="https://github.com/epi052/feroxbuster/actions?query=workflow%3A%22CI+Pipeline%22">
+    <img src="https://img.shields.io/github/workflow/status/epi052/feroxbuster/CI%20Pipeline/master?logo=github">
+  </a>
 
-Happy Hacktoberfest!   
+  <a href="https://github.com/epi052/feroxbuster/releases">
+    <img src="https://img.shields.io/github/downloads/epi052/feroxbuster/total?label=downloads&logo=github&color=inactive" alt="github downloads">
+  </a>
 
-# feroxbuster
+  <a href="https://github.com/epi052/feroxbuster/commits/master">
+    <img src="https://img.shields.io/github/last-commit/epi052/feroxbuster?logo=github">
+  </a>
 
-`feroxbuster` is a fast, simple, recursive content discovery tool written in Rust.
+  <a href="https://crates.io/crates/feroxbuster">
+    <img src="https://img.shields.io/crates/v/feroxbuster?color=blue&label=version&logo=rust">
+  </a>
+ 
+  <a href="https://crates.io/crates/feroxbuster">
+    <img src="https://img.shields.io/crates/d/feroxbuster?label=downloads&logo=rust&color=inactive">
+  </a>
 
-Table of Contents
+  <a href="https://codecov.io/gh/epi052/feroxbuster">
+    <img src="https://codecov.io/gh/epi052/feroxbuster/branch/master/graph/badge.svg" />
+  </a>
+</p>
+
+![demo](img/demo.gif)
+
+<p align="center">
+  🦀
+  <a href="https://github.com/epi052/feroxbuster/releases">Releases</a> ✨
+  <a href="#-example-usage">Example Usage</a> ✨
+  <a href="https://github.com/epi052/feroxbuster/blob/master/CONTRIBUTING.md">Contributing</a> ✨
+  <a href="https://docs.rs/feroxbuster/latest/feroxbuster/">Documentation</a>
+  🦀
+</p>
+
+## 😕 What the heck is a ferox anyway?
+
+Ferox is short for Ferric Oxide. Ferric Oxide, simply put, is rust.  The name rustbuster was taken, so I decided on a variation.  🤷	
+
+## 🤔 What's it do tho? 
+
+`feroxbuster` is a tool designed to perform [Forced Browsing](https://owasp.org/www-community/attacks/Forced_browsing).  
+
+Forced browsing is an attack where the aim is to enumerate and access resources that are not referenced by the web application, but are still accessible by an attacker.
+
+`feroxbuster` uses brute force combined with a wordlist to search for unlinked content in target directories. These resources may store sensitive information about web applications and operational systems, such as source code, credentials, internal network addressing, etc...
+
+This attack is also known as Predictable Resource Location, File Enumeration, Directory Enumeration, and Resource Enumeration.
+
+📖 Table of Contents
 -----------------
-- [Downloads](#downloads)
-- [Installation](#installation)
-- [Configuration](#configuration)
+- [Installation](#-installation)
+    - [Download a Release](#download-a-release)
+    - [Snap Install](#snap-install)
+    - [Homebrew on MacOS and Linux](#homebrew-on-macos-and-linux)
+    - [Cargo Install](#cargo-install)
+    - [apt Install](#apt-install)
+    - [AUR Install](#aur-install)
+    - [Docker Install](#docker-install)
+- [Configuration](#%EF%B8%8F-configuration)
     - [Default Values](#default-values)
+    - [Threads and Connection Limits At A High-Level](#threads-and-connection-limits-at-a-high-level)
     - [ferox-config.toml](#ferox-configtoml)
     - [Command Line Parsing](#command-line-parsing)
-- [Example Usage](#example-usage)
-- [Comparison w/ Similar Tools](#comparison-w-similar-tools)
+- [Example Usage](#-example-usage)
+    - [Multiple Values](#multiple-values)
+    - [Include Headers](#include-headers)
+    - [IPv6, Non-recursive scan with INFO logging enabled](#ipv6-non-recursive-scan-with-info-level-logging-enabled)
+    - [Read urls from STDIN; pipe only resulting urls out to another tool](#read-urls-from-stdin-pipe-only-resulting-urls-out-to-another-tool)
+    - [Proxy traffic through Burp](#proxy-traffic-through-burp)
+    - [Proxy traffic through a SOCKS proxy](#proxy-traffic-through-a-socks-proxy)
+    - [Pass auth token via query parameter](#pass-auth-token-via-query-parameter)
+    - [Extract Links from Response Body (new in `v1.1.0`)](#extract-links-from-response-body-new-in-v110)
+    - [Limit Total Number of Concurrent Scans (new in `v1.2.0`)](#limit-total-number-of-concurrent-scans-new-in-v120)
+    - [Filter Response by Status Code  (new in `v1.3.0`)](#filter-response-by-status-code--new-in-v130)
+    - [Pause an Active Scan (new in `v1.4.0`)](#pause-an-active-scan-new-in-v140)
+    - [Replay Responses to a Proxy based on Status Code (new in `v1.5.0`)](#replay-responses-to-a-proxy-based-on-status-code-new-in-v150)
+    - [Filter Response Using a Regular Expression (new in `v1.8.0`)](#filter-response-using-a-regular-expression-new-in-v180)
+    - [Stop and Resume Scans (save scan's state to disk) (new in `v1.9.0`)](#stop-and-resume-scans---resume-from-file-new-in-v190)
+- [Comparison w/ Similar Tools](#-comparison-w-similar-tools)
+- [Common Problems/Issues (FAQ)](#-common-problemsissues-faq)
+    - [No file descriptors available](#no-file-descriptors-available)
+    - [Progress bars print one line at a time](#progress-bars-print-one-line-at-a-time)
 
-## Downloads
-There are pre-built binaries for the following systems:
+## 💿 Installation
 
-- [Linux x86](https://github.com/epi052/feroxbuster/releases/latest/download/x86-linux-feroxbuster.zip)
-- [Linux x86_64](https://github.com/epi052/feroxbuster/releases/latest/download/x86_64-linux-feroxbuster.zip)
-- [MacOS x86_64](https://github.com/epi052/feroxbuster/releases/latest/download/x86_64-macos-feroxbuster.zip)
-- [Windows x86](https://github.com/epi052/feroxbuster/releases/latest/download/x86-windows-feroxbuster.exe.zip)
-- [Windows x86_64](https://github.com/epi052/feroxbuster/releases/latest/download/x86_64-windows-feroxbuster.exe.zip)
+### Download a Release
 
-## Installation
-## Configuration
+Releases for multiple architectures can be found in the [Releases](https://github.com/epi052/feroxbuster/releases) section.  The latest release for each of the following systems can be downloaded and executed as shown below.
+
+#### Linux (32 and 64-bit) & MacOS
+```
+curl -sL https://raw.githubusercontent.com/epi052/feroxbuster/master/install-nix.sh | bash
+```
+
+#### Windows x86
+
+```
+https://github.com/epi052/feroxbuster/releases/latest/download/x86-windows-feroxbuster.exe.zip
+Expand-Archive .\feroxbuster.zip
+.\feroxbuster\feroxbuster.exe -V
+```
+
+#### Windows x86_64
+
+```
+Invoke-WebRequest https://github.com/epi052/feroxbuster/releases/latest/download/x86_64-windows-feroxbuster.exe.zip -OutFile feroxbuster.zip
+Expand-Archive .\feroxbuster.zip
+.\feroxbuster\feroxbuster.exe -V
+```
+
+### Snap Install
+
+Install using `snap`
+
+```
+sudo snap install feroxbuster
+```
+
+The only gotcha here is that the snap package can only read wordlists from a few specific locations. There are a few 
+possible solutions, of which two are shown below.
+
+If the wordlist is on the same partition as your home directory, it can be hard-linked into `~/snap/feroxbuster/common`
+
+```
+ln /path/to/the/wordlist ~/snap/feroxbuster/common
+./feroxbuster -u http://localhost -w ~/snap/feroxbuster/common/wordlist
+``` 
+
+If the wordlist is on a separate partition, hard-linking won't work.  You'll need to copy it into the snap directory.
+
+```
+cp /path/to/the/wordlist ~/snap/feroxbuster/common
+./feroxbuster -u http://localhost -w ~/snap/feroxbuster/common/wordlist
+``` 
+
+### Homebrew on MacOS and Linux
+
+Install using Homebrew via tap
+
+🍏 [MacOS](https://github.com/TGotwig/homebrew-feroxbuster/blob/main/feroxbuster.rb)
+
+```shell
+brew tap tgotwig/feroxbuster
+brew install feroxbuster
+```
+
+🐧 [Linux](https://github.com/TGotwig/homebrew-linux-feroxbuster/blob/main/feroxbuster.rb)
+
+```shell
+brew tap tgotwig/linux-feroxbuster
+brew install feroxbuster
+```
+
+### Cargo Install
+
+`feroxbuster` is published on crates.io, making it easy to install if you already have rust installed on your system.
+
+```
+cargo install feroxbuster
+```
+
+### apt Install
+
+Download `feroxbuster_amd64.deb` from the [Releases](https://github.com/epi052/feroxbuster/releases) section.  After that, use your favorite package manager to install the `.deb`.
+
+```
+wget -sLO https://github.com/epi052/feroxbuster/releases/latest/download/feroxbuster_amd64.deb.zip
+unzip feroxbuster_amd64.deb.zip
+sudo apt install ./feroxbuster_amd64.deb
+```
+
+### AUR Install
+
+Install `feroxbuster-git` on Arch Linux with your AUR helper of choice:
+
+```
+yay -S feroxbuster-git
+```
+
+### Docker Install
+
+> The following steps assume you have docker installed / setup
+
+First, clone the repository.
+
+```
+git clone https://github.com/epi052/feroxbuster.git
+cd feroxbuster
+```
+
+Next, build the image.
+
+```
+sudo docker build -t feroxbuster .
+```
+
+After that, you should be able to use `docker run` to perform scans with `feroxbuster`.
+
+#### Basic usage
+
+```
+sudo docker run --init -it feroxbuster -u http://example.com -x js,html
+```
+
+#### Piping from stdin and proxying all requests through socks5 proxy
+
+```
+cat targets.txt | sudo docker run --net=host --init -i feroxbuster --stdin -x js,html --proxy socks5://127.0.0.1:9050
+```
+
+#### Mount a volume to pass in `ferox-config.toml`
+
+You've got some options available if you want to pass in a config file.  [`ferox-buster.toml`](#ferox-configtoml) can live in multiple locations and still be valid, so it's up to you how you'd like to pass it in.  Below are a few valid examples:
+
+```
+sudo docker run --init -v $(pwd)/ferox-config.toml:/etc/feroxbuster/ferox-config.toml -it feroxbuster -u http://example.com
+```
+
+```
+sudo docker run --init -v ~/.config/feroxbuster:/root/.config/feroxbuster -it feroxbuster -u http://example.com
+```
+
+Note: If you are on a SELinux enforced system, you will need to pass the `:Z` attribute also.
+
+```
+docker run --init -v (pwd)/ferox-config.toml:/etc/feroxbuster/ferox-config.toml:Z -it feroxbuster -u http://example.com
+```
+
+#### Define an alias for simplicity
+
+```
+alias feroxbuster="sudo docker run --init -v ~/.config/feroxbuster:/root/.config/feroxbuster -i feroxbuster"
+```
+
+## ⚙️ Configuration
 ### Default Values
 Configuration begins with with the following built-in default values baked into the binary:
 
@@ -44,17 +252,51 @@ Configuration begins with with the following built-in default values baked into
 - wordlist: `/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt`
 - threads: `50`
 - verbosity: `0` (no logging enabled)
-- statuscodes: `200 204 301 302 307 308 401 403 405`
-- useragent: `feroxbuster/VERSION`
+- scan_limit: `0` (no limit imposed on concurrent scans)
+- status_codes: `200 204 301 302 307 308 401 403 405`
+- user_agent: `feroxbuster/VERSION`
 - recursion depth: `4`
 - auto-filter wildcards - `true`
 - output: `stdout`
+- save_state: `true` (create a state file in cwd when `Ctrl+C` is received)
+
+### Threads and Connection Limits At A High-Level
+
+This section explains how the `-t` and `-L` options work together to determine the overall aggressiveness of a scan. The combination of the two values set by these options determines how hard your target will get hit and to some extent also determines how many resources will be consumed on your local machine.
+
+#### A Note on Green Threads
+
+`feroxbuster` uses so-called [green threads](https://en.wikipedia.org/wiki/Green_threads) as opposed to traditional kernel/OS threads. This means (at a high-level) that the threads are implemented entirely in userspace, within a single running process. As a result, a scan with 30 green threads will appear to the OS to be a single process with no additional light-weight processes associated with it as far as the kernel is concerned. As such, there will not be any impact to process (`nproc`) limits when specifying larger values for `-t`. However, these threads will still consume file descriptors, so you will need to ensure that you have a suitable `nlimit` set when scaling up the amount of threads. More detailed documentation on setting appropriate `nlimit` values can be found in the [No File Descriptors Available](#no-file-descriptors-available) section of the FAQ
+
+#### Threads and Connection Limits: The Implementation
+
+* Threads: The `-t` option specifies the maximum amount of active threads *per-directory* during a scan
+* Connection Limits: The `-L` option specifies the maximum amount of active connections per thread
+
+#### Threads and Connection Limits: Examples
+
+To truly have only 30 active requests to a site at any given time, `-t 30 -L 1` is necessary. Using `-t 30 -L 2` will result in a maximum of 60 total requests being processed at any given time for that site. And so on. For a conversation on this, please see [Issue #126](https://github.com/epi052/feroxbuster/issues/126) which may provide more (or less) clarity :wink:
 
 ### ferox-config.toml
 After setting built-in default values, any values defined in a `ferox-config.toml` config file will override the
-built-in defaults.  If `ferox-config.toml` is not found in the **same directory** as `feroxbuster`, nothing happens at this stage. 
+built-in defaults.  
 
-For example, say that we prefer to use a different wordlist as our default when scanning; we can
+`feroxbuster` searches for `ferox-config.toml` in the following locations (in the order shown):
+- `/etc/feroxbuster/` (global)
+- `CONFIG_DIR/ferxobuster/` (per-user)
+- The same directory as the `feroxbuster` executable (per-user)
+- The user's current working directory (per-target)
+
+> `CONFIG_DIR` is defined as the following:
+> - Linux: `$XDG_CONFIG_HOME` or `$HOME/.config` i.e. `/home/bob/.config`
+> - MacOs: `$HOME/Library/Application Support` i.e. `/Users/bob/Library/Application Support`
+> - Windows: `{FOLDERID_RoamingAppData}` i.e. `C:\Users\Bob\AppData\Roaming`
+
+If more than one valid configuration file is found, each one overwrites the values found previously.  
+
+If no configuration file is found, nothing happens at this stage.
+
+As an example, let's say that we prefer to use a different wordlist as our default when scanning; we can
 set the `wordlist` value in the config file to override the baked-in default.
 
 Notes of interest:
@@ -80,24 +322,35 @@ A pre-made configuration file with examples of all available settings can be fou
 # Any setting used here can be overridden by the corresponding command line option/argument
 #
 # wordlist = "/wordlists/jhaddix/all.txt"
-# statuscodes = [200, 500]
+# status_codes = [200, 500]
+# filter_status = [301]
 # threads = 1
 # timeout = 5
 # proxy = "http://127.0.0.1:8080"
+# replay_proxy = "http://127.0.0.1:8081"
+# replay_codes = [200, 302]
 # verbosity = 1
+# scan_limit = 6
 # quiet = true
+# json = true
 # output = "/targets/ellingson_mineral_company/gibson.txt"
-# useragent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0"
+# debug_log = "/var/log/find-the-derp.log"
+# user_agent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0"
 # redirects = true
 # insecure = true
 # extensions = ["php", "html"]
-# norecursion = true
-# addslash = true
+# no_recursion = true
+# add_slash = true
 # stdin = true
-# dontfilter = true
+# dont_filter = true
+# extract_links = true
 # depth = 1
-# sizefilters = [5174]
+# filter_size = [5174]
+# filter_regex = ["^ignore me$"]
+# filter_word_count = [993]
+# filter_line_count = [35, 36]
 # queries = [["name","value"], ["rick", "astley"]]
+# save_state = false
 
 # headers can be specified on multiple lines or as an inline table
 #
@@ -121,36 +374,248 @@ USAGE:
     feroxbuster [FLAGS] [OPTIONS] --url <URL>...
 
 FLAGS:
-    -f, --addslash       Append / to each request
-    -D, --dontfilter     Don't auto-filter wildcard responses
-    -h, --help           Prints help information
-    -k, --insecure       Disables TLS certificate validation
-    -n, --norecursion    Do not scan recursively
-    -q, --quiet          Only print URLs; Don't print status codes, response size, running config, etc...
-    -r, --redirects      Follow redirects
-        --stdin          Read url(s) from STDIN
-    -V, --version        Prints version information
-    -v, --verbosity      Increase verbosity level (use -vv or more for greater effect)
+    -f, --add-slash        Append / to each request
+    -D, --dont-filter      Don't auto-filter wildcard responses
+    -e, --extract-links    Extract links from response body (html, javascript, etc...); make new requests based on
+                           findings (default: false)
+    -h, --help             Prints help information
+    -k, --insecure         Disables TLS certificate validation
+        --json             Emit JSON logs to --output and --debug-log instead of normal text
+    -n, --no-recursion     Do not scan recursively
+    -q, --quiet            Only print URLs; Don't print status codes, response size, running config, etc...
+    -r, --redirects        Follow redirects
+        --stdin            Read url(s) from STDIN
+    -V, --version          Prints version information
+    -v, --verbosity        Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v's is probably
+                           too much)
 
 OPTIONS:
+        --debug-log <FILE>                  Output file to write log entries (use w/ --json for JSON entries)
     -d, --depth <RECURSION_DEPTH>           Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)
     -x, --extensions <FILE_EXTENSION>...    File extension(s) to search for (ex: -x php -x pdf js)
+    -N, --filter-lines <LINES>...           Filter out messages of a particular line count (ex: -N 20 -N 31,30)
+    -X, --filter-regex <REGEX>...           Filter out messages via regular expression matching on the response's body
+                                            (ex: -X '^ignore me$')
+    -S, --filter-size <SIZE>...             Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)
+    -C, --filter-status <STATUS_CODE>...    Filter out status codes (deny list) (ex: -C 200 -C 401)
+    -W, --filter-words <WORDS>...           Filter out messages of a particular word count (ex: -W 312 -W 91,82)
     -H, --headers <HEADER>...               Specify HTTP headers (ex: -H Header:val 'stuff: things')
-    -o, --output <FILE>                     Output file to write results to (default: stdout)
+    -o, --output <FILE>                     Output file to write results to (use w/ --json for JSON entries)
     -p, --proxy <PROXY>                     Proxy to use for requests (ex: http(s)://host:port, socks5://host:port)
     -Q, --query <QUERY>...                  Specify URL query parameters (ex: -Q token=stuff -Q secret=key)
-    -S, --sizefilter <SIZE>...              Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)
-    -s, --statuscodes <STATUS_CODE>...      Status Codes of interest (default: 200 204 301 302 307 308 401 403 405)
+    -R, --replay-codes <REPLAY_CODE>...     Status Codes to send through a Replay Proxy when found (default: --status
+                                            -codes value)
+    -P, --replay-proxy <REPLAY_PROXY>       Send only unfiltered requests through a Replay Proxy, instead of all
+                                            requests
+        --resume-from <STATE_FILE>          State file from which to resume a partially complete scan (ex. --resume-from
+                                            ferox-1606586780.state)
+    -L, --scan-limit <SCAN_LIMIT>           Limit total number of concurrent scans (default: 0, i.e. no limit)
+    -s, --status-codes <STATUS_CODE>...     Status Codes to include (allow list) (default: 200 204 301 302 307 308 401
+                                            403 405)
     -t, --threads <THREADS>                 Number of concurrent threads (default: 50)
     -T, --timeout <SECONDS>                 Number of seconds before a request times out (default: 7)
     -u, --url <URL>...                      The target URL(s) (required, unless --stdin used)
-    -a, --useragent <USER_AGENT>            Sets the User-Agent (default: feroxbuster/VERSION)
+    -a, --user-agent <USER_AGENT>           Sets the User-Agent (default: feroxbuster/VERSION)
     -w, --wordlist <FILE>                   Path to the wordlist
 ```
 
-## Example Usage
+## 🧰 Example Usage
 
-## Comparison w/ Similar Tools
+### Multiple Values
+
+Options that take multiple values are very flexible.  Consider the following ways of specifying extensions:
+
+```
+./feroxbuster -u http://127.1 -x pdf -x js,html -x php txt json,docx
+```
+
+The command above adds .pdf, .js, .html, .php, .txt, .json, and .docx to each url
+
+All of the methods above (multiple flags, space separated, comma separated, etc...) are valid and interchangeable.  The same goes for urls, headers, status codes, queries, and size filters.
+
+### Include Headers
+
+```
+./feroxbuster -u http://127.1 -H Accept:application/json "Authorization: Bearer {token}"
+```
+
+### IPv6, non-recursive scan with INFO-level logging enabled
+
+```
+./feroxbuster -u http://[::1] --no-recursion -vv
+```
+
+### Read urls from STDIN; pipe only resulting urls out to another tool
+
+```
+cat targets | ./feroxbuster --stdin --quiet -s 200 301 302 --redirects -x js | fff -s 200 -o js-files
+```
+
+### Proxy traffic through Burp
+
+```
+./feroxbuster -u http://127.1 --insecure --proxy http://127.0.0.1:8080
+```
+
+### Proxy traffic through a SOCKS proxy
+
+```
+./feroxbuster -u http://127.1 --proxy socks5://127.0.0.1:9050
+```
+
+### Pass auth token via query parameter 
+
+```
+./feroxbuster -u http://127.1 --query token=0123456789ABCDEF
+```
+
+### Extract Links from Response Body (New in `v1.1.0`) 
+
+Search through the body of valid responses (html, javascript, etc...) for additional endpoints to scan. This turns
+`feroxbuster` into a hybrid that looks for both linked and unlinked content. 
+
+Example request/response with `--extract-links` enabled:
+- Make request to `http://example.com/index.html`
+- Receive, and read in, the `body` of the response
+- Search the `body` for absolute and relative links (i.e. `homepage/assets/img/icons/handshake.svg`)
+- Add the following directories for recursive scanning:
+    - `http://example.com/homepage`
+    - `http://example.com/homepage/assets`
+    - `http://example.com/homepage/assets/img`
+    - `http://example.com/homepage/assets/img/icons`
+- Make a single request to `http://example.com/homepage/assets/img/icons/handshake.svg`
+
+```
+./feroxbuster -u http://127.1 --extract-links
+```
+
+Here's a comparison of a wordlist-only scan vs `--extract-links` using [Feline](https://www.hackthebox.eu/home/machines/profile/274) from Hack the Box:
+
+Wordlist only
+
+![normal-scan-cmp-extract](img/normal-scan-cmp-extract.gif)
+
+With `--extract-links`
+
+![extract-scan-cmp-normal](img/extract-scan-cmp-normal.gif)
+
+### Limit Total Number of Concurrent Scans (new in `v1.2.0`)
+
+Limit the number of scans permitted to run at any given time.  Recursion will still identify new directories, but newly
+discovered directories can only begin scanning when the total number of active scans drops below the value passed to 
+`--scan-limit`.
+
+```
+./feroxbuster -u http://127.1 --scan-limit 2
+```
+
+![limit-demo](img/limit-demo.gif)
+
+### Filter Response by Status Code  (new in `v1.3.0`)
+
+Version 1.3.0 included an overhaul to the filtering system which will allow for a wide array of filters to be added 
+with minimal effort. The first such filter is a Status Code Filter. As responses come back from the scanned server,
+each one is checked against a list of known filters and either displayed or not according to which filters are set.
+
+```
+./feroxbuster -u http://127.1 --filter-status 301
+```
+
+### Pause an Active Scan (new in `v1.4.0`)
+
+Scans can be paused and resumed by pressing the ENTER key (shown below)
+
+![pause-resume-demo](img/pause-resume-demo.gif)
+
+### Replay Responses to a Proxy based on Status Code (new in `v1.5.0`)
+
+The `--replay-proxy` and `--replay-codes` options were added as a way to only send a select few responses to a proxy.  This is in stark contrast to `--proxy` which proxies EVERY request.  
+
+Imagine you only care about proxying responses that have either the status code `200` or `302` (or you just don't want to clutter up your Burp history).  These two options will allow you to fine-tune what gets proxied and what doesn't.  
+
+```
+./feroxbuster -u http://127.1 --replay-proxy http://localhost:8080 --replay-codes 200 302 --insecure
+```
+
+Of note: this means that for every response that matches your replay criteria, you'll end up sending the request that generated that response a second time.  Depending on the target and your engagement terms (if any), it may not make sense from a traffic generated perspective.
+
+![replay-proxy-demo](img/replay-proxy-demo.gif)
+
+### Filter Response Using a Regular Expression (new in `v1.8.0`) 
+
+Version 1.3.0 included an overhaul to the filtering system which will allow for a wide array of filters to be added 
+with minimal effort. The latest addition is a Regular Expression Filter. As responses come back from the scanned server,
+the **body** of the response is checked against the filter's regular expression.  If the expression is found in the 
+body, then that response is filtered out.  
+
+**NOTE: Using regular expressions to filter large responses or many regular expressions may negatively impact performance.**  
+
+```
+./feroxbuster -u http://127.1 --filter-regex '[aA]ccess [dD]enied.?' --output results.txt --json
+```
+
+### Stop and Resume Scans (`--resume-from FILE`) (new in `v1.9.0`)
+
+Version 1.9.0 adds a few features that allow for completely stopping a scan, and resuming that same scan from a file on disk. 
+
+A simple `Ctrl+C` during a scan will create a file that contains information about the scan that was cancelled.
+
+![save-state](img/save-state.png)
+
+```json
+// example snippet of state file
+
+{
+   "scans":[
+      {
+         "id":"057016a14769414aac9a7a62707598cb",
+         "url":"https://localhost.com",
+         "scan_type":"Directory",
+         "complete":true
+      },
+      {
+         "id":"400b2323a16f43468a04ffcbbeba34c6",
+         "url":"https://localhost.com/css",
+         "scan_type":"Directory",
+         "complete":false
+      }
+   ],
+   "config":{
+      "wordlist":"/wordlists/seclists/Discovery/Web-Content/common.txt",
+      "...":"..."
+   },
+   "responses":[
+      {
+         "type":"response",
+         "url":"https://localhost.com/Login",
+         "path":"/Login",
+         "wildcard":false,
+         "status":302,
+         "content_length":0,
+         "line_count":0,
+         "word_count":0,
+         "headers":{
+            "content-length":"0",
+            "server":"nginx/1.16.1"
+         }
+      }
+   ]
+},
+```
+
+Based on the example image above, the same scan can be resumed by using `feroxbuster --resume-from ferox-http_localhost-1606947491.state`.  Directories that were already complete are not rescanned, however partially complete scans are started from the beginning.  
+
+![resumed-scan](img/resumed-scan.gif)
+
+In order to prevent state file creation when `Ctrl+C` is pressed, you can simply add the entry below to your `ferox-config.toml`.
+
+```toml
+# ferox-config.toml
+
+save_state = false
+```
+
+## 🧐 Comparison w/ Similar Tools
 
 There are quite a few similar tools for forced browsing/content discovery.  Burp Suite Pro, Dirb, Dirbuster, etc... 
 However, in my opinion, there are two that set the standard: [gobuster](https://github.com/OJ/gobuster) and 
@@ -164,29 +629,111 @@ a few of the use-cases in which feroxbuster may be a better fit:
 - You want to be able to run your content discovery as part of some crazy 12 command unix **pipeline extravaganza**
 - You want to scan through a **SOCKS** proxy
 - You want **auto-filtering** of Wildcard responses by default
+- You want an integrated **link extractor** to increase discovered endpoints
 - You want **recursion** along with some other thing mentioned above (ffuf also does recursion)
 - You want a **configuration file** option for overriding built-in default values for your scans
 
-|                                                     | feroxbuster        | gobuster           | ffuf               |
-|-----------------------------------------------------|--------------------|--------------------|--------------------|
-| fast                                                | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: |
-| easy to use                                         | :heavy_check_mark: | :heavy_check_mark: |                    |
-| blacklist status codes (in addition to whitelist)   |                    | :heavy_check_mark: | :heavy_check_mark: |
-| allows recursion                                    | :heavy_check_mark: |                    | :heavy_check_mark: |
-| can specify query parameters                        | :heavy_check_mark: |                    | :heavy_check_mark: |
-| SOCKS proxy support                                 | :heavy_check_mark: |                    |                    |
-| multiple target scan (via stdin or multiple -u)     | :heavy_check_mark: |                    |                    |
-| configuration file for default value override       | :heavy_check_mark: |                    | :heavy_check_mark: |
-| can accept urls via STDIN as part of a pipeline     | :heavy_check_mark: |                    |                    |
-| can accept wordlists via STDIN                      |                    | :heavy_check_mark: |                    |
-| filter by response size                             | :heavy_check_mark: |                    | :heavy_check_mark: |
-| auto-filter wildcard responses                      | :heavy_check_mark: |                    | :heavy_check_mark: |
-| performs other scans (vhost, dns, etc)              |                    | :heavy_check_mark: | :heavy_check_mark: |
-| time delay / rate limiting                          |                    | :heavy_check_mark: | :heavy_check_mark: |
-| **huge** number of other options                    |                    |                    | :heavy_check_mark: |
+|                                                                  | feroxbuster | gobuster | ffuf |
+|------------------------------------------------------------------|---|---|---|
+| fast                                                             | ✔ | ✔ | ✔ |
+| easy to use                                                      | ✔ | ✔ |   |
+| filter out responses by status code (new in `v1.3.0`)            | ✔ | ✔ | ✔ |
+| allows recursion                                                 | ✔ |   | ✔ |
+| can specify query parameters                                     | ✔ |   | ✔ |
+| SOCKS proxy support                                              | ✔ |   |   |
+| extracts links from response body to increase scan coverage      | ✔ |   |   |
+| multiple target scan (via stdin or multiple -u)                  | ✔ |   | ✔ |
+| configuration file for default value override                    | ✔ |   | ✔ |
+| can accept urls via STDIN as part of a pipeline                  | ✔ |   | ✔ |
+| can accept wordlists via STDIN                                   |   | ✔ | ✔ |
+| filter based on response size, wordcount, and linecount          | ✔ |   | ✔ |
+| auto-filter wildcard responses                                   | ✔ |   | ✔ |
+| performs other scans (vhost, dns, etc)                           |   | ✔ | ✔ |
+| time delay / rate limiting                                       |   | ✔ | ✔ |
+| **huge** number of other options                                 |   |   | ✔ |
 
 Of note, there's another written-in-rust content discovery tool, [rustbuster](https://github.com/phra/rustbuster). I 
-came across rustbuster when I was naming my tool (:cry:). I don't have any experience using it, but it appears to 
+came across rustbuster when I was naming my tool (😢). I don't have any experience using it, but it appears to 
 be able to do POST requests with an HTTP body, has SOCKS support, and has an 8.3 shortname scanner (in addition to vhost
 dns, directory, etc...).  In short, it definitely looks interesting and may be what you're looking for as it has some 
 capability I haven't seen in similar tools.  
+
+## 🤯 Common Problems/Issues (FAQ)
+
+### No file descriptors available
+
+Why do I get a bunch of `No file descriptors available (os error 24)` errors?
+
+---
+
+There are a few potential causes of this error.  The simplest is that your operating system sets an open file limit that is aggressively low.  Through personal testing, I've found that `4096` is a reasonable open file limit (this will vary based on your exact setup).
+
+There are quite a few options to solve this particular problem, of which a handful are shown below.  
+
+#### Increase the Number of Open Files
+
+We'll start by increasing the number of open files the OS allows. On my Kali install, the default was `1024`, and I know some MacOS installs use `256` 😕.
+
+##### Edit `/etc/security/limits.conf`
+
+One option to up the limit is to edit `/etc/security/limits.conf` so that it includes the two lines below.  
+
+- `*` represents all users
+- `hard` and `soft` indicate the hard and soft limits for the OS 
+- `nofile` is the number of open files option. 
+
+```
+/etc/security/limits.conf
+-------------------------
+...
+*        soft nofile 4096
+*        hard nofile 8192
+...
+```
+
+##### Use `ulimit` directly
+
+A faster option, that is **not** persistent, is to simply use the `ulimit` command to change the setting.
+
+```
+ulimit -n 4096
+```
+
+#### Additional Tweaks (may not be needed)
+
+If you still find yourself hitting the file limit with the above changes, there are a few additional tweaks that may help.  
+
+> This section was shamelessly stolen from this [stackoverflow answer](https://stackoverflow.com/a/3923785).  More information is included in that post and is recommended reading if you end up needing to use this section.
+
+✨ Special thanks to HTB user [@sparkla](https://www.hackthebox.eu/home/users/profile/221599) for their help with identifying these additional tweaks ✨
+
+##### Increase the ephemeral port range, and decrease the tcp_fin_timeout.
+
+The ephermal port range defines the maximum number of outbound sockets a host can create from a particular I.P. address. The fin_timeout defines the minimum time these sockets will stay in TIME_WAIT state (unusable after being used once). Usual system defaults are
+
+- `net.ipv4.ip_local_port_range = 32768   61000`
+- `net.ipv4.tcp_fin_timeout = 60`
+
+This basically means your system cannot consistently guarantee more than `(61000 - 32768) / 60 = 470` sockets per second.
+
+```
+sudo sysctl net.ipv4.ip_local_port_range="15000 61000"
+sudo sysctl net.ipv4.tcp_fin_timeout=30
+```
+
+##### Allow socket reuse while in a `TIME_WAIT` status
+
+This allows fast cycling of sockets in time_wait state and re-using them. Make sure to read post [Coping with the TCP TIME-WAIT](https://vincent.bernat.ch/en/blog/2014-tcp-time-wait-state-linux) from Vincent Bernat to understand the implications.
+
+```
+sudo sysctl net.ipv4.tcp_tw_reuse=1 
+```
+
+### Progress bars print one line at a time
+
+`feroxbuster` needs a terminal width of at least the size of what's being printed in order to do progress bar printing correctly.  If your width is too small, you may see output like what's shown below.
+
+![small-term](img/small-term.png)
+
+If you can, simply make the terminal wider and rerun.  If you're unable to make your terminal wider
+consider using `-q` to suppress the progress bars.

ferox-config.toml.example

@@ -8,24 +8,35 @@
 # Any setting used here can be overridden by the corresponding command line option/argument
 #
 # wordlist = "/wordlists/seclists/Discovery/Web-Content/raft-medium-directories.txt"
-# statuscodes = [200, 500]
+# status_codes = [200, 500]
+# filter_status = [301]
 # threads = 1
 # timeout = 5
 # proxy = "http://127.0.0.1:8080"
+# replay_proxy = "http://127.0.0.1:8081"
+# replay_codes = [200, 302]
 # verbosity = 1
+# scan_limit = 6
 # quiet = true
+# json = true
 # output = "/targets/ellingson_mineral_company/gibson.txt"
-# useragent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0"
+# debug_log = "/var/log/find-the-derp.log"
+# user_agent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0"
 # redirects = true
 # insecure = true
 # extensions = ["php", "html"]
-# norecursion = true
-# addslash = true
+# no_recursion = true
+# add_slash = true
 # stdin = true
-# dontfilter = true
+# dont_filter = true
+# extract_links = true
 # depth = 1
-# sizefilters = [5174]
+# filter_size = [5174]
+# filter_regex = ["^ignore me$"]
+# filter_word_count = [993]
+# filter_line_count = [35, 36]
 # queries = [["name","value"], ["rick", "astley"]]
+# save_state = false
 
 # headers can be specified on multiple lines or as an inline table
 #

install-nix.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+
+BASE_URL=https://github.com/epi052/feroxbuster/releases/latest/download
+
+MAC_ZIP=x86_64-macos-feroxbuster.zip
+MAC_URL="${BASE_URL}/${MAC_ZIP}"
+
+LIN32_ZIP=x86-linux-feroxbuster.zip
+LIN32_URL="${BASE_URL}/${LIN32_ZIP}"
+
+LIN64_ZIP=x86_64-linux-feroxbuster.zip
+LIN64_URL="${BASE_URL}/${LIN64_ZIP}"
+
+EMOJI_URL=https://gist.github.com/epi052/8196b550ea51d0907ad4b93751b1b57d/raw/6112c9f32ae07922983fdc549c54fd3fb9a38e4c/NotoColorEmoji.ttf
+
+echo "[+] Installing feroxbuster!"
+
+if [[ "$(uname)" == "Darwin" ]]; then
+    echo "[=] Found MacOS, downloading from ${MAC_URL}"
+
+    curl -sLO "${MAC_URL}"
+    unzip -o "${MAC_ZIP}" > /dev/null
+    rm "${MAC_ZIP}"
+elif [[ "$(expr substr $(uname -s) 1 5)" == "Linux" ]]; then
+    if [[ $(getconf LONG_BIT) == 32 ]]; then
+        echo "[=] Found 32-bit Linux, downloading from ${LIN32_URL}"
+
+        curl -sLO "${LIN32_URL}"
+        unzip -o "${LIN32_ZIP}" > /dev/null
+        rm "${LIN32_ZIP}"
+    else
+        echo "[=] Found 64-bit Linux, downloading from ${LIN64_URL}"
+
+        curl -sLO "${LIN64_URL}"
+        unzip -o "${LIN64_ZIP}" > /dev/null
+        rm "${LIN64_ZIP}"
+    fi
+
+    echo "[=] Installing Noto Emoji Font"
+    mkdir -p ~/.fonts
+    pushd ~/.fonts 2>&1 >/dev/null
+
+    curl -sLO "${EMOJI_URL}"
+
+    fc-cache -f -v >/dev/null
+
+    popd 2>&1 >/dev/null
+    echo "[+] Noto Emoji Font installed"
+fi
+
+chmod +x ./feroxbuster
+
+echo "[+] Installed feroxbuster version $(./feroxbuster -V)"
+
+
+

snapcraft.yaml

@@ -0,0 +1,41 @@
+name: feroxbuster
+version: git
+summary: A simple, fast, recursive content discovery tool written in Rust
+description: |
+  feroxbuster is a tool designed to perform Forced Browsing.
+
+  Forced browsing is an attack where the aim is to enumerate and access resources that are not referenced by the web application, but are still accessible by an attacker.
+
+  feroxbuster uses brute force combined with a wordlist to search for unlinked content in target directories. These resources may store sensitive information about web applications and operational systems, such as source code, credentials, internal network addressing, etc...
+
+  This attack is also known as Predictable Resource Location, File Enumeration, Directory Enumeration, and Resource Enumeration.
+
+
+base: core18
+
+plugs:
+  etc-feroxbuster:
+    interface: system-files
+    read:
+    - /etc/feroxbuster
+  dot-config-feroxbuster:
+    interface: personal-files
+    read:
+    - $HOME/.config/feroxbuster
+
+architectures:
+  - build-on: amd64
+  - build-on: i386
+
+parts:
+  feroxbuster:
+    plugin: rust
+    source: .
+
+apps:
+  feroxbuster:
+    command: bin/feroxbuster
+    plugs:
+      - etc-feroxbuster
+      - dot-config-feroxbuster
+      - network

src/banner.rs

@@ -1,5 +1,11 @@
-use crate::{config::CONFIGURATION, utils::status_colorizer, VERSION};
+use crate::config::{Configuration, CONFIGURATION};
+use crate::utils::{make_request, status_colorizer};
+use console::style;
+use reqwest::{Client, Url};
+use serde_json::Value;
+use std::io::Write;
 
+/// macro helper to abstract away repetitive string formatting
 macro_rules! format_banner_entry_helper {
     // \u{0020} -> unicode space
     // \u{2502} -> vertical box drawing character, i.e. │
@@ -26,6 +32,7 @@ macro_rules! format_banner_entry_helper {
     };
 }
 
+/// macro that wraps another macro helper to abstract away repetitive string formatting
 macro_rules! format_banner_entry {
     // 4 -> unicode emoji padding width
     // 22 -> column width (when unicode rune is 4 bytes wide, 23 when it's 3)
@@ -38,196 +45,655 @@ macro_rules! format_banner_entry {
     };
 }
 
+/// Url used to query github's api; specifically used to look for the latest tagged release name
+const UPDATE_URL: &str = "https://api.github.com/repos/epi052/feroxbuster/releases/latest";
+
+/// Simple enum to hold three different update states
+#[derive(Debug)]
+enum UpdateStatus {
+    /// this version and latest release are the same
+    UpToDate,
+
+    /// this version and latest release are not the same
+    OutOfDate,
+
+    /// some error occurred during version check
+    Unknown,
+}
+
+/// Makes a request to the given url, expecting to receive a JSON response that contains a field
+/// named `tag_name` that holds a value representing the latest tagged release of this tool.
+///
+/// ex: v1.1.0
+///
+/// Returns `UpdateStatus`
+async fn needs_update(client: &Client, url: &str, bin_version: &str) -> UpdateStatus {
+    log::trace!("enter: needs_update({:?}, {})", client, url);
+
+    let unknown = UpdateStatus::Unknown;
+
+    let api_url = match Url::parse(url) {
+        Ok(url) => url,
+        Err(e) => {
+            log::error!("{}", e);
+            log::trace!("exit: needs_update -> {:?}", unknown);
+            return unknown;
+        }
+    };
+
+    if let Ok(response) = make_request(&client, &api_url).await {
+        let body = response.text().await.unwrap_or_default();
+
+        let json_response: Value = serde_json::from_str(&body).unwrap_or_default();
+
+        if json_response.is_null() {
+            // unwrap_or_default above should result in a null value for the json_response variable
+            log::error!("Could not parse JSON from response body");
+            log::trace!("exit: needs_update -> {:?}", unknown);
+            return unknown;
+        }
+
+        let latest_version = match json_response["tag_name"].as_str() {
+            Some(tag) => tag.trim_start_matches('v'),
+            None => {
+                log::error!("Could not get version field from JSON response");
+                log::debug!("{}", json_response);
+                log::trace!("exit: needs_update -> {:?}", unknown);
+                return unknown;
+            }
+        };
+
+        // if we've gotten this far, we have a string in the form of X.X.X where X is a number
+        // all that's left is to compare the current version with the version found above
+
+        return if latest_version == bin_version {
+            // there's really only two possible outcomes if we accept that the tag conforms to
+            // the X.X.X pattern:
+            //   1. the version strings match, meaning we're up to date
+            //   2. the version strings do not match, meaning we're out of date
+            //
+            // except for developers working on this code, nobody should ever be in a situation
+            // where they have a version greater than the latest tagged release
+            log::trace!("exit: needs_update -> UpdateStatus::UpToDate");
+            UpdateStatus::UpToDate
+        } else {
+            log::trace!("exit: needs_update -> UpdateStatus::OutOfDate");
+            UpdateStatus::OutOfDate
+        };
+    }
+
+    log::trace!("exit: needs_update -> {:?}", unknown);
+    unknown
+}
+
 /// Prints the banner to stdout.
 ///
 /// Only prints those settings which are either always present, or passed in by the user.
-pub fn initialize(targets: &[String]) {
+pub async fn initialize<W>(targets: &[String], config: &Configuration, version: &str, mut writer: W)
+where
+    W: Write,
+{
     let artwork = format!(
         r#"
  ___  ___  __   __     __      __         __   ___
 |__  |__  |__) |__) | /  `    /  \ \_/ | |  \ |__
 |    |___ |  \ |  \ | \__,    \__/ / \ | |__/ |___
 by Ben "epi" Risher {}                  ver: {}"#,
-        '\u{1F913}', VERSION
+        '\u{1F913}', version
     );
 
+    let status = needs_update(&CONFIGURATION.client, UPDATE_URL, version).await;
+
     let top = "───────────────────────────┬──────────────────────";
+    let addl_section = "──────────────────────────────────────────────────";
     let bottom = "───────────────────────────┴──────────────────────";
 
-    eprintln!("{}", artwork);
-    eprintln!("{}", top);
+    writeln!(&mut writer, "{}", artwork).unwrap_or_default();
+    writeln!(&mut writer, "{}", top).unwrap_or_default();
 
     // begin with always printed items
     for target in targets {
-        eprintln!(
+        writeln!(
+            &mut writer,
             "{}",
             format_banner_entry!("\u{1F3af}", "Target Url", target)
-        ); // 🎯
+        )
+        .unwrap_or_default(); // 🎯
     }
 
     let mut codes = vec![];
 
-    for code in &CONFIGURATION.statuscodes {
+    for code in &config.status_codes {
         codes.push(status_colorizer(&code.to_string()))
     }
 
-    eprintln!(
+    writeln!(
+        &mut writer,
         "{}",
-        format_banner_entry!("\u{1F680}", "Threads", CONFIGURATION.threads)
-    ); // 🚀
-    eprintln!(
+        format_banner_entry!("\u{1F680}", "Threads", config.threads)
+    )
+    .unwrap_or_default(); // 🚀
+
+    writeln!(
+        &mut writer,
         "{}",
-        format_banner_entry!("\u{1f4d6}", "Wordlist", CONFIGURATION.wordlist)
-    ); // 📖
-    eprintln!(
+        format_banner_entry!("\u{1f4d6}", "Wordlist", config.wordlist)
+    )
+    .unwrap_or_default(); // 📖
+
+    writeln!(
+        &mut writer,
         "{}",
         format_banner_entry!(
             "\u{1F197}",
             "Status Codes",
             format!("[{}]", codes.join(", "))
         )
-    ); // 🆗
-    eprintln!(
+    )
+    .unwrap_or_default(); // 🆗
+
+    if !config.filter_status.is_empty() {
+        // exception here for optional print due to me wanting the allows and denys to be printed
+        // one after the other
+        let mut code_filters = vec![];
+
+        for code in &config.filter_status {
+            code_filters.push(status_colorizer(&code.to_string()))
+        }
+
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!(
+                "\u{1f5d1}",
+                "Status Code Filters",
+                format!("[{}]", code_filters.join(", "))
+            )
+        )
+        .unwrap_or_default(); // 🗑
+    }
+
+    writeln!(
+        &mut writer,
         "{}",
-        format_banner_entry!("\u{1f4a5}", "Timeout (secs)", CONFIGURATION.timeout)
-    ); // 💥
-    eprintln!(
+        format_banner_entry!("\u{1f4a5}", "Timeout (secs)", config.timeout)
+    )
+    .unwrap_or_default(); // 💥
+
+    writeln!(
+        &mut writer,
         "{}",
-        format_banner_entry!("\u{1F9a1}", "User-Agent", CONFIGURATION.useragent)
-    ); // 🦡
+        format_banner_entry!("\u{1F9a1}", "User-Agent", config.user_agent)
+    )
+    .unwrap_or_default(); // 🦡
 
     // followed by the maybe printed or variably displayed values
-    if !CONFIGURATION.proxy.is_empty() {
-        eprintln!(
+    if !config.config.is_empty() {
+        writeln!(
+            &mut writer,
             "{}",
-            format_banner_entry!("\u{1f48e}", "Proxy", CONFIGURATION.proxy)
-        ); // 💎
+            format_banner_entry!("\u{1f489}", "Config File", config.config)
+        )
+        .unwrap_or_default(); // 💉
     }
 
-    if !CONFIGURATION.headers.is_empty() {
-        for (name, value) in &CONFIGURATION.headers {
-            eprintln!(
+    if !config.proxy.is_empty() {
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!("\u{1f48e}", "Proxy", config.proxy)
+        )
+        .unwrap_or_default(); // 💎
+    }
+
+    if !config.replay_proxy.is_empty() {
+        // i include replay codes logic here because in config.rs, replay codes are set to the
+        // value in status codes, meaning it's never empty
+
+        let mut replay_codes = vec![];
+
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!("\u{1f3a5}", "Replay Proxy", config.replay_proxy)
+        )
+        .unwrap_or_default(); // 🎥
+
+        for code in &config.replay_codes {
+            replay_codes.push(status_colorizer(&code.to_string()))
+        }
+
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!(
+                "\u{1f4fc}",
+                "Replay Proxy Codes",
+                format!("[{}]", replay_codes.join(", "))
+            )
+        )
+        .unwrap_or_default(); // 📼
+    }
+
+    if !config.headers.is_empty() {
+        for (name, value) in &config.headers {
+            writeln!(
+                &mut writer,
                 "{}",
                 format_banner_entry!("\u{1f92f}", "Header", name, value)
-            ); // 🤯
+            )
+            .unwrap_or_default(); // 🤯
         }
     }
 
-    if !CONFIGURATION.sizefilters.is_empty() {
-        for filter in &CONFIGURATION.sizefilters {
-            eprintln!(
+    if !config.filter_size.is_empty() {
+        for filter in &config.filter_size {
+            writeln!(
+                &mut writer,
                 "{}",
                 format_banner_entry!("\u{1f4a2}", "Size Filter", filter)
-            ); // 💢
+            )
+            .unwrap_or_default(); // 💢
         }
     }
 
-    if !CONFIGURATION.queries.is_empty() {
-        for query in &CONFIGURATION.queries {
-            eprintln!(
+    for filter in &config.filter_word_count {
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!("\u{1f4a2}", "Word Count Filter", filter)
+        )
+        .unwrap_or_default(); // 💢
+    }
+
+    for filter in &config.filter_line_count {
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!("\u{1f4a2}", "Line Count Filter", filter)
+        )
+        .unwrap_or_default(); // 💢
+    }
+
+    for filter in &config.filter_regex {
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!("\u{1f4a2}", "Regex Filter", filter)
+        )
+        .unwrap_or_default(); // 💢
+    }
+
+    if config.extract_links {
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!("\u{1F50E}", "Extract Links", config.extract_links)
+        )
+        .unwrap_or_default(); // 🔎
+    }
+
+    if config.json {
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!("\u{1F9d4}", "JSON Output", config.json)
+        )
+        .unwrap_or_default(); // 🧔
+    }
+
+    if !config.queries.is_empty() {
+        for query in &config.queries {
+            writeln!(
+                &mut writer,
                 "{}",
                 format_banner_entry!(
                     "\u{1f914}",
                     "Query Parameter",
                     format!("{}={}", query.0, query.1)
                 )
-            ); // 🤔
+            )
+            .unwrap_or_default(); // 🤔
         }
     }
 
-    if !CONFIGURATION.output.is_empty() {
-        eprintln!(
+    if !config.output.is_empty() {
+        writeln!(
+            &mut writer,
             "{}",
-            format_banner_entry!("\u{1f4be}", "Output File", CONFIGURATION.output)
-        ); // 💾
+            format_banner_entry!("\u{1f4be}", "Output File", config.output)
+        )
+        .unwrap_or_default(); // 💾
     }
 
-    if !CONFIGURATION.extensions.is_empty() {
-        eprintln!(
+    if !config.debug_log.is_empty() {
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!("\u{1fab2}", "Debugging Log", config.debug_log)
+        )
+        .unwrap_or_default(); // 🪲
+    }
+
+    if !config.extensions.is_empty() {
+        writeln!(
+            &mut writer,
             "{}",
             format_banner_entry!(
                 "\u{1f4b2}",
                 "Extensions",
-                format!("[{}]", CONFIGURATION.extensions.join(", "))
+                format!("[{}]", config.extensions.join(", "))
             )
-        ); // 💲
+        )
+        .unwrap_or_default(); // 💲
     }
 
-    if CONFIGURATION.insecure {
-        eprintln!(
+    if config.insecure {
+        writeln!(
+            &mut writer,
             "{}",
-            format_banner_entry!("\u{1f513}", "Insecure", CONFIGURATION.insecure)
-        ); // 🔓
+            format_banner_entry!("\u{1f513}", "Insecure", config.insecure)
+        )
+        .unwrap_or_default(); // 🔓
     }
 
-    if CONFIGURATION.redirects {
-        eprintln!(
+    if config.redirects {
+        writeln!(
+            &mut writer,
             "{}",
-            format_banner_entry!("\u{1f4cd}", "Follow Redirects", CONFIGURATION.redirects)
-        ); // 📍
+            format_banner_entry!("\u{1f4cd}", "Follow Redirects", config.redirects)
+        )
+        .unwrap_or_default(); // 📍
     }
 
-    if CONFIGURATION.dontfilter {
-        eprintln!(
+    if config.dont_filter {
+        writeln!(
+            &mut writer,
             "{}",
-            format_banner_entry!("\u{1f92a}", "Filter Wildcards", !CONFIGURATION.dontfilter)
-        ); // 🤪
+            format_banner_entry!("\u{1f92a}", "Filter Wildcards", !config.dont_filter)
+        )
+        .unwrap_or_default(); // 🤪
     }
 
-    match CONFIGURATION.verbosity {
+    match config.verbosity {
         //speaker medium volume (increasing with verbosity to loudspeaker)
         1 => {
-            eprintln!(
+            writeln!(
+                &mut writer,
                 "{}",
-                format_banner_entry!("\u{1f508}", "Verbosity", CONFIGURATION.verbosity)
-            ); // 🔈
+                format_banner_entry!("\u{1f508}", "Verbosity", config.verbosity)
+            )
+            .unwrap_or_default(); // 🔈
         }
         2 => {
-            eprintln!(
+            writeln!(
+                &mut writer,
                 "{}",
-                format_banner_entry!("\u{1f509}", "Verbosity", CONFIGURATION.verbosity)
-            ); // 🔉
+                format_banner_entry!("\u{1f509}", "Verbosity", config.verbosity)
+            )
+            .unwrap_or_default(); // 🔉
         }
         3 => {
-            eprintln!(
+            writeln!(
+                &mut writer,
                 "{}",
-                format_banner_entry!("\u{1f50a}", "Verbosity", CONFIGURATION.verbosity)
-            ); // 🔊
+                format_banner_entry!("\u{1f50a}", "Verbosity", config.verbosity)
+            )
+            .unwrap_or_default(); // 🔊
         }
         4 => {
-            eprintln!(
+            writeln!(
+                &mut writer,
                 "{}",
-                format_banner_entry!("\u{1f4e2}", "Verbosity", CONFIGURATION.verbosity)
-            ); // 📢
+                format_banner_entry!("\u{1f4e2}", "Verbosity", config.verbosity)
+            )
+            .unwrap_or_default(); // 📢
         }
         _ => {}
     }
 
-    if CONFIGURATION.addslash {
-        eprintln!(
+    if config.add_slash {
+        writeln!(
+            &mut writer,
             "{}",
-            format_banner_entry!("\u{1fa93}", "Add Slash", CONFIGURATION.addslash)
-        ); // 🪓
+            format_banner_entry!("\u{1fa93}", "Add Slash", config.add_slash)
+        )
+        .unwrap_or_default(); // 🪓
     }
 
-    if !CONFIGURATION.norecursion {
-        if CONFIGURATION.depth == 0 {
-            eprintln!(
+    if !config.no_recursion {
+        if config.depth == 0 {
+            writeln!(
+                &mut writer,
                 "{}",
                 format_banner_entry!("\u{1f503}", "Recursion Depth", "INFINITE")
-            ); // 🔃
+            )
+            .unwrap_or_default(); // 🔃
         } else {
-            eprintln!(
+            writeln!(
+                &mut writer,
                 "{}",
-                format_banner_entry!("\u{1f503}", "Recursion Depth", CONFIGURATION.depth)
-            ); // 🔃
+                format_banner_entry!("\u{1f503}", "Recursion Depth", config.depth)
+            )
+            .unwrap_or_default(); // 🔃
         }
     } else {
-        eprintln!(
+        writeln!(
+            &mut writer,
             "{}",
-            format_banner_entry!("\u{1f6ab}", "Do Not Recurse", CONFIGURATION.norecursion)
-        ); // 🚫
+            format_banner_entry!("\u{1f6ab}", "Do Not Recurse", config.no_recursion)
+        )
+        .unwrap_or_default(); // 🚫
     }
 
-    eprintln!("{}", bottom);
+    if CONFIGURATION.scan_limit > 0 {
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!("\u{1f9a5}", "Concurrent Scan Limit", config.scan_limit)
+        )
+        .unwrap_or_default(); // 🦥
+    }
+
+    if matches!(status, UpdateStatus::OutOfDate) {
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!(
+                "\u{1f389}",
+                "New Version Available",
+                "https://github.com/epi052/feroxbuster/releases/latest"
+            )
+        )
+        .unwrap_or_default(); // 🎉
+    }
+
+    writeln!(&mut writer, "{}", bottom).unwrap_or_default();
+    // ⏯
+    writeln!(
+        &mut writer,
+        " \u{23ef}   Press [{}] to {}|{} your scan",
+        style("ENTER").yellow(),
+        style("pause").red(),
+        style("resume").green()
+    )
+    .unwrap_or_default();
+    writeln!(&mut writer, "{}", addl_section).unwrap_or_default();
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::VERSION;
+    use httpmock::Method::GET;
+    use httpmock::{Mock, MockServer};
+    use std::fs::read_to_string;
+    use std::io::stderr;
+    use std::time::Duration;
+    use tempfile::NamedTempFile;
+
+    #[tokio::test(core_threads = 1)]
+    /// test to hit no execution of targets for loop in banner
+    async fn banner_intialize_without_targets() {
+        let config = Configuration::default();
+        initialize(&[], &config, VERSION, stderr()).await;
+    }
+
+    #[tokio::test(core_threads = 1)]
+    /// test to hit no execution of statuscode for loop in banner
+    async fn banner_intialize_without_status_codes() {
+        let mut config = Configuration::default();
+        config.status_codes = vec![];
+        initialize(
+            &[String::from("http://localhost")],
+            &config,
+            VERSION,
+            stderr(),
+        )
+        .await;
+    }
+
+    #[tokio::test(core_threads = 1)]
+    /// test to hit an empty config file
+    async fn banner_intialize_without_config_file() {
+        let mut config = Configuration::default();
+        config.config = String::new();
+        initialize(
+            &[String::from("http://localhost")],
+            &config,
+            VERSION,
+            stderr(),
+        )
+        .await;
+    }
+
+    #[tokio::test(core_threads = 1)]
+    /// test to hit an empty config file
+    async fn banner_intialize_without_queries() {
+        let mut config = Configuration::default();
+        config.queries = vec![(String::new(), String::new())];
+        initialize(
+            &[String::from("http://localhost")],
+            &config,
+            VERSION,
+            stderr(),
+        )
+        .await;
+    }
+
+    #[tokio::test(core_threads = 1)]
+    /// test to show that a new version is available for download
+    async fn banner_intialize_with_mismatched_version() {
+        let config = Configuration::default();
+        let file = NamedTempFile::new().unwrap();
+        initialize(
+            &[String::from("http://localhost")],
+            &config,
+            "mismatched-version",
+            &file,
+        )
+        .await;
+        let contents = read_to_string(file.path()).unwrap();
+        println!("contents: {}", contents);
+        assert!(contents.contains("New Version Available"));
+        assert!(contents.contains("https://github.com/epi052/feroxbuster/releases/latest"));
+    }
+
+    #[tokio::test(core_threads = 1)]
+    /// test that
+    async fn banner_needs_update_returns_unknown_with_bad_url() {
+        let result = needs_update(&CONFIGURATION.client, &"", VERSION).await;
+        assert!(matches!(result, UpdateStatus::Unknown));
+    }
+
+    #[tokio::test(core_threads = 1)]
+    /// test return value of good url to needs_update
+    async fn banner_needs_update_returns_up_to_date() {
+        let srv = MockServer::start();
+
+        let mock = Mock::new()
+            .expect_method(GET)
+            .expect_path("/latest")
+            .return_status(200)
+            .return_body("{\"tag_name\":\"v1.1.0\"}")
+            .create_on(&srv);
+
+        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.1.0").await;
+
+        assert_eq!(mock.times_called(), 1);
+        assert!(matches!(result, UpdateStatus::UpToDate));
+    }
+
+    #[tokio::test(core_threads = 1)]
+    /// test return value of good url to needs_update that returns a newer version than current
+    async fn banner_needs_update_returns_out_of_date() {
+        let srv = MockServer::start();
+
+        let mock = Mock::new()
+            .expect_method(GET)
+            .expect_path("/latest")
+            .return_status(200)
+            .return_body("{\"tag_name\":\"v1.1.0\"}")
+            .create_on(&srv);
+
+        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1").await;
+
+        assert_eq!(mock.times_called(), 1);
+        assert!(matches!(result, UpdateStatus::OutOfDate));
+    }
+
+    #[tokio::test(core_threads = 1)]
+    /// test return value of good url that times out
+    async fn banner_needs_update_returns_unknown_on_timeout() {
+        let srv = MockServer::start();
+
+        let mock = Mock::new()
+            .expect_method(GET)
+            .expect_path("/latest")
+            .return_status(200)
+            .return_body("{\"tag_name\":\"v1.1.0\"}")
+            .return_with_delay(Duration::from_secs(8))
+            .create_on(&srv);
+
+        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1").await;
+
+        assert_eq!(mock.times_called(), 1);
+        assert!(matches!(result, UpdateStatus::Unknown));
+    }
+
+    #[tokio::test(core_threads = 1)]
+    /// test return value of good url with bad json response
+    async fn banner_needs_update_returns_unknown_on_bad_json_response() {
+        let srv = MockServer::start();
+
+        let mock = Mock::new()
+            .expect_method(GET)
+            .expect_path("/latest")
+            .return_status(200)
+            .return_body("not json")
+            .create_on(&srv);
+
+        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1").await;
+
+        assert_eq!(mock.times_called(), 1);
+        assert!(matches!(result, UpdateStatus::Unknown));
+    }
+
+    #[tokio::test(core_threads = 1)]
+    /// test return value of good url with json response that lacks the tag_name field
+    async fn banner_needs_update_returns_unknown_on_json_without_correct_tag() {
+        let srv = MockServer::start();
+
+        let mock = Mock::new()
+            .expect_method(GET)
+            .expect_path("/latest")
+            .return_status(200)
+            .return_body("{\"no tag_name\": \"doesn't exist\"}")
+            .create_on(&srv);
+
+        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1").await;
+
+        assert_eq!(mock.times_called(), 1);
+        assert!(matches!(result, UpdateStatus::Unknown));
+    }
 }

src/client.rs

@@ -1,15 +1,16 @@
-use crate::utils::status_colorizer;
+use crate::utils::{module_colorizer, status_colorizer};
 use reqwest::header::HeaderMap;
 use reqwest::{redirect::Policy, Client, Proxy};
 use std::collections::HashMap;
 use std::convert::TryInto;
+#[cfg(not(test))]
 use std::process::exit;
 use std::time::Duration;
 
 /// Create and return an instance of [reqwest::Client](https://docs.rs/reqwest/latest/reqwest/struct.Client.html)
 pub fn initialize(
     timeout: u64,
-    useragent: &str,
+    user_agent: &str,
     redirects: bool,
     insecure: bool,
     headers: &HashMap<String, String>,
@@ -21,55 +22,80 @@ pub fn initialize(
         Policy::none()
     };
 
-    let header_map: HeaderMap = match headers.try_into() {
-        Ok(map) => map,
-        Err(e) => {
-            eprintln!(
-                "[{}] - Client::initialize: {}",
-                status_colorizer("ERROR"),
-                e
-            );
-            exit(1);
-        }
-    };
+    // try_into returns infallible as its error, unwrap is safe here
+    let header_map: HeaderMap = headers.try_into().unwrap();
 
     let client = Client::builder()
         .timeout(Duration::new(timeout, 0))
-        .user_agent(useragent)
+        .user_agent(user_agent)
         .danger_accept_invalid_certs(insecure)
         .default_headers(header_map)
         .redirect(policy);
 
-    let client = if proxy.is_some() && !proxy.unwrap().is_empty() {
-        match Proxy::all(proxy.unwrap()) {
-            Ok(proxy_obj) => client.proxy(proxy_obj),
-            Err(e) => {
-                eprintln!(
-                    "[{}] - Could not add proxy ({:?}) to Client configuration",
-                    status_colorizer("ERROR"),
-                    proxy
-                );
-                eprintln!(
-                    "[{}] - Client::initialize: {}",
-                    status_colorizer("ERROR"),
-                    e
-                );
-                exit(1);
+    let client = match proxy {
+        // a proxy is specified, need to add it to the client
+        Some(some_proxy) => {
+            if !some_proxy.is_empty() {
+                // it's not an empty string
+                match Proxy::all(some_proxy) {
+                    Ok(proxy_obj) => client.proxy(proxy_obj),
+                    Err(e) => {
+                        eprintln!(
+                            "{} {} {}",
+                            status_colorizer("ERROR"),
+                            module_colorizer("Client::initialize"),
+                            e
+                        );
+
+                        #[cfg(test)]
+                        panic!();
+                        #[cfg(not(test))]
+                        exit(1);
+                    }
+                }
+            } else {
+                client // Some("") was used?
             }
         }
-    } else {
-        client
+        // no proxy specified
+        None => client,
     };
 
     match client.build() {
         Ok(client) => client,
         Err(e) => {
             eprintln!(
-                "[{}] - Could not create a Client with the given configuration, exiting.",
-                status_colorizer("ERROR")
+                "{} {} {}",
+                status_colorizer("ERROR"),
+                module_colorizer("Client::build"),
+                e
             );
-            eprintln!("[{}] - Client::build: {}", status_colorizer("ERROR"), e);
+
+            #[cfg(test)]
+            panic!();
+            #[cfg(not(test))]
             exit(1);
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    #[should_panic]
+    /// create client with a bad proxy, expect panic
+    fn client_with_bad_proxy() {
+        let headers = HashMap::new();
+        initialize(0, "stuff", true, false, &headers, Some("not a valid proxy"));
+    }
+
+    #[test]
+    /// create client with a proxy, expect no error
+    fn client_with_good_proxy() {
+        let headers = HashMap::new();
+        let proxy = "http://127.0.0.1:8080";
+        initialize(0, "stuff", true, true, &headers, Some(proxy));
+    }
+}

src/extractor.rs

@@ -0,0 +1,269 @@
+use crate::FeroxResponse;
+use lazy_static::lazy_static;
+use regex::Regex;
+use reqwest::Url;
+use std::collections::HashSet;
+
+/// Regular expression used in [LinkFinder](https://github.com/GerbenJavado/LinkFinder)
+///
+/// Incorporates change from this [Pull Request](https://github.com/GerbenJavado/LinkFinder/pull/66/files)
+const LINKFINDER_REGEX: &str = r#"(?:"|')(((?:[a-zA-Z]{1,10}://|//)[^"'/]{1,}\.[a-zA-Z]{2,}[^"']{0,})|((?:/|\.\./|\./)[^"'><,;| *()(%%$^/\\\[\]][^"'><,;|()]{1,})|([a-zA-Z0-9_\-/]{1,}/[a-zA-Z0-9_\-/]{1,}\.(?:[a-zA-Z]{1,4}|action)(?:[\?|#][^"|']{0,}|))|([a-zA-Z0-9_\-/]{1,}/[a-zA-Z0-9_\-/]{3,}(?:[\?|#][^"|']{0,}|))|([a-zA-Z0-9_\-.]{1,}\.(?:php|asp|aspx|jsp|json|action|html|js|txt|xml)(?:[\?|#][^"|']{0,}|)))(?:"|')"#;
+
+lazy_static! {
+    /// `LINKFINDER_REGEX` as a regex::Regex type
+    static ref REGEX: Regex = Regex::new(LINKFINDER_REGEX).unwrap();
+}
+
+/// Iterate over a given path, return a list of every sub-path found
+///
+/// example: `path` contains a link fragment `homepage/assets/img/icons/handshake.svg`
+/// the following fragments would be returned:
+///   - homepage/assets/img/icons/handshake.svg
+///   - homepage/assets/img/icons/
+///   - homepage/assets/img/
+///   - homepage/assets/
+///   - homepage/
+fn get_sub_paths_from_path(path: &str) -> Vec<String> {
+    log::trace!("enter: get_sub_paths_from_path({})", path);
+    let mut paths = vec![];
+
+    // filter out any empty strings caused by .split
+    let mut parts: Vec<&str> = path.split('/').filter(|s| !s.is_empty()).collect();
+
+    let length = parts.len();
+
+    for _ in 0..length {
+        // iterate over all parts of the path
+        if parts.is_empty() {
+            // pop left us with an empty vector, we're done
+            break;
+        }
+
+        let possible_path = parts.join("/");
+
+        if possible_path.is_empty() {
+            // .join can result in an empty string, which we don't need, ignore
+            continue;
+        }
+
+        paths.push(possible_path); // good sub-path found
+        parts.pop(); // use .pop() to remove the last part of the path and continue iteration
+    }
+
+    log::trace!("exit: get_sub_paths_from_path -> {:?}", paths);
+    paths
+}
+
+/// simple helper to stay DRY, trys to join a url + fragment and add it to the `links` HashSet
+fn add_link_to_set_of_links(link: &str, url: &Url, links: &mut HashSet<String>) {
+    log::trace!(
+        "enter: add_link_to_set_of_links({}, {}, {:?})",
+        link,
+        url.to_string(),
+        links
+    );
+    match url.join(&link) {
+        Ok(new_url) => {
+            links.insert(new_url.to_string());
+        }
+        Err(e) => {
+            log::error!("Could not join given url to the base url: {}", e);
+        }
+    }
+    log::trace!("exit: add_link_to_set_of_links");
+}
+
+/// Given a `reqwest::Response`, perform the following actions
+///   - parse the response's text for links using the linkfinder regex
+///   - for every link found take its url path and parse each sub-path
+///     - example: Response contains a link fragment `homepage/assets/img/icons/handshake.svg`
+///       with a base url of http://localhost, the following urls would be returned:
+///         - homepage/assets/img/icons/handshake.svg
+///         - homepage/assets/img/icons/
+///         - homepage/assets/img/
+///         - homepage/assets/
+///         - homepage/
+pub async fn get_links(response: &FeroxResponse) -> HashSet<String> {
+    log::trace!("enter: get_links({})", response.url().as_str());
+
+    let mut links = HashSet::<String>::new();
+
+    let body = response.text();
+
+    for capture in REGEX.captures_iter(&body) {
+        // remove single & double quotes from both ends of the capture
+        // capture[0] is the entire match, additional capture groups start at [1]
+        let link = capture[0].trim_matches(|c| c == '\'' || c == '"');
+
+        match Url::parse(link) {
+            Ok(absolute) => {
+                if absolute.domain() != response.url().domain()
+                    || absolute.host() != response.url().host()
+                {
+                    // domains/ips are not the same, don't scan things that aren't part of the original
+                    // target url
+                    continue;
+                }
+
+                for sub_path in get_sub_paths_from_path(absolute.path()) {
+                    // take a url fragment like homepage/assets/img/icons/handshake.svg and
+                    // incrementally add
+                    //     - homepage/assets/img/icons/
+                    //     - homepage/assets/img/
+                    //     - homepage/assets/
+                    //     - homepage/
+                    log::debug!("Adding {} to {:?}", sub_path, links);
+                    add_link_to_set_of_links(&sub_path, &response.url(), &mut links);
+                }
+            }
+            Err(e) => {
+                // this is the expected error that happens when we try to parse a url fragment
+                //     ex: Url::parse("/login") -> Err("relative URL without a base")
+                // while this is technically an error, these are good results for us
+                if e.to_string().contains("relative URL without a base") {
+                    for sub_path in get_sub_paths_from_path(link) {
+                        // incrementally save all sub-paths that led to the relative url's resource
+                        log::debug!("Adding {} to {:?}", sub_path, links);
+                        add_link_to_set_of_links(&sub_path, &response.url(), &mut links);
+                    }
+                } else {
+                    // unexpected error has occurred
+                    log::error!("Could not parse given url: {}", e);
+                }
+            }
+        }
+    }
+
+    log::trace!("exit: get_links -> {:?}", links);
+    links
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::utils::make_request;
+    use httpmock::Method::GET;
+    use httpmock::{Mock, MockServer};
+    use reqwest::Client;
+
+    #[test]
+    /// extract sub paths from the given url fragment; expect 4 sub paths and that all are
+    /// in the expected array
+    fn extractor_get_sub_paths_from_path_with_multiple_paths() {
+        let path = "homepage/assets/img/icons/handshake.svg";
+        let paths = get_sub_paths_from_path(&path);
+        let expected = vec![
+            "homepage",
+            "homepage/assets",
+            "homepage/assets/img",
+            "homepage/assets/img/icons",
+            "homepage/assets/img/icons/handshake.svg",
+        ];
+
+        assert_eq!(paths.len(), expected.len());
+        for expected_path in expected {
+            assert_eq!(paths.contains(&expected_path.to_string()), true);
+        }
+    }
+
+    #[test]
+    /// extract sub paths from the given url fragment; expect 2 sub paths and that all are
+    /// in the expected array. the fragment is wrapped in slashes to ensure no empty strings are
+    /// returned
+    fn extractor_get_sub_paths_from_path_with_enclosing_slashes() {
+        let path = "/homepage/assets/";
+        let paths = get_sub_paths_from_path(&path);
+        let expected = vec!["homepage", "homepage/assets"];
+
+        assert_eq!(paths.len(), expected.len());
+        for expected_path in expected {
+            assert_eq!(paths.contains(&expected_path.to_string()), true);
+        }
+    }
+
+    #[test]
+    /// extract sub paths from the given url fragment; expect 1 sub path, no forward slashes are
+    /// included
+    fn extractor_get_sub_paths_from_path_with_only_a_word() {
+        let path = "homepage";
+        let paths = get_sub_paths_from_path(&path);
+        let expected = vec!["homepage"];
+
+        assert_eq!(paths.len(), expected.len());
+        for expected_path in expected {
+            assert_eq!(paths.contains(&expected_path.to_string()), true);
+        }
+    }
+
+    #[test]
+    /// extract sub paths from the given url fragment; expect 1 sub path, forward slash removed
+    fn extractor_get_sub_paths_from_path_with_an_absolute_word() {
+        let path = "/homepage";
+        let paths = get_sub_paths_from_path(&path);
+        let expected = vec!["homepage"];
+
+        assert_eq!(paths.len(), expected.len());
+        for expected_path in expected {
+            assert_eq!(paths.contains(&expected_path.to_string()), true);
+        }
+    }
+
+    #[test]
+    /// test that a full url and fragment are joined correctly, then added to the given list
+    /// i.e. the happy path
+    fn extractor_add_link_to_set_of_links_happy_path() {
+        let url = Url::parse("https://localhost").unwrap();
+        let mut links = HashSet::<String>::new();
+        let link = "admin";
+
+        assert_eq!(links.len(), 0);
+        add_link_to_set_of_links(link, &url, &mut links);
+
+        assert_eq!(links.len(), 1);
+        assert!(links.contains("https://localhost/admin"));
+    }
+
+    #[test]
+    /// test that an invalid path fragment doesn't add anything to the set of links
+    fn extractor_add_link_to_set_of_links_with_non_base_url() {
+        let url = Url::parse("https://localhost").unwrap();
+        let mut links = HashSet::<String>::new();
+        let link = "\\\\\\\\";
+
+        assert_eq!(links.len(), 0);
+        add_link_to_set_of_links(link, &url, &mut links);
+
+        assert_eq!(links.len(), 0);
+        assert!(links.is_empty());
+    }
+
+    #[tokio::test(core_threads = 1)]
+    /// use make_request to generate a Response, and use the Response to test get_links;
+    /// the response will contain an absolute path to a domain that is not part of the scanned
+    /// domain; expect an empty set returned
+    async fn extractor_get_links_with_absolute_url_that_differs_from_target_domain(
+    ) -> Result<(), Box<dyn std::error::Error>> {
+        let srv = MockServer::start();
+
+        let mock = Mock::new()
+            .expect_method(GET)
+            .expect_path("/some-path")
+            .return_status(200)
+            .return_body("\"http://defintely.not.a.thing.probably.com/homepage/assets/img/icons/handshake.svg\"")
+            .create_on(&srv);
+
+        let client = Client::new();
+        let url = Url::parse(&srv.url("/some-path")).unwrap();
+
+        let response = make_request(&client, &url).await.unwrap();
+
+        let ferox_response = FeroxResponse::from(response, true).await;
+
+        let links = get_links(&ferox_response).await;
+
+        assert!(links.is_empty());
+
+        assert_eq!(mock.times_called(), 1);
+        Ok(())
+    }
+}

src/filters.rs

@@ -0,0 +1,422 @@
+use crate::config::CONFIGURATION;
+use crate::utils::get_url_path_length;
+use crate::FeroxResponse;
+use regex::Regex;
+use std::any::Any;
+use std::fmt::Debug;
+
+// references:
+//   https://dev.to/magnusstrale/rust-trait-objects-in-a-vector-non-trivial-4co5
+//   https://stackoverflow.com/questions/25339603/how-to-test-for-equality-between-trait-objects
+
+/// FeroxFilter trait; represents different types of possible filters that can be applied to
+/// responses
+pub trait FeroxFilter: Debug + Send + Sync {
+    /// Determine whether or not this particular filter should be applied or not
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool;
+
+    /// delegates to the FeroxFilter-implementing type which gives us the actual type of self
+    fn box_eq(&self, other: &dyn Any) -> bool;
+
+    /// gives us `other` as Any in box_eq
+    fn as_any(&self) -> &dyn Any;
+}
+
+/// implementation of PartialEq, necessary long-form due to "trait cannot be made into an object"
+/// error when attempting to derive PartialEq on the trait itself
+impl PartialEq for Box<dyn FeroxFilter> {
+    /// Perform a comparison of two implementors of the FeroxFilter trait
+    fn eq(&self, other: &Box<dyn FeroxFilter>) -> bool {
+        self.box_eq(other.as_any())
+    }
+}
+
+/// Data holder for two pieces of data needed when auto-filtering out wildcard responses
+///
+/// `dynamic` is the size of the response that will later be combined with the length
+/// of the path of the url requested and used to determine interesting pages from custom
+/// 404s where the requested url is reflected back in the response
+///
+/// `size` is size of the response that should be included with filters passed via runtime
+/// configuration and any static wildcard lengths.
+#[derive(Debug, Default, Clone, PartialEq)]
+pub struct WildcardFilter {
+    /// size of the response that will later be combined with the length of the path of the url
+    /// requested
+    pub dynamic: u64,
+
+    /// size of the response that should be included with filters passed via runtime configuration
+    pub size: u64,
+}
+
+/// implementation of FeroxFilter for WildcardFilter
+impl FeroxFilter for WildcardFilter {
+    /// Examine size, dynamic, and content_len to determine whether or not the response received
+    /// is a wildcard response and therefore should be filtered out
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        // quick return if dont_filter is set
+        if CONFIGURATION.dont_filter {
+            // --dont-filter applies specifically to wildcard filters, it is not a 100% catch all
+            // for not filtering anything.  As such, it should live in the implementation of
+            // a wildcard filter
+            return false;
+        }
+
+        if self.size > 0 && self.size == response.content_length() {
+            // static wildcard size found during testing
+            // size isn't default, size equals response length, and auto-filter is on
+            log::debug!("static wildcard: filtered out {}", response.url());
+            log::trace!("exit: should_filter_response -> true");
+            return true;
+        }
+
+        if self.dynamic > 0 {
+            // dynamic wildcard offset found during testing
+
+            // I'm about to manually split this url path instead of using reqwest::Url's
+            // builtin parsing. The reason is that they call .split() on the url path
+            // except that I don't want an empty string taking up the last index in the
+            // event that the url ends with a forward slash.  It's ugly enough to be split
+            // into its own function for readability.
+            let url_len = get_url_path_length(&response.url());
+
+            if url_len + self.dynamic == response.content_length() {
+                log::debug!("dynamic wildcard: filtered out {}", response.url());
+                log::trace!("exit: should_filter_response -> true");
+                return true;
+            }
+        }
+        log::trace!("exit: should_filter_response -> false");
+        false
+    }
+
+    /// Compare one WildcardFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
+
+/// Simple implementor of FeroxFilter; used to filter out status codes specified using
+/// -C|--filter-status
+#[derive(Default, Debug, PartialEq)]
+pub struct StatusCodeFilter {
+    /// Status code that should not be displayed to the user
+    pub filter_code: u16,
+}
+
+/// implementation of FeroxFilter for StatusCodeFilter
+impl FeroxFilter for StatusCodeFilter {
+    /// Check `filter_code` against what was passed in via -C|--filter-status
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        if response.status().as_u16() == self.filter_code {
+            log::debug!(
+                "filtered out {} based on --filter-status of {}",
+                response.url(),
+                self.filter_code
+            );
+            log::trace!("exit: should_filter_response -> true");
+            return true;
+        }
+
+        log::trace!("exit: should_filter_response -> false");
+        false
+    }
+
+    /// Compare one StatusCodeFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
+
+/// Simple implementor of FeroxFilter; used to filter out responses based on the number of lines
+/// in a Response body; specified using -N|--filter-lines
+#[derive(Default, Debug, PartialEq)]
+pub struct LinesFilter {
+    /// Number of lines in a Response's body that should be filtered
+    pub line_count: usize,
+}
+
+/// implementation of FeroxFilter for LinesFilter
+impl FeroxFilter for LinesFilter {
+    /// Check `line_count` against what was passed in via -N|--filter-lines
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        let result = response.line_count() == self.line_count;
+
+        log::trace!("exit: should_filter_response -> {}", result);
+
+        result
+    }
+
+    /// Compare one LinesFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
+
+/// Simple implementor of FeroxFilter; used to filter out responses based on the number of words
+/// in a Response body; specified using -W|--filter-words
+#[derive(Default, Debug, PartialEq)]
+pub struct WordsFilter {
+    /// Number of words in a Response's body that should be filtered
+    pub word_count: usize,
+}
+
+/// implementation of FeroxFilter for WordsFilter
+impl FeroxFilter for WordsFilter {
+    /// Check `word_count` against what was passed in via -W|--filter-words
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        let result = response.word_count() == self.word_count;
+
+        log::trace!("exit: should_filter_response -> {}", result);
+
+        result
+    }
+
+    /// Compare one WordsFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
+
+/// Simple implementor of FeroxFilter; used to filter out responses based on the length of a
+/// Response body; specified using -S|--filter-size
+#[derive(Default, Debug, PartialEq)]
+pub struct SizeFilter {
+    /// Overall length of a Response's body that should be filtered
+    pub content_length: u64,
+}
+
+/// implementation of FeroxFilter for SizeFilter
+impl FeroxFilter for SizeFilter {
+    /// Check `content_length` against what was passed in via -S|--filter-size
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        let result = response.content_length() == self.content_length;
+
+        log::trace!("exit: should_filter_response -> {}", result);
+
+        result
+    }
+
+    /// Compare one SizeFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
+
+/// Simple implementor of FeroxFilter; used to filter out responses based on a given regular
+/// expression; specified using -X|--filter-regex
+#[derive(Debug)]
+pub struct RegexFilter {
+    /// Regular expression to be applied to the response body for filtering, compiled
+    pub compiled: Regex,
+
+    /// Regular expression as passed in on the command line, not compiled
+    pub raw_string: String,
+}
+
+/// implementation of FeroxFilter for RegexFilter
+impl FeroxFilter for RegexFilter {
+    /// Check `expression` against the response body, if the expression matches, the response
+    /// should be filtered out
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        let result = self.compiled.is_match(response.text());
+
+        log::trace!("exit: should_filter_response -> {}", result);
+
+        result
+    }
+
+    /// Compare one SizeFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
+
+/// PartialEq implementation for RegexFilter
+impl PartialEq for RegexFilter {
+    /// Simple comparison of the raw string passed in via the command line
+    fn eq(&self, other: &RegexFilter) -> bool {
+        self.raw_string == other.raw_string
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use reqwest::Url;
+
+    #[test]
+    /// just a simple test to increase code coverage by hitting as_any and the inner value
+    fn lines_filter_as_any() {
+        let filter = LinesFilter { line_count: 1 };
+
+        assert_eq!(filter.line_count, 1);
+        assert_eq!(
+            *filter.as_any().downcast_ref::<LinesFilter>().unwrap(),
+            filter
+        );
+    }
+
+    #[test]
+    /// just a simple test to increase code coverage by hitting as_any and the inner value
+    fn words_filter_as_any() {
+        let filter = WordsFilter { word_count: 1 };
+
+        assert_eq!(filter.word_count, 1);
+        assert_eq!(
+            *filter.as_any().downcast_ref::<WordsFilter>().unwrap(),
+            filter
+        );
+    }
+
+    #[test]
+    /// just a simple test to increase code coverage by hitting as_any and the inner value
+    fn size_filter_as_any() {
+        let filter = SizeFilter { content_length: 1 };
+
+        assert_eq!(filter.content_length, 1);
+        assert_eq!(
+            *filter.as_any().downcast_ref::<SizeFilter>().unwrap(),
+            filter
+        );
+    }
+
+    #[test]
+    /// just a simple test to increase code coverage by hitting as_any and the inner value
+    fn status_code_filter_as_any() {
+        let filter = StatusCodeFilter { filter_code: 200 };
+
+        assert_eq!(filter.filter_code, 200);
+        assert_eq!(
+            *filter.as_any().downcast_ref::<StatusCodeFilter>().unwrap(),
+            filter
+        );
+    }
+
+    #[test]
+    /// just a simple test to increase code coverage by hitting as_any and the inner value
+    fn regex_filter_as_any() {
+        let raw = r".*\.txt$";
+        let compiled = Regex::new(raw).unwrap();
+        let filter = RegexFilter {
+            compiled,
+            raw_string: raw.to_string(),
+        };
+
+        assert_eq!(filter.raw_string, r".*\.txt$");
+        assert_eq!(
+            *filter.as_any().downcast_ref::<RegexFilter>().unwrap(),
+            filter
+        );
+    }
+
+    #[test]
+    /// test should_filter on WilcardFilter where static logic matches
+    fn wildcard_should_filter_when_static_wildcard_found() {
+        let resp = FeroxResponse {
+            text: String::new(),
+            wildcard: true,
+            url: Url::parse("http://localhost").unwrap(),
+            content_length: 100,
+            word_count: 50,
+            line_count: 25,
+            headers: reqwest::header::HeaderMap::new(),
+            status: reqwest::StatusCode::OK,
+        };
+
+        let filter = WildcardFilter {
+            size: 100,
+            dynamic: 0,
+        };
+
+        assert!(filter.should_filter_response(&resp));
+    }
+
+    #[test]
+    /// test should_filter on WilcardFilter where dynamic logic matches
+    fn wildcard_should_filter_when_dynamic_wildcard_found() {
+        let resp = FeroxResponse {
+            text: String::new(),
+            wildcard: true,
+            url: Url::parse("http://localhost/stuff").unwrap(),
+            content_length: 100,
+            word_count: 50,
+            line_count: 25,
+            headers: reqwest::header::HeaderMap::new(),
+            status: reqwest::StatusCode::OK,
+        };
+
+        let filter = WildcardFilter {
+            size: 0,
+            dynamic: 95,
+        };
+
+        assert!(filter.should_filter_response(&resp));
+    }
+
+    #[test]
+    /// test should_filter on RegexFilter where regex matches body
+    fn regexfilter_should_filter_when_regex_matches_on_response_body() {
+        let resp = FeroxResponse {
+            text: String::from("im a body response hurr durr!"),
+            wildcard: false,
+            url: Url::parse("http://localhost/stuff").unwrap(),
+            content_length: 100,
+            word_count: 50,
+            line_count: 25,
+            headers: reqwest::header::HeaderMap::new(),
+            status: reqwest::StatusCode::OK,
+        };
+
+        let raw = r"response...rr";
+
+        let filter = RegexFilter {
+            raw_string: raw.to_string(),
+            compiled: Regex::new(raw).unwrap(),
+        };
+
+        assert!(filter.should_filter_response(&resp));
+    }
+}

src/heuristics.rs

@@ -1,28 +1,18 @@
-use crate::config::{CONFIGURATION, PROGRESS_PRINTER};
-use crate::scanner::{format_url, make_request};
-use crate::utils::{ferox_print, get_url_path_length, status_colorizer};
-use ansi_term::Color::{Cyan, Yellow};
+use crate::{
+    config::{CONFIGURATION, PROGRESS_PRINTER},
+    filters::WildcardFilter,
+    scanner::should_filter_response,
+    utils::{ferox_print, format_url, get_url_path_length, make_request, status_colorizer},
+    FeroxResponse,
+};
+use console::style;
 use indicatif::ProgressBar;
-use reqwest::Response;
-use std::process;
+use tokio::sync::mpsc::UnboundedSender;
 use uuid::Uuid;
 
+/// length of a standard UUID, used when determining wildcard responses
 const UUID_LENGTH: u64 = 32;
 
-/// Data holder for two pieces of data needed when auto-filtering out wildcard responses
-///
-/// `dynamic` is the size of the response that will later be combined with the length
-/// of the path of the url requested and used to determine interesting pages from custom
-/// 404s where the requested url is reflected back in the response
-///
-/// `size` is size of the response that should be included with filters passed via runtime
-/// configuration and any static wildcard lengths.
-#[derive(Default, Debug)]
-pub struct WildcardFilter {
-    pub dynamic: u64,
-    pub size: u64,
-}
-
 /// Simple helper to return a uuid, formatted as lowercase without hyphens
 ///
 /// `length` determines the number of uuids to string together. Each uuid
@@ -46,22 +36,34 @@ fn unique_string(length: usize) -> String {
 ///
 /// In the event that url returns a wildcard response, a
 /// [WildcardFilter](struct.WildcardFilter.html) is created and returned to the caller.
-pub async fn wildcard_test(target_url: &str, bar: ProgressBar) -> Option<WildcardFilter> {
-    log::trace!("enter: wildcard_test({:?})", target_url);
+pub async fn wildcard_test(
+    target_url: &str,
+    bar: ProgressBar,
+    tx_term: UnboundedSender<FeroxResponse>,
+) -> Option<WildcardFilter> {
+    log::trace!(
+        "enter: wildcard_test({:?}, {:?}, {:?})",
+        target_url,
+        bar,
+        tx_term
+    );
 
-    if CONFIGURATION.dontfilter {
-        // early return, dontfilter scans don't need tested
+    if CONFIGURATION.dont_filter {
+        // early return, dont_filter scans don't need tested
         log::trace!("exit: wildcard_test -> None");
         return None;
     }
 
-    if let Some(resp_one) = make_wildcard_request(&target_url, 1).await {
+    let tx_clone_one = tx_term.clone();
+    let tx_clone_two = tx_term.clone();
+
+    if let Some(ferox_response) = make_wildcard_request(&target_url, 1, tx_clone_one).await {
         bar.inc(1);
 
         // found a wildcard response
         let mut wildcard = WildcardFilter::default();
 
-        let wc_length = resp_one.content_length().unwrap_or(0);
+        let wc_length = ferox_response.content_length();
 
         if wc_length == 0 {
             log::trace!("exit: wildcard_test -> Some({:?})", wildcard);
@@ -70,42 +72,49 @@ pub async fn wildcard_test(target_url: &str, bar: ProgressBar) -> Option<Wildcar
 
         // content length of wildcard is non-zero, perform additional tests:
         //   make a second request, with a known-sized (64) longer request
-        if let Some(resp_two) = make_wildcard_request(&target_url, 3).await {
+        if let Some(resp_two) = make_wildcard_request(&target_url, 3, tx_clone_two).await {
             bar.inc(1);
 
-            let wc2_length = resp_two.content_length().unwrap_or(0);
+            let wc2_length = resp_two.content_length();
 
             if wc2_length == wc_length + (UUID_LENGTH * 2) {
                 // second length is what we'd expect to see if the requested url is
                 // reflected in the response along with some static content; aka custom 404
-                let url_len = get_url_path_length(&resp_one.url());
-
-                if !CONFIGURATION.quiet {
-                    ferox_print(
-                    &format!(
-                            "{} {:>10} Wildcard response is dynamic; {} ({} + url length) responses; toggle this behavior by using {}",
-                            status_colorizer("WLD"),
-                            wc_length - url_len,
-                            Yellow.paint("auto-filtering"),
-                            Cyan.paint(format!("{}", wc_length - url_len)),
-                            Yellow.paint("--dontfilter")
-                        ), &PROGRESS_PRINTER
-                    );
-                }
+                let url_len = get_url_path_length(&ferox_response.url());
 
                 wildcard.dynamic = wc_length - url_len;
-            } else if wc_length == wc2_length {
+
                 if !CONFIGURATION.quiet {
-                    ferox_print(&format!(
-                        "{} {:>10} Wildcard response is static; {} {} responses; toggle this behavior by using {}",
-                        status_colorizer("WLD"),
-                        wc_length,
-                        Yellow.paint("auto-filtering"),
-                        Cyan.paint(format!("{}", wc_length)),
-                        Yellow.paint("--dontfilter")
-                    ), &PROGRESS_PRINTER);
+                    let msg = format!(
+                            "{} {:>9} {:>9} {:>9} Wildcard response is dynamic; {} ({} + url length) responses; toggle this behavior by using {}\n",
+                            status_colorizer("WLD"),
+                            "-",
+                            "-",
+                            "-",
+                            style("auto-filtering").yellow(),
+                            style(wc_length - url_len).cyan(),
+                            style("--dont-filter").yellow()
+                    );
+
+                    ferox_print(&msg, &PROGRESS_PRINTER);
                 }
+            } else if wc_length == wc2_length {
                 wildcard.size = wc_length;
+
+                if !CONFIGURATION.quiet {
+                    let msg = format!(
+                        "{} {:>9} {:>9} {:>9} Wildcard response is static; {} {} responses; toggle this behavior by using {}\n",
+                        status_colorizer("WLD"),
+                        "-",
+                        "-",
+                        "-",
+                        style("auto-filtering").yellow(),
+                        style(wc_length).cyan(),
+                        style("--dont-filter").yellow()
+                    );
+
+                    ferox_print(&msg, &PROGRESS_PRINTER);
+                }
             }
         } else {
             bar.inc(2);
@@ -125,15 +134,24 @@ pub async fn wildcard_test(target_url: &str, bar: ProgressBar) -> Option<Wildcar
 /// Once the unique url is created, the request is sent to the server. If the server responds
 /// back with a valid status code, the response is considered to be a wildcard response. If that
 /// wildcard response has a 3xx status code, that redirection location is displayed to the user.
-async fn make_wildcard_request(target_url: &str, length: usize) -> Option<Response> {
-    log::trace!("enter: make_wildcard_request({}, {})", target_url, length);
+async fn make_wildcard_request(
+    target_url: &str,
+    length: usize,
+    tx_file: UnboundedSender<FeroxResponse>,
+) -> Option<FeroxResponse> {
+    log::trace!(
+        "enter: make_wildcard_request({}, {}, {:?})",
+        target_url,
+        length,
+        tx_file
+    );
 
     let unique_str = unique_string(length);
 
     let nonexistent = match format_url(
         target_url,
         &unique_str,
-        CONFIGURATION.addslash,
+        CONFIGURATION.add_slash,
         &CONFIGURATION.queries,
         None,
     ) {
@@ -145,65 +163,25 @@ async fn make_wildcard_request(target_url: &str, length: usize) -> Option<Respon
         }
     };
 
-    let wildcard = status_colorizer("WLD");
-
     match make_request(&CONFIGURATION.client, &nonexistent.to_owned()).await {
         Ok(response) => {
             if CONFIGURATION
-                .statuscodes
+                .status_codes
                 .contains(&response.status().as_u16())
             {
                 // found a wildcard response
-                let url_len = get_url_path_length(&response.url());
-                let content_len = response.content_length().unwrap_or(0);
+                let mut ferox_response = FeroxResponse::from(response, true).await;
+                ferox_response.wildcard = true;
 
-                if !CONFIGURATION.quiet {
-                    ferox_print(
-                        &format!(
-                            "{} {:>10} Got {} for {} (url length: {})",
-                            wildcard,
-                            content_len,
-                            status_colorizer(&response.status().as_str()),
-                            response.url(),
-                            url_len
-                        ),
-                        &PROGRESS_PRINTER,
-                    );
+                if !CONFIGURATION.quiet
+                    && !should_filter_response(&ferox_response)
+                    && tx_file.send(ferox_response.clone()).is_err()
+                {
+                    return None;
                 }
-                if response.status().is_redirection() {
-                    // show where it goes, if possible
-                    if let Some(next_loc) = response.headers().get("Location") {
-                        if let Ok(next_loc_str) = next_loc.to_str() {
-                            if !CONFIGURATION.quiet {
-                                ferox_print(
-                                    &format!(
-                                        "{} {:>10} {} redirects to => {}",
-                                        wildcard,
-                                        content_len,
-                                        response.url(),
-                                        next_loc_str
-                                    ),
-                                    &PROGRESS_PRINTER,
-                                );
-                            }
-                        } else {
-                            if !CONFIGURATION.quiet {
-                                ferox_print(
-                                    &format!(
-                                        "{} {:>10} {} redirects to => {:?}",
-                                        wildcard,
-                                        content_len,
-                                        response.url(),
-                                        next_loc
-                                    ),
-                                    &PROGRESS_PRINTER,
-                                );
-                            }
-                        }
-                    }
-                }
-                log::trace!("exit: make_wildcard_request -> {:?}", response);
-                return Some(response);
+
+                log::trace!("exit: make_wildcard_request -> {}", ferox_response);
+                return Some(ferox_response);
             }
         }
         Err(e) => {
@@ -230,7 +208,7 @@ pub async fn connectivity_test(target_urls: &[String]) -> Vec<String> {
         let request = match format_url(
             target_url,
             "",
-            CONFIGURATION.addslash,
+            CONFIGURATION.add_slash,
             &CONFIGURATION.queries,
             None,
         ) {
@@ -247,7 +225,6 @@ pub async fn connectivity_test(target_urls: &[String]) -> Vec<String> {
             }
             Err(e) => {
                 if !CONFIGURATION.quiet {
-                    // todo unwrap
                     ferox_print(
                         &format!("Could not connect to {}, skipping...", target_url),
                         &PROGRESS_PRINTER,
@@ -260,8 +237,6 @@ pub async fn connectivity_test(target_urls: &[String]) -> Vec<String> {
 
     if good_urls.is_empty() {
         log::error!("Could not connect to any target provided, exiting.");
-        log::trace!("exit: connectivity_test");
-        process::exit(1);
     }
 
     log::trace!("exit: connectivity_test -> {:?}", good_urls);
@@ -274,9 +249,18 @@ mod tests {
     use super::*;
 
     #[test]
-    fn unique_string_returns_correct_length() {
+    /// request a unique string of 32bytes * a value returns correct result
+    fn heuristics_unique_string_returns_correct_length() {
         for i in 0..10 {
             assert_eq!(unique_string(i).len(), i * 32);
         }
     }
+
+    #[test]
+    /// simply test the default values for wildcardfilter, expect 0, 0
+    fn heuristics_wildcardfilter_dafaults() {
+        let wcf = WildcardFilter::default();
+        assert_eq!(wcf.size, 0);
+        assert_eq!(wcf.dynamic, 0);
+    }
 }

src/lib.rs

@@ -1,22 +1,56 @@
 pub mod banner;
 pub mod client;
 pub mod config;
+pub mod extractor;
+pub mod filters;
 pub mod heuristics;
 pub mod logger;
 pub mod parser;
 pub mod progress;
+pub mod reporter;
+pub mod scan_manager;
 pub mod scanner;
 pub mod utils;
 
-use reqwest::StatusCode;
+use crate::utils::{get_url_path_length, status_colorizer};
+use console::{style, Color};
+use reqwest::header::{HeaderName, HeaderValue};
+use reqwest::{header::HeaderMap, Response, StatusCode, Url};
+use serde::{ser::SerializeStruct, Deserialize, Deserializer, Serialize, Serializer};
+use serde_json::Value;
+use std::collections::HashMap;
+use std::convert::{TryFrom, TryInto};
+use std::str::FromStr;
+use std::{error, fmt};
+use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender};
 
 /// Generic Result type to ease error handling in async contexts
-pub type FeroxResult<T> =
-    std::result::Result<T, Box<dyn std::error::Error + Send + Sync + 'static>>;
+pub type FeroxResult<T> = std::result::Result<T, Box<dyn error::Error + Send + Sync + 'static>>;
+
+/// Simple Error implementation to allow for custom error returns
+#[derive(Debug, Default)]
+pub struct FeroxError {
+    /// fancy string that can be printed via Display
+    pub message: String,
+}
+
+impl error::Error for FeroxError {}
+
+impl fmt::Display for FeroxError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}", &self.message)
+    }
+}
+
+/// Generic mpsc::unbounded_channel type to tidy up some code
+pub type FeroxChannel<T> = (UnboundedSender<T>, UnboundedReceiver<T>);
 
 /// Version pulled from Cargo.toml at compile time
 pub const VERSION: &str = env!("CARGO_PKG_VERSION");
 
+/// Maximum number of file descriptors that can be opened during a scan
+pub const DEFAULT_OPEN_FILE_LIMIT: usize = 8192;
+
 /// Default wordlist to use when `-w|--wordlist` isn't specified and not `wordlist` isn't set
 /// in a [ferox-config.toml](constant.DEFAULT_CONFIG_NAME.html) config file.
 ///
@@ -25,6 +59,9 @@ pub const VERSION: &str = env!("CARGO_PKG_VERSION");
 pub const DEFAULT_WORDLIST: &str =
     "/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt";
 
+/// Number of milliseconds to wait between polls of `PAUSE_SCAN` when user pauses a scan
+pub static SLEEP_DURATION: u64 = 500;
+
 /// Default list of status codes to report
 ///
 /// * 200 Ok
@@ -52,3 +89,508 @@ pub const DEFAULT_STATUS_CODES: [StatusCode; 9] = [
 ///
 /// Expected location is in the same directory as the feroxbuster binary.
 pub const DEFAULT_CONFIG_NAME: &str = "ferox-config.toml";
+
+/// FeroxSerialize trait; represents different types that are Serialize and also implement
+/// as_str / as_json methods
+pub trait FeroxSerialize: Serialize {
+    /// Return a String representation of the object, generally the human readable version of the
+    /// implementor
+    fn as_str(&self) -> String;
+
+    /// Return an NDJSON representation of the object
+    fn as_json(&self) -> String;
+}
+
+/// A `FeroxResponse`, derived from a `Response` to a submitted `Request`
+#[derive(Debug, Clone)]
+pub struct FeroxResponse {
+    /// The final `Url` of this `FeroxResponse`
+    url: Url,
+
+    /// The `StatusCode` of this `FeroxResponse`
+    status: StatusCode,
+
+    /// The full response text
+    text: String,
+
+    /// The content-length of this response, if known
+    content_length: u64,
+
+    /// The number of lines contained in the body of this response, if known
+    line_count: usize,
+
+    /// The number of words contained in the body of this response, if known
+    word_count: usize,
+
+    /// The `Headers` of this `FeroxResponse`
+    headers: HeaderMap,
+
+    /// Wildcard response status
+    wildcard: bool,
+}
+
+/// Implement Display for FeroxResponse
+impl fmt::Display for FeroxResponse {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(
+            f,
+            "FeroxResponse {{ url: {}, status: {}, content-length: {} }}",
+            self.url(),
+            self.status(),
+            self.content_length()
+        )
+    }
+}
+
+/// `FeroxResponse` implementation
+impl FeroxResponse {
+    /// Get the `StatusCode` of this `FeroxResponse`
+    pub fn status(&self) -> &StatusCode {
+        &self.status
+    }
+
+    /// Get the final `Url` of this `FeroxResponse`.
+    pub fn url(&self) -> &Url {
+        &self.url
+    }
+
+    /// Get the full response text
+    pub fn text(&self) -> &str {
+        &self.text
+    }
+
+    /// Get the `Headers` of this `FeroxResponse`
+    pub fn headers(&self) -> &HeaderMap {
+        &self.headers
+    }
+
+    /// Get the content-length of this response, if known
+    pub fn content_length(&self) -> u64 {
+        self.content_length
+    }
+
+    /// Set `FeroxResponse`'s `url` attribute, has no affect if an error occurs
+    pub fn set_url(&mut self, url: &str) {
+        match Url::parse(&url) {
+            Ok(url) => {
+                self.url = url;
+            }
+            Err(e) => {
+                log::error!("Could not parse {} into a Url: {}", url, e);
+            }
+        };
+    }
+
+    /// Make a reasonable guess at whether the response is a file or not
+    ///
+    /// Examines the last part of a path to determine if it has an obvious extension
+    /// i.e. http://localhost/some/path/stuff.js where stuff.js indicates a file
+    ///
+    /// Additionally, inspects query parameters, as they're also often indicative of a file
+    pub fn is_file(&self) -> bool {
+        let has_extension = match self.url.path_segments() {
+            Some(path) => {
+                if let Some(last) = path.last() {
+                    last.contains('.') // last segment has some sort of extension, probably
+                } else {
+                    false
+                }
+            }
+            None => false,
+        };
+
+        self.url.query_pairs().count() > 0 || has_extension
+    }
+
+    /// Returns line count of the response text.
+    pub fn line_count(&self) -> usize {
+        self.line_count
+    }
+
+    /// Returns word count of the response text.
+    pub fn word_count(&self) -> usize {
+        self.word_count
+    }
+
+    /// Create a new `FeroxResponse` from the given `Response`
+    pub async fn from(response: Response, read_body: bool) -> Self {
+        let url = response.url().clone();
+        let status = response.status();
+        let headers = response.headers().clone();
+        let content_length = response.content_length().unwrap_or(0);
+
+        let text = if read_body {
+            // .text() consumes the response, must be called last
+            // additionally, --extract-links is currently the only place we use the body of the
+            // response, so we forego the processing if not performing extraction
+            match response.text().await {
+                // await the response's body
+                Ok(text) => text,
+                Err(e) => {
+                    log::error!("Could not parse body from response: {}", e);
+                    String::new()
+                }
+            }
+        } else {
+            String::new()
+        };
+
+        let line_count = text.lines().count();
+        let word_count = text.lines().map(|s| s.split_whitespace().count()).sum();
+
+        FeroxResponse {
+            url,
+            status,
+            content_length,
+            text,
+            headers,
+            line_count,
+            word_count,
+            wildcard: false,
+        }
+    }
+}
+
+/// Implement FeroxSerialusize::from(ize for FeroxRespons)e
+impl FeroxSerialize for FeroxResponse {
+    /// Simple wrapper around create_report_string
+    fn as_str(&self) -> String {
+        let lines = self.line_count().to_string();
+        let words = self.word_count().to_string();
+        let chars = self.content_length().to_string();
+        let status = self.status().as_str();
+        let wild_status = status_colorizer("WLD");
+
+        if self.wildcard {
+            // response is a wildcard, special messages abound when this is the case...
+
+            // create the base message
+            let mut message = format!(
+                "{} {:>8}l {:>8}w {:>8}c Got {} for {} (url length: {})\n",
+                wild_status,
+                lines,
+                words,
+                chars,
+                status_colorizer(&status),
+                self.url(),
+                get_url_path_length(&self.url())
+            );
+
+            if self.status().is_redirection() {
+                // when it's a redirect, show where it goes, if possible
+                if let Some(next_loc) = self.headers().get("Location") {
+                    let next_loc_str = next_loc.to_str().unwrap_or("Unknown");
+
+                    let redirect_msg = format!(
+                        "{} {:>9} {:>9} {:>9} {} redirects to => {}\n",
+                        wild_status,
+                        "-",
+                        "-",
+                        "-",
+                        self.url(),
+                        next_loc_str
+                    );
+
+                    message.push_str(&redirect_msg);
+                }
+            }
+
+            // base message + redirection message (if appropriate)
+            message
+        } else {
+            // not a wildcard, just create a normal entry
+            utils::create_report_string(
+                self.status.as_str(),
+                &lines,
+                &words,
+                &chars,
+                self.url().as_str(),
+            )
+        }
+    }
+
+    /// Create an NDJSON representation of the FeroxResponse
+    ///
+    /// (expanded for clarity)
+    /// ex:
+    /// {
+    ///    "type":"response",
+    ///    "url":"https://localhost.com/images",
+    ///    "path":"/images",
+    ///    "status":301,
+    ///    "content_length":179,
+    ///    "line_count":10,
+    ///    "word_count":16,
+    ///    "headers":{
+    ///       "x-content-type-options":"nosniff",
+    ///       "strict-transport-security":"max-age=31536000; includeSubDomains",
+    ///       "x-frame-options":"SAMEORIGIN",
+    ///       "connection":"keep-alive",
+    ///       "server":"nginx/1.16.1",
+    ///       "content-type":"text/html; charset=UTF-8",
+    ///       "referrer-policy":"origin-when-cross-origin",
+    ///       "content-security-policy":"default-src 'none'",
+    ///       "access-control-allow-headers":"X-Requested-With",
+    ///       "x-xss-protection":"1; mode=block",
+    ///       "content-length":"179",
+    ///       "date":"Mon, 23 Nov 2020 15:33:24 GMT",
+    ///       "location":"/images/",
+    ///       "access-control-allow-origin":"https://localhost.com"
+    ///    }
+    /// }\n
+    fn as_json(&self) -> String {
+        if let Ok(mut json) = serde_json::to_string(&self) {
+            json.push('\n');
+            json
+        } else {
+            format!("{{\"error\":\"could not convert {} to json\"}}", self.url())
+        }
+    }
+}
+
+/// Serialize implementation for FeroxResponse
+impl Serialize for FeroxResponse {
+    /// Function that handles serialization of a FeroxResponse to NDJSON
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let mut headers = HashMap::new();
+        let mut state = serializer.serialize_struct("FeroxResponse", 7)?;
+
+        // need to convert the HeaderMap to a HashMap in order to pass it to the serializer
+        for (key, value) in &self.headers {
+            let k = key.as_str().to_owned();
+            let v = String::from_utf8_lossy(value.as_bytes());
+            headers.insert(k, v);
+        }
+
+        state.serialize_field("type", "response")?;
+        state.serialize_field("url", self.url.as_str())?;
+        state.serialize_field("path", self.url.path())?;
+        state.serialize_field("wildcard", &self.wildcard)?;
+        state.serialize_field("status", &self.status.as_u16())?;
+        state.serialize_field("content_length", &self.content_length)?;
+        state.serialize_field("line_count", &self.line_count)?;
+        state.serialize_field("word_count", &self.word_count)?;
+        state.serialize_field("headers", &headers)?;
+
+        state.end()
+    }
+}
+
+/// Deserialize implementation for FeroxResponse
+impl<'de> Deserialize<'de> for FeroxResponse {
+    /// Deserialize a FeroxResponse from a serde_json::Value
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let mut response = Self {
+            url: Url::parse("http://localhost").unwrap(),
+            status: StatusCode::OK,
+            text: String::new(),
+            content_length: 0,
+            headers: HeaderMap::new(),
+            wildcard: false,
+            line_count: 0,
+            word_count: 0,
+        };
+
+        let map: HashMap<String, Value> = HashMap::deserialize(deserializer)?;
+
+        for (key, value) in &map {
+            match key.as_str() {
+                "url" => {
+                    if let Some(url) = value.as_str() {
+                        if let Ok(parsed) = Url::parse(url) {
+                            response.url = parsed;
+                        }
+                    }
+                }
+                "status" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(smaller) = u16::try_from(num) {
+                            if let Ok(status) = StatusCode::from_u16(smaller) {
+                                response.status = status;
+                            }
+                        }
+                    }
+                }
+                "content_length" => {
+                    if let Some(num) = value.as_u64() {
+                        response.content_length = num;
+                    }
+                }
+                "line_count" => {
+                    if let Some(num) = value.as_u64() {
+                        response.line_count = num.try_into().unwrap_or_default();
+                    }
+                }
+                "word_count" => {
+                    if let Some(num) = value.as_u64() {
+                        response.word_count = num.try_into().unwrap_or_default();
+                    }
+                }
+                "headers" => {
+                    let mut headers = HeaderMap::<HeaderValue>::default();
+
+                    if let Some(map_headers) = value.as_object() {
+                        for (h_key, h_value) in map_headers {
+                            let h_value_str = h_value.as_str().unwrap_or("");
+                            let h_name = HeaderName::from_str(h_key)
+                                .unwrap_or_else(|_| HeaderName::from_str("Unknown").unwrap());
+                            let h_value_parsed = HeaderValue::from_str(h_value_str)
+                                .unwrap_or_else(|_| HeaderValue::from_str("Unknown").unwrap());
+                            headers.insert(h_name, h_value_parsed);
+                        }
+                    }
+
+                    response.headers = headers;
+                }
+                "wildcard" => {
+                    if let Some(result) = value.as_bool() {
+                        response.wildcard = result;
+                    }
+                }
+                _ => {}
+            }
+        }
+
+        Ok(response)
+    }
+}
+
+#[derive(Serialize, Deserialize, Default)]
+/// Representation of a log entry, can be represented as a human readable string or JSON
+pub struct FeroxMessage {
+    #[serde(rename = "type")]
+    /// Name of this type of struct, used for serialization, i.e. `{"type":"log"}`
+    kind: String,
+
+    /// The log message
+    pub message: String,
+
+    /// The log level
+    pub level: String,
+
+    /// The number of seconds elapsed since the scan started
+    pub time_offset: f32,
+
+    /// The module from which log::* was called
+    pub module: String,
+}
+
+/// Implementation of FeroxMessage
+impl FeroxSerialize for FeroxMessage {
+    /// Create an NDJSON representation of the log message
+    ///
+    /// (expanded for clarity)
+    /// ex:
+    /// {
+    ///   "type": "log",
+    ///   "message": "Sent https://localhost/api to file handler",
+    ///   "level": "DEBUG",
+    ///   "time_offset": 0.86333454,
+    ///   "module": "feroxbuster::reporter"
+    /// }\n
+    fn as_json(&self) -> String {
+        if let Ok(mut json) = serde_json::to_string(&self) {
+            json.push('\n');
+            json
+        } else {
+            String::from("{\"error\":\"could not convert to json\"}")
+        }
+    }
+
+    /// Create a string representation of the log message
+    ///
+    /// ex:  301       10l       16w      173c https://localhost/api
+    fn as_str(&self) -> String {
+        let (level_name, level_color) = match self.level.as_str() {
+            "ERROR" => ("ERR", Color::Red),
+            "WARN" => ("WRN", Color::Red),
+            "INFO" => ("INF", Color::Cyan),
+            "DEBUG" => ("DBG", Color::Yellow),
+            "TRACE" => ("TRC", Color::Magenta),
+            "WILDCARD" => ("WLD", Color::Cyan),
+            _ => ("UNK", Color::White),
+        };
+
+        format!(
+            "{} {:10.03} {} {}\n",
+            style(level_name).bg(level_color).black(),
+            style(self.time_offset).dim(),
+            self.module,
+            style(&self.message).dim(),
+        )
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    /// asserts default config name is correct
+    fn default_config_name() {
+        assert_eq!(DEFAULT_CONFIG_NAME, "ferox-config.toml");
+    }
+
+    #[test]
+    /// asserts default wordlist is correct
+    fn default_wordlist() {
+        assert_eq!(
+            DEFAULT_WORDLIST,
+            "/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt"
+        );
+    }
+
+    #[test]
+    /// asserts default version is correct
+    fn default_version() {
+        assert_eq!(VERSION, env!("CARGO_PKG_VERSION"));
+    }
+
+    #[test]
+    /// test as_str method of FeroxMessage
+    fn ferox_message_as_str_returns_string_with_newline() {
+        let message = FeroxMessage {
+            message: "message".to_string(),
+            module: "utils".to_string(),
+            time_offset: 1.0,
+            level: "INFO".to_string(),
+            kind: "log".to_string(),
+        };
+        let message_str = message.as_str();
+
+        assert!(message_str.contains("INF"));
+        assert!(message_str.contains("1.000"));
+        assert!(message_str.contains("utils"));
+        assert!(message_str.contains("message"));
+        assert!(message_str.ends_with('\n'));
+    }
+
+    #[test]
+    /// test as_json method of FeroxMessage
+    fn ferox_message_as_json_returns_json_representation_of_ferox_message_with_newline() {
+        let message = FeroxMessage {
+            message: "message".to_string(),
+            module: "utils".to_string(),
+            time_offset: 1.0,
+            level: "INFO".to_string(),
+            kind: "log".to_string(),
+        };
+
+        let message_str = message.as_json();
+
+        let error_margin = f32::EPSILON;
+
+        let json: FeroxMessage = serde_json::from_str(&message_str).unwrap();
+        assert_eq!(json.module, message.module);
+        assert_eq!(json.message, message.message);
+        assert!((json.time_offset - message.time_offset).abs() < error_margin);
+        assert_eq!(json.level, message.level);
+        assert_eq!(json.kind, message.kind);
+    }
+}

src/logger.rs

@@ -1,10 +1,16 @@
-use crate::config::PROGRESS_PRINTER;
-use console::{style, Color};
+use crate::{
+    config::{CONFIGURATION, PROGRESS_PRINTER},
+    reporter::safe_file_write,
+    utils::open_file,
+    FeroxMessage, FeroxSerialize,
+};
 use env_logger::Builder;
 use std::env;
 use std::time::Instant;
 
-/// Create an instance of an [Logger](struct.Logger.html) and set the log level based on `verbosity`
+/// Create a customized instance of
+/// [env_logger::Logger](https://docs.rs/env_logger/latest/env_logger/struct.Logger.html)
+/// with timer offset/color and set the log level based on `verbosity`
 pub fn initialize(verbosity: u8) {
     // use occurrences of -v on commandline to or verbosity = N in feroxconfig.toml to set
     // log level for the application; respects already specified RUST_LOG environment variable
@@ -16,8 +22,8 @@ pub fn initialize(verbosity: u8) {
                 0 => (),
                 1 => env::set_var("RUST_LOG", "warn"),
                 2 => env::set_var("RUST_LOG", "info"),
-                3 => env::set_var("RUST_LOG", "debug,hyper=info,reqwest=info"),
-                _ => env::set_var("RUST_LOG", "trace,hyper=info,reqwest=info"),
+                3 => env::set_var("RUST_LOG", "feroxbuster=debug,info"),
+                _ => env::set_var("RUST_LOG", "feroxbuster=trace,info"),
             }
         }
     }
@@ -25,27 +31,29 @@ pub fn initialize(verbosity: u8) {
     let start = Instant::now();
     let mut builder = Builder::from_default_env();
 
+    let debug_file = open_file(&CONFIGURATION.debug_log);
+
+    if let Some(buffered_file) = debug_file.clone() {
+        // write out the configuration to the debug file if it exists
+        safe_file_write(&*CONFIGURATION, buffered_file, CONFIGURATION.json);
+    }
+
     builder
         .format(move |_, record| {
-            let t = start.elapsed().as_secs_f32();
-            let level = record.level();
-
-            let (level_name, level_color) = match level {
-                log::Level::Error => ("ERR", Color::Red),
-                log::Level::Warn => ("WRN", Color::Red),
-                log::Level::Info => ("INF", Color::Cyan),
-                log::Level::Debug => ("DBG", Color::Yellow),
-                log::Level::Trace => ("TRC", Color::Magenta),
+            let log_entry = FeroxMessage {
+                message: record.args().to_string(),
+                level: record.level().to_string(),
+                time_offset: start.elapsed().as_secs_f32(),
+                module: record.target().to_string(),
+                kind: "log".to_string(),
             };
 
-            let msg = format!(
-                "{} {:10.03} {}",
-                style(format!("{}", level_name)).bg(level_color).black(),
-                style(t).dim(),
-                style(record.args()).dim(),
-            );
+            PROGRESS_PRINTER.println(&log_entry.as_str());
+
+            if let Some(buffered_file) = debug_file.clone() {
+                safe_file_write(&log_entry, buffered_file, CONFIGURATION.json);
+            }
 
-            PROGRESS_PRINTER.println(msg);
             Ok(())
         })
         .init();

src/main.rs

@@ -1,15 +1,64 @@
-use feroxbuster::config::{CONFIGURATION, PROGRESS_PRINTER};
-use feroxbuster::scanner::scan_url;
-use feroxbuster::utils::get_current_depth;
-use feroxbuster::{banner, heuristics, logger, FeroxResult};
+use crossterm::event::{self, Event, KeyCode};
+use feroxbuster::progress::add_bar;
+use feroxbuster::{
+    banner,
+    config::{CONFIGURATION, PROGRESS_BAR, PROGRESS_PRINTER},
+    heuristics, logger, reporter,
+    scan_manager::{self, PAUSE_SCAN},
+    scanner::{self, scan_url, RESPONSES, SCANNED_URLS},
+    utils::{ferox_print, get_current_depth, module_colorizer, status_colorizer},
+    FeroxError, FeroxResponse, FeroxResult, FeroxSerialize, SLEEP_DURATION, VERSION,
+};
+#[cfg(not(target_os = "windows"))]
+use feroxbuster::{utils::set_open_file_limit, DEFAULT_OPEN_FILE_LIMIT};
 use futures::StreamExt;
-use std::collections::HashSet;
-use std::fs::File;
-use std::io::{BufRead, BufReader};
-use std::sync::Arc;
-use tokio::io;
+use std::convert::TryInto;
+use std::{
+    collections::HashSet,
+    fs::File,
+    io::{stderr, BufRead, BufReader},
+    process,
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        Arc,
+    },
+    time::Duration,
+};
+use tokio::{io, sync::mpsc::UnboundedSender, task::JoinHandle};
 use tokio_util::codec::{FramedRead, LinesCodec};
 
+/// Atomic boolean flag, used to determine whether or not the terminal input handler should exit
+pub static SCAN_COMPLETE: AtomicBool = AtomicBool::new(false);
+
+/// Handles specific key events triggered by the user over stdin
+fn terminal_input_handler() {
+    log::trace!("enter: terminal_input_handler");
+
+    loop {
+        if event::poll(Duration::from_millis(SLEEP_DURATION)).unwrap_or(false) {
+            // It's guaranteed that the `read()` won't block when the `poll()`
+            // function returns `true`
+
+            if let Ok(key_pressed) = event::read() {
+                if key_pressed == Event::Key(KeyCode::Enter.into()) {
+                    // if the user presses Enter, toggle the value stored in PAUSE_SCAN
+                    // ignore any other keys
+                    let current = PAUSE_SCAN.load(Ordering::Acquire);
+
+                    PAUSE_SCAN.store(!current, Ordering::Release);
+                }
+            }
+        } else {
+            // Timeout expired and no `Event` is available; use the timeout to check SCAN_COMPLETE
+            if SCAN_COMPLETE.load(Ordering::Relaxed) {
+                // scan has been marked complete by main, time to exit the loop
+                break;
+            }
+        }
+    }
+    log::trace!("exit: terminal_input_handler");
+}
+
 /// Create a HashSet of Strings from the given wordlist then stores it inside an Arc
 fn get_unique_words_from_wordlist(path: &str) -> FeroxResult<Arc<HashSet<String>>> {
     log::trace!("enter: get_unique_words_from_wordlist({})", path);
@@ -29,26 +78,30 @@ fn get_unique_words_from_wordlist(path: &str) -> FeroxResult<Arc<HashSet<String>
     let mut words = HashSet::new();
 
     for line in reader.lines() {
-        match line {
-            Ok(word) => {
-                words.insert(word);
-            }
-            Err(e) => {
-                log::warn!("Could not parse current line from wordlist : {}", e);
-            }
+        let result = line?;
+
+        if result.starts_with('#') || result.is_empty() {
+            continue;
         }
+
+        words.insert(result);
     }
 
     log::trace!(
         "exit: get_unique_words_from_wordlist -> Arc<wordlist[{} words...]>",
         words.len()
     );
+
     Ok(Arc::new(words))
 }
 
 /// Determine whether it's a single url scan or urls are coming from stdin, then scan as needed
-async fn scan(targets: Vec<String>) -> FeroxResult<()> {
-    log::trace!("enter: scan");
+async fn scan(
+    targets: Vec<String>,
+    tx_term: UnboundedSender<FeroxResponse>,
+    tx_file: UnboundedSender<FeroxResponse>,
+) -> FeroxResult<()> {
+    log::trace!("enter: scan({:?}, {:?}, {:?})", targets, tx_term, tx_file);
     // cloning an Arc is cheap (it's basically a pointer into the heap)
     // so that will allow for cheap/safe sharing of a single wordlist across multi-target scans
     // as well as additional directories found as part of recursion
@@ -56,14 +109,49 @@ async fn scan(targets: Vec<String>) -> FeroxResult<()> {
         tokio::spawn(async move { get_unique_words_from_wordlist(&CONFIGURATION.wordlist) })
             .await??;
 
+    if words.len() == 0 {
+        let mut err = FeroxError::default();
+        err.message = format!("Did not find any words in {}", CONFIGURATION.wordlist);
+        return Err(Box::new(err));
+    }
+
+    scanner::initialize(words.len(), &CONFIGURATION);
+
+    if CONFIGURATION.resumed {
+        if let Ok(scans) = SCANNED_URLS.scans.lock() {
+            for scan in scans.iter() {
+                if let Ok(locked_scan) = scan.lock() {
+                    if locked_scan.complete {
+                        // these scans are complete, and just need to be shown to the user
+                        let pb = add_bar(
+                            &locked_scan.url,
+                            words.len().try_into().unwrap_or_default(),
+                            false,
+                            true,
+                        );
+                        pb.finish();
+                    }
+                }
+            }
+        }
+
+        if let Ok(responses) = RESPONSES.responses.read() {
+            for response in responses.iter() {
+                PROGRESS_PRINTER.println(response.as_str());
+            }
+        }
+    }
+
     let mut tasks = vec![];
 
     for target in targets {
-        let wordclone = words.clone();
+        let word_clone = words.clone();
+        let term_clone = tx_term.clone();
+        let file_clone = tx_file.clone();
 
         let task = tokio::spawn(async move {
             let base_depth = get_current_depth(&target);
-            scan_url(&target, wordclone, base_depth).await;
+            scan_url(&target, word_clone, base_depth, term_clone, file_clone).await;
         });
 
         tasks.push(task);
@@ -76,7 +164,8 @@ async fn scan(targets: Vec<String>) -> FeroxResult<()> {
     Ok(())
 }
 
-async fn get_targets() -> Vec<String> {
+/// Get targets from either commandline or stdin, pass them back to the caller as a Result<Vec>
+async fn get_targets() -> FeroxResult<Vec<String>> {
     log::trace!("enter: get_targets");
 
     let mut targets = vec![];
@@ -88,12 +177,21 @@ async fn get_targets() -> Vec<String> {
         let mut reader = FramedRead::new(stdin, LinesCodec::new());
 
         while let Some(line) = reader.next().await {
-            match line {
-                Ok(target) => {
-                    targets.push(target);
-                }
-                Err(e) => {
-                    log::error!("{}", e);
+            targets.push(line?);
+        }
+    } else if CONFIGURATION.resumed {
+        // resume-from can't be used with any other flag, making it mutually exclusive from either
+        // of the other two options
+        if let Ok(scans) = SCANNED_URLS.scans.lock() {
+            for scan in scans.iter() {
+                // SCANNED_URLS gets deserialized scans added to it at program start if --resume-from
+                // is used, so scans that aren't marked complete still need to be scanned
+                if let Ok(locked_scan) = scan.lock() {
+                    if locked_scan.complete {
+                        // this one's already done, ignore it
+                        continue;
+                    }
+                    targets.push(locked_scan.url.to_owned());
                 }
             }
         }
@@ -103,35 +201,157 @@ async fn get_targets() -> Vec<String> {
 
     log::trace!("exit: get_targets -> {:?}", targets);
 
-    targets
+    Ok(targets)
 }
 
-#[tokio::main]
-async fn main() {
-    logger::initialize(CONFIGURATION.verbosity);
+/// async main called from real main, broken out in this way to allow for some synchronous code
+/// to be executed before bringing the tokio runtime online
+async fn wrapped_main() {
+    // join can only be called once, otherwise it causes the thread to panic
+    tokio::task::spawn_blocking(move || {
+        // ok, lazy_static! uses (unsurprisingly in retrospect) a lazy loading model where the
+        // thing obtained through deref isn't actually created until it's used. This created a
+        // problem when initializing the logger as it relied on PROGRESS_PRINTER which may or may
+        // not have been created by the time it was needed for logging (really only occurred in
+        // heuristics / banner / main). In order to initialize logging properly, we need to ensure
+        // PROGRESS_PRINTER and PROGRESS_BAR have been used at least once.  This call satisfies
+        // that constraint
+        PROGRESS_PRINTER.println("");
+        PROGRESS_BAR.join().unwrap();
+    });
 
+    // can't trace main until after logger is initialized and the above task is started
     log::trace!("enter: main");
-    log::debug!("{:#?}", *CONFIGURATION);
+
+    // spawn a thread that listens for keyboard input on stdin, when a user presses enter
+    // the input handler will toggle PAUSE_SCAN, which in turn is used to pause and resume
+    // scans that are already running
+    tokio::task::spawn_blocking(terminal_input_handler);
+
+    let save_output = !CONFIGURATION.output.is_empty(); // was -o used?
+
+    let (tx_term, tx_file, term_handle, file_handle) =
+        reporter::initialize(&CONFIGURATION.output, save_output);
 
     // get targets from command line or stdin
-    let targets = get_targets().await;
+    let targets = match get_targets().await {
+        Ok(t) => t,
+        Err(e) => {
+            // should only happen in the event that there was an error reading from stdin
+            log::error!("{} {}", module_colorizer("main::get_targets"), e);
+            clean_up(tx_term, term_handle, tx_file, file_handle, save_output).await;
+            return;
+        }
+    };
 
     if !CONFIGURATION.quiet {
         // only print banner if -q isn't used
-        banner::initialize(&targets);
+        let std_stderr = stderr(); // std::io::stderr
+        banner::initialize(&targets, &CONFIGURATION, &VERSION, std_stderr).await;
     }
 
     // discard non-responsive targets
     let live_targets = heuristics::connectivity_test(&targets).await;
 
-    match scan(live_targets).await {
+    if live_targets.is_empty() {
+        clean_up(tx_term, term_handle, tx_file, file_handle, save_output).await;
+        return;
+    }
+
+    // kick off a scan against any targets determined to be responsive
+    match scan(live_targets, tx_term.clone(), tx_file.clone()).await {
         Ok(_) => {
-            log::info!("Done");
+            log::info!("All scans complete!");
+        }
+        Err(e) => {
+            ferox_print(
+                &format!("{} while scanning: {}", status_colorizer("Error"), e),
+                &PROGRESS_PRINTER,
+            );
+            clean_up(tx_term, term_handle, tx_file, file_handle, save_output).await;
+            process::exit(1);
         }
-        Err(e) => log::error!("An error occurred: {}", e),
     };
 
-    PROGRESS_PRINTER.finish();
+    clean_up(tx_term, term_handle, tx_file, file_handle, save_output).await;
 
     log::trace!("exit: main");
 }
+
+/// Single cleanup function that handles all the necessary drops/finishes etc required to gracefully
+/// shutdown the program
+async fn clean_up(
+    tx_term: UnboundedSender<FeroxResponse>,
+    term_handle: JoinHandle<()>,
+    tx_file: UnboundedSender<FeroxResponse>,
+    file_handle: Option<JoinHandle<()>>,
+    save_output: bool,
+) {
+    log::trace!(
+        "enter: clean_up({:?}, {:?}, {:?}, {:?}, {})",
+        tx_term,
+        term_handle,
+        tx_file,
+        file_handle,
+        save_output
+    );
+
+    drop(tx_term);
+    log::trace!("dropped terminal output handler's transmitter");
+
+    log::trace!("awaiting terminal output handler's receiver");
+    // after dropping tx, we can await the future where rx lived
+    match term_handle.await {
+        Ok(_) => {}
+        Err(e) => {
+            log::error!("error awaiting terminal output handler's receiver: {}", e);
+        }
+    }
+    log::trace!("done awaiting terminal output handler's receiver");
+
+    log::trace!("tx_file: {:?}", tx_file);
+    // the same drop/await process used on the terminal handler is repeated for the file handler
+    // we drop the file transmitter every time, because it's created no matter what
+    drop(tx_file);
+
+    log::trace!("dropped file output handler's transmitter");
+    if save_output {
+        // but we only await if -o was specified
+        log::trace!("awaiting file output handler's receiver");
+        match file_handle.unwrap().await {
+            Ok(_) => {}
+            Err(e) => {
+                log::error!("error awaiting file output handler's receiver: {}", e);
+            }
+        }
+        log::trace!("done awaiting file output handler's receiver");
+    }
+
+    // mark all scans complete so the terminal input handler will exit cleanly
+    SCAN_COMPLETE.store(true, Ordering::Relaxed);
+
+    // clean-up function for the MultiProgress bar; must be called last in order to still see
+    // the final trace messages above
+    PROGRESS_PRINTER.finish();
+
+    log::trace!("exit: clean_up");
+}
+
+fn main() {
+    // setup logging based on the number of -v's used
+    logger::initialize(CONFIGURATION.verbosity);
+
+    if CONFIGURATION.save_state {
+        // start the ctrl+c handler
+        scan_manager::initialize();
+    }
+
+    // this function uses rlimit, which is not supported on windows
+    #[cfg(not(target_os = "windows"))]
+    set_open_file_limit(DEFAULT_OPEN_FILE_LIMIT);
+
+    if let Ok(mut runtime) = tokio::runtime::Runtime::new() {
+        let future = wrapped_main();
+        runtime.block_on(future);
+    }
+}

src/parser.rs

@@ -1,5 +1,5 @@
 use crate::VERSION;
-use clap::{App, Arg};
+use clap::{App, Arg, ArgGroup};
 
 /// Create and return an instance of [clap::App](https://docs.rs/clap/latest/clap/struct.App.html), i.e. the Command Line Interface's configuration
 pub fn initialize() -> App<'static, 'static> {
@@ -19,7 +19,7 @@ pub fn initialize() -> App<'static, 'static> {
             Arg::with_name("url")
                 .short("u")
                 .long("url")
-                .required_unless("stdin")
+                .required_unless_one(&["stdin", "resume_from"])
                 .value_name("URL")
                 .multiple(true)
                 .use_delimiter(true)
@@ -55,7 +55,7 @@ pub fn initialize() -> App<'static, 'static> {
                 .long("verbosity")
                 .takes_value(false)
                 .multiple(true)
-                .help("Increase verbosity level (use -vv or more for greater effect)"),
+                .help("Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v's is probably too much)"),
         )
         .arg(
             Arg::with_name("proxy")
@@ -68,15 +68,38 @@ pub fn initialize() -> App<'static, 'static> {
                 ),
         )
         .arg(
-            Arg::with_name("statuscodes")
+            Arg::with_name("replay_proxy")
+                .short("P")
+                .long("replay-proxy")
+                .takes_value(true)
+                .value_name("REPLAY_PROXY")
+                .help(
+                    "Send only unfiltered requests through a Replay Proxy, instead of all requests",
+                ),
+        )
+        .arg(
+            Arg::with_name("replay_codes")
+                .short("R")
+                .long("replay-codes")
+                .value_name("REPLAY_CODE")
+                .takes_value(true)
+                .multiple(true)
+                .use_delimiter(true)
+                .requires("replay_proxy")
+                .help(
+                    "Status Codes to send through a Replay Proxy when found (default: --status-codes value)",
+                ),
+        )
+        .arg(
+            Arg::with_name("status_codes")
                 .short("s")
-                .long("statuscodes")
+                .long("status-codes")
                 .value_name("STATUS_CODE")
                 .takes_value(true)
                 .multiple(true)
                 .use_delimiter(true)
                 .help(
-                    "Status Codes of interest (default: 200 204 301 302 307 308 401 403 405)",
+                    "Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)",
                 ),
         )
         .arg(
@@ -87,9 +110,16 @@ pub fn initialize() -> App<'static, 'static> {
                 .help("Only print URLs; Don't print status codes, response size, running config, etc...")
         )
         .arg(
-            Arg::with_name("dontfilter")
+            Arg::with_name("json")
+                .long("json")
+                .takes_value(false)
+                .requires("output_files")
+                .help("Emit JSON logs to --output and --debug-log instead of normal text")
+        )
+        .arg(
+            Arg::with_name("dont_filter")
                 .short("D")
-                .long("dontfilter")
+                .long("dont-filter")
                 .takes_value(false)
                 .help("Don't auto-filter wildcard responses")
         )
@@ -98,13 +128,28 @@ pub fn initialize() -> App<'static, 'static> {
                 .short("o")
                 .long("output")
                 .value_name("FILE")
-                .help("Output file to write results to (default: stdout)")
+                .help("Output file to write results to (use w/ --json for JSON entries)")
                 .takes_value(true),
         )
         .arg(
-            Arg::with_name("useragent")
+            Arg::with_name("resume_from")
+                .long("resume-from")
+                .value_name("STATE_FILE")
+                .help("State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)")
+                .conflicts_with_all(&["wordlist", "url", "threads", "depth", "timeout", "verbosity", "proxy", "replay_proxy", "replay_codes", "status_codes", "quiet", "json", "dont_filter", "output", "debug_log", "user_agent", "redirects", "insecure", "extensions", "headers", "queries", "no_recursion", "add_slash", "stdin", "filter_size", "filter_regex", "filter_words", "filter_lines", "filter_status", "extract_links", "scan_limit"])
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name("debug_log")
+                .long("debug-log")
+                .value_name("FILE")
+                .help("Output file to write log entries (use w/ --json for JSON entries)")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::with_name("user_agent")
                 .short("a")
-                .long("useragent")
+                .long("user-agent")
                 .value_name("USER_AGENT")
                 .takes_value(true)
                 .help(
@@ -162,16 +207,16 @@ pub fn initialize() -> App<'static, 'static> {
                 ),
         )
         .arg(
-            Arg::with_name("norecursion")
+            Arg::with_name("no_recursion")
                 .short("n")
-                .long("norecursion")
+                .long("no-recursion")
                 .takes_value(false)
                 .help("Do not scan recursively")
         )
         .arg(
-            Arg::with_name("addslash")
+            Arg::with_name("add_slash")
                 .short("f")
-                .long("addslash")
+                .long("add-slash")
                 .takes_value(false)
                 .conflicts_with("extensions")
                 .help("Append / to each request")
@@ -184,9 +229,9 @@ pub fn initialize() -> App<'static, 'static> {
                 .conflicts_with("url")
         )
         .arg(
-            Arg::with_name("sizefilters")
+            Arg::with_name("filter_size")
                 .short("S")
-                .long("sizefilter")
+                .long("filter-size")
                 .value_name("SIZE")
                 .takes_value(true)
                 .multiple(true)
@@ -195,7 +240,73 @@ pub fn initialize() -> App<'static, 'static> {
                     "Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)",
                 ),
         )
-
+        .arg(
+            Arg::with_name("filter_regex")
+                .short("X")
+                .long("filter-regex")
+                .value_name("REGEX")
+                .takes_value(true)
+                .multiple(true)
+                .use_delimiter(true)
+                .help(
+                    "Filter out messages via regular expression matching on the response's body (ex: -X '^ignore me$')",
+                ),
+        )
+        .arg(
+            Arg::with_name("filter_words")
+                .short("W")
+                .long("filter-words")
+                .value_name("WORDS")
+                .takes_value(true)
+                .multiple(true)
+                .use_delimiter(true)
+                .help(
+                    "Filter out messages of a particular word count (ex: -W 312 -W 91,82)",
+                ),
+        )
+        .arg(
+            Arg::with_name("filter_lines")
+                .short("N")
+                .long("filter-lines")
+                .value_name("LINES")
+                .takes_value(true)
+                .multiple(true)
+                .use_delimiter(true)
+                .help(
+                    "Filter out messages of a particular line count (ex: -N 20 -N 31,30)",
+                ),
+        )
+        .arg(
+            Arg::with_name("filter_status")
+                .short("C")
+                .long("filter-status")
+                .value_name("STATUS_CODE")
+                .takes_value(true)
+                .multiple(true)
+                .use_delimiter(true)
+                .help(
+                    "Filter out status codes (deny list) (ex: -C 200 -C 401)",
+                ),
+        )
+        .arg(
+            Arg::with_name("extract_links")
+                .short("e")
+                .long("extract-links")
+                .takes_value(false)
+                .help("Extract links from response body (html, javascript, etc...); make new requests based on findings (default: false)")
+        )
+        .arg(
+            Arg::with_name("scan_limit")
+                .short("L")
+                .long("scan-limit")
+                .value_name("SCAN_LIMIT")
+                .takes_value(true)
+                .help("Limit total number of concurrent scans (default: 0, i.e. no limit)")
+        )
+        .group(ArgGroup::with_name("output_files")
+            .args(&["debug_log", "output"])
+            .multiple(true)
+        )
         .after_help(r#"NOTE:
     Options that take multiple values are very flexible.  Consider the following ways of specifying
     extensions:
@@ -211,7 +322,7 @@ EXAMPLES:
         ./feroxbuster -u http://127.1 -H Accept:application/json "Authorization: Bearer {token}"
 
     IPv6, non-recursive scan with INFO-level logging enabled:
-        ./feroxbuster -u http://[::1] --norecursion -vv
+        ./feroxbuster -u http://[::1] --no-recursion -vv
 
     Read urls from STDIN; pipe only resulting urls out to another tool
         cat targets | ./feroxbuster --stdin --quiet -s 200 301 302 --redirects -x js | fff -s 200 -o js-files
@@ -225,7 +336,22 @@ EXAMPLES:
     Pass auth token via query parameter
         ./feroxbuster -u http://127.1 --query token=0123456789ABCDEF
 
+    Find links in javascript/html and make additional requests based on results
+        ./feroxbuster -u http://127.1 --extract-links
+
     Ludicrous speed... go!
         ./feroxbuster -u http://127.1 -t 200
     "#)
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    /// initalize parser, expect a clap::App returned
+    fn parser_initialize_gives_defaults() {
+        let app = initialize();
+        assert_eq!(app.get_name(), "feroxbuster");
+    }
+}

src/progress.rs

@@ -1,9 +1,18 @@
 use crate::config::{CONFIGURATION, PROGRESS_BAR};
 use indicatif::{ProgressBar, ProgressStyle};
 
-pub fn add_bar(prefix: &str, length: u64, hidden: bool) -> ProgressBar {
+/// Add an [indicatif::ProgressBar](https://docs.rs/indicatif/latest/indicatif/struct.ProgressBar.html)
+/// to the global [PROGRESS_BAR](../config/struct.PROGRESS_BAR.html)
+pub fn add_bar(prefix: &str, length: u64, hidden: bool, hide_per_sec: bool) -> ProgressBar {
     let style = if hidden || CONFIGURATION.quiet {
         ProgressStyle::default_bar().template("")
+    } else if hide_per_sec {
+        ProgressStyle::default_bar()
+            .template(&format!(
+                "[{{bar:.cyan/blue}}] - {{elapsed:<4}} {{pos:>7}}/{{len:7}} {:7} {{prefix}}",
+                "-"
+            ))
+            .progress_chars("#>-")
     } else {
         ProgressStyle::default_bar()
             .template("[{bar:.cyan/blue}] - {elapsed:<4} {pos:>7}/{len:7} {per_sec:7} {prefix}")
@@ -18,3 +27,24 @@ pub fn add_bar(prefix: &str, length: u64, hidden: bool) -> ProgressBar {
 
     progress_bar
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    /// hit all code branches for add_bar
+    fn add_bar_with_all_configurations() {
+        let p1 = add_bar("prefix", 2, true, false); // hidden
+        let p2 = add_bar("prefix", 2, false, true); // no per second field
+        let p3 = add_bar("prefix", 2, false, false); // normal bar
+
+        p1.finish();
+        p2.finish();
+        p3.finish();
+
+        assert!(p1.is_finished());
+        assert!(p2.is_finished());
+        assert!(p3.is_finished());
+    }
+}

src/reporter.rs

@@ -0,0 +1,227 @@
+use crate::{
+    config::{CONFIGURATION, PROGRESS_PRINTER},
+    scanner::RESPONSES,
+    utils::{ferox_print, make_request, open_file},
+    FeroxChannel, FeroxResponse, FeroxSerialize,
+};
+use console::strip_ansi_codes;
+use std::io::Write;
+use std::sync::{Arc, Once, RwLock};
+use std::{fs, io};
+use tokio::sync::mpsc::{self, UnboundedReceiver, UnboundedSender};
+use tokio::task::JoinHandle;
+
+/// Singleton buffered file behind an Arc/RwLock; used for file writes from two locations:
+///     - [logger::initialize](../logger/fn.initialize.html) (specifically a closure on the global logger instance)
+///     - `reporter::spawn_file_handler`
+pub static mut LOCKED_FILE: Option<Arc<RwLock<io::BufWriter<fs::File>>>> = None;
+
+/// An initializer Once variable used to create `LOCKED_FILE`
+static INIT: Once = Once::new();
+
+// Accessing a `static mut` is unsafe much of the time, but if we do so
+// in a synchronized fashion (e.g., write once or read all) then we're
+// good to go!
+//
+// This function will only call `open_file` once, and will
+// otherwise always return the value returned from the first invocation.
+pub fn get_cached_file_handle(filename: &str) -> Option<Arc<RwLock<io::BufWriter<fs::File>>>> {
+    unsafe {
+        INIT.call_once(|| {
+            LOCKED_FILE = open_file(&filename);
+        });
+        LOCKED_FILE.clone()
+    }
+}
+
+/// Creates all required output handlers (terminal, file) and returns
+/// the transmitter sides of each mpsc along with each receiver's future's JoinHandle to be awaited
+///
+/// Any other module that needs to write a Response to stdout or output results to a file should
+/// be passed a clone of the appropriate returned transmitter
+pub fn initialize(
+    output_file: &str,
+    save_output: bool,
+) -> (
+    UnboundedSender<FeroxResponse>,
+    UnboundedSender<FeroxResponse>,
+    JoinHandle<()>,
+    Option<JoinHandle<()>>,
+) {
+    log::trace!("enter: initialize({}, {})", output_file, save_output);
+
+    let (tx_rpt, rx_rpt): FeroxChannel<FeroxResponse> = mpsc::unbounded_channel();
+    let (tx_file, rx_file): FeroxChannel<FeroxResponse> = mpsc::unbounded_channel();
+
+    let file_clone = tx_file.clone();
+
+    let term_reporter =
+        tokio::spawn(async move { spawn_terminal_reporter(rx_rpt, file_clone, save_output).await });
+
+    let file_reporter = if save_output {
+        // -o used, need to spawn the thread for writing to disk
+        let file_clone = output_file.to_string();
+        Some(tokio::spawn(async move {
+            spawn_file_reporter(rx_file, &file_clone).await
+        }))
+    } else {
+        None
+    };
+
+    log::trace!(
+        "exit: initialize -> ({:?}, {:?}, {:?}, {:?})",
+        tx_rpt,
+        tx_file,
+        term_reporter,
+        file_reporter
+    );
+    (tx_rpt, tx_file, term_reporter, file_reporter)
+}
+
+/// Spawn a single consumer task (sc side of mpsc)
+///
+/// The consumer simply receives responses and prints them if they meet the given
+/// reporting criteria
+async fn spawn_terminal_reporter(
+    mut resp_chan: UnboundedReceiver<FeroxResponse>,
+    file_chan: UnboundedSender<FeroxResponse>,
+    save_output: bool,
+) {
+    log::trace!(
+        "enter: spawn_terminal_reporter({:?}, {:?}, {})",
+        resp_chan,
+        file_chan,
+        save_output
+    );
+
+    while let Some(resp) = resp_chan.recv().await {
+        log::trace!("received {} on reporting channel", resp.url());
+
+        let contains_sentry = CONFIGURATION.status_codes.contains(&resp.status().as_u16());
+        let unknown_sentry = !RESPONSES.contains(&resp); // !contains == unknown
+        let should_process_response = contains_sentry && unknown_sentry;
+
+        if should_process_response {
+            // print to stdout
+            ferox_print(&resp.as_str(), &PROGRESS_PRINTER);
+
+            if save_output {
+                // -o used, need to send the report to be written out to disk
+                match file_chan.send(resp.clone()) {
+                    Ok(_) => {
+                        log::debug!("Sent {} to file handler", resp.url());
+                    }
+                    Err(e) => {
+                        log::error!("Could not send {} to file handler: {}", resp.url(), e);
+                    }
+                }
+            }
+        }
+        log::trace!("report complete: {}", resp.url());
+
+        if CONFIGURATION.replay_client.is_some() && should_process_response {
+            // replay proxy specified/client created and this response's status code is one that
+            // should be replayed
+            match make_request(CONFIGURATION.replay_client.as_ref().unwrap(), &resp.url()).await {
+                Ok(_) => {}
+                Err(e) => {
+                    log::error!("{}", e);
+                }
+            }
+        }
+
+        if should_process_response {
+            // add response to RESPONSES for serialization in case of ctrl+c
+            // placed all by its lonesome like this so that RESPONSES can take ownership
+            // of the FeroxResponse
+            RESPONSES.insert(resp);
+        }
+    }
+    log::trace!("exit: spawn_terminal_reporter");
+}
+
+/// Spawn a single consumer task (sc side of mpsc)
+///
+/// The consumer simply receives responses and writes them to the given output file if they meet
+/// the given reporting criteria
+async fn spawn_file_reporter(
+    mut report_channel: UnboundedReceiver<FeroxResponse>,
+    output_file: &str,
+) {
+    let buffered_file = match get_cached_file_handle(&CONFIGURATION.output) {
+        Some(file) => file,
+        None => {
+            log::trace!("exit: spawn_file_reporter");
+            return;
+        }
+    };
+
+    log::trace!(
+        "enter: spawn_file_reporter({:?}, {})",
+        report_channel,
+        output_file
+    );
+
+    log::info!("Writing scan results to {}", output_file);
+
+    while let Some(response) = report_channel.recv().await {
+        safe_file_write(&response, buffered_file.clone(), CONFIGURATION.json);
+    }
+
+    log::trace!("exit: spawn_file_reporter");
+}
+
+/// Given a string and a reference to a locked buffered file, write the contents and flush
+/// the buffer to disk.
+pub fn safe_file_write<T>(
+    value: &T,
+    locked_file: Arc<RwLock<io::BufWriter<fs::File>>>,
+    convert_to_json: bool,
+) where
+    T: FeroxSerialize,
+{
+    // note to future self: adding logging of anything other than error to this function
+    // is a bad idea. we call this function while processing records generated by the logger.
+    // If we then call log::... while already processing some logging output, it results in
+    // the second log entry being injected into the first.
+
+    let contents = if convert_to_json {
+        value.as_json()
+    } else {
+        value.as_str()
+    };
+
+    let contents = strip_ansi_codes(&contents);
+
+    if let Ok(mut handle) = locked_file.write() {
+        // write lock acquired
+        match handle.write(contents.as_bytes()) {
+            Ok(_) => {}
+            Err(e) => {
+                log::error!("could not write report to disk: {}", e);
+            }
+        }
+
+        match handle.flush() {
+            // this function is used within async functions/loops, so i'm flushing so that in
+            // the event of a ctrl+c or w/e results seen so far are saved instead of left lying
+            // around in the buffer
+            Ok(_) => {}
+            Err(e) => {
+                log::error!("error writing to file: {}", e);
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    #[should_panic]
+    /// asserts that an empty string for a filename returns None
+    fn reporter_get_cached_file_handle_without_filename_returns_none() {
+        let _used = get_cached_file_handle(&"").unwrap();
+    }
+}

src/scan_manager.rs

@@ -0,0 +1,992 @@
+use crate::config::Configuration;
+use crate::reporter::safe_file_write;
+use crate::utils::open_file;
+use crate::{
+    config::{CONFIGURATION, PROGRESS_PRINTER},
+    progress,
+    scanner::{NUMBER_OF_REQUESTS, RESPONSES, SCANNED_URLS},
+    FeroxResponse, FeroxSerialize, SLEEP_DURATION,
+};
+use console::style;
+use indicatif::{ProgressBar, ProgressStyle};
+use lazy_static::lazy_static;
+use serde::{
+    ser::{SerializeSeq, SerializeStruct},
+    Deserialize, Deserializer, Serialize, Serializer,
+};
+use serde_json::Value;
+use std::collections::HashMap;
+use std::{
+    cmp::PartialEq,
+    fmt,
+    fs::File,
+    io::BufReader,
+    sync::{Arc, Mutex, RwLock},
+    time::{SystemTime, UNIX_EPOCH},
+};
+use std::{
+    io::{stderr, Write},
+    sync::atomic::{AtomicBool, AtomicUsize, Ordering},
+};
+use tokio::{task::JoinHandle, time};
+use uuid::Uuid;
+
+lazy_static! {
+    /// A clock spinner protected with a RwLock to allow for a single thread to use at a time
+    // todo remove this when issue #107 is resolved
+    static ref SINGLE_SPINNER: RwLock<ProgressBar> = RwLock::new(get_single_spinner());
+}
+
+/// Single atomic number that gets incremented once, used to track first thread to interact with
+/// when pausing a scan
+static INTERACTIVE_BARRIER: AtomicUsize = AtomicUsize::new(0);
+
+/// Atomic boolean flag, used to determine whether or not a scan should pause or resume
+pub static PAUSE_SCAN: AtomicBool = AtomicBool::new(false);
+
+/// Simple enum used to flag a `FeroxScan` as likely a directory or file
+#[derive(Debug, Serialize, Deserialize)]
+pub enum ScanType {
+    File,
+    Directory,
+}
+
+/// Default implementation for ScanType
+impl Default for ScanType {
+    /// Return ScanType::File as default
+    fn default() -> Self {
+        Self::File
+    }
+}
+
+/// Struct to hold scan-related state
+///
+/// The purpose of this container is to open up the pathway to aborting currently running tasks and
+/// serialization of all scan state into a state file in order to resume scans that were cut short
+#[derive(Debug)]
+pub struct FeroxScan {
+    /// UUID that uniquely ID's the scan
+    pub id: String,
+
+    /// The URL that to be scanned
+    pub url: String,
+
+    /// The type of scan
+    pub scan_type: ScanType,
+
+    /// Whether or not this scan has completed
+    pub complete: bool,
+
+    /// The spawned tokio task performing this scan
+    pub task: Option<JoinHandle<()>>,
+
+    /// The progress bar associated with this scan
+    pub progress_bar: Option<ProgressBar>,
+}
+
+/// Default implementation for FeroxScan
+impl Default for FeroxScan {
+    /// Create a default FeroxScan, populates ID with a new UUID
+    fn default() -> Self {
+        let new_id = Uuid::new_v4().to_simple().to_string();
+
+        FeroxScan {
+            id: new_id,
+            task: None,
+            complete: false,
+            url: String::new(),
+            progress_bar: None,
+            scan_type: ScanType::File,
+        }
+    }
+}
+
+/// Implementation of FeroxScan
+impl FeroxScan {
+    /// Stop a currently running scan
+    pub fn abort(&self) {
+        self.stop_progress_bar();
+
+        if let Some(_task) = &self.task {
+            // task.abort();  todo uncomment once upgraded to tokio 0.3 (issue #107)
+        }
+    }
+
+    /// Simple helper to call .finish on the scan's progress bar
+    fn stop_progress_bar(&self) {
+        if let Some(pb) = &self.progress_bar {
+            pb.finish();
+        }
+    }
+
+    /// Simple helper get a progress bar
+    pub fn progress_bar(&mut self) -> ProgressBar {
+        if let Some(pb) = &self.progress_bar {
+            pb.clone()
+        } else {
+            let num_requests = NUMBER_OF_REQUESTS.load(Ordering::Relaxed);
+            let pb = progress::add_bar(&self.url, num_requests, false, false);
+
+            pb.reset_elapsed();
+
+            self.progress_bar = Some(pb.clone());
+
+            pb
+        }
+    }
+
+    /// Given a URL and ProgressBar, create a new FeroxScan, wrap it in an Arc and return it
+    pub fn new(url: &str, scan_type: ScanType, pb: Option<ProgressBar>) -> Arc<Mutex<Self>> {
+        let mut me = Self::default();
+
+        me.url = url.to_string();
+        me.scan_type = scan_type;
+        me.progress_bar = pb;
+
+        Arc::new(Mutex::new(me))
+    }
+
+    /// Mark the scan as complete and stop the scan's progress bar
+    pub fn finish(&mut self) {
+        self.complete = true;
+        self.stop_progress_bar();
+    }
+}
+
+/// Display implementation
+impl fmt::Display for FeroxScan {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let complete = if self.complete {
+            style("complete").green()
+        } else {
+            style("incomplete").red()
+        };
+
+        write!(f, "{:10} {}", complete, self.url)
+    }
+}
+
+/// PartialEq implementation; uses FeroxScan.id for comparison
+impl PartialEq for FeroxScan {
+    fn eq(&self, other: &Self) -> bool {
+        self.id == other.id
+    }
+}
+
+/// Serialize implementation for FeroxScan
+impl Serialize for FeroxScan {
+    /// Function that handles serialization of a FeroxScan
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let mut state = serializer.serialize_struct("FeroxScan", 4)?;
+
+        state.serialize_field("id", &self.id)?;
+        state.serialize_field("url", &self.url)?;
+        state.serialize_field("scan_type", &self.scan_type)?;
+        state.serialize_field("complete", &self.complete)?;
+
+        state.end()
+    }
+}
+
+/// Deserialize implementation for FeroxScan
+impl<'de> Deserialize<'de> for FeroxScan {
+    /// Deserialize a FeroxScan from a serde_json::Value
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let mut scan = Self::default();
+
+        let map: HashMap<String, Value> = HashMap::deserialize(deserializer)?;
+
+        for (key, value) in &map {
+            match key.as_str() {
+                "id" => {
+                    if let Some(id) = value.as_str() {
+                        scan.id = id.to_string();
+                    }
+                }
+                "scan_type" => {
+                    if let Some(scan_type) = value.as_str() {
+                        scan.scan_type = match scan_type {
+                            "File" => ScanType::File,
+                            "Directory" => ScanType::Directory,
+                            _ => ScanType::File,
+                        }
+                    }
+                }
+                "complete" => {
+                    if let Some(complete) = value.as_bool() {
+                        scan.complete = complete;
+                    }
+                }
+                "url" => {
+                    if let Some(url) = value.as_str() {
+                        scan.url = url.to_string();
+                    }
+                }
+                _ => {}
+            }
+        }
+
+        Ok(scan)
+    }
+}
+
+/// Container around a locked hashset of `FeroxScan`s, adds wrappers for insertion and searching
+#[derive(Debug, Default)]
+pub struct FeroxScans {
+    /// Internal structure: locked hashset of `FeroxScan`s
+    pub scans: Mutex<Vec<Arc<Mutex<FeroxScan>>>>,
+}
+
+/// Serialize implementation for FeroxScans
+impl Serialize for FeroxScans {
+    /// Function that handles serialization of FeroxScans
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        if let Ok(scans) = self.scans.lock() {
+            let mut seq = serializer.serialize_seq(Some(scans.len()))?;
+
+            for scan in scans.iter() {
+                if let Ok(unlocked) = scan.lock() {
+                    seq.serialize_element(&*unlocked)?;
+                }
+            }
+
+            seq.end()
+        } else {
+            // if for some reason we can't unlock the mutex, just write an empty list
+            let seq = serializer.serialize_seq(Some(0))?;
+            seq.end()
+        }
+    }
+}
+
+/// Implementation of `FeroxScans`
+impl FeroxScans {
+    /// Add a `FeroxScan` to the internal container
+    ///
+    /// If the internal container did NOT contain the scan, true is returned; else false
+    pub fn insert(&self, scan: Arc<Mutex<FeroxScan>>) -> bool {
+        let sentry = match scan.lock() {
+            Ok(locked_scan) => {
+                // If the container did contain the scan, set sentry to false
+                // If the container did not contain the scan, set sentry to true
+                !self.contains(&locked_scan.url)
+            }
+            Err(e) => {
+                // poisoned lock
+                log::error!("FeroxScan's ({:?}) mutex is poisoned: {}", self, e);
+                false
+            }
+        };
+
+        if sentry {
+            // can't update the internal container while the scan itself is locked, so first
+            // lock the scan and check the container for the scan's presence, then add if
+            // not found
+            match self.scans.lock() {
+                Ok(mut scans) => {
+                    scans.push(scan);
+                }
+                Err(e) => {
+                    log::error!("FeroxScans' container's mutex is poisoned: {}", e);
+                    return false;
+                }
+            }
+        }
+
+        sentry
+    }
+
+    /// Simple check for whether or not a FeroxScan is contained within the inner container based
+    /// on the given URL
+    pub fn contains(&self, url: &str) -> bool {
+        match self.scans.lock() {
+            Ok(scans) => {
+                for scan in scans.iter() {
+                    if let Ok(locked_scan) = scan.lock() {
+                        if locked_scan.url == url {
+                            return true;
+                        }
+                    }
+                }
+            }
+            Err(e) => {
+                log::error!("FeroxScans' container's mutex is poisoned: {}", e);
+            }
+        }
+        false
+    }
+
+    /// Find and return a `FeroxScan` based on the given URL
+    pub fn get_scan_by_url(&self, url: &str) -> Option<Arc<Mutex<FeroxScan>>> {
+        if let Ok(scans) = self.scans.lock() {
+            for scan in scans.iter() {
+                if let Ok(locked_scan) = scan.lock() {
+                    if locked_scan.url == url {
+                        return Some(scan.clone());
+                    }
+                }
+            }
+        }
+        None
+    }
+
+    /// Print all FeroxScans of type Directory
+    ///
+    /// Example:
+    ///   0: complete   https://10.129.45.20
+    ///   9: complete   https://10.129.45.20/images
+    ///  10: complete   https://10.129.45.20/assets
+    pub fn display_scans(&self) {
+        if let Ok(scans) = self.scans.lock() {
+            for (i, scan) in scans.iter().enumerate() {
+                if let Ok(unlocked_scan) = scan.lock() {
+                    match unlocked_scan.scan_type {
+                        ScanType::Directory => {
+                            PROGRESS_PRINTER.println(format!("{:3}: {}", i, unlocked_scan));
+                        }
+                        ScanType::File => {
+                            // we're only interested in displaying directory scans, as those are
+                            // the only ones that make sense to be stopped
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    /// Forced the calling thread into a busy loop
+    ///
+    /// Every `SLEEP_DURATION` milliseconds, the function examines the result stored in `PAUSE_SCAN`
+    ///
+    /// When the value stored in `PAUSE_SCAN` becomes `false`, the function returns, exiting the busy
+    /// loop
+    pub async fn pause(&self, get_user_input: bool) {
+        // function uses tokio::time, not std
+
+        // local testing showed a pretty slow increase (less than linear) in CPU usage as # of
+        // concurrent scans rose when SLEEP_DURATION was set to 500, using that as the default for now
+        let mut interval = time::interval(time::Duration::from_millis(SLEEP_DURATION));
+
+        // ignore any error returned
+        let _ = stderr().flush();
+
+        if INTERACTIVE_BARRIER.load(Ordering::Relaxed) == 0 {
+            INTERACTIVE_BARRIER.fetch_add(1, Ordering::Relaxed);
+
+            if get_user_input {
+                self.display_scans();
+
+                let mut user_input = String::new();
+                std::io::stdin().read_line(&mut user_input).unwrap();
+                // todo (issue #107) actual logic for parsing user input in a way that allows for
+                // calling .abort on the scan retrieved based on the input
+            }
+        }
+
+        if SINGLE_SPINNER.read().unwrap().is_finished() {
+            // todo remove this when issue #107 is resolved
+
+            // in order to not leave draw artifacts laying around in the terminal, we call
+            // finish_and_clear on the progress bar when resuming scans. For this reason, we need to
+            // check if the spinner is finished, and repopulate the RwLock with a new spinner if
+            // necessary
+            if let Ok(mut guard) = SINGLE_SPINNER.write() {
+                *guard = get_single_spinner();
+            }
+        }
+
+        if let Ok(spinner) = SINGLE_SPINNER.write() {
+            spinner.enable_steady_tick(120);
+        }
+
+        loop {
+            // first tick happens immediately, all others wait the specified duration
+            interval.tick().await;
+
+            if !PAUSE_SCAN.load(Ordering::Acquire) {
+                // PAUSE_SCAN is false, so we can exit the busy loop
+
+                if INTERACTIVE_BARRIER.load(Ordering::Relaxed) == 1 {
+                    INTERACTIVE_BARRIER.fetch_sub(1, Ordering::Relaxed);
+                }
+
+                if let Ok(spinner) = SINGLE_SPINNER.write() {
+                    // todo remove this when issue #107 is resolved
+                    spinner.finish_and_clear();
+                }
+
+                let _ = stderr().flush();
+
+                log::trace!("exit: pause_scan");
+                return;
+            }
+        }
+    }
+
+    /// Given a url, create a new `FeroxScan` and add it to `FeroxScans`
+    ///
+    /// If `FeroxScans` did not already contain the scan, return true; otherwise return false
+    ///
+    /// Also return a reference to the new `FeroxScan`
+    fn add_scan(&self, url: &str, scan_type: ScanType) -> (bool, Arc<Mutex<FeroxScan>>) {
+        let bar = match scan_type {
+            ScanType::Directory => {
+                let progress_bar = progress::add_bar(
+                    &url,
+                    NUMBER_OF_REQUESTS.load(Ordering::Relaxed),
+                    false,
+                    false,
+                );
+
+                progress_bar.reset_elapsed();
+
+                Some(progress_bar)
+            }
+            ScanType::File => None,
+        };
+
+        let ferox_scan = FeroxScan::new(&url, scan_type, bar);
+
+        // If the set did not contain the scan, true is returned.
+        // If the set did contain the scan, false is returned.
+        let response = self.insert(ferox_scan.clone());
+
+        (response, ferox_scan)
+    }
+
+    /// Given a url, create a new `FeroxScan` and add it to `FeroxScans` as a Directory Scan
+    ///
+    /// If `FeroxScans` did not already contain the scan, return true; otherwise return false
+    ///
+    /// Also return a reference to the new `FeroxScan`
+    pub fn add_directory_scan(&self, url: &str) -> (bool, Arc<Mutex<FeroxScan>>) {
+        self.add_scan(&url, ScanType::Directory)
+    }
+
+    /// Given a url, create a new `FeroxScan` and add it to `FeroxScans` as a File Scan
+    ///
+    /// If `FeroxScans` did not already contain the scan, return true; otherwise return false
+    ///
+    /// Also return a reference to the new `FeroxScan`
+    pub fn add_file_scan(&self, url: &str) -> (bool, Arc<Mutex<FeroxScan>>) {
+        self.add_scan(&url, ScanType::File)
+    }
+}
+
+/// Return a clock spinner, used when scans are paused
+// todo remove this when issue #107 is resolved
+fn get_single_spinner() -> ProgressBar {
+    log::trace!("enter: get_single_spinner");
+
+    let spinner = ProgressBar::new_spinner().with_style(
+        ProgressStyle::default_spinner()
+            .tick_strings(&[
+                "🕛", "🕐", "🕑", "🕒", "🕓", "🕔", "🕕", "🕖", "🕗", "🕘", "🕙", "🕚",
+            ])
+            .template(&format!(
+                "\t-= All Scans {{spinner}} {} =-",
+                style("Paused").red()
+            )),
+    );
+
+    log::trace!("exit: get_single_spinner -> {:?}", spinner);
+    spinner
+}
+
+/// Container around a locked vector of `FeroxResponse`s, adds wrappers for insertion and search
+#[derive(Debug, Default)]
+pub struct FeroxResponses {
+    /// Internal structure: locked hashset of `FeroxScan`s
+    pub responses: Arc<RwLock<Vec<FeroxResponse>>>,
+}
+
+/// Serialize implementation for FeroxResponses
+impl Serialize for FeroxResponses {
+    /// Function that handles serialization of FeroxResponses
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        if let Ok(responses) = self.responses.read() {
+            let mut seq = serializer.serialize_seq(Some(responses.len()))?;
+
+            for response in responses.iter() {
+                seq.serialize_element(response)?;
+            }
+
+            seq.end()
+        } else {
+            // if for some reason we can't unlock the mutex, just write an empty list
+            let seq = serializer.serialize_seq(Some(0))?;
+            seq.end()
+        }
+    }
+}
+
+/// Implementation of `FeroxResponses`
+impl FeroxResponses {
+    /// Add a `FeroxResponse` to the internal container
+    pub fn insert(&self, response: FeroxResponse) {
+        match self.responses.write() {
+            Ok(mut responses) => {
+                responses.push(response);
+            }
+            Err(e) => {
+                log::error!("FeroxResponses' container's mutex is poisoned: {}", e);
+            }
+        }
+    }
+
+    /// Simple check for whether or not a FeroxResponse is contained within the inner container
+    pub fn contains(&self, other: &FeroxResponse) -> bool {
+        match self.responses.read() {
+            Ok(responses) => {
+                for response in responses.iter() {
+                    if response.url == other.url {
+                        return true;
+                    }
+                }
+            }
+            Err(e) => {
+                log::error!("FeroxResponses' container's mutex is poisoned: {}", e);
+            }
+        }
+        false
+    }
+}
+/// Data container for (de)?serialization of multiple items
+#[derive(Serialize, Debug)]
+pub struct FeroxState {
+    /// Known scans
+    scans: &'static FeroxScans,
+
+    /// Current running config
+    config: &'static Configuration,
+
+    /// Known responses
+    responses: &'static FeroxResponses,
+}
+
+/// FeroxSerialize implementation for FeroxState
+impl FeroxSerialize for FeroxState {
+    /// Simply return debug format of FeroxState to satisfy as_str
+    fn as_str(&self) -> String {
+        format!("{:?}", self)
+    }
+
+    /// Simple call to produce a JSON string using the given FeroxState
+    fn as_json(&self) -> String {
+        serde_json::to_string(&self).unwrap_or_default()
+    }
+}
+
+/// Initialize the ctrl+c handler that saves scan state to disk
+pub fn initialize() {
+    log::trace!("enter: initialize");
+
+    let result = ctrlc::set_handler(move || {
+        let ts = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .unwrap()
+            .as_secs();
+
+        let slug = if !CONFIGURATION.target_url.is_empty() {
+            // target url populated
+            CONFIGURATION
+                .target_url
+                .replace("://", "_")
+                .replace("/", "_")
+                .replace(".", "_")
+        } else {
+            // stdin used
+            "stdin".to_string()
+        };
+
+        let filename = format!("ferox-{}-{}.state", slug, ts);
+        let warning = format!(
+            "🚨 Caught {} 🚨 saving scan state to {} ...",
+            style("ctrl+c").yellow(),
+            filename
+        );
+
+        PROGRESS_PRINTER.println(warning);
+
+        let state = FeroxState {
+            config: &CONFIGURATION,
+            scans: &SCANNED_URLS,
+            responses: &RESPONSES,
+        };
+
+        let state_file = open_file(&filename);
+
+        if let Some(buffered_file) = state_file {
+            safe_file_write(&state, buffered_file, true);
+        }
+
+        std::process::exit(1);
+    });
+
+    if result.is_err() {
+        log::error!("Could not set Ctrl+c handler");
+        std::process::exit(1);
+    }
+
+    log::trace!("exit: initialize");
+}
+
+/// Primary logic used to load a Configuration from disk and populate the appropriate data
+/// structures
+pub fn resume_scan(filename: &str) -> Configuration {
+    log::trace!("enter: resume_scan({})", filename);
+
+    let file = File::open(filename).unwrap_or_else(|e| {
+        log::error!("{}", e);
+        log::error!("Could not open state file, exiting");
+        std::process::exit(1);
+    });
+
+    let reader = BufReader::new(file);
+    let state: serde_json::Value = serde_json::from_reader(reader).unwrap();
+
+    let conf = state.get("config").unwrap_or_else(|| {
+        log::error!("Could not load configuration from state file, exiting");
+        std::process::exit(1);
+    });
+
+    let config = serde_json::from_value(conf.clone()).unwrap_or_else(|e| {
+        log::error!("{}", e);
+        log::error!("Could not deserialize configuration found in state file, exiting");
+        std::process::exit(1);
+    });
+
+    // let scans: FeroxScans = serde_json::from_value(state.get("scans").unwrap().clone()).unwrap();
+    if let Some(responses) = state.get("responses") {
+        if let Some(arr_responses) = responses.as_array() {
+            for response in arr_responses {
+                if let Ok(deser_resp) = serde_json::from_value(response.clone()) {
+                    RESPONSES.insert(deser_resp);
+                }
+            }
+        }
+    }
+
+    if let Some(scans) = state.get("scans") {
+        if let Some(arr_scans) = scans.as_array() {
+            for scan in arr_scans {
+                let deser_scan: FeroxScan =
+                    serde_json::from_value(scan.clone()).unwrap_or_default();
+                // need to determine if it's complete and based on that create a progress bar
+                // populate it accordingly based on completion
+                SCANNED_URLS.insert(Arc::new(Mutex::new(deser_scan)));
+            }
+        }
+    }
+
+    log::trace!("exit: resume_scan -> {:?}", config);
+    config
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use predicates::prelude::*;
+
+    #[test]
+    /// test that ScanType's default is File
+    fn default_scantype_is_file() {
+        match ScanType::default() {
+            ScanType::File => {}
+            ScanType::Directory => panic!(),
+        }
+    }
+
+    #[test]
+    /// test that get_single_spinner returns the correct spinner
+    // todo remove this when issue #107 is resolved
+    fn scanner_get_single_spinner_returns_spinner() {
+        let spinner = get_single_spinner();
+        assert!(!spinner.is_finished());
+    }
+
+    #[tokio::test(core_threads = 1)]
+    /// tests that pause_scan pauses execution and releases execution when PAUSE_SCAN is toggled
+    /// the spinner used during the test has had .finish_and_clear called on it, meaning that
+    /// a new one will be created, taking the if branch within the function
+    async fn scanner_pause_scan_with_finished_spinner() {
+        let now = time::Instant::now();
+        let urls = FeroxScans::default();
+
+        PAUSE_SCAN.store(true, Ordering::Relaxed);
+
+        let expected = time::Duration::from_secs(2);
+
+        tokio::spawn(async move {
+            time::delay_for(expected).await;
+            PAUSE_SCAN.store(false, Ordering::Relaxed);
+        });
+
+        urls.pause(false).await;
+
+        assert!(now.elapsed() > expected);
+    }
+
+    #[test]
+    /// add an unknown url to the hashset, expect true
+    fn add_url_to_list_of_scanned_urls_with_unknown_url() {
+        let urls = FeroxScans::default();
+        let url = "http://unknown_url";
+        let (result, _scan) = urls.add_scan(url, ScanType::Directory);
+        assert_eq!(result, true);
+    }
+
+    #[test]
+    /// add a known url to the hashset, with a trailing slash, expect false
+    fn add_url_to_list_of_scanned_urls_with_known_url() {
+        let urls = FeroxScans::default();
+        let pb = ProgressBar::new(1);
+        let url = "http://unknown_url/";
+        let scan = FeroxScan::new(url, ScanType::Directory, Some(pb));
+
+        assert_eq!(urls.insert(scan), true);
+
+        let (result, _scan) = urls.add_scan(url, ScanType::Directory);
+
+        assert_eq!(result, false);
+    }
+
+    #[test]
+    /// abort should call stop_progress_bar, marking it as finished
+    fn abort_stops_progress_bar() {
+        let pb = ProgressBar::new(1);
+        let url = "http://unknown_url/";
+        let scan = FeroxScan::new(url, ScanType::Directory, Some(pb));
+
+        assert_eq!(
+            scan.lock()
+                .unwrap()
+                .progress_bar
+                .as_ref()
+                .unwrap()
+                .is_finished(),
+            false
+        );
+
+        scan.lock().unwrap().abort();
+
+        assert_eq!(
+            scan.lock()
+                .unwrap()
+                .progress_bar
+                .as_ref()
+                .unwrap()
+                .is_finished(),
+            true
+        );
+    }
+
+    #[test]
+    /// add a known url to the hashset, without a trailing slash, expect false
+    fn add_url_to_list_of_scanned_urls_with_known_url_without_slash() {
+        let urls = FeroxScans::default();
+        let url = "http://unknown_url";
+        let scan = FeroxScan::new(url, ScanType::File, None);
+
+        assert_eq!(urls.insert(scan), true);
+
+        let (result, _scan) = urls.add_scan(url, ScanType::File);
+
+        assert_eq!(result, false);
+    }
+
+    #[test]
+    /// just increasing coverage, no real expectations
+    fn call_display_scans() {
+        let urls = FeroxScans::default();
+        let pb = ProgressBar::new(1);
+        let pb_two = ProgressBar::new(2);
+        let url = "http://unknown_url/";
+        let url_two = "http://unknown_url/fa";
+        let scan = FeroxScan::new(url, ScanType::Directory, Some(pb));
+        let scan_two = FeroxScan::new(url_two, ScanType::Directory, Some(pb_two));
+
+        scan_two.lock().unwrap().finish(); // one complete, one incomplete
+
+        assert_eq!(urls.insert(scan), true);
+
+        urls.display_scans();
+    }
+
+    #[test]
+    /// ensure that PartialEq compares FeroxScan.id fields
+    fn partial_eq_compares_the_id_field() {
+        let url = "http://unknown_url/";
+        let scan = FeroxScan::new(url, ScanType::Directory, None);
+        let scan_two = FeroxScan::new(url, ScanType::Directory, None);
+
+        assert!(!scan.lock().unwrap().eq(&scan_two.lock().unwrap()));
+
+        scan_two.lock().unwrap().id = scan.lock().unwrap().id.clone();
+
+        assert!(scan.lock().unwrap().eq(&scan_two.lock().unwrap()));
+    }
+
+    #[test]
+    /// show that a new progress bar is created if one doesn't exist
+    fn ferox_scan_get_progress_bar_when_none_is_set() {
+        let mut scan = FeroxScan::default();
+
+        assert!(scan.progress_bar.is_none()); // no pb exists
+
+        let pb = scan.progress_bar();
+
+        assert!(scan.progress_bar.is_some()); // new pb created
+        assert!(!pb.is_finished()) // not finished
+    }
+
+    #[test]
+    /// given a JSON entry representing a FeroxScan, test that it deserializes into the proper type
+    /// with the right attributes
+    fn ferox_scan_deserialize() {
+        let fs_json = r#"{"id":"057016a14769414aac9a7a62707598cb","url":"https://spiritanimal.com","scan_type":"Directory","complete":true}"#;
+        let fs_json_two = r#"{"id":"057016a14769414aac9a7a62707598cb","url":"https://spiritanimal.com","scan_type":"Not Correct","complete":true}"#;
+
+        let fs: FeroxScan = serde_json::from_str(fs_json).unwrap();
+        let fs_two: FeroxScan = serde_json::from_str(fs_json_two).unwrap();
+        assert_eq!(fs.url, "https://spiritanimal.com");
+
+        match fs.scan_type {
+            ScanType::Directory => {}
+            ScanType::File => {
+                panic!();
+            }
+        }
+        match fs_two.scan_type {
+            ScanType::Directory => {
+                panic!();
+            }
+            ScanType::File => {}
+        }
+
+        match fs.progress_bar {
+            None => {}
+            Some(_) => {
+                panic!();
+            }
+        }
+        assert_eq!(fs.complete, true);
+        assert_eq!(fs.id, "057016a14769414aac9a7a62707598cb");
+    }
+
+    #[test]
+    /// given a FeroxScan, test that it serializes into the proper JSON entry
+    fn ferox_scan_serialize() {
+        let fs = FeroxScan::new("https://spiritanimal.com", ScanType::Directory, None);
+        let fs_json = format!(
+            r#"{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","complete":false}}"#,
+            fs.lock().unwrap().id
+        );
+        assert_eq!(
+            fs_json,
+            serde_json::to_string(&*fs.lock().unwrap()).unwrap()
+        );
+    }
+
+    #[test]
+    /// given a FeroxScans, test that it serializes into the proper JSON entry
+    fn ferox_scans_serialize() {
+        let ferox_scan = FeroxScan::new("https://spiritanimal.com", ScanType::Directory, None);
+        let ferox_scans = FeroxScans::default();
+        let ferox_scans_json = format!(
+            r#"[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","complete":false}}]"#,
+            ferox_scan.lock().unwrap().id
+        );
+        ferox_scans.scans.lock().unwrap().push(ferox_scan);
+        assert_eq!(
+            ferox_scans_json,
+            serde_json::to_string(&ferox_scans).unwrap()
+        );
+    }
+
+    #[test]
+    /// given a FeroxResponses, test that it serializes into the proper JSON entry
+    fn ferox_responses_serialize() {
+        let json_response = r#"{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{"server":"nginx/1.16.1"}}"#;
+        let response: FeroxResponse = serde_json::from_str(json_response).unwrap();
+
+        let responses = FeroxResponses::default();
+        responses.insert(response);
+        // responses has a response now
+
+        // serialized should be a list of responses
+        let expected = format!("[{}]", json_response);
+
+        let serialized = serde_json::to_string(&responses).unwrap();
+        assert_eq!(expected, serialized);
+    }
+
+    #[test]
+    /// given a FeroxResponse, test that it serializes into the proper JSON entry
+    fn ferox_response_serialize_and_deserialize() {
+        // deserialize
+        let json_response = r#"{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{"server":"nginx/1.16.1"}}"#;
+        let response: FeroxResponse = serde_json::from_str(json_response).unwrap();
+
+        assert_eq!(response.url.as_str(), "https://nerdcore.com/css");
+        assert_eq!(response.url.path(), "/css");
+        assert_eq!(response.wildcard, true);
+        assert_eq!(response.status.as_u16(), 301);
+        assert_eq!(response.content_length, 173);
+        assert_eq!(response.line_count, 10);
+        assert_eq!(response.word_count, 16);
+        assert_eq!(response.headers.get("server").unwrap(), "nginx/1.16.1");
+
+        // serialize, however, this can fail when headers are out of order
+        let new_json = serde_json::to_string(&response).unwrap();
+        assert_eq!(json_response, new_json);
+    }
+
+    #[test]
+    /// test FeroxSerialize implementation of FeroxState
+    fn feroxstates_feroxserialize_implementation() {
+        let ferox_scan = FeroxScan::new("https://spiritanimal.com", ScanType::Directory, None);
+        let saved_id = ferox_scan.lock().unwrap().id.clone();
+        SCANNED_URLS.insert(ferox_scan);
+
+        let json_response = r#"{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{"server":"nginx/1.16.1"}}"#;
+        let response: FeroxResponse = serde_json::from_str(json_response).unwrap();
+        RESPONSES.insert(response);
+
+        let ferox_state = FeroxState {
+            scans: &SCANNED_URLS,
+            responses: &RESPONSES,
+            config: &CONFIGURATION,
+        };
+
+        let expected_strs = predicates::str::contains("scans: FeroxScans").and(
+            predicate::str::contains("config: Configuration")
+                .and(predicate::str::contains("responses: FeroxResponses"))
+                .and(predicate::str::contains("nerdcore.com"))
+                .and(predicate::str::contains("/css"))
+                .and(predicate::str::contains("https://spiritanimal.com")),
+        );
+
+        assert!(expected_strs.eval(&ferox_state.as_str()));
+
+        let json_state = ferox_state.as_json();
+        let expected = format!(
+            r#"{{"scans":[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","complete":false}}],"config":{{"type":"configuration","wordlist":"/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt","config":"","proxy":"","replay_proxy":"","target_url":"","status_codes":[200,204,301,302,307,308,401,403,405],"replay_codes":[200,204,301,302,307,308,401,403,405],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"quiet":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/1.9.0","redirects":false,"insecure":false,"extensions":[],"headers":{{}},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":false,"depth":4,"scan_limit":0,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false,"resumed":false,"save_state":true}},"responses":[{{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{{"server":"nginx/1.16.1"}}}}]}}"#,
+            saved_id
+        );
+
+        assert!(predicates::str::similar(expected).eval(&json_state));
+    }
+}

src/utils.rs

@@ -1,8 +1,41 @@
-use ansi_term::Color::{Blue, Cyan, Green, Red, Yellow};
-use console::{strip_ansi_codes, user_attended};
+use crate::{
+    config::{CONFIGURATION, PROGRESS_PRINTER},
+    FeroxError, FeroxResult,
+};
+use console::{strip_ansi_codes, style, user_attended};
 use indicatif::ProgressBar;
-use reqwest::Url;
+use reqwest::{Client, Response, Url};
+#[cfg(not(target_os = "windows"))]
+use rlimit::{getrlimit, setrlimit, Resource, Rlim};
 use std::convert::TryInto;
+use std::sync::{Arc, RwLock};
+use std::{fs, io};
+
+/// Given the path to a file, open the file in append mode (create it if it doesn't exist) and
+/// return a reference to the file that is buffered and locked
+pub fn open_file(filename: &str) -> Option<Arc<RwLock<io::BufWriter<fs::File>>>> {
+    log::trace!("enter: open_file({})", filename);
+
+    match fs::OpenOptions::new() // std fs
+        .create(true)
+        .append(true)
+        .open(filename)
+    {
+        Ok(file) => {
+            let writer = io::BufWriter::new(file); // std io
+
+            let locked_file = Some(Arc::new(RwLock::new(writer)));
+
+            log::trace!("exit: open_file -> {:?}", locked_file);
+            locked_file
+        }
+        Err(e) => {
+            log::error!("{}", e);
+            log::trace!("exit: open_file -> None");
+            None
+        }
+    }
+}
 
 /// Helper function that determines the current depth of a given url
 ///
@@ -18,13 +51,7 @@ use std::convert::TryInto;
 pub fn get_current_depth(target: &str) -> usize {
     log::trace!("enter: get_current_depth({})", target);
 
-    let target = if !target.ends_with('/') {
-        // target url doesn't end with a /, for the purposes of determining depth, we'll normalize
-        // all urls to end in a / and then calculate accordingly
-        format!("{}/", target)
-    } else {
-        String::from(target)
-    };
+    let target = normalize_url(target);
 
     match Url::parse(&target) {
         Ok(url) => {
@@ -61,17 +88,24 @@ pub fn get_current_depth(target: &str) -> usize {
 /// Takes in a string and examines the first character to return a color version of the same string
 pub fn status_colorizer(status: &str) -> String {
     match status.chars().next() {
-        Some('1') => Blue.paint(status).to_string(), // informational
-        Some('2') => Green.bold().paint(status).to_string(), // success
-        Some('3') => Yellow.paint(status).to_string(), // redirects
-        Some('4') => Red.paint(status).to_string(),  // client error
-        Some('5') => Red.paint(status).to_string(),  // server error
-        Some('W') => Cyan.paint(status).to_string(), // wildcard
-        Some('E') => Red.paint(status).to_string(),  // wildcard
-        _ => status.to_string(),                     // ¯\_(ツ)_/¯
+        Some('1') => style(status).blue().to_string(), // informational
+        Some('2') => style(status).green().to_string(), // success
+        Some('3') => style(status).yellow().to_string(), // redirects
+        Some('4') => style(status).red().to_string(),  // client error
+        Some('5') => style(status).red().to_string(),  // server error
+        Some('W') => style(status).cyan().to_string(), // wildcard
+        Some('E') => style(status).red().to_string(),  // error
+        _ => status.to_string(),                       // ¯\_(ツ)_/¯
     }
 }
 
+/// Takes in a string and colors it using console::style
+///
+/// mainly putting this here in case i want to change the color later, making any changes easy
+pub fn module_colorizer(modname: &str) -> String {
+    style(modname).cyan().to_string()
+}
+
 /// Gets the length of a url's path
 ///
 /// example: http://localhost/stuff -> 5
@@ -80,8 +114,8 @@ pub fn get_url_path_length(url: &Url) -> u64 {
 
     let path = url.path();
 
-    let segments = if path.starts_with('/') {
-        path[1..].split_terminator('/')
+    let segments = if let Some(split) = path.strip_prefix('/') {
+        split.split_terminator('/')
     } else {
         log::trace!("exit: get_url_path_length -> 0");
         return 0;
@@ -126,31 +160,434 @@ pub fn ferox_print(msg: &str, bar: &ProgressBar) {
     }
 }
 
+/// Simple helper to generate a `Url`
+///
+/// Errors during parsing `url` or joining `word` are propagated up the call stack
+pub fn format_url(
+    url: &str,
+    word: &str,
+    add_slash: bool,
+    queries: &[(String, String)],
+    extension: Option<&str>,
+) -> FeroxResult<Url> {
+    log::trace!(
+        "enter: format_url({}, {}, {}, {:?} {:?})",
+        url,
+        word,
+        add_slash,
+        queries,
+        extension
+    );
+
+    if Url::parse(&word).is_ok() {
+        // when a full url is passed in as a word to be joined to a base url using
+        // reqwest::Url::join, the result is that the word (url) completely overwrites the base
+        // url, potentially resulting in requests to places that aren't actually the target
+        // specified.
+        //
+        // in order to resolve the issue, we check if the word from the wordlist is a parsable URL
+        // and if so, don't do any further processing
+        let message = format!(
+            "word ({}) from the wordlist is actually a URL, skipping...",
+            word
+        );
+        log::warn!("{}", message);
+
+        let mut err = FeroxError::default();
+        err.message = message;
+
+        log::trace!("exit: format_url -> {}", err);
+        return Err(Box::new(err));
+    }
+
+    // from reqwest::Url::join
+    //   Note: a trailing slash is significant. Without it, the last path component
+    //   is considered to be a “file” name to be removed to get at the “directory”
+    //   that is used as the base
+    //
+    // the transforms that occur here will need to keep this in mind, i.e. add a slash to preserve
+    // the current directory sent as part of the url
+    let url = if word.is_empty() {
+        // v1.0.6: added during --extract-links feature inplementation to support creating urls
+        // that were extracted from response bodies, i.e. http://localhost/some/path/js/main.js
+        url.to_string()
+    } else if !url.ends_with('/') {
+        format!("{}/", url)
+    } else {
+        url.to_string()
+    };
+
+    let base_url = reqwest::Url::parse(&url)?;
+
+    // extensions and slashes are mutually exclusive cases
+    let word = if extension.is_some() {
+        format!("{}.{}", word, extension.unwrap())
+    } else if add_slash && !word.ends_with('/') {
+        // -f used, and word doesn't already end with a /
+        format!("{}/", word)
+    } else {
+        String::from(word)
+    };
+
+    match base_url.join(&word) {
+        Ok(request) => {
+            if queries.is_empty() {
+                // no query params to process
+                log::trace!("exit: format_url -> {}", request);
+                Ok(request)
+            } else {
+                match reqwest::Url::parse_with_params(request.as_str(), queries) {
+                    Ok(req_w_params) => {
+                        log::trace!("exit: format_url -> {}", req_w_params);
+                        Ok(req_w_params) // request with params attached
+                    }
+                    Err(e) => {
+                        log::error!(
+                            "Could not add query params {:?} to {}: {}",
+                            queries,
+                            request,
+                            e
+                        );
+                        log::trace!("exit: format_url -> {}", request);
+                        Ok(request) // couldn't process params, return initially ok url
+                    }
+                }
+            }
+        }
+        Err(e) => {
+            log::trace!("exit: format_url -> {}", e);
+            log::error!("Could not join {} with {}", word, base_url);
+            Err(Box::new(e))
+        }
+    }
+}
+
+/// Initiate request to the given `Url` using `Client`
+pub async fn make_request(client: &Client, url: &Url) -> FeroxResult<Response> {
+    log::trace!("enter: make_request(CONFIGURATION.Client, {})", url);
+
+    match client.get(url.to_owned()).send().await {
+        Ok(resp) => {
+            log::trace!("exit: make_request -> {:?}", resp);
+            Ok(resp)
+        }
+        Err(e) => {
+            log::trace!("exit: make_request -> {}", e);
+            if e.to_string().contains("operation timed out") {
+                // only warn for timeouts, while actual errors are still left as errors
+                log::warn!("Error while making request: {}", e);
+            } else if e.is_redirect() {
+                if let Some(last_redirect) = e.url() {
+                    // get where we were headed (last_redirect) and where we came from (url)
+                    let fancy_message = format!("{} !=> {}", url, last_redirect);
+
+                    let report = if let Some(msg_status) = e.status() {
+                        create_report_string(msg_status.as_str(), "-1", "-1", "-1", &fancy_message)
+                    } else {
+                        create_report_string("UNK", "-1", "-1", "-1", &fancy_message)
+                    };
+
+                    ferox_print(&report, &PROGRESS_PRINTER)
+                };
+            } else {
+                log::error!("Error while making request: {}", e);
+            }
+            Err(Box::new(e))
+        }
+    }
+}
+
+/// Helper to create the standard line for output to file/terminal
+///
+/// example output:
+/// 200      127l      283w     4134c http://localhost/faq
+pub fn create_report_string(
+    status: &str,
+    line_count: &str,
+    word_count: &str,
+    content_length: &str,
+    url: &str,
+) -> String {
+    if CONFIGURATION.quiet {
+        // -q used, just need the url
+        format!("{}\n", url)
+    } else {
+        // normal printing with status and sizes
+        let color_status = status_colorizer(status);
+        format!(
+            "{} {:>8}l {:>8}w {:>8}c {}\n",
+            color_status, line_count, word_count, content_length, url
+        )
+    }
+}
+
+/// Attempts to set the soft limit for the RLIMIT_NOFILE resource
+///
+/// RLIMIT_NOFILE is the maximum number of file descriptors that can be opened by this process
+///
+/// The soft limit is the value that the kernel enforces for the corresponding resource.
+/// The hard limit acts as a ceiling for the soft limit: an unprivileged process may set only its
+/// soft limit to a value in the range from 0 up to the hard limit, and (irreversibly) lower its
+/// hard limit.
+///
+/// A child process created via fork(2) inherits its parent's resource limits. Resource limits are
+/// per-process attributes that are shared by all of the threads in a process.
+///
+/// Based on the above information, no attempt is made to restore the limit to its pre-scan value
+/// as the adjustment made here is only valid for the scan itself (and any child processes, of which
+/// there are none).
+#[cfg(not(target_os = "windows"))]
+pub fn set_open_file_limit(limit: usize) -> bool {
+    log::trace!("enter: set_open_file_limit");
+
+    if let Ok((soft, hard)) = getrlimit(Resource::NOFILE) {
+        if hard.as_usize() > limit {
+            // our default open file limit is less than the current hard limit, this means we can
+            // set the soft limit to our default
+            let new_soft_limit = Rlim::from_usize(limit);
+
+            if setrlimit(Resource::NOFILE, new_soft_limit, hard).is_ok() {
+                log::debug!("set open file descriptor limit to {}", limit);
+
+                log::trace!("exit: set_open_file_limit -> {}", true);
+                return true;
+            }
+        } else if soft != hard {
+            // hard limit is lower than our default, the next best option is to set the soft limit as
+            // high as the hard limit will allow
+            if setrlimit(Resource::NOFILE, hard, hard).is_ok() {
+                log::debug!("set open file descriptor limit to {}", limit);
+
+                log::trace!("exit: set_open_file_limit -> {}", true);
+                return true;
+            }
+        }
+    }
+
+    // failed to set a new limit, as limit adjustments are a 'nice to have', we'll just log
+    // and move along
+    log::warn!("could not set open file descriptor limit to {}", limit);
+
+    log::trace!("exit: set_open_file_limit -> {}", false);
+    false
+}
+
+/// Simple helper to abstract away adding a forward-slash to a url if not present
+///
+/// used mostly for deduplication purposes and url state tracking
+pub fn normalize_url(url: &str) -> String {
+    log::trace!("enter: normalize_url({})", url);
+
+    let normalized = if url.ends_with('/') {
+        url.to_string()
+    } else {
+        format!("{}/", url)
+    };
+
+    log::trace!("exit: normalize_url -> {}", normalized);
+    normalized
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
 
     #[test]
-    fn base_url_returns_1() {
+    /// set_open_file_limit with a low requested limit succeeds
+    fn utils_set_open_file_limit_with_low_requested_limit() {
+        let (_, hard) = getrlimit(Resource::NOFILE).unwrap();
+        let lower_limit = hard.as_usize() - 1;
+        assert!(set_open_file_limit(lower_limit));
+    }
+
+    #[test]
+    /// set_open_file_limit with a high requested limit succeeds
+    fn utils_set_open_file_limit_with_high_requested_limit() {
+        let (_, hard) = getrlimit(Resource::NOFILE).unwrap();
+        let higher_limit = hard.as_usize() + 1;
+        // calculate a new soft to ensure soft != hard and hit that logic branch
+        let new_soft = Rlim::from_usize(hard.as_usize() - 1);
+        setrlimit(Resource::NOFILE, new_soft, hard).unwrap();
+        assert!(set_open_file_limit(higher_limit));
+    }
+
+    #[test]
+    /// set_open_file_limit should fail when hard == soft
+    fn utils_set_open_file_limit_with_fails_when_both_limits_are_equal() {
+        let (_, hard) = getrlimit(Resource::NOFILE).unwrap();
+        // calculate a new soft to ensure soft == hard and hit the failure logic branch
+        setrlimit(Resource::NOFILE, hard, hard).unwrap();
+        assert!(!set_open_file_limit(hard.as_usize())); // returns false
+    }
+
+    #[test]
+    /// base url returns 1
+    fn get_current_depth_base_url_returns_1() {
         let depth = get_current_depth("http://localhost");
         assert_eq!(depth, 1);
     }
 
     #[test]
-    fn base_url_with_slash_returns_1() {
+    /// base url with slash returns 1
+    fn get_current_depth_base_url_with_slash_returns_1() {
         let depth = get_current_depth("http://localhost/");
         assert_eq!(depth, 1);
     }
 
     #[test]
-    fn one_dir_returns_2() {
+    /// base url + 1 dir returns 2
+    fn get_current_depth_one_dir_returns_2() {
         let depth = get_current_depth("http://localhost/src");
         assert_eq!(depth, 2);
     }
 
     #[test]
-    fn one_dir_with_slash_returns_2() {
+    /// base url + 1 dir and slash returns 2
+    fn get_current_depth_one_dir_with_slash_returns_2() {
         let depth = get_current_depth("http://localhost/src/");
         assert_eq!(depth, 2);
     }
+
+    #[test]
+    /// base url + 1 dir and slash returns 2
+    fn get_current_depth_single_forward_slash_is_zero() {
+        let depth = get_current_depth("");
+        assert_eq!(depth, 0);
+    }
+
+    #[test]
+    /// base url + 1 word + no slash + no extension
+    fn format_url_normal() {
+        assert_eq!(
+            format_url("http://localhost", "stuff", false, &Vec::new(), None).unwrap(),
+            reqwest::Url::parse("http://localhost/stuff").unwrap()
+        );
+    }
+
+    #[test]
+    /// base url + no word + no slash + no extension
+    fn format_url_no_word() {
+        assert_eq!(
+            format_url("http://localhost", "", false, &Vec::new(), None).unwrap(),
+            reqwest::Url::parse("http://localhost").unwrap()
+        );
+    }
+
+    #[test]
+    /// base url + word + no slash + no extension + queries
+    fn format_url_joins_queries() {
+        assert_eq!(
+            format_url(
+                "http://localhost",
+                "lazer",
+                false,
+                &[(String::from("stuff"), String::from("things"))],
+                None
+            )
+            .unwrap(),
+            reqwest::Url::parse("http://localhost/lazer?stuff=things").unwrap()
+        );
+    }
+
+    #[test]
+    /// base url + no word + no slash + no extension + queries
+    fn format_url_without_word_joins_queries() {
+        assert_eq!(
+            format_url(
+                "http://localhost",
+                "",
+                false,
+                &[(String::from("stuff"), String::from("things"))],
+                None
+            )
+            .unwrap(),
+            reqwest::Url::parse("http://localhost/?stuff=things").unwrap()
+        );
+    }
+
+    #[test]
+    #[should_panic]
+    /// no base url is an error
+    fn format_url_no_url() {
+        format_url("", "stuff", false, &Vec::new(), None).unwrap();
+    }
+
+    #[test]
+    /// word prepended with slash is adjusted for correctness
+    fn format_url_word_with_preslash() {
+        assert_eq!(
+            format_url("http://localhost", "/stuff", false, &Vec::new(), None).unwrap(),
+            reqwest::Url::parse("http://localhost/stuff").unwrap()
+        );
+    }
+
+    #[test]
+    /// word with appended slash allows the slash to persist
+    fn format_url_word_with_postslash() {
+        assert_eq!(
+            format_url("http://localhost", "stuff/", false, &Vec::new(), None).unwrap(),
+            reqwest::Url::parse("http://localhost/stuff/").unwrap()
+        );
+    }
+
+    #[test]
+    /// word that is a fully formed url, should return an error
+    fn format_url_word_that_is_a_url() {
+        let url = format_url(
+            "http://localhost",
+            "http://schmocalhost",
+            false,
+            &Vec::new(),
+            None,
+        );
+        assert!(url.is_err());
+    }
+
+    #[test]
+    /// status colorizer uses red for 500s
+    fn status_colorizer_uses_red_for_500s() {
+        assert_eq!(status_colorizer("500"), style("500").red().to_string());
+    }
+
+    #[test]
+    /// status colorizer uses red for 400s
+    fn status_colorizer_uses_red_for_400s() {
+        assert_eq!(status_colorizer("400"), style("400").red().to_string());
+    }
+
+    #[test]
+    /// status colorizer uses red for errors
+    fn status_colorizer_uses_red_for_errors() {
+        assert_eq!(status_colorizer("ERROR"), style("ERROR").red().to_string());
+    }
+
+    #[test]
+    /// status colorizer uses cyan for wildcards
+    fn status_colorizer_uses_cyan_for_wildcards() {
+        assert_eq!(status_colorizer("WLD"), style("WLD").cyan().to_string());
+    }
+
+    #[test]
+    /// status colorizer uses blue for 100s
+    fn status_colorizer_uses_blue_for_100s() {
+        assert_eq!(status_colorizer("100"), style("100").blue().to_string());
+    }
+
+    #[test]
+    /// status colorizer uses green for 200s
+    fn status_colorizer_uses_green_for_200s() {
+        assert_eq!(status_colorizer("200"), style("200").green().to_string());
+    }
+
+    #[test]
+    /// status colorizer uses yellow for 300s
+    fn status_colorizer_uses_yellow_for_300s() {
+        assert_eq!(status_colorizer("300"), style("300").yellow().to_string());
+    }
+
+    #[test]
+    /// status colorizer doesnt color anything else
+    fn status_colorizer_returns_as_is() {
+        assert_eq!(status_colorizer("farfignewton"), "farfignewton".to_string());
+    }
 }

tests/test_banner.rs

@@ -0,0 +1,785 @@
+mod utils;
+use assert_cmd::Command;
+use predicates::prelude::*;
+use utils::{setup_tmp_directory, teardown_tmp_directory};
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + proxy
+fn banner_prints_proxy() -> Result<(), Box<dyn std::error::Error>> {
+    let urls = vec![
+        String::from("http://localhost"),
+        String::from("http://schmocalhost"),
+    ];
+    let (tmp_dir, file) = setup_tmp_directory(&urls, "wordlist")?;
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--stdin")
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--proxy")
+        .arg("http://127.0.0.1:8080")
+        .pipe_stdin(file)
+        .unwrap()
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("http://schmocalhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Proxy"))
+                .and(predicate::str::contains("http://127.0.0.1:8080"))
+                .and(predicate::str::contains("─┴─")),
+        );
+
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + replay proxy
+fn banner_prints_replay_proxy() -> Result<(), Box<dyn std::error::Error>> {
+    let urls = vec![
+        String::from("http://localhost"),
+        String::from("http://schmocalhost"),
+    ];
+    let (tmp_dir, file) = setup_tmp_directory(&urls, "wordlist")?;
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--stdin")
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--replay-proxy")
+        .arg("http://127.0.0.1:8081")
+        .pipe_stdin(file)
+        .unwrap()
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("http://schmocalhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Replay Proxy"))
+                .and(predicate::str::contains("http://127.0.0.1:8081"))
+                .and(predicate::str::contains("─┴─")),
+        );
+
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + multiple headers
+fn banner_prints_headers() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--headers")
+        .arg("stuff:things")
+        .arg("-H")
+        .arg("mostuff:mothings")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Header"))
+                .and(predicate::str::contains("stuff: things"))
+                .and(predicate::str::contains("mostuff: mothings"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + multiple size filters
+fn banner_prints_filter_sizes() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-S")
+        .arg("789456123")
+        .arg("--filter-size")
+        .arg("44444444")
+        .arg("-N")
+        .arg("678")
+        .arg("--filter-lines")
+        .arg("679")
+        .arg("-W")
+        .arg("93")
+        .arg("--filter-words")
+        .arg("94")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Size Filter"))
+                .and(predicate::str::contains("Word Count Filter"))
+                .and(predicate::str::contains("Line Count Filter"))
+                .and(predicate::str::contains("789456123"))
+                .and(predicate::str::contains("44444444"))
+                .and(predicate::str::contains("93"))
+                .and(predicate::str::contains("94"))
+                .and(predicate::str::contains("678"))
+                .and(predicate::str::contains("679"))
+                .and(predicate::str::contains("─┴─")),
+        );
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + queries
+fn banner_prints_queries() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-Q")
+        .arg("token=supersecret")
+        .arg("--query")
+        .arg("stuff=things")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Query Parameter"))
+                .and(predicate::str::contains("token=supersecret"))
+                .and(predicate::str::contains("stuff=things"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + status codes
+fn banner_prints_status_codes() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-s")
+        .arg("201,301,401")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("[201, 301, 401]"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + replay codes
+fn banner_prints_replay_codes() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--replay-codes")
+        .arg("200,302")
+        .arg("--replay-proxy")
+        .arg("http://localhost:8081")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Replay Proxy"))
+                .and(predicate::str::contains("http://localhost:8081"))
+                .and(predicate::str::contains("Replay Proxy Codes"))
+                .and(predicate::str::contains("[200, 302]"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + output file
+fn banner_prints_output_file() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--output")
+        .arg("/super/cool/path")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Output File"))
+                .and(predicate::str::contains("/super/cool/path"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + insecure
+fn banner_prints_insecure() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-k")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Insecure"))
+                .and(predicate::str::contains("true"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + follow redirects
+fn banner_prints_redirects() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-r")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Follow Redirects"))
+                .and(predicate::str::contains("true"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + extensions
+fn banner_prints_extensions() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-x")
+        .arg("js")
+        .arg("--extensions")
+        .arg("pdf")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Extensions"))
+                .and(predicate::str::contains("[js, pdf]"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + dont_filter
+fn banner_prints_dont_filter() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--dont-filter")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Filter Wildcards"))
+                .and(predicate::str::contains("false"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + verbosity=1
+fn banner_prints_verbosity_one() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-v")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Verbosity"))
+                .and(predicate::str::contains("│ 1"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + verbosity=2
+fn banner_prints_verbosity_two() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-vv")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Verbosity"))
+                .and(predicate::str::contains("│ 2"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + verbosity=3
+fn banner_prints_verbosity_three() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-vvv")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Verbosity"))
+                .and(predicate::str::contains("│ 3"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + verbosity=4
+fn banner_prints_verbosity_four() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-vvvv")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Verbosity"))
+                .and(predicate::str::contains("│ 4"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + add slash
+fn banner_prints_add_slash() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-f")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Add Slash"))
+                .and(predicate::str::contains("true"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + INFINITE recursion
+fn banner_prints_infinite_depth() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--depth")
+        .arg("0")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Recursion Depth"))
+                .and(predicate::str::contains("INFINITE"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + recursion depth
+fn banner_prints_recursion_depth() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--depth")
+        .arg("343214")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Recursion Depth"))
+                .and(predicate::str::contains("343214"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + no recursion
+fn banner_prints_no_recursion() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-n")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Do Not Recurse"))
+                .and(predicate::str::contains("true"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see nothing
+fn banner_doesnt_print() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-q")
+        .assert()
+        .success()
+        .stderr(predicate::str::is_empty());
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + extract-links
+fn banner_prints_extract_links() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-e")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Extract Links"))
+                .and(predicate::str::contains("true"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + scan-limit
+fn banner_prints_scan_limit() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-L")
+        .arg("4")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Concurrent Scan Limit"))
+                .and(predicate::str::contains("│ 4"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + filter-status
+fn banner_prints_filter_status() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-C")
+        .arg("200")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Status Code Filters"))
+                .and(predicate::str::contains("│ [200]"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + json
+fn banner_prints_json() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--json")
+        .arg("--output")
+        .arg("/dev/null")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("JSON Output"))
+                .and(predicate::str::contains("│ true"))
+                .and(predicate::str::contains("─┴─")),
+        );
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + json
+fn banner_prints_debug_log() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--debug-log")
+        .arg("/dev/null")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Debugging Log"))
+                .and(predicate::str::contains("│ /dev/null"))
+                .and(predicate::str::contains("─┴─")),
+        );
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + regex filters
+fn banner_prints_filter_regex() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--filter-regex")
+        .arg("^ignore me$")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Regex Filter"))
+                .and(predicate::str::contains("│ ^ignore me$"))
+                .and(predicate::str::contains("─┴─")),
+        );
+}

tests/test_config.rs

@@ -0,0 +1,27 @@
+mod utils;
+use assert_cmd::prelude::*;
+use predicates::prelude::*;
+use std::process::Command;
+use utils::{setup_tmp_directory, teardown_tmp_directory};
+
+#[test]
+/// send a single valid request, expect a 200 response
+fn read_in_config_file_for_settings() -> Result<(), Box<dyn std::error::Error>> {
+    let (tmp_dir, file) = setup_tmp_directory(&["threads = 37".to_string()], "ferox-config.toml")?;
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .current_dir(&tmp_dir)
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-vvvv")
+        .assert()
+        .success()
+        .stderr(predicate::str::contains("│ 37"));
+
+    teardown_tmp_directory(tmp_dir);
+
+    Ok(())
+}

tests/test_extractor.rs

@@ -0,0 +1,229 @@
+mod utils;
+use assert_cmd::prelude::*;
+use httpmock::Method::GET;
+use httpmock::{Mock, MockServer};
+use predicates::prelude::*;
+use std::process::Command;
+use utils::{setup_tmp_directory, teardown_tmp_directory};
+
+#[test]
+/// send a request to a page that contains a relative link, --extract-links should find the link
+/// and make a request to the new link
+fn extractor_finds_absolute_url() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/LICENSE")
+        .return_status(200)
+        .return_body(&srv.url("'/homepage/assets/img/icons/handshake.svg'"))
+        .create_on(&srv);
+
+    let mock_two = Mock::new()
+        .expect_method(GET)
+        .expect_path("/homepage/assets/img/icons/handshake.svg")
+        .return_status(200)
+        .create_on(&srv);
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--extract-links")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .and(predicate::str::contains("200"))
+            .and(predicate::str::contains(
+                "/homepage/assets/img/icons/handshake.svg",
+            )),
+    );
+
+    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock_two.times_called(), 1);
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// send a request to a page that contains an absolute link to another domain, scanner should not
+/// follow
+fn extractor_finds_absolute_url_to_different_domain() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/LICENSE")
+        .return_status(200)
+        .return_body("\"http://localhost/homepage/assets/img/icons/handshake.svg\"")
+        .create_on(&srv);
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--extract-links")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .and(predicate::str::contains("200"))
+            .and(predicate::str::contains(
+                "/homepage/assets/img/icons/handshake.svg",
+            ))
+            .not(),
+    );
+
+    assert_eq!(mock.times_called(), 1);
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// send a request to a page that contains a relative link, should follow
+fn extractor_finds_relative_url() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/LICENSE")
+        .return_status(200)
+        .return_body("\"/homepage/assets/img/icons/handshake.svg\"")
+        .create_on(&srv);
+
+    let mock_two = Mock::new()
+        .expect_method(GET)
+        .expect_path("/homepage/assets/img/icons/handshake.svg")
+        .return_status(200)
+        .create_on(&srv);
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--extract-links")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .and(predicate::str::contains("200"))
+            .and(predicate::str::contains(
+                "/homepage/assets/img/icons/handshake.svg",
+            )),
+    );
+
+    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock_two.times_called(), 1);
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// send a request to a page that contains an relative link, follow it, and find the same link again
+/// should follow then filter
+fn extractor_finds_same_relative_url_twice() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) =
+        setup_tmp_directory(&["LICENSE".to_string(), "README".to_string()], "wordlist").unwrap();
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/LICENSE")
+        .return_status(200)
+        .return_body(&srv.url("\"/homepage/assets/img/icons/handshake.svg\""))
+        .create_on(&srv);
+
+    let mock_two = Mock::new()
+        .expect_method(GET)
+        .expect_path("/README")
+        .return_body(&srv.url("\"/homepage/assets/img/icons/handshake.svg\""))
+        .return_status(200)
+        .create_on(&srv);
+
+    let mock_three = Mock::new()
+        .expect_method(GET)
+        .expect_path("/homepage/assets/img/icons/handshake.svg")
+        .return_status(200)
+        .create_on(&srv);
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--extract-links")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .and(predicate::str::contains("200"))
+            .and(predicate::str::contains(
+                "/homepage/assets/img/icons/handshake.svg",
+            )),
+    );
+
+    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock_two.times_called(), 1);
+    assert!(mock_three.times_called() <= 2); // todo: sometimes this is 2 instead of 1
+                                             // the expectation is one, suggesting a race condition... investigate and fix
+    teardown_tmp_directory(tmp_dir);
+}
+
+#[test]
+/// send a request to a page that contains an absolute link that leads to a page with a filter_size
+/// that should filter it out, expect not to see the second response reported
+fn extractor_finds_filtered_content() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) =
+        setup_tmp_directory(&["LICENSE".to_string(), "README".to_string()], "wordlist")?;
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/LICENSE")
+        .return_status(200)
+        .return_body(&srv.url("\"/homepage/assets/img/icons/handshake.svg\""))
+        .create_on(&srv);
+
+    let mock_two = Mock::new()
+        .expect_method(GET)
+        .expect_path("/homepage/assets/img/icons/handshake.svg")
+        .return_body("im a little teapot")
+        .return_status(200)
+        .create_on(&srv);
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--extract-links")
+        .arg("--filter-size")
+        .arg("18")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .and(predicate::str::contains("200"))
+            .and(predicate::str::contains(
+                "/homepage/assets/img/icons/handshake.svg",
+            ))
+            .not(),
+    );
+
+    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock_two.times_called(), 1);
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}

tests/test_filters.rs

@@ -0,0 +1,204 @@
+mod utils;
+use assert_cmd::prelude::*;
+use httpmock::Method::GET;
+use httpmock::{Mock, MockServer};
+use predicates::prelude::*;
+use std::process::Command;
+use utils::{setup_tmp_directory, teardown_tmp_directory};
+
+#[test]
+/// create a FeroxResponse that should elicit a true from
+/// StatusCodeFilter::should_filter_response
+fn filters_status_code_should_filter_response() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) =
+        setup_tmp_directory(&["LICENSE".to_string(), "file.js".to_string()], "wordlist").unwrap();
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/LICENSE")
+        .return_status(302)
+        .return_body("this is a test")
+        .create_on(&srv);
+
+    let mock_two = Mock::new()
+        .expect_method(GET)
+        .expect_path("/file.js")
+        .return_status(200)
+        .return_body("this is also a test of some import")
+        .create_on(&srv);
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-vvvv")
+        .arg("--filter-status")
+        .arg("302")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .not()
+            .and(predicate::str::contains("302"))
+            .not()
+            .and(predicate::str::contains("14c"))
+            .not()
+            .and(predicate::str::contains("/file.js"))
+            .and(predicate::str::contains("200"))
+            .and(predicate::str::contains("34c")),
+    );
+
+    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock_two.times_called(), 1);
+    teardown_tmp_directory(tmp_dir);
+}
+
+#[test]
+/// create a FeroxResponse that should elicit a true from
+/// LinesFilter::should_filter_response
+fn filters_lines_should_filter_response() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) =
+        setup_tmp_directory(&["LICENSE".to_string(), "file.js".to_string()], "wordlist").unwrap();
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/LICENSE")
+        .return_status(302)
+        .return_body("this is a test")
+        .create_on(&srv);
+
+    let mock_two = Mock::new()
+        .expect_method(GET)
+        .expect_path("/file.js")
+        .return_status(200)
+        .return_body("this is also a test of some import\nwith 2 lines, no less")
+        .create_on(&srv);
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--filter-lines")
+        .arg("2")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .and(predicate::str::contains("302"))
+            .and(predicate::str::contains("14"))
+            .and(predicate::str::contains("/file.js"))
+            .not()
+            .and(predicate::str::contains("200"))
+            .not()
+            .and(predicate::str::contains("2l"))
+            .not(),
+    );
+
+    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock_two.times_called(), 1);
+    teardown_tmp_directory(tmp_dir);
+}
+
+#[test]
+/// create a FeroxResponse that should elicit a true from
+/// WordsFilter::should_filter_response
+fn filters_words_should_filter_response() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) =
+        setup_tmp_directory(&["LICENSE".to_string(), "file.js".to_string()], "wordlist").unwrap();
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/LICENSE")
+        .return_status(302)
+        .return_body("this is a test")
+        .create_on(&srv);
+
+    let mock_two = Mock::new()
+        .expect_method(GET)
+        .expect_path("/file.js")
+        .return_status(200)
+        .return_body("this is also a test of some import\nwith 2 lines, no less")
+        .create_on(&srv);
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--filter-words")
+        .arg("13")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .and(predicate::str::contains("302"))
+            .and(predicate::str::contains("14"))
+            .and(predicate::str::contains("/file.js"))
+            .not()
+            .and(predicate::str::contains("200"))
+            .not()
+            .and(predicate::str::contains("13w"))
+            .not(),
+    );
+
+    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock_two.times_called(), 1);
+    teardown_tmp_directory(tmp_dir);
+}
+
+#[test]
+/// create a FeroxResponse that should elicit a true from
+/// SizeFilter::should_filter_response
+fn filters_size_should_filter_response() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) =
+        setup_tmp_directory(&["LICENSE".to_string(), "file.js".to_string()], "wordlist").unwrap();
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/LICENSE")
+        .return_status(302)
+        .return_body("this is a test")
+        .create_on(&srv);
+
+    let mock_two = Mock::new()
+        .expect_method(GET)
+        .expect_path("/file.js")
+        .return_status(200)
+        .return_body("this is also a test of some import\nwith 2 lines, no less")
+        .create_on(&srv);
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--filter-size")
+        .arg("56")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .and(predicate::str::contains("302"))
+            .and(predicate::str::contains("14"))
+            .and(predicate::str::contains("/file.js"))
+            .not()
+            .and(predicate::str::contains("200"))
+            .not()
+            .and(predicate::str::contains("56c"))
+            .not(),
+    );
+
+    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock_two.times_called(), 1);
+    teardown_tmp_directory(tmp_dir);
+}

tests/test_heuristics.rs

@@ -10,19 +10,19 @@ use utils::{setup_tmp_directory, teardown_tmp_directory};
 /// test passes one bad target via -u to the scanner, expected result is that the
 /// scanner dies
 fn test_single_target_cannot_connect() -> Result<(), Box<dyn std::error::Error>> {
-    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()])?;
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
-    let cmd = std::panic::catch_unwind(|| {
-        Command::cargo_bin("feroxbuster")
-            .unwrap()
-            .arg("--url")
-            .arg("http://fjdksafjkdsajfkdsajkfdsajkfsdjkdsfdsafdsafdsajkr3l2ajfdskafdsjk")
-            .arg("--wordlist")
-            .arg(file.as_os_str())
-            .unwrap()
-    });
-
-    assert!(cmd.is_err());
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://fjdksafjkdsajfkdsajkfdsajkfsdjkdsfdsafdsafdsajkr3l2ajfdskafdsjk")
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .assert()
+        .success()
+        .stdout(
+            predicate::str::contains("Could not connect to http://fjdksafjkdsajfkdsajkfdsajkfsdjkdsfdsafdsafdsajkr3l2ajfdskafdsjk, skipping...", )
+        );
 
     teardown_tmp_directory(tmp_dir);
     Ok(())
@@ -35,20 +35,20 @@ fn test_two_targets_cannot_connect() -> Result<(), Box<dyn std::error::Error>> {
     let not_real =
         String::from("http://fjdksafjkdsajfkdsajkfdsajkfsdjkdsfdsafdsafdsajkr3l2ajfdskafdsjk");
     let urls = vec![not_real.clone(), not_real];
-    let (tmp_dir, file) = setup_tmp_directory(&urls)?;
+    let (tmp_dir, file) = setup_tmp_directory(&urls, "wordlist")?;
 
-    let cmd = std::panic::catch_unwind(|| {
-        Command::cargo_bin("feroxbuster")
-            .unwrap()
-            .arg("--stdin")
-            .arg("--wordlist")
-            .arg(file.as_os_str())
-            .pipe_stdin(file)
-            .unwrap()
-            .unwrap()
-    });
-
-    assert!(cmd.is_err());
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--stdin")
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .pipe_stdin(file)
+        .unwrap()
+        .assert()
+        .success()
+        .stdout(
+            predicate::str::contains("Could not connect to http://fjdksafjkdsajfkdsajkfdsajkfsdjkdsfdsafdsafdsajkr3l2ajfdskafdsjk, skipping...", )
+        );
 
     teardown_tmp_directory(tmp_dir);
     Ok(())
@@ -63,7 +63,7 @@ fn test_one_good_and_one_bad_target_scan_succeeds() -> Result<(), Box<dyn std::e
     let not_real =
         String::from("http://fjdksafjkdsajfkdsajkfdsajkfsdjkdsfdsafdsafdsajkr3l2ajfdskafdsjk");
     let urls = vec![not_real, srv.url("/"), String::from("LICENSE")];
-    let (tmp_dir, file) = setup_tmp_directory(&urls)?;
+    let (tmp_dir, file) = setup_tmp_directory(&urls, "wordlist")?;
 
     let mock = Mock::new()
         .expect_method(GET)
@@ -83,8 +83,8 @@ fn test_one_good_and_one_bad_target_scan_succeeds() -> Result<(), Box<dyn std::e
         .success()
         .stdout(
             predicate::str::contains("/LICENSE")
-                .and(predicate::str::contains("200 OK"))
-                .and(predicate::str::contains("[14 bytes]")),
+                .and(predicate::str::contains("200"))
+                .and(predicate::str::contains("14")),
         );
     assert_eq!(mock.times_called(), 1);
 
@@ -93,9 +93,10 @@ fn test_one_good_and_one_bad_target_scan_succeeds() -> Result<(), Box<dyn std::e
 }
 
 #[test]
+/// test finds a static wildcard and reports as much to stdout
 fn test_static_wildcard_request_found() -> Result<(), Box<dyn std::error::Error>> {
     let srv = MockServer::start();
-    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()])?;
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
     let mock = Mock::new()
         .expect_method(GET)
@@ -110,12 +111,12 @@ fn test_static_wildcard_request_found() -> Result<(), Box<dyn std::error::Error>
         .arg(srv.url("/"))
         .arg("--wordlist")
         .arg(file.as_os_str())
-        .arg("--addslash")
+        .arg("--add-slash")
         .unwrap();
 
     teardown_tmp_directory(tmp_dir);
 
-    cmd.assert().success().stderr(
+    cmd.assert().success().stdout(
         predicate::str::contains("WLD")
             .and(predicate::str::contains("Got"))
             .and(predicate::str::contains("200"))
@@ -127,9 +128,11 @@ fn test_static_wildcard_request_found() -> Result<(), Box<dyn std::error::Error>
 }
 
 #[test]
-fn test_dynamic_wildcard_request_found() -> Result<(), Box<dyn std::error::Error>> {
+/// test finds a dynamic wildcard and reports as much to stdout and a file
+fn test_dynamic_wildcard_request_found() {
     let srv = MockServer::start();
-    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()])?;
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
+    let outfile = tmp_dir.path().join("outfile");
 
     let mock = Mock::new()
         .expect_method(GET)
@@ -151,7 +154,88 @@ fn test_dynamic_wildcard_request_found() -> Result<(), Box<dyn std::error::Error
         .arg(srv.url("/"))
         .arg("--wordlist")
         .arg(file.as_os_str())
-        .arg("--addslash")
+        .arg("--add-slash")
+        .arg("--output")
+        .arg(outfile.as_os_str())
+        .unwrap();
+
+    let contents = std::fs::read_to_string(outfile).unwrap();
+
+    teardown_tmp_directory(tmp_dir);
+
+    assert_eq!(contents.contains("WLD"), true);
+    assert_eq!(contents.contains("Got"), true);
+    assert_eq!(contents.contains("200"), true);
+    assert_eq!(contents.contains("(url length: 32)"), true);
+    assert_eq!(contents.contains("(url length: 96)"), true);
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("WLD")
+            .and(predicate::str::contains("Got"))
+            .and(predicate::str::contains("200"))
+            .and(predicate::str::contains("(url length: 32)"))
+            .and(predicate::str::contains("(url length: 96)")),
+    );
+
+    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock2.times_called(), 1);
+}
+
+#[test]
+/// uses dont_filter, so the normal wildcard test should never happen
+fn heuristics_static_wildcard_request_with_dont_filter() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap())
+        .return_status(200)
+        .return_body("this is a test")
+        .create_on(&srv);
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--dont-filter")
+        .unwrap();
+
+    teardown_tmp_directory(tmp_dir);
+
+    assert_eq!(mock.times_called(), 0);
+    Ok(())
+}
+
+#[test]
+/// test finds a static wildcard and reports as much to stdout
+fn heuristics_wildcard_test_with_two_static_wildcards() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap())
+        .return_status(200)
+        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
+        .create_on(&srv);
+
+    let mock2 = Mock::new()
+        .expect_method(GET)
+        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap())
+        .return_status(200)
+        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
+        .create_on(&srv);
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--add-slash")
         .unwrap();
 
     teardown_tmp_directory(tmp_dir);
@@ -162,13 +246,167 @@ fn test_dynamic_wildcard_request_found() -> Result<(), Box<dyn std::error::Error
             .and(predicate::str::contains("200"))
             .and(predicate::str::contains("(url length: 32)"))
             .and(predicate::str::contains("(url length: 96)"))
-            .and(predicate::str::contains("Wildcard response is dynamic;"))
-            .and(predicate::str::contains("auto-filtering"))
             .and(predicate::str::contains(
-                "(14 + url length) responses; toggle this behavior by using",
+                "Wildcard response is static; auto-filtering 46",
             )),
     );
 
+    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock2.times_called(), 1);
+}
+
+#[test]
+/// test finds a static wildcard and reports nothing to stdout
+fn heuristics_wildcard_test_with_two_static_wildcards_with_quiet_enabled(
+) -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap())
+        .return_status(200)
+        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
+        .create_on(&srv);
+
+    let mock2 = Mock::new()
+        .expect_method(GET)
+        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap())
+        .return_status(200)
+        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
+        .create_on(&srv);
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--add-slash")
+        .arg("-q")
+        .unwrap();
+
+    teardown_tmp_directory(tmp_dir);
+
+    cmd.assert().success().stdout(predicate::str::is_empty());
+
+    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock2.times_called(), 1);
+    Ok(())
+}
+
+#[test]
+/// test finds a static wildcard and reports as much to stdout and a file
+fn heuristics_wildcard_test_with_two_static_wildcards_and_output_to_file() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
+    let outfile = tmp_dir.path().join("outfile");
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap())
+        .return_status(200)
+        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
+        .create_on(&srv);
+
+    let mock2 = Mock::new()
+        .expect_method(GET)
+        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap())
+        .return_status(200)
+        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
+        .create_on(&srv);
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--add-slash")
+        .arg("--output")
+        .arg(outfile.as_os_str())
+        .unwrap();
+
+    let contents = std::fs::read_to_string(outfile).unwrap();
+
+    teardown_tmp_directory(tmp_dir);
+
+    assert_eq!(contents.contains("WLD"), true);
+    assert_eq!(contents.contains("Got"), true);
+    assert_eq!(contents.contains("200"), true);
+    assert_eq!(contents.contains("(url length: 32)"), true);
+    assert_eq!(contents.contains("(url length: 96)"), true);
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("WLD")
+            .and(predicate::str::contains("Got"))
+            .and(predicate::str::contains("200"))
+            .and(predicate::str::contains("(url length: 32)"))
+            .and(predicate::str::contains("(url length: 96)"))
+            .and(predicate::str::contains(
+                "Wildcard response is static; auto-filtering 46",
+            )),
+    );
+
+    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock2.times_called(), 1);
+}
+
+#[test]
+/// test finds a static wildcard that returns 3xx, expect redirects to => in response as well as
+/// in the output file
+fn heuristics_wildcard_test_with_redirect_as_response_code(
+) -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+    let outfile = tmp_dir.path().join("outfile");
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap())
+        .return_status(301)
+        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
+        .create_on(&srv);
+
+    let mock2 = Mock::new()
+        .expect_method(GET)
+        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap())
+        .return_status(301)
+        .return_header("Location", &srv.url("/some-redirect"))
+        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
+        .create_on(&srv);
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--add-slash")
+        .arg("--output")
+        .arg(outfile.as_os_str())
+        .unwrap();
+
+    let contents = std::fs::read_to_string(outfile).unwrap();
+
+    teardown_tmp_directory(tmp_dir);
+
+    assert_eq!(contents.contains("WLD"), true);
+    assert_eq!(contents.contains("301"), true);
+    assert_eq!(contents.contains("/some-redirect"), true);
+    assert_eq!(contents.contains("redirects to => "), true);
+    assert_eq!(contents.contains(&srv.url("/")), true);
+    assert_eq!(contents.contains("(url length: 32)"), true);
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("redirects to => ")
+            .and(predicate::str::contains("/some-redirect"))
+            .and(predicate::str::contains("301"))
+            .and(predicate::str::contains(srv.url("/")))
+            .and(predicate::str::contains("(url length: 32)"))
+            .and(predicate::str::contains("WLD")),
+    );
+
     assert_eq!(mock.times_called(), 1);
     assert_eq!(mock2.times_called(), 1);
     Ok(())

tests/test_main.rs

@@ -0,0 +1,92 @@
+mod utils;
+use assert_cmd::Command;
+use httpmock::Method::GET;
+use httpmock::{Mock, MockServer};
+use predicates::prelude::*;
+use utils::{setup_tmp_directory, teardown_tmp_directory};
+
+#[test]
+/// send the function a file to which we dont have permission in order to execute error branch
+fn main_use_root_owned_file_as_wordlist() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/")
+        .return_status(200)
+        .return_body("this is a test")
+        .create_on(&srv);
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg("/etc/shadow")
+        .arg("-vvvv")
+        .assert()
+        .failure()
+        .stdout(predicate::str::contains("Permission denied (os error 13)"));
+
+    // connectivity test hits it once
+    assert_eq!(mock.times_called(), 1);
+    Ok(())
+}
+
+#[test]
+/// send the function an empty file
+fn main_use_empty_wordlist() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&[], "wordlist")?;
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/")
+        .return_status(200)
+        .return_body("this is a test")
+        .create_on(&srv);
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-vvvv")
+        .assert()
+        .failure()
+        .stdout(predicate::str::contains("Did not find any words in"));
+
+    assert_eq!(mock.times_called(), 1);
+
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// send nothing over stdin, expect heuristics to be upset during connectivity test
+fn main_use_empty_stdin_targets() -> Result<(), Box<dyn std::error::Error>> {
+    let (tmp_dir, file) = setup_tmp_directory(&[], "wordlist")?;
+
+    // get_targets is called before scan, so the empty wordlist shouldn't trigger
+    // the 'Did not find any words' error
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--stdin")
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-vvv")
+        .pipe_stdin(file)
+        .unwrap()
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("Could not connect to any target provided")
+                .and(predicate::str::contains("Target Url"))
+                .not(), // no target url found
+        );
+
+    teardown_tmp_directory(tmp_dir);
+
+    Ok(())
+}

tests/test_scan_manager.rs

@@ -0,0 +1,93 @@
+mod utils;
+use assert_cmd::Command;
+use httpmock::Method::GET;
+use httpmock::{Mock, MockServer};
+use predicates::prelude::*;
+use utils::{setup_tmp_directory, teardown_tmp_directory};
+
+#[test]
+/// pass a known serialized scan with 1 scan complete and 1 not. expect the incomplete scan to
+/// start and the complete to not start. expect the responses, scans, and configuration structures
+/// to be populated based off the contents of the given state file
+fn resume_scan_works() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) =
+        setup_tmp_directory(&["css".to_string(), "stuff".to_string()], "wordlist").unwrap();
+
+    // localhost:PORT/ <- complete
+    // localhost:PORT/js <- will get scanned with /css and /stuff
+    let complete_scan = format!(
+        r#"{{"id":"057016a14769414aac9a7a62707598cb","url":"{}","scan_type":"Directory","complete":true}}"#,
+        srv.url("/")
+    );
+    let incomplete_scan = format!(
+        r#"{{"id":"400b2323a16f43468a04ffcbbeba34c6","url":"{}","scan_type":"Directory","complete":false}}"#,
+        srv.url("/js")
+    );
+    let scans = format!(r#""scans":[{},{}]"#, complete_scan, incomplete_scan);
+
+    let config = format!(
+        r#""config": {{"type":"configuration","wordlist":"{}","config":"","proxy":"","replay_proxy":"","target_url":"{}","status_codes":[200,204,301,302,307,308,401,403,405],"replay_codes":[200,204,301,302,307,308,401,403,405],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"quiet":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/1.9.0","redirects":false,"insecure":false,"extensions":[],"headers":{{}},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":false,"depth":2,"scan_limit":1,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false}}"#,
+        file.to_string_lossy(),
+        srv.url("/")
+    );
+
+    // // localhost:PORT/js/css has already been seen, expect not to be scanned
+    let response = format!(
+        r#"{{"type":"response","url":"{}","path":"/js/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{{"server":"nginx/1.16.1"}}}}"#,
+        srv.url("/js/css")
+    );
+    let responses = format!(r#""responses":[{}]"#, response);
+
+    // not scanned because /js is not complete, and /js/stuff response is not known
+    let not_scanned_yet = Mock::new()
+        .expect_method(GET)
+        .expect_path("/js/stuff")
+        .return_status(200)
+        .return_body("i expect to be scanned")
+        .create_on(&srv);
+
+    // will get scanned because /js is not complete, but because response of /js/css is known, the
+    // response will not be in stdout
+    let already_scanned = Mock::new()
+        .expect_method(GET)
+        .expect_path("/js/css")
+        .return_status(200)
+        .create_on(&srv);
+
+    // already scanned because scan on / is complete
+    let also_already_scanned = Mock::new()
+        .expect_method(GET)
+        .expect_path("/css")
+        .return_status(200)
+        .return_body("two words")
+        .create_on(&srv);
+
+    let state_file_contents = format!("{{{},{},{}}}", scans, config, responses);
+    let (tmp_dir2, state_file) = setup_tmp_directory(&[state_file_contents], "state-file").unwrap();
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--resume-from")
+        .arg(state_file.as_os_str())
+        .assert()
+        .success()
+        .stdout(
+            predicate::str::contains("/js/stuff")
+                .and(predicate::str::contains("22c"))
+                .and(predicate::str::contains("5w"))
+                .and(predicate::str::contains("/js/css"))
+                .not()
+                .and(predicate::str::contains("2w"))
+                .not()
+                .and(predicate::str::contains("9c"))
+                .not(),
+        );
+
+    teardown_tmp_directory(tmp_dir);
+    teardown_tmp_directory(tmp_dir2);
+
+    assert_eq!(already_scanned.times_called(), 1);
+    assert_eq!(also_already_scanned.times_called(), 0);
+    assert_eq!(not_scanned_yet.times_called(), 1);
+}

tests/test_scanner.rs

@@ -7,9 +7,10 @@ use std::process::Command;
 use utils::{setup_tmp_directory, teardown_tmp_directory};
 
 #[test]
-fn test_single_request_scan() -> Result<(), Box<dyn std::error::Error>> {
+/// send a single valid request, expect a 200 response
+fn scanner_single_request_scan() -> Result<(), Box<dyn std::error::Error>> {
     let srv = MockServer::start();
-    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()])?;
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
     let mock = Mock::new()
         .expect_method(GET)
@@ -24,15 +25,565 @@ fn test_single_request_scan() -> Result<(), Box<dyn std::error::Error>> {
         .arg(srv.url("/"))
         .arg("--wordlist")
         .arg(file.as_os_str())
+        .arg("-vvvv")
         .unwrap();
 
     cmd.assert().success().stdout(
         predicate::str::contains("/LICENSE")
-            .and(predicate::str::contains("200 OK"))
-            .and(predicate::str::contains("[14 bytes]")),
+            .and(predicate::str::contains("200"))
+            .and(predicate::str::contains("14")),
     );
 
     assert_eq!(mock.times_called(), 1);
     teardown_tmp_directory(tmp_dir);
     Ok(())
 }
+
+#[test]
+/// send a valid request, follow redirects into new directories, expect 301/200 responses
+fn scanner_recursive_request_scan() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let urls = [
+        "js".to_string(),
+        "prod".to_string(),
+        "dev".to_string(),
+        "file.js".to_string(),
+    ];
+    let (tmp_dir, file) = setup_tmp_directory(&urls, "wordlist")?;
+
+    let js_mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/js")
+        .return_status(301)
+        .return_header("Location", &srv.url("/js/"))
+        .create_on(&srv);
+
+    let js_prod_mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/js/prod")
+        .return_status(301)
+        .return_header("Location", &srv.url("/js/prod/"))
+        .create_on(&srv);
+
+    let js_dev_mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/js/dev")
+        .return_status(301)
+        .return_header("Location", &srv.url("/js/dev/"))
+        .create_on(&srv);
+
+    let js_dev_file_mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/js/dev/file.js")
+        .return_status(200)
+        .return_body("this is a test and is more bytes than other ones")
+        .create_on(&srv);
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-vvvv")
+        .arg("-t")
+        .arg("1")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::is_match("301.*js")
+            .unwrap()
+            .and(predicate::str::is_match("301.*js/prod").unwrap())
+            .and(predicate::str::is_match("301.*js/dev").unwrap())
+            .and(predicate::str::is_match("200.*js/dev/file.js").unwrap()),
+    );
+
+    assert_eq!(js_mock.times_called(), 1);
+    assert_eq!(js_prod_mock.times_called(), 1);
+    assert_eq!(js_dev_mock.times_called(), 1);
+    assert_eq!(js_dev_file_mock.times_called(), 1);
+
+    teardown_tmp_directory(tmp_dir);
+
+    Ok(())
+}
+
+#[test]
+/// send a valid request, follow 200s into new directories, expect 200 responses
+fn scanner_recursive_request_scan_using_only_success_responses(
+) -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let urls = [
+        "js/".to_string(),
+        "prod/".to_string(),
+        "dev/".to_string(),
+        "file.js".to_string(),
+    ];
+    let (tmp_dir, file) = setup_tmp_directory(&urls, "wordlist")?;
+
+    let js_mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/js/")
+        .return_status(200)
+        .return_header("Location", &srv.url("/js/"))
+        .create_on(&srv);
+
+    let js_prod_mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/js/prod/")
+        .return_status(200)
+        .return_header("Location", &srv.url("/js/prod/"))
+        .create_on(&srv);
+
+    let js_dev_mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/js/dev/")
+        .return_status(200)
+        .return_header("Location", &srv.url("/js/dev/"))
+        .create_on(&srv);
+
+    let js_dev_file_mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/js/dev/file.js")
+        .return_status(200)
+        .return_body("this is a test and is more bytes than other ones")
+        .create_on(&srv);
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-vvvv")
+        .arg("-t")
+        .arg("1")
+        .arg("--redirects")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::is_match("200.*js")
+            .unwrap()
+            .and(predicate::str::is_match("200.*js/prod").unwrap())
+            .and(predicate::str::is_match("200.*js/dev").unwrap())
+            .and(predicate::str::is_match("200.*js/dev/file.js").unwrap()),
+    );
+
+    assert_eq!(js_mock.times_called(), 1);
+    assert_eq!(js_prod_mock.times_called(), 1);
+    assert_eq!(js_dev_mock.times_called(), 1);
+    assert_eq!(js_dev_file_mock.times_called(), 1);
+
+    teardown_tmp_directory(tmp_dir);
+
+    Ok(())
+}
+
+#[test]
+/// send a single valid request, get a response, and write it to disk
+fn scanner_single_request_scan_with_file_output() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/LICENSE")
+        .return_status(200)
+        .return_body("this is a test")
+        .create_on(&srv);
+
+    let outfile = tmp_dir.path().join("output");
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-vvvv")
+        .arg("-o")
+        .arg(outfile.as_os_str())
+        .unwrap();
+
+    let contents = std::fs::read_to_string(outfile)?;
+
+    assert!(contents.contains("/LICENSE"));
+    assert!(contents.contains("200"));
+    assert!(contents.contains("14"));
+
+    assert_eq!(mock.times_called(), 1);
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// send a single valid request with -q, get a response, and write only the url to disk
+fn scanner_single_request_scan_with_file_output_and_tack_q(
+) -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/LICENSE")
+        .return_status(200)
+        .return_body("this is a test")
+        .create_on(&srv);
+
+    let outfile = tmp_dir.path().join("output");
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-vvvv")
+        .arg("-q")
+        .arg("-o")
+        .arg(outfile.as_os_str())
+        .unwrap();
+
+    let contents = std::fs::read_to_string(outfile)?;
+
+    let url = srv.url("/LICENSE");
+    assert!(contents.contains(&url));
+
+    assert_eq!(mock.times_called(), 1);
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// send an invalid output file, expect nothing to be written to disk
+fn scanner_single_request_scan_with_invalid_file_output() -> Result<(), Box<dyn std::error::Error>>
+{
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/LICENSE")
+        .return_status(200)
+        .return_body("this is a test")
+        .create_on(&srv);
+
+    let outfile = tmp_dir.path(); // outfile is a directory
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-vvvv")
+        .arg("-q")
+        .arg("-o")
+        .arg(outfile.as_os_str())
+        .unwrap();
+
+    let contents = std::fs::read_to_string(outfile);
+    assert!(contents.is_err());
+
+    assert_eq!(mock.times_called(), 1);
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// send a single valid request using -q, expect only the url on stdout
+fn scanner_single_request_quiet_scan() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/LICENSE")
+        .return_status(200)
+        .return_body("this is a test")
+        .create_on(&srv);
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-x")
+        .arg("js,html")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains(srv.url("/LICENSE"))
+            .and(predicate::str::contains("200"))
+            .not()
+            .and(predicate::str::contains("14"))
+            .not(),
+    );
+
+    assert_eq!(mock.times_called(), 1);
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// send single valid request, get back a 301 without a Location header
+/// expect response_is_directory to return false when called
+fn scanner_single_request_returns_301_without_location_header(
+) -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/LICENSE")
+        .return_body("this is a test")
+        .return_status(301)
+        .create_on(&srv);
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--timeout")
+        .arg("5")
+        .arg("--user-agent")
+        .arg("some-user-agent-string")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains(srv.url("/LICENSE"))
+            .and(predicate::str::contains("301"))
+            .and(predicate::str::contains("14")),
+    );
+
+    assert_eq!(mock.times_called(), 1);
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// send a single valid request, filter the size of the response, expect one out of 2 urls
+fn scanner_single_request_scan_with_filtered_result() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) =
+        setup_tmp_directory(&["LICENSE".to_string(), "ignored".to_string()], "wordlist")?;
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/LICENSE")
+        .return_status(200)
+        .return_body("this is a not a test")
+        .create_on(&srv);
+
+    let filtered_mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/ignored")
+        .return_status(200)
+        .return_body("this is a test")
+        .create_on(&srv);
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-n")
+        .arg("-S")
+        .arg("14")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .and(predicate::str::contains("200"))
+            .and(predicate::str::contains("20"))
+            .and(predicate::str::contains("ignored"))
+            .not()
+            .and(predicate::str::contains(" 14 "))
+            .not(),
+    );
+
+    assert_eq!(mock.times_called(), 1);
+    assert_eq!(filtered_mock.times_called(), 1);
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// send a single valid request, expect a 200 response that then gets routed to the replay
+/// proxy
+fn scanner_single_request_replayed_to_proxy() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let proxy = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/LICENSE")
+        .return_status(200)
+        .return_body("this is a test")
+        .create_on(&srv);
+
+    let mock_two = Mock::new()
+        .expect_method(GET)
+        .expect_path("/LICENSE")
+        .return_status(200)
+        .return_body("this is a test")
+        .create_on(&proxy);
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--replay-proxy")
+        .arg(format!("http://{}", proxy.address().to_string()))
+        .arg("--replay-codes")
+        .arg("200")
+        .unwrap();
+
+    cmd.assert()
+        .success()
+        .stdout(
+            predicate::str::contains("/LICENSE")
+                .and(predicate::str::contains("200"))
+                .and(predicate::str::contains("14")),
+        )
+        .stderr(predicate::str::contains("Replay Proxy Codes"));
+
+    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock_two.times_called(), 1);
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// send a single valid request, get a response, and write the logging messages to disk
+fn scanner_single_request_scan_with_debug_logging() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/LICENSE")
+        .return_status(200)
+        .return_body("this is a test")
+        .create_on(&srv);
+
+    let outfile = tmp_dir.path().join("debug.log");
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-vvvv")
+        .arg("--debug-log")
+        .arg(outfile.as_os_str())
+        .unwrap();
+
+    let contents = std::fs::read_to_string(outfile).unwrap();
+    println!("{}", contents);
+    assert!(contents.starts_with("Configuration {"));
+    assert!(contents.contains("TRC"));
+    assert!(contents.contains("DBG"));
+    assert!(contents.contains("INF"));
+    assert!(contents.contains("feroxbuster All scans complete!"));
+    assert!(contents.contains("feroxbuster exit: terminal_input_handler"));
+
+    assert_eq!(mock.times_called(), 1);
+    teardown_tmp_directory(tmp_dir);
+}
+
+#[test]
+/// send a single valid request, get a response, and write the logging messages to disk as NDJSON
+fn scanner_single_request_scan_with_debug_logging_as_json() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/LICENSE")
+        .return_status(200)
+        .return_body("this is a test")
+        .create_on(&srv);
+
+    let outfile = tmp_dir.path().join("debug.log");
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-vvvv")
+        .arg("--debug-log")
+        .arg(outfile.as_os_str())
+        .arg("--json")
+        .unwrap();
+
+    let contents = std::fs::read_to_string(outfile).unwrap();
+    println!("{}", contents);
+    assert!(contents.starts_with("{\"type\":\"configuration\""));
+    assert!(contents.contains("\"level\":\"TRACE\""));
+    assert!(contents.contains("\"level\":\"DEBUG\""));
+    assert!(contents.contains("\"level\":\"INFO\""));
+    assert!(contents.contains("time_offset"));
+    assert!(contents.contains("\"module\":\"feroxbuster::scanner\""));
+    assert!(contents.contains("All scans complete!"));
+    assert!(contents.contains("exit: terminal_input_handler"));
+
+    assert_eq!(mock.times_called(), 1);
+    teardown_tmp_directory(tmp_dir);
+}
+
+#[test]
+/// send a single valid request, filter the response by regex, expect one out of 2 urls
+fn scanner_single_request_scan_with_regex_filtered_result() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) =
+        setup_tmp_directory(&["LICENSE".to_string(), "ignored".to_string()], "wordlist").unwrap();
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/LICENSE")
+        .return_status(200)
+        .return_body("this is a not a test")
+        .create_on(&srv);
+
+    let filtered_mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/ignored")
+        .return_status(200)
+        .return_body("this is a test\nThat rug really tied the room together")
+        .create_on(&srv);
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--filter-regex")
+        .arg("'That rug.*together$'")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .and(predicate::str::contains("200"))
+            .and(predicate::str::contains("20"))
+            .and(predicate::str::contains("ignored"))
+            .not()
+            .and(predicate::str::contains(" 14 "))
+            .not(),
+    );
+
+    assert_eq!(mock.times_called(), 1);
+    assert_eq!(filtered_mock.times_called(), 1);
+    teardown_tmp_directory(tmp_dir);
+}

tests/utils/mod.rs

@@ -2,15 +2,20 @@ use std::fs::{remove_dir_all, write};
 use std::path::PathBuf;
 use tempfile::TempDir;
 
+/// integration test helper: creates a temp directory, and writes `words` to
+/// a file named `filename` in the temp directory
 pub fn setup_tmp_directory(
     words: &[String],
+    filename: &str,
 ) -> Result<(TempDir, PathBuf), Box<dyn std::error::Error>> {
     let tmp_dir = TempDir::new()?;
-    let file = tmp_dir.path().join("wordlist");
+    let file = tmp_dir.path().join(&filename);
     write(&file, words.join("\n"))?;
     Ok((tmp_dir, file))
 }
 
+/// integration test helper: removes a temporary directory, presumably created with
+/// [setup_tmp_directory](fn.setup_tmp_directory.html)
 pub fn teardown_tmp_directory(directory: TempDir) {
     remove_dir_all(directory).unwrap();
 }