docs: add aldamd as a contributor for code (#1309)

* docs: update README.md [skip ci]

* docs: update .all-contributorsrc [skip ci]

---------

Co-authored-by: allcontributors[bot] <46447321+allcontributors[bot]@users.noreply.github.com>

.cargo/config.toml

@@ -0,0 +1,5 @@
+[target.armv7-unknown-linux-gnueabihf]
+linker = "arm-linux-gnueabihf-gcc"
+
+[target.aarch64-unknown-linux-gnu]
+linker = "aarch64-linux-gnu-gcc"

.github/FUNDING.yml

@@ -0,0 +1,4 @@
+# These are supported funding model platforms
+
+github: [epi052]
+ko_fi: epi052

.github/dependabot.yml

@@ -0,0 +1,7 @@
+version: 2
+updates:
+- package-ecosystem: cargo
+  directory: "/"
+  schedule:
+    interval: daily
+  open-pull-requests-limit: 10

.github/pull_request_template.md

@@ -4,19 +4,22 @@ Long form explanations of most of the items below can be found in the [CONTRIBUT
 
 ## Branching checklist
 - [ ] There is an issue associated with your PR (bug, feature, etc.. if not, create one)
-- [ ] Your PR description references the associated issue (i.e. fixes #123)
+- [ ] Your PR description references the associated issue (i.e. fixes #123456)
 - [ ] Code is in its own branch
 - [ ] Branch name is related to the PR contents
-- [ ] PR targets master
+- [ ] PR targets main
 
 ## Static analysis checks
 - [ ] All rust files are formatted using `cargo fmt`
-- [ ] All `clippy` checks pass when running `cargo clippy --all-targets --all-features -- -D warnings -A clippy::unnecessary_unwrap`
+- [ ] All `clippy` checks pass when running `cargo clippy --all-targets --all-features -- -D warnings -A clippy::mutex-atomic`
 - [ ] All existing tests pass
 
 ## Documentation
 - [ ] New code is documented using [doc comments](https://doc.rust-lang.org/stable/rust-by-example/meta/doc.html)
-- [ ] Documentation about your PR is included in the README, as needed
+- [ ] Documentation about your PR is included in the `docs`, as needed. The docs live in a [separate repository](https://epi052.github.io/feroxbuster-docs/). Update the appropriate pages at the links below.
+  - [ ] update [example config file section](https://epi052.github.io/feroxbuster-docs/configuration/ferox-config-toml/)
+  - [ ] update [help output section](https://epi052.github.io/feroxbuster-docs/configuration/command-line/)
+  - [ ] add an [example](https://epi052.github.io/feroxbuster-docs/examples/auto-tune/)
 
 ## Additional Tests
 - [ ] New code is unit tested

.github/stale.yml

@@ -0,0 +1,18 @@
+# Number of days of inactivity before an issue becomes stale
+daysUntilStale: 14
+# Number of days of inactivity before a stale issue is closed
+daysUntilClose: 7
+# Issues with these labels will never be considered stale
+exemptLabels:
+  - pinned
+  - security
+  - confirmed
+# Label to use when marking an issue as stale
+staleLabel: stale
+# Comment to post when marking an issue as stale. Set to `false` to disable
+markComment: >
+  This issue has been automatically marked as stale because it has not had
+  recent activity. It will be closed if no further activity occurs. Thank you
+  for your contributions.
+# Comment to post when closing a stale issue. Set to `false` to disable
+closeComment: false

.github/workflows/build.yml

@@ -4,11 +4,13 @@ on: [push]
 
 jobs:
   build-nix:
+    env:
+      IN_PIPELINE: true
     runs-on: ${{ matrix.os }}
-    if: github.ref == 'refs/heads/master'
+    if: github.ref == 'refs/heads/main'
     strategy:
       matrix:
-        type: [ubuntu-x64, ubuntu-x86]
+        type: [ubuntu-x64, ubuntu-x86, armv7, aarch64]
         include:
           - type: ubuntu-x64
             os: ubuntu-latest
@@ -22,82 +24,178 @@ jobs:
             name: x86-linux-feroxbuster
             path: target/i686-unknown-linux-musl/release/feroxbuster
             pkg_config_path: /usr/lib/i686-linux-gnu/pkgconfig
+          - type: armv7
+            os: ubuntu-latest
+            target: armv7-unknown-linux-gnueabihf
+            name: armv7-linux-feroxbuster
+            path: target/armv7-unknown-linux-gnueabihf/release/feroxbuster
+            pkg_config_path: /usr/lib/x86_64-linux-gnu/pkgconfig
+          - type: aarch64
+            os: ubuntu-latest
+            target: aarch64-unknown-linux-gnu
+            name: aarch64-linux-feroxbuster
+            path: target/aarch64-unknown-linux-gnu/release/feroxbuster
+            pkg_config_path: /usr/lib/x86_64-linux-gnu/pkgconfig
     steps:
-      - uses: actions/checkout@v2
-      - name: Install System Dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y --no-install-recommends libssl-dev pkg-config
-      - uses: actions-rs/toolchain@v1
+      - uses: actions/checkout@v4
+      - name: Cache cargo & target directories
+        uses: Swatinem/rust-cache@v2
+      - name: Build binary
+        uses: houseabsolute/actions-rust-cross@v0
         with:
-          toolchain: stable
-          target: ${{ matrix.target }}
-          override: true
-      - uses: actions-rs/cargo@v1
-        env:
-          PKG_CONFIG_PATH: ${{ matrix.pkg_config_path }}
-          OPENSSL_DIR: /usr/lib/ssl
-        with:
-          use-cross: true
           command: build
-          args: --release --target=${{ matrix.target }}
+          target: ${{ matrix.target }}
+          args: "--locked --release"
+          strip: true
+          toolchain: stable
       - name: Build tar.gz for homebrew installs
         if: matrix.type == 'ubuntu-x64'
         run: |
           tar czf ${{ matrix.name }}.tar.gz -C target/x86_64-unknown-linux-musl/release feroxbuster
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.name }}
           path: ${{ matrix.path }}
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v4
         if: matrix.type == 'ubuntu-x64'
         with:
           name: ${{ matrix.name }}.tar.gz
           path: ${{ matrix.name }}.tar.gz
 
+  build-debug:
+    env:
+      IN_PIPELINE: true
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install System Dependencies
+        run: |
+          env
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends libssl-dev pkg-config musl-tools
+      - name: Set up Rust toolchain
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: stable
+          target: x86_64-unknown-linux-musl
+
+      - name: Build the project
+        env:
+          PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig
+          OPENSSL_DIR: /usr/lib/ssl
+        run: cargo build --target=x86_64-unknown-linux-musl
+      - uses: actions/upload-artifact@v4
+        with:
+          name: x86_64-linux-debug-feroxbuster
+          path: target/x86_64-unknown-linux-musl/debug/feroxbuster
+
+  build-debug-windows:
+    env:
+      IN_PIPELINE: true
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Rust toolchain
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: stable
+          target: x86_64-pc-windows-msvc
+
+      - name: Build the project
+        run: cargo build --target=x86_64-pc-windows-msvc
+      - uses: actions/upload-artifact@v4
+        with:
+          name: x86_64-windows-debug-feroxbuster.exe
+          path: target\x86_64-pc-windows-msvc\debug\feroxbuster.exe
+
   build-deb:
     needs: [build-nix]
     runs-on: ubuntu-latest
+    env:
+      IN_PIPELINE: true
     steps:
-      - uses: actions/checkout@master
+      - uses: actions/checkout@v4
+      - name: Install cargo-deb
+        run: cargo install -f cargo-deb
+      - uses: awalsh128/cache-apt-pkgs-action@v1
+        with:
+          packages: musl-tools # provides musl-gcc
+          version: 1.0
+      - name: Install musl toolchain
+        run: rustup target add x86_64-unknown-linux-musl
       - name: Deb Build
-        uses: ebbflow-io/cargo-deb-amd64-ubuntu@1.0
+        run: cargo deb --target=x86_64-unknown-linux-musl
       - name: Upload Deb Artifact
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: feroxbuster_amd64.deb
           path: ./target/x86_64-unknown-linux-musl/debian/*
 
   build-macos:
+    env:
+      IN_PIPELINE: true
     runs-on: macos-latest
-    if: github.ref == 'refs/heads/master'
+    if: github.ref == 'refs/heads/main'
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
+      - uses: actions/checkout@v4
+      - name: Cache cargo & target directories
+        uses: Swatinem/rust-cache@v2
+      - name: Build binary
+        uses: houseabsolute/actions-rust-cross@v0
         with:
-          toolchain: stable
-          target: x86_64-apple-darwin
-          override: true
-      - uses: actions-rs/cargo@v1
-        with:
-          use-cross: true
           command: build
-          args: --release --target=x86_64-apple-darwin
+          target: x86_64-apple-darwin
+          args: "--locked --release"
+          strip: true
+          toolchain: stable
       - name: Build tar.gz for homebrew installs
         run: |
           tar czf x86_64-macos-feroxbuster.tar.gz -C target/x86_64-apple-darwin/release feroxbuster
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v4
         with:
           name: x86_64-macos-feroxbuster
           path: target/x86_64-apple-darwin/release/feroxbuster
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v4
         with:
           name: x86_64-macos-feroxbuster.tar.gz
           path: x86_64-macos-feroxbuster.tar.gz
+  
+  build-macos-aarch64:
+    env:
+      IN_PIPELINE: true
+    runs-on: macos-latest
+    if: github.ref == 'refs/heads/main'
+    steps:
+      - uses: actions/checkout@v4
+      - name: Cache cargo & target directories
+        uses: Swatinem/rust-cache@v2
+      - name: Build binary
+        uses: houseabsolute/actions-rust-cross@v0
+        with:
+          command: build
+          target: aarch64-apple-darwin
+          args: "--locked --release"
+          strip: true
+          toolchain: stable
+      - name: Build tar.gz for homebrew installs
+        run: |
+          tar czf aarch64-macos-feroxbuster.tar.gz -C target/aarch64-apple-darwin/release feroxbuster
+      - uses: actions/upload-artifact@v4
+        with:
+          name: aarch64-macos-feroxbuster
+          path: target/aarch64-apple-darwin/release/feroxbuster
+      - uses: actions/upload-artifact@v4
+        with:
+          name: aarch64-macos-feroxbuster.tar.gz
+          path: aarch64-macos-feroxbuster.tar.gz
 
   build-windows:
+    env:
+      IN_PIPELINE: true
     runs-on: ${{ matrix.os }}
-    if: github.ref == 'refs/heads/master'
+    if: github.ref == 'refs/heads/main'
     strategy:
       matrix:
         type: [windows-x64, windows-x86]
@@ -113,19 +211,18 @@ jobs:
             name: x86-windows-feroxbuster.exe
             path: target\i686-pc-windows-msvc\release\feroxbuster.exe
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
+      - uses: actions/checkout@v4
+      - name: Cache cargo & target directories
+        uses: Swatinem/rust-cache@v2
+      - name: Build binary
+        uses: houseabsolute/actions-rust-cross@v0
         with:
-          toolchain: stable
-          target: ${{ matrix.target }}
-          override: true
-      - uses: actions-rs/cargo@v1
-        with:
-          use-cross: true
           command: build
-          args: --release --target=${{ matrix.target }}
-      - uses: actions/upload-artifact@v2
+          target:  ${{ matrix.target }}
+          args: "--locked --release"
+          strip: true
+          toolchain: stable
+      - uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.name }}
           path: ${{ matrix.path }}
-

.github/workflows/check.yml

@@ -1,64 +1,51 @@
 name: CI Pipeline
 
-on: [push]
+on: [push, pull_request]
 
 jobs:
   check:
     name: Check
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-      - uses: actions-rs/cargo@v1
-        with:
-          command: check
+      - uses: actions/checkout@v4
+      - name: Cache cargo & target directories
+        uses: Swatinem/rust-cache@v2
+      - uses: dtolnay/rust-toolchain@stable
+      - run: cargo check
 
   test:
     name: Test Suite
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-      - uses: actions-rs/cargo@v1
-        with:
-          command: test
+      - uses: actions/checkout@v4
+      - name: Cache cargo & target directories
+        uses: Swatinem/rust-cache@v2
+      - name: Install latest nextest release
+        uses: taiki-e/install-action@nextest
+      - uses: dtolnay/rust-toolchain@stable
+      - name: Test with latest nextest release
+        run: cargo nextest run --all-features --all-targets --retries 4 --no-fail-fast
 
   fmt:
     name: Rust fmt
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
+      - uses: actions/checkout@v4
+      - name: Cache cargo & target directories
+        uses: Swatinem/rust-cache@v2
+      - uses: dtolnay/rust-toolchain@stable
         with:
-          profile: minimal
-          toolchain: stable
-          override: true
-      - run: rustup component add rustfmt
-      - uses: actions-rs/cargo@v1
-        with:
-          command: fmt
-          args: --all -- --check
+          components: rustfmt
+      - run: cargo fmt --all -- --check
 
   clippy:
     name: Clippy
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
+      - uses: actions/checkout@v4
+      - name: Cache cargo & target directories
+        uses: Swatinem/rust-cache@v2
+      - uses: dtolnay/rust-toolchain@stable
         with:
-          profile: minimal
-          toolchain: stable
-          override: true
-      - run: rustup component add clippy
-      - uses: actions-rs/cargo@v1
-        with:
-          command: clippy
-          args: --all-targets --all-features -- -D warnings -A clippy::unnecessary_unwrap
+          components: clippy
+      - run: cargo clippy --all-targets --all-features -- -D warnings

.github/workflows/cicd-to-dockerhub.yml

@@ -0,0 +1,60 @@
+name: ci-to-dockerhub
+
+on:
+  push:
+    branches: [ main ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and push
+        id: docker_build
+        uses: docker/build-push-action@v6
+        with:
+          context: ./
+          file: ./Dockerfile
+          push: true
+          tags: ${{ secrets.DOCKER_HUB_USERNAME }}/feroxbuster:latest
+
+      - name: Image digest
+        run: echo ${{ steps.docker_build.outputs.digest }}
+
+      - name: Verify pushed image
+        run: |
+          # Wait a moment for the image to be available
+          sleep 5
+          
+          # Pull the image we just pushed
+          docker pull ${{ secrets.DOCKER_HUB_USERNAME }}/feroxbuster:latest
+          
+          # Get the digest of the pulled image
+          PULLED_DIGEST=$(docker inspect --format='{{index .RepoDigests 0}}' ${{ secrets.DOCKER_HUB_USERNAME }}/feroxbuster:latest | cut -d'@' -f2)
+          PUSHED_DIGEST="${{ steps.docker_build.outputs.digest }}"
+          
+          echo "Pushed digest: $PUSHED_DIGEST"
+          echo "Pulled digest: $PULLED_DIGEST"
+          
+          # Verify they match
+          if [ "$PULLED_DIGEST" = "$PUSHED_DIGEST" ]; then
+            echo "✓ Verification successful: Pulled image matches pushed image"
+            
+            # Test that the binary works
+            docker run --rm ${{ secrets.DOCKER_HUB_USERNAME }}/feroxbuster:latest --version
+          else
+            echo "✗ Verification failed: Digests do not match"
+            exit 1
+          fi

.github/workflows/coverage.yml

@@ -3,34 +3,23 @@ on: [push]
 name: Code Coverage Pipeline
 
 jobs:
-  upload-coverage:
+  coverage:
+    name: LLVM Coverage
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v1
-      - uses: actions-rs/toolchain@v1
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
         with:
-          toolchain: nightly
-          override: true
-      - uses: actions-rs/cargo@v1
+          components: llvm-tools-preview
+      - name: Install cargo-llvm-cov and cargo-nextest
+        uses: taiki-e/install-action@v2
         with:
-          command: clean
-      - uses: actions-rs/cargo@v1
+          tool: cargo-nextest,cargo-llvm-cov
+      - name: Generate code coverage
+        run: cargo llvm-cov nextest --all-features --no-fail-fast --lcov --retries 4 --output-path lcov.info
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v4
         with:
-          command: test
-          args: --all-features --no-fail-fast
-        env:
-          CARGO_INCREMENTAL: '0'
-          RUSTFLAGS: '-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort'
-          RUSTDOCFLAGS: '-Cpanic=abort'
-      - uses: actions-rs/grcov@v0.1
-      - name: Convert lcov to xml
-        run: |
-          curl -O https://raw.githubusercontent.com/eriwen/lcov-to-cobertura-xml/master/lcov_cobertura/lcov_cobertura.py
-          chmod +x lcov_cobertura.py
-          ./lcov_cobertura.py ./lcov.info
-      - uses: codecov/codecov-action@v1
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          file: ./coverage.xml
-          name: codecov-umbrella
+          files: lcov.info
           fail_ci_if_error: true
+          token: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/winget.yml

@@ -0,0 +1,21 @@
+name: Publish to Winget
+on:
+  release:
+    types: [released]
+  workflow_dispatch:
+    inputs:
+      tag_name:
+        description: 'Tag name of release'
+        required: true
+        type: string
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: vedantmgoyal2009/winget-releaser@main
+        with:
+          identifier: epi052.feroxbuster
+          installers-regex: '-windows-feroxbuster\.exe\.zip$'
+          token: ${{ secrets.WINGET_TOKEN }}
+          release-tag: ${{ inputs.tag_name || github.event.release.tag_name || github.ref_name }}

.gitignore

@@ -3,16 +3,15 @@
 debug/
 target/
 
-# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
-# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
-Cargo.lock
-
 # These are backup files generated by rustfmt
 **/*.rs.bk
 
 # jetbrains metadata folder
 .idea/
 
+# vscode metadata folder
+.vscode/
+
 # personal feroxbuster config for testing
 ferox-config.toml
 
@@ -22,3 +21,15 @@ img/**
 # scripts to check code coverage using nightly compiler
 check-coverage.sh
 lcov_cobertura.py
+
+# dockerignore file that makes it so i can work on the docker config without copying a 4GB manifest or w/e it is
+.dockerignore
+
+# state file created during tests
+ferox-*.state
+
+# python stuff cuz reasons
+Pipfile*
+
+# ignore choco_package generated nupkg
+/choco_package/*.nupkg

.rustfmt.toml

@@ -0,0 +1 @@
+reorder_modules = false

CONTRIBUTING.md

@@ -76,35 +76,35 @@ Now that you have a copy of your fork, there is work you will need to do to keep
 
 Do this prior to every time you create a branch for a PR:
 
-1. Make sure you are on the `master` branch
+1. Make sure you are on the `main` branch
 
   > ```sh
   > $ git status
-  > On branch master
-  > Your branch is up-to-date with 'origin/master'.
+  > On branch main
+  > Your branch is up-to-date with 'origin/main'.
   > ```
 
-  > If your aren't on `master`, resolve outstanding files and commits and checkout the `master` branch
+  > If your aren't on `main`, resolve outstanding files and commits and checkout the `main` branch
 
   > ```sh
-  > $ git checkout master
+  > $ git checkout main
   > ```
 
 2. Do a pull with rebase against `upstream`
 
   > ```sh
-  > $ git pull --rebase upstream master
+  > $ git pull --rebase upstream main
   > ```
 
-  > This will pull down all of the changes to the official master branch, without making an additional commit in your local repo.
+  > This will pull down all of the changes to the official main branch, without making an additional commit in your local repo.
 
-3. (_Optional_) Force push your updated master branch to your GitHub fork
+3. (_Optional_) Force push your updated main branch to your GitHub fork
 
   > ```sh
-  > $ git push origin master --force
+  > $ git push origin main --force
   > ```
 
-  > This will overwrite the master branch of your fork.
+  > This will overwrite the main branch of your fork.
 
 ### Creating a branch
 
@@ -166,7 +166,7 @@ primarily related to continuous integration and release deployment.
 
 feroxbuster uses the [`clippy`](https://rust-lang.github.io/rust-clippy/) code linter.
 
-The command that will ultimately be used in the CI pipeline for linting is `cargo clippy --all-targets --all-features -- -D warnings -A clippy::unnecessary_unwrap`.  
+The command that will ultimately be used in the CI pipeline for linting is `cargo clippy --all-targets --all-features -- -D warnings -A clippy::mutex-atomic`.
 
 Before submitting a Pull Request, the above command should be run. Please do not ignore any linting errors in code you write or modify, as they are meant to **help** by ensuring a clean and simple code base.
 
@@ -182,14 +182,17 @@ Test coverage can be checked using [grcov](https://github.com/mozilla/grcov).  I
 
 ```sh
 cargo install grcov
+rustup component add llvm-tools
 rustup install nightly
 rustup default nightly
 export CARGO_INCREMENTAL=0
-export RUSTFLAGS="-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort"
+export RUSTFLAGS="-Cinstrument-coverage -Clink-dead-code -Ccodegen-units=1 -Coverflow-checks=off"
+export LLVM_PROFILE_FILE="target/debug/coverage/profraw/feroxbuster-%p-%m.profraw"
 export RUSTDOCFLAGS="-Cpanic=abort"
+rm -r target/debug/coverage/profraw
 cargo build
 cargo test
-grcov ./target/debug/ -s . -t html --llvm --branch --ignore-not-existing -o ./target/debug/coverage/
+grcov . --source-dir . --keep-only "src/*" --binary-path ./target/debug/ -t html --branch --ignore-not-existing -o ./target/debug/coverage/
 firefox target/debug/coverage/index.html
 ```
 
@@ -214,20 +217,20 @@ GitHub has a good guide on how to contribute to open source [here](https://opens
 
 ##### Editing via your local fork
 
-1.  Perform the maintenance step of rebasing `master`
-2.  Ensure you're on the `master` branch using `git status`:
+1.  Perform the maintenance step of rebasing `main`
+2.  Ensure you're on the `main` branch using `git status`:
 
 ```sh
 $ git status
-On branch master
-Your branch is up-to-date with 'origin/master'.
+On branch main
+Your branch is up-to-date with 'origin/main'.
 
 nothing to commit, working directory clean
 ```
 
-1.  If you're not on master or your working directory is not clean, resolve
-    any outstanding files/commits and checkout master `git checkout master`
-2.  Create a branch off of `master` with git: `git checkout -B
+1.  If you're not on main or your working directory is not clean, resolve
+    any outstanding files/commits and checkout main `git checkout main`
+2.  Create a branch off of `main` with git: `git checkout -B
     branch/name-here`
 3.  Edit your file(s) locally with the editor of your choice
 4.  Check your `git status` to see unstaged files
@@ -239,8 +242,8 @@ nothing to commit, working directory clean
 8.  Push your commits to your GitHub Fork: `git push -u origin branch/name-here`
 9.  Once the edits have been committed, you will be prompted to create a pull
     request on your fork's GitHub page
-10.  By default, all pull requests should be against the `master` branch
-11.  Submit a pull request from your branch to feroxbuster's `master` branch
+10.  By default, all pull requests should be against the `main` branch
+11.  Submit a pull request from your branch to feroxbuster's `main` branch
 12.  The title (also called the subject) of your PR should be descriptive of your
     changes and succinctly indicate what is being fixed
     -   Examples: `Add test cases for Unicode support`; `Correct typo in overview documentation`

Cargo.toml

@@ -1,43 +1,76 @@
 [package]
 name = "feroxbuster"
-version = "1.1.1"
-authors = ["Ben 'epi' Risher <epibar052@gmail.com>"]
+version = "2.13.1"
+authors = ["Ben 'epi' Risher (@epi052)"]
 license = "MIT"
-edition = "2018"
+edition = "2021"
 homepage = "https://github.com/epi052/feroxbuster"
 repository = "https://github.com/epi052/feroxbuster"
 description = "A fast, simple, recursive content discovery tool."
 categories = ["command-line-utilities"]
-keywords = ["pentest", "enumeration", "url-bruteforce", "content-discovery", "web"]
+keywords = [
+    "pentest",
+    "enumeration",
+    "url-bruteforce",
+    "content-discovery",
+    "web",
+]
 exclude = [".github/*", "img/*", "check-coverage.sh"]
+build = "build.rs"
 
 [badges]
 maintenance = { status = "actively-developed" }
 
+[build-dependencies]
+clap = { version = "4.5", features = ["wrap_help", "cargo"] }
+clap_complete = "4.5"
+regex = "1.11"
+lazy_static = "1.5"
+dirs = "5.0"
+
 [dependencies]
-futures = { version = "0.3"}
-tokio = { version = "0.2", features = ["full"] }
-tokio-util = {version = "0.3", features = ["codec"]}
+scraper = "0.19"
+futures = "0.3"
+tokio = { version = "1.47", features = ["full"] }
+tokio-util = { version = "0.7", features = ["codec"] }
 log = "0.4"
-env_logger = "0.7"
-reqwest = { version = "0.10", features = ["socks"] }
-clap = "2"
-lazy_static = "1.4"
-toml = "0.5"
-serde = { version = "1.0", features = ["derive"] }
+env_logger = "0.11"
+reqwest = { version = "0.12", features = ["socks", "native-tls-alpn"] }
+# uses feature unification to add 'serde' to reqwest::Url
+url = { version = "2.5", features = ["serde"] }
+serde_regex = "1.1"
+clap = { version = "4.5", features = ["wrap_help", "cargo"] }
+lazy_static = "1.5"
+toml = "0.8"
+serde = { version = "1.0", features = ["derive", "rc"] }
 serde_json = "1.0"
-uuid = { version = "0.8", features = ["v4"] }
-indicatif = "0.15"
-console = "0.12"
+uuid = { version = "1.17", features = ["v4"] }
+indicatif = { version = "0.17.11" }
+console = "0.15"
 openssl = { version = "0.10", features = ["vendored"] }
-dirs = "3.0"
-regex = "1"
+dirs = "5.0"
+regex = "1.11"
+crossterm = "0.27"
+rlimit = "0.10"
+ctrlc = "3.4"
+anyhow = "1.0"
+leaky-bucket = "1.1"
+gaoya = "0.2"
+# 0.37+ relies on the broken version of indicatif and forces
+# the broken version to be used regardless of the version
+# specified above 
+self_update = { version = "0.40", features = [
+    "archive-tar",
+    "compression-flate2",
+    "archive-zip",
+    "compression-zip-deflate",
+] }
 
 [dev-dependencies]
-tempfile = "3.1"
-httpmock = "0.4.5"
-assert_cmd = "1.0.1"
-predicates = "1.0.5"
+tempfile = "3.20"
+httpmock = "0.7"
+assert_cmd = "2.1"
+predicates = "3.1"
 
 [profile.release]
 lto = true
@@ -49,6 +82,29 @@ section = "utility"
 license-file = ["LICENSE", "4"]
 conf-files = ["/etc/feroxbuster/ferox-config.toml"]
 assets = [
-    ["target/release/feroxbuster", "/usr/bin/", "755"],
-    ["ferox-config.toml.example", "/etc/feroxbuster/ferox-config.toml", "644"],
+    [
+        "target/release/feroxbuster",
+        "/usr/bin/",
+        "755",
+    ],
+    [
+        "ferox-config.toml.example",
+        "/etc/feroxbuster/ferox-config.toml",
+        "644",
+    ],
+    [
+        "shell_completions/feroxbuster.bash",
+        "/usr/share/bash-completion/completions/feroxbuster.bash",
+        "644",
+    ],
+    [
+        "shell_completions/feroxbuster.fish",
+        "/usr/share/fish/completions/feroxbuster.fish",
+        "644",
+    ],
+    [
+        "shell_completions/_feroxbuster",
+        "/usr/share/zsh/vendor-completions/_feroxbuster",
+        "644",
+    ],
 ]

Dockerfile

@@ -1,12 +1,23 @@
-FROM alpine:latest
+FROM alpine:3.17.1 AS build
 LABEL maintainer="wfnintr@null.net"
 
-# download default wordlists 
-RUN apk add --no-cache --virtual .depends subversion && \
-	svn export https://github.com/danielmiessler/SecLists/trunk/Discovery/Web-Content /usr/share/seclists/Discovery/Web-Content && \
-	apk del .depends
+RUN apk upgrade --update-cache --available && apk add --update openssl
 
-# install latest release
-RUN wget https://github.com/epi052/feroxbuster/releases/latest/download/x86_64-linux-feroxbuster.zip -qO feroxbuster.zip && unzip -d /usr/local/bin/ feroxbuster.zip feroxbuster && rm feroxbuster.zip && chmod +x /usr/local/bin/feroxbuster
+# Download latest release
+RUN wget https://github.com/epi052/feroxbuster/releases/latest/download/x86_64-linux-feroxbuster.zip -qO feroxbuster.zip \
+    && unzip -d /tmp/ feroxbuster.zip feroxbuster \
+    && chmod +x /tmp/feroxbuster \
+    && wget https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/Web-Content/raft-medium-directories.txt -O /tmp/raft-medium-directories.txt
+
+FROM alpine:3.17.1 AS release
+COPY --from=build /tmp/raft-medium-directories.txt /usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt
+COPY --from=build /tmp/feroxbuster /usr/local/bin/feroxbuster
+
+RUN adduser \
+    --gecos "" \
+    --disabled-password \
+    feroxbuster
+
+USER feroxbuster
 
 ENTRYPOINT ["feroxbuster"]

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2020 epi
+Copyright (c) 2020-2026 epi
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile

@@ -0,0 +1,80 @@
+default_prefix = /usr/local
+prefix ?= $(default_prefix)
+exec_prefix = $(prefix)
+bindir = $(exec_prefix)/bin
+datarootdir = $(prefix)/share
+datadir = $(datarootdir)
+example_config = ferox-config.toml.example
+config_file = ferox-config.toml
+completion_dir = shell_completions
+completion_prefix = $(completion_dir)/$(BIN)
+
+BIN=feroxbuster
+SHR_SOURCES = $(shell find src -type f -wholename '*src/*.rs') Cargo.toml Cargo.lock
+
+RELEASE = debug
+DEBUG ?= 0
+
+ifeq (0, $(DEBUG))
+	ARGS = --release
+	RELEASE = release
+endif
+
+VENDORED ?= 0
+ifeq (1,$(VENDORED))
+    ARGS += --frozen
+endif
+
+TARGET = target/$(RELEASE)
+
+.PHONY: all clean install uninstall test update
+
+all: cli
+cli: $(TARGET)/$(BIN) $(TARGET)/$(BIN).1.gz $(SHR_SOURCES)
+install: all install-cli
+
+verify:
+	cargo fmt
+	cargo clippy --all-targets --all-features -- -D warnings -A clippy::mutex-atomic
+	cargo test
+
+clean:
+	cargo clean
+
+vendor: vendor.tar
+
+vendor.tar:
+	cargo vendor
+	tar pcf vendor.tar vendor
+	rm -rf vendor
+
+install-cli: cli
+	install -Dm 0644 "$(completion_prefix).bash" "$(DESTDIR)/usr/share/bash-completion/completions/$(BIN).bash"
+	install -Dm 0644 "$(completion_prefix).fish" "$(DESTDIR)/usr/share/fish/completions/$(BIN).fish"
+	install -Dm 0644 "$(completion_dir)/_$(BIN)" "$(DESTDIR)/usr/share/zsh/vendor-completions/_$(BIN)"
+	install -sDm 0755 "$(TARGET)/$(BIN)" "$(DESTDIR)$(bindir)/$(BIN)"
+	install -Dm 0644 "$(TARGET)/$(BIN).1.gz" "$(DESTDIR)$(datadir)/man/man1/$(BIN).1.gz"
+	install -Dm 0644 "$(example_config)" "$(DESTDIR)/etc/$(BIN)/$(config_file)"
+
+uninstall:
+	rm -f "$(DESTDIR)$(bindir)/$(BIN)"
+	rm -f "$(DESTDIR)$(datadir)/man/man1/$(BIN).1.gz"
+	rm -rf "$(DESTDIR)/etc/$(BIN)/"
+	rm -f "$(DESTDIR)/usr/share/bash-completion/completions/$(BIN).bash"
+	rm -f "$(DESTDIR)/usr/share/zsh/vendor-completions/_$(BIN)"
+	rm -f "$(DESTDIR)/usr/share/fish/completions/$(BIN).fish"
+
+extract:
+ifeq (1, $(VENDORED))
+	tar pxf vendor.tar
+endif
+
+$(TARGET)/$(BIN): extract
+	mkdir -p .cargo debian
+	touch debian/cargo.config
+	cp debian/cargo.config .cargo/config.toml
+	cargo build $(ARGS)
+
+$(TARGET)/$(BIN).1.gz: $(TARGET)/$(BIN)
+	help2man --no-info $< | gzip -c > $@.partial
+	mv $@.partial $@

Makefile.toml

@@ -0,0 +1,53 @@
+# composite tasks
+[tasks.upgrade]
+dependencies = ["upgrade-deps", "update"]
+
+[tasks.check]
+dependencies = ["fmt", "clippy", "test"]
+
+# cleaning
+[tasks.clean-state]
+script = """
+rm ferox-*.state
+"""
+
+# dependency management
+[tasks.upgrade-deps]
+command = "cargo"
+args = ["upgrade", "--exclude", "self_update"]
+
+[tasks.update]
+command = "cargo"
+args = ["update"]
+
+# clippy / lint
+[tasks.clippy]
+clear = true
+script = """
+cargo clippy --all-targets --all-features -- -D warnings
+"""
+
+[tasks.fmt]
+clear = true
+script = """
+cargo fmt --all
+"""
+
+# tests
+[tasks.test]
+clear = true
+dependencies = ["test-local", "test-remote"]
+
+[tasks.test-remote]
+condition = { env_set = ["CI"] }
+clear = true
+script = """
+cargo nextest run --all-features --all-targets --retries 4 --no-fail-fast
+"""
+
+[tasks.test-local]
+condition = { env_not_set = ["CI"] }
+clear = true
+script = """
+cargo nextest run --all-features --all-targets --no-fail-fast --run-ignored all --retries 4
+"""

README.md

@@ -1,3 +1,18 @@
+> [!WARNING]
+> **Security Notice – Domain Impersonation**
+>
+> The domain **feroxbuster.com** is **NOT affiliated** with this project, its maintainers, or any official feroxbuster releases.
+>
+> Official feroxbuster downloads are distributed **ONLY** through:
+>
+> - [https://github.com/epi052/feroxbuster](https://github.com/epi052/feroxbuster/releases) (open source)
+> - [https://www.feroxbuster.pro](https://www.feroxbuster.pro) (commercial)
+> - package repositories listed in this README
+> - package repositories listed in the [installation docs](https://epi052.github.io/feroxbuster-docs/installation/android/)
+>
+> We do **not** distribute software from feroxbuster.com, and we cannot vouch for the authenticity or safety of files hosted there.
+> If you downloaded feroxbuster from any other domain, we strongly recommend deleting it and reinstalling from an official source.
+
 <h1 align="center">
   <br>
   <a href="https://github.com/epi052/feroxbuster"><img src="img/logo/default-cropped.png" alt="feroxbuster"></a>
@@ -8,7 +23,7 @@
 
 <p align="center">
   <a href="https://github.com/epi052/feroxbuster/actions?query=workflow%3A%22CI+Pipeline%22">
-    <img src="https://img.shields.io/github/workflow/status/epi052/feroxbuster/CI%20Pipeline/master?logo=github">
+    <img src="https://img.shields.io/github/actions/workflow/status/epi052/feroxbuster/.github/workflows/check.yml?branch=main&logo=github">
   </a>
 
   <a href="https://github.com/epi052/feroxbuster/releases">
@@ -22,7 +37,7 @@
   <a href="https://crates.io/crates/feroxbuster">
     <img src="https://img.shields.io/crates/v/feroxbuster?color=blue&label=version&logo=rust">
   </a>
- 
+
   <a href="https://crates.io/crates/feroxbuster">
     <img src="https://img.shields.io/crates/d/feroxbuster?label=downloads&logo=rust&color=inactive">
   </a>
@@ -30,93 +45,84 @@
   <a href="https://codecov.io/gh/epi052/feroxbuster">
     <img src="https://codecov.io/gh/epi052/feroxbuster/branch/master/graph/badge.svg" />
   </a>
+  <!--
+  <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section 
+    [![All Contributors](https://img.shields.io/badge/all_contributors-15-orange.svg?style=flat-square)](#contributors-)
+  <!-- ALL-CONTRIBUTORS-BADGE:END -->
+  <a href="https://github.com/epi052/feroxbuster/graphs/contributors">
+    <img src="https://img.shields.io/badge/all_contributors-31-orange.svg" />
+  </a>
+
 </p>
 
 ![demo](img/demo.gif)
 
 <p align="center">
-  🦀
-  <a href="https://github.com/epi052/feroxbuster/releases">Releases</a> ✨
-  <a href="#-example-usage">Example Usage</a> ✨
-  <a href="https://github.com/epi052/feroxbuster/blob/master/CONTRIBUTING.md">Contributing</a> ✨
-  <a href="https://docs.rs/feroxbuster/latest/feroxbuster/">Documentation</a>
-  🦀
+  <a href="https://github.com/epi052/feroxbuster/releases"><img src="https://img.shields.io/badge/Releases-CF4F4B?style=flat-square&logo=github&logoColor=white" alt="Releases"></a>&nbsp;
+  <a href="https://epi052.github.io/feroxbuster-docs/examples/auto-tune/"><img src="https://img.shields.io/badge/Examples-CF4F4B?style=flat-square" alt="Example Usage"></a>&nbsp;
+  <a href="https://github.com/epi052/feroxbuster/blob/main/CONTRIBUTING.md"><img src="https://img.shields.io/badge/Contributing-CF4F4B?style=flat-square" alt="Contributing"></a>&nbsp;
+  <a href="https://epi052.github.io/feroxbuster-docs/overview"><img src="https://img.shields.io/badge/Documentation-CF4F4B?style=flat-square&logo=bookstack&logoColor=white" alt="Documentation"></a>&nbsp;
+  <a href="https://www.feroxbuster.pro"><img src="https://img.shields.io/badge/Pro-CF4F4B?style=flat-square" alt="Pro"></a>
 </p>
 
-## 😕 What the heck is a ferox anyway?
+---
 
-Ferox is short for Ferric Oxide. Ferric Oxide, simply put, is rust.  The name rustbuster was taken, so I decided on a variation.  🤷	
+> [!TIP]
+> **Documentation has moved!** &mdash; Instead of having a 1300 line `README.md` (sorry...), feroxbuster's documentation has moved to GitHub Pages. The move to hosting documentation on Pages should make it a LOT easier to find the information you're looking for, whatever that may be. Please check it out for anything you need beyond a quick-start.
+>
+> **[View the full documentation &#8594;](https://epi052.github.io/feroxbuster-docs/overview)**
 
-## 🤔 What's it do tho? 
+## What the heck is a ferox anyway?
 
-`feroxbuster` is a tool designed to perform [Forced Browsing](https://owasp.org/www-community/attacks/Forced_browsing).  
+Ferox is short for Ferric Oxide. Ferric Oxide, simply put, is rust. The name rustbuster was taken, so I decided on a
+variation.
 
-Forced browsing is an attack where the aim is to enumerate and access resources that are not referenced by the web application, but are still accessible by an attacker.
+## What's it do tho?
 
-`feroxbuster` uses brute force combined with a wordlist to search for unlinked content in target directories. These resources may store sensitive information about web applications and operational systems, such as source code, credentials, internal network addressing, etc...
+`feroxbuster` is a tool designed to perform [Forced Browsing](https://owasp.org/www-community/attacks/Forced_browsing).
 
-This attack is also known as Predictable Resource Location, File Enumeration, Directory Enumeration, and Resource Enumeration.
+Forced browsing is an attack where the aim is to enumerate and access resources that are not referenced by the web
+application, but are still accessible by an attacker.
 
-📖 Table of Contents
------------------
-- [Installation](#-installation)
-    - [Download a Release](#download-a-release)
-    - [Homebrew on MacOS and Linux](#homebrew-on-macos-and-linux)
-    - [Cargo Install](#cargo-install)
-    - [apt Install](#apt-install)
-    - [AUR Install](#aur-install)
-    - [Docker Install](#docker-install)
-- [Configuration](#%EF%B8%8F-configuration)
-    - [Default Values](#default-values)
-    - [ferox-config.toml](#ferox-configtoml)
-    - [Command Line Parsing](#command-line-parsing)
-- [Example Usage](#-example-usage)
-    - [Multiple Values](#multiple-values)
-    - [Extract Links from Response Body (new in `v1.1.0`)](#extract-links-from-response-body-new-in-v110)
-    - [Include Headers](#include-headers)
-    - [IPv6, Non-recursive scan with INFO logging enabled](#ipv6-non-recursive-scan-with-info-level-logging-enabled)
-    - [Read urls from STDIN; pipe only resulting urls out to another tool](#read-urls-from-stdin-pipe-only-resulting-urls-out-to-another-tool)
-    - [Proxy traffic through Burp](#proxy-traffic-through-burp)
-    - [Proxy traffic through a SOCKS proxy](#proxy-traffic-through-a-socks-proxy)
-    - [Pass auth token via query parameter](#pass-auth-token-via-query-parameter)
-- [Comparison w/ Similar Tools](#-comparison-w-similar-tools)
+`feroxbuster` uses brute force combined with a wordlist to search for unlinked content in target directories. These
+resources may store sensitive information about web applications and operational systems, such as source code,
+credentials, internal network addressing, etc...
 
-## 💿 Installation
+This attack is also known as Predictable Resource Location, File Enumeration, Directory Enumeration, and Resource
+Enumeration.
 
-### Download a Release
+## Quick Start
 
-Releases for multiple architectures can be found in the [Releases](https://github.com/epi052/feroxbuster/releases) section.  The latest release for each of the following systems can be downloaded and executed as shown below.
+This section will cover the minimum amount of information to get up and running with feroxbuster. Please refer the the [documentation](https://epi052.github.io/feroxbuster-docs/overview/), as it's much more comprehensive.
 
-#### Linux x86
-```
-curl -sLO https://github.com/epi052/feroxbuster/releases/latest/download/x86-linux-feroxbuster.zip
-unzip x86-linux-feroxbuster.zip
-chmod +x ./feroxbuster
-./feroxbuster -V
-```
-#### Linux x86_64
+### Installation
+
+There are quite a few other [installation methods](https://epi052.github.io/feroxbuster-docs/installation/android/), but these snippets should cover the majority of users. 
+
+#### Kali 
+
+If you're using kali, this is the preferred install method. Installing from the repos adds a [**ferox-config.toml**](https://epi052.github.io/feroxbuster-docs/configuration/ferox-config-toml/) in `/etc/feroxbuster/`, adds command completion for bash, fish, and zsh, includes a man page entry, and installs `feroxbuster` itself. 
 
 ```
-curl -sLO https://github.com/epi052/feroxbuster/releases/latest/download/x86_64-linux-feroxbuster.zip
-unzip x86_64-linux-feroxbuster.zip
-chmod +x ./feroxbuster
-./feroxbuster -V
+sudo apt update && sudo apt install -y feroxbuster
 ```
 
-#### MacOS x86_64
+#### Linux (32 and 64-bit) & MacOS
+
+Install to a particular directory
 ```
-curl -sLO https://github.com/epi052/feroxbuster/releases/latest/download/x86_64-macos-feroxbuster.zip
-unzip x86_64-macos-feroxbuster.zip
-chmod +x ./feroxbuster
-./feroxbuster -V
+curl -sL https://raw.githubusercontent.com/epi052/feroxbuster/main/install-nix.sh | bash -s $HOME/.local/bin
 ```
 
-#### Windows x86
+Install to current working directory
+```
+curl -sL https://raw.githubusercontent.com/epi052/feroxbuster/main/install-nix.sh | bash
+```
+
+#### MacOS via Homebrew 
 
 ```
-https://github.com/epi052/feroxbuster/releases/latest/download/x86-windows-feroxbuster.exe.zip
-Expand-Archive .\feroxbuster.zip
-.\feroxbuster\feroxbuster.exe -V
+brew install feroxbuster
 ```
 
 #### Windows x86_64
@@ -127,241 +133,35 @@ Expand-Archive .\feroxbuster.zip
 .\feroxbuster\feroxbuster.exe -V
 ```
 
-### Homebrew on MacOS and Linux
-
-Installable by Homebrew throughout own formulas:
-
-🍏 [MacOS](https://github.com/TGotwig/homebrew-feroxbuster/blob/main/feroxbuster.rb)
-
-```shell
-brew tap tgotwig/feroxbuster
-brew install feroxbuster
-```
-
-🐧 [Linux](https://github.com/TGotwig/homebrew-linux-feroxbuster/blob/main/feroxbuster.rb)
-
-```shell
-brew tap tgotwig/linux-feroxbuster
-brew install feroxbuster
-```
-
-### Cargo Install
-
-`feroxbuster` is published on crates.io, making it easy to install if you already have rust installed on your system.
+#### Windows via Winget
 
 ```
-cargo install feroxbuster
+winget install epi052.feroxbuster
 ```
 
-### apt Install
-
-Download `feroxbuster_amd64.deb` from the [Releases](https://github.com/epi052/feroxbuster/releases) section.  After that, use your favorite package manager to install the `.deb`.
+#### Windows via Chocolatey
 
 ```
-wget -sLO https://github.com/epi052/feroxbuster/releases/latest/download/feroxbuster_amd64.deb.zip
-unzip feroxbuster_amd64.deb.zip
-sudo apt install ./feroxbuster_amd64.deb
+choco install feroxbuster
 ```
 
-### AUR Install
+#### All others 
 
-Install `feroxbuster-git` on Arch Linux with your AUR helper of choice:
+Please refer the the [documentation](https://epi052.github.io/feroxbuster-docs/overview).
+
+### Updating feroxbuster (new in v2.9.1)
 
 ```
-yay -S feroxbuster-git
+./feroxbuster --update
 ```
 
-### Docker Install
+## Example Usage
 
-> The following steps assume you have docker installed / setup
-
-First, clone the repository.
-
-```
-git clone https://github.com/epi052/feroxbuster.git
-cd feroxbuster
-```
-
-Next, build the image.
-
-```
-sudo docker build -t feroxbuster .
-```
-
-After that, you should be able to use `docker run` to perform scans with `feroxbuster`.
-
-#### Basic usage
-
-```
-sudo docker run --init -it feroxbuster -u http://example.com -x js,html
-```
-
-#### Piping from stdin and proxying all requests through socks5 proxy
-
-```
-cat targets.txt | sudo docker run --net=host --init -i feroxbuster --stdin -x js,html --proxy socks5://127.0.0.1:9050
-```
-
-#### Mount a volume to pass in `ferox-config.toml`
-
-You've got some options available if you want to pass in a config file.  [`ferox-buster.toml`](#ferox-configtoml) can live in multiple locations and still be valid, so it's up to you how you'd like to pass it in.  Below are a few valid examples:
-
-```
-sudo docker run --init -v $(pwd)/ferox-config.toml:/etc/feroxbuster/ferox-config.toml -it feroxbuster -u http://example.com
-```
-
-```
-sudo docker run --init -v ~/.config/feroxbuster:/root/.config/feroxbuster -it feroxbuster -u http://example.com
-```
-
-Note: If you are on a SELinux enforced system, you will need to pass the `:Z` attribute also.
-
-```
-docker run --init -v (pwd)/ferox-config.toml:/etc/feroxbuster/ferox-config.toml:Z -it feroxbuster -u http://example.com
-```
-
-#### Define an alias for simplicity
-
-```
-alias feroxbuster="sudo docker run --init -v ~/.config/feroxbuster:/root/.config/feroxbuster -i feroxbuster"
-```
-
-## ⚙️ Configuration
-### Default Values
-Configuration begins with with the following built-in default values baked into the binary:
-
-- timeout: `7` seconds
-- follow redirects: `false`
-- wordlist: `/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt`
-- threads: `50`
-- verbosity: `0` (no logging enabled)
-- statuscodes: `200 204 301 302 307 308 401 403 405`
-- useragent: `feroxbuster/VERSION`
-- recursion depth: `4`
-- auto-filter wildcards - `true`
-- output: `stdout`
-
-### ferox-config.toml
-After setting built-in default values, any values defined in a `ferox-config.toml` config file will override the
-built-in defaults.  
-
-`feroxbuster` searches for `ferox-config.toml` in the following locations (in the order shown):
-- `/etc/feroxbuster/` (global)
-- `CONFIG_DIR/ferxobuster/` (per-user)
-- The same directory as the `feroxbuster` executable (per-user)
-- The user's current working directory (per-target)
-
-> `CONFIG_DIR` is defined as the following:
-> - Linux: `$XDG_CONFIG_HOME` or `$HOME/.config` i.e. `/home/bob/.config`
-> - MacOs: `$HOME/Library/Application Support` i.e. `/Users/bob/Library/Application Support`
-> - Windows: `{FOLDERID_RoamingAppData}` i.e. `C:\Users\Bob\AppData\Roaming`
-
-If more than one valid configuration file is found, each one overwrites the values found previously.  
-
-If no configuration file is found, nothing happens at this stage.
-
-As an example, let's say that we prefer to use a different wordlist as our default when scanning; we can
-set the `wordlist` value in the config file to override the baked-in default.
-
-Notes of interest:
-- it's ok to only specify values you want to change without specifying anything else
-- variable names in `ferox-config.toml` must match their command-line counterpart
-
-```toml
-# ferox-config.toml
-
-wordlist = "/wordlists/jhaddix/all.txt"
-```
-
-A pre-made configuration file with examples of all available settings can be found in `ferox-config.toml.example`.
-```toml
-# ferox-config.toml
-# Example configuration for feroxbuster
-#
-# If you wish to provide persistent settings to feroxbuster, rename this file to ferox-config.toml and make sure
-# it resides in the same directory as the feroxbuster binary.
-#
-# After that, uncomment any line to override the default value provided by the binary itself.
-#
-# Any setting used here can be overridden by the corresponding command line option/argument
-#
-# wordlist = "/wordlists/jhaddix/all.txt"
-# statuscodes = [200, 500]
-# threads = 1
-# timeout = 5
-# proxy = "http://127.0.0.1:8080"
-# verbosity = 1
-# quiet = true
-# output = "/targets/ellingson_mineral_company/gibson.txt"
-# useragent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0"
-# redirects = true
-# insecure = true
-# extensions = ["php", "html"]
-# norecursion = true
-# addslash = true
-# stdin = true
-# dontfilter = true
-# extract_links = true
-# depth = 1
-# sizefilters = [5174]
-# queries = [["name","value"], ["rick", "astley"]]
-
-# headers can be specified on multiple lines or as an inline table
-#
-# inline example
-# headers = {"stuff" = "things"}
-#
-# multi-line example
-#   note: if multi-line is used, all key/value pairs under it belong to the headers table until the next table
-#         is found or the end of the file is reached
-#
-# [headers]
-# stuff = "things"
-# more = "headers"
-```
-
-### Command Line Parsing
-Finally, after parsing the available config file, any options/arguments given on the commandline will override any values that were set as a built-in or config-file value.
-
-```
-USAGE:
-    feroxbuster [FLAGS] [OPTIONS] --url <URL>...
-
-FLAGS:
-    -f, --addslash         Append / to each request
-    -D, --dontfilter       Don't auto-filter wildcard responses
-    -e, --extract-links    Extract links from response body (html, javascript, etc...); make new requests based on
-                           findings (default: false)
-    -h, --help             Prints help information
-    -k, --insecure         Disables TLS certificate validation
-    -n, --norecursion      Do not scan recursively
-    -q, --quiet            Only print URLs; Don't print status codes, response size, running config, etc...
-    -r, --redirects        Follow redirects
-        --stdin            Read url(s) from STDIN
-    -V, --version          Prints version information
-    -v, --verbosity        Increase verbosity level (use -vv or more for greater effect)
-
-OPTIONS:
-    -d, --depth <RECURSION_DEPTH>           Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)
-    -x, --extensions <FILE_EXTENSION>...    File extension(s) to search for (ex: -x php -x pdf js)
-    -H, --headers <HEADER>...               Specify HTTP headers (ex: -H Header:val 'stuff: things')
-    -o, --output <FILE>                     Output file to write results to (default: stdout)
-    -p, --proxy <PROXY>                     Proxy to use for requests (ex: http(s)://host:port, socks5://host:port)
-    -Q, --query <QUERY>...                  Specify URL query parameters (ex: -Q token=stuff -Q secret=key)
-    -S, --sizefilter <SIZE>...              Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)
-    -s, --statuscodes <STATUS_CODE>...      Status Codes of interest (default: 200 204 301 302 307 308 401 403 405)
-    -t, --threads <THREADS>                 Number of concurrent threads (default: 50)
-    -T, --timeout <SECONDS>                 Number of seconds before a request times out (default: 7)
-    -u, --url <URL>...                      The target URL(s) (required, unless --stdin used)
-    -a, --useragent <USER_AGENT>            Sets the User-Agent (default: feroxbuster/VERSION)
-    -w, --wordlist <FILE>                   Path to the wordlist
-```
-
-## 🧰 Example Usage
+Here are a few brief examples to get you started.  Please note, feroxbuster can do a **lot more** than what's listed below.  As a result, there are **many more** examples, with **demonstration gifs** that highlight specific features, in the [documentation](https://epi052.github.io/feroxbuster-docs/overview).
 
 ### Multiple Values
 
-Options that take multiple values are very flexible.  Consider the following ways of specifying extensions:
+Options that take multiple values are very flexible. Consider the following ways of specifying extensions:
 
 ```
 ./feroxbuster -u http://127.1 -x pdf -x js,html -x php txt json,docx
@@ -369,7 +169,8 @@ Options that take multiple values are very flexible.  Consider the following way
 
 The command above adds .pdf, .js, .html, .php, .txt, .json, and .docx to each url
 
-All of the methods above (multiple flags, space separated, comma separated, etc...) are valid and interchangeable.  The same goes for urls, headers, status codes, queries, and size filters.
+All of the methods above (multiple flags, space separated, comma separated, etc...) are valid and interchangeable. The
+same goes for urls, headers, status codes, queries, and size filters.
 
 ### Include Headers
 
@@ -377,36 +178,16 @@ All of the methods above (multiple flags, space separated, comma separated, etc.
 ./feroxbuster -u http://127.1 -H Accept:application/json "Authorization: Bearer {token}"
 ```
 
-### Extract Links from Response Body (New in `v1.1.0`) 
-
-Search through the body of valid responses (html, javascript, etc...) for additional endpoints to scan. This turns
-`feroxbuster` into a hybrid that looks for both linked and unlinked content. 
-
-Example request/response with `--extract-links` enabled:
-- Make request to `http://example.com/index.html`
-- Receive, and read in, the `body` of the response
-- Search the `body` for absolute and relative links (i.e. `homepage/assets/img/icons/handshake.svg`)
-- Add the following directories for recursive scanning:
-    - `http://example.com/homepage`
-    - `http://example.com/homepage/assets`
-    - `http://example.com/homepage/assets/img`
-    - `http://example.com/homepage/assets/img/icons`
-- Make a single request to `http://example.com/homepage/assets/img/icons/handshake.svg`
-
-```
-./feroxbuster -u http://127.1 --extract-links
-```
-
 ### IPv6, non-recursive scan with INFO-level logging enabled
 
 ```
-./feroxbuster -u http://[::1] --norecursion -vv
+./feroxbuster -u http://[::1] --no-recursion -vv
 ```
 
 ### Read urls from STDIN; pipe only resulting urls out to another tool
 
 ```
-cat targets | ./feroxbuster --stdin --quiet -s 200 301 302 --redirects -x js | fff -s 200 -o js-files
+cat targets | ./feroxbuster --stdin --silent -s 200 301 302 --redirects -x js | fff -s 200 -o js-files
 ```
 
 ### Proxy traffic through Burp
@@ -415,10 +196,10 @@ cat targets | ./feroxbuster --stdin --quiet -s 200 301 302 --redirects -x js | f
 ./feroxbuster -u http://127.1 --insecure --proxy http://127.0.0.1:8080
 ```
 
-### Proxy traffic through a SOCKS proxy
+### Proxy traffic through a SOCKS proxy (including DNS lookups)
 
 ```
-./feroxbuster -u http://127.1 --proxy socks5://127.0.0.1:9050
+./feroxbuster -u http://127.1 --proxy socks5h://127.0.0.1:9050
 ```
 
 ### Pass auth token via query parameter
@@ -427,46 +208,179 @@ cat targets | ./feroxbuster --stdin --quiet -s 200 301 302 --redirects -x js | f
 ./feroxbuster -u http://127.1 --query token=0123456789ABCDEF
 ```
 
+### Set the Content-Type of the body automatically with --data-json --data-urlencoded
 
-## 🧐 Comparison w/ Similar Tools
+```
+./feroxbuster -u http://127.1 --data-json '{"some": "payload"}'
+./feroxbuster -u http://127.1 --data-json @payload.json
+./feroxbuster -u http://127.1 --data-urlencoded 'some=payload'
+./feroxbuster -u http://127.1 --data-urlencoded @file.payload
+```
 
-There are quite a few similar tools for forced browsing/content discovery.  Burp Suite Pro, Dirb, Dirbuster, etc... 
-However, in my opinion, there are two that set the standard: [gobuster](https://github.com/OJ/gobuster) and 
-[ffuf](https://github.com/ffuf/ffuf).  Both are mature, feature-rich, and all-around incredible tools to use.
+> [!TIP]
+> For realsies, there used to be over 1300 lines in this README, but it's all been moved to the [new documentation site](https://epi052.github.io/feroxbuster-docs/overview). Go check it out!
+>
+> **[View the full documentation &#8594;](https://epi052.github.io/feroxbuster-docs/overview)**
 
-So, why would you ever want to use feroxbuster over ffuf/gobuster?  In most cases, you probably won't.  ffuf in particular
-can do the vast majority of things that feroxbuster can, while still offering boatloads more functionality.  Here are
-a few of the use-cases in which feroxbuster may be a better fit:
+## Contributors
 
-- You want a **simple** tool usage experience
-- You want to be able to run your content discovery as part of some crazy 12 command unix **pipeline extravaganza**
-- You want to scan through a **SOCKS** proxy
-- You want **auto-filtering** of Wildcard responses by default
-- You want an integrated **link extractor** to increase discovered endpoints
-- You want **recursion** along with some other thing mentioned above (ffuf also does recursion)
-- You want a **configuration file** option for overriding built-in default values for your scans
+Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
 
-|                                                                  | feroxbuster | gobuster | ffuf |
-|------------------------------------------------------------------|---|---|---|
-| fast                                                             | ✔ | ✔ | ✔ |
-| easy to use                                                      | ✔ | ✔ |   |
-| blacklist status codes (in addition to whitelist)                |   | ✔ | ✔ |
-| allows recursion                                                 | ✔ |   | ✔ |
-| can specify query parameters                                     | ✔ |   | ✔ |
-| SOCKS proxy support                                              | ✔ |   |   |
-| extracts links from response body to increase scan coverage      | ✔ |   |   |
-| multiple target scan (via stdin or multiple -u)                  | ✔ |   | ✔ |
-| configuration file for default value override                    | ✔ |   | ✔ |
-| can accept urls via STDIN as part of a pipeline                  | ✔ |   | ✔ |
-| can accept wordlists via STDIN                                   |   | ✔ | ✔ |
-| filter by response size                                          | ✔ |   | ✔ |
-| auto-filter wildcard responses                                   | ✔ |   | ✔ |
-| performs other scans (vhost, dns, etc)                           |   | ✔ | ✔ |
-| time delay / rate limiting                                       |   | ✔ | ✔ |
-| **huge** number of other options                                 |   |   | ✔ |
+<!-- ALL-CONTRIBUTORS-LIST:START - Do not remove or modify this section -->
+<!-- prettier-ignore-start -->
+<!-- markdownlint-disable -->
+<table>
+  <tbody>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://io.fi"><img src="https://avatars.githubusercontent.com/u/5235109?v=4?s=100" width="100px;" alt="Joona Hoikkala"/><br /><sub><b>Joona Hoikkala</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/commits?author=joohoi" title="Documentation">📖</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/jsav0"><img src="https://avatars.githubusercontent.com/u/20546041?v=4?s=100" width="100px;" alt="J Savage"/><br /><sub><b>J Savage</b></sub></a><br /><a href="#infra-jsav0" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="https://github.com/epi052/feroxbuster/commits?author=jsav0" title="Documentation">📖</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="http://www.tgotwig.dev"><img src="https://avatars.githubusercontent.com/u/30773779?v=4?s=100" width="100px;" alt="Thomas Gotwig"/><br /><sub><b>Thomas Gotwig</b></sub></a><br /><a href="#infra-TGotwig" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="https://github.com/epi052/feroxbuster/commits?author=TGotwig" title="Documentation">📖</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/spikecodes"><img src="https://avatars.githubusercontent.com/u/19519553?v=4?s=100" width="100px;" alt="Spike"/><br /><sub><b>Spike</b></sub></a><br /><a href="#infra-spikecodes" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="https://github.com/epi052/feroxbuster/commits?author=spikecodes" title="Documentation">📖</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/evanrichter"><img src="https://avatars.githubusercontent.com/u/330292?v=4?s=100" width="100px;" alt="Evan Richter"/><br /><sub><b>Evan Richter</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/commits?author=evanrichter" title="Code">💻</a> <a href="https://github.com/epi052/feroxbuster/commits?author=evanrichter" title="Documentation">📖</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/mzpqnxow"><img src="https://avatars.githubusercontent.com/u/8016228?v=4?s=100" width="100px;" alt="AG"/><br /><sub><b>AG</b></sub></a><br /><a href="#ideas-mzpqnxow" title="Ideas, Planning, & Feedback">🤔</a> <a href="https://github.com/epi052/feroxbuster/commits?author=mzpqnxow" title="Documentation">📖</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://n-thumann.de/"><img src="https://avatars.githubusercontent.com/u/46975855?v=4?s=100" width="100px;" alt="Nicolas Thumann"/><br /><sub><b>Nicolas Thumann</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/commits?author=n-thumann" title="Code">💻</a> <a href="https://github.com/epi052/feroxbuster/commits?author=n-thumann" title="Documentation">📖</a></td>
+    </tr>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/tomtastic"><img src="https://avatars.githubusercontent.com/u/302127?v=4?s=100" width="100px;" alt="Tom Matthews"/><br /><sub><b>Tom Matthews</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/commits?author=tomtastic" title="Documentation">📖</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/bsysop"><img src="https://avatars.githubusercontent.com/u/9998303?v=4?s=100" width="100px;" alt="bsysop"/><br /><sub><b>bsysop</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/commits?author=bsysop" title="Documentation">📖</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="http://bpsizemore.me"><img src="https://avatars.githubusercontent.com/u/11645898?v=4?s=100" width="100px;" alt="Brian Sizemore"/><br /><sub><b>Brian Sizemore</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/commits?author=bpsizemore" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://pwn.by/noraj"><img src="https://avatars.githubusercontent.com/u/16578570?v=4?s=100" width="100px;" alt="Alexandre ZANNI"/><br /><sub><b>Alexandre ZANNI</b></sub></a><br /><a href="#infra-noraj" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="https://github.com/epi052/feroxbuster/commits?author=noraj" title="Documentation">📖</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/craig"><img src="https://avatars.githubusercontent.com/u/99729?v=4?s=100" width="100px;" alt="Craig"/><br /><sub><b>Craig</b></sub></a><br /><a href="#infra-craig" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://www.reddit.com/u/EONRaider"><img src="https://avatars.githubusercontent.com/u/15611424?v=4?s=100" width="100px;" alt="EONRaider"/><br /><sub><b>EONRaider</b></sub></a><br /><a href="#infra-EONRaider" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/wtwver"><img src="https://avatars.githubusercontent.com/u/53866088?v=4?s=100" width="100px;" alt="wtwver"/><br /><sub><b>wtwver</b></sub></a><br /><a href="#infra-wtwver" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a></td>
+    </tr>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://tib3rius.com"><img src="https://avatars.githubusercontent.com/u/48113936?v=4?s=100" width="100px;" alt="Tib3rius"/><br /><sub><b>Tib3rius</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3ATib3rius" title="Bug reports">🐛</a> <a href="#ideas-Tib3rius" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/0xdf"><img src="https://avatars.githubusercontent.com/u/1489045?v=4?s=100" width="100px;" alt="0xdf"/><br /><sub><b>0xdf</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3A0xdf" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="http://secure77.de"><img src="https://avatars.githubusercontent.com/u/31564517?v=4?s=100" width="100px;" alt="secure-77"/><br /><sub><b>secure-77</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3Asecure-77" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/sbrun"><img src="https://avatars.githubusercontent.com/u/7712154?v=4?s=100" width="100px;" alt="Sophie Brun"/><br /><sub><b>Sophie Brun</b></sub></a><br /><a href="#infra-sbrun" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/black-A"><img src="https://avatars.githubusercontent.com/u/30686803?v=4?s=100" width="100px;" alt="black-A"/><br /><sub><b>black-A</b></sub></a><br /><a href="#ideas-black-A" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/dinosn"><img src="https://avatars.githubusercontent.com/u/3851678?v=4?s=100" width="100px;" alt="Nicolas Krassas"/><br /><sub><b>Nicolas Krassas</b></sub></a><br /><a href="#ideas-dinosn" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/N0ur5"><img src="https://avatars.githubusercontent.com/u/24260009?v=4?s=100" width="100px;" alt="N0ur5"/><br /><sub><b>N0ur5</b></sub></a><br /><a href="#ideas-N0ur5" title="Ideas, Planning, & Feedback">🤔</a> <a href="https://github.com/epi052/feroxbuster/issues?q=author%3AN0ur5" title="Bug reports">🐛</a></td>
+    </tr>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/moscowchill"><img src="https://avatars.githubusercontent.com/u/72578879?v=4?s=100" width="100px;" alt="mchill"/><br /><sub><b>mchill</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3Amoscowchill" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="http://BitThr3at.github.io"><img src="https://avatars.githubusercontent.com/u/45028933?v=4?s=100" width="100px;" alt="Naman"/><br /><sub><b>Naman</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3ABitThr3at" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/Sicks3c"><img src="https://avatars.githubusercontent.com/u/32225186?v=4?s=100" width="100px;" alt="Ayoub Elaich"/><br /><sub><b>Ayoub Elaich</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3Asicks3c" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/HenryHoggard"><img src="https://avatars.githubusercontent.com/u/1208121?v=4?s=100" width="100px;" alt="Henry"/><br /><sub><b>Henry</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3AHenryHoggard" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/SleepiPanda"><img src="https://avatars.githubusercontent.com/u/6428561?v=4?s=100" width="100px;" alt="SleepiPanda"/><br /><sub><b>SleepiPanda</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3ASleepiPanda" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/uBadRequest"><img src="https://avatars.githubusercontent.com/u/47282747?v=4?s=100" width="100px;" alt="Bad Requests"/><br /><sub><b>Bad Requests</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3AuBadRequest" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://home.dnaka91.rocks"><img src="https://avatars.githubusercontent.com/u/36804488?v=4?s=100" width="100px;" alt="Dominik Nakamura"/><br /><sub><b>Dominik Nakamura</b></sub></a><br /><a href="#infra-dnaka91" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a></td>
+    </tr>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/hunter0x8"><img src="https://avatars.githubusercontent.com/u/46222314?v=4?s=100" width="100px;" alt="Muhammad Ahsan"/><br /><sub><b>Muhammad Ahsan</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3Ahunter0x8" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/cortantief"><img src="https://avatars.githubusercontent.com/u/34527333?v=4?s=100" width="100px;" alt="cortantief"/><br /><sub><b>cortantief</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3Acortantief" title="Bug reports">🐛</a> <a href="https://github.com/epi052/feroxbuster/commits?author=cortantief" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/dsaxton"><img src="https://avatars.githubusercontent.com/u/2658661?v=4?s=100" width="100px;" alt="Daniel Saxton"/><br /><sub><b>Daniel Saxton</b></sub></a><br /><a href="#ideas-dsaxton" title="Ideas, Planning, & Feedback">🤔</a> <a href="https://github.com/epi052/feroxbuster/commits?author=dsaxton" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/n0kovo"><img src="https://avatars.githubusercontent.com/u/16690056?v=4?s=100" width="100px;" alt="n0kovo"/><br /><sub><b>n0kovo</b></sub></a><br /><a href="#ideas-n0kovo" title="Ideas, Planning, & Feedback">🤔</a> <a href="https://github.com/epi052/feroxbuster/issues?q=author%3An0kovo" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://ring0.lol"><img src="https://avatars.githubusercontent.com/u/1893909?v=4?s=100" width="100px;" alt="Justin Steven"/><br /><sub><b>Justin Steven</b></sub></a><br /><a href="#ideas-justinsteven" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/7047payloads"><img src="https://avatars.githubusercontent.com/u/95562424?v=4?s=100" width="100px;" alt="7047payloads"/><br /><sub><b>7047payloads</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/commits?author=7047payloads" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/unkn0wnsyst3m"><img src="https://avatars.githubusercontent.com/u/21272239?v=4?s=100" width="100px;" alt="unkn0wnsyst3m"/><br /><sub><b>unkn0wnsyst3m</b></sub></a><br /><a href="#ideas-unkn0wnsyst3m" title="Ideas, Planning, & Feedback">🤔</a></td>
+    </tr>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://ironwort.me/"><img src="https://avatars.githubusercontent.com/u/15280042?v=4?s=100" width="100px;" alt="0x08"/><br /><sub><b>0x08</b></sub></a><br /><a href="#ideas-its0x08" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/MD-Levitan"><img src="https://avatars.githubusercontent.com/u/12116508?v=4?s=100" width="100px;" alt="kusok"/><br /><sub><b>kusok</b></sub></a><br /><a href="#ideas-MD-Levitan" title="Ideas, Planning, & Feedback">🤔</a> <a href="https://github.com/epi052/feroxbuster/commits?author=MD-Levitan" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/godylockz"><img src="https://avatars.githubusercontent.com/u/81207744?v=4?s=100" width="100px;" alt="godylockz"/><br /><sub><b>godylockz</b></sub></a><br /><a href="#ideas-godylockz" title="Ideas, Planning, & Feedback">🤔</a> <a href="https://github.com/epi052/feroxbuster/commits?author=godylockz" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="http://ryanmontgomery.me"><img src="https://avatars.githubusercontent.com/u/44453666?v=4?s=100" width="100px;" alt="Ryan Montgomery"/><br /><sub><b>Ryan Montgomery</b></sub></a><br /><a href="#ideas-0dayCTF" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/IppSec"><img src="https://avatars.githubusercontent.com/u/24677271?v=4?s=100" width="100px;" alt="ippsec"/><br /><sub><b>ippsec</b></sub></a><br /><a href="#ideas-ippsec" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/gtjamesa"><img src="https://avatars.githubusercontent.com/u/2078364?v=4?s=100" width="100px;" alt="James"/><br /><sub><b>James</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3Agtjamesa" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://twitter.com/Jhaddix"><img src="https://avatars.githubusercontent.com/u/3488554?v=4?s=100" width="100px;" alt="Jason Haddix"/><br /><sub><b>Jason Haddix</b></sub></a><br /><a href="#ideas-jhaddix" title="Ideas, Planning, & Feedback">🤔</a> <a href="https://github.com/epi052/feroxbuster/issues?q=author%3Ajhaddix" title="Bug reports">🐛</a></td>
+    </tr>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/ThisLimn0"><img src="https://avatars.githubusercontent.com/u/67125885?v=4?s=100" width="100px;" alt="Limn0"/><br /><sub><b>Limn0</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3AThisLimn0" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/0xdf223"><img src="https://avatars.githubusercontent.com/u/76954092?v=4?s=100" width="100px;" alt="0xdf"/><br /><sub><b>0xdf</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3A0xdf223" title="Bug reports">🐛</a> <a href="#ideas-0xdf223" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/Flangyver"><img src="https://avatars.githubusercontent.com/u/59575870?v=4?s=100" width="100px;" alt="Flangyver"/><br /><sub><b>Flangyver</b></sub></a><br /><a href="#ideas-Flangyver" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/DonatoReis"><img src="https://avatars.githubusercontent.com/u/93531354?v=4?s=100" width="100px;" alt="PeakyBlinder"/><br /><sub><b>PeakyBlinder</b></sub></a><br /><a href="#ideas-DonatoReis" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://postmodern.github.io/"><img src="https://avatars.githubusercontent.com/u/12671?v=4?s=100" width="100px;" alt="Postmodern"/><br /><sub><b>Postmodern</b></sub></a><br /><a href="#ideas-postmodern" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/herrcykel"><img src="https://avatars.githubusercontent.com/u/1936757?v=4?s=100" width="100px;" alt="O"/><br /><sub><b>O</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/commits?author=herrcykel" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="http://udoprog.github.io/"><img src="https://avatars.githubusercontent.com/u/111092?v=4?s=100" width="100px;" alt="John-John Tedro"/><br /><sub><b>John-John Tedro</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/commits?author=udoprog" title="Code">💻</a></td>
+    </tr>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/kmanc"><img src="https://avatars.githubusercontent.com/u/14863147?v=4?s=100" width="100px;" alt="kmanc"/><br /><sub><b>kmanc</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3Akmanc" title="Bug reports">🐛</a> <a href="https://github.com/epi052/feroxbuster/commits?author=kmanc" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/hakdogpinas"><img src="https://avatars.githubusercontent.com/u/71529469?v=4?s=100" width="100px;" alt="hakdogpinas"/><br /><sub><b>hakdogpinas</b></sub></a><br /><a href="#ideas-hakdogpinas" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/duokebei"><img src="https://avatars.githubusercontent.com/u/75022552?v=4?s=100" width="100px;" alt="多可悲"/><br /><sub><b>多可悲</b></sub></a><br /><a href="#ideas-duokebei" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://blog.ah34.net/"><img src="https://avatars.githubusercontent.com/u/58670593?v=4?s=100" width="100px;" alt="Aidan Hall"/><br /><sub><b>Aidan Hall</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/commits?author=aidanhall34" title="Code">💻</a> <a href="#infra-aidanhall34" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://hachyderm.io/@JohnnyCiocca"><img src="https://avatars.githubusercontent.com/u/6473725?v=4?s=100" width="100px;" alt="João Ciocca"/><br /><sub><b>João Ciocca</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3Ajoaociocca" title="Bug reports">🐛</a> <a href="#ideas-joaociocca" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/f3rn0s"><img src="https://avatars.githubusercontent.com/u/1351279?v=4?s=100" width="100px;" alt="f3rn0s"/><br /><sub><b>f3rn0s</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3Af3rn0s" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://sth.sh"><img src="https://avatars.githubusercontent.com/u/2099767?v=4?s=100" width="100px;" alt="LongCat"/><br /><sub><b>LongCat</b></sub></a><br /><a href="#ideas-pich4ya" title="Ideas, Planning, & Feedback">🤔</a></td>
+    </tr>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/xaeroborg"><img src="https://avatars.githubusercontent.com/u/33274680?v=4?s=100" width="100px;" alt="xaeroborg"/><br /><sub><b>xaeroborg</b></sub></a><br /><a href="#ideas-xaeroborg" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/Luoooio"><img src="https://avatars.githubusercontent.com/u/26653157?v=4?s=100" width="100px;" alt="Luoooio"/><br /><sub><b>Luoooio</b></sub></a><br /><a href="#ideas-Luoooio" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://petruknisme.com"><img src="https://avatars.githubusercontent.com/u/6284204?v=4?s=100" width="100px;" alt="Aan"/><br /><sub><b>Aan</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/commits?author=aancw" title="Code">💻</a> <a href="#infra-aancw" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#ideas-aancw" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/imBigo"><img src="https://avatars.githubusercontent.com/u/54672433?v=4?s=100" width="100px;" alt="Simon"/><br /><sub><b>Simon</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3AimBigo" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://acut3.github.io/"><img src="https://avatars.githubusercontent.com/u/17295243?v=4?s=100" width="100px;" alt="Nicolas Christin"/><br /><sub><b>Nicolas Christin</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3Aacut3" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/DrorDvash"><img src="https://avatars.githubusercontent.com/u/8413651?v=4?s=100" width="100px;" alt="DrDv"/><br /><sub><b>DrDv</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3ADrorDvash" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/aroly"><img src="https://avatars.githubusercontent.com/u/1257705?v=4?s=100" width="100px;" alt="Antoine Roly"/><br /><sub><b>Antoine Roly</b></sub></a><br /><a href="#ideas-aroly" title="Ideas, Planning, & Feedback">🤔</a></td>
+    </tr>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="http://lavafroth.is-a.dev"><img src="https://avatars.githubusercontent.com/u/107522312?v=4?s=100" width="100px;" alt="Himadri Bhattacharjee"/><br /><sub><b>Himadri Bhattacharjee</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/commits?author=lavafroth" title="Code">💻</a> <a href="#ideas-lavafroth" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/AkechiShiro"><img src="https://avatars.githubusercontent.com/u/14914796?v=4?s=100" width="100px;" alt="Samy Lahfa"/><br /><sub><b>Samy Lahfa</b></sub></a><br /><a href="#ideas-AkechiShiro" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/sectroyer"><img src="https://avatars.githubusercontent.com/u/6706818?v=4?s=100" width="100px;" alt="sectroyer"/><br /><sub><b>sectroyer</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3Asectroyer" title="Bug reports">🐛</a> <a href="#ideas-sectroyer" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://medium.com/@b3rm1nG"><img src="https://avatars.githubusercontent.com/u/19836003?v=4?s=100" width="100px;" alt="ktecv2000"/><br /><sub><b>ktecv2000</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3Aktecv2000" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="http://untrue.me"><img src="https://avatars.githubusercontent.com/u/56048157?v=4?s=100" width="100px;" alt="Andrea De Murtas"/><br /><sub><b>Andrea De Murtas</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/commits?author=andreademurtas" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/sawmj"><img src="https://avatars.githubusercontent.com/u/30024085?v=4?s=100" width="100px;" alt="sawmj"/><br /><sub><b>sawmj</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3Asawmj" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/devx00"><img src="https://avatars.githubusercontent.com/u/6897405?v=4?s=100" width="100px;" alt="Zach Hanson"/><br /><sub><b>Zach Hanson</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3Adevx00" title="Bug reports">🐛</a></td>
+    </tr>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/ocervell"><img src="https://avatars.githubusercontent.com/u/9629314?v=4?s=100" width="100px;" alt="Olivier Cervello"/><br /><sub><b>Olivier Cervello</b></sub></a><br /><a href="#ideas-ocervell" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/RavySena"><img src="https://avatars.githubusercontent.com/u/67729597?v=4?s=100" width="100px;" alt="RavySena"/><br /><sub><b>RavySena</b></sub></a><br /><a href="#ideas-RavySena" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/stuhlmann"><img src="https://avatars.githubusercontent.com/u/11061864?v=4?s=100" width="100px;" alt="Florian Stuhlmann"/><br /><sub><b>Florian Stuhlmann</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3Astuhlmann" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/Mister7F"><img src="https://avatars.githubusercontent.com/u/35213773?v=4?s=100" width="100px;" alt="Mister7F"/><br /><sub><b>Mister7F</b></sub></a><br /><a href="#ideas-Mister7F" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/manugramm"><img src="https://avatars.githubusercontent.com/u/145961515?v=4?s=100" width="100px;" alt="manugramm"/><br /><sub><b>manugramm</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3Amanugramm" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/ArthurMuraro"><img src="https://avatars.githubusercontent.com/u/73059809?v=4?s=100" width="100px;" alt="ArthurMuraro"/><br /><sub><b>ArthurMuraro</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3AArthurMuraro" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/amiremami"><img src="https://avatars.githubusercontent.com/u/15929497?v=4?s=100" width="100px;" alt="Shadow"/><br /><sub><b>Shadow</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3Aamiremami" title="Bug reports">🐛</a></td>
+    </tr>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/dirhamgithub"><img src="https://avatars.githubusercontent.com/u/115349974?v=4?s=100" width="100px;" alt="dirhamgithub"/><br /><sub><b>dirhamgithub</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3Adirhamgithub" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/FieldOfRice"><img src="https://avatars.githubusercontent.com/u/85353?v=4?s=100" width="100px;" alt="FieldOfRice"/><br /><sub><b>FieldOfRice</b></sub></a><br /><a href="#infra-FieldOfRice" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/NotoriousRebel"><img src="https://avatars.githubusercontent.com/u/36310667?v=4?s=100" width="100px;" alt="Matt"/><br /><sub><b>Matt</b></sub></a><br /><a href="#ideas-NotoriousRebel" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/tritoke"><img src="https://avatars.githubusercontent.com/u/34941249?v=4?s=100" width="100px;" alt="Sam Leonard"/><br /><sub><b>Sam Leonard</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/commits?author=tritoke" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/rew1nter"><img src="https://avatars.githubusercontent.com/u/64508791?v=4?s=100" width="100px;" alt="Rewinter"/><br /><sub><b>Rewinter</b></sub></a><br /><a href="#ideas-rew1nter" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/deadloot"><img src="https://avatars.githubusercontent.com/u/92878901?v=4?s=100" width="100px;" alt="deadloot"/><br /><sub><b>deadloot</b></sub></a><br /><a href="#ideas-deadloot" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/Spidle"><img src="https://avatars.githubusercontent.com/u/90011249?v=4?s=100" width="100px;" alt="Spidle"/><br /><sub><b>Spidle</b></sub></a><br /><a href="#ideas-Spidle" title="Ideas, Planning, & Feedback">🤔</a></td>
+    </tr>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/JulianGR"><img src="https://avatars.githubusercontent.com/u/53094530?v=4?s=100" width="100px;" alt="Julián Gómez"/><br /><sub><b>Julián Gómez</b></sub></a><br /><a href="#ideas-JulianGR" title="Ideas, Planning, & Feedback">🤔</a> <a href="#infra-JulianGR" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="https://github.com/epi052/feroxbuster/commits?author=JulianGR" title="Documentation">📖</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/soutzis"><img src="https://avatars.githubusercontent.com/u/25797286?v=4?s=100" width="100px;" alt="Petros"/><br /><sub><b>Petros</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3Asoutzis" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/sitiom"><img src="https://avatars.githubusercontent.com/u/56180050?v=4?s=100" width="100px;" alt="Ryan"/><br /><sub><b>Ryan</b></sub></a><br /><a href="#infra-sitiom" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="https://github.com/epi052/feroxbuster/commits?author=sitiom" title="Documentation">📖</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/wikamp-collaborator"><img src="https://avatars.githubusercontent.com/u/147445097?v=4?s=100" width="100px;" alt="wikamp-collaborator"/><br /><sub><b>wikamp-collaborator</b></sub></a><br /><a href="#ideas-wikamp-collaborator" title="Ideas, Planning, & Feedback">🤔</a> <a href="#infra-wikamp-collaborator" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="http://lino.codes"><img src="https://avatars.githubusercontent.com/u/123986259?v=4?s=100" width="100px;" alt="Lino"/><br /><sub><b>Lino</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3AL1-0" title="Bug reports">🐛</a> <a href="#ideas-L1-0" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://danthesalmon.com"><img src="https://avatars.githubusercontent.com/u/3712226?v=4?s=100" width="100px;" alt="Dan Salmon"/><br /><sub><b>Dan Salmon</b></sub></a><br /><a href="#ideas-sa7mon" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/swordfish0x0"><img src="https://avatars.githubusercontent.com/u/21209130?v=4?s=100" width="100px;" alt="swordfish0x0"/><br /><sub><b>swordfish0x0</b></sub></a><br /><a href="#ideas-swordfish0x0" title="Ideas, Planning, & Feedback">🤔</a></td>
+    </tr>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/libklein"><img src="https://avatars.githubusercontent.com/u/42714034?v=4?s=100" width="100px;" alt="Patrick Klein"/><br /><sub><b>Patrick Klein</b></sub></a><br /><a href="#ideas-libklein" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/Raymond-JV"><img src="https://avatars.githubusercontent.com/u/23642921?v=4?s=100" width="100px;" alt="Raymond"/><br /><sub><b>Raymond</b></sub></a><br /><a href="#ideas-Raymond-JV" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/zer0x64"><img src="https://avatars.githubusercontent.com/u/17575242?v=4?s=100" width="100px;" alt="zer0x64"/><br /><sub><b>zer0x64</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/commits?author=zer0x64" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://zar3bski.com"><img src="https://avatars.githubusercontent.com/u/22128014?v=4?s=100" width="100px;" alt="zar3bski"/><br /><sub><b>zar3bski</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/commits?author=zar3bski" title="Code">💻</a> <a href="#ideas-zar3bski" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/karanabe"><img src="https://avatars.githubusercontent.com/u/152078880?v=4?s=100" width="100px;" alt="karanabe"/><br /><sub><b>karanabe</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/commits?author=karanabe" title="Documentation">📖</a> <a href="https://github.com/epi052/feroxbuster/commits?author=karanabe" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/h121h"><img src="https://avatars.githubusercontent.com/u/616758?v=4?s=100" width="100px;" alt="h121h"/><br /><sub><b>h121h</b></sub></a><br /><a href="#ideas-h121h" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/s0i37"><img src="https://avatars.githubusercontent.com/u/22872513?v=4?s=100" width="100px;" alt="s0i37"/><br /><sub><b>s0i37</b></sub></a><br /><a href="#ideas-s0i37" title="Ideas, Planning, & Feedback">🤔</a></td>
+    </tr>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/wilco375"><img src="https://avatars.githubusercontent.com/u/7385023?v=4?s=100" width="100px;" alt="Wilco"/><br /><sub><b>Wilco</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3Awilco375" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/HenriBom"><img src="https://avatars.githubusercontent.com/u/46447744?v=4?s=100" width="100px;" alt="HenriBom"/><br /><sub><b>HenriBom</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3AHenriBom" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/0x7274"><img src="https://avatars.githubusercontent.com/u/85586890?v=4?s=100" width="100px;" alt="R̝͖̱͖͕̤̰̯͙ͫ͒̀ͮȁ̤͔̝̘̪̻͕̝̖ͧͪͤu̗̠̜̩̗͇͑̀ͣ̃͂̔͂c̫͔͚̲̬̓̂̿͌̿͊̐͗h͚̲̤̟͓̟̥̊ͬͪ̏̍̍ T̟̜̞͉͙̙ͣ́ͪ͗̓̇ͭo͍̰͎̼͓̟̽ͧ̓̉ͬ̐͐b͇̖̳̫̰̗̭͍ͧ̄̄̌̈i̙̪̤̝̟͓̹̋̽͋̀ͧ̒a͕̭̱͎̪̦̤ͤ͊̊̑ͣ̄s̪̯͖̰̯͍ͫ̋͑̄ͭͅͅ"/><br /><sub><b>R̝͖̱͖͕̤̰̯͙ͫ͒̀ͮȁ̤͔̝̘̪̻͕̝̖ͧͪͤu̗̠̜̩̗͇͑̀ͣ̃͂̔͂c̫͔͚̲̬̓̂̿͌̿͊̐͗h͚̲̤̟͓̟̥̊ͬͪ̏̍̍ T̟̜̞͉͙̙ͣ́ͪ͗̓̇ͭo͍̰͎̼͓̟̽ͧ̓̉ͬ̐͐b͇̖̳̫̰̗̭͍ͧ̄̄̌̈i̙̪̤̝̟͓̹̋̽͋̀ͧ̒a͕̭̱͎̪̦̤ͤ͊̊̑ͣ̄s̪̯͖̰̯͍ͫ̋͑̄ͭͅͅ</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3A0x7274" title="Bug reports">🐛</a> <a href="#ideas-0x7274" title="Ideas, Planning, & Feedback">🤔</a> <a href="https://github.com/epi052/feroxbuster/commits?author=0x7274" title="Documentation">📖</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/4FunAndProfit"><img src="https://avatars.githubusercontent.com/u/174417079?v=4?s=100" width="100px;" alt="4FunAndProfit"/><br /><sub><b>4FunAndProfit</b></sub></a><br /><a href="#ideas-4FunAndProfit" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/lidorelias3"><img src="https://avatars.githubusercontent.com/u/41958137?v=4?s=100" width="100px;" alt="lidorelias3"/><br /><sub><b>lidorelias3</b></sub></a><br /><a href="#ideas-lidorelias3" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="http://adnanullahkhan.com"><img src="https://avatars.githubusercontent.com/u/75381620?v=4?s=100" width="100px;" alt="Adnan Ullah Khan (auk0x01)"/><br /><sub><b>Adnan Ullah Khan (auk0x01)</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/commits?author=auk0x01" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/mzember"><img src="https://avatars.githubusercontent.com/u/61412285?v=4?s=100" width="100px;" alt="Martin Žember"/><br /><sub><b>Martin Žember</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3Amzember" title="Bug reports">🐛</a></td>
+    </tr>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/pg9051"><img src="https://avatars.githubusercontent.com/u/202219877?v=4?s=100" width="100px;" alt="pg9051"/><br /><sub><b>pg9051</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/commits?author=pg9051" title="Documentation">📖</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/sebastiaanspeck"><img src="https://avatars.githubusercontent.com/u/12570668?v=4?s=100" width="100px;" alt="Sebastiaan Speck"/><br /><sub><b>Sebastiaan Speck</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3Asebastiaanspeck" title="Bug reports">🐛</a> <a href="https://github.com/epi052/feroxbuster/commits?author=sebastiaanspeck" title="Documentation">📖</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/OpenSourceKyle"><img src="https://avatars.githubusercontent.com/u/173112933?v=4?s=100" width="100px;" alt="OpenSourceKyle"/><br /><sub><b>OpenSourceKyle</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/commits?author=OpenSourceKyle" title="Documentation">📖</a> <a href="https://github.com/epi052/feroxbuster/issues?q=author%3AOpenSourceKyle" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/Antonio-R1"><img src="https://avatars.githubusercontent.com/u/54741970?v=4?s=100" width="100px;" alt="Antonio"/><br /><sub><b>Antonio</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/commits?author=Antonio-R1" title="Code">💻</a> <a href="https://github.com/epi052/feroxbuster/issues?q=author%3AAntonio-R1" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/redacean"><img src="https://avatars.githubusercontent.com/u/125687454?v=4?s=100" width="100px;" alt="Redacean"/><br /><sub><b>Redacean</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3Aredacean" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/ghsdpolley"><img src="https://avatars.githubusercontent.com/u/19826831?v=4?s=100" width="100px;" alt="ghsdpolley"/><br /><sub><b>ghsdpolley</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3Aghsdpolley" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/aldamd"><img src="https://avatars.githubusercontent.com/u/178115486?v=4?s=100" width="100px;" alt="Daniel Aldam"/><br /><sub><b>Daniel Aldam</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/commits?author=aldamd" title="Code">💻</a></td>
+    </tr>
+  </tbody>
+</table>
 
-Of note, there's another written-in-rust content discovery tool, [rustbuster](https://github.com/phra/rustbuster). I 
-came across rustbuster when I was naming my tool (😢). I don't have any experience using it, but it appears to 
-be able to do POST requests with an HTTP body, has SOCKS support, and has an 8.3 shortname scanner (in addition to vhost
-dns, directory, etc...).  In short, it definitely looks interesting and may be what you're looking for as it has some 
-capability I haven't seen in similar tools.  
+<!-- markdownlint-restore -->
+<!-- prettier-ignore-end -->
+
+<!-- ALL-CONTRIBUTORS-LIST:END -->
+
+This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome!

build.rs

@@ -0,0 +1,80 @@
+use std::fs::{copy, create_dir_all, OpenOptions};
+use std::io::{Read, Seek, Write};
+
+use clap_complete::{generate_to, shells};
+
+include!("src/parser.rs");
+
+fn main() {
+    println!("cargo:rerun-if-env-changed=src/parser.rs");
+
+    if std::env::var("DOCS_RS").is_ok() {
+        return; // only build when we're not generating docs
+    }
+
+    let outdir = "shell_completions";
+
+    let mut app = initialize();
+
+    generate_to(shells::Bash, &mut app, "feroxbuster", outdir).unwrap();
+    generate_to(shells::Zsh, &mut app, "feroxbuster", outdir).unwrap();
+    generate_to(shells::Fish, &mut app, "feroxbuster", outdir).unwrap();
+    generate_to(shells::PowerShell, &mut app, "feroxbuster", outdir).unwrap();
+    generate_to(shells::Elvish, &mut app, "feroxbuster", outdir).unwrap();
+
+    // 0xdf pointed out an oddity when tab-completing options that expect file paths, the fix we
+    // landed on was to add -o plusdirs to the bash completion script. The following code aims to
+    // automate that fix and have it present in all future builds
+    let mut contents = String::new();
+
+    let mut bash_file = OpenOptions::new()
+        .read(true)
+        .write(true)
+        .open(format!("{outdir}/feroxbuster.bash"))
+        .expect("Couldn't open bash completion script");
+
+    bash_file
+        .read_to_string(&mut contents)
+        .expect("Couldn't read bash completion script");
+
+    contents = contents.replace("default feroxbuster", "default -o plusdirs feroxbuster");
+
+    bash_file
+        .rewind()
+        .expect("Couldn't seek to position 0 in bash completion script");
+
+    bash_file
+        .write_all(contents.as_bytes())
+        .expect("Couldn't write updated bash completion script to disk");
+
+    // hunter0x8 let me know that when installing via cargo, it would be nice if we dropped a
+    // config file during the build process. The following code will place an example config in
+    // the user's configuration directory
+    //   - linux: $XDG_CONFIG_HOME or $HOME/.config
+    //   - macOS: $HOME/Library/Application Support
+    //   - windows: {FOLDERID_RoamingAppData}
+    let mut config_dir = dirs::config_dir().expect("Couldn't resolve user's config directory");
+    config_dir = config_dir.join("feroxbuster"); // $HOME/.config/feroxbuster
+
+    if !config_dir.exists() {
+        // recursively create the feroxbuster directory and all of its parent components if
+        // they are missing
+        if create_dir_all(&config_dir).is_err() {
+            // only copy the config file when we're not running in the CI/CD pipeline
+            // which fails with permission denied
+            eprintln!("Couldn't create one or more directories needed to copy the config file");
+            return;
+        }
+    }
+
+    // hard-coding config name here to not rely on the crate we're building, if DEFAULT_CONFIG_NAME
+    // ever changes, this will need to be updated
+    let config_file = config_dir.join("ferox-config.toml");
+
+    if !config_file.exists() {
+        // config file doesn't exist, add it to the config directory
+        if copy("ferox-config.toml.example", config_file).is_err() {
+            eprintln!("Couldn't copy example config into config directory");
+        }
+    }
+}

choco_package/feroxbuster.nuspec

@@ -0,0 +1,79 @@
+<?xml version="1.0" encoding="utf-8"?>
+<package xmlns="http://schemas.microsoft.com/packaging/2015/06/nuspec.xsd">
+  <metadata>
+    <id>feroxbuster</id>
+    <version>2.8.0</version>
+    <packageSourceUrl>https://github.com/epi052/feroxbuster/releases/</packageSourceUrl>
+    <owners>epi052</owners>
+    <title>feroxbuster (Install)</title>
+    <authors>epi052</authors>
+    <projectUrl>https://github.com/epi052/feroxbuster</projectUrl>
+    <iconUrl>https://rawcdn.githack.com/epi052/feroxbuster/2d381e7e057ce60c580b324dd36c9abaf30c2ec7/img/logo/logo.png</iconUrl>
+    <copyright>2020-2023</copyright>
+    <licenseUrl>https://github.com/epi052/feroxbuster/blob/main/LICENSE</licenseUrl>
+    <requireLicenseAcceptance>true</requireLicenseAcceptance>
+    <projectSourceUrl>https://github.com/epi052/feroxbuster</projectSourceUrl>
+    <docsUrl>https://epi052.github.io/feroxbuster-docs/</docsUrl>
+    <!--<mailingListUrl></mailingListUrl>-->
+    <bugTrackerUrl>https://github.com/epi052/feroxbuster/issues</bugTrackerUrl>
+    <tags>content-discovery pentesting-tool url-bruteforcer</tags>
+    <summary>A simple, fast, recursive content discovery tool written in Rust</summary>
+    <description>
+    A simple, fast, recursive content discovery tool written in Rust
+[![Feroxbuster](https://github.com/epi052/feroxbuster/raw/main/img/logo/default-cropped.png)](https://github.com/epi052/feroxbuster)
+
+## What the heck is a ferox anyway?
+
+Ferox is short for Ferric Oxide. Ferric Oxide, simply put, is rust. The name rustbuster was taken, so I decided on a
+variation. 
+
+## What's it do tho?
+
+`feroxbuster` is a tool designed to perform [Forced Browsing](https://owasp.org/www-community/attacks/Forced_browsing).
+
+Forced browsing is an attack where the aim is to enumerate and access resources that are not referenced by the web
+application, but are still accessible by an attacker.
+
+`feroxbuster` uses brute force combined with a wordlist to search for unlinked content in target directories. These
+resources may store sensitive information about web applications and operational systems, such as source code,
+credentials, internal network addressing, etc...
+
+This attack is also known as Predictable Resource Location, File Enumeration, Directory Enumeration, and Resource
+Enumeration.
+
+## Quick Start
+
+This section will cover the minimum amount of information to get up and running with feroxbuster. Please refer the the [documentation](https://epi052.github.io/feroxbuster-docs/overview), as it's much more comprehensive.
+
+### Installation
+
+There are quite a few other [installation methods](https://epi052.github.io/feroxbuster-docs/installation/android/), but these snippets should cover the majority of users. 
+
+#### All others Docs
+
+Please refer the the [documentation](https://epi052.github.io/feroxbuster-docs/overview).
+
+## Example Usage
+
+Here are a few brief examples to get you started.  Please note, feroxbuster can do a **lot more** than what's listed below.  As a result, there are **many more** examples, with **demonstration gifs** that highlight specific features, in the [documentation](https://epi052.github.io/feroxbuster-docs/overview).
+
+### Multiple Values
+
+Options that take multiple values are very flexible. Consider the following ways of specifying extensions:
+
+```
+./feroxbuster -u http://127.1 -x pdf -x js,html -x php txt json,docx
+```
+
+The command above adds .pdf, .js, .html, .php, .txt, .json, and .docx to each url
+
+All of the methods above (multiple flags, space separated, comma separated, etc...) are valid and interchangeable. The
+same goes for urls, headers, status codes, queries, and size filters.
+</description>
+    <!-- <releaseNotes>__REPLACE_OR_REMOVE__MarkDown_Okay</releaseNotes> -->
+  </metadata>
+  <files>
+    <!-- this section controls what actually gets packaged into the Chocolatey package -->
+    <file src="tools\**" target="tools" />
+  </files>
+</package>

choco_package/legal/LICENSE.txt

@@ -0,0 +1,26 @@
+
+From: https://github.com/epi052/feroxbuster/blob/main/LICENSE
+
+LICENSE
+
+MIT License
+
+Copyright (c) 2020-2023 epi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

choco_package/legal/VERIFICATION.txt

@@ -0,0 +1,5 @@
+
+VERIFICATION
+
+checksum -t sha512 -f .\x86-windows-feroxbuster.exe.zip
+checksum -t sha512 -f .\x86_64-windows-feroxbuster.exe.zip

choco_package/tools/chocolateyinstall.ps1

@@ -0,0 +1,27 @@
+$ErrorActionPreference = 'Stop' 
+
+$toolsDir   = "$(Split-Path -parent $MyInvocation.MyCommand.Definition)"
+$version    = '2.8.0'
+$url        = "https://github.com/epi052/feroxbuster/releases/download/v$version/x86-windows-feroxbuster.exe.zip"
+$url64      = "https://github.com/epi052/feroxbuster/releases/download/v$version/x86_64-windows-feroxbuster.exe.zip"
+
+$packageArgs = @{
+  packageName   = $env:ChocolateyPackageName
+  unzipLocation = $toolsDir
+  fileType      = 'exe' #only one of these: exe, msi, msu
+  url           = $url
+  url64bit      = $url64
+  #file         = $fileLocation
+
+  softwareName  = 'feroxbuster*' 
+
+  # Checksums are now required as of 0.10.0.
+  # To determine checksums, you can get that from the original site if provided.
+  # You can also use checksum.exe (choco install checksum) and use it
+  # e.g. checksum -t sha256 -f path\to\file
+  checksum      = 'e5cac59c737260233903a17706a68bac11fe0d7a15169e1c5a9637cc221e7230fd6ddbfc1a7243833dde6472ad053c033449ca8338164654f7354363da54ba88'
+  checksumType  = 'sha512'
+  checksum64    = 'cce58d6eacef7e12c31076f5a00fee9742a4e3fdfc69d807d98736200e50469f77359978e137ecafd87b14460845c65c6808d1f8b23ae561f7e7c637e355dee3'
+  checksumType64= 'sha512'
+}
+Install-ChocolateyZipPackage @packageArgs # https://docs.chocolatey.org/en-us/create/functions/install-chocolateyzippackage

choco_package/tools/chocolateyuninstall.ps1

@@ -0,0 +1,47 @@
+$ErrorActionPreference = 'Stop' # stop on all errors
+$packageArgs = @{
+  packageName   = $env:ChocolateyPackageName
+  softwareName  = 'feroxbuster*'  #part or all of the Display Name as you see it in Programs and Features. It should be enough to be unique
+  fileType      = 'exe' #only one of these: MSI or EXE (ignore MSU for now)
+}
+
+# Get-UninstallRegistryKey is new to 0.9.10, if supporting 0.9.9.x and below,
+# take a dependency on "chocolatey-core.extension" in your nuspec file.
+# This is only a fuzzy search if $softwareName includes '*'. Otherwise it is
+# exact. In the case of versions in key names, we recommend removing the version
+# and using '*'.
+[array]$key = Get-UninstallRegistryKey -SoftwareName $packageArgs['softwareName']
+
+if ($key.Count -eq 1) {
+  $key | % {
+    $packageArgs['file'] = "$($_.UninstallString)" #NOTE: You may need to split this if it contains spaces, see below
+
+    if ($packageArgs['fileType'] -eq 'MSI') {
+      # The Product Code GUID is all that should be passed for MSI, and very
+      # FIRST, because it comes directly after /x, which is already set in the
+      # Uninstall-ChocolateyPackage msiargs (facepalm).
+      $packageArgs['silentArgs'] = "$($_.PSChildName) $($packageArgs['silentArgs'])"
+
+      # Don't pass anything for file, it is ignored for msi (facepalm number 2)
+      # Alternatively if you need to pass a path to an msi, determine that and
+      # use it instead of the above in silentArgs, still very first
+      $packageArgs['file'] = ''
+    } else {
+      # NOTES:
+      # - You probably will need to sanitize $packageArgs['file'] as it comes from the registry and could be in a variety of fun but unusable formats
+      # - Split args from exe in $packageArgs['file'] and pass those args through $packageArgs['silentArgs'] or ignore them
+      # - Ensure you don't pass double quotes in $file (aka $packageArgs['file']) - otherwise you will get "Illegal characters in path when you attempt to run this"
+      # - Review the code for auto-uninstaller for all of the fun things it does in sanitizing - https://github.com/chocolatey/choco/blob/bfe351b7d10c798014efe4bfbb100b171db25099/src/chocolatey/infrastructure.app/services/AutomaticUninstallerService.cs#L142-L192
+    }
+
+    Uninstall-ChocolateyPackage @packageArgs
+  }
+} elseif ($key.Count -eq 0) {
+  Write-Warning "$packageName has already been uninstalled by other means."
+} elseif ($key.Count -gt 1) {
+  Write-Warning "$($key.Count) matches found!"
+  Write-Warning "To prevent accidental data loss, no programs will be uninstalled."
+  Write-Warning "Please alert package maintainer the following keys were matched:"
+  $key | % {Write-Warning "- $($_.DisplayName)"}
+}
+

docs/index.html

@@ -0,0 +1,11 @@
+<html>
+<head>
+   <meta http-equiv="refresh"
+   content="0; url=https://epi052.github.io/feroxbuster-docs/">
+</head>
+<body>
+   <p>The page has moved to:
+   <a href="https://epi052.github.io/feroxbuster-docs/">feroxbuster-docs</a></p>
+</body>
+</html>
+

ferox-config.toml.example

@@ -8,25 +8,65 @@
 # Any setting used here can be overridden by the corresponding command line option/argument
 #
 # wordlist = "/wordlists/seclists/Discovery/Web-Content/raft-medium-directories.txt"
-# statuscodes = [200, 500]
+# status_codes = [200, 500]
+# filter_status = [301]
 # threads = 1
 # timeout = 5
 # proxy = "http://127.0.0.1:8080"
+# replay_proxy = "http://127.0.0.1:8081"
+# replay_codes = [200, 302]
 # verbosity = 1
+# parallel = 8
+# scan_limit = 6
+# rate_limit = 250
 # quiet = true
+# silent = true
+# auto_tune = true
+# auto_bail = true
+# json = true
 # output = "/targets/ellingson_mineral_company/gibson.txt"
-# useragent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0"
+# debug_log = "/var/log/find-the-derp.log"
+# user_agent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0"
+# random_agent = false
 # redirects = true
 # insecure = true
+# collect_words = true
+# collect_backups = true
+# collect_extensions = true
 # extensions = ["php", "html"]
-# norecursion = true
-# addslash = true
+# dont_collect = ["png", "gif", "jpg", "jpeg"]
+# methods = ["GET", "POST"]
+# data = [11, 12, 13, 14, 15]
+# url_denylist = ["http://dont-scan.me", "https://also-not.me"]
+# any subdomain of a domain provided to scope is implicitly allowed also.
+# so things like "api.other.com" and "sub.third.com" would also be considered
+# in-scope given the example config below.
+# scope = ["example.com", "other.com", "third.com"]
+# regex_denylist = ["/deny.*"]
+# no_recursion = true
+# add_slash = true
 # stdin = true
-# dontfilter = true
+# dont_filter = true
 # extract_links = true
 # depth = 1
-# sizefilters = [5174]
+# limit_bars = 3
+# force_recursion = true
+# filter_size = [5174]
+# filter_regex = ["^ignore me$"]
+# filter_similar = ["https://somesite.com/soft404"]
+# filter_word_count = [993]
+# filter_line_count = [35, 36]
 # queries = [["name","value"], ["rick", "astley"]]
+# save_state = false
+# time_limit = "10m"
+# server_certs = ["/some/cert.pem", "/some/other/cert.pem"]
+# client_cert = "/some/client/cert.pem"
+# client_key = "/some/client/key.pem"
+# request_file = "/some/raw/request/file"
+# protocol = "http"
+# scan_dir_listings = true
+# unique = true
+# response_size_limit = 4194304
 
 # headers can be specified on multiple lines or as an inline table
 #
@@ -37,6 +77,7 @@
 #   note: if multi-line is used, all key/value pairs under it belong to the headers table until the next table
 #         is found or the end of the file is reached
 #
+# If you want to use [headers], UNCOMMENT the line below
 # [headers]
 # stuff = "things"
 # more = "headers"

install-nix.sh

@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+
+BASE_URL=https://github.com/epi052/feroxbuster/releases/latest/download
+
+MAC_ZIP=x86_64-macos-feroxbuster.zip
+MAC_URL="$BASE_URL/$MAC_ZIP"
+
+LIN32_ZIP=x86-linux-feroxbuster.zip
+LIN32_URL="$BASE_URL/$LIN32_ZIP"
+
+LIN64_ZIP=x86_64-linux-feroxbuster.zip
+LIN64_URL="$BASE_URL/$LIN64_ZIP"
+
+EMOJI_URL=https://gist.github.com/epi052/8196b550ea51d0907ad4b93751b1b57d/raw/6112c9f32ae07922983fdc549c54fd3fb9a38e4c/NotoColorEmoji.ttf
+
+INSTALL_DIR="${1:-$(pwd)}"
+
+echo "[+] Installing feroxbuster to ${INSTALL_DIR}!"
+
+which unzip &>/dev/null
+if [ "$?" != "0" ]; then
+  echo "[!] unzip not found, exiting. "
+  exit -1
+fi
+
+if [[ "$(uname)" == "Darwin" ]]; then
+  echo "[=] Found MacOS, downloading from $MAC_URL"
+
+  curl -sLO "$MAC_URL"
+  unzip -o "$MAC_ZIP" -d "${INSTALL_DIR}" >/dev/null
+  rm "$MAC_ZIP"
+elif [[ "$(expr substr $(uname -s) 1 5)" == "Linux" ]]; then
+  if [[ $(getconf LONG_BIT) == 32 ]]; then
+    echo "[=] Found 32-bit Linux, downloading from $LIN32_URL"
+
+    curl -sLO "$LIN32_URL"
+    unzip -o "$LIN32_ZIP" -d "${INSTALL_DIR}" >/dev/null
+    rm "$LIN32_ZIP"
+  else
+    echo "[=] Found 64-bit Linux, downloading from $LIN64_URL"
+
+    curl -sLO "$LIN64_URL"
+    unzip -o "$LIN64_ZIP" -d "${INSTALL_DIR}" >/dev/null
+    rm "$LIN64_ZIP"
+  fi
+
+  if [[ "$(fc-list NotoColorEmoji | wc -l)" -gt 0 ]]; then
+    echo "[=] Found Noto Emoji Font, skipping install"
+  else
+    echo "[=] Installing Noto Emoji Font"
+    mkdir -p ~/.fonts
+    pushd ~/.fonts 2>&1 >/dev/null
+
+    curl -sLO "$EMOJI_URL"
+
+    fc-cache -f -v >/dev/null
+
+    popd 2>&1 >/dev/null
+    echo "[+] Noto Emoji Font installed"
+  fi
+fi
+
+chmod +x "${INSTALL_DIR}/feroxbuster"
+
+echo "[+] Installed feroxbuster"
+echo "  [-] path: ${INSTALL_DIR}/feroxbuster"
+echo "  [-] version: $(${INSTALL_DIR}/feroxbuster -V | awk '{print $2}')"

shell_completions/_feroxbuster

@@ -0,0 +1,139 @@
+#compdef feroxbuster
+
+autoload -U is-at-least
+
+_feroxbuster() {
+    typeset -A opt_args
+    typeset -a _arguments_options
+    local ret=1
+
+    if is-at-least 5.2; then
+        _arguments_options=(-s -S -C)
+    else
+        _arguments_options=(-s -C)
+    fi
+
+    local context curcontext="$curcontext" state line
+    _arguments "${_arguments_options[@]}" : \
+'-u+[The target URL (required, unless \[--stdin || --resume-from || --request-file\] used)]:URL:_urls' \
+'--url=[The target URL (required, unless \[--stdin || --resume-from || --request-file\] used)]:URL:_urls' \
+'(-u --url)--resume-from=[State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)]:STATE_FILE:_files' \
+'(-u --url)--request-file=[Raw HTTP request file to use as a template for all requests]:REQUEST_FILE:_files' \
+'(--data --data-json)--data-urlencoded=[Set -H '\''Content-Type\: application/x-www-form-urlencoded'\'', --data to <data-urlencoded> (supports @file) and -m to POST]:DATA:_default' \
+'(--data --data-urlencoded)--data-json=[Set -H '\''Content-Type\: application/json'\'', --data to <data-json> (supports @file) and -m to POST]:DATA:_default' \
+'-p+[Proxy to use for requests (ex\: http(s)\://host\:port, socks5(h)\://host\:port)]:PROXY:_urls' \
+'--proxy=[Proxy to use for requests (ex\: http(s)\://host\:port, socks5(h)\://host\:port)]:PROXY:_urls' \
+'-P+[Send only unfiltered requests through a Replay Proxy, instead of all requests]:REPLAY_PROXY:_urls' \
+'--replay-proxy=[Send only unfiltered requests through a Replay Proxy, instead of all requests]:REPLAY_PROXY:_urls' \
+'*-R+[Status Codes to send through a Replay Proxy when found (default\: --status-codes value)]:REPLAY_CODE:_default' \
+'*--replay-codes=[Status Codes to send through a Replay Proxy when found (default\: --status-codes value)]:REPLAY_CODE:_default' \
+'-a+[Sets the User-Agent (default\: feroxbuster/2.13.1)]:USER_AGENT:_default' \
+'--user-agent=[Sets the User-Agent (default\: feroxbuster/2.13.1)]:USER_AGENT:_default' \
+'*-x+[File extension(s) to search for (ex\: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex\: @ext.txt)]:FILE_EXTENSION:_default' \
+'*--extensions=[File extension(s) to search for (ex\: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex\: @ext.txt)]:FILE_EXTENSION:_default' \
+'*-m+[Which HTTP request method(s) should be sent (default\: GET)]:HTTP_METHODS:_default' \
+'*--methods=[Which HTTP request method(s) should be sent (default\: GET)]:HTTP_METHODS:_default' \
+'--data=[Request'\''s Body; can read data from a file if input starts with an @ (ex\: @post.bin)]:DATA:_default' \
+'*-H+[Specify HTTP headers to be used in each request (ex\: -H Header\:val -H '\''stuff\: things'\'')]:HEADER:_default' \
+'*--headers=[Specify HTTP headers to be used in each request (ex\: -H Header\:val -H '\''stuff\: things'\'')]:HEADER:_default' \
+'*-b+[Specify HTTP cookies to be used in each request (ex\: -b stuff=things)]:COOKIE:_default' \
+'*--cookies=[Specify HTTP cookies to be used in each request (ex\: -b stuff=things)]:COOKIE:_default' \
+'*-Q+[Request'\''s URL query parameters (ex\: -Q token=stuff -Q secret=key)]:QUERY:_default' \
+'*--query=[Request'\''s URL query parameters (ex\: -Q token=stuff -Q secret=key)]:QUERY:_default' \
+'--protocol=[Specify the protocol to use when targeting via --request-file or --url with domain only (default\: https)]:PROTOCOL:_default' \
+'*--dont-scan=[URL(s) or Regex Pattern(s) to exclude from recursion/scans]:URL:_default' \
+'*--scope=[Additional domains/URLs to consider in-scope for scanning (in addition to current domain)]:URL:_default' \
+'*-S+[Filter out messages of a particular size (ex\: -S 5120 -S 4927,1970)]:SIZE:_default' \
+'*--filter-size=[Filter out messages of a particular size (ex\: -S 5120 -S 4927,1970)]:SIZE:_default' \
+'*-X+[Filter out messages via regular expression matching on the response'\''s body/headers (ex\: -X '\''^ignore me\$'\'')]:REGEX:_default' \
+'*--filter-regex=[Filter out messages via regular expression matching on the response'\''s body/headers (ex\: -X '\''^ignore me\$'\'')]:REGEX:_default' \
+'*-W+[Filter out messages of a particular word count (ex\: -W 312 -W 91,82)]:WORDS:_default' \
+'*--filter-words=[Filter out messages of a particular word count (ex\: -W 312 -W 91,82)]:WORDS:_default' \
+'*-N+[Filter out messages of a particular line count (ex\: -N 20 -N 31,30)]:LINES:_default' \
+'*--filter-lines=[Filter out messages of a particular line count (ex\: -N 20 -N 31,30)]:LINES:_default' \
+'(-s --status-codes)*-C+[Filter out status codes (deny list) (ex\: -C 200 -C 401)]:STATUS_CODE:_default' \
+'(-s --status-codes)*--filter-status=[Filter out status codes (deny list) (ex\: -C 200 -C 401)]:STATUS_CODE:_default' \
+'*--filter-similar-to=[Filter out pages that are similar to the given page (ex. --filter-similar-to http\://site.xyz/soft404)]:UNWANTED_PAGE:_urls' \
+'*-s+[Status Codes to include (allow list) (default\: All Status Codes)]:STATUS_CODE:_default' \
+'*--status-codes=[Status Codes to include (allow list) (default\: All Status Codes)]:STATUS_CODE:_default' \
+'-T+[Number of seconds before a client'\''s request times out (default\: 7)]:SECONDS:_default' \
+'--timeout=[Number of seconds before a client'\''s request times out (default\: 7)]:SECONDS:_default' \
+'--server-certs=[Add custom root certificate(s) for servers with unknown certificates]:PEM|DER:_files' \
+'--client-cert=[Add a PEM encoded certificate for mutual authentication (mTLS)]:PEM:_files' \
+'--client-key=[Add a PEM encoded private key for mutual authentication (mTLS)]:PEM:_files' \
+'-t+[Number of concurrent threads (default\: 50)]:THREADS:_default' \
+'--threads=[Number of concurrent threads (default\: 50)]:THREADS:_default' \
+'-d+[Maximum recursion depth, a depth of 0 is infinite recursion (default\: 4)]:RECURSION_DEPTH:_default' \
+'--depth=[Maximum recursion depth, a depth of 0 is infinite recursion (default\: 4)]:RECURSION_DEPTH:_default' \
+'-L+[Limit total number of concurrent scans (default\: 0, i.e. no limit)]:SCAN_LIMIT:_default' \
+'--scan-limit=[Limit total number of concurrent scans (default\: 0, i.e. no limit)]:SCAN_LIMIT:_default' \
+'(-v --verbosity -u --url)--parallel=[Run parallel feroxbuster instances (one child process per url passed via stdin)]:PARALLEL_SCANS:_default' \
+'--rate-limit=[Limit number of requests per second (per directory) (default\: 0, i.e. no limit)]:RATE_LIMIT:_default' \
+'--response-size-limit=[Limit size of response body to read in bytes (default\: 4MB)]:BYTES:_default' \
+'--time-limit=[Limit total run time of all scans (ex\: --time-limit 10m)]:TIME_SPEC:_default' \
+'-w+[Path or URL of the wordlist]:FILE:_files' \
+'--wordlist=[Path or URL of the wordlist]:FILE:_files' \
+'-B+[Automatically request likely backup extensions for "found" urls (default\: ~, .bak, .bak2, .old, .1)]' \
+'--collect-backups=[Automatically request likely backup extensions for "found" urls (default\: ~, .bak, .bak2, .old, .1)]' \
+'*-I+[File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)]:FILE_EXTENSION:_default' \
+'*--dont-collect=[File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)]:FILE_EXTENSION:_default' \
+'-o+[Output file to write results to (use w/ --json for JSON entries)]:FILE:_files' \
+'--output=[Output file to write results to (use w/ --json for JSON entries)]:FILE:_files' \
+'--debug-log=[Output file to write log entries (use w/ --json for JSON entries)]:FILE:_files' \
+'--limit-bars=[Number of directory scan bars to show at any given time (default\: no limit)]:NUM_BARS_TO_SHOW:_default' \
+'(-u --url)--stdin[Read url(s) from STDIN]' \
+'(-p --proxy -k --insecure --burp-replay)--burp[Set --proxy to http\://127.0.0.1\:8080 and set --insecure to true]' \
+'(-P --replay-proxy -k --insecure)--burp-replay[Set --replay-proxy to http\://127.0.0.1\:8080 and set --insecure to true]' \
+'(--rate-limit --auto-bail)--smart[Set --auto-tune, --collect-words, and --collect-backups to true]' \
+'(--rate-limit --auto-bail)--thorough[Use the same settings as --smart and set --collect-extensions and --scan-dir-listings to true]' \
+'-A[Use a random User-Agent]' \
+'--random-agent[Use a random User-Agent]' \
+'-f[Append / to each request'\''s URL]' \
+'--add-slash[Append / to each request'\''s URL]' \
+'--unique[Only show unique responses]' \
+'-r[Allow client to follow redirects]' \
+'--redirects[Allow client to follow redirects]' \
+'-k[Disables TLS certificate validation in the client]' \
+'--insecure[Disables TLS certificate validation in the client]' \
+'-n[Do not scan recursively]' \
+'--no-recursion[Do not scan recursively]' \
+'(-n --no-recursion)--force-recursion[Force recursion attempts on all '\''found'\'' endpoints (still respects recursion depth)]' \
+'-e[Extract links from response body (html, javascript, etc...); make new requests based on findings (default\: true)]' \
+'--extract-links[Extract links from response body (html, javascript, etc...); make new requests based on findings (default\: true)]' \
+'--dont-extract-links[Don'\''t extract links from response body (html, javascript, etc...)]' \
+'(--auto-bail)--auto-tune[Automatically lower scan rate when an excessive amount of errors are encountered]' \
+'--auto-bail[Automatically stop scanning when an excessive amount of errors are encountered]' \
+'-D[Don'\''t auto-filter wildcard responses]' \
+'--dont-filter[Don'\''t auto-filter wildcard responses]' \
+'-E[Automatically discover extensions and add them to --extensions (unless they'\''re in --dont-collect)]' \
+'--collect-extensions[Automatically discover extensions and add them to --extensions (unless they'\''re in --dont-collect)]' \
+'-g[Automatically discover important words from within responses and add them to the wordlist]' \
+'--collect-words[Automatically discover important words from within responses and add them to the wordlist]' \
+'--scan-dir-listings[Force scans to recurse into directory listings]' \
+'(--silent)*-v[Increase verbosity level (use -vv or more for greater effect. \[CAUTION\] 4 -v'\''s is probably too much)]' \
+'(--silent)*--verbosity[Increase verbosity level (use -vv or more for greater effect. \[CAUTION\] 4 -v'\''s is probably too much)]' \
+'(-q --quiet)--silent[Only print URLs (or JSON w/ --json) + turn off logging (good for piping a list of urls to other commands)]' \
+'-q[Hide progress bars and banner (good for tmux windows w/ notifications)]' \
+'--quiet[Hide progress bars and banner (good for tmux windows w/ notifications)]' \
+'--json[Emit JSON logs to --output and --debug-log instead of normal text]' \
+'--no-state[Disable state output file (*.state)]' \
+'-U[Update feroxbuster to the latest version]' \
+'--update[Update feroxbuster to the latest version]' \
+'-h[Print help (see more with '\''--help'\'')]' \
+'--help[Print help (see more with '\''--help'\'')]' \
+'-V[Print version]' \
+'--version[Print version]' \
+&& ret=0
+}
+
+(( $+functions[_feroxbuster_commands] )) ||
+_feroxbuster_commands() {
+    local commands; commands=()
+    _describe -t commands 'feroxbuster commands' commands "$@"
+}
+
+if [ "$funcstack[1]" = "_feroxbuster" ]; then
+    _feroxbuster "$@"
+else
+    compdef _feroxbuster feroxbuster
+fi

shell_completions/_feroxbuster.ps1

@@ -0,0 +1,138 @@
+
+using namespace System.Management.Automation
+using namespace System.Management.Automation.Language
+
+Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
+    param($wordToComplete, $commandAst, $cursorPosition)
+
+    $commandElements = $commandAst.CommandElements
+    $command = @(
+        'feroxbuster'
+        for ($i = 1; $i -lt $commandElements.Count; $i++) {
+            $element = $commandElements[$i]
+            if ($element -isnot [StringConstantExpressionAst] -or
+                $element.StringConstantType -ne [StringConstantType]::BareWord -or
+                $element.Value.StartsWith('-') -or
+                $element.Value -eq $wordToComplete) {
+                break
+        }
+        $element.Value
+    }) -join ';'
+
+    $completions = @(switch ($command) {
+        'feroxbuster' {
+            [CompletionResult]::new('-u', '-u', [CompletionResultType]::ParameterName, 'The target URL (required, unless [--stdin || --resume-from || --request-file] used)')
+            [CompletionResult]::new('--url', '--url', [CompletionResultType]::ParameterName, 'The target URL (required, unless [--stdin || --resume-from || --request-file] used)')
+            [CompletionResult]::new('--resume-from', '--resume-from', [CompletionResultType]::ParameterName, 'State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)')
+            [CompletionResult]::new('--request-file', '--request-file', [CompletionResultType]::ParameterName, 'Raw HTTP request file to use as a template for all requests')
+            [CompletionResult]::new('--data-urlencoded', '--data-urlencoded', [CompletionResultType]::ParameterName, 'Set -H ''Content-Type: application/x-www-form-urlencoded'', --data to <data-urlencoded> (supports @file) and -m to POST')
+            [CompletionResult]::new('--data-json', '--data-json', [CompletionResultType]::ParameterName, 'Set -H ''Content-Type: application/json'', --data to <data-json> (supports @file) and -m to POST')
+            [CompletionResult]::new('-p', '-p', [CompletionResultType]::ParameterName, 'Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)')
+            [CompletionResult]::new('--proxy', '--proxy', [CompletionResultType]::ParameterName, 'Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)')
+            [CompletionResult]::new('-P', '-P ', [CompletionResultType]::ParameterName, 'Send only unfiltered requests through a Replay Proxy, instead of all requests')
+            [CompletionResult]::new('--replay-proxy', '--replay-proxy', [CompletionResultType]::ParameterName, 'Send only unfiltered requests through a Replay Proxy, instead of all requests')
+            [CompletionResult]::new('-R', '-R ', [CompletionResultType]::ParameterName, 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)')
+            [CompletionResult]::new('--replay-codes', '--replay-codes', [CompletionResultType]::ParameterName, 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)')
+            [CompletionResult]::new('-a', '-a', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.13.1)')
+            [CompletionResult]::new('--user-agent', '--user-agent', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.13.1)')
+            [CompletionResult]::new('-x', '-x', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex: @ext.txt)')
+            [CompletionResult]::new('--extensions', '--extensions', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex: @ext.txt)')
+            [CompletionResult]::new('-m', '-m', [CompletionResultType]::ParameterName, 'Which HTTP request method(s) should be sent (default: GET)')
+            [CompletionResult]::new('--methods', '--methods', [CompletionResultType]::ParameterName, 'Which HTTP request method(s) should be sent (default: GET)')
+            [CompletionResult]::new('--data', '--data', [CompletionResultType]::ParameterName, 'Request''s Body; can read data from a file if input starts with an @ (ex: @post.bin)')
+            [CompletionResult]::new('-H', '-H ', [CompletionResultType]::ParameterName, 'Specify HTTP headers to be used in each request (ex: -H Header:val -H ''stuff: things'')')
+            [CompletionResult]::new('--headers', '--headers', [CompletionResultType]::ParameterName, 'Specify HTTP headers to be used in each request (ex: -H Header:val -H ''stuff: things'')')
+            [CompletionResult]::new('-b', '-b', [CompletionResultType]::ParameterName, 'Specify HTTP cookies to be used in each request (ex: -b stuff=things)')
+            [CompletionResult]::new('--cookies', '--cookies', [CompletionResultType]::ParameterName, 'Specify HTTP cookies to be used in each request (ex: -b stuff=things)')
+            [CompletionResult]::new('-Q', '-Q ', [CompletionResultType]::ParameterName, 'Request''s URL query parameters (ex: -Q token=stuff -Q secret=key)')
+            [CompletionResult]::new('--query', '--query', [CompletionResultType]::ParameterName, 'Request''s URL query parameters (ex: -Q token=stuff -Q secret=key)')
+            [CompletionResult]::new('--protocol', '--protocol', [CompletionResultType]::ParameterName, 'Specify the protocol to use when targeting via --request-file or --url with domain only (default: https)')
+            [CompletionResult]::new('--dont-scan', '--dont-scan', [CompletionResultType]::ParameterName, 'URL(s) or Regex Pattern(s) to exclude from recursion/scans')
+            [CompletionResult]::new('--scope', '--scope', [CompletionResultType]::ParameterName, 'Additional domains/URLs to consider in-scope for scanning (in addition to current domain)')
+            [CompletionResult]::new('-S', '-S ', [CompletionResultType]::ParameterName, 'Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)')
+            [CompletionResult]::new('--filter-size', '--filter-size', [CompletionResultType]::ParameterName, 'Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)')
+            [CompletionResult]::new('-X', '-X ', [CompletionResultType]::ParameterName, 'Filter out messages via regular expression matching on the response''s body/headers (ex: -X ''^ignore me$'')')
+            [CompletionResult]::new('--filter-regex', '--filter-regex', [CompletionResultType]::ParameterName, 'Filter out messages via regular expression matching on the response''s body/headers (ex: -X ''^ignore me$'')')
+            [CompletionResult]::new('-W', '-W ', [CompletionResultType]::ParameterName, 'Filter out messages of a particular word count (ex: -W 312 -W 91,82)')
+            [CompletionResult]::new('--filter-words', '--filter-words', [CompletionResultType]::ParameterName, 'Filter out messages of a particular word count (ex: -W 312 -W 91,82)')
+            [CompletionResult]::new('-N', '-N ', [CompletionResultType]::ParameterName, 'Filter out messages of a particular line count (ex: -N 20 -N 31,30)')
+            [CompletionResult]::new('--filter-lines', '--filter-lines', [CompletionResultType]::ParameterName, 'Filter out messages of a particular line count (ex: -N 20 -N 31,30)')
+            [CompletionResult]::new('-C', '-C ', [CompletionResultType]::ParameterName, 'Filter out status codes (deny list) (ex: -C 200 -C 401)')
+            [CompletionResult]::new('--filter-status', '--filter-status', [CompletionResultType]::ParameterName, 'Filter out status codes (deny list) (ex: -C 200 -C 401)')
+            [CompletionResult]::new('--filter-similar-to', '--filter-similar-to', [CompletionResultType]::ParameterName, 'Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)')
+            [CompletionResult]::new('-s', '-s', [CompletionResultType]::ParameterName, 'Status Codes to include (allow list) (default: All Status Codes)')
+            [CompletionResult]::new('--status-codes', '--status-codes', [CompletionResultType]::ParameterName, 'Status Codes to include (allow list) (default: All Status Codes)')
+            [CompletionResult]::new('-T', '-T ', [CompletionResultType]::ParameterName, 'Number of seconds before a client''s request times out (default: 7)')
+            [CompletionResult]::new('--timeout', '--timeout', [CompletionResultType]::ParameterName, 'Number of seconds before a client''s request times out (default: 7)')
+            [CompletionResult]::new('--server-certs', '--server-certs', [CompletionResultType]::ParameterName, 'Add custom root certificate(s) for servers with unknown certificates')
+            [CompletionResult]::new('--client-cert', '--client-cert', [CompletionResultType]::ParameterName, 'Add a PEM encoded certificate for mutual authentication (mTLS)')
+            [CompletionResult]::new('--client-key', '--client-key', [CompletionResultType]::ParameterName, 'Add a PEM encoded private key for mutual authentication (mTLS)')
+            [CompletionResult]::new('-t', '-t', [CompletionResultType]::ParameterName, 'Number of concurrent threads (default: 50)')
+            [CompletionResult]::new('--threads', '--threads', [CompletionResultType]::ParameterName, 'Number of concurrent threads (default: 50)')
+            [CompletionResult]::new('-d', '-d', [CompletionResultType]::ParameterName, 'Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)')
+            [CompletionResult]::new('--depth', '--depth', [CompletionResultType]::ParameterName, 'Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)')
+            [CompletionResult]::new('-L', '-L ', [CompletionResultType]::ParameterName, 'Limit total number of concurrent scans (default: 0, i.e. no limit)')
+            [CompletionResult]::new('--scan-limit', '--scan-limit', [CompletionResultType]::ParameterName, 'Limit total number of concurrent scans (default: 0, i.e. no limit)')
+            [CompletionResult]::new('--parallel', '--parallel', [CompletionResultType]::ParameterName, 'Run parallel feroxbuster instances (one child process per url passed via stdin)')
+            [CompletionResult]::new('--rate-limit', '--rate-limit', [CompletionResultType]::ParameterName, 'Limit number of requests per second (per directory) (default: 0, i.e. no limit)')
+            [CompletionResult]::new('--response-size-limit', '--response-size-limit', [CompletionResultType]::ParameterName, 'Limit size of response body to read in bytes (default: 4MB)')
+            [CompletionResult]::new('--time-limit', '--time-limit', [CompletionResultType]::ParameterName, 'Limit total run time of all scans (ex: --time-limit 10m)')
+            [CompletionResult]::new('-w', '-w', [CompletionResultType]::ParameterName, 'Path or URL of the wordlist')
+            [CompletionResult]::new('--wordlist', '--wordlist', [CompletionResultType]::ParameterName, 'Path or URL of the wordlist')
+            [CompletionResult]::new('-B', '-B ', [CompletionResultType]::ParameterName, 'Automatically request likely backup extensions for "found" urls (default: ~, .bak, .bak2, .old, .1)')
+            [CompletionResult]::new('--collect-backups', '--collect-backups', [CompletionResultType]::ParameterName, 'Automatically request likely backup extensions for "found" urls (default: ~, .bak, .bak2, .old, .1)')
+            [CompletionResult]::new('-I', '-I ', [CompletionResultType]::ParameterName, 'File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)')
+            [CompletionResult]::new('--dont-collect', '--dont-collect', [CompletionResultType]::ParameterName, 'File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)')
+            [CompletionResult]::new('-o', '-o', [CompletionResultType]::ParameterName, 'Output file to write results to (use w/ --json for JSON entries)')
+            [CompletionResult]::new('--output', '--output', [CompletionResultType]::ParameterName, 'Output file to write results to (use w/ --json for JSON entries)')
+            [CompletionResult]::new('--debug-log', '--debug-log', [CompletionResultType]::ParameterName, 'Output file to write log entries (use w/ --json for JSON entries)')
+            [CompletionResult]::new('--limit-bars', '--limit-bars', [CompletionResultType]::ParameterName, 'Number of directory scan bars to show at any given time (default: no limit)')
+            [CompletionResult]::new('--stdin', '--stdin', [CompletionResultType]::ParameterName, 'Read url(s) from STDIN')
+            [CompletionResult]::new('--burp', '--burp', [CompletionResultType]::ParameterName, 'Set --proxy to http://127.0.0.1:8080 and set --insecure to true')
+            [CompletionResult]::new('--burp-replay', '--burp-replay', [CompletionResultType]::ParameterName, 'Set --replay-proxy to http://127.0.0.1:8080 and set --insecure to true')
+            [CompletionResult]::new('--smart', '--smart', [CompletionResultType]::ParameterName, 'Set --auto-tune, --collect-words, and --collect-backups to true')
+            [CompletionResult]::new('--thorough', '--thorough', [CompletionResultType]::ParameterName, 'Use the same settings as --smart and set --collect-extensions and --scan-dir-listings to true')
+            [CompletionResult]::new('-A', '-A ', [CompletionResultType]::ParameterName, 'Use a random User-Agent')
+            [CompletionResult]::new('--random-agent', '--random-agent', [CompletionResultType]::ParameterName, 'Use a random User-Agent')
+            [CompletionResult]::new('-f', '-f', [CompletionResultType]::ParameterName, 'Append / to each request''s URL')
+            [CompletionResult]::new('--add-slash', '--add-slash', [CompletionResultType]::ParameterName, 'Append / to each request''s URL')
+            [CompletionResult]::new('--unique', '--unique', [CompletionResultType]::ParameterName, 'Only show unique responses')
+            [CompletionResult]::new('-r', '-r', [CompletionResultType]::ParameterName, 'Allow client to follow redirects')
+            [CompletionResult]::new('--redirects', '--redirects', [CompletionResultType]::ParameterName, 'Allow client to follow redirects')
+            [CompletionResult]::new('-k', '-k', [CompletionResultType]::ParameterName, 'Disables TLS certificate validation in the client')
+            [CompletionResult]::new('--insecure', '--insecure', [CompletionResultType]::ParameterName, 'Disables TLS certificate validation in the client')
+            [CompletionResult]::new('-n', '-n', [CompletionResultType]::ParameterName, 'Do not scan recursively')
+            [CompletionResult]::new('--no-recursion', '--no-recursion', [CompletionResultType]::ParameterName, 'Do not scan recursively')
+            [CompletionResult]::new('--force-recursion', '--force-recursion', [CompletionResultType]::ParameterName, 'Force recursion attempts on all ''found'' endpoints (still respects recursion depth)')
+            [CompletionResult]::new('-e', '-e', [CompletionResultType]::ParameterName, 'Extract links from response body (html, javascript, etc...); make new requests based on findings (default: true)')
+            [CompletionResult]::new('--extract-links', '--extract-links', [CompletionResultType]::ParameterName, 'Extract links from response body (html, javascript, etc...); make new requests based on findings (default: true)')
+            [CompletionResult]::new('--dont-extract-links', '--dont-extract-links', [CompletionResultType]::ParameterName, 'Don''t extract links from response body (html, javascript, etc...)')
+            [CompletionResult]::new('--auto-tune', '--auto-tune', [CompletionResultType]::ParameterName, 'Automatically lower scan rate when an excessive amount of errors are encountered')
+            [CompletionResult]::new('--auto-bail', '--auto-bail', [CompletionResultType]::ParameterName, 'Automatically stop scanning when an excessive amount of errors are encountered')
+            [CompletionResult]::new('-D', '-D ', [CompletionResultType]::ParameterName, 'Don''t auto-filter wildcard responses')
+            [CompletionResult]::new('--dont-filter', '--dont-filter', [CompletionResultType]::ParameterName, 'Don''t auto-filter wildcard responses')
+            [CompletionResult]::new('-E', '-E ', [CompletionResultType]::ParameterName, 'Automatically discover extensions and add them to --extensions (unless they''re in --dont-collect)')
+            [CompletionResult]::new('--collect-extensions', '--collect-extensions', [CompletionResultType]::ParameterName, 'Automatically discover extensions and add them to --extensions (unless they''re in --dont-collect)')
+            [CompletionResult]::new('-g', '-g', [CompletionResultType]::ParameterName, 'Automatically discover important words from within responses and add them to the wordlist')
+            [CompletionResult]::new('--collect-words', '--collect-words', [CompletionResultType]::ParameterName, 'Automatically discover important words from within responses and add them to the wordlist')
+            [CompletionResult]::new('--scan-dir-listings', '--scan-dir-listings', [CompletionResultType]::ParameterName, 'Force scans to recurse into directory listings')
+            [CompletionResult]::new('-v', '-v', [CompletionResultType]::ParameterName, 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v''s is probably too much)')
+            [CompletionResult]::new('--verbosity', '--verbosity', [CompletionResultType]::ParameterName, 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v''s is probably too much)')
+            [CompletionResult]::new('--silent', '--silent', [CompletionResultType]::ParameterName, 'Only print URLs (or JSON w/ --json) + turn off logging (good for piping a list of urls to other commands)')
+            [CompletionResult]::new('-q', '-q', [CompletionResultType]::ParameterName, 'Hide progress bars and banner (good for tmux windows w/ notifications)')
+            [CompletionResult]::new('--quiet', '--quiet', [CompletionResultType]::ParameterName, 'Hide progress bars and banner (good for tmux windows w/ notifications)')
+            [CompletionResult]::new('--json', '--json', [CompletionResultType]::ParameterName, 'Emit JSON logs to --output and --debug-log instead of normal text')
+            [CompletionResult]::new('--no-state', '--no-state', [CompletionResultType]::ParameterName, 'Disable state output file (*.state)')
+            [CompletionResult]::new('-U', '-U ', [CompletionResultType]::ParameterName, 'Update feroxbuster to the latest version')
+            [CompletionResult]::new('--update', '--update', [CompletionResultType]::ParameterName, 'Update feroxbuster to the latest version')
+            [CompletionResult]::new('-h', '-h', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
+            [CompletionResult]::new('--help', '--help', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
+            [CompletionResult]::new('-V', '-V ', [CompletionResultType]::ParameterName, 'Print version')
+            [CompletionResult]::new('--version', '--version', [CompletionResultType]::ParameterName, 'Print version')
+            break
+        }
+    })
+
+    $completions.Where{ $_.CompletionText -like "$wordToComplete*" } |
+        Sort-Object -Property ListItemText
+}

shell_completions/feroxbuster.bash

@@ -0,0 +1,420 @@
+_feroxbuster() {
+    local i cur prev opts cmd
+    COMPREPLY=()
+    if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
+        cur="$2"
+    else
+        cur="${COMP_WORDS[COMP_CWORD]}"
+    fi
+    prev="$3"
+    cmd=""
+    opts=""
+
+    for i in "${COMP_WORDS[@]:0:COMP_CWORD}"
+    do
+        case "${cmd},${i}" in
+            ",$1")
+                cmd="feroxbuster"
+                ;;
+            *)
+                ;;
+        esac
+    done
+
+    case "${cmd}" in
+        feroxbuster)
+            opts="-u -p -P -R -a -A -x -m -H -b -Q -f -S -X -W -N -C -s -T -r -k -t -n -d -e -L -w -D -E -B -g -I -v -q -o -U -h -V --url --stdin --resume-from --request-file --burp --burp-replay --data-urlencoded --data-json --smart --thorough --proxy --replay-proxy --replay-codes --user-agent --random-agent --extensions --methods --data --headers --cookies --query --add-slash --protocol --dont-scan --scope --filter-size --filter-regex --filter-words --filter-lines --filter-status --filter-similar-to --status-codes --unique --timeout --redirects --insecure --server-certs --client-cert --client-key --threads --no-recursion --depth --force-recursion --extract-links --dont-extract-links --scan-limit --parallel --rate-limit --response-size-limit --time-limit --wordlist --auto-tune --auto-bail --dont-filter --collect-extensions --collect-backups --collect-words --dont-collect --scan-dir-listings --verbosity --silent --quiet --json --output --debug-log --no-state --limit-bars --update --help --version"
+            if [[ ${cur} == -* || ${COMP_CWORD} -eq 1 ]] ; then
+                COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
+                return 0
+            fi
+            case "${prev}" in
+                --url)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -u)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --resume-from)
+                    local oldifs
+                    if [ -n "${IFS+x}" ]; then
+                        oldifs="$IFS"
+                    fi
+                    IFS=$'\n'
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    if [ -n "${oldifs+x}" ]; then
+                        IFS="$oldifs"
+                    fi
+                    if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
+                        compopt -o filenames
+                    fi
+                    return 0
+                    ;;
+                --request-file)
+                    local oldifs
+                    if [ -n "${IFS+x}" ]; then
+                        oldifs="$IFS"
+                    fi
+                    IFS=$'\n'
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    if [ -n "${oldifs+x}" ]; then
+                        IFS="$oldifs"
+                    fi
+                    if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
+                        compopt -o filenames
+                    fi
+                    return 0
+                    ;;
+                --data-urlencoded)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --data-json)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --proxy)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -p)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --replay-proxy)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -P)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --replay-codes)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -R)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --user-agent)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -a)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --extensions)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -x)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --methods)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -m)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --data)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --headers)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -H)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --cookies)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -b)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --query)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -Q)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --protocol)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --dont-scan)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --scope)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --filter-size)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -S)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --filter-regex)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -X)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --filter-words)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -W)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --filter-lines)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -N)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --filter-status)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -C)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --filter-similar-to)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --status-codes)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -s)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --timeout)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -T)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --server-certs)
+                    local oldifs
+                    if [ -n "${IFS+x}" ]; then
+                        oldifs="$IFS"
+                    fi
+                    IFS=$'\n'
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    if [ -n "${oldifs+x}" ]; then
+                        IFS="$oldifs"
+                    fi
+                    if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
+                        compopt -o filenames
+                    fi
+                    return 0
+                    ;;
+                --client-cert)
+                    local oldifs
+                    if [ -n "${IFS+x}" ]; then
+                        oldifs="$IFS"
+                    fi
+                    IFS=$'\n'
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    if [ -n "${oldifs+x}" ]; then
+                        IFS="$oldifs"
+                    fi
+                    if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
+                        compopt -o filenames
+                    fi
+                    return 0
+                    ;;
+                --client-key)
+                    local oldifs
+                    if [ -n "${IFS+x}" ]; then
+                        oldifs="$IFS"
+                    fi
+                    IFS=$'\n'
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    if [ -n "${oldifs+x}" ]; then
+                        IFS="$oldifs"
+                    fi
+                    if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
+                        compopt -o filenames
+                    fi
+                    return 0
+                    ;;
+                --threads)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -t)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --depth)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -d)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --scan-limit)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -L)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --parallel)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --rate-limit)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --response-size-limit)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --time-limit)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --wordlist)
+                    local oldifs
+                    if [ -n "${IFS+x}" ]; then
+                        oldifs="$IFS"
+                    fi
+                    IFS=$'\n'
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    if [ -n "${oldifs+x}" ]; then
+                        IFS="$oldifs"
+                    fi
+                    if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
+                        compopt -o filenames
+                    fi
+                    return 0
+                    ;;
+                -w)
+                    local oldifs
+                    if [ -n "${IFS+x}" ]; then
+                        oldifs="$IFS"
+                    fi
+                    IFS=$'\n'
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    if [ -n "${oldifs+x}" ]; then
+                        IFS="$oldifs"
+                    fi
+                    if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
+                        compopt -o filenames
+                    fi
+                    return 0
+                    ;;
+                --collect-backups)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -B)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --dont-collect)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -I)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --output)
+                    local oldifs
+                    if [ -n "${IFS+x}" ]; then
+                        oldifs="$IFS"
+                    fi
+                    IFS=$'\n'
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    if [ -n "${oldifs+x}" ]; then
+                        IFS="$oldifs"
+                    fi
+                    if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
+                        compopt -o filenames
+                    fi
+                    return 0
+                    ;;
+                -o)
+                    local oldifs
+                    if [ -n "${IFS+x}" ]; then
+                        oldifs="$IFS"
+                    fi
+                    IFS=$'\n'
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    if [ -n "${oldifs+x}" ]; then
+                        IFS="$oldifs"
+                    fi
+                    if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
+                        compopt -o filenames
+                    fi
+                    return 0
+                    ;;
+                --debug-log)
+                    local oldifs
+                    if [ -n "${IFS+x}" ]; then
+                        oldifs="$IFS"
+                    fi
+                    IFS=$'\n'
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    if [ -n "${oldifs+x}" ]; then
+                        IFS="$oldifs"
+                    fi
+                    if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
+                        compopt -o filenames
+                    fi
+                    return 0
+                    ;;
+                --limit-bars)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                *)
+                    COMPREPLY=()
+                    ;;
+            esac
+            COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
+            return 0
+            ;;
+    esac
+}
+
+if [[ "${BASH_VERSINFO[0]}" -eq 4 && "${BASH_VERSINFO[1]}" -ge 4 || "${BASH_VERSINFO[0]}" -gt 4 ]]; then
+    complete -F _feroxbuster -o nosort -o bashdefault -o default -o plusdirs feroxbuster
+else
+    complete -F _feroxbuster -o bashdefault -o default -o plusdirs feroxbuster
+fi

shell_completions/feroxbuster.elv

@@ -0,0 +1,132 @@
+
+use builtin;
+use str;
+
+set edit:completion:arg-completer[feroxbuster] = {|@words|
+    fn spaces {|n|
+        builtin:repeat $n ' ' | str:join ''
+    }
+    fn cand {|text desc|
+        edit:complex-candidate $text &display=$text' '(spaces (- 14 (wcswidth $text)))$desc
+    }
+    var command = 'feroxbuster'
+    for word $words[1..-1] {
+        if (str:has-prefix $word '-') {
+            break
+        }
+        set command = $command';'$word
+    }
+    var completions = [
+        &'feroxbuster'= {
+            cand -u 'The target URL (required, unless [--stdin || --resume-from || --request-file] used)'
+            cand --url 'The target URL (required, unless [--stdin || --resume-from || --request-file] used)'
+            cand --resume-from 'State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)'
+            cand --request-file 'Raw HTTP request file to use as a template for all requests'
+            cand --data-urlencoded 'Set -H ''Content-Type: application/x-www-form-urlencoded'', --data to <data-urlencoded> (supports @file) and -m to POST'
+            cand --data-json 'Set -H ''Content-Type: application/json'', --data to <data-json> (supports @file) and -m to POST'
+            cand -p 'Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)'
+            cand --proxy 'Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)'
+            cand -P 'Send only unfiltered requests through a Replay Proxy, instead of all requests'
+            cand --replay-proxy 'Send only unfiltered requests through a Replay Proxy, instead of all requests'
+            cand -R 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)'
+            cand --replay-codes 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)'
+            cand -a 'Sets the User-Agent (default: feroxbuster/2.13.1)'
+            cand --user-agent 'Sets the User-Agent (default: feroxbuster/2.13.1)'
+            cand -x 'File extension(s) to search for (ex: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex: @ext.txt)'
+            cand --extensions 'File extension(s) to search for (ex: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex: @ext.txt)'
+            cand -m 'Which HTTP request method(s) should be sent (default: GET)'
+            cand --methods 'Which HTTP request method(s) should be sent (default: GET)'
+            cand --data 'Request''s Body; can read data from a file if input starts with an @ (ex: @post.bin)'
+            cand -H 'Specify HTTP headers to be used in each request (ex: -H Header:val -H ''stuff: things'')'
+            cand --headers 'Specify HTTP headers to be used in each request (ex: -H Header:val -H ''stuff: things'')'
+            cand -b 'Specify HTTP cookies to be used in each request (ex: -b stuff=things)'
+            cand --cookies 'Specify HTTP cookies to be used in each request (ex: -b stuff=things)'
+            cand -Q 'Request''s URL query parameters (ex: -Q token=stuff -Q secret=key)'
+            cand --query 'Request''s URL query parameters (ex: -Q token=stuff -Q secret=key)'
+            cand --protocol 'Specify the protocol to use when targeting via --request-file or --url with domain only (default: https)'
+            cand --dont-scan 'URL(s) or Regex Pattern(s) to exclude from recursion/scans'
+            cand --scope 'Additional domains/URLs to consider in-scope for scanning (in addition to current domain)'
+            cand -S 'Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)'
+            cand --filter-size 'Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)'
+            cand -X 'Filter out messages via regular expression matching on the response''s body/headers (ex: -X ''^ignore me$'')'
+            cand --filter-regex 'Filter out messages via regular expression matching on the response''s body/headers (ex: -X ''^ignore me$'')'
+            cand -W 'Filter out messages of a particular word count (ex: -W 312 -W 91,82)'
+            cand --filter-words 'Filter out messages of a particular word count (ex: -W 312 -W 91,82)'
+            cand -N 'Filter out messages of a particular line count (ex: -N 20 -N 31,30)'
+            cand --filter-lines 'Filter out messages of a particular line count (ex: -N 20 -N 31,30)'
+            cand -C 'Filter out status codes (deny list) (ex: -C 200 -C 401)'
+            cand --filter-status 'Filter out status codes (deny list) (ex: -C 200 -C 401)'
+            cand --filter-similar-to 'Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)'
+            cand -s 'Status Codes to include (allow list) (default: All Status Codes)'
+            cand --status-codes 'Status Codes to include (allow list) (default: All Status Codes)'
+            cand -T 'Number of seconds before a client''s request times out (default: 7)'
+            cand --timeout 'Number of seconds before a client''s request times out (default: 7)'
+            cand --server-certs 'Add custom root certificate(s) for servers with unknown certificates'
+            cand --client-cert 'Add a PEM encoded certificate for mutual authentication (mTLS)'
+            cand --client-key 'Add a PEM encoded private key for mutual authentication (mTLS)'
+            cand -t 'Number of concurrent threads (default: 50)'
+            cand --threads 'Number of concurrent threads (default: 50)'
+            cand -d 'Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)'
+            cand --depth 'Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)'
+            cand -L 'Limit total number of concurrent scans (default: 0, i.e. no limit)'
+            cand --scan-limit 'Limit total number of concurrent scans (default: 0, i.e. no limit)'
+            cand --parallel 'Run parallel feroxbuster instances (one child process per url passed via stdin)'
+            cand --rate-limit 'Limit number of requests per second (per directory) (default: 0, i.e. no limit)'
+            cand --response-size-limit 'Limit size of response body to read in bytes (default: 4MB)'
+            cand --time-limit 'Limit total run time of all scans (ex: --time-limit 10m)'
+            cand -w 'Path or URL of the wordlist'
+            cand --wordlist 'Path or URL of the wordlist'
+            cand -B 'Automatically request likely backup extensions for "found" urls (default: ~, .bak, .bak2, .old, .1)'
+            cand --collect-backups 'Automatically request likely backup extensions for "found" urls (default: ~, .bak, .bak2, .old, .1)'
+            cand -I 'File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)'
+            cand --dont-collect 'File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)'
+            cand -o 'Output file to write results to (use w/ --json for JSON entries)'
+            cand --output 'Output file to write results to (use w/ --json for JSON entries)'
+            cand --debug-log 'Output file to write log entries (use w/ --json for JSON entries)'
+            cand --limit-bars 'Number of directory scan bars to show at any given time (default: no limit)'
+            cand --stdin 'Read url(s) from STDIN'
+            cand --burp 'Set --proxy to http://127.0.0.1:8080 and set --insecure to true'
+            cand --burp-replay 'Set --replay-proxy to http://127.0.0.1:8080 and set --insecure to true'
+            cand --smart 'Set --auto-tune, --collect-words, and --collect-backups to true'
+            cand --thorough 'Use the same settings as --smart and set --collect-extensions and --scan-dir-listings to true'
+            cand -A 'Use a random User-Agent'
+            cand --random-agent 'Use a random User-Agent'
+            cand -f 'Append / to each request''s URL'
+            cand --add-slash 'Append / to each request''s URL'
+            cand --unique 'Only show unique responses'
+            cand -r 'Allow client to follow redirects'
+            cand --redirects 'Allow client to follow redirects'
+            cand -k 'Disables TLS certificate validation in the client'
+            cand --insecure 'Disables TLS certificate validation in the client'
+            cand -n 'Do not scan recursively'
+            cand --no-recursion 'Do not scan recursively'
+            cand --force-recursion 'Force recursion attempts on all ''found'' endpoints (still respects recursion depth)'
+            cand -e 'Extract links from response body (html, javascript, etc...); make new requests based on findings (default: true)'
+            cand --extract-links 'Extract links from response body (html, javascript, etc...); make new requests based on findings (default: true)'
+            cand --dont-extract-links 'Don''t extract links from response body (html, javascript, etc...)'
+            cand --auto-tune 'Automatically lower scan rate when an excessive amount of errors are encountered'
+            cand --auto-bail 'Automatically stop scanning when an excessive amount of errors are encountered'
+            cand -D 'Don''t auto-filter wildcard responses'
+            cand --dont-filter 'Don''t auto-filter wildcard responses'
+            cand -E 'Automatically discover extensions and add them to --extensions (unless they''re in --dont-collect)'
+            cand --collect-extensions 'Automatically discover extensions and add them to --extensions (unless they''re in --dont-collect)'
+            cand -g 'Automatically discover important words from within responses and add them to the wordlist'
+            cand --collect-words 'Automatically discover important words from within responses and add them to the wordlist'
+            cand --scan-dir-listings 'Force scans to recurse into directory listings'
+            cand -v 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v''s is probably too much)'
+            cand --verbosity 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v''s is probably too much)'
+            cand --silent 'Only print URLs (or JSON w/ --json) + turn off logging (good for piping a list of urls to other commands)'
+            cand -q 'Hide progress bars and banner (good for tmux windows w/ notifications)'
+            cand --quiet 'Hide progress bars and banner (good for tmux windows w/ notifications)'
+            cand --json 'Emit JSON logs to --output and --debug-log instead of normal text'
+            cand --no-state 'Disable state output file (*.state)'
+            cand -U 'Update feroxbuster to the latest version'
+            cand --update 'Update feroxbuster to the latest version'
+            cand -h 'Print help (see more with ''--help'')'
+            cand --help 'Print help (see more with ''--help'')'
+            cand -V 'Print version'
+            cand --version 'Print version'
+        }
+    ]
+    $completions[$command]
+}

shell_completions/feroxbuster.fish

@@ -0,0 +1,70 @@
+complete -c feroxbuster -s u -l url -d 'The target URL (required, unless [--stdin || --resume-from || --request-file] used)' -r -f
+complete -c feroxbuster -l resume-from -d 'State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)' -r -F
+complete -c feroxbuster -l request-file -d 'Raw HTTP request file to use as a template for all requests' -r -F
+complete -c feroxbuster -l data-urlencoded -d 'Set -H \'Content-Type: application/x-www-form-urlencoded\', --data to <data-urlencoded> (supports @file) and -m to POST' -r
+complete -c feroxbuster -l data-json -d 'Set -H \'Content-Type: application/json\', --data to <data-json> (supports @file) and -m to POST' -r
+complete -c feroxbuster -s p -l proxy -d 'Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)' -r -f
+complete -c feroxbuster -s P -l replay-proxy -d 'Send only unfiltered requests through a Replay Proxy, instead of all requests' -r -f
+complete -c feroxbuster -s R -l replay-codes -d 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)' -r
+complete -c feroxbuster -s a -l user-agent -d 'Sets the User-Agent (default: feroxbuster/2.13.1)' -r
+complete -c feroxbuster -s x -l extensions -d 'File extension(s) to search for (ex: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex: @ext.txt)' -r
+complete -c feroxbuster -s m -l methods -d 'Which HTTP request method(s) should be sent (default: GET)' -r
+complete -c feroxbuster -l data -d 'Request\'s Body; can read data from a file if input starts with an @ (ex: @post.bin)' -r
+complete -c feroxbuster -s H -l headers -d 'Specify HTTP headers to be used in each request (ex: -H Header:val -H \'stuff: things\')' -r
+complete -c feroxbuster -s b -l cookies -d 'Specify HTTP cookies to be used in each request (ex: -b stuff=things)' -r
+complete -c feroxbuster -s Q -l query -d 'Request\'s URL query parameters (ex: -Q token=stuff -Q secret=key)' -r
+complete -c feroxbuster -l protocol -d 'Specify the protocol to use when targeting via --request-file or --url with domain only (default: https)' -r
+complete -c feroxbuster -l dont-scan -d 'URL(s) or Regex Pattern(s) to exclude from recursion/scans' -r
+complete -c feroxbuster -l scope -d 'Additional domains/URLs to consider in-scope for scanning (in addition to current domain)' -r
+complete -c feroxbuster -s S -l filter-size -d 'Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)' -r
+complete -c feroxbuster -s X -l filter-regex -d 'Filter out messages via regular expression matching on the response\'s body/headers (ex: -X \'^ignore me$\')' -r
+complete -c feroxbuster -s W -l filter-words -d 'Filter out messages of a particular word count (ex: -W 312 -W 91,82)' -r
+complete -c feroxbuster -s N -l filter-lines -d 'Filter out messages of a particular line count (ex: -N 20 -N 31,30)' -r
+complete -c feroxbuster -s C -l filter-status -d 'Filter out status codes (deny list) (ex: -C 200 -C 401)' -r
+complete -c feroxbuster -l filter-similar-to -d 'Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)' -r -f
+complete -c feroxbuster -s s -l status-codes -d 'Status Codes to include (allow list) (default: All Status Codes)' -r
+complete -c feroxbuster -s T -l timeout -d 'Number of seconds before a client\'s request times out (default: 7)' -r
+complete -c feroxbuster -l server-certs -d 'Add custom root certificate(s) for servers with unknown certificates' -r -F
+complete -c feroxbuster -l client-cert -d 'Add a PEM encoded certificate for mutual authentication (mTLS)' -r -F
+complete -c feroxbuster -l client-key -d 'Add a PEM encoded private key for mutual authentication (mTLS)' -r -F
+complete -c feroxbuster -s t -l threads -d 'Number of concurrent threads (default: 50)' -r
+complete -c feroxbuster -s d -l depth -d 'Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)' -r
+complete -c feroxbuster -s L -l scan-limit -d 'Limit total number of concurrent scans (default: 0, i.e. no limit)' -r
+complete -c feroxbuster -l parallel -d 'Run parallel feroxbuster instances (one child process per url passed via stdin)' -r
+complete -c feroxbuster -l rate-limit -d 'Limit number of requests per second (per directory) (default: 0, i.e. no limit)' -r
+complete -c feroxbuster -l response-size-limit -d 'Limit size of response body to read in bytes (default: 4MB)' -r
+complete -c feroxbuster -l time-limit -d 'Limit total run time of all scans (ex: --time-limit 10m)' -r
+complete -c feroxbuster -s w -l wordlist -d 'Path or URL of the wordlist' -r -F
+complete -c feroxbuster -s B -l collect-backups -d 'Automatically request likely backup extensions for "found" urls (default: ~, .bak, .bak2, .old, .1)' -r
+complete -c feroxbuster -s I -l dont-collect -d 'File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)' -r
+complete -c feroxbuster -s o -l output -d 'Output file to write results to (use w/ --json for JSON entries)' -r -F
+complete -c feroxbuster -l debug-log -d 'Output file to write log entries (use w/ --json for JSON entries)' -r -F
+complete -c feroxbuster -l limit-bars -d 'Number of directory scan bars to show at any given time (default: no limit)' -r
+complete -c feroxbuster -l stdin -d 'Read url(s) from STDIN'
+complete -c feroxbuster -l burp -d 'Set --proxy to http://127.0.0.1:8080 and set --insecure to true'
+complete -c feroxbuster -l burp-replay -d 'Set --replay-proxy to http://127.0.0.1:8080 and set --insecure to true'
+complete -c feroxbuster -l smart -d 'Set --auto-tune, --collect-words, and --collect-backups to true'
+complete -c feroxbuster -l thorough -d 'Use the same settings as --smart and set --collect-extensions and --scan-dir-listings to true'
+complete -c feroxbuster -s A -l random-agent -d 'Use a random User-Agent'
+complete -c feroxbuster -s f -l add-slash -d 'Append / to each request\'s URL'
+complete -c feroxbuster -l unique -d 'Only show unique responses'
+complete -c feroxbuster -s r -l redirects -d 'Allow client to follow redirects'
+complete -c feroxbuster -s k -l insecure -d 'Disables TLS certificate validation in the client'
+complete -c feroxbuster -s n -l no-recursion -d 'Do not scan recursively'
+complete -c feroxbuster -l force-recursion -d 'Force recursion attempts on all \'found\' endpoints (still respects recursion depth)'
+complete -c feroxbuster -s e -l extract-links -d 'Extract links from response body (html, javascript, etc...); make new requests based on findings (default: true)'
+complete -c feroxbuster -l dont-extract-links -d 'Don\'t extract links from response body (html, javascript, etc...)'
+complete -c feroxbuster -l auto-tune -d 'Automatically lower scan rate when an excessive amount of errors are encountered'
+complete -c feroxbuster -l auto-bail -d 'Automatically stop scanning when an excessive amount of errors are encountered'
+complete -c feroxbuster -s D -l dont-filter -d 'Don\'t auto-filter wildcard responses'
+complete -c feroxbuster -s E -l collect-extensions -d 'Automatically discover extensions and add them to --extensions (unless they\'re in --dont-collect)'
+complete -c feroxbuster -s g -l collect-words -d 'Automatically discover important words from within responses and add them to the wordlist'
+complete -c feroxbuster -l scan-dir-listings -d 'Force scans to recurse into directory listings'
+complete -c feroxbuster -s v -l verbosity -d 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v\'s is probably too much)'
+complete -c feroxbuster -l silent -d 'Only print URLs (or JSON w/ --json) + turn off logging (good for piping a list of urls to other commands)'
+complete -c feroxbuster -s q -l quiet -d 'Hide progress bars and banner (good for tmux windows w/ notifications)'
+complete -c feroxbuster -l json -d 'Emit JSON logs to --output and --debug-log instead of normal text'
+complete -c feroxbuster -l no-state -d 'Disable state output file (*.state)'
+complete -c feroxbuster -s U -l update -d 'Update feroxbuster to the latest version'
+complete -c feroxbuster -s h -l help -d 'Print help (see more with \'--help\')'
+complete -c feroxbuster -s V -l version -d 'Print version'

snapcraft.yaml

@@ -0,0 +1,41 @@
+name: feroxbuster
+version: git
+summary: A simple, fast, recursive content discovery tool written in Rust
+description: |
+  feroxbuster is a tool designed to perform Forced Browsing.
+
+  Forced browsing is an attack where the aim is to enumerate and access resources that are not referenced by the web application, but are still accessible by an attacker.
+
+  feroxbuster uses brute force combined with a wordlist to search for unlinked content in target directories. These resources may store sensitive information about web applications and operational systems, such as source code, credentials, internal network addressing, etc...
+
+  This attack is also known as Predictable Resource Location, File Enumeration, Directory Enumeration, and Resource Enumeration.
+
+
+base: core18
+
+plugs:
+  etc-feroxbuster:
+    interface: system-files
+    read:
+    - /etc/feroxbuster
+  dot-config-feroxbuster:
+    interface: personal-files
+    read:
+    - $HOME/.config/feroxbuster
+
+architectures:
+  - build-on: amd64
+  - build-on: i386
+
+parts:
+  feroxbuster:
+    plugin: rust
+    source: .
+
+apps:
+  feroxbuster:
+    command: bin/feroxbuster
+    plugs:
+      - etc-feroxbuster
+      - dot-config-feroxbuster
+      - network

src/banner.rs

@@ -1,583 +0,0 @@
-use crate::config::{Configuration, CONFIGURATION};
-use crate::utils::{make_request, status_colorizer};
-use reqwest::{Client, Url};
-use serde_json::Value;
-use std::io::Write;
-
-/// macro helper to abstract away repetitive string formatting
-macro_rules! format_banner_entry_helper {
-    // \u{0020} -> unicode space
-    // \u{2502} -> vertical box drawing character, i.e. │
-    ($rune:expr, $name:expr, $value:expr, $indent:expr, $col_width:expr) => {
-        format!(
-            "\u{0020}{:\u{0020}<indent$}{:\u{0020}<col_w$}\u{2502}\u{0020}{}",
-            $rune,
-            $name,
-            $value,
-            indent = $indent,
-            col_w = $col_width
-        )
-    };
-    ($rune:expr, $name:expr, $value:expr, $value2:expr, $indent:expr, $col_width:expr) => {
-        format!(
-            "\u{0020}{:\u{0020}<indent$}{:\u{0020}<col_w$}\u{2502}\u{0020}{}:\u{0020}{}",
-            $rune,
-            $name,
-            $value,
-            $value2,
-            indent = $indent,
-            col_w = $col_width
-        )
-    };
-}
-
-/// macro that wraps another macro helper to abstract away repetitive string formatting
-macro_rules! format_banner_entry {
-    // 4 -> unicode emoji padding width
-    // 22 -> column width (when unicode rune is 4 bytes wide, 23 when it's 3)
-    // hardcoded since macros don't allow let statements
-    ($rune:expr, $name:expr, $value:expr) => {
-        format_banner_entry_helper!($rune, $name, $value, 3, 22)
-    };
-    ($rune:expr, $name:expr, $value1:expr, $value2:expr) => {
-        format_banner_entry_helper!($rune, $name, $value1, $value2, 3, 22)
-    };
-}
-
-/// Url used to query github's api; specifically used to look for the latest tagged release name
-const UPDATE_URL: &str = "https://api.github.com/repos/epi052/feroxbuster/releases/latest";
-
-/// Simple enum to hold three different update states
-#[derive(Debug)]
-enum UpdateStatus {
-    /// this version and latest release are the same
-    UpToDate,
-
-    /// this version and latest release are not the same
-    OutOfDate,
-
-    /// some error occurred during version check
-    Unknown,
-}
-
-/// Makes a request to the given url, expecting to receive a JSON response that contains a field
-/// named `tag_name` that holds a value representing the latest tagged release of this tool.
-///
-/// ex: v1.1.0
-///
-/// Returns `UpdateStatus`
-async fn needs_update(client: &Client, url: &str, bin_version: &str) -> UpdateStatus {
-    log::trace!("enter: needs_update({:?}, {})", client, url);
-
-    let unknown = UpdateStatus::Unknown;
-
-    let api_url = match Url::parse(url) {
-        Ok(url) => url,
-        Err(e) => {
-            log::error!("{}", e);
-            log::trace!("exit: needs_update -> {:?}", unknown);
-            return unknown;
-        }
-    };
-
-    if let Ok(response) = make_request(&client, &api_url).await {
-        let body = response.text().await.unwrap_or_default();
-
-        let json_response: Value = serde_json::from_str(&body).unwrap_or_default();
-
-        if json_response.is_null() {
-            // unwrap_or_default above should result in a null value for the json_response variable
-            log::error!("Could not parse JSON from response body");
-            log::trace!("exit: needs_update -> {:?}", unknown);
-            return unknown;
-        }
-
-        let latest_version = match json_response["tag_name"].as_str() {
-            Some(tag) => tag.trim_start_matches('v'),
-            None => {
-                log::error!("Could not get version field from JSON response");
-                log::debug!("{}", json_response);
-                log::trace!("exit: needs_update -> {:?}", unknown);
-                return unknown;
-            }
-        };
-
-        // if we've gotten this far, we have a string in the form of X.X.X where X is a number
-        // all that's left is to compare the current version with the version found above
-
-        return if latest_version == bin_version {
-            // there's really only two possible outcomes if we accept that the tag conforms to
-            // the X.X.X pattern:
-            //   1. the version strings match, meaning we're up to date
-            //   2. the version strings do not match, meaning we're out of date
-            //
-            // except for developers working on this code, nobody should ever be in a situation
-            // where they have a version greater than the latest tagged release
-            log::trace!("exit: needs_update -> UpdateStatus::UpToDate");
-            UpdateStatus::UpToDate
-        } else {
-            log::trace!("exit: needs_update -> UpdateStatus::OutOfDate");
-            UpdateStatus::OutOfDate
-        };
-    }
-
-    log::trace!("exit: needs_update -> {:?}", unknown);
-    unknown
-}
-
-/// Prints the banner to stdout.
-///
-/// Only prints those settings which are either always present, or passed in by the user.
-pub async fn initialize<W>(targets: &[String], config: &Configuration, version: &str, mut writer: W)
-where
-    W: Write,
-{
-    let artwork = format!(
-        r#"
- ___  ___  __   __     __      __         __   ___
-|__  |__  |__) |__) | /  `    /  \ \_/ | |  \ |__
-|    |___ |  \ |  \ | \__,    \__/ / \ | |__/ |___
-by Ben "epi" Risher {}                  ver: {}"#,
-        '\u{1F913}', version
-    );
-
-    let status = needs_update(&CONFIGURATION.client, UPDATE_URL, version).await;
-
-    let top = "───────────────────────────┬──────────────────────";
-    let bottom = "───────────────────────────┴──────────────────────";
-
-    writeln!(&mut writer, "{}", artwork).unwrap_or_default();
-    writeln!(&mut writer, "{}", top).unwrap_or_default();
-
-    // begin with always printed items
-    for target in targets {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1F3af}", "Target Url", target)
-        )
-        .unwrap_or_default(); // 🎯
-    }
-
-    let mut codes = vec![];
-
-    for code in &config.statuscodes {
-        codes.push(status_colorizer(&code.to_string()))
-    }
-
-    writeln!(
-        &mut writer,
-        "{}",
-        format_banner_entry!("\u{1F680}", "Threads", config.threads)
-    )
-    .unwrap_or_default(); // 🚀
-
-    writeln!(
-        &mut writer,
-        "{}",
-        format_banner_entry!("\u{1f4d6}", "Wordlist", config.wordlist)
-    )
-    .unwrap_or_default(); // 📖
-
-    writeln!(
-        &mut writer,
-        "{}",
-        format_banner_entry!(
-            "\u{1F197}",
-            "Status Codes",
-            format!("[{}]", codes.join(", "))
-        )
-    )
-    .unwrap_or_default(); // 🆗
-
-    writeln!(
-        &mut writer,
-        "{}",
-        format_banner_entry!("\u{1f4a5}", "Timeout (secs)", config.timeout)
-    )
-    .unwrap_or_default(); // 💥
-
-    writeln!(
-        &mut writer,
-        "{}",
-        format_banner_entry!("\u{1F9a1}", "User-Agent", config.useragent)
-    )
-    .unwrap_or_default(); // 🦡
-
-    // followed by the maybe printed or variably displayed values
-    if !config.config.is_empty() {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1f489}", "Config File", config.config)
-        )
-        .unwrap_or_default(); // 💉
-    }
-
-    if !config.proxy.is_empty() {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1f48e}", "Proxy", config.proxy)
-        )
-        .unwrap_or_default(); // 💎
-    }
-
-    if !config.headers.is_empty() {
-        for (name, value) in &config.headers {
-            writeln!(
-                &mut writer,
-                "{}",
-                format_banner_entry!("\u{1f92f}", "Header", name, value)
-            )
-            .unwrap_or_default(); // 🤯
-        }
-    }
-
-    if !config.sizefilters.is_empty() {
-        for filter in &config.sizefilters {
-            writeln!(
-                &mut writer,
-                "{}",
-                format_banner_entry!("\u{1f4a2}", "Size Filter", filter)
-            )
-            .unwrap_or_default(); // 💢
-        }
-    }
-
-    if config.extract_links {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1F50E}", "Extract Links", config.extract_links)
-        )
-        .unwrap_or_default(); // 🔎
-    }
-
-    if !config.queries.is_empty() {
-        for query in &config.queries {
-            writeln!(
-                &mut writer,
-                "{}",
-                format_banner_entry!(
-                    "\u{1f914}",
-                    "Query Parameter",
-                    format!("{}={}", query.0, query.1)
-                )
-            )
-            .unwrap_or_default(); // 🤔
-        }
-    }
-
-    if !config.output.is_empty() {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1f4be}", "Output File", config.output)
-        )
-        .unwrap_or_default(); // 💾
-    }
-
-    if !config.extensions.is_empty() {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!(
-                "\u{1f4b2}",
-                "Extensions",
-                format!("[{}]", config.extensions.join(", "))
-            )
-        )
-        .unwrap_or_default(); // 💲
-    }
-
-    if config.insecure {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1f513}", "Insecure", config.insecure)
-        )
-        .unwrap_or_default(); // 🔓
-    }
-
-    if config.redirects {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1f4cd}", "Follow Redirects", config.redirects)
-        )
-        .unwrap_or_default(); // 📍
-    }
-
-    if config.dontfilter {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1f92a}", "Filter Wildcards", !config.dontfilter)
-        )
-        .unwrap_or_default(); // 🤪
-    }
-
-    match config.verbosity {
-        //speaker medium volume (increasing with verbosity to loudspeaker)
-        1 => {
-            writeln!(
-                &mut writer,
-                "{}",
-                format_banner_entry!("\u{1f508}", "Verbosity", config.verbosity)
-            )
-            .unwrap_or_default(); // 🔈
-        }
-        2 => {
-            writeln!(
-                &mut writer,
-                "{}",
-                format_banner_entry!("\u{1f509}", "Verbosity", config.verbosity)
-            )
-            .unwrap_or_default(); // 🔉
-        }
-        3 => {
-            writeln!(
-                &mut writer,
-                "{}",
-                format_banner_entry!("\u{1f50a}", "Verbosity", config.verbosity)
-            )
-            .unwrap_or_default(); // 🔊
-        }
-        4 => {
-            writeln!(
-                &mut writer,
-                "{}",
-                format_banner_entry!("\u{1f4e2}", "Verbosity", config.verbosity)
-            )
-            .unwrap_or_default(); // 📢
-        }
-        _ => {}
-    }
-
-    if config.addslash {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1fa93}", "Add Slash", config.addslash)
-        )
-        .unwrap_or_default(); // 🪓
-    }
-
-    if !config.norecursion {
-        if config.depth == 0 {
-            writeln!(
-                &mut writer,
-                "{}",
-                format_banner_entry!("\u{1f503}", "Recursion Depth", "INFINITE")
-            )
-            .unwrap_or_default(); // 🔃
-        } else {
-            writeln!(
-                &mut writer,
-                "{}",
-                format_banner_entry!("\u{1f503}", "Recursion Depth", config.depth)
-            )
-            .unwrap_or_default(); // 🔃
-        }
-    } else {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!("\u{1f6ab}", "Do Not Recurse", config.norecursion)
-        )
-        .unwrap_or_default(); // 🚫
-    }
-
-    if matches!(status, UpdateStatus::OutOfDate) {
-        writeln!(
-            &mut writer,
-            "{}",
-            format_banner_entry!(
-                "\u{1f389}",
-                "New Version Available",
-                "https://github.com/epi052/feroxbuster/releases/latest"
-            )
-        )
-        .unwrap_or_default(); // 🎉
-    }
-
-    writeln!(&mut writer, "{}", bottom).unwrap_or_default();
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::VERSION;
-    use httpmock::Method::GET;
-    use httpmock::{Mock, MockServer};
-    use std::fs::read_to_string;
-    use std::io::stderr;
-    use std::time::Duration;
-    use tempfile::NamedTempFile;
-
-    #[tokio::test(core_threads = 1)]
-    /// test to hit no execution of targets for loop in banner
-    async fn banner_intialize_without_targets() {
-        let config = Configuration::default();
-        initialize(&[], &config, VERSION, stderr()).await;
-    }
-
-    #[tokio::test(core_threads = 1)]
-    /// test to hit no execution of statuscode for loop in banner
-    async fn banner_intialize_without_status_codes() {
-        let mut config = Configuration::default();
-        config.statuscodes = vec![];
-        initialize(
-            &[String::from("http://localhost")],
-            &config,
-            VERSION,
-            stderr(),
-        )
-        .await;
-    }
-
-    #[tokio::test(core_threads = 1)]
-    /// test to hit an empty config file
-    async fn banner_intialize_without_config_file() {
-        let mut config = Configuration::default();
-        config.config = String::new();
-        initialize(
-            &[String::from("http://localhost")],
-            &config,
-            VERSION,
-            stderr(),
-        )
-        .await;
-    }
-
-    #[tokio::test(core_threads = 1)]
-    /// test to hit an empty config file
-    async fn banner_intialize_without_queries() {
-        let mut config = Configuration::default();
-        config.queries = vec![(String::new(), String::new())];
-        initialize(
-            &[String::from("http://localhost")],
-            &config,
-            VERSION,
-            stderr(),
-        )
-        .await;
-    }
-
-    #[tokio::test(core_threads = 1)]
-    /// test to show that a new version is available for download
-    async fn banner_intialize_with_mismatched_version() {
-        let config = Configuration::default();
-        let file = NamedTempFile::new().unwrap();
-        initialize(
-            &[String::from("http://localhost")],
-            &config,
-            "mismatched-version",
-            &file,
-        )
-        .await;
-        let contents = read_to_string(file.path()).unwrap();
-        println!("contents: {}", contents);
-        assert!(contents.contains("New Version Available"));
-        assert!(contents.contains("https://github.com/epi052/feroxbuster/releases/latest"));
-    }
-
-    #[tokio::test(core_threads = 1)]
-    /// test that
-    async fn banner_needs_update_returns_unknown_with_bad_url() {
-        let result = needs_update(&CONFIGURATION.client, &"", VERSION).await;
-        assert!(matches!(result, UpdateStatus::Unknown));
-    }
-
-    #[tokio::test(core_threads = 1)]
-    /// test return value of good url to needs_update
-    async fn banner_needs_update_returns_up_to_date() {
-        let srv = MockServer::start();
-
-        let mock = Mock::new()
-            .expect_method(GET)
-            .expect_path("/latest")
-            .return_status(200)
-            .return_body("{\"tag_name\":\"v1.1.0\"}")
-            .create_on(&srv);
-
-        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.1.0").await;
-
-        assert_eq!(mock.times_called(), 1);
-        assert!(matches!(result, UpdateStatus::UpToDate));
-    }
-
-    #[tokio::test(core_threads = 1)]
-    /// test return value of good url to needs_update that returns a newer version than current
-    async fn banner_needs_update_returns_out_of_date() {
-        let srv = MockServer::start();
-
-        let mock = Mock::new()
-            .expect_method(GET)
-            .expect_path("/latest")
-            .return_status(200)
-            .return_body("{\"tag_name\":\"v1.1.0\"}")
-            .create_on(&srv);
-
-        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1").await;
-
-        assert_eq!(mock.times_called(), 1);
-        assert!(matches!(result, UpdateStatus::OutOfDate));
-    }
-
-    #[tokio::test(core_threads = 1)]
-    /// test return value of good url that times out
-    async fn banner_needs_update_returns_unknown_on_timeout() {
-        let srv = MockServer::start();
-
-        let mock = Mock::new()
-            .expect_method(GET)
-            .expect_path("/latest")
-            .return_status(200)
-            .return_body("{\"tag_name\":\"v1.1.0\"}")
-            .return_with_delay(Duration::from_secs(8))
-            .create_on(&srv);
-
-        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1").await;
-
-        assert_eq!(mock.times_called(), 1);
-        assert!(matches!(result, UpdateStatus::Unknown));
-    }
-
-    #[tokio::test(core_threads = 1)]
-    /// test return value of good url with bad json response
-    async fn banner_needs_update_returns_unknown_on_bad_json_response() {
-        let srv = MockServer::start();
-
-        let mock = Mock::new()
-            .expect_method(GET)
-            .expect_path("/latest")
-            .return_status(200)
-            .return_body("not json")
-            .create_on(&srv);
-
-        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1").await;
-
-        assert_eq!(mock.times_called(), 1);
-        assert!(matches!(result, UpdateStatus::Unknown));
-    }
-
-    #[tokio::test(core_threads = 1)]
-    /// test return value of good url with json response that lacks the tag_name field
-    async fn banner_needs_update_returns_unknown_on_json_without_correct_tag() {
-        let srv = MockServer::start();
-
-        let mock = Mock::new()
-            .expect_method(GET)
-            .expect_path("/latest")
-            .return_status(200)
-            .return_body("{\"no tag_name\": \"doesn't exist\"}")
-            .create_on(&srv);
-
-        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1").await;
-
-        assert_eq!(mock.times_called(), 1);
-        assert!(matches!(result, UpdateStatus::Unknown));
-    }
-}

src/banner/container.rs

@@ -0,0 +1,854 @@
+use super::entry::BannerEntry;
+use crate::{
+    client,
+    config::Configuration,
+    event_handlers::Handles,
+    utils::{make_request, parse_url_with_raw_path, status_colorizer},
+    DEFAULT_IGNORED_EXTENSIONS, DEFAULT_METHOD, DEFAULT_STATUS_CODES, VERSION,
+};
+use anyhow::{bail, Result};
+use console::{style, Emoji};
+use serde_json::Value;
+use std::collections::HashMap;
+use std::{io::Write, sync::Arc};
+
+/// Url used to query github's api; specifically used to look for the latest tagged release name
+pub const UPDATE_URL: &str = "https://api.github.com/repos/epi052/feroxbuster/releases/latest";
+
+/// Simple enum to hold three different update states
+#[derive(Debug)]
+pub(super) enum UpdateStatus {
+    /// this version and latest release are the same
+    UpToDate,
+
+    /// this version and latest release are not the same
+    OutOfDate,
+
+    /// some error occurred during version check
+    Unknown,
+}
+
+/// Banner object, contains multiple BannerEntry's and knows how to display itself
+pub struct Banner {
+    /// all live targets
+    targets: Vec<BannerEntry>,
+
+    /// represents Configuration.status_codes
+    status_codes: BannerEntry,
+
+    /// represents Configuration.filter_status
+    filter_status: BannerEntry,
+
+    /// represents Configuration.threads
+    threads: BannerEntry,
+
+    /// represents Configuration.wordlist
+    wordlist: BannerEntry,
+
+    /// represents Configuration.timeout
+    timeout: BannerEntry,
+
+    /// represents Configuration.user_agent
+    user_agent: BannerEntry,
+
+    /// represents Configuration.random_agent
+    random_agent: BannerEntry,
+
+    /// represents Configuration.config
+    config: BannerEntry,
+
+    /// represents Configuration.proxy
+    proxy: BannerEntry,
+
+    /// represents Configuration.client_key
+    client_key: BannerEntry,
+
+    /// represents Configuration.client_cert
+    client_cert: BannerEntry,
+
+    /// represents Configuration.server_certs
+    server_certs: BannerEntry,
+
+    /// represents Configuration.replay_proxy
+    replay_proxy: BannerEntry,
+
+    /// represents Configuration.replay_codes
+    replay_codes: BannerEntry,
+
+    /// represents Configuration.headers
+    headers: Vec<BannerEntry>,
+
+    /// represents Configuration.filter_size
+    filter_size: Vec<BannerEntry>,
+
+    /// represents Configuration.filter_similar
+    filter_similar: Vec<BannerEntry>,
+
+    /// represents Configuration.filter_word_count
+    filter_word_count: Vec<BannerEntry>,
+
+    /// represents Configuration.filter_line_count
+    filter_line_count: Vec<BannerEntry>,
+
+    /// represents Configuration.filter_regex
+    filter_regex: Vec<BannerEntry>,
+
+    /// represents Configuration.extract_links
+    extract_links: BannerEntry,
+
+    /// represents Configuration.json
+    json: BannerEntry,
+
+    /// represents Configuration.output
+    output: BannerEntry,
+
+    /// represents Configuration.debug_log
+    debug_log: BannerEntry,
+
+    /// represents Configuration.extensions
+    extensions: BannerEntry,
+
+    /// represents Configuration.methods
+    methods: BannerEntry,
+
+    /// represents Configuration.data
+    data: BannerEntry,
+
+    /// represents Configuration.insecure
+    insecure: BannerEntry,
+
+    /// represents Configuration.redirects
+    redirects: BannerEntry,
+
+    /// represents Configuration.dont_filter
+    dont_filter: BannerEntry,
+
+    /// represents Configuration.queries
+    queries: Vec<BannerEntry>,
+
+    /// represents Configuration.verbosity
+    verbosity: BannerEntry,
+
+    /// represents Configuration.add_slash
+    add_slash: BannerEntry,
+
+    /// represents Configuration.no_recursion
+    no_recursion: BannerEntry,
+
+    /// represents Configuration.scan_limit
+    scan_limit: BannerEntry,
+
+    /// represents Configuration.time_limit
+    time_limit: BannerEntry,
+
+    /// represents Configuration.rate_limit
+    rate_limit: BannerEntry,
+
+    /// represents Configuration.parallel
+    parallel: BannerEntry,
+
+    /// represents Configuration.auto_tune
+    auto_tune: BannerEntry,
+
+    /// represents Configuration.auto_bail
+    auto_bail: BannerEntry,
+
+    /// represents Configuration.url_denylist
+    url_denylist: Vec<BannerEntry>,
+
+    /// represents Configuration.scope
+    scope: Vec<BannerEntry>,
+
+    /// current version of feroxbuster
+    pub(super) version: String,
+
+    /// whether or not there is a known new version
+    pub(super) update_status: UpdateStatus,
+
+    /// represents Configuration.collect_extensions
+    collect_extensions: BannerEntry,
+
+    /// represents Configuration.dont_collect
+    dont_collect: BannerEntry,
+
+    /// represents Configuration.collect_backups
+    collect_backups: BannerEntry,
+
+    /// represents Configuration.collect_words
+    collect_words: BannerEntry,
+
+    /// represents Configuration.collect_words
+    force_recursion: BannerEntry,
+
+    /// represents Configuration.protocol
+    protocol: BannerEntry,
+
+    /// represents Configuration.scan_dir_listings
+    scan_dir_listings: BannerEntry,
+
+    /// represents Configuration.limit_bars
+    limit_bars: BannerEntry,
+
+    /// represents Configuration.unique
+    unique: BannerEntry,
+
+    /// represents Configuration.response_size_limit
+    response_size_limit: BannerEntry,
+}
+
+/// implementation of Banner
+impl Banner {
+    /// Create a new Banner from a Configuration and live targets
+    pub fn new(tgts: &[String], config: &Configuration) -> Self {
+        let mut targets = Vec::new();
+        let mut url_denylist = Vec::new();
+        let mut scope = Vec::new();
+        let mut code_filters = Vec::new();
+        let mut replay_codes = Vec::new();
+        let mut headers = Vec::new();
+        let mut filter_size = Vec::new();
+        let mut filter_similar = Vec::new();
+        let mut filter_word_count = Vec::new();
+        let mut filter_line_count = Vec::new();
+        let mut filter_regex = Vec::new();
+        let mut queries = Vec::new();
+
+        for target in tgts {
+            targets.push(BannerEntry::new("🎯", "Target Url", target));
+        }
+
+        for denied_url in &config.url_denylist {
+            url_denylist.push(BannerEntry::new(
+                "🚫",
+                "Don't Scan Url",
+                denied_url.as_str(),
+            ));
+        }
+
+        for denied_regex in &config.regex_denylist {
+            url_denylist.push(BannerEntry::new(
+                "🚫",
+                "Don't Scan Regex",
+                denied_regex.as_str(),
+            ));
+        }
+
+        for scope_url in &config.scope {
+            let value = match scope_url.host() {
+                Some(host) => host.to_string(),
+                None => scope_url.as_str().to_string(),
+            };
+
+            scope.push(BannerEntry::new("🚩", "In-Scope Url", &value));
+        }
+
+        // the +2 is for the 2 experimental status codes we add to the default list manually
+        let status_codes = if config.status_codes.len() == DEFAULT_STATUS_CODES.len() + 2 {
+            let all_str = format!(
+                "{} {} {}{}",
+                style("All").cyan(),
+                style("Status").green(),
+                style("Codes").yellow(),
+                style("!").red()
+            );
+            BannerEntry::new("👌", "Status Codes", &all_str)
+        } else {
+            let mut codes = vec![];
+
+            for code in &config.status_codes {
+                codes.push(status_colorizer(&code.to_string()))
+            }
+
+            BannerEntry::new("👌", "Status Codes", &format!("[{}]", codes.join(", ")))
+        };
+
+        for code in &config.filter_status {
+            code_filters.push(status_colorizer(&code.to_string()))
+        }
+        let filter_status = BannerEntry::new(
+            "💢",
+            "Status Code Filters",
+            &format!("[{}]", code_filters.join(", ")),
+        );
+
+        for code in &config.replay_codes {
+            replay_codes.push(status_colorizer(&code.to_string()))
+        }
+        let replay_codes = BannerEntry::new(
+            "📼",
+            "Replay Proxy Codes",
+            &format!("[{}]", replay_codes.join(", ")),
+        );
+
+        for (name, value) in &config.headers {
+            headers.push(BannerEntry::new(
+                "🤯",
+                "Header",
+                &format!("{name}: {value}"),
+            ));
+        }
+
+        for filter in &config.filter_size {
+            filter_size.push(BannerEntry::new("💢", "Size Filter", &filter.to_string()));
+        }
+
+        for filter in &config.filter_similar {
+            filter_similar.push(BannerEntry::new("💢", "Similarity Filter", filter));
+        }
+
+        for filter in &config.filter_word_count {
+            filter_word_count.push(BannerEntry::new(
+                "💢",
+                "Word Count Filter",
+                &filter.to_string(),
+            ));
+        }
+
+        for filter in &config.filter_line_count {
+            filter_line_count.push(BannerEntry::new(
+                "💢",
+                "Line Count Filter",
+                &filter.to_string(),
+            ));
+        }
+
+        for filter in &config.filter_regex {
+            filter_regex.push(BannerEntry::new("💢", "Regex Filter", filter));
+        }
+
+        for query in &config.queries {
+            queries.push(BannerEntry::new(
+                "🤔",
+                "Query Parameter",
+                &format!("{}={}", query.0, query.1),
+            ));
+        }
+
+        let volume = ["🔈", "🔉", "🔊", "📢"];
+        let verbosity = if let 1..=4 = config.verbosity {
+            //speaker medium volume (increasing with verbosity to loudspeaker)
+            BannerEntry::new(
+                volume[config.verbosity as usize - 1],
+                "Verbosity",
+                &config.verbosity.to_string(),
+            )
+        } else {
+            BannerEntry::default()
+        };
+
+        let no_recursion = if !config.no_recursion {
+            let depth = if config.depth == 0 {
+                "INFINITE".to_string()
+            } else {
+                config.depth.to_string()
+            };
+
+            BannerEntry::new("🔃", "Recursion Depth", &depth)
+        } else {
+            BannerEntry::new("🚫", "Do Not Recurse", &config.no_recursion.to_string())
+        };
+
+        let protocol = if config.protocol.to_lowercase() == "http" {
+            BannerEntry::new("🔓", "Default Protocol", &config.protocol)
+        } else {
+            BannerEntry::new("🔒", "Default Protocol", &config.protocol)
+        };
+
+        let scan_limit = BannerEntry::new(
+            "🦥",
+            "Concurrent Scan Limit",
+            &config.scan_limit.to_string(),
+        );
+
+        let force_recursion =
+            BannerEntry::new("🤘", "Force Recursion", &config.force_recursion.to_string());
+        let replay_proxy = BannerEntry::new("🎥", "Replay Proxy", &config.replay_proxy);
+        let auto_tune = BannerEntry::new("🎶", "Auto Tune", &config.auto_tune.to_string());
+        let auto_bail = BannerEntry::new("🙅", "Auto Bail", &config.auto_bail.to_string());
+        let scan_dir_listings = BannerEntry::new(
+            "📂",
+            "Scan Dir Listings",
+            &config.scan_dir_listings.to_string(),
+        );
+        let cfg = BannerEntry::new("💉", "Config File", &config.config);
+        let proxy = BannerEntry::new("💎", "Proxy", &config.proxy);
+        let server_certs = BannerEntry::new(
+            "🏅",
+            "Server Certificates",
+            &format!("[{}]", config.server_certs.join(", ")),
+        );
+        let client_cert = BannerEntry::new("🏅", "Client Certificate", &config.client_cert);
+        let client_key = BannerEntry::new("🔑", "Client Key", &config.client_key);
+        let threads = BannerEntry::new("🚀", "Threads", &config.threads.to_string());
+        let limit_bars =
+            BannerEntry::new("📊", "Limit Dir Scan Bars", &config.limit_bars.to_string());
+        let wordlist = BannerEntry::new("📖", "Wordlist", &config.wordlist);
+        let timeout = BannerEntry::new("💥", "Timeout (secs)", &config.timeout.to_string());
+        let user_agent = BannerEntry::new("🦡", "User-Agent", &config.user_agent);
+        let random_agent = BannerEntry::new("🦡", "User-Agent", "Random");
+        let extract_links =
+            BannerEntry::new("🔎", "Extract Links", &config.extract_links.to_string());
+        let json = BannerEntry::new("🧔", "JSON Output", &config.json.to_string());
+        let output = BannerEntry::new("💾", "Output File", &config.output);
+        let debug_log = BannerEntry::new("🪲", "Debugging Log", &config.debug_log);
+        let extensions = BannerEntry::new(
+            "💲",
+            "Extensions",
+            &format!("[{}]", config.extensions.join(", ")),
+        );
+        let methods = BannerEntry::new(
+            "🏁",
+            "HTTP methods",
+            &format!("[{}]", config.methods.join(", ")),
+        );
+
+        let dont_collect = if config.dont_collect == DEFAULT_IGNORED_EXTENSIONS {
+            // default has 30+ extensions, just trim it up
+            BannerEntry::new(
+                "💸",
+                "Ignored Extensions",
+                "[Images, Movies, Audio, etc...]",
+            )
+        } else {
+            BannerEntry::new(
+                "💸",
+                "Ignored Extensions",
+                &format!("[{}]", config.dont_collect.join(", ")),
+            )
+        };
+
+        let offset = std::cmp::min(config.data.len(), 30);
+        let data = String::from_utf8(config.data[..offset].to_vec())
+            .unwrap_or_else(|_err| {
+                format!(
+                    "{:x?} ...",
+                    &config.data[..std::cmp::min(config.data.len(), 13)]
+                )
+            })
+            .replace('\n', " ")
+            .replace('\r', "");
+        let data = BannerEntry::new("💣", "HTTP Body", &data);
+        let insecure = BannerEntry::new("🔓", "Insecure", &config.insecure.to_string());
+        let redirects = BannerEntry::new("📍", "Follow Redirects", &config.redirects.to_string());
+        let dont_filter =
+            BannerEntry::new("🤪", "Filter Wildcards", &(!config.dont_filter).to_string());
+        let add_slash = BannerEntry::new("🪓", "Add Slash", &config.add_slash.to_string());
+        let time_limit = BannerEntry::new("🕖", "Time Limit", &config.time_limit);
+        let parallel = BannerEntry::new("🛤", "Parallel Scans", &config.parallel.to_string());
+        let rate_limit =
+            BannerEntry::new("🚧", "Requests per Second", &config.rate_limit.to_string());
+        let collect_extensions = BannerEntry::new(
+            "💰",
+            "Collect Extensions",
+            &config.collect_extensions.to_string(),
+        );
+        let collect_backups =
+            BannerEntry::new("🏦", "Collect Backups", &config.collect_backups.to_string());
+
+        let collect_words =
+            BannerEntry::new("🤑", "Collect Words", &config.collect_words.to_string());
+
+        let unique = BannerEntry::new("🎲", "Unique Responses", &config.unique.to_string());
+
+        let response_size_limit = BannerEntry::new(
+            "📏",
+            "Response Size Limit",
+            &format!("{} bytes", config.response_size_limit),
+        );
+
+        Self {
+            targets,
+            status_codes,
+            threads,
+            wordlist,
+            filter_status,
+            timeout,
+            user_agent,
+            random_agent,
+            auto_bail,
+            auto_tune,
+            proxy,
+            client_cert,
+            client_key,
+            server_certs,
+            replay_codes,
+            replay_proxy,
+            headers,
+            filter_size,
+            filter_similar,
+            filter_word_count,
+            filter_line_count,
+            filter_regex,
+            extract_links,
+            parallel,
+            json,
+            queries,
+            output,
+            debug_log,
+            extensions,
+            methods,
+            data,
+            insecure,
+            dont_filter,
+            redirects,
+            verbosity,
+            add_slash,
+            no_recursion,
+            rate_limit,
+            scan_limit,
+            force_recursion,
+            time_limit,
+            url_denylist,
+            scope,
+            collect_extensions,
+            collect_backups,
+            collect_words,
+            dont_collect,
+            config: cfg,
+            scan_dir_listings,
+            protocol,
+            limit_bars,
+            unique,
+            response_size_limit,
+            version: VERSION.to_string(),
+            update_status: UpdateStatus::Unknown,
+        }
+    }
+
+    /// get a fancy header for the banner
+    fn header(&self) -> String {
+        let artwork = format!(
+            r#"
+ ___  ___  __   __     __      __         __   ___
+|__  |__  |__) |__) | /  `    /  \ \_/ | |  \ |__
+|    |___ |  \ |  \ | \__,    \__/ / \ | |__/ |___
+by Ben "epi" Risher {}                 ver: {}"#,
+            Emoji("🤓", &format!("{:<2}", "\u{0020}")),
+            self.version
+        );
+
+        let top = "───────────────────────────┬──────────────────────";
+
+        format!("{artwork}\n{top}")
+    }
+
+    /// get a fancy footer for the banner
+    fn footer(&self) -> String {
+        let addl_section = "──────────────────────────────────────────────────";
+        let bottom = "───────────────────────────┴──────────────────────";
+
+        let instructions = format!(
+            " 🏁  Press [{}] to use the {}™",
+            style("ENTER").yellow(),
+            style("Scan Management Menu").bright().yellow(),
+        );
+
+        format!("{bottom}\n{instructions}\n{addl_section}")
+    }
+
+    /// Makes a request to the given url, expecting to receive a JSON response that contains a field
+    /// named `tag_name` that holds a value representing the latest tagged release of this tool.
+    ///
+    /// ex: v1.1.0
+    pub async fn check_for_updates(&mut self, url: &str, handles: Arc<Handles>) -> Result<()> {
+        log::trace!("enter: needs_update({url}, {handles:?})");
+
+        let api_url = parse_url_with_raw_path(url)?;
+
+        // we don't want to leak sensitive header info / include auth headers
+        // with the github api request, so we'll build a client specifically
+        // for this task. thanks to @stuhlmann for the suggestion!
+        let headers = HashMap::new();
+        let client_cert = if handles.config.client_cert.is_empty() {
+            None
+        } else {
+            Some(handles.config.client_cert.as_str())
+        };
+        let client_key = if handles.config.client_key.is_empty() {
+            None
+        } else {
+            Some(handles.config.client_key.as_str())
+        };
+        let proxy = if handles.config.proxy.is_empty() {
+            None
+        } else {
+            Some(handles.config.proxy.as_str())
+        };
+        let client_config = client::ClientConfig {
+            timeout: handles.config.timeout,
+            user_agent: "feroxbuster-update-check",
+            redirects: handles.config.redirects,
+            insecure: handles.config.insecure,
+            headers: &headers,
+            proxy,
+            server_certs: Some(&handles.config.server_certs),
+            client_cert,
+            client_key,
+            scope: &handles.config.scope,
+        };
+        let client = client::initialize(client_config)?;
+        let level = handles.config.output_level;
+        let tx_stats = handles.stats.tx.clone();
+
+        let result = make_request(
+            &client,
+            &api_url,
+            DEFAULT_METHOD,
+            None,
+            level,
+            &handles.config,
+            tx_stats,
+        )
+        .await?;
+
+        let body = result.text().await?;
+
+        let json_response: Value = serde_json::from_str(&body)?;
+
+        let latest_version = match json_response["tag_name"].as_str() {
+            Some(tag) => tag.trim_start_matches('v'),
+            None => {
+                bail!("JSON has no tag_name: {json_response}");
+            }
+        };
+
+        // if we've gotten this far, we have a string in the form of X.X.X where X is a number
+        // all that's left is to compare the current version with the version found above
+
+        if latest_version == self.version {
+            // there's really only two possible outcomes if we accept that the tag conforms to
+            // the X.X.X pattern:
+            //   1. the version strings match, meaning we're up to date
+            //   2. the version strings do not match, meaning we're out of date
+            //
+            // except for developers working on this code, nobody should ever be in a situation
+            // where they have a version greater than the latest tagged release
+            self.update_status = UpdateStatus::UpToDate;
+        } else {
+            self.update_status = UpdateStatus::OutOfDate;
+        }
+
+        log::trace!("exit: check_for_updates -> {:?}", self.update_status);
+        Ok(())
+    }
+
+    /// display the banner on Write writer
+    pub fn print_to<W>(&self, mut writer: W, config: Arc<Configuration>) -> Result<()>
+    where
+        W: Write,
+    {
+        writeln!(&mut writer, "{}", self.header())?;
+
+        // begin with always printed items
+        for target in &self.targets {
+            writeln!(&mut writer, "{target}")?;
+        }
+
+        for denied_url in &self.url_denylist {
+            writeln!(&mut writer, "{denied_url}")?;
+        }
+
+        for scoped_url in &self.scope {
+            writeln!(&mut writer, "{scoped_url}")?;
+        }
+
+        writeln!(&mut writer, "{}", self.threads)?;
+        writeln!(&mut writer, "{}", self.wordlist)?;
+
+        if config.filter_status.is_empty() {
+            // -C and -s are mutually exclusive, and -s meaning changes when -C is used
+            // so only print one or the other
+            writeln!(&mut writer, "{}", self.status_codes)?;
+        } else {
+            writeln!(&mut writer, "{}", self.filter_status)?;
+        }
+
+        writeln!(&mut writer, "{}", self.timeout)?;
+
+        if config.random_agent {
+            writeln!(&mut writer, "{}", self.random_agent)?;
+        } else {
+            writeln!(&mut writer, "{}", self.user_agent)?;
+        }
+
+        // followed by the maybe printed or variably displayed values
+        if !config.request_file.is_empty() {
+            writeln!(&mut writer, "{}", self.protocol)?;
+        }
+
+        if config.limit_bars > 0 {
+            writeln!(&mut writer, "{}", self.limit_bars)?;
+        }
+
+        if !config.config.is_empty() {
+            writeln!(&mut writer, "{}", self.config)?;
+        }
+
+        if !config.proxy.is_empty() {
+            writeln!(&mut writer, "{}", self.proxy)?;
+        }
+
+        if !config.client_cert.is_empty() {
+            writeln!(&mut writer, "{}", self.client_cert)?;
+        }
+
+        if !config.client_key.is_empty() {
+            writeln!(&mut writer, "{}", self.client_key)?;
+        }
+
+        if !config.server_certs.is_empty() {
+            writeln!(&mut writer, "{}", self.server_certs)?;
+        }
+
+        if !config.replay_proxy.is_empty() {
+            // i include replay codes logic here because in config.rs, replay codes are set to the
+            // value in status codes, meaning it's never empty
+            writeln!(&mut writer, "{}", self.replay_proxy)?;
+            writeln!(&mut writer, "{}", self.replay_codes)?;
+        }
+
+        for header in &self.headers {
+            writeln!(&mut writer, "{header}")?;
+        }
+
+        for filter in &self.filter_size {
+            writeln!(&mut writer, "{filter}")?;
+        }
+
+        for filter in &self.filter_similar {
+            writeln!(&mut writer, "{filter}")?;
+        }
+
+        for filter in &self.filter_word_count {
+            writeln!(&mut writer, "{filter}")?;
+        }
+
+        for filter in &self.filter_line_count {
+            writeln!(&mut writer, "{filter}")?;
+        }
+
+        for filter in &self.filter_regex {
+            writeln!(&mut writer, "{filter}")?;
+        }
+
+        if config.extract_links {
+            writeln!(&mut writer, "{}", self.extract_links)?;
+        }
+
+        if config.json {
+            writeln!(&mut writer, "{}", self.json)?;
+        }
+
+        for query in &self.queries {
+            writeln!(&mut writer, "{query}")?;
+        }
+
+        if !config.output.is_empty() {
+            writeln!(&mut writer, "{}", self.output)?;
+        }
+
+        if config.scan_dir_listings {
+            writeln!(&mut writer, "{}", self.scan_dir_listings)?;
+        }
+
+        if !config.debug_log.is_empty() {
+            writeln!(&mut writer, "{}", self.debug_log)?;
+        }
+
+        if !config.extensions.is_empty() {
+            writeln!(&mut writer, "{}", self.extensions)?;
+        }
+
+        if config.collect_extensions {
+            // dont-collect is active only when collect-extensions is used
+            writeln!(&mut writer, "{}", self.collect_extensions)?;
+            writeln!(&mut writer, "{}", self.dont_collect)?;
+        }
+
+        if config.collect_backups {
+            writeln!(&mut writer, "{}", self.collect_backups)?;
+        }
+
+        if config.collect_words {
+            writeln!(&mut writer, "{}", self.collect_words)?;
+        }
+
+        if !config.methods.is_empty() {
+            writeln!(&mut writer, "{}", self.methods)?;
+        }
+
+        if !config.data.is_empty() {
+            writeln!(&mut writer, "{}", self.data)?;
+        }
+
+        if config.insecure {
+            writeln!(&mut writer, "{}", self.insecure)?;
+        }
+
+        if config.auto_bail {
+            writeln!(&mut writer, "{}", self.auto_bail)?;
+        }
+        if config.auto_tune {
+            writeln!(&mut writer, "{}", self.auto_tune)?;
+        }
+
+        if config.redirects {
+            writeln!(&mut writer, "{}", self.redirects)?;
+        }
+
+        if config.dont_filter {
+            writeln!(&mut writer, "{}", self.dont_filter)?;
+        }
+
+        if let 1..=4 = config.verbosity {
+            writeln!(&mut writer, "{}", self.verbosity)?;
+        }
+
+        if config.add_slash {
+            writeln!(&mut writer, "{}", self.add_slash)?;
+        }
+
+        writeln!(&mut writer, "{}", self.no_recursion)?;
+
+        if config.force_recursion {
+            writeln!(&mut writer, "{}", self.force_recursion)?;
+        }
+
+        if config.scan_limit > 0 {
+            writeln!(&mut writer, "{}", self.scan_limit)?;
+        }
+
+        if config.parallel > 0 {
+            writeln!(&mut writer, "{}", self.parallel)?;
+        }
+
+        if config.rate_limit > 0 {
+            writeln!(&mut writer, "{}", self.rate_limit)?;
+        }
+
+        if !config.time_limit.is_empty() {
+            writeln!(&mut writer, "{}", self.time_limit)?;
+        }
+
+        if config.unique {
+            writeln!(&mut writer, "{}", self.unique)?;
+        }
+
+        if config.response_size_limit != 4194304 {
+            writeln!(&mut writer, "{}", self.response_size_limit)?;
+        }
+
+        if matches!(self.update_status, UpdateStatus::OutOfDate) {
+            let update = BannerEntry::new(
+                "🎉",
+                "New Version Available",
+                "https://github.com/epi052/feroxbuster/releases/latest",
+            );
+            writeln!(&mut writer, "{update}")?;
+        }
+
+        writeln!(&mut writer, "{}", self.footer())?;
+
+        Ok(())
+    }
+}

src/banner/entry.rs

@@ -0,0 +1,57 @@
+use console::{measure_text_width, Emoji};
+use std::fmt;
+
+/// Initial visual indentation size used in formatting banner entries
+const INDENT: usize = 3;
+
+/// Column width used in formatting banner entries
+const COL_WIDTH: usize = 22;
+
+/// Represents a single line on the banner
+#[derive(Default)]
+pub(super) struct BannerEntry {
+    /// emoji used in the banner entry
+    emoji: String,
+
+    /// title used in the banner entry
+    title: String,
+
+    /// value passed in via config/cli/defaults
+    value: String,
+}
+
+/// implementation of a banner entry
+impl BannerEntry {
+    /// Create a new banner entry from given fields
+    pub fn new(emoji: &str, title: &str, value: &str) -> Self {
+        BannerEntry {
+            emoji: emoji.to_string(),
+            title: title.to_string(),
+            value: value.to_string(),
+        }
+    }
+
+    /// Simple wrapper for emoji or fallback when terminal doesn't support emoji
+    fn format_emoji(&self) -> String {
+        let width = measure_text_width(&self.emoji);
+        let pad_len = width * width;
+        let pad = format!("{:<pad_len$}", "\u{0020}", pad_len = pad_len);
+        Emoji(&self.emoji, &pad).to_string()
+    }
+}
+
+/// Display implementation for a banner entry
+impl fmt::Display for BannerEntry {
+    /// Display formatter for the given banner entry
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "\u{0020}{:\u{0020}<indent$}{:\u{0020}<width$}\u{2502}\u{0020}{}",
+            self.format_emoji(),
+            self.title,
+            self.value,
+            indent = INDENT,
+            width = COL_WIDTH
+        )
+    }
+}

src/banner/mod.rs

@@ -0,0 +1,8 @@
+//! all logic related to building/printing the banner seen when scans start
+mod container;
+mod entry;
+
+#[cfg(test)]
+mod tests;
+
+pub use self::container::{Banner, UPDATE_URL};

src/banner/tests.rs

@@ -0,0 +1,174 @@
+use super::container::UpdateStatus;
+use super::*;
+use crate::{config::Configuration, event_handlers::Handles, scan_manager::FeroxScans};
+use httpmock::Method::GET;
+use httpmock::MockServer;
+use std::{io::stderr, sync::Arc, time::Duration};
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test to hit no execution of targets for loop in banner
+async fn banner_intialize_without_targets() {
+    let config = Configuration::new().unwrap();
+    let banner = Banner::new(&[], &config);
+    banner.print_to(stderr(), Arc::new(config)).unwrap();
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test to hit no execution of statuscode for loop in banner
+async fn banner_intialize_without_status_codes() {
+    let config = Configuration {
+        status_codes: vec![],
+        ..Default::default()
+    };
+
+    let banner = Banner::new(&[String::from("http://localhost")], &config);
+    banner.print_to(stderr(), Arc::new(config)).unwrap();
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test to hit an empty config file
+async fn banner_intialize_without_config_file() {
+    let config = Configuration {
+        config: String::new(),
+        ..Default::default()
+    };
+
+    let banner = Banner::new(&[String::from("http://localhost")], &config);
+    banner.print_to(stderr(), Arc::new(config)).unwrap();
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test to hit an empty queries
+async fn banner_intialize_without_queries() {
+    let config = Configuration {
+        queries: vec![(String::new(), String::new())],
+        ..Default::default()
+    };
+
+    let banner = Banner::new(&[String::from("http://localhost")], &config);
+    banner.print_to(stderr(), Arc::new(config)).unwrap();
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test that
+async fn banner_needs_update_returns_unknown_with_bad_url() {
+    let handles = Arc::new(Handles::for_testing(None, None).0);
+
+    let mut banner = Banner::new(
+        &[String::from("http://localhost")],
+        &Configuration::new().unwrap(),
+    );
+
+    let _ = banner.check_for_updates("", handles).await;
+
+    assert!(matches!(banner.update_status, UpdateStatus::Unknown));
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test return value of good url to needs_update
+async fn banner_needs_update_returns_up_to_date() {
+    let srv = MockServer::start();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/latest");
+        then.status(200).body("{\"tag_name\":\"v1.1.0\"}");
+    });
+    let scans = Arc::new(FeroxScans::default());
+
+    let handles = Arc::new(Handles::for_testing(Some(scans), None).0);
+
+    let mut banner = Banner::new(&[srv.url("")], &Configuration::new().unwrap());
+    banner.version = String::from("1.1.0");
+
+    let _ = banner.check_for_updates(&srv.url("/latest"), handles).await;
+
+    assert_eq!(mock.hits(), 1);
+    assert!(matches!(banner.update_status, UpdateStatus::UpToDate));
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test return value of good url to needs_update that returns a newer version than current
+async fn banner_needs_update_returns_out_of_date() {
+    let srv = MockServer::start();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/latest");
+        then.status(200).body("{\"tag_name\":\"v1.1.0\"}");
+    });
+
+    let scans = Arc::new(FeroxScans::default());
+
+    let handles = Arc::new(Handles::for_testing(Some(scans), None).0);
+
+    let mut banner = Banner::new(&[srv.url("")], &Configuration::new().unwrap());
+    banner.version = String::from("1.0.1");
+
+    let _ = banner.check_for_updates(&srv.url("/latest"), handles).await;
+
+    assert_eq!(mock.hits(), 1);
+    assert!(matches!(banner.update_status, UpdateStatus::OutOfDate));
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test return value of good url that times out
+async fn banner_needs_update_returns_unknown_on_timeout() {
+    let srv = MockServer::start();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/latest");
+        then.status(200)
+            .body("{\"tag_name\":\"v1.1.0\"}")
+            .delay(Duration::from_secs(8));
+    });
+
+    let handles = Arc::new(Handles::for_testing(None, None).0);
+
+    let mut banner = Banner::new(&[srv.url("")], &Configuration::new().unwrap());
+
+    let _ = banner.check_for_updates(&srv.url("/latest"), handles).await;
+
+    assert_eq!(mock.hits(), 1);
+    assert!(matches!(banner.update_status, UpdateStatus::Unknown));
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test return value of good url with bad json response
+async fn banner_needs_update_returns_unknown_on_bad_json_response() {
+    let srv = MockServer::start();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/latest");
+        then.status(200).body("not json");
+    });
+
+    let handles = Arc::new(Handles::for_testing(None, None).0);
+
+    let mut banner = Banner::new(&[srv.url("")], &Configuration::new().unwrap());
+
+    let _ = banner.check_for_updates(&srv.url("/latest"), handles).await;
+
+    assert_eq!(mock.hits(), 1);
+    assert!(matches!(banner.update_status, UpdateStatus::Unknown));
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test return value of good url with json response that lacks the tag_name field
+async fn banner_needs_update_returns_unknown_on_json_without_correct_tag() {
+    let srv = MockServer::start();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/latest");
+        then.status(200)
+            .body("{\"no tag_name\": \"doesn't exist\"}");
+    });
+
+    let handles = Arc::new(Handles::for_testing(None, None).0);
+
+    let mut banner = Banner::new(&[srv.url("")], &Configuration::new().unwrap());
+    banner.version = String::from("1.0.1");
+
+    let _ = banner.check_for_updates(&srv.url("/latest"), handles).await;
+
+    assert_eq!(mock.hits(), 1);
+    assert!(matches!(banner.update_status, UpdateStatus::Unknown));
+}

src/client.rs

@@ -1,85 +1,132 @@
-use crate::utils::{module_colorizer, status_colorizer};
+use crate::url::UrlExt;
+use anyhow::{Context, Result};
 use reqwest::header::HeaderMap;
 use reqwest::{redirect::Policy, Client, Proxy};
 use std::collections::HashMap;
 use std::convert::TryInto;
-#[cfg(not(test))]
-use std::process::exit;
+use std::path::Path;
 use std::time::Duration;
+use url::Url;
 
-/// Create and return an instance of [reqwest::Client](https://docs.rs/reqwest/latest/reqwest/struct.Client.html)
-pub fn initialize(
-    timeout: u64,
-    useragent: &str,
-    redirects: bool,
-    insecure: bool,
-    headers: &HashMap<String, String>,
-    proxy: Option<&str>,
-) -> Client {
-    let policy = if redirects {
+/// Configuration struct for initializing a reqwest client
+pub struct ClientConfig<'a, I>
+where
+    I: IntoIterator,
+    I::Item: AsRef<Path> + std::fmt::Debug,
+{
+    /// The timeout for requests in seconds
+    pub timeout: u64,
+    /// The User-Agent string to use for requests
+    pub user_agent: &'a str,
+    /// Whether to follow redirects
+    pub redirects: bool,
+    /// Whether to allow insecure connections
+    pub insecure: bool,
+    /// Headers to include in requests
+    pub headers: &'a HashMap<String, String>,
+    /// Proxy server to use for requests
+    pub proxy: Option<&'a str>,
+    /// Server certificates to use for requests
+    pub server_certs: Option<I>,
+    /// Client certificate to use for requests
+    pub client_cert: Option<&'a str>,
+    /// Client key to use for requests
+    pub client_key: Option<&'a str>,
+    /// scope for redirect handling
+    pub scope: &'a [Url],
+}
+
+/// Create a redirect policy based on the provided config
+fn create_redirect_policy<I>(config: &ClientConfig<'_, I>) -> Policy
+where
+    I: IntoIterator,
+    I::Item: AsRef<Path> + std::fmt::Debug,
+{
+    // old behavior set Policy::limited(10) if redirects were enabled
+    // and Policy::none() if they were not. New policy behavior is
+    // scope-aware when redirects are enabled and scope is provided.
+
+    if config.redirects && config.scope.is_empty() {
+        // scope should never be empty, so this should never be hit, just a fallback
         Policy::limited(10)
+    } else if config.redirects {
+        // create a custom policy that checks scope for each redirect
+        let scoped_urls = config.scope.to_vec();
+
+        Policy::custom(move |attempt| {
+            let redirect_url = attempt.url();
+
+            if redirect_url.is_in_scope(&scoped_urls) {
+                attempt.follow()
+            } else {
+                attempt.stop()
+            }
+        })
     } else {
         Policy::none()
-    };
+    }
+}
 
-    // try_into returns infallible as its error, unwrap is safe here
-    let header_map: HeaderMap = headers.try_into().unwrap();
+/// Create and return an instance of [reqwest::Client](https://docs.rs/reqwest/latest/reqwest/struct.Client.html)
+/// with optional scope-aware redirect handling
+pub fn initialize<I>(config: ClientConfig<'_, I>) -> Result<Client>
+where
+    I: IntoIterator,
+    I::Item: AsRef<Path> + std::fmt::Debug,
+{
+    let policy = create_redirect_policy(&config);
 
-    let client = Client::builder()
-        .timeout(Duration::new(timeout, 0))
-        .user_agent(useragent)
-        .danger_accept_invalid_certs(insecure)
+    let header_map: HeaderMap = config.headers.try_into()?;
+
+    let mut client = Client::builder()
+        .timeout(Duration::new(config.timeout, 0))
+        .user_agent(config.user_agent)
+        .danger_accept_invalid_certs(config.insecure)
         .default_headers(header_map)
-        .redirect(policy);
+        .redirect(policy)
+        .http1_title_case_headers();
 
-    let client = if proxy.is_some() && !proxy.unwrap().is_empty() {
-        match Proxy::all(proxy.unwrap()) {
-            Ok(proxy_obj) => client.proxy(proxy_obj),
-            Err(e) => {
-                eprintln!(
-                    "{} {} Could not add proxy ({:?}) to Client configuration",
-                    status_colorizer("ERROR"),
-                    module_colorizer("Client::initialize"),
-                    proxy
-                );
-                eprintln!(
-                    "{} {} {}",
-                    status_colorizer("ERROR"),
-                    module_colorizer("Client::initialize"),
-                    e
-                );
-
-                #[cfg(test)]
-                panic!();
-                #[cfg(not(test))]
-                exit(1);
-            }
-        }
-    } else {
-        client
-    };
-
-    match client.build() {
-        Ok(client) => client,
-        Err(e) => {
-            eprintln!(
-                "{} {} Could not create a Client with the given configuration, exiting.",
-                status_colorizer("ERROR"),
-                module_colorizer("Client::build")
-            );
-            eprintln!(
-                "{} {} {}",
-                status_colorizer("ERROR"),
-                module_colorizer("Client::build"),
-                e
-            );
-
-            #[cfg(test)]
-            panic!();
-            #[cfg(not(test))]
-            exit(1);
+    if let Some(some_proxy) = config.proxy {
+        if !some_proxy.is_empty() {
+            // it's not an empty string; set the proxy
+            let proxy_obj = Proxy::all(some_proxy)?;
+            // just add the proxy to the client
+            // don't build and return it just yet
+            client = client.proxy(proxy_obj);
         }
     }
+
+    for cert_path in config.server_certs.into_iter().flatten() {
+        let buf = std::fs::read(&cert_path)?;
+
+        let cert = match reqwest::Certificate::from_pem(&buf) {
+            Ok(cert) => cert,
+            Err(err) => reqwest::Certificate::from_der(&buf).with_context(|| {
+                format!(
+                    "{:?} does not contain a valid PEM or DER certificate\n{}",
+                    &cert_path, err
+                )
+            })?,
+        };
+
+        client = client.add_root_certificate(cert);
+    }
+
+    if let (Some(cert_path), Some(key_path)) = (config.client_cert, config.client_key) {
+        if !cert_path.is_empty() && !key_path.is_empty() {
+            let cert = std::fs::read(cert_path)?;
+            let key = std::fs::read(key_path)?;
+
+            let identity = reqwest::Identity::from_pkcs8_pem(&cert, &key).with_context(|| {
+                format!(
+                    "either {cert_path} or {key_path} are invalid; expecting PEM encoded certificate and key")
+            })?;
+
+            client = client.identity(identity);
+        }
+    }
+
+    Ok(client.build()?)
 }
 
 #[cfg(test)]
@@ -91,7 +138,19 @@ mod tests {
     /// create client with a bad proxy, expect panic
     fn client_with_bad_proxy() {
         let headers = HashMap::new();
-        initialize(0, "stuff", true, false, &headers, Some("not a valid proxy"));
+        let client_config = ClientConfig {
+            timeout: 0,
+            user_agent: "stuff",
+            redirects: true,
+            insecure: false,
+            headers: &headers,
+            proxy: Some("not a valid proxy"),
+            server_certs: Option::<Vec<String>>::None,
+            client_cert: None,
+            client_key: None,
+            scope: &Vec::new(),
+        };
+        initialize(client_config).unwrap();
     }
 
     #[test]
@@ -99,6 +158,154 @@ mod tests {
     fn client_with_good_proxy() {
         let headers = HashMap::new();
         let proxy = "http://127.0.0.1:8080";
-        initialize(0, "stuff", true, true, &headers, Some(proxy));
+        let client_config = ClientConfig {
+            timeout: 0,
+            user_agent: "stuff",
+            redirects: true,
+            insecure: true,
+            headers: &headers,
+            proxy: Some(proxy),
+            server_certs: Option::<Vec<String>>::None,
+            client_cert: None,
+            client_key: None,
+            scope: &Vec::new(),
+        };
+        initialize(client_config).unwrap();
+    }
+
+    #[test]
+    /// create client with a server cert in pem format, expect no error
+    fn client_with_valid_server_pem() {
+        let headers = HashMap::new();
+        let server_certs = vec!["tests/mutual-auth/certs/server/server.crt.1".to_string()];
+        let client_config = ClientConfig {
+            timeout: 0,
+            user_agent: "stuff",
+            redirects: true,
+            insecure: true,
+            headers: &headers,
+            proxy: None,
+            server_certs: Some(server_certs),
+            client_cert: None,
+            client_key: None,
+            scope: &Vec::new(),
+        };
+        initialize(client_config).unwrap();
+    }
+
+    #[test]
+    /// create client with a server cert in der format, expect no error
+    fn client_with_valid_server_der() {
+        let headers = HashMap::new();
+        let server_certs = vec!["tests/mutual-auth/certs/server/server.der".to_string()];
+        let client_config = ClientConfig {
+            timeout: 0,
+            user_agent: "stuff",
+            redirects: true,
+            insecure: true,
+            headers: &headers,
+            proxy: None,
+            server_certs: Some(server_certs),
+            client_cert: None,
+            client_key: None,
+            scope: &Vec::new(),
+        };
+        initialize(client_config).unwrap();
+    }
+
+    #[test]
+    /// create client with two server certs (pem and der), expect no error
+    fn client_with_valid_server_pem_and_der() {
+        let headers = HashMap::new();
+        let server_certs = vec![
+            "tests/mutual-auth/certs/server/server.crt.1".to_string(),
+            "tests/mutual-auth/certs/server/server.der".to_string(),
+        ];
+
+        println!("{}", std::env::current_dir().unwrap().display());
+
+        let client_config = ClientConfig {
+            timeout: 0,
+            user_agent: "stuff",
+            redirects: true,
+            insecure: true,
+            headers: &headers,
+            proxy: None,
+            server_certs: Some(server_certs),
+            client_cert: None,
+            client_key: None,
+            scope: &Vec::new(),
+        };
+        initialize(client_config).unwrap();
+    }
+
+    /// create client with invalid certificate, expect panic
+    #[test]
+    #[should_panic]
+    fn client_with_invalid_server_cert() {
+        let headers = HashMap::new();
+        let server_certs = vec!["tests/mutual-auth/certs/client/client.key".to_string()];
+        let client_config = ClientConfig {
+            timeout: 0,
+            user_agent: "stuff",
+            redirects: true,
+            insecure: true,
+            headers: &headers,
+            proxy: None,
+            server_certs: Some(server_certs),
+            client_cert: None,
+            client_key: None,
+            scope: &Vec::new(),
+        };
+        initialize(client_config).unwrap();
+    }
+
+    #[test]
+    /// test that scope-aware client can be created with valid parameters
+    fn initialize_with_scope_creates_client() {
+        let headers = HashMap::new();
+        let scope = vec![
+            Url::parse("https://api.example.com").unwrap(),
+            Url::parse("https://cdn.example.com").unwrap(),
+        ];
+
+        let client_config = ClientConfig {
+            timeout: 5,
+            user_agent: "test-agent",
+            redirects: true,
+            insecure: false,
+            headers: &headers,
+            proxy: None,
+            server_certs: Option::<Vec<String>>::None,
+            client_cert: None,
+            client_key: None,
+            scope: &scope,
+        };
+        let client = initialize(client_config);
+
+        assert!(client.is_ok());
+    }
+
+    #[test]
+    /// test that scope-aware client works without scope (should use default behavior)
+    fn initialize_with_scope_empty_scope() {
+        let headers = HashMap::new();
+        let scope = vec![];
+
+        let client_config = ClientConfig {
+            timeout: 5,
+            user_agent: "test-agent",
+            redirects: true,
+            insecure: false,
+            headers: &headers,
+            proxy: None,
+            server_certs: Option::<Vec<String>>::None,
+            client_cert: None,
+            client_key: None,
+            scope: &scope,
+        };
+        let client = initialize(client_config);
+
+        assert!(client.is_ok());
     }
 }

src/config.rs

@@ -1,770 +0,0 @@
-use crate::utils::{module_colorizer, status_colorizer};
-use crate::{client, parser, progress};
-use crate::{DEFAULT_CONFIG_NAME, DEFAULT_STATUS_CODES, DEFAULT_WORDLIST, VERSION};
-use clap::value_t;
-use indicatif::{MultiProgress, ProgressBar, ProgressDrawTarget};
-use lazy_static::lazy_static;
-use reqwest::{Client, StatusCode};
-use serde::Deserialize;
-use std::collections::HashMap;
-use std::env::{current_dir, current_exe};
-use std::fs::read_to_string;
-use std::path::PathBuf;
-use std::process::exit;
-
-lazy_static! {
-    /// Global configuration state
-    pub static ref CONFIGURATION: Configuration = Configuration::new();
-
-    /// Global progress bar that houses other progress bars
-    pub static ref PROGRESS_BAR: MultiProgress = MultiProgress::with_draw_target(ProgressDrawTarget::stdout());
-
-    /// Global progress bar that is only used for printing messages that don't jack up other bars
-    pub static ref PROGRESS_PRINTER: ProgressBar = progress::add_bar("", 0, true);
-}
-
-/// Represents the final, global configuration of the program.
-///
-/// This struct is the combination of the following:
-/// - default configuration values
-/// - plus overrides read from a configuration file
-/// - plus command-line options
-///
-/// In that order.
-///
-/// Inspired by and derived from https://github.com/PhilipDaniels/rust-config-example
-#[derive(Debug, Clone, Deserialize)]
-pub struct Configuration {
-    /// Path to the wordlist
-    #[serde(default = "wordlist")]
-    pub wordlist: String,
-
-    /// Path to the config file used
-    #[serde(default)]
-    pub config: String,
-
-    /// Proxy to use for requests (ex: http(s)://host:port, socks5://host:port)
-    #[serde(default)]
-    pub proxy: String,
-
-    /// The target URL
-    #[serde(default)]
-    pub target_url: String,
-
-    /// Status Codes of interest (default: 200 204 301 302 307 308 401 403 405)
-    #[serde(default = "statuscodes")]
-    pub statuscodes: Vec<u16>,
-
-    /// Instance of [reqwest::Client](https://docs.rs/reqwest/latest/reqwest/struct.Client.html)
-    #[serde(skip)]
-    pub client: Client,
-
-    /// Number of concurrent threads (default: 50)
-    #[serde(default = "threads")]
-    pub threads: usize,
-
-    /// Number of seconds before a request times out (default: 7)
-    #[serde(default = "timeout")]
-    pub timeout: u64,
-
-    /// Level of verbosity, equates to log level
-    #[serde(default)]
-    pub verbosity: u8,
-
-    /// Only print URLs
-    #[serde(default)]
-    pub quiet: bool,
-
-    /// Output file to write results to (default: stdout)
-    #[serde(default)]
-    pub output: String,
-
-    /// Sets the User-Agent (default: feroxbuster/VERSION)
-    #[serde(default = "useragent")]
-    pub useragent: String,
-
-    /// Follow redirects
-    #[serde(default)]
-    pub redirects: bool,
-
-    /// Disables TLS certificate validation
-    #[serde(default)]
-    pub insecure: bool,
-
-    /// File extension(s) to search for
-    #[serde(default)]
-    pub extensions: Vec<String>,
-
-    /// HTTP headers to be used in each request
-    #[serde(default)]
-    pub headers: HashMap<String, String>,
-
-    /// URL query parameters
-    #[serde(default)]
-    pub queries: Vec<(String, String)>,
-
-    /// Do not scan recursively
-    #[serde(default)]
-    pub norecursion: bool,
-
-    /// Extract links from html/javscript
-    #[serde(default)]
-    pub extract_links: bool,
-
-    /// Append / to each request
-    #[serde(default)]
-    pub addslash: bool,
-
-    /// Read url(s) from STDIN
-    #[serde(default)]
-    pub stdin: bool,
-
-    /// Maximum recursion depth, a depth of 0 is infinite recursion
-    #[serde(default = "depth")]
-    pub depth: usize,
-
-    /// Filter out messages of a particular size
-    #[serde(default)]
-    pub sizefilters: Vec<u64>,
-
-    /// Don't auto-filter wildcard responses
-    #[serde(default)]
-    pub dontfilter: bool,
-}
-
-// functions timeout, threads, statuscodes, useragent, wordlist, and depth are used to provide
-// defaults in the event that a ferox-config.toml is found but one or more of the values below
-// aren't listed in the config.  This way, we get the correct defaults upon Deserialization
-
-/// default timeout value
-fn timeout() -> u64 {
-    7
-}
-
-/// default threads value
-fn threads() -> usize {
-    50
-}
-
-/// default status codes
-fn statuscodes() -> Vec<u16> {
-    DEFAULT_STATUS_CODES
-        .iter()
-        .map(|code| code.as_u16())
-        .collect()
-}
-
-/// default wordlist
-fn wordlist() -> String {
-    String::from(DEFAULT_WORDLIST)
-}
-
-/// default useragent
-fn useragent() -> String {
-    format!("feroxbuster/{}", VERSION)
-}
-
-/// default recursion depth
-fn depth() -> usize {
-    4
-}
-
-impl Default for Configuration {
-    /// Builds the default Configuration for feroxbuster
-    fn default() -> Self {
-        let timeout = timeout();
-        let useragent = useragent();
-        let client = client::initialize(timeout, &useragent, false, false, &HashMap::new(), None);
-
-        Configuration {
-            client,
-            timeout,
-            useragent,
-            dontfilter: false,
-            quiet: false,
-            stdin: false,
-            verbosity: 0,
-            addslash: false,
-            insecure: false,
-            redirects: false,
-            norecursion: false,
-            extract_links: false,
-            proxy: String::new(),
-            config: String::new(),
-            output: String::new(),
-            target_url: String::new(),
-            queries: Vec::new(),
-            extensions: Vec::new(),
-            sizefilters: Vec::new(),
-            headers: HashMap::new(),
-            threads: threads(),
-            depth: depth(),
-            wordlist: wordlist(),
-            statuscodes: statuscodes(),
-        }
-    }
-}
-
-impl Configuration {
-    /// Creates a [Configuration](struct.Configuration.html) object with the following
-    /// built-in default values
-    ///
-    /// - **timeout**: `5` seconds
-    /// - **redirects**: `false`
-    /// - **extract-links**: `false`
-    /// - **wordlist**: [`DEFAULT_WORDLIST`](constant.DEFAULT_WORDLIST.html)
-    /// - **config**: `None`
-    /// - **threads**: `50`
-    /// - **timeout**: `7` seconds
-    /// - **verbosity**: `0` (no logging enabled)
-    /// - **proxy**: `None`
-    /// - **statuscodes**: [`DEFAULT_RESPONSE_CODES`](constant.DEFAULT_RESPONSE_CODES.html)
-    /// - **output**: `None` (print to stdout)
-    /// - **quiet**: `false`
-    /// - **useragent**: `feroxer/VERSION`
-    /// - **insecure**: `false` (don't be insecure, i.e. don't allow invalid certs)
-    /// - **extensions**: `None`
-    /// - **sizefilters**: `None`
-    /// - **headers**: `None`
-    /// - **queries**: `None`
-    /// - **norecursion**: `false` (recursively scan enumerated sub-directories)
-    /// - **addslash**: `false`
-    /// - **stdin**: `false`
-    /// - **dontfilter**: `false` (auto filter wildcard responses)
-    /// - **depth**: `4` (maximum recursion depth)
-    ///
-    /// After which, any values defined in a
-    /// [ferox-config.toml](constant.DEFAULT_CONFIG_NAME.html) config file will override the
-    /// built-in defaults.
-    ///
-    /// `ferox-config.toml` can be placed in any of the following locations (in the order shown):
-    /// - `/etc/feroxbuster/`
-    /// - `CONFIG_DIR/ferxobuster/`
-    /// - The same directory as the `feroxbuster` executable
-    /// - The user's current working directory
-    ///
-    /// If more than one valid configuration file is found, each one overwrites the values found previously.
-    ///
-    /// Finally, any options/arguments given on the commandline will override both built-in and
-    /// config-file specified values.
-    ///
-    /// The resulting [Configuration](struct.Configuration.html) is a singleton with a `static`
-    /// lifetime.
-    pub fn new() -> Self {
-        // when compiling for test, we want to eliminate the runtime dependency of the parser
-        if cfg!(test) {
-            return Configuration::default();
-        }
-
-        // Get the default configuration, this is what will apply if nothing
-        // else is specified.
-        let mut config = Configuration::default();
-
-        // Next, we parse the ferox-config.toml file, if present and set the values
-        // therein to overwrite our default values. Deserialized defaults are specified
-        // in the Configuration struct so that we don't change anything that isn't
-        // actually specified in the config file
-        //
-        // search for a config using the following order of precedence
-        //   - /etc/feroxbuster/
-        //   - CONFIG_DIR/ferxobuster/
-        //   - same directory as feroxbuster executable
-        //   - current directory
-
-        // merge a config found at /etc/feroxbuster/ferox-config.toml
-        let config_file = PathBuf::new()
-            .join("/etc/feroxbuster")
-            .join(DEFAULT_CONFIG_NAME);
-        Self::parse_and_merge_config(config_file, &mut config);
-
-        // merge a config found at ~/.config/feroxbuster/ferox-config.toml
-        if let Some(config_dir) = dirs::config_dir() {
-            // config_dir() resolves to one of the following
-            //   - linux: $XDG_CONFIG_HOME or $HOME/.config
-            //   - macOS: $HOME/Library/Application Support
-            //   - windows: {FOLDERID_RoamingAppData}
-
-            let config_file = config_dir.join("feroxbuster").join(DEFAULT_CONFIG_NAME);
-            Self::parse_and_merge_config(config_file, &mut config);
-        };
-
-        // merge a config found in same the directory as feroxbuster executable
-        if let Ok(exe_path) = current_exe() {
-            if let Some(bin_dir) = exe_path.parent() {
-                let config_file = bin_dir.join(DEFAULT_CONFIG_NAME);
-                Self::parse_and_merge_config(config_file, &mut config);
-            };
-        };
-
-        // merge a config found in the user's current working directory
-        if let Ok(cwd) = current_dir() {
-            let config_file = cwd.join(DEFAULT_CONFIG_NAME);
-            Self::parse_and_merge_config(config_file, &mut config);
-        }
-
-        let args = parser::initialize().get_matches();
-
-        // the .is_some appears clunky, but it allows default values to be incrementally
-        // overwritten from Struct defaults, to file config, to command line args, soooo ¯\_(ツ)_/¯
-        if args.value_of("threads").is_some() {
-            let threads = value_t!(args.value_of("threads"), usize).unwrap_or_else(|e| e.exit());
-            config.threads = threads;
-        }
-
-        if args.value_of("depth").is_some() {
-            let depth = value_t!(args.value_of("depth"), usize).unwrap_or_else(|e| e.exit());
-            config.depth = depth;
-        }
-
-        if args.value_of("wordlist").is_some() {
-            config.wordlist = String::from(args.value_of("wordlist").unwrap());
-        }
-
-        if args.value_of("output").is_some() {
-            config.output = String::from(args.value_of("output").unwrap());
-        }
-
-        if args.values_of("statuscodes").is_some() {
-            config.statuscodes = args
-                .values_of("statuscodes")
-                .unwrap() // already known good
-                .map(|code| {
-                    StatusCode::from_bytes(code.as_bytes())
-                        .unwrap_or_else(|e| {
-                            eprintln!(
-                                "{} {}: {}",
-                                status_colorizer("ERROR"),
-                                module_colorizer("Configuration::new"),
-                                e
-                            );
-                            exit(1)
-                        })
-                        .as_u16()
-                })
-                .collect();
-        }
-
-        if args.values_of("extensions").is_some() {
-            config.extensions = args
-                .values_of("extensions")
-                .unwrap()
-                .map(|val| val.to_string())
-                .collect();
-        }
-
-        if args.values_of("sizefilters").is_some() {
-            config.sizefilters = args
-                .values_of("sizefilters")
-                .unwrap() // already known good
-                .map(|size| {
-                    size.parse::<u64>().unwrap_or_else(|e| {
-                        eprintln!(
-                            "{} {}: {}",
-                            status_colorizer("ERROR"),
-                            module_colorizer("Configuration::new"),
-                            e
-                        );
-                        exit(1)
-                    })
-                })
-                .collect();
-        }
-
-        if args.is_present("quiet") {
-            // the reason this is protected by an if statement:
-            // consider a user specifying quiet = true in ferox-config.toml
-            // if the line below is outside of the if, we'd overwrite true with
-            // false if no -q is used on the command line
-            config.quiet = args.is_present("quiet");
-        }
-
-        if args.is_present("dontfilter") {
-            config.dontfilter = args.is_present("dontfilter");
-        }
-
-        if args.occurrences_of("verbosity") > 0 {
-            // occurrences_of returns 0 if none are found; this is protected in
-            // an if block for the same reason as the quiet option
-            config.verbosity = args.occurrences_of("verbosity") as u8;
-        }
-
-        if args.is_present("norecursion") {
-            config.norecursion = args.is_present("norecursion");
-        }
-
-        if args.is_present("addslash") {
-            config.addslash = args.is_present("addslash");
-        }
-
-        if args.is_present("extract_links") {
-            config.extract_links = args.is_present("extract_links");
-        }
-
-        if args.is_present("stdin") {
-            config.stdin = args.is_present("stdin");
-        } else {
-            config.target_url = String::from(args.value_of("url").unwrap());
-        }
-
-        ////
-        // organizational breakpoint; all options below alter the Client configuration
-        ////
-        if args.value_of("proxy").is_some() {
-            config.proxy = String::from(args.value_of("proxy").unwrap());
-        }
-
-        if args.value_of("useragent").is_some() {
-            config.useragent = String::from(args.value_of("useragent").unwrap());
-        }
-
-        if args.value_of("timeout").is_some() {
-            let timeout = value_t!(args.value_of("timeout"), u64).unwrap_or_else(|e| e.exit());
-            config.timeout = timeout;
-        }
-
-        if args.is_present("redirects") {
-            config.redirects = args.is_present("redirects");
-        }
-
-        if args.is_present("insecure") {
-            config.insecure = args.is_present("insecure");
-        }
-
-        if args.values_of("headers").is_some() {
-            for val in args.values_of("headers").unwrap() {
-                let mut split_val = val.split(':');
-
-                // explicitly take first split value as header's name
-                let name = split_val.next().unwrap().trim();
-
-                // all other items in the iterator returned by split, when combined with the
-                // original split deliminator (:), make up the header's final value
-                let value = split_val.collect::<Vec<&str>>().join(":");
-                config.headers.insert(name.to_string(), value.to_string());
-            }
-        }
-
-        if args.values_of("queries").is_some() {
-            for val in args.values_of("queries").unwrap() {
-                // same basic logic used as reading in the headers HashMap above
-                let mut split_val = val.split('=');
-
-                let name = split_val.next().unwrap().trim();
-
-                let value = split_val.collect::<Vec<&str>>().join("=");
-
-                config.queries.push((name.to_string(), value.to_string()));
-            }
-        }
-
-        // this if statement determines if we've gotten a Client configuration change from
-        // either the config file or command line arguments; if we have, we need to rebuild
-        // the client and store it in the config struct
-        if !config.proxy.is_empty()
-            || config.timeout != timeout()
-            || config.useragent != useragent()
-            || config.redirects
-            || config.insecure
-            || !config.headers.is_empty()
-        {
-            if config.proxy.is_empty() {
-                config.client = client::initialize(
-                    config.timeout,
-                    &config.useragent,
-                    config.redirects,
-                    config.insecure,
-                    &config.headers,
-                    None,
-                )
-            } else {
-                config.client = client::initialize(
-                    config.timeout,
-                    &config.useragent,
-                    config.redirects,
-                    config.insecure,
-                    &config.headers,
-                    Some(&config.proxy),
-                )
-            }
-        }
-
-        config
-    }
-
-    /// Given a configuration file's location and an instance of `Configuration`, read in
-    /// the config file if found and update the current settings with the settings found therein
-    fn parse_and_merge_config(config_file: PathBuf, mut config: &mut Self) {
-        if config_file.exists() {
-            // save off a string version of the path before it goes out of scope
-            let conf_str = match config_file.to_str() {
-                Some(cs) => String::from(cs),
-                None => String::new(),
-            };
-
-            if let Some(settings) = Self::parse_config(config_file) {
-                // set the config used for viewing in the banner
-                config.config = conf_str;
-
-                // update the settings
-                Self::merge_config(&mut config, settings);
-            }
-        }
-    }
-
-    /// Given two Configurations, overwrite `settings` with the fields found in `settings_to_merge`
-    fn merge_config(settings: &mut Self, settings_to_merge: Self) {
-        settings.threads = settings_to_merge.threads;
-        settings.wordlist = settings_to_merge.wordlist;
-        settings.statuscodes = settings_to_merge.statuscodes;
-        settings.proxy = settings_to_merge.proxy;
-        settings.timeout = settings_to_merge.timeout;
-        settings.verbosity = settings_to_merge.verbosity;
-        settings.quiet = settings_to_merge.quiet;
-        settings.output = settings_to_merge.output;
-        settings.useragent = settings_to_merge.useragent;
-        settings.redirects = settings_to_merge.redirects;
-        settings.insecure = settings_to_merge.insecure;
-        settings.extract_links = settings_to_merge.extract_links;
-        settings.extensions = settings_to_merge.extensions;
-        settings.headers = settings_to_merge.headers;
-        settings.queries = settings_to_merge.queries;
-        settings.norecursion = settings_to_merge.norecursion;
-        settings.addslash = settings_to_merge.addslash;
-        settings.stdin = settings_to_merge.stdin;
-        settings.depth = settings_to_merge.depth;
-        settings.sizefilters = settings_to_merge.sizefilters;
-        settings.dontfilter = settings_to_merge.dontfilter;
-    }
-
-    /// If present, read in `DEFAULT_CONFIG_NAME` and deserialize the specified values
-    ///
-    /// uses serde to deserialize the toml into a `Configuration` struct
-    fn parse_config(config_file: PathBuf) -> Option<Self> {
-        if let Ok(content) = read_to_string(config_file) {
-            match toml::from_str(content.as_str()) {
-                Ok(config) => {
-                    return Some(config);
-                }
-                Err(e) => {
-                    println!(
-                        "{} {} {}",
-                        status_colorizer("ERROR"),
-                        module_colorizer("config::parse_config"),
-                        e
-                    );
-                }
-            }
-        }
-        None
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use std::fs::write;
-    use tempfile::TempDir;
-
-    /// creates a dummy configuration file for testing
-    fn setup_config_test() -> Configuration {
-        let data = r#"
-            wordlist = "/some/path"
-            statuscodes = [201, 301, 401]
-            threads = 40
-            timeout = 5
-            proxy = "http://127.0.0.1:8080"
-            quiet = true
-            verbosity = 1
-            output = "/some/otherpath"
-            redirects = true
-            insecure = true
-            extensions = ["html", "php", "js"]
-            headers = {stuff = "things", mostuff = "mothings"}
-            queries = [["name","value"], ["rick", "astley"]]
-            norecursion = true
-            addslash = true
-            stdin = true
-            dontfilter = true
-            extract_links = true
-            depth = 1
-            sizefilters = [4120]
-        "#;
-        let tmp_dir = TempDir::new().unwrap();
-        let file = tmp_dir.path().join(DEFAULT_CONFIG_NAME);
-        write(&file, data).unwrap();
-        Configuration::parse_config(file).unwrap()
-    }
-
-    #[test]
-    /// test that all default config values meet expectations
-    fn default_configuration() {
-        let config = Configuration::default();
-        assert_eq!(config.wordlist, wordlist());
-        assert_eq!(config.proxy, String::new());
-        assert_eq!(config.target_url, String::new());
-        assert_eq!(config.config, String::new());
-        assert_eq!(config.statuscodes, statuscodes());
-        assert_eq!(config.threads, threads());
-        assert_eq!(config.depth, depth());
-        assert_eq!(config.timeout, timeout());
-        assert_eq!(config.verbosity, 0);
-        assert_eq!(config.quiet, false);
-        assert_eq!(config.dontfilter, false);
-        assert_eq!(config.norecursion, false);
-        assert_eq!(config.stdin, false);
-        assert_eq!(config.addslash, false);
-        assert_eq!(config.redirects, false);
-        assert_eq!(config.extract_links, false);
-        assert_eq!(config.insecure, false);
-        assert_eq!(config.queries, Vec::new());
-        assert_eq!(config.extensions, Vec::<String>::new());
-        assert_eq!(config.sizefilters, Vec::<u64>::new());
-        assert_eq!(config.headers, HashMap::new());
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_wordlist() {
-        let config = setup_config_test();
-        assert_eq!(config.wordlist, "/some/path");
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_statuscodes() {
-        let config = setup_config_test();
-        assert_eq!(config.statuscodes, vec![201, 301, 401]);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_threads() {
-        let config = setup_config_test();
-        assert_eq!(config.threads, 40);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_depth() {
-        let config = setup_config_test();
-        assert_eq!(config.depth, 1);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_timeout() {
-        let config = setup_config_test();
-        assert_eq!(config.timeout, 5);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_proxy() {
-        let config = setup_config_test();
-        assert_eq!(config.proxy, "http://127.0.0.1:8080");
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_quiet() {
-        let config = setup_config_test();
-        assert_eq!(config.quiet, true);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_verbosity() {
-        let config = setup_config_test();
-        assert_eq!(config.verbosity, 1);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_output() {
-        let config = setup_config_test();
-        assert_eq!(config.output, "/some/otherpath");
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_redirects() {
-        let config = setup_config_test();
-        assert_eq!(config.redirects, true);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_insecure() {
-        let config = setup_config_test();
-        assert_eq!(config.insecure, true);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_norecursion() {
-        let config = setup_config_test();
-        assert_eq!(config.norecursion, true);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_stdin() {
-        let config = setup_config_test();
-        assert_eq!(config.stdin, true);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_dontfilter() {
-        let config = setup_config_test();
-        assert_eq!(config.dontfilter, true);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_addslash() {
-        let config = setup_config_test();
-        assert_eq!(config.addslash, true);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_extract_links() {
-        let config = setup_config_test();
-        assert_eq!(config.extract_links, true);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_extensions() {
-        let config = setup_config_test();
-        assert_eq!(config.extensions, vec!["html", "php", "js"]);
-    }
-
-    #[test]
-    /// parse the test config and see that the value parsed is correct
-    fn config_reads_sizefilters() {
-        let config = setup_config_test();
-        assert_eq!(config.sizefilters, vec![4120]);
-    }
-
-    #[test]
-    /// parse the test config and see that the values parsed are correct
-    fn config_reads_headers() {
-        let config = setup_config_test();
-        let mut headers = HashMap::new();
-        headers.insert("stuff".to_string(), "things".to_string());
-        headers.insert("mostuff".to_string(), "mothings".to_string());
-        assert_eq!(config.headers, headers);
-    }
-
-    #[test]
-    /// parse the test config and see that the values parsed are correct
-    fn config_reads_queries() {
-        let config = setup_config_test();
-        let mut queries = vec![];
-        queries.push(("name".to_string(), "value".to_string()));
-        queries.push(("rick".to_string(), "astley".to_string()));
-        assert_eq!(config.queries, queries);
-    }
-}

src/config/mod.rs

@@ -0,0 +1,9 @@
+//! all logic related to instantiating a running configuration
+
+mod container;
+mod utils;
+#[cfg(test)]
+mod tests;
+
+pub use self::container::Configuration;
+pub use self::utils::{determine_output_level, OutputLevel, RequesterPolicy};

src/config/tests.rs

@@ -0,0 +1,614 @@
+use super::utils::*;
+use super::*;
+use crate::{traits::FeroxSerialize, DEFAULT_CONFIG_NAME};
+use regex::Regex;
+use reqwest::Url;
+use std::{collections::HashMap, fs::write};
+use tempfile::TempDir;
+
+/// creates a dummy configuration file for testing
+fn setup_config_test() -> Configuration {
+    let data = r#"
+            wordlist = "/some/path"
+            status_codes = [201, 301, 401]
+            replay_codes = [201, 301]
+            threads = 40
+            timeout = 5
+            proxy = "http://127.0.0.1:8080"
+            replay_proxy = "http://127.0.0.1:8081"
+            quiet = true
+            silent = true
+            auto_tune = true
+            auto_bail = true
+            verbosity = 1
+            scan_limit = 6
+            parallel = 14
+            rate_limit = 250
+            time_limit = "10m"
+            output = "/some/otherpath"
+            debug_log = "/yet/anotherpath"
+            resume_from = "/some/state/file"
+            redirects = true
+            insecure = true
+            collect_backups = true
+            collect_extensions = true
+            collect_words = true
+            extensions = ["html", "php", "js"]
+            dont_collect = ["png", "gif", "jpg", "jpeg"]
+            methods = ["GET", "PUT", "DELETE"]
+            data = [31, 32, 33, 34]
+            url_denylist = ["http://dont-scan.me", "https://also-not.me"]
+            scope = ["http://example.com", "https://other.com"]
+            regex_denylist = ["/deny.*"]
+            headers = {stuff = "things", mostuff = "mothings"}
+            queries = [["name","value"], ["rick", "astley"]]
+            no_recursion = true
+            add_slash = true
+            stdin = true
+            dont_filter = true
+            extract_links = false
+            json = true
+            save_state = false
+            depth = 1
+            limit_bars = 3
+            protocol = "http"
+            request_file = "/some/request/file"
+            scan_dir_listings = true
+            force_recursion = true
+            filter_size = [4120]
+            filter_regex = ["^ignore me$"]
+            filter_similar = ["https://somesite.com/soft404"]
+            filter_word_count = [994, 992]
+            filter_line_count = [34]
+            filter_status = [201]
+            server_certs = ["/some/cert.pem", "/some/other/cert.pem"]
+            client_cert = "/some/client/cert.pem"
+            client_key = "/some/client/key.pem"
+            backup_extensions = [".save"]
+            unique = true
+            response_size_limit = 8388608
+        "#;
+    let tmp_dir = TempDir::new().unwrap();
+    let file = tmp_dir.path().join(DEFAULT_CONFIG_NAME);
+    write(&file, data).unwrap();
+    Configuration::parse_config(file).unwrap()
+}
+
+#[test]
+/// test that all default config values meet expectations
+fn default_configuration() {
+    let config = Configuration::default();
+    assert_eq!(config.wordlist, wordlist());
+    assert_eq!(config.proxy, String::new());
+    assert_eq!(config.target_url, String::new());
+    assert_eq!(config.time_limit, String::new());
+    assert_eq!(config.resume_from, String::new());
+    assert_eq!(config.debug_log, String::new());
+    assert_eq!(config.config, String::new());
+    assert_eq!(config.replay_proxy, String::new());
+    assert_eq!(config.status_codes, status_codes());
+    assert_eq!(config.replay_codes, config.status_codes);
+    assert!(config.replay_client.is_none());
+    assert_eq!(config.threads, threads());
+    assert_eq!(config.depth, depth());
+    assert_eq!(config.timeout, timeout());
+    assert_eq!(config.verbosity, 0);
+    assert_eq!(config.scan_limit, 0);
+    assert_eq!(config.limit_bars, 0);
+    assert!(!config.silent);
+    assert!(!config.quiet);
+    assert_eq!(config.output_level, OutputLevel::Default);
+    assert!(!config.dont_filter);
+    assert!(!config.auto_tune);
+    assert!(!config.auto_bail);
+    assert_eq!(config.requester_policy, RequesterPolicy::Default);
+    assert!(!config.no_recursion);
+    assert!(!config.random_agent);
+    assert!(!config.json);
+    assert!(config.save_state);
+    assert!(!config.stdin);
+    assert!(!config.add_slash);
+    assert!(!config.force_recursion);
+    assert!(!config.redirects);
+    assert!(config.extract_links);
+    assert!(!config.insecure);
+    assert!(!config.collect_extensions);
+    assert!(!config.collect_backups);
+    assert!(!config.collect_words);
+    assert!(!config.scan_dir_listings);
+    assert!(config.regex_denylist.is_empty());
+    assert_eq!(config.queries, Vec::new());
+    assert_eq!(config.filter_size, Vec::<u64>::new());
+    assert_eq!(config.extensions, Vec::<String>::new());
+    assert_eq!(config.methods, vec!["GET"]);
+    assert_eq!(config.data, Vec::<u8>::new());
+    assert_eq!(config.url_denylist, Vec::<Url>::new());
+    assert_eq!(config.scope, Vec::<Url>::new());
+    assert_eq!(config.dont_collect, ignored_extensions());
+    assert_eq!(config.filter_regex, Vec::<String>::new());
+    assert_eq!(config.filter_similar, Vec::<String>::new());
+    assert_eq!(config.filter_word_count, Vec::<usize>::new());
+    assert_eq!(config.filter_line_count, Vec::<usize>::new());
+    assert_eq!(config.filter_status, Vec::<u16>::new());
+    assert_eq!(config.headers, HashMap::new());
+    assert_eq!(config.server_certs, Vec::<String>::new());
+    assert_eq!(config.client_cert, String::new());
+    assert_eq!(config.client_key, String::new());
+    assert_eq!(config.backup_extensions, backup_extensions());
+    assert_eq!(config.protocol, request_protocol());
+    assert_eq!(config.request_file, String::new());
+    assert!(!config.unique);
+    assert_eq!(config.response_size_limit, 4194304); // 4MB
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_wordlist() {
+    let config = setup_config_test();
+    assert_eq!(config.wordlist, "/some/path");
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_debug_log() {
+    let config = setup_config_test();
+    assert_eq!(config.debug_log, "/yet/anotherpath");
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_status_codes() {
+    let config = setup_config_test();
+    assert_eq!(config.status_codes, vec![201, 301, 401]);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_replay_codes() {
+    let config = setup_config_test();
+    assert_eq!(config.replay_codes, vec![201, 301]);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_threads() {
+    let config = setup_config_test();
+    assert_eq!(config.threads, 40);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_depth() {
+    let config = setup_config_test();
+    assert_eq!(config.depth, 1);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_scan_limit() {
+    let config = setup_config_test();
+    assert_eq!(config.scan_limit, 6);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_parallel() {
+    let config = setup_config_test();
+    assert_eq!(config.parallel, 14);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_rate_limit() {
+    let config = setup_config_test();
+    assert_eq!(config.rate_limit, 250);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_timeout() {
+    let config = setup_config_test();
+    assert_eq!(config.timeout, 5);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_proxy() {
+    let config = setup_config_test();
+    assert_eq!(config.proxy, "http://127.0.0.1:8080");
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_replay_proxy() {
+    let config = setup_config_test();
+    assert_eq!(config.replay_proxy, "http://127.0.0.1:8081");
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_silent() {
+    let config = setup_config_test();
+    assert!(config.silent);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_force_recursion() {
+    let config = setup_config_test();
+    assert!(config.force_recursion);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_quiet() {
+    let config = setup_config_test();
+    assert!(config.quiet);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_json() {
+    let config = setup_config_test();
+    assert!(config.json);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_auto_bail() {
+    let config = setup_config_test();
+    assert!(config.auto_bail);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_auto_tune() {
+    let config = setup_config_test();
+    assert!(config.auto_tune);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_verbosity() {
+    let config = setup_config_test();
+    assert_eq!(config.verbosity, 1);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_limit_bars() {
+    let config = setup_config_test();
+    assert_eq!(config.limit_bars, 3);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_output() {
+    let config = setup_config_test();
+    assert_eq!(config.output, "/some/otherpath");
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_redirects() {
+    let config = setup_config_test();
+    assert!(config.redirects);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_insecure() {
+    let config = setup_config_test();
+    assert!(config.insecure);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_no_recursion() {
+    let config = setup_config_test();
+    assert!(config.no_recursion);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_stdin() {
+    let config = setup_config_test();
+    assert!(config.stdin);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_dont_filter() {
+    let config = setup_config_test();
+    assert!(config.dont_filter);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_add_slash() {
+    let config = setup_config_test();
+    assert!(config.add_slash);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_extract_links() {
+    let config = setup_config_test();
+    assert!(!config.extract_links);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_collect_extensions() {
+    let config = setup_config_test();
+    assert!(config.collect_extensions);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_collect_backups() {
+    let config = setup_config_test();
+    assert!(config.collect_backups);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_collect_words() {
+    let config = setup_config_test();
+    assert!(config.collect_words);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_extensions() {
+    let config = setup_config_test();
+    assert_eq!(config.extensions, vec!["html", "php", "js"]);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_dont_collect() {
+    let config = setup_config_test();
+    assert_eq!(config.dont_collect, vec!["png", "gif", "jpg", "jpeg"]);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_methods() {
+    let config = setup_config_test();
+    assert_eq!(config.methods, vec!["GET", "PUT", "DELETE"]);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_data() {
+    let config = setup_config_test();
+    assert_eq!(config.data, vec![31, 32, 33, 34]);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_regex_denylist() {
+    let config = setup_config_test();
+    assert_eq!(
+        config.regex_denylist[0].as_str(),
+        Regex::new("/deny.*").unwrap().as_str()
+    );
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_url_denylist() {
+    let config = setup_config_test();
+    assert_eq!(
+        config.url_denylist,
+        vec![
+            Url::parse("http://dont-scan.me").unwrap(),
+            Url::parse("https://also-not.me").unwrap(),
+        ]
+    );
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_scope() {
+    let config = setup_config_test();
+    assert_eq!(
+        config.scope,
+        vec![
+            Url::parse("http://example.com").unwrap(),
+            Url::parse("https://other.com").unwrap(),
+        ]
+    );
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_filter_regex() {
+    let config = setup_config_test();
+    assert_eq!(config.filter_regex, vec!["^ignore me$"]);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_filter_similar() {
+    let config = setup_config_test();
+    assert_eq!(config.filter_similar, vec!["https://somesite.com/soft404"]);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_filter_size() {
+    let config = setup_config_test();
+    assert_eq!(config.filter_size, vec![4120]);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_filter_word_count() {
+    let config = setup_config_test();
+    assert_eq!(config.filter_word_count, vec![994, 992]);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_filter_line_count() {
+    let config = setup_config_test();
+    assert_eq!(config.filter_line_count, vec![34]);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_filter_status() {
+    let config = setup_config_test();
+    assert_eq!(config.filter_status, vec![201]);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_save_state() {
+    let config = setup_config_test();
+    assert!(!config.save_state);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_time_limit() {
+    let config = setup_config_test();
+    assert_eq!(config.time_limit, "10m");
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_scan_dir_listings() {
+    let config = setup_config_test();
+    assert!(config.scan_dir_listings);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_protocol() {
+    let config = setup_config_test();
+    assert_eq!(config.protocol, "http");
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_request_file() {
+    let config = setup_config_test();
+    assert_eq!(config.request_file, String::new());
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_resume_from() {
+    let config = setup_config_test();
+    assert_eq!(config.resume_from, "/some/state/file");
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_server_certs() {
+    let config = setup_config_test();
+    assert_eq!(
+        config.server_certs,
+        ["/some/cert.pem", "/some/other/cert.pem"]
+    );
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_backup_extensions() {
+    let config = setup_config_test();
+    assert_eq!(config.backup_extensions, [".save"]);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_client_cert() {
+    let config = setup_config_test();
+    assert_eq!(config.client_cert, "/some/client/cert.pem");
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_client_key() {
+    let config = setup_config_test();
+    assert_eq!(config.client_key, "/some/client/key.pem");
+}
+
+#[test]
+/// parse the test config and see that the values parsed are correct
+fn config_reads_headers() {
+    let config = setup_config_test();
+    let mut headers = HashMap::new();
+    headers.insert("stuff".to_string(), "things".to_string());
+    headers.insert("mostuff".to_string(), "mothings".to_string());
+    assert_eq!(config.headers, headers);
+}
+
+#[test]
+/// parse the test config and see that the values parsed are correct
+fn config_reads_queries() {
+    let config = setup_config_test();
+    let queries = vec![
+        ("name".to_string(), "value".to_string()),
+        ("rick".to_string(), "astley".to_string()),
+    ];
+    assert_eq!(config.queries, queries);
+}
+
+#[test]
+fn config_default_not_random_agent() {
+    let config = setup_config_test();
+    assert!(!config.random_agent);
+}
+
+#[test]
+#[should_panic]
+/// test that an error message is printed and panic is called when report_and_exit is called
+fn config_report_and_exit_works() {
+    report_and_exit("some message");
+}
+
+#[test]
+/// test as_str method of Configuration
+fn as_str_returns_string_with_newline() {
+    let config = Configuration::new().unwrap();
+    let config_str = config.as_str();
+    println!("{config_str}");
+    assert!(config_str.starts_with("Configuration {"));
+    assert!(config_str.ends_with("}\n"));
+    assert!(config_str.contains("replay_codes:"));
+    assert!(config_str.contains("client: Client {"));
+    assert!(config_str.contains("user_agent: \"feroxbuster"));
+}
+
+#[test]
+/// test as_json method of Configuration
+fn as_json_returns_json_representation_of_configuration_with_newline() {
+    let mut config = Configuration::new().unwrap();
+    config.timeout = 12;
+    config.depth = 2;
+    let config_str = config.as_json().unwrap();
+    let json: Configuration = serde_json::from_str(&config_str).unwrap();
+    assert_eq!(json.config, config.config);
+    assert_eq!(json.wordlist, config.wordlist);
+    assert_eq!(json.replay_codes, config.replay_codes);
+    assert_eq!(json.timeout, config.timeout);
+    assert_eq!(json.depth, config.depth);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_unique() {
+    let config = setup_config_test();
+    assert!(config.unique);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_response_size_limit() {
+    let config = setup_config_test();
+    assert_eq!(config.response_size_limit, 8388608); // 8MB as set in setup_config_test
+}

src/event_handlers/command.rs

@@ -0,0 +1,101 @@
+use std::sync::Arc;
+use std::time::Duration;
+
+use reqwest::StatusCode;
+use tokio::sync::oneshot::Sender;
+
+use crate::response::FeroxResponse;
+use crate::{
+    event_handlers::Handles,
+    message::FeroxMessage,
+    statistics::{StatError, StatField},
+    traits::FeroxFilter,
+};
+
+/// Protocol definition for updating an event handler via mpsc
+#[derive(Debug)]
+pub enum Command {
+    /// Add one to the total number of requests
+    AddRequest,
+
+    /// Add one to the proper field(s) based on the given `StatError`
+    AddError(StatError),
+
+    /// Add one to the proper field(s) based on the given `StatusCode`
+    AddStatus(StatusCode),
+
+    /// Create the progress bar (`BarType::Total`) that is updated from the stats thread
+    ///
+    /// the u64 value is the offset at which to start the progress bar (can be 0)
+    CreateBar(u64),
+
+    /// Add to a `Stats` field that corresponds to the given `StatField` by the given `usize` value
+    AddToUsizeField(StatField, usize),
+
+    /// Subtract from a `Stats` field that corresponds to the given `StatField` by the given `usize` value
+    SubtractFromUsizeField(StatField, usize),
+
+    /// Update a `Stats` field that corresponds to the given `StatField` by the given `f64` value
+    AddToF64Field(StatField, f64),
+
+    /// Save a `Stats` object to disk using `reporter::get_cached_file_handle`
+    Save,
+
+    /// Load a `Stats` object from disk
+    LoadStats(String),
+
+    /// Add a `FeroxFilter` implementor to `FilterHandler`'s instance of `FeroxFilters`
+    AddFilter(Box<dyn FeroxFilter>),
+
+    /// Remove a set of `FeroxFilter` implementors from `FeroxFilters` by index
+    RemoveFilters(Vec<usize>),
+
+    /// Send a `FeroxResponse` to the output handler for reporting
+    Report(Box<FeroxResponse>),
+
+    /// Send a group of urls to be scanned (only used for the urls passed in explicitly by the user)
+    ScanInitialUrls(Vec<String>),
+
+    /// Send a single url to be scanned (presumably added from the interactive menu)
+    ScanNewUrl(String),
+
+    /// Determine whether or not recursion is appropriate, given a FeroxResponse, if so start a scan
+    TryRecursion(Box<FeroxResponse>),
+
+    /// Send a pointer to the wordlist to the recursion handler
+    UpdateWordlist(Arc<Vec<String>>),
+
+    /// Instruct the ScanHandler to join on all known scans, use sender to notify main when done
+    JoinTasks(Sender<bool>),
+
+    /// Command used to test that a spawned task succeeded in initialization
+    Ping,
+
+    /// Just receive a sender and reply, used for slowing down the main thread
+    Sync(Sender<bool>),
+
+    /// Notify event handler that a new extension has been seen
+    AddDiscoveredExtension(String),
+
+    /// Write an arbitrary string to disk
+    WriteToDisk(Box<FeroxMessage>),
+
+    /// Break out of the (infinite) mpsc receive loop
+    Exit,
+
+    /// Give a handler access to an Arc<Handles> instance after the handler has
+    /// already been initialized
+    AddHandles(Arc<Handles>),
+
+    /// inform the Stats object about which targets are being scanned
+    UpdateTargets(Vec<String>),
+
+    /// query the Stats handler about the position of the overall progress bar
+    QueryOverallBarEta(Sender<Duration>),
+
+    /// Add permits to the scan limiter (semaphore)
+    AddScanPermits(usize),
+
+    /// Subtract permits from the scan limiter (semaphore)
+    SubtractScanPermits(usize),
+}

src/event_handlers/container.rs

@@ -0,0 +1,183 @@
+use super::*;
+use crate::config::Configuration;
+use crate::event_handlers::scans::ScanHandle;
+use crate::scan_manager::FeroxScans;
+use crate::Joiner;
+#[cfg(test)]
+use crate::{filters::FeroxFilters, statistics::Stats, Command};
+use anyhow::{bail, Result};
+use std::collections::HashSet;
+use std::sync::{Arc, RwLock};
+#[cfg(test)]
+use tokio::sync::mpsc::{self, UnboundedReceiver};
+
+#[derive(Debug)]
+/// Simple container for multiple JoinHandles
+pub struct Tasks {
+    /// JoinHandle for terminal handler
+    pub terminal: Joiner,
+
+    /// JoinHandle for statistics handler
+    pub stats: Joiner,
+
+    /// JoinHandle for filters handler
+    pub filters: Joiner,
+
+    /// JoinHandle for scans handler
+    pub scans: Joiner,
+}
+
+/// Tasks implementation
+impl Tasks {
+    /// Given JoinHandles for terminal, statistics, and filters create a new Tasks object
+    pub fn new(terminal: Joiner, stats: Joiner, filters: Joiner, scans: Joiner) -> Self {
+        Self {
+            terminal,
+            stats,
+            filters,
+            scans,
+        }
+    }
+}
+
+#[derive(Debug)]
+/// Container for the different *Handles that will be shared across modules
+pub struct Handles {
+    /// Handle for statistics
+    pub stats: StatsHandle,
+
+    /// Handle for filters
+    pub filters: FiltersHandle,
+
+    /// Handle for output (terminal/file)
+    pub output: TermOutHandle,
+
+    /// Handle for Configuration
+    pub config: Arc<Configuration>,
+
+    /// Handle for recursion
+    pub scans: RwLock<Option<ScanHandle>>,
+
+    /// Pointer to the list of words generated from reading in the wordlist
+    pub wordlist: Arc<Vec<String>>,
+}
+
+/// implementation of Handles
+impl Handles {
+    /// Given a StatsHandle, FiltersHandle, and OutputHandle, create a Handles object
+    pub fn new(
+        stats: StatsHandle,
+        filters: FiltersHandle,
+        output: TermOutHandle,
+        config: Arc<Configuration>,
+        wordlist: Arc<Vec<String>>,
+    ) -> Self {
+        Self {
+            stats,
+            filters,
+            output,
+            config,
+            scans: RwLock::new(None),
+            wordlist,
+        }
+    }
+
+    /// create a Handles object suitable for unit testing (non-functional)
+    #[cfg(test)]
+    pub fn for_testing(
+        scanned_urls: Option<Arc<FeroxScans>>,
+        config: Option<Arc<Configuration>>,
+    ) -> (Self, UnboundedReceiver<Command>) {
+        let configuration = config.unwrap_or_else(|| Arc::new(Configuration::new().unwrap()));
+        let (tx, rx) = mpsc::unbounded_channel::<Command>();
+        let terminal_handle = TermOutHandle::new(tx.clone(), tx.clone());
+        let stats_handle = StatsHandle::new(Arc::new(Stats::new(configuration.json)), tx.clone());
+        let filters_handle = FiltersHandle::new(Arc::new(FeroxFilters::default()), tx.clone());
+        let wordlist = Arc::new(vec![String::from("this_is_a_test")]);
+        let handles = Self::new(
+            stats_handle,
+            filters_handle,
+            terminal_handle,
+            configuration,
+            wordlist,
+        );
+        if let Some(sh) = scanned_urls {
+            let scan_handle = ScanHandle::new(sh, tx);
+            handles.set_scan_handle(scan_handle);
+        }
+        (handles, rx)
+    }
+
+    /// Set the ScanHandle object
+    pub fn set_scan_handle(&self, handle: ScanHandle) {
+        if let Ok(mut guard) = self.scans.write() {
+            if guard.is_none() {
+                guard.replace(handle);
+            }
+        }
+    }
+
+    /// Helper to easily send a Command over the (locked) underlying CommandSender object
+    pub fn send_scan_command(&self, command: Command) -> Result<()> {
+        if let Ok(guard) = self.scans.read().as_ref() {
+            if let Some(handle) = guard.as_ref() {
+                handle.send(command)?;
+                return Ok(());
+            }
+        }
+
+        bail!("Could not get underlying CommandSender object")
+    }
+
+    /// wrapper to reach into `FeroxScans` and yank out the length of `collected_extensions`
+    pub fn num_collected_extensions(&self) -> usize {
+        if !self.config.collect_extensions {
+            // if --collect-extensions wasn't used, simply return 0 and forego unlocking
+            return 0;
+        }
+
+        self.collected_extensions().len()
+    }
+
+    /// wrapper to reach into `FeroxScans` and yank out the length of `collected_extensions`
+    pub fn collected_extensions(&self) -> HashSet<String> {
+        if let Ok(scans) = self.ferox_scans() {
+            if let Ok(extensions) = scans.collected_extensions.read() {
+                return extensions.clone();
+            }
+        }
+
+        HashSet::new()
+    }
+
+    /// number of words in the wordlist, multiplied by `expected_num_requests_multiplier`
+    pub fn expected_num_requests_per_dir(&self) -> usize {
+        let num_words = self.wordlist.len();
+        let multiplier = self.expected_num_requests_multiplier();
+        multiplier * num_words
+    }
+
+    /// estimate of HTTP requests per word = (base + static extensions + collected extensions)
+    /// multiplied by the number of request methods
+    pub fn expected_num_requests_multiplier(&self) -> usize {
+        let methods = self.config.methods.len().max(1);
+        let base_requests = 1; // the bare word (with optional slash)
+        let static_extensions = self.config.extensions.len();
+        let dynamic_extensions = self.num_collected_extensions();
+
+        let total_paths = base_requests + static_extensions + dynamic_extensions;
+
+        total_paths * methods
+    }
+
+    /// Helper to easily get the (locked) underlying FeroxScans object
+    pub fn ferox_scans(&self) -> Result<Arc<FeroxScans>> {
+        if let Ok(guard) = self.scans.read().as_ref() {
+            if let Some(handle) = guard.as_ref() {
+                return Ok(handle.data.clone());
+            }
+        }
+
+        bail!("Could not get underlying FeroxScans")
+    }
+}

src/event_handlers/filters.rs

@@ -0,0 +1,144 @@
+use super::*;
+use crate::filters::EmptyFilter;
+use crate::{filters::FeroxFilters, CommandSender, FeroxChannel, Joiner};
+use anyhow::Result;
+use std::sync::Arc;
+use tokio::sync::{
+    mpsc::{self, UnboundedReceiver},
+    oneshot,
+};
+
+#[derive(Debug)]
+/// Container for filters transmitter and FeroxFilters object
+pub struct FiltersHandle {
+    /// FeroxFilters object used across modules to track active filters
+    pub data: Arc<FeroxFilters>,
+
+    /// transmitter used to update `data`
+    pub tx: CommandSender,
+}
+
+/// implementation of FiltersHandle
+impl FiltersHandle {
+    /// Given an Arc-wrapped FeroxFilters and CommandSender, create a new FiltersHandle
+    pub fn new(data: Arc<FeroxFilters>, tx: CommandSender) -> Self {
+        Self { data, tx }
+    }
+
+    /// Send the given Command over `tx`
+    pub fn send(&self, command: Command) -> Result<()> {
+        self.tx.send(command)?;
+        Ok(())
+    }
+
+    /// Sync the handle with the handler
+    pub async fn sync(&self) -> Result<()> {
+        let (tx, rx) = oneshot::channel::<bool>();
+        self.send(Command::Sync(tx))?;
+        rx.await?;
+        Ok(())
+    }
+}
+
+/// event handler for updating a single data structure of all active filters
+#[derive(Debug)]
+pub struct FiltersHandler {
+    /// collection of FeroxFilters
+    data: Arc<FeroxFilters>,
+
+    /// Receiver half of mpsc from which `Command`s are processed
+    receiver: UnboundedReceiver<Command>,
+}
+
+/// implementation of event handler for filters
+impl FiltersHandler {
+    /// create new event handler
+    pub fn new(data: Arc<FeroxFilters>, receiver: UnboundedReceiver<Command>) -> Self {
+        Self { data, receiver }
+    }
+
+    /// Initialize new `FeroxFilters` and the sc side of an mpsc channel that is responsible for
+    /// updates to the aforementioned object.
+    pub fn initialize() -> (Joiner, FiltersHandle) {
+        log::trace!("enter: initialize");
+
+        let data = Arc::new(FeroxFilters::default());
+        let (tx, rx): FeroxChannel<Command> = mpsc::unbounded_channel();
+
+        let mut handler = Self::new(data.clone(), rx);
+
+        let task = tokio::spawn(async move { handler.start().await });
+
+        let event_handle = FiltersHandle::new(data, tx);
+
+        log::trace!("exit: initialize -> ({task:?}, {event_handle:?})");
+
+        (task, event_handle)
+    }
+
+    /// Start a single consumer task (sc side of mpsc)
+    ///
+    /// The consumer simply receives `Command` and acts accordingly
+    pub async fn start(&mut self) -> Result<()> {
+        log::trace!("enter: start({self:?})");
+
+        while let Some(command) = self.receiver.recv().await {
+            match command {
+                Command::AddFilter(filter) => {
+                    if filter.as_any().downcast_ref::<EmptyFilter>().is_none() {
+                        // don't add an empty filter
+                        self.data.push(filter)?;
+                    }
+                }
+                Command::RemoveFilters(mut indices) => self.data.remove(&mut indices),
+                Command::Sync(sender) => {
+                    log::debug!("filters: {self:?}");
+                    sender.send(true).unwrap_or_default();
+                }
+                Command::Exit => break,
+                _ => {} // no other commands needed for FilterHandler
+            }
+        }
+
+        log::trace!("exit: start");
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::filters::WordsFilter;
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    async fn empty_filter_skipped() {
+        let data = Arc::new(FeroxFilters::default());
+        let (tx, rx): FeroxChannel<Command> = mpsc::unbounded_channel();
+
+        let mut handler = FiltersHandler::new(data.clone(), rx);
+
+        let event_handle = FiltersHandle::new(data, tx);
+
+        let _task = tokio::spawn(async move { handler.start().await });
+
+        event_handle
+            .send(Command::AddFilter(Box::new(EmptyFilter {})))
+            .unwrap();
+
+        let (tx, rx) = oneshot::channel::<bool>();
+        event_handle.send(Command::Sync(tx)).unwrap();
+        rx.await.unwrap();
+
+        assert!(event_handle.data.filters.read().unwrap().is_empty());
+
+        event_handle
+            .send(Command::AddFilter(Box::new(WordsFilter { word_count: 1 })))
+            .unwrap();
+
+        let (tx, rx) = oneshot::channel::<bool>();
+        event_handle.send(Command::Sync(tx)).unwrap();
+        rx.await.unwrap();
+
+        assert_eq!(event_handle.data.filters.read().unwrap().len(), 1);
+    }
+}

src/event_handlers/inputs.rs

@@ -0,0 +1,178 @@
+use super::*;
+use crate::{
+    progress::PROGRESS_PRINTER,
+    scan_manager::{FeroxState, PAUSE_SCAN},
+    scanner::RESPONSES,
+    statistics::StatError,
+    utils::slugify_filename,
+    utils::{open_file, write_to},
+    SLEEP_DURATION,
+};
+use anyhow::Result;
+use console::style;
+use crossterm::event::{self, Event, KeyCode};
+use std::{
+    env::temp_dir,
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        Arc,
+    },
+    thread::sleep,
+    time::Duration,
+};
+
+/// Atomic boolean flag, used to determine whether or not the terminal input handler should exit
+pub static SCAN_COMPLETE: AtomicBool = AtomicBool::new(false);
+
+/// Container for filters transmitter and FeroxFilters object
+pub struct TermInputHandler {
+    /// handles to other handlers
+    handles: Arc<Handles>,
+}
+
+/// implementation of event handler for terminal input
+///
+/// kicks off the following handlers related to terminal input:
+///     ctrl+c handler that saves scan state to disk
+///     enter handler that listens for enter during scans to drop into interactive scan management menu
+impl TermInputHandler {
+    /// Create new event handler
+    pub fn new(handles: Arc<Handles>) -> Self {
+        Self { handles }
+    }
+
+    /// Initialize the sigint and enter handlers that are responsible for handling initial user
+    /// interaction during scans
+    pub fn initialize(handles: Arc<Handles>) {
+        log::trace!("enter: initialize({handles:?})");
+
+        let handler = Self::new(handles);
+        handler.start();
+
+        log::trace!("exit: initialize");
+    }
+
+    /// wrapper around sigint_handler and enter_handler
+    fn start(&self) {
+        tokio::task::spawn_blocking(Self::enter_handler);
+
+        if self.handles.config.save_state {
+            // start the ctrl+c handler
+            let cloned = self.handles.clone();
+
+            let result = ctrlc::set_handler(move || {
+                let _ = Self::sigint_handler(cloned.clone());
+            });
+
+            if result.is_err() {
+                log::warn!("Could not set Ctrl+c handler; scan state will not be saved");
+                self.handles
+                    .stats
+                    .send(Command::AddError(StatError::Other))
+                    .unwrap_or_default();
+            }
+        }
+    }
+
+    /// Writes the current state of the program to disk (if save_state is true) and then exits
+    pub fn sigint_handler(handles: Arc<Handles>) -> Result<()> {
+        log::trace!("enter: sigint_handler({handles:?})");
+
+        // check for STATE_FILENAME env var first; credit to Tobias Rauch for the idea
+        let filename = if let Ok(path) = std::env::var("STATE_FILENAME") {
+            path
+        } else if !handles.config.target_url.is_empty() {
+            // target url populated
+            slugify_filename(&handles.config.target_url, "ferox", "state")
+        } else {
+            // stdin used
+            slugify_filename("stdin", "ferox", "state")
+        };
+
+        let warning = format!(
+            "🚨 Caught {} 🚨 saving scan state to {} ...",
+            style("ctrl+c").yellow(),
+            filename
+        );
+
+        PROGRESS_PRINTER.println(warning);
+
+        let state = FeroxState::new(
+            handles.ferox_scans()?,
+            handles.config.clone(),
+            &RESPONSES,
+            handles.stats.data.clone(),
+            handles.filters.data.clone(),
+        );
+
+        // User didn't set the --no-state flag (so saved_state is still the default true)
+        if handles.config.save_state {
+            let Ok(mut state_file) = open_file(&filename) else {
+                // couldn't open the file, let the user know we're going to try again
+                let error = format!(
+                    "❌ Could not save {}, falling back to {}",
+                    filename,
+                    temp_dir().to_string_lossy()
+                );
+                PROGRESS_PRINTER.println(error);
+
+                let temp_filename = temp_dir().join(&filename);
+
+                let Ok(mut state_file) = open_file(&temp_filename.to_string_lossy()) else {
+                    // couldn't open the fallback file, let the user know
+                    let error = format!("❌❌ Could not save {temp_filename:?}, giving up...");
+                    PROGRESS_PRINTER.println(error);
+
+                    log::trace!("exit: sigint_handler (failed to write)");
+                    std::process::exit(1);
+                };
+
+                write_to(&state, &mut state_file, true)?;
+
+                let msg = format!("✅ Saved scan state to {temp_filename:?}");
+                PROGRESS_PRINTER.println(msg);
+
+                log::trace!("exit: sigint_handler (saved to temp folder)");
+                std::process::exit(1);
+            };
+
+            write_to(&state, &mut state_file, true)?;
+        }
+
+        log::trace!("exit: sigint_handler (end of program)");
+        std::process::exit(1);
+    }
+
+    /// Handles specific key events triggered by the user over stdin
+    fn enter_handler() {
+        // todo eventually move away from atomics, the blocking recv is the problem
+        log::trace!("enter: start_enter_handler");
+
+        loop {
+            if PAUSE_SCAN.load(Ordering::Relaxed) {
+                // if the scan is already paused, we don't want this event poller fighting the user
+                // over stdin
+                sleep(Duration::from_millis(SLEEP_DURATION));
+            } else if event::poll(Duration::from_millis(SLEEP_DURATION)).unwrap_or(false) {
+                // It's guaranteed that the `read()` won't block when the `poll()`
+                // function returns `true`
+
+                if let Ok(key_pressed) = event::read() {
+                    // ignore any other keys
+                    if key_pressed == Event::Key(KeyCode::Enter.into()) {
+                        // if the user presses Enter, set PAUSE_SCAN to true. The interactive menu
+                        // will be triggered and will handle setting PAUSE_SCAN to false
+                        PAUSE_SCAN.store(true, Ordering::Release);
+                    }
+                }
+            } else {
+                // Timeout expired and no `Event` is available; use the timeout to check SCAN_COMPLETE
+                if SCAN_COMPLETE.load(Ordering::Relaxed) {
+                    // scan has been marked complete by main, time to exit the loop
+                    break;
+                }
+            }
+        }
+        log::trace!("exit: start_enter_handler");
+    }
+}

src/event_handlers/mod.rs

@@ -0,0 +1,16 @@
+//! collection of event handlers (typically long-running tokio spawned tasks)
+mod statistics;
+mod filters;
+mod container;
+mod command;
+mod outputs;
+mod scans;
+mod inputs;
+
+pub use self::command::Command;
+pub use self::container::{Handles, Tasks};
+pub use self::filters::{FiltersHandle, FiltersHandler};
+pub use self::inputs::{TermInputHandler, SCAN_COMPLETE};
+pub use self::outputs::{TermOutHandle, TermOutHandler};
+pub use self::scans::{ScanHandle, ScanHandler};
+pub use self::statistics::{StatsHandle, StatsHandler};

src/event_handlers/outputs.rs

@@ -0,0 +1,618 @@
+use super::Command::AddToUsizeField;
+use super::*;
+
+use anyhow::{Context, Result};
+use futures::future::{BoxFuture, FutureExt};
+use tokio::sync::{mpsc, oneshot};
+
+use crate::{
+    config::Configuration,
+    filters::SimilarityFilter,
+    progress::PROGRESS_PRINTER,
+    response::FeroxResponse,
+    scanner::RESPONSES,
+    send_command, skip_fail,
+    statistics::StatField::{ResourcesDiscovered, TotalExpected},
+    traits::FeroxSerialize,
+    utils::{ferox_print, fmt_err, make_request, open_file, write_to},
+    CommandReceiver, CommandSender, Joiner, UNIQUE_DISTANCE,
+};
+
+use std::sync::Arc;
+use url::Url;
+
+#[derive(Debug, Copy, Clone)]
+/// Simple enum for semantic clarity around calling expectations for `process_response`
+enum ProcessResponseCall {
+    /// call should allow recursion
+    Recursive,
+
+    /// call should not allow recursion
+    NotRecursive,
+}
+
+#[derive(Debug)]
+/// Container for terminal output transmitter
+pub struct TermOutHandle {
+    /// Transmitter that sends to the TermOutHandler handler
+    pub tx: CommandSender,
+
+    /// Transmitter that sends to the FileOutHandler handler
+    pub tx_file: CommandSender,
+}
+
+/// implementation of OutputHandle
+impl TermOutHandle {
+    /// Given a CommandSender, create a new OutputHandle
+    pub fn new(tx: CommandSender, tx_file: CommandSender) -> Self {
+        Self { tx, tx_file }
+    }
+
+    /// Send the given Command over `tx`
+    pub fn send(&self, command: Command) -> Result<()> {
+        self.tx.send(command)?;
+        Ok(())
+    }
+
+    /// Sync the handle with the handler
+    pub async fn sync(&self, send_to_file: bool) -> Result<()> {
+        let (tx, rx) = oneshot::channel::<bool>();
+        self.send(Command::Sync(tx))?;
+
+        if send_to_file {
+            let (tx, rx) = oneshot::channel::<bool>();
+            self.tx_file.send(Command::Sync(tx))?;
+            rx.await?;
+        }
+
+        rx.await?;
+        Ok(())
+    }
+}
+
+#[derive(Debug)]
+/// Event handler for files
+pub struct FileOutHandler {
+    /// file output handler's receiver
+    receiver: CommandReceiver,
+
+    /// pointer to "global" configuration struct
+    config: Arc<Configuration>,
+}
+
+impl FileOutHandler {
+    /// Given a file tx/rx pair along with a filename and awaitable task, create
+    /// a FileOutHandler
+    fn new(rx: CommandReceiver, config: Arc<Configuration>) -> Self {
+        Self {
+            receiver: rx,
+            config,
+        }
+    }
+
+    /// Spawn a single consumer task (sc side of mpsc)
+    ///
+    /// The consumer simply receives responses from the terminal handler and writes them to disk
+    async fn start(&mut self, tx_stats: CommandSender) -> Result<()> {
+        log::trace!("enter: start_file_handler({tx_stats:?})");
+
+        let mut file = open_file(&self.config.output)?;
+
+        log::info!("Writing scan results to {}", self.config.output);
+
+        write_to(&*self.config, &mut file, self.config.json)?;
+
+        while let Some(command) = self.receiver.recv().await {
+            match command {
+                Command::Report(response) => {
+                    skip_fail!(write_to(&*response, &mut file, self.config.json));
+                }
+                Command::WriteToDisk(message) => {
+                    // todo consider making report accept dyn FeroxSerialize; would mean adding
+                    //  as_any/box_eq/PartialEq to the trait and then adding them to the
+                    //  implementing structs
+                    skip_fail!(write_to(&*message, &mut file, self.config.json));
+                }
+                Command::Exit => {
+                    break;
+                }
+                Command::Sync(sender) => {
+                    skip_fail!(sender.send(true));
+                }
+                _ => {} // no more needed
+            }
+        }
+
+        // close the file before we tell statistics to save current data to the same file
+        drop(file);
+
+        send_command!(tx_stats, Command::Save);
+
+        log::trace!("exit: start_file_handler");
+        Ok(())
+    }
+}
+
+#[derive(Debug)]
+/// Event handler for terminal
+pub struct TermOutHandler {
+    /// terminal output handler's receiver
+    receiver: CommandReceiver,
+
+    /// file handler
+    tx_file: CommandSender,
+
+    /// optional file handler task
+    file_task: Option<Joiner>,
+
+    /// pointer to "global" configuration struct
+    config: Arc<Configuration>,
+
+    /// handles instance
+    handles: Option<Arc<Handles>>,
+}
+
+/// implementation of TermOutHandler
+impl TermOutHandler {
+    /// Given a terminal receiver along with a file transmitter and filename, create
+    /// an OutputHandler
+    fn new(
+        receiver: CommandReceiver,
+        tx_file: CommandSender,
+        file_task: Option<Joiner>,
+        config: Arc<Configuration>,
+    ) -> Self {
+        Self {
+            receiver,
+            tx_file,
+            file_task,
+            config,
+            handles: None,
+        }
+    }
+
+    /// Creates all required output handlers (terminal, file) and updates the given Handles/Tasks
+    pub fn initialize(
+        config: Arc<Configuration>,
+        tx_stats: CommandSender,
+    ) -> (Joiner, TermOutHandle) {
+        log::trace!("enter: initialize({config:?}, {tx_stats:?})");
+
+        let (tx_term, rx_term) = mpsc::unbounded_channel::<Command>();
+        let (tx_file, rx_file) = mpsc::unbounded_channel::<Command>();
+
+        let mut file_handler = FileOutHandler::new(rx_file, config.clone());
+
+        let tx_stats_clone = tx_stats.clone();
+
+        let file_task = if !config.output.is_empty() {
+            // -o used, need to spawn the thread for writing to disk
+            Some(tokio::spawn(async move {
+                file_handler.start(tx_stats_clone).await
+            }))
+        } else {
+            None
+        };
+
+        let mut term_handler = Self::new(rx_term, tx_file.clone(), file_task, config);
+        let term_task = tokio::spawn(async move { term_handler.start(tx_stats).await });
+
+        let event_handle = TermOutHandle::new(tx_term, tx_file);
+
+        log::trace!("exit: initialize -> ({term_task:?}, {event_handle:?})");
+
+        (term_task, event_handle)
+    }
+
+    /// Start a single consumer task (sc side of mpsc)
+    ///
+    /// The consumer simply receives `Command` and acts accordingly
+    async fn start(&mut self, tx_stats: CommandSender) -> Result<()> {
+        log::trace!("enter: start({tx_stats:?})");
+
+        while let Some(command) = self.receiver.recv().await {
+            match command {
+                Command::Report(resp) => {
+                    if let Err(err) = self
+                        .process_response(tx_stats.clone(), resp, ProcessResponseCall::Recursive)
+                        .await
+                    {
+                        log::warn!("{err}");
+                    }
+                }
+                Command::Sync(sender) => {
+                    sender.send(true).unwrap_or_default();
+                }
+                Command::AddHandles(handles) => {
+                    self.handles = Some(handles);
+                }
+                Command::Exit => {
+                    if self.tx_file.send(Command::Exit).is_ok() {
+                        if let Some(task) = self.file_task.as_mut() {
+                            task.await??; // wait for death
+                        }
+                    }
+                    break;
+                }
+                _ => {} // no more commands needed
+            }
+        }
+        log::trace!("exit: start");
+        Ok(())
+    }
+
+    /// upon receiving a `FeroxResponse` from the mpsc, handle printing, sending to the replay
+    /// proxy, checking for backups of the `FeroxResponse`'s url, and tracking the response.
+    fn process_response(
+        &self,
+        tx_stats: CommandSender,
+        mut resp: Box<FeroxResponse>,
+        call_type: ProcessResponseCall,
+    ) -> BoxFuture<'_, Result<()>> {
+        log::trace!("enter: process_response({resp:?}, {call_type:?})");
+
+        async move {
+            let contains_sentry = if !self.config.filter_status.is_empty() {
+                // -C was used, meaning -s was not and we should ignore the defaults
+                // https://github.com/epi052/feroxbuster/issues/535
+                // -C indicates that we should filter that status code, but allow all others
+                !self.config.filter_status.contains(&resp.status().as_u16())
+            } else {
+                // -C wasn't used, so, we defer to checking the -s values
+                self.config.status_codes.contains(&resp.status().as_u16())
+            };
+
+            let unknown_sentry = !RESPONSES.contains(&resp); // !contains == unknown
+            let should_process_response = contains_sentry && unknown_sentry;
+
+            if should_process_response {
+                // print to stdout
+                ferox_print(&resp.as_str(), &PROGRESS_PRINTER);
+
+                send_command!(tx_stats, AddToUsizeField(ResourcesDiscovered, 1));
+
+                if self.file_task.is_some() {
+                    // -o used, need to send the report to be written out to disk
+                    self.tx_file
+                        .send(Command::Report(resp.clone()))
+                        .with_context(|| {
+                            fmt_err(&format!("Could not send {resp} to file handler"))
+                        })?;
+                }
+            }
+            log::trace!("report complete: {}", resp.url());
+
+            if should_process_response {
+                if let Some(client) = self.config.replay_client.as_ref() {
+                    // replay proxy specified/client created and this response's status code is one that
+                    // should be replayed; not using logged_request due to replay proxy client
+                    let data = if self.config.data.is_empty() {
+                        None
+                    } else {
+                        Some(self.config.data.as_slice())
+                    };
+
+                    make_request(
+                        client,
+                        resp.url(),
+                        resp.method().as_str(),
+                        data,
+                        self.config.output_level,
+                        &self.config,
+                        tx_stats.clone(),
+                    )
+                    .await
+                    .with_context(|| "Could not replay request through replay proxy")?;
+                } else {
+                    // replay proxy not configured, skip replay without exiting response processing
+                    log::trace!("replay proxy not configured, skipping replay");
+                }
+            }
+
+            if self.config.collect_backups
+                && should_process_response
+                && matches!(call_type, ProcessResponseCall::Recursive)
+            {
+                // --collect-backups was used; the response is one we care about, and the function
+                // call came from the loop in `.start` (i.e. recursive was specified)
+                let backup_urls = self.generate_backup_urls(&resp).await;
+
+                // need to manually adjust stats
+                send_command!(tx_stats, AddToUsizeField(TotalExpected, backup_urls.len()));
+
+                for backup_url in &backup_urls {
+                    let backup_response = make_request(
+                        &self.config.client,
+                        backup_url,
+                        resp.method().as_str(),
+                        None,
+                        self.config.output_level,
+                        &self.config,
+                        tx_stats.clone(),
+                    )
+                    .await
+                    .with_context(|| {
+                        format!("Could not request backup of {}", resp.url().as_str())
+                    })?;
+
+                    let ferox_response = FeroxResponse::from(
+                        backup_response,
+                        resp.url().as_str(),
+                        resp.method().as_str(),
+                        resp.output_level,
+                        self.config.response_size_limit,
+                    )
+                    .await;
+
+                    let Some(handles) = self.handles.as_ref() else {
+                        // shouldn't ever happen, but we'll log and return early if it does
+                        log::error!("handles were unexpectedly None, this shouldn't happen");
+                        return Ok(());
+                    };
+
+                    if handles
+                        .filters
+                        .data
+                        .should_filter_response(&ferox_response, tx_stats.clone())
+                    {
+                        // response was filtered for one reason or another, don't process it
+                        continue;
+                    }
+
+                    if handles.config.unique {
+                        let mut unique_filter = SimilarityFilter::from(&ferox_response);
+                        unique_filter.cutoff = UNIQUE_DISTANCE;
+                        handles.filters.data.push(Box::new(unique_filter))?;
+                    }
+
+                    self.process_response(
+                        tx_stats.clone(),
+                        Box::new(ferox_response),
+                        ProcessResponseCall::NotRecursive,
+                    )
+                    .await?;
+                }
+            }
+
+            if should_process_response {
+                // add response to RESPONSES for serialization in case of ctrl+c
+                // placed all by its lonesome like this so that RESPONSES can take ownership
+                // of the FeroxResponse
+
+                // before ownership is transferred, there's no real reason to keep the body anymore
+                // so we can free that piece of data, reducing memory usage
+                resp.drop_text();
+
+                RESPONSES.insert(*resp);
+            }
+            log::trace!("exit: process_response");
+            Ok(())
+        }
+        .boxed()
+    }
+
+    /// internal helper to stay DRY
+    fn add_new_url_to_vec(&self, url: &Url, new_name: &str, urls: &mut Vec<Url>) {
+        if let Ok(joined) = url.join(new_name) {
+            urls.push(joined);
+        }
+    }
+
+    /// given a `FeroxResponse`, generate either 6 or 7 urls that are likely backups of the
+    /// original.
+    ///
+    /// example:
+    ///     original: LICENSE.txt
+    ///     backups:    
+    ///         - LICENSE.txt~
+    ///         - LICENSE.txt.bak
+    ///         - LICENSE.txt.bak2
+    ///         - LICENSE.txt.old
+    ///         - LICENSE.txt.1
+    ///         - LICENSE.bak
+    ///         - .LICENSE.txt.swp
+    async fn generate_backup_urls(&self, response: &FeroxResponse) -> Vec<Url> {
+        log::trace!("enter: generate_backup_urls({response:?})");
+
+        let mut urls = vec![];
+        let url = response.url();
+
+        // confirmed safe: see src/response.rs for comments
+        let filename = url.path_segments().unwrap().next_back().unwrap();
+
+        if !filename.is_empty() {
+            // append rules
+            for suffix in &self.config.backup_extensions {
+                self.add_new_url_to_vec(url, &format!("{filename}{suffix}"), &mut urls);
+            }
+
+            // vim swap rule
+            self.add_new_url_to_vec(url, &format!(".{filename}.swp"), &mut urls);
+
+            // replace original extension rule
+            let parts: Vec<_> = filename
+                .split('.')
+                // keep things like /.bash_history out of results
+                .filter(|part| !part.is_empty())
+                .collect();
+
+            if parts.len() > 1 {
+                // filename + at least one extension, i.e. whatever.js becomes ["whatever", "js"]
+                self.add_new_url_to_vec(url, &format!("{}.bak", parts.first().unwrap()), &mut urls);
+            }
+        }
+
+        log::trace!("exit: generate_backup_urls -> {urls:?}");
+        urls
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::event_handlers::Command;
+
+    #[test]
+    /// try to hit struct field coverage of FileOutHandler
+    fn struct_fields_of_file_out_handler() {
+        let (_, rx) = mpsc::unbounded_channel::<Command>();
+        let config = Arc::new(Configuration::new().unwrap());
+        let foh = FileOutHandler {
+            config,
+            receiver: rx,
+        };
+        println!("{foh:?}");
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// try to hit struct field coverage of TermOutHandler
+    async fn struct_fields_of_term_out_handler() {
+        let (tx, rx) = mpsc::unbounded_channel::<Command>();
+        let (tx_file, _) = mpsc::unbounded_channel::<Command>();
+        let config = Arc::new(Configuration::new().unwrap());
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+
+        let toh = TermOutHandler {
+            config,
+            file_task: None,
+            receiver: rx,
+            tx_file,
+            handles: Some(handles),
+        };
+
+        println!("{toh:?}");
+        tx.send(Command::Exit).unwrap();
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// when the feroxresponse's url contains an extension, there should be 7 urls returned
+    async fn generate_backup_urls_creates_correct_urls_when_extension_present() {
+        let (tx, rx) = mpsc::unbounded_channel::<Command>();
+        let (tx_file, _) = mpsc::unbounded_channel::<Command>();
+        let config = Arc::new(Configuration::new().unwrap());
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+
+        let toh = TermOutHandler {
+            config,
+            file_task: None,
+            receiver: rx,
+            tx_file,
+            handles: Some(handles),
+        };
+
+        let expected: Vec<_> = vec![
+            "derp.php~",
+            "derp.php.bak",
+            "derp.php.bak2",
+            "derp.php.old",
+            "derp.php.1",
+            ".derp.php.swp",
+            "derp.bak",
+        ];
+
+        let mut fr = FeroxResponse::default();
+        fr.set_url("http://localhost/derp.php");
+
+        let urls = toh.generate_backup_urls(&fr).await;
+
+        let paths: Vec<_> = urls
+            .iter()
+            .map(|url| url.path_segments().unwrap().next_back().unwrap())
+            .collect();
+
+        assert_eq!(urls.len(), 7);
+
+        for path in paths {
+            assert!(expected.contains(&path));
+        }
+
+        tx.send(Command::Exit).unwrap();
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// when the feroxresponse's url doesn't contain an extension, there should be 6 urls returned
+    async fn generate_backup_urls_creates_correct_urls_when_extension_not_present() {
+        let (tx, rx) = mpsc::unbounded_channel::<Command>();
+        let (tx_file, _) = mpsc::unbounded_channel::<Command>();
+        let config = Arc::new(Configuration::new().unwrap());
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+
+        let toh = TermOutHandler {
+            config,
+            file_task: None,
+            receiver: rx,
+            tx_file,
+            handles: Some(handles),
+        };
+
+        let expected: Vec<_> = vec![
+            "derp~",
+            "derp.bak",
+            "derp.bak2",
+            "derp.old",
+            "derp.1",
+            ".derp.swp",
+        ];
+
+        let mut fr = FeroxResponse::default();
+        fr.set_url("http://localhost/derp");
+
+        let urls = toh.generate_backup_urls(&fr).await;
+
+        let paths: Vec<_> = urls
+            .iter()
+            .map(|url| url.path_segments().unwrap().next_back().unwrap())
+            .collect();
+
+        assert_eq!(urls.len(), 6);
+
+        for path in paths {
+            assert!(expected.contains(&path));
+        }
+
+        tx.send(Command::Exit).unwrap();
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// test to ensure that backups are requested from the directory in which they were found
+    /// re: issue #513
+    async fn generate_backup_urls_creates_correct_urls_when_not_at_root() {
+        let (tx, rx) = mpsc::unbounded_channel::<Command>();
+        let (tx_file, _) = mpsc::unbounded_channel::<Command>();
+        let config = Arc::new(Configuration::new().unwrap());
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+
+        let toh = TermOutHandler {
+            config,
+            file_task: None,
+            receiver: rx,
+            tx_file,
+            handles: Some(handles),
+        };
+
+        let expected: Vec<_> = vec![
+            "http://localhost/wordpress/derp.php~",
+            "http://localhost/wordpress/derp.php.bak",
+            "http://localhost/wordpress/derp.php.bak2",
+            "http://localhost/wordpress/derp.php.old",
+            "http://localhost/wordpress/derp.php.1",
+            "http://localhost/wordpress/.derp.php.swp",
+            "http://localhost/wordpress/derp.bak",
+        ];
+
+        let mut fr = FeroxResponse::default();
+        fr.set_url("http://localhost/wordpress/derp.php");
+
+        let urls = toh.generate_backup_urls(&fr).await;
+
+        let url_strs: Vec<_> = urls.iter().map(|url| url.as_str()).collect();
+
+        assert_eq!(urls.len(), 7);
+
+        for url_str in url_strs {
+            assert!(expected.contains(&url_str));
+        }
+
+        tx.send(Command::Exit).unwrap();
+    }
+}

src/event_handlers/scans.rs

@@ -0,0 +1,490 @@
+use std::sync::Arc;
+
+use anyhow::{bail, Result};
+use tokio::sync::mpsc;
+
+use crate::{
+    response::FeroxResponse,
+    scan_manager::{FeroxScan, FeroxScans, ScanOrder},
+    scanner::{FeroxScanner, RESPONSES},
+    statistics::StatField::TotalScans,
+    sync::DynamicSemaphore,
+    url::FeroxUrl,
+    utils::should_deny_url,
+    CommandReceiver, CommandSender, FeroxChannel, Joiner, SLEEP_DURATION,
+};
+
+use super::command::Command::AddToUsizeField;
+use super::*;
+use crate::statistics::StatField;
+use crate::utils::parse_url_with_raw_path;
+use tokio::time::Duration;
+
+#[derive(Debug)]
+/// Container for recursion transmitter and FeroxScans object
+pub struct ScanHandle {
+    /// FeroxScans object used across modules to track scans
+    pub data: Arc<FeroxScans>,
+
+    /// transmitter used to update `data`
+    pub tx: CommandSender,
+}
+
+/// implementation of RecursionHandle
+impl ScanHandle {
+    /// Given an Arc-wrapped FeroxScans and CommandSender, create a new RecursionHandle
+    pub fn new(data: Arc<FeroxScans>, tx: CommandSender) -> Self {
+        Self { data, tx }
+    }
+
+    /// Send the given Command over `tx`
+    pub fn send(&self, command: Command) -> Result<()> {
+        self.tx.send(command)?;
+        Ok(())
+    }
+}
+
+/// event handler for updating a single data structure of all FeroxScans
+#[derive(Debug)]
+pub struct ScanHandler {
+    /// collection of FeroxScans
+    data: Arc<FeroxScans>,
+
+    /// handles to other handlers needed to kick off a scan while already past main
+    handles: Arc<Handles>,
+
+    /// Receiver half of mpsc from which `Command`s are processed
+    receiver: CommandReceiver,
+
+    /// wordlist (re)used for each scan
+    wordlist: std::sync::Mutex<Option<Arc<Vec<String>>>>,
+
+    /// group of scans that need to be joined
+    tasks: Vec<Arc<FeroxScan>>,
+
+    /// Maximum recursion depth, a depth of 0 is infinite recursion
+    max_depth: usize,
+
+    /// depths associated with the initial targets provided by the user
+    depths: Vec<(String, usize)>,
+
+    /// Bounded semaphore used as a barrier to limit concurrent scans
+    limiter: Arc<DynamicSemaphore>,
+}
+
+/// implementation of event handler for filters
+impl ScanHandler {
+    /// create new event handler
+    pub fn new(
+        data: Arc<FeroxScans>,
+        handles: Arc<Handles>,
+        max_depth: usize,
+        receiver: CommandReceiver,
+    ) -> Self {
+        let limit = handles.config.scan_limit;
+        let limiter = DynamicSemaphore::new(limit);
+
+        if limit == 0 {
+            // scan_limit == 0 means no limit should be imposed... however, scoping the Semaphore
+            // permit is tricky, so as a workaround, we'll add a ridiculous number of permits to
+            // the semaphore (1,152,921,504,606,846,975 to be exact) and call that 'unlimited'
+
+            // note to self: the docs say max is usize::MAX >> 3, however, threads will panic if
+            // that value is used (says adding (1) will overflow the semaphore, even though none
+            // are being added...)
+            limiter.increase_capacity(usize::MAX >> 4);
+        }
+
+        Self {
+            data,
+            handles,
+            receiver,
+            max_depth,
+            tasks: Vec::new(),
+            depths: Vec::new(),
+            limiter: Arc::new(limiter),
+            wordlist: std::sync::Mutex::new(None),
+        }
+    }
+
+    /// Set the wordlist
+    fn wordlist(&self, wordlist: Arc<Vec<String>>) {
+        if let Ok(mut guard) = self.wordlist.lock() {
+            if guard.is_none() {
+                guard.replace(wordlist);
+            }
+        }
+    }
+
+    /// Initialize new `FeroxScans` and the sc side of an mpsc channel that is responsible for
+    /// updates to the aforementioned object.
+    pub fn initialize(handles: Arc<Handles>) -> (Joiner, ScanHandle) {
+        log::trace!("enter: initialize");
+
+        let data = Arc::new(FeroxScans::new(
+            handles.config.output_level,
+            handles.config.limit_bars,
+        ));
+        let (tx, rx): FeroxChannel<Command> = mpsc::unbounded_channel();
+
+        let max_depth = handles.config.depth;
+
+        let mut handler = Self::new(data.clone(), handles, max_depth, rx);
+
+        let task = tokio::spawn(async move { handler.start().await });
+
+        let event_handle = ScanHandle::new(data, tx);
+
+        log::trace!("exit: initialize -> ({task:?}, {event_handle:?})");
+
+        (task, event_handle)
+    }
+
+    /// Start a single consumer task (sc side of mpsc)
+    ///
+    /// The consumer simply receives `Command` and acts accordingly
+    pub async fn start(&mut self) -> Result<()> {
+        log::trace!("enter: start({self:?})");
+
+        while let Some(command) = self.receiver.recv().await {
+            match command {
+                Command::ScanInitialUrls(targets) => {
+                    self.ordered_scan_url(targets, ScanOrder::Initial).await?;
+                }
+                Command::ScanNewUrl(target) => {
+                    // added as part of interactive menu ability (2.4.1) to add a new scan.
+                    // we don't have a way of knowing if they're adding a new url entirely (i.e.
+                    // new base url), or simply adding a new sub-directory found some other way.
+                    // Since we can't know, we'll start a scan as though we received the scan
+                    // from -u | --stdin
+                    self.ordered_scan_url(vec![target], ScanOrder::Initial)
+                        .await?;
+                }
+                Command::UpdateWordlist(wordlist) => {
+                    self.wordlist(wordlist);
+                }
+                Command::JoinTasks(sender) => {
+                    let ferox_scans = self.handles.ferox_scans().unwrap_or_default();
+                    let limiter_clone = self.limiter.clone();
+
+                    tokio::spawn(async move {
+                        while ferox_scans.has_active_scans() {
+                            tokio::time::sleep(Duration::from_millis(SLEEP_DURATION + 250)).await;
+                        }
+                        limiter_clone.close();
+                        sender.send(true).expect("oneshot channel failed");
+                    });
+                }
+                Command::TryRecursion(response) => {
+                    self.try_recursion(response).await?;
+                }
+                Command::Sync(sender) => {
+                    sender.send(true).unwrap_or_default();
+                }
+                Command::AddDiscoveredExtension(new_extension) => {
+                    // if --collect-extensions was used, AND the new extension isn't in
+                    // the --dont-collect list AND it's also not in the --extensions list, AND
+                    // we actually added a new extension (i.e. wasn't previously known), add
+                    // it to FeroxScans.collected_extensions
+                    if self.handles.config.collect_extensions
+                        && !self.handles.config.dont_collect.contains(&new_extension)
+                        && !self.handles.config.extensions.contains(&new_extension)
+                        && self.data.add_discovered_extension(new_extension)
+                    {
+                        self.update_all_bar_lengths()?;
+                        self.handles
+                            .stats
+                            .send(Command::AddToUsizeField(StatField::ExtensionsCollected, 1))
+                            .unwrap_or_default();
+                    }
+                }
+                Command::AddScanPermits(value) => {
+                    let current = self.limiter.current_capacity();
+
+                    self.limiter.increase_capacity(current + value);
+
+                    log::debug!(
+                        "increased scan permits to {} (was {current})",
+                        current + value
+                    );
+                }
+                Command::SubtractScanPermits(value) => {
+                    let current = self.limiter.current_capacity();
+                    let new_capacity = current.saturating_sub(value);
+
+                    self.limiter.reduce_capacity(new_capacity);
+
+                    log::debug!("decreased scan permits to {new_capacity} (was {current})");
+                }
+                _ => {} // no other commands needed for RecursionHandler
+            }
+        }
+
+        log::trace!("exit: start");
+        Ok(())
+    }
+
+    /// update all current and future bar lengths
+    ///
+    /// updating all bar lengths correctly requires a few different actions on our part.
+    /// - get the current number of requests expected per scan (dynamic when --collect-extensions
+    ///   is used)
+    /// - update the overall progress bar via the statistics handler (total expected)
+    /// - update the expected per scan value tracked in the statistics handler
+    /// - update progress bars on each FeroxScan (type::directory) that are running/not-started
+    /// - update progress bar length on FeroxScans (this is used when creating new a FeroxScan and
+    ///   determines the new scan's progress bar length)
+    fn update_all_bar_lengths(&self) -> Result<()> {
+        log::trace!("enter: update_all_bar_lengths");
+
+        // current number of requests expected per scan
+        // ExpectedPerScan and TotalExpected are a += action, so we need the wordlist length to
+        // update them while the other updates use expected_num_requests_per_dir
+        let num_words = self.get_wordlist(0)?.len();
+        let current_expectation = self.handles.expected_num_requests_per_dir() as u64;
+
+        // used in the calculation of bar width down below, see explanation there
+        let divisor = (self.handles.expected_num_requests_multiplier() as u64 - 1).max(1);
+
+        // add another `wordlist.len` to the expected per scan tracker in the statistics handler
+        self.handles
+            .stats
+            .send(AddToUsizeField(StatField::ExpectedPerScan, num_words))?;
+
+        // since we're adding extensions in the middle of scans (potentially), we need to take
+        // current number of requests into account, new_total will be used as an accumulator
+        // used to increment the overall progress bar
+        let mut new_total = 0;
+
+        if let Ok(ferox_scans) = self.handles.ferox_scans() {
+            // update progress bar length on FeroxScans, which used when creating a new FeroxScan's
+            // progress bar and should mirror the expected_per_scan field on Statistics
+            ferox_scans.set_bar_length(current_expectation);
+
+            if let Ok(scans_guard) = ferox_scans.scans.read() {
+                // update progress bars on each FeroxScan where its scan type is directory and
+                // scan status is either running or not-started
+                for scan in scans_guard.iter() {
+                    if scan.is_active() {
+                        // current number of words left in the 'to-scan' bin, for example:
+                        //
+                        // say we have a 2000 word wordlist, have `-x js` on the command line, and
+                        // just found `php` as a new extension
+                        //
+                        // that puts our state at:
+                        // - wordlist length: 2000
+                        // - total expected: 4000 (original length * 2 for -x js)
+                        //
+                        // let's assume the current scan has sent 3000 requests so far
+                        // that means to get the number of `words` left to send, we need to take
+                        // the difference of 4000 and 3000 and then divide that by the current
+                        // multiplier (2 in the example)
+                        //
+                        // (4000 - 3000) / 2 => 500 words left to send
+                        //
+                        // the remaining 500 words will be sent as 3 variations (word, word.js,
+                        // word.php). So, we would then need to increment the bar by 500 to
+                        // reflect the dynamism of adding extensions mid-scan.
+                        let bar = scan.progress_bar();
+
+                        // (4000 - 3000) / 2 => 500 words left to send
+                        let length = bar.length().unwrap_or(1);
+                        let num_words_left = (length - bar.position()) / divisor;
+
+                        // accumulate each bar's increment value for incrementing the total bar
+                        new_total += num_words_left;
+
+                        bar.inc_length(num_words_left);
+                    }
+                }
+            }
+
+            // add the total number of newly expected requests to the overall progress bar
+            // via the statistics handler
+            self.handles.stats.send(AddToUsizeField(
+                StatField::TotalExpected,
+                new_total as usize,
+            ))?;
+        }
+
+        log::trace!("exit: update_all_bar_lengths");
+        Ok(())
+    }
+
+    /// Helper to easily get the (locked) underlying wordlist
+    pub fn get_wordlist(&self, offset: usize) -> Result<Arc<Vec<String>>> {
+        if let Ok(guard) = self.wordlist.lock().as_ref() {
+            if let Some(list) = guard.as_ref() {
+                return if offset > 0 {
+                    Ok(Arc::new(list[offset..].to_vec()))
+                } else {
+                    Ok(list.clone())
+                };
+            }
+        }
+
+        bail!("Could not get underlying wordlist")
+    }
+
+    /// wrapper around scanning a url to stay DRY
+    async fn ordered_scan_url(&mut self, targets: Vec<String>, order: ScanOrder) -> Result<()> {
+        log::trace!("enter: ordered_scan_url({targets:?}, {order:?})");
+        let should_test_deny = !self.handles.config.url_denylist.is_empty()
+            || !self.handles.config.regex_denylist.is_empty();
+
+        for target in targets {
+            if self.data.contains(&target) && matches!(order, ScanOrder::Latest) {
+                // FeroxScans knows about this url and scan isn't an Initial scan
+                // initial scans are skipped because when resuming from a .state file, the scans
+                // will already be populated in FeroxScans, so we need to not skip kicking off
+                // their scans
+                continue;
+            }
+
+            let scan = if let Some(ferox_scan) = self.data.get_scan_by_url(&target) {
+                ferox_scan // scan already known
+            } else {
+                self.data
+                    .add_directory_scan(&target, order, self.handles.clone())
+                    .1 // add the new target; return FeroxScan
+            };
+
+            if should_test_deny
+                && should_deny_url(&parse_url_with_raw_path(&target)?, self.handles.clone())?
+            {
+                // response was caught by a user-provided deny list
+                // checking this last, since it's most susceptible to longer runtimes due to what
+                // input is received
+                continue;
+            }
+
+            let divisor = self.handles.expected_num_requests_multiplier();
+
+            let list = if divisor > 1 && scan.requests() > 0 {
+                // if there were extensions provided and/or more than a single method used, and some
+                // number of requests have already been sent, we need to adjust the offset into the
+                // wordlist to ensure we don't index out of bounds
+
+                let adjusted = scan.requests_made_so_far() as f64 / (divisor as f64 - 1.0).max(1.0);
+                self.get_wordlist(adjusted as usize)?
+            } else {
+                self.get_wordlist(scan.requests_made_so_far() as usize)?
+            };
+
+            log::info!("scan handler received {target} - beginning scan");
+
+            if matches!(order, ScanOrder::Initial) {
+                // keeps track of the initial targets' scan depths in order to enforce the
+                // maximum recursion depth on any identified sub-directories
+                let url = FeroxUrl::from_string(&target, self.handles.clone());
+                let depth = url.depth().unwrap_or(0);
+                self.depths.push((target.clone(), depth));
+            }
+
+            let scanner = FeroxScanner::new(
+                &target,
+                order,
+                list,
+                self.limiter.clone(),
+                self.handles.clone(),
+            );
+
+            let task = tokio::spawn(async move {
+                if let Err(e) = scanner.scan_url().await {
+                    log::warn!("{e}");
+                }
+            });
+
+            self.handles.stats.send(AddToUsizeField(TotalScans, 1))?;
+
+            scan.set_task(task).await?;
+
+            self.tasks.push(scan.clone());
+        }
+
+        log::trace!("exit: ordered_scan_url");
+        Ok(())
+    }
+
+    async fn try_recursion(&mut self, response: Box<FeroxResponse>) -> Result<()> {
+        log::trace!("enter: try_recursion({response:?})",);
+
+        if !self.handles.config.force_recursion && !response.is_directory() {
+            // not a directory and --force-recursion wasn't used, quick exit
+            return Ok(());
+        }
+
+        let mut base_depth = 1_usize;
+
+        for (base_url, base_url_depth) in &self.depths {
+            if response.url().as_str().starts_with(base_url) {
+                base_depth = *base_url_depth;
+            }
+        }
+
+        if response.reached_max_depth(base_depth, self.max_depth, self.handles.clone()) {
+            // at or past recursion depth
+            return Ok(());
+        }
+
+        if let Ok(responses) = RESPONSES.responses.read() {
+            for maybe_wild in responses.iter() {
+                if !maybe_wild.wildcard() || !maybe_wild.is_directory() {
+                    // if the stored response isn't a wildcard, skip it
+                    // if the stored response isn't a directory, skip it
+                    // we're only interested in preventing recursion into wildcard directories
+                    continue;
+                }
+
+                if maybe_wild.method() != response.method() {
+                    // methods don't match, skip it
+                    continue;
+                }
+
+                // methods match and is a directory wildcard
+                // need to check the wildcard's parent directory
+                // for equality with the incoming response's parent directory
+                //
+                // if the parent directories match, we need to prevent recursion
+                // into the wildcard directory
+
+                match (
+                    maybe_wild.url().path_segments(),
+                    response.url().path_segments(),
+                ) {
+                    // both urls must have path segments
+                    (Some(mut maybe_wild_segments), Some(mut response_segments)) => {
+                        match (
+                            maybe_wild_segments.nth_back(1),
+                            response_segments.nth_back(1),
+                        ) {
+                            // both urls must have at least 2 path segments, the next to last being the parent
+                            (Some(maybe_wild_parent), Some(response_parent)) => {
+                                if maybe_wild_parent == response_parent {
+                                    // the parent directories match, so we need to prevent recursion
+                                    return Ok(());
+                                }
+                            }
+                            _ => {
+                                // we couldn't get the parent directory, so we'll skip this
+                                continue;
+                            }
+                        }
+                    }
+                    _ => {
+                        // we couldn't get the path segments, so we'll skip this
+                        continue;
+                    }
+                }
+            }
+        }
+
+        let targets = vec![response.url().to_string()];
+        self.ordered_scan_url(targets, ScanOrder::Latest).await?;
+
+        log::info!("Added new directory to recursive scan: {}", response.url());
+
+        log::trace!("exit: try_recursion");
+        Ok(())
+    }
+}

src/event_handlers/statistics.rs

@@ -0,0 +1,183 @@
+use super::*;
+use crate::{
+    config::Configuration,
+    progress::{add_bar, BarType},
+    statistics::{StatField, Stats},
+    CommandSender, FeroxChannel, Joiner,
+};
+use anyhow::Result;
+use console::style;
+use indicatif::ProgressBar;
+use std::{sync::Arc, time::Instant};
+use tokio::sync::{
+    mpsc::{self, UnboundedReceiver},
+    oneshot,
+};
+
+#[derive(Debug)]
+/// Container for statistics transmitter and Stats object
+pub struct StatsHandle {
+    /// Stats object used across modules to track statistics
+    pub data: Arc<Stats>,
+
+    /// transmitter used to update `data`
+    pub tx: CommandSender,
+}
+
+/// implementation of StatsHandle
+impl StatsHandle {
+    /// Given an Arc-wrapped Stats and CommandSender, create a new StatsHandle
+    pub fn new(data: Arc<Stats>, tx: CommandSender) -> Self {
+        Self { data, tx }
+    }
+
+    /// Send the given Command over `tx`
+    pub fn send(&self, command: Command) -> Result<()> {
+        self.tx.send(command)?;
+        Ok(())
+    }
+
+    /// Sync the handle with the handler
+    pub async fn sync(&self) -> Result<()> {
+        let (tx, rx) = oneshot::channel::<bool>();
+        self.send(Command::Sync(tx))?;
+        rx.await?;
+        Ok(())
+    }
+}
+
+/// event handler struct for updating statistics
+#[derive(Debug)]
+pub struct StatsHandler {
+    /// overall scan's progress bar
+    bar: ProgressBar,
+
+    /// Receiver half of mpsc from which `StatCommand`s are processed
+    receiver: UnboundedReceiver<Command>,
+
+    /// data class that stores all statistics updates
+    stats: Arc<Stats>,
+}
+
+/// implementation of event handler for statistics
+impl StatsHandler {
+    /// create new event handler
+    fn new(stats: Arc<Stats>, rx_stats: UnboundedReceiver<Command>) -> Self {
+        // will be updated later via StatCommand; delay is for banner to print first
+        let bar = ProgressBar::hidden();
+
+        Self {
+            bar,
+            stats,
+            receiver: rx_stats,
+        }
+    }
+
+    /// Start a single consumer task (sc side of mpsc)
+    ///
+    /// The consumer simply receives `StatCommands` and updates the given `Stats` object as appropriate
+    async fn start(&mut self, output_file: &str) -> Result<()> {
+        log::trace!("enter: start({self:?})");
+
+        let start = Instant::now();
+
+        while let Some(command) = self.receiver.recv().await {
+            match command as Command {
+                Command::AddError(err) => {
+                    self.stats.add_error(err);
+                    self.increment_bar();
+                }
+                Command::AddStatus(status) => {
+                    self.stats.add_status_code(status);
+
+                    self.increment_bar();
+                }
+                Command::AddRequest => {
+                    self.stats.add_request();
+                    self.increment_bar();
+                }
+                Command::Save => {
+                    self.stats
+                        .save(start.elapsed().as_secs_f64(), output_file)?;
+                }
+                Command::AddToUsizeField(field, value) => {
+                    self.stats.update_usize_field(field, value);
+
+                    if matches!(field, StatField::TotalScans | StatField::TotalExpected) {
+                        self.bar.set_length(self.stats.total_expected() as u64);
+                    }
+                }
+                Command::SubtractFromUsizeField(field, value) => {
+                    self.stats.subtract_from_usize_field(field, value);
+
+                    if matches!(field, StatField::TotalExpected) {
+                        self.bar.set_length(self.stats.total_expected() as u64);
+                    }
+                }
+                Command::AddToF64Field(field, value) => self.stats.update_f64_field(field, value),
+                Command::CreateBar(offset) => {
+                    self.bar = add_bar("", self.stats.total_expected() as u64, BarType::Total);
+                    self.bar.set_position(offset);
+                }
+                Command::LoadStats(filename) => {
+                    self.stats.merge_from(&filename)?;
+                }
+                Command::Sync(sender) => {
+                    sender.send(true).unwrap_or_default();
+                }
+                Command::QueryOverallBarEta(sender) => {
+                    sender.send(self.bar.eta()).unwrap_or_default();
+                }
+                Command::UpdateTargets(targets) => {
+                    self.stats.update_targets(targets);
+                }
+                Command::Exit => break,
+                _ => {} // no more commands needed
+            }
+        }
+
+        self.bar.finish();
+
+        log::info!("{:#?}", *self.stats);
+        log::trace!("exit: start");
+        Ok(())
+    }
+
+    /// Wrapper around incrementing the overall scan's progress bar
+    fn increment_bar(&self) {
+        let msg = format!(
+            "{}:{:<7} {}:{:<7}",
+            style("found").green(),
+            self.stats.resources_discovered(),
+            style("errors").red(),
+            self.stats.errors(),
+        );
+
+        self.bar.set_message(msg);
+
+        if self.bar.position() < self.stats.total_expected() as u64 {
+            // don't run off the end when we're a few requests over the expected total
+            // due to the heuristics tests
+            self.bar.inc(1);
+        }
+    }
+
+    /// Initialize new `Stats` object and the sc side of an mpsc channel that is responsible for
+    /// updates to the aforementioned object.
+    pub fn initialize(config: Arc<Configuration>) -> (Joiner, StatsHandle) {
+        log::trace!("enter: initialize");
+
+        let data = Arc::new(Stats::new(config.json));
+        let (tx, rx): FeroxChannel<Command> = mpsc::unbounded_channel();
+
+        let mut handler = StatsHandler::new(data.clone(), rx);
+
+        let task = tokio::spawn(async move { handler.start(&config.output).await });
+
+        let event_handle = StatsHandle::new(data, tx);
+
+        log::trace!("exit: initialize -> ({task:?}, {event_handle:?})");
+
+        (task, event_handle)
+    }
+}

src/extractor.rs

@@ -1,269 +0,0 @@
-use crate::FeroxResponse;
-use lazy_static::lazy_static;
-use regex::Regex;
-use reqwest::Url;
-use std::collections::HashSet;
-
-/// Regular expression used in [LinkFinder](https://github.com/GerbenJavado/LinkFinder)
-///
-/// Incorporates change from this [Pull Request](https://github.com/GerbenJavado/LinkFinder/pull/66/files)
-const LINKFINDER_REGEX: &str = r#"(?:"|')(((?:[a-zA-Z]{1,10}://|//)[^"'/]{1,}\.[a-zA-Z]{2,}[^"']{0,})|((?:/|\.\./|\./)[^"'><,;| *()(%%$^/\\\[\]][^"'><,;|()]{1,})|([a-zA-Z0-9_\-/]{1,}/[a-zA-Z0-9_\-/]{1,}\.(?:[a-zA-Z]{1,4}|action)(?:[\?|#][^"|']{0,}|))|([a-zA-Z0-9_\-/]{1,}/[a-zA-Z0-9_\-/]{3,}(?:[\?|#][^"|']{0,}|))|([a-zA-Z0-9_\-.]{1,}\.(?:php|asp|aspx|jsp|json|action|html|js|txt|xml)(?:[\?|#][^"|']{0,}|)))(?:"|')"#;
-
-lazy_static! {
-    /// `LINKFINDER_REGEX` as a regex::Regex type
-    static ref REGEX: Regex = Regex::new(LINKFINDER_REGEX).unwrap();
-}
-
-/// Iterate over a given path, return a list of every sub-path found
-///
-/// example: `path` contains a link fragment `homepage/assets/img/icons/handshake.svg`
-/// the following fragments would be returned:
-///   - homepage/assets/img/icons/handshake.svg
-///   - homepage/assets/img/icons/
-///   - homepage/assets/img/
-///   - homepage/assets/
-///   - homepage/
-fn get_sub_paths_from_path(path: &str) -> Vec<String> {
-    log::trace!("enter: get_sub_paths_from_path({})", path);
-    let mut paths = vec![];
-
-    // filter out any empty strings caused by .split
-    let mut parts: Vec<&str> = path.split('/').filter(|s| !s.is_empty()).collect();
-
-    let length = parts.len();
-
-    for _ in 0..length {
-        // iterate over all parts of the path
-        if parts.is_empty() {
-            // pop left us with an empty vector, we're done
-            break;
-        }
-
-        let possible_path = parts.join("/");
-
-        if possible_path.is_empty() {
-            // .join can result in an empty string, which we don't need, ignore
-            continue;
-        }
-
-        paths.push(possible_path); // good sub-path found
-        parts.pop(); // use .pop() to remove the last part of the path and continue iteration
-    }
-
-    log::trace!("exit: get_sub_paths_from_path -> {:?}", paths);
-    paths
-}
-
-/// simple helper to stay DRY, trys to join a url + fragment and add it to the `links` HashSet
-fn add_link_to_set_of_links(link: &str, url: &Url, links: &mut HashSet<String>) {
-    log::trace!(
-        "enter: add_link_to_set_of_links({}, {}, {:?})",
-        link,
-        url.to_string(),
-        links
-    );
-    match url.join(&link) {
-        Ok(new_url) => {
-            links.insert(new_url.to_string());
-        }
-        Err(e) => {
-            log::error!("Could not join given url to the base url: {}", e);
-        }
-    }
-    log::trace!("exit: add_link_to_set_of_links");
-}
-
-/// Given a `reqwest::Response`, perform the following actions
-///   - parse the response's text for links using the linkfinder regex
-///   - for every link found take its url path and parse each sub-path
-///     - example: Response contains a link fragment `homepage/assets/img/icons/handshake.svg`
-///       with a base url of http://localhost, the following urls would be returned:
-///         - homepage/assets/img/icons/handshake.svg
-///         - homepage/assets/img/icons/
-///         - homepage/assets/img/
-///         - homepage/assets/
-///         - homepage/
-pub async fn get_links(response: &FeroxResponse) -> HashSet<String> {
-    log::trace!("enter: get_links({})", response.url().as_str());
-
-    let mut links = HashSet::<String>::new();
-
-    let body = response.text();
-
-    for capture in REGEX.captures_iter(&body) {
-        // remove single & double quotes from both ends of the capture
-        // capture[0] is the entire match, additional capture groups start at [1]
-        let link = capture[0].trim_matches(|c| c == '\'' || c == '"');
-
-        match Url::parse(link) {
-            Ok(absolute) => {
-                if absolute.domain() != response.url().domain()
-                    || absolute.host() != response.url().host()
-                {
-                    // domains/ips are not the same, don't scan things that aren't part of the original
-                    // target url
-                    continue;
-                }
-
-                for sub_path in get_sub_paths_from_path(absolute.path()) {
-                    // take a url fragment like homepage/assets/img/icons/handshake.svg and
-                    // incrementally add
-                    //     - homepage/assets/img/icons/
-                    //     - homepage/assets/img/
-                    //     - homepage/assets/
-                    //     - homepage/
-                    log::debug!("Adding {} to {:?}", sub_path, links);
-                    add_link_to_set_of_links(&sub_path, &response.url(), &mut links);
-                }
-            }
-            Err(e) => {
-                // this is the expected error that happens when we try to parse a url fragment
-                //     ex: Url::parse("/login") -> Err("relative URL without a base")
-                // while this is technically an error, these are good results for us
-                if e.to_string().contains("relative URL without a base") {
-                    for sub_path in get_sub_paths_from_path(link) {
-                        // incrementally save all sub-paths that led to the relative url's resource
-                        log::debug!("Adding {} to {:?}", sub_path, links);
-                        add_link_to_set_of_links(&sub_path, &response.url(), &mut links);
-                    }
-                } else {
-                    // unexpected error has occurred
-                    log::error!("Could not parse given url: {}", e);
-                }
-            }
-        }
-    }
-
-    log::trace!("exit: get_links -> {:?}", links);
-    links
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::utils::make_request;
-    use httpmock::Method::GET;
-    use httpmock::{Mock, MockServer};
-    use reqwest::Client;
-
-    #[test]
-    /// extract sub paths from the given url fragment; expect 4 sub paths and that all are
-    /// in the expected array
-    fn extractor_get_sub_paths_from_path_with_multiple_paths() {
-        let path = "homepage/assets/img/icons/handshake.svg";
-        let paths = get_sub_paths_from_path(&path);
-        let expected = vec![
-            "homepage",
-            "homepage/assets",
-            "homepage/assets/img",
-            "homepage/assets/img/icons",
-            "homepage/assets/img/icons/handshake.svg",
-        ];
-
-        assert_eq!(paths.len(), expected.len());
-        for expected_path in expected {
-            assert_eq!(paths.contains(&expected_path.to_string()), true);
-        }
-    }
-
-    #[test]
-    /// extract sub paths from the given url fragment; expect 2 sub paths and that all are
-    /// in the expected array. the fragment is wrapped in slashes to ensure no empty strings are
-    /// returned
-    fn extractor_get_sub_paths_from_path_with_enclosing_slashes() {
-        let path = "/homepage/assets/";
-        let paths = get_sub_paths_from_path(&path);
-        let expected = vec!["homepage", "homepage/assets"];
-
-        assert_eq!(paths.len(), expected.len());
-        for expected_path in expected {
-            assert_eq!(paths.contains(&expected_path.to_string()), true);
-        }
-    }
-
-    #[test]
-    /// extract sub paths from the given url fragment; expect 1 sub path, no forward slashes are
-    /// included
-    fn extractor_get_sub_paths_from_path_with_only_a_word() {
-        let path = "homepage";
-        let paths = get_sub_paths_from_path(&path);
-        let expected = vec!["homepage"];
-
-        assert_eq!(paths.len(), expected.len());
-        for expected_path in expected {
-            assert_eq!(paths.contains(&expected_path.to_string()), true);
-        }
-    }
-
-    #[test]
-    /// extract sub paths from the given url fragment; expect 1 sub path, forward slash removed
-    fn extractor_get_sub_paths_from_path_with_an_absolute_word() {
-        let path = "/homepage";
-        let paths = get_sub_paths_from_path(&path);
-        let expected = vec!["homepage"];
-
-        assert_eq!(paths.len(), expected.len());
-        for expected_path in expected {
-            assert_eq!(paths.contains(&expected_path.to_string()), true);
-        }
-    }
-
-    #[test]
-    /// test that a full url and fragment are joined correctly, then added to the given list
-    /// i.e. the happy path
-    fn extractor_add_link_to_set_of_links_happy_path() {
-        let url = Url::parse("https://localhost").unwrap();
-        let mut links = HashSet::<String>::new();
-        let link = "admin";
-
-        assert_eq!(links.len(), 0);
-        add_link_to_set_of_links(link, &url, &mut links);
-
-        assert_eq!(links.len(), 1);
-        assert!(links.contains("https://localhost/admin"));
-    }
-
-    #[test]
-    /// test that an invalid path fragment doesn't add anything to the set of links
-    fn extractor_add_link_to_set_of_links_with_non_base_url() {
-        let url = Url::parse("https://localhost").unwrap();
-        let mut links = HashSet::<String>::new();
-        let link = "\\\\\\\\";
-
-        assert_eq!(links.len(), 0);
-        add_link_to_set_of_links(link, &url, &mut links);
-
-        assert_eq!(links.len(), 0);
-        assert!(links.is_empty());
-    }
-
-    #[tokio::test(core_threads = 1)]
-    /// use make_request to generate a Response, and use the Response to test get_links;
-    /// the response will contain an absolute path to a domain that is not part of the scanned
-    /// domain; expect an empty set returned
-    async fn extractor_get_links_with_absolute_url_that_differs_from_target_domain(
-    ) -> Result<(), Box<dyn std::error::Error>> {
-        let srv = MockServer::start();
-
-        let mock = Mock::new()
-            .expect_method(GET)
-            .expect_path("/some-path")
-            .return_status(200)
-            .return_body("\"http://defintely.not.a.thing.probably.com/homepage/assets/img/icons/handshake.svg\"")
-            .create_on(&srv);
-
-        let client = Client::new();
-        let url = Url::parse(&srv.url("/some-path")).unwrap();
-
-        let response = make_request(&client, &url).await.unwrap();
-
-        let ferox_response = FeroxResponse::from(response, true).await;
-
-        let links = get_links(&ferox_response).await;
-
-        assert!(links.is_empty());
-
-        assert_eq!(mock.times_called(), 1);
-        Ok(())
-    }
-}

src/extractor/builder.rs

@@ -0,0 +1,145 @@
+use super::*;
+use crate::event_handlers::Handles;
+use anyhow::{bail, Result};
+
+/// Regular expression used in [LinkFinder](https://github.com/GerbenJavado/LinkFinder)
+///
+/// updated on 8 August 2025 to commit 1debac5dace4724fd6187c06f133578dae51c86f
+///
+/// NOTE: the ` ? or # mark with parameters` lines need to have the # character escaped as `\#`
+/// to avoid being interpreted as a comment by the Rust compiler
+pub(super) const LINKFINDER_REGEX: &str = r#"(?x)
+  (?:"|')                               # Start newline delimiter
+
+  (
+    ((?:[a-zA-Z]{1,10}://|//)           # Match a scheme [a-Z]*1-10 or //
+    [^"'/]{1,}\.                        # Match a domainname (any character + dot)
+    [a-zA-Z]{2,}[^"']{0,})              # The domainextension and/or path
+
+    |
+
+    ((?:/|\.\./|\./)                    # Start with /,../,./
+    [^"'><,;| *()(%%$^/\\\[\]]          # Next character can't be...
+    [^"'><,;|()]{1,})                   # Rest of the characters can't be
+
+    |
+
+    ([a-zA-Z0-9_\-/]{1,}/               # Relative endpoint with /
+    [a-zA-Z0-9_\-/.]{1,}                # Resource name
+    \.(?:[a-zA-Z]{1,4}|action)          # Rest + extension (length 1-4 or action)
+    (?:[\?|\#][^"|']{0,}|))             # ? or # mark with parameters
+
+    |
+
+    ([a-zA-Z0-9_\-/]{1,}/               # REST API (no extension) with /
+    [a-zA-Z0-9_\-/]{3,}                 # Proper REST endpoints usually have 3+ chars
+    (?:[\?|\#][^"|']{0,}|))             # ? or # mark with parameters
+
+    |
+
+    ([a-zA-Z0-9_\-]{1,}                 # filename
+    \.(?:php|asp|aspx|jsp|json|
+         action|html|js|txt|xml)        # . + extension
+    (?:[\?|\#][^"|']{0,}|))             # ? or # mark with parameters
+
+  )
+
+  (?:"|')                               # End newline delimiter
+"#;
+
+/// Regular expression to pull url paths from robots.txt
+///
+/// ref: https://developers.google.com/search/reference/robots_txt
+pub(super) const ROBOTS_TXT_REGEX: &str =
+    r#"(?m)^[ \t]*(?i)(allow|disallow)[ \t]*:[ \t]*(?P<url_path>[^ \t\r\n#$]*)?[ \t]*\$?(?:#.*)?$"#; // multi-line (?m), case-insensitive (?i)
+
+/// Regular expression to filter bad characters from extracted url paths
+///
+/// ref: https://www.rfc-editor.org/rfc/rfc3986#section-2
+pub(super) const URL_CHARS_REGEX: &str = r#"["<>\\^`{|} ]"#;
+
+/// Which type of extraction should be performed
+#[derive(Debug, Copy, Clone)]
+pub enum ExtractionTarget {
+    /// Examine a response body and extract javascript and html links (multiple tags)
+    ResponseBody,
+
+    /// Examine robots.txt (specifically) and extract links
+    RobotsTxt,
+
+    /// Extract all <a> tags from a page
+    DirectoryListing,
+}
+
+/// responsible for building an `Extractor`
+pub struct ExtractorBuilder<'a> {
+    /// Response from which to extract links
+    response: Option<&'a FeroxResponse>,
+
+    /// URL of where to extract links
+    url: String,
+
+    /// Handles object to house the underlying mpsc transmitters
+    handles: Option<Arc<Handles>>,
+
+    /// type of extraction to be performed
+    target: ExtractionTarget,
+}
+
+/// ExtractorBuilder implementation
+impl<'a> Default for ExtractorBuilder<'a> {
+    fn default() -> Self {
+        Self {
+            response: None,
+            url: "".to_string(),
+            handles: None,
+            target: ExtractionTarget::ResponseBody,
+        }
+    }
+}
+
+/// ExtractorBuilder implementation
+impl<'a> ExtractorBuilder<'a> {
+    /// builder call to set `handles`
+    pub fn handles(&mut self, handles: Arc<Handles>) -> &mut Self {
+        self.handles = Some(handles);
+        self
+    }
+
+    /// builder call to set `url`
+    pub fn url(&mut self, url: &str) -> &mut Self {
+        self.url = url.to_string();
+        self
+    }
+
+    /// builder call to set `target`
+    pub fn target(&mut self, target: ExtractionTarget) -> &mut Self {
+        self.target = target;
+        self
+    }
+
+    /// builder call to set `response`
+    pub fn response(&mut self, response: &'a FeroxResponse) -> &mut Self {
+        self.response = Some(response);
+        self
+    }
+
+    /// finalize configuration of `ExtractorBuilder` and return an `Extractor`
+    ///
+    /// requires either `with_url` or `with_response` to have been used in the build process
+    pub fn build(&self) -> Result<Extractor<'a>> {
+        if (self.url.is_empty() && self.response.is_none()) || self.handles.is_none() {
+            bail!("Extractor requires a URL or a FeroxResponse be specified as well as a Handles object")
+        }
+
+        Ok(Extractor {
+            links_regex: Regex::new(LINKFINDER_REGEX).unwrap(),
+            robots_regex: Regex::new(ROBOTS_TXT_REGEX).unwrap(),
+            url_regex: Regex::new(URL_CHARS_REGEX).unwrap(),
+            response: self.response,
+            url: self.url.to_owned(),
+            handles: self.handles.as_ref().unwrap().clone(),
+            target: self.target,
+        })
+    }
+}

src/extractor/container.rs

@@ -0,0 +1,724 @@
+use super::*;
+use crate::{
+    client,
+    event_handlers::{
+        Command::{AddError, AddToUsizeField},
+        Handles,
+    },
+    filters::SimilarityFilter,
+    scan_manager::ScanOrder,
+    statistics::{
+        StatError::Other,
+        StatField::{LinksExtracted, TotalExpected},
+    },
+    url::{FeroxUrl, UrlExt},
+    utils::{
+        logged_request, make_request, parse_url_with_raw_path, send_try_recursion_command,
+        should_deny_url,
+    },
+    ExtractionResult, DEFAULT_METHOD, UNIQUE_DISTANCE,
+};
+use anyhow::{bail, Context, Result};
+use futures::StreamExt;
+use reqwest::{Client, Response, StatusCode, Url};
+use scraper::{Html, Selector};
+use std::{borrow::Cow, collections::HashSet};
+
+/// Wrapper around link extraction logic
+///   - create a new Url object based on cli options/args
+///   - check if the new Url has already been seen/scanned -> None
+///   - make a request to the new Url ? -> Some(response) : None
+pub(super) async fn request_link(url: &str, handles: Arc<Handles>) -> Result<Response> {
+    log::trace!("enter: request_link({url})");
+
+    let ferox_url = FeroxUrl::from_string(url, handles.clone());
+
+    // create a url based on the given command line options
+    let new_url = ferox_url.format("", None)?;
+
+    let scanned_urls = handles.ferox_scans()?;
+
+    if scanned_urls.get_scan_by_url(new_url.as_ref()).is_some() {
+        //we've seen the url before and don't need to scan again
+        log::trace!("exit: request_link -> None");
+        bail!("previously seen url");
+    }
+
+    if (!handles.config.url_denylist.is_empty() || !handles.config.regex_denylist.is_empty())
+        && should_deny_url(&new_url, handles.clone())?
+    {
+        // can't allow a denied url to be requested
+        bail!(
+            "prevented request to {} due to {:?} || {:?}",
+            url,
+            handles.config.url_denylist,
+            handles.config.regex_denylist,
+        );
+    }
+
+    // make the request and store the response
+    let new_response = logged_request(&new_url, DEFAULT_METHOD, None, handles.clone()).await?;
+
+    log::trace!("exit: request_link -> {new_response:?}");
+
+    Ok(new_response)
+}
+
+/// Whether an active scan is recursive or not
+#[derive(Debug, Copy, Clone)]
+enum RecursionStatus {
+    /// Scan is recursive
+    Recursive,
+
+    /// Scan is not recursive
+    NotRecursive,
+}
+
+/// Handles all logic related to extracting links from requested source code
+#[derive(Debug)]
+pub struct Extractor<'a> {
+    /// `LINKFINDER_REGEX` as a regex::Regex type
+    pub(super) links_regex: Regex,
+
+    /// `ROBOTS_TXT_REGEX` as a regex::Regex type
+    pub(super) robots_regex: Regex,
+
+    /// regex to validate a url
+    pub(super) url_regex: Regex,
+
+    /// Response from which to extract links
+    pub(super) response: Option<&'a FeroxResponse>,
+
+    /// URL of where to extract links
+    pub(super) url: String,
+
+    /// Handles object to house the underlying mpsc transmitters
+    pub(super) handles: Arc<Handles>,
+
+    /// type of extraction to be performed
+    pub(super) target: ExtractionTarget,
+}
+
+/// Extractor implementation
+impl<'a> Extractor<'a> {
+    /// perform extraction from the given target and return any links found
+    pub async fn extract(&self) -> Result<ExtractionResult> {
+        log::trace!(
+            "enter: extract({:?}) (this fn has no associated trace exit msg)",
+            self.target
+        );
+        match self.target {
+            ExtractionTarget::ResponseBody => Ok(self.extract_from_body().await?),
+            ExtractionTarget::RobotsTxt => Ok(self.extract_from_robots().await?),
+            ExtractionTarget::DirectoryListing => Ok(self.extract_from_dir_listing().await?),
+        }
+    }
+
+    /// wrapper around logic that performs the following:
+    /// - parses `url_to_parse`
+    /// - bails if the parsed url doesn't belong to the list of in-scope urls
+    /// - otherwise, calls `add_all_sub_paths` with the parsed result
+    fn parse_url_and_add_subpaths(
+        &self,
+        url_to_parse: &str,
+        links: &mut HashSet<String>,
+    ) -> Result<()> {
+        log::trace!("enter: parse_url_and_add_subpaths({links:?})");
+
+        match parse_url_with_raw_path(url_to_parse) {
+            Ok(absolute) => {
+                if !absolute.is_in_scope(&self.handles.config.scope) {
+                    // URL is not in scope based on domain/scope configuration
+                    bail!("parsed url is not in scope");
+                }
+
+                if self.add_all_sub_paths(absolute.path(), links).is_err() {
+                    log::warn!("could not add sub-paths from {absolute} to {links:?}");
+                }
+            }
+            Err(e) => {
+                // this is the expected error that happens when we try to parse a url fragment
+                //     ex: Url::parse("/login") -> Err("relative URL without a base")
+                // while this is technically an error, these are good results for us
+                if e.to_string().contains("relative URL without a base") {
+                    // scope for these should be enforced in add_all_sub_paths since
+                    // we join the fragment with the base url there and can check
+                    // the full Url against scope
+                    if self.add_all_sub_paths(url_to_parse, links).is_err() {
+                        log::warn!("could not add sub-paths from {url_to_parse} to {links:?}");
+                    }
+                } else {
+                    // unexpected error has occurred
+                    log::warn!("Could not parse given url: {e}");
+                    self.handles.stats.send(AddError(Other)).unwrap_or_default();
+                }
+            }
+        }
+
+        log::trace!("exit: parse_url_and_add_subpaths");
+        Ok(())
+    }
+
+    /// given a set of links from a normal http body response, task the request handler to make
+    /// the requests
+    pub async fn request_links(
+        &mut self,
+        links: HashSet<String>,
+    ) -> Result<Option<tokio::task::JoinHandle<()>>> {
+        log::trace!("enter: request_links({links:?})");
+
+        if links.is_empty() {
+            return Ok(None);
+        }
+
+        self.update_stats(links.len())?;
+
+        // create clones/remove use of self of/from everything the async move block will need to function
+        let cloned_scanned_urls = self.handles.ferox_scans()?;
+        let cloned_handles = self.handles.clone();
+        let cloned_url = self.url.clone();
+        let threads = self.handles.config.threads;
+        let recursive = if self.handles.config.no_recursion {
+            RecursionStatus::NotRecursive
+        } else {
+            RecursionStatus::Recursive
+        };
+
+        let link_request_task = tokio::spawn(async move {
+            let producers = futures::stream::iter(links.into_iter())
+                .map(|link| {
+                    // another clone to satisfy the async move block
+                    let inner_clone = cloned_handles.clone();
+
+                    (
+                        tokio::spawn(async move { request_link(&link, inner_clone).await }),
+                        cloned_handles.clone(),
+                        cloned_scanned_urls.clone(),
+                        recursive,
+                        cloned_url.clone(),
+                    )
+                })
+                .for_each_concurrent(
+                    threads,
+                    |(join_handle, c_handles, c_scanned_urls, c_recursive, og_url)| async move {
+                        match join_handle.await {
+                            Ok(Ok(reqwest_response)) => {
+                                let mut resp = FeroxResponse::from(
+                                    reqwest_response,
+                                    &og_url,
+                                    DEFAULT_METHOD,
+                                    c_handles.config.output_level,
+                                    c_handles.config.response_size_limit,
+                                )
+                                .await;
+
+                                // filter if necessary
+                                if c_handles
+                                    .filters
+                                    .data
+                                    .should_filter_response(&resp, c_handles.stats.tx.clone())
+                                {
+                                    return;
+                                }
+
+                                if c_handles.config.unique {
+                                    // if the filter above didn't filter it out, add it as a unique filter
+                                    let mut unique_filter = SimilarityFilter::from(&resp);
+                                    unique_filter.cutoff = UNIQUE_DISTANCE;
+                                    c_handles
+                                        .filters
+                                        .data
+                                        .push(Box::new(unique_filter))
+                                        .unwrap_or_default();
+                                }
+
+                                // request and report assumed file
+                                if !resp.is_directory() && !c_handles.config.force_recursion {
+                                    log::debug!("Extracted File: {resp}");
+
+                                    c_scanned_urls.add_file_scan(
+                                        resp.url().as_str(),
+                                        ScanOrder::Latest,
+                                        c_handles.clone(),
+                                    );
+
+                                    if c_handles.config.collect_extensions {
+                                        // no real reason this should fail
+                                        resp.parse_extension(c_handles.clone()).unwrap();
+                                    }
+
+                                    if let Err(e) = resp.send_report(c_handles.output.tx.clone()) {
+                                        log::warn!(
+                                            "Could not send FeroxResponse to output handler: {e}"
+                                        );
+                                    }
+
+                                    return;
+                                }
+
+                                if matches!(c_recursive, RecursionStatus::Recursive) {
+                                    log::debug!("Extracted Directory: {resp}");
+
+                                    if !resp.url().as_str().ends_with('/')
+                                        && (resp.status().is_success()
+                                            || matches!(resp.status(), &StatusCode::FORBIDDEN))
+                                    {
+                                        // if the url doesn't end with a /
+                                        // and the response code is either a 2xx or 403
+
+                                        // since all of these are 2xx or 403, recursion is only attempted if the
+                                        // url ends in a /. I am actually ok with adding the slash and not
+                                        // adding it, as both have merit.  Leaving it in for now to see how
+                                        // things turn out (current as of: v1.1.0)
+                                        resp.set_url(&format!("{}/", resp.url()));
+                                    }
+
+                                    if c_handles.config.filter_status.is_empty() {
+                                        // -C wasn't used, so -s is the only 'filter' left to account for
+                                        if c_handles
+                                            .config
+                                            .status_codes
+                                            .contains(&resp.status().as_u16())
+                                        {
+                                            send_try_recursion_command(c_handles.clone(), resp)
+                                                .await
+                                                .unwrap_or_default();
+                                        }
+                                    } else {
+                                        // -C was used, that means the filters above would have removed
+                                        // those responses, and anything else should be let through
+                                        send_try_recursion_command(c_handles.clone(), resp)
+                                            .await
+                                            .unwrap_or_default();
+                                    }
+                                }
+                            }
+                            Ok(Err(err)) => {
+                                log::warn!("Error during link extraction: {err}");
+                            }
+                            Err(err) => {
+                                log::warn!("JoinError during link extraction: {err}");
+                            }
+                        }
+                    },
+                );
+
+            // wait for the requests to finish
+            producers.await;
+        });
+
+        log::trace!("exit: request_links");
+        Ok(Some(link_request_task))
+    }
+
+    /// wrapper around link extraction via html attributes
+    fn extract_all_links_from_html_tags(
+        &self,
+        resp_url: &Url,
+        links: &mut HashSet<String>,
+        html: &Html,
+    ) {
+        self.extract_links_by_attr(resp_url, links, html, "a", "href");
+        self.extract_links_by_attr(resp_url, links, html, "img", "src");
+        self.extract_links_by_attr(resp_url, links, html, "form", "action");
+        self.extract_links_by_attr(resp_url, links, html, "script", "src");
+        self.extract_links_by_attr(resp_url, links, html, "iframe", "src");
+        self.extract_links_by_attr(resp_url, links, html, "div", "src");
+        self.extract_links_by_attr(resp_url, links, html, "frame", "src");
+        self.extract_links_by_attr(resp_url, links, html, "embed", "src");
+        self.extract_links_by_attr(resp_url, links, html, "link", "href");
+    }
+
+    /// Given the body of a `reqwest::Response`, perform the following actions
+    ///   - parse the body for links using the linkfinder regex
+    ///   - for every link found take its url path and parse each sub-path
+    ///     - example: Response contains a link fragment `homepage/assets/img/icons/handshake.svg`
+    ///       with a base url of http://localhost, the following urls would be returned:
+    ///         - homepage/assets/img/icons/handshake.svg
+    ///         - homepage/assets/img/icons/
+    ///         - homepage/assets/img/
+    ///         - homepage/assets/
+    ///         - homepage/
+    fn extract_all_links_from_javascript(
+        &self,
+        response_body: &str,
+        response_url: &Url,
+        links: &mut HashSet<String>,
+    ) {
+        log::trace!(
+            "enter: extract_all_links_from_javascript(html body..., {}, {:?})",
+            response_url.as_str(),
+            links
+        );
+
+        for capture in self.links_regex.captures_iter(response_body) {
+            // remove single & double quotes from both ends of the capture
+            // capture[0] is the entire match, additional capture groups start at [1]
+            let link = capture[0].trim_matches(|c| c == '\'' || c == '"');
+
+            if self.parse_url_and_add_subpaths(link, links).is_err() {
+                // purposely not logging the error here, due to the frequency with which it gets hit
+            }
+        }
+
+        log::trace!("exit: extract_all_links_from_javascript");
+    }
+
+    /// take a url fragment like homepage/assets/img/icons/handshake.svg and
+    /// incrementally add
+    ///   - homepage/assets/img/icons/
+    ///   - homepage/assets/img/
+    ///   - homepage/assets/
+    ///   - homepage/
+    fn add_all_sub_paths(&self, url_path: &str, links: &mut HashSet<String>) -> Result<()> {
+        log::trace!("enter: add_all_sub_paths({url_path}, {links:?})");
+
+        for sub_path in self.get_sub_paths_from_path(url_path) {
+            self.add_link_to_set_of_links(&sub_path, links)?;
+        }
+
+        log::trace!("exit: add_all_sub_paths");
+        Ok(())
+    }
+
+    /// given a url path, trim whitespace, remove slashes, and queries/fragments; return the
+    /// normalized string
+    pub(super) fn normalize_url_path(&self, path: &str) -> String {
+        log::trace!("enter: normalize_url_path({path})");
+
+        // remove whitespace and leading '/'
+        let path_str: String = path
+            .trim()
+            .trim_start_matches('/')
+            .chars()
+            .filter(|char| !char.is_whitespace())
+            .collect();
+
+        // snippets from rfc-3986:
+        //
+        //          foo://example.com:8042/over/there?name=ferret#nose
+        //          \_/   \______________/\_________/ \_________/ \__/
+        //           |           |            |            |        |
+        //        scheme     authority       path        query   fragment
+        //
+        // The path component is terminated
+        //    by the first question mark ("?") or number sign ("#") character, or
+        //    by the end of the URI.
+        //
+        // The query component is indicated by the first question
+        //    mark ("?") character and terminated by a number sign ("#") character
+        //    or by the end of the URI.
+        let (path_str, _discarded) = path_str
+            .split_once('?')
+            // if there isn't a '?', try to remove a fragment
+            .unwrap_or_else(|| {
+                // if there isn't a '#', return (original, empty)
+                path_str.split_once('#').unwrap_or((&path_str, ""))
+            });
+
+        log::trace!("exit: normalize_url_path -> {path_str}");
+        path_str.into()
+    }
+
+    /// Iterate over a given path, return a list of every sub-path found
+    ///
+    /// example: `path` contains a link fragment `homepage/assets/img/icons/handshake.svg`
+    /// the following fragments would be returned:
+    ///   - homepage/assets/img/icons/handshake.svg
+    ///   - homepage/assets/img/icons/
+    ///   - homepage/assets/img/
+    ///   - homepage/assets/
+    ///   - homepage/
+    pub(super) fn get_sub_paths_from_path(&self, path: &str) -> Vec<String> {
+        log::trace!("enter: get_sub_paths_from_path({path})");
+        let mut paths = vec![];
+
+        let normalized_path = self.normalize_url_path(path);
+
+        // filter out any empty strings caused by .split
+        let mut parts: Vec<Cow<_>> = normalized_path
+            .split('/')
+            .map(|s| self.url_regex.replace_all(s, ""))
+            .filter(|s| !s.is_empty())
+            .collect();
+
+        let length = parts.len();
+
+        for i in 0..length {
+            // iterate over all parts of the path
+            if parts.is_empty() {
+                // pop left us with an empty vector, we're done
+                break;
+            }
+
+            let mut possible_path = parts.join("/");
+
+            if possible_path.is_empty() {
+                // .join can result in an empty string, which we don't need, ignore
+                continue;
+            }
+
+            if i > 0 {
+                // this isn't the last index of the parts array
+                // ex: /buried/misc/stupidfile.php
+                // this block skips the file but sees all parent folders
+                possible_path = format!("{possible_path}/");
+            }
+
+            paths.push(possible_path); // good sub-path found
+            parts.pop(); // use .pop() to remove the last part of the path and continue iteration
+        }
+
+        log::trace!("exit: get_sub_paths_from_path -> {paths:?}");
+        paths
+    }
+
+    /// simple helper to stay DRY, tries to join a url + fragment and add it to the `links` HashSet
+    pub(super) fn add_link_to_set_of_links(
+        &self,
+        link: &str,
+        links: &mut HashSet<String>,
+    ) -> Result<()> {
+        log::trace!("enter: add_link_to_set_of_links({link}, {links:?})");
+
+        let old_url = match self.target {
+            ExtractionTarget::ResponseBody | ExtractionTarget::DirectoryListing => {
+                self.response.unwrap().url().clone()
+            }
+            ExtractionTarget::RobotsTxt => match parse_url_with_raw_path(&self.url) {
+                Ok(u) => u,
+                Err(e) => {
+                    bail!("Could not parse {}: {}", self.url, e);
+                }
+            },
+        };
+
+        let new_url = old_url
+            .join(link)
+            .with_context(|| format!("Could not join {old_url} with {link}"))?;
+
+        if !new_url.is_in_scope(&self.handles.config.scope) {
+            // URL is not in scope based on domain/scope configuration
+            log::debug!("Skipping {new_url} because it's not in scope");
+            log::trace!("exit: add_link_to_set_of_links");
+            return Ok(());
+        }
+
+        links.insert(new_url.to_string());
+
+        log::trace!("exit: add_link_to_set_of_links");
+
+        Ok(())
+    }
+
+    /// Entry point to perform link extraction from robots.txt
+    ///
+    /// `base_url` can have paths and subpaths, however robots.txt will be requested from the
+    /// root of the url
+    /// given the url:
+    ///     http://localhost/stuff/things
+    /// this function requests:
+    ///     http://localhost/robots.txt
+    pub(super) async fn extract_from_robots(&self) -> Result<ExtractionResult> {
+        log::trace!("enter: extract_robots_txt");
+
+        let mut result: HashSet<_> = ExtractionResult::new();
+
+        // request
+        let response = self.make_extract_request("/robots.txt").await?;
+        let body = response.text();
+
+        for capture in self.robots_regex.captures_iter(body) {
+            if let Some(new_path) = capture.name("url_path") {
+                let mut new_url = parse_url_with_raw_path(&self.url)?;
+
+                new_url.set_path(new_path.as_str());
+
+                if self.add_all_sub_paths(new_url.path(), &mut result).is_err() {
+                    log::warn!("could not add sub-paths from {new_url} to {result:?}");
+                }
+            }
+        }
+
+        log::trace!("exit: extract_robots_txt -> {result:?}");
+        Ok(result)
+    }
+
+    /// outer-most wrapper for parsing html response bodies in search of additional content.
+    /// performs the following high-level steps:
+    /// - requests the page, if necessary
+    /// - checks the page to see if directory listing is enabled and sucks up all the links, if so
+    /// - uses the linkfinder regex to grab links from embedded javascript/javascript files
+    /// - extracts many different types of link sources from the html itself
+    pub(super) async fn extract_from_body(&self) -> Result<ExtractionResult> {
+        log::trace!("enter: extract_from_body");
+
+        let mut result = ExtractionResult::new();
+
+        let response = self.response.unwrap();
+        let resp_url = response.url();
+        let body = response.text();
+        let html = Html::parse_document(body);
+
+        // extract links from html tags/attributes and embedded javascript
+        self.extract_all_links_from_html_tags(resp_url, &mut result, &html);
+        self.extract_all_links_from_javascript(body, resp_url, &mut result);
+
+        log::trace!("exit: extract_from_body -> {result:?}");
+        Ok(result)
+    }
+
+    /// parses html response bodies in search of <a> tags.
+    ///
+    /// the assumption is that directory listing is turned on and this extraction target simply
+    /// scoops up all the links for the given directory. The test to detect a directory listing
+    /// is located in `HeuristicTests`
+    pub async fn extract_from_dir_listing(&self) -> Result<ExtractionResult> {
+        log::trace!("enter: extract_from_dir_listing");
+
+        let mut result = ExtractionResult::new();
+
+        let response = self.response.unwrap();
+        let html = Html::parse_document(response.text());
+
+        self.extract_links_by_attr(response.url(), &mut result, &html, "a", "href");
+
+        log::trace!("exit: extract_from_dir_listing -> {result:?}");
+        Ok(result)
+    }
+
+    /// simple helper to get html links by tag/attribute and add it to the `links` HashSet
+    fn extract_links_by_attr(
+        &self,
+        resp_url: &Url,
+        links: &mut HashSet<String>,
+        html: &Html,
+        html_tag: &str,
+        html_attr: &str,
+    ) {
+        log::trace!("enter: extract_links_by_attr");
+
+        let Some(selector) = Selector::parse(html_tag).ok() else {
+            log::warn!("Failed to parse selector for tag: {html_tag}");
+            return;
+        };
+
+        let tags = html
+            .select(&selector)
+            .filter(|a| a.value().attrs().any(|attr| attr.0 == html_attr));
+
+        for tag in tags {
+            if let Some(link) = tag.value().attr(html_attr) {
+                log::debug!("Parsed link \"{}\" from {}", link, resp_url.as_str());
+
+                if self.parse_url_and_add_subpaths(link, links).is_err() {
+                    log::debug!("link didn't belong to the target domain/host: {link}");
+                }
+            }
+        }
+
+        log::trace!("exit: extract_links_by_attr");
+    }
+
+    /// helper function that simply requests at <location> on the given url's base url
+    ///
+    /// example:
+    ///     http://localhost/api/users -> http://localhost/<location>
+    pub(super) async fn make_extract_request(&self, location: &str) -> Result<FeroxResponse> {
+        log::trace!("enter: make_extract_request");
+
+        // need late binding here to avoid 'creates a temporary which is freed...' in the
+        // `let ... if` below to avoid cloning the client out of config
+        let mut client = Client::new();
+
+        if location == "/robots.txt" {
+            // more often than not, domain/robots.txt will redirect to www.domain/robots.txt or something
+            // similar; to account for that, create a client that will follow redirects, regardless of
+            // what the user specified for the scanning client. Other than redirects, it will respect
+            // all other user specified settings
+            let follow_redirects = true;
+
+            let proxy = if self.handles.config.proxy.is_empty() {
+                None
+            } else {
+                Some(self.handles.config.proxy.as_str())
+            };
+
+            let server_certs = &self.handles.config.server_certs;
+
+            let client_cert = if self.handles.config.client_cert.is_empty() {
+                None
+            } else {
+                Some(self.handles.config.client_cert.as_str())
+            };
+
+            let client_key = if self.handles.config.client_key.is_empty() {
+                None
+            } else {
+                Some(self.handles.config.client_key.as_str())
+            };
+
+            let client_config = client::ClientConfig {
+                timeout: self.handles.config.timeout,
+                user_agent: &self.handles.config.user_agent,
+                redirects: follow_redirects,
+                insecure: self.handles.config.insecure,
+                headers: &self.handles.config.headers,
+                proxy,
+                server_certs: Some(server_certs),
+                client_cert,
+                client_key,
+                scope: &self.handles.config.scope,
+            };
+            client = client::initialize(client_config)?;
+        }
+
+        let client = if location != "/robots.txt" {
+            &self.handles.config.client
+        } else {
+            &client
+        };
+
+        let mut url = parse_url_with_raw_path(&self.url)?;
+        url.set_path(location); // overwrite existing path
+
+        // purposefully not using logged_request here due to using the special client
+        let response = make_request(
+            client,
+            &url,
+            DEFAULT_METHOD,
+            None,
+            self.handles.config.output_level,
+            &self.handles.config,
+            self.handles.stats.tx.clone(),
+        )
+        .await?;
+
+        let ferox_response = FeroxResponse::from(
+            response,
+            &self.url,
+            DEFAULT_METHOD,
+            self.handles.config.output_level,
+            self.handles.config.response_size_limit,
+        )
+        .await;
+        // note: don't call parse_extension here. If we call it here, it gets called on robots.txt
+
+        log::trace!("exit: make_extract_request -> {ferox_response}");
+        Ok(ferox_response)
+    }
+
+    /// update total number of links extracted and expected responses
+    fn update_stats(&self, num_links: usize) -> Result<()> {
+        let multiplier = self.handles.expected_num_requests_multiplier();
+
+        self.handles
+            .stats
+            .send(AddToUsizeField(LinksExtracted, num_links))?;
+        self.handles
+            .stats
+            .send(AddToUsizeField(TotalExpected, num_links * multiplier))?;
+
+        Ok(())
+    }
+}

src/extractor/mod.rs

@@ -0,0 +1,13 @@
+//! extract links from html source and robots.txt
+mod builder;
+mod container;
+#[cfg(test)]
+mod tests;
+
+pub use self::builder::ExtractionTarget;
+pub use self::builder::ExtractorBuilder;
+pub use self::container::Extractor;
+
+use crate::response::FeroxResponse;
+use regex::Regex;
+use std::sync::Arc;

src/extractor/tests.rs

@@ -0,0 +1,416 @@
+use super::builder::{LINKFINDER_REGEX, ROBOTS_TXT_REGEX, URL_CHARS_REGEX};
+use super::container::request_link;
+use super::*;
+use crate::config::{Configuration, OutputLevel};
+use crate::scan_manager::ScanOrder;
+use crate::{
+    event_handlers::Handles, scan_manager::FeroxScans, utils::make_request, Command, FeroxChannel,
+    DEFAULT_METHOD,
+};
+use anyhow::Result;
+use httpmock::{Method::GET, MockServer};
+use lazy_static::lazy_static;
+use reqwest::{Client, StatusCode, Url};
+use std::collections::HashSet;
+use tokio::sync::mpsc;
+
+lazy_static! {
+    /// Extractor for testing robots.txt
+    static ref ROBOTS_EXT: Extractor<'static> = setup_extractor(ExtractionTarget::RobotsTxt, Arc::new(FeroxScans::default()));
+
+    /// Extractor for testing response bodies
+    static ref BODY_EXT: Extractor<'static> = setup_extractor(ExtractionTarget::ResponseBody, Arc::new(FeroxScans::default()));
+
+    /// Extractor for testing paring html
+    static ref PARSEHTML_EXT: Extractor<'static> = setup_extractor(ExtractionTarget::DirectoryListing, Arc::new(FeroxScans::default()));
+
+    /// FeroxResponse for Extractor
+    static ref RESPONSE: FeroxResponse = get_test_response();
+}
+
+/// constructor for the default FeroxResponse used during testing
+fn get_test_response() -> FeroxResponse {
+    let mut resp = FeroxResponse::default();
+    resp.set_text("nulla pharetra diam sit amet nisl suscipit adipiscing bibendum est");
+    resp
+}
+
+/// creates a single extractor that can be used to test standalone functions
+fn setup_extractor(target: ExtractionTarget, scanned_urls: Arc<FeroxScans>) -> Extractor<'static> {
+    let mut builder = ExtractorBuilder::default();
+
+    let builder = match target {
+        ExtractionTarget::ResponseBody => builder
+            .target(ExtractionTarget::ResponseBody)
+            .response(&RESPONSE),
+        ExtractionTarget::RobotsTxt => builder
+            .url("http://localhost")
+            .target(ExtractionTarget::RobotsTxt),
+        ExtractionTarget::DirectoryListing => builder
+            .url("http://localhost")
+            .target(ExtractionTarget::DirectoryListing),
+    };
+
+    // need to add scope to the config to allow extracted links to make it through the
+    // full pipeline
+    let mut config = Configuration::new().unwrap();
+    config.scope.push(Url::parse("http://localhost").unwrap());
+
+    let config = Arc::new(config);
+    let handles = Arc::new(Handles::for_testing(Some(scanned_urls), Some(config)).0);
+
+    builder.handles(handles).build().unwrap()
+}
+
+#[test]
+/// extract sub paths from the given url fragment; expect 4 sub paths and that all are
+/// in the expected array
+fn extractor_get_sub_paths_from_path_with_multiple_paths() {
+    let path = "homepage/assets/img/icons/handshake.svg";
+    let r_paths = ROBOTS_EXT.get_sub_paths_from_path(path);
+    let b_paths = BODY_EXT.get_sub_paths_from_path(path);
+    let expected = vec![
+        "homepage/",
+        "homepage/assets/",
+        "homepage/assets/img/",
+        "homepage/assets/img/icons/",
+        "homepage/assets/img/icons/handshake.svg",
+    ];
+
+    assert_eq!(r_paths.len(), expected.len());
+    assert_eq!(b_paths.len(), expected.len());
+    for expected_path in expected {
+        assert!(r_paths.contains(&expected_path.to_string()));
+        assert!(b_paths.contains(&expected_path.to_string()));
+    }
+}
+
+#[test]
+/// extract sub paths from the given url fragment; expect 2 sub paths and that all are
+/// in the expected array. the fragment is wrapped in slashes to ensure no empty strings are
+/// returned
+fn extractor_get_sub_paths_from_path_with_enclosing_slashes() {
+    let path = "/homepage/assets/";
+    let r_paths = ROBOTS_EXT.get_sub_paths_from_path(path);
+    let b_paths = BODY_EXT.get_sub_paths_from_path(path);
+    let expected = vec!["homepage/", "homepage/assets"];
+
+    assert_eq!(r_paths.len(), expected.len());
+    assert_eq!(b_paths.len(), expected.len());
+    for expected_path in expected {
+        assert!(r_paths.contains(&expected_path.to_string()));
+        assert!(b_paths.contains(&expected_path.to_string()));
+    }
+}
+
+#[test]
+/// extract sub paths from the given url fragment; expect 1 sub path, no forward slashes are
+/// included
+fn extractor_get_sub_paths_from_path_with_only_a_word() {
+    let path = "homepage";
+    let r_paths = ROBOTS_EXT.get_sub_paths_from_path(path);
+    let b_paths = BODY_EXT.get_sub_paths_from_path(path);
+    let expected = vec!["homepage"];
+
+    assert_eq!(r_paths.len(), expected.len());
+    assert_eq!(b_paths.len(), expected.len());
+    for expected_path in expected {
+        assert!(r_paths.contains(&expected_path.to_string()));
+        assert!(b_paths.contains(&expected_path.to_string()));
+    }
+}
+
+#[test]
+/// extract sub paths from the given url fragment; expect 1 sub path, forward slash removed
+fn extractor_get_sub_paths_from_path_with_an_absolute_word() {
+    let path = "/homepage";
+    let r_paths = ROBOTS_EXT.get_sub_paths_from_path(path);
+    let b_paths = BODY_EXT.get_sub_paths_from_path(path);
+    let expected = vec!["homepage"];
+
+    assert_eq!(r_paths.len(), expected.len());
+    assert_eq!(b_paths.len(), expected.len());
+    for expected_path in expected {
+        assert!(r_paths.contains(&expected_path.to_string()));
+        assert!(b_paths.contains(&expected_path.to_string()));
+    }
+}
+
+#[test]
+/// test that an ExtractorBuilder without a FeroxResponse and without a URL bails
+fn extractor_builder_bails_when_neither_required_field_is_set() {
+    let handles = Arc::new(Handles::for_testing(None, None).0);
+
+    let extractor = ExtractorBuilder::default()
+        .url("")
+        .target(ExtractionTarget::RobotsTxt)
+        .handles(handles)
+        .build();
+
+    assert!(extractor.is_err());
+}
+
+#[test]
+/// Extractor with a non-base url bails
+fn extractor_with_non_base_url_bails() -> Result<()> {
+    let mut links = HashSet::<String>::new();
+    let link = "admin";
+    let handles = Arc::new(Handles::for_testing(None, None).0);
+
+    let extractor = ExtractorBuilder::default()
+        .url("\\\\\\")
+        .handles(handles)
+        .target(ExtractionTarget::RobotsTxt)
+        .build()?;
+
+    let result = extractor.add_link_to_set_of_links(link, &mut links);
+
+    assert!(result.is_err());
+    Ok(())
+}
+
+#[test]
+/// test that a full url and fragment are joined correctly, then added to the given list
+/// i.e. the happy path
+fn extractor_add_link_to_set_of_links_happy_path() {
+    let mut r_links = HashSet::<String>::new();
+    let r_link = "admin";
+    let mut b_links = HashSet::<String>::new();
+    let b_link = "shmadmin";
+
+    assert_eq!(r_links.len(), 0);
+    ROBOTS_EXT
+        .add_link_to_set_of_links(r_link, &mut r_links)
+        .unwrap();
+
+    assert_eq!(r_links.len(), 1);
+    assert!(r_links.contains("http://localhost/admin"));
+
+    assert_eq!(b_links.len(), 0);
+
+    BODY_EXT
+        .add_link_to_set_of_links(b_link, &mut b_links)
+        .unwrap();
+
+    assert_eq!(b_links.len(), 1);
+    assert!(b_links.contains("http://localhost/shmadmin"));
+}
+
+#[test]
+/// test that an invalid path fragment doesn't add anything to the set of links
+fn extractor_add_link_to_set_of_links_with_non_base_url() {
+    let mut links = HashSet::<String>::new();
+    let link = "\\\\\\\\";
+    assert_eq!(links.len(), 0);
+    assert!(ROBOTS_EXT
+        .add_link_to_set_of_links(link, &mut links)
+        .is_err());
+    assert!(BODY_EXT.add_link_to_set_of_links(link, &mut links).is_err());
+
+    assert_eq!(links.len(), 0);
+    assert!(links.is_empty());
+}
+
+#[test]
+/// test for filtering queries and fragments
+fn normalize_url_path_filters_queries_and_fragments() {
+    let handles = Arc::new(Handles::for_testing(None, None).0);
+    let extractor = ExtractorBuilder::default()
+        .url("doesnt matter")
+        .target(ExtractionTarget::RobotsTxt)
+        .handles(handles)
+        .build()
+        .unwrap();
+
+    let test_strings = [
+        "over/there?name=ferret#nose",
+        "over/there?name=ferret",
+        "over/there#nose",
+        "over/there",
+        "over/there?name#nose",
+        "over/there?name",
+        "   over/there?name=ferret#nose  ",
+        "over/there?name=ferret   ",
+        "   over/there#nose",
+    ];
+    test_strings.iter().for_each(|&ts| {
+        let normed = extractor.normalize_url_path(ts);
+        assert_eq!(normed, "over/there");
+    });
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// use make_request to generate a Response, and use the Response to test get_links;
+/// the response will contain an absolute path to a domain that is not part of the scanned
+/// domain; expect an empty set returned
+async fn extractor_get_links_with_absolute_url_that_differs_from_target_domain() -> Result<()> {
+    let (tx_stats, _): FeroxChannel<Command> = mpsc::unbounded_channel();
+
+    let srv = MockServer::start();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/some-path");
+        then.status(200).body(
+            "\"http://definitely.not.a.thing.probably.com/homepage/assets/img/icons/handshake.svg\"",
+        );
+    });
+
+    let client = Client::new();
+    let url = Url::parse(&srv.url("/some-path")).unwrap();
+    let config = Configuration::new().unwrap();
+
+    let response = make_request(
+        &client,
+        &url,
+        DEFAULT_METHOD,
+        None,
+        OutputLevel::Default,
+        &config,
+        tx_stats.clone(),
+    )
+    .await
+    .unwrap();
+    let (handles, _rx) = Handles::for_testing(None, None);
+
+    let handles = Arc::new(handles);
+    let ferox_response = FeroxResponse::from(
+        response,
+        &srv.url(""),
+        DEFAULT_METHOD,
+        OutputLevel::Default,
+        4194304,
+    )
+    .await;
+
+    let extractor = Extractor {
+        links_regex: Regex::new(LINKFINDER_REGEX).unwrap(),
+        robots_regex: Regex::new(ROBOTS_TXT_REGEX).unwrap(),
+        url_regex: Regex::new(URL_CHARS_REGEX).unwrap(),
+        response: Some(&ferox_response),
+        url: String::new(),
+        target: ExtractionTarget::ResponseBody,
+        handles: handles.clone(),
+    };
+
+    let links = extractor.extract_from_body().await?;
+
+    assert!(links.is_empty());
+    assert_eq!(mock.hits(), 1);
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test that /robots.txt is correctly requested given a base url (happy path)
+async fn request_robots_txt_without_proxy() -> Result<()> {
+    let handles = Arc::new(Handles::for_testing(None, None).0);
+
+    let srv = MockServer::start();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/robots.txt");
+        then.status(200).body("this is a test");
+    });
+
+    let extractor = Extractor {
+        links_regex: Regex::new(LINKFINDER_REGEX).unwrap(),
+        robots_regex: Regex::new(ROBOTS_TXT_REGEX).unwrap(),
+        url_regex: Regex::new(URL_CHARS_REGEX).unwrap(),
+        response: None,
+        url: srv.url("/api/users/stuff/things"),
+        target: ExtractionTarget::RobotsTxt,
+        handles,
+    };
+
+    let resp = extractor.make_extract_request("/robots.txt").await?;
+
+    assert!(matches!(resp.status(), &StatusCode::OK));
+    println!("{resp}");
+    assert_eq!(resp.content_length(), 14);
+    assert_eq!(mock.hits(), 1);
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// test that /robots.txt is correctly requested given a base url (happy path) when a proxy is used
+async fn request_robots_txt_with_proxy() -> Result<()> {
+    let handles = Arc::new(Handles::for_testing(None, None).0);
+    let mut config = Configuration::new()?;
+
+    let srv = MockServer::start();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/robots.txt");
+        then.status(200).body("this is also a test");
+    });
+
+    // note: the proxy doesn't actually do anything other than hit a different code branch
+    // in this unit test; it would however have an effect on an integration test
+    config.proxy = srv.url("/ima-proxy");
+    config.no_recursion = true;
+
+    let extractor = ExtractorBuilder::default()
+        .url(&srv.url("/api/different/path"))
+        .target(ExtractionTarget::RobotsTxt)
+        .handles(handles)
+        .build()?;
+
+    let resp = extractor.make_extract_request("/robots.txt").await?;
+
+    assert!(matches!(resp.status(), &StatusCode::OK));
+    assert_eq!(resp.content_length(), 19);
+    assert_eq!(mock.hits(), 1);
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// request_link's happy path, expect back a FeroxResponse
+async fn request_link_happy_path() -> Result<()> {
+    let srv = MockServer::start();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/login.php");
+        then.status(200).body("this is a test");
+    });
+
+    let r_resp = request_link(&srv.url("/login.php"), ROBOTS_EXT.handles.clone()).await?;
+    let b_resp = request_link(&srv.url("/login.php"), BODY_EXT.handles.clone()).await?;
+
+    assert!(matches!(r_resp.status(), StatusCode::OK));
+    assert!(matches!(b_resp.status(), StatusCode::OK));
+    assert_eq!(r_resp.content_length().unwrap(), 14);
+    assert_eq!(b_resp.content_length().unwrap(), 14);
+    assert_eq!(mock.hits(), 2);
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+/// request_link should bail in the event that the url is already in scanned_urls
+async fn request_link_bails_on_seen_url() -> Result<()> {
+    let url = "/unique-for-this-test.php";
+    let srv = MockServer::start();
+    let served = srv.url(url);
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path(url);
+        then.status(200)
+            .body("this is a unique test, don't reuse the endpoint");
+    });
+
+    let scans = Arc::new(FeroxScans::default());
+    scans.add_file_scan(
+        &served,
+        ScanOrder::Latest,
+        Arc::new(Handles::for_testing(None, None).0),
+    );
+
+    let robots = setup_extractor(ExtractionTarget::RobotsTxt, scans.clone());
+    let body = setup_extractor(ExtractionTarget::ResponseBody, scans);
+
+    let r_resp = request_link(&served, robots.handles.clone()).await;
+    let b_resp = request_link(&served, body.handles.clone()).await;
+
+    assert!(r_resp.is_err());
+    assert!(b_resp.is_err());
+    assert_eq!(mock.hits(), 0); // function exits before requests can happen
+    Ok(())
+}

src/filters/container.rs

@@ -0,0 +1,131 @@
+use std::sync::RwLock;
+
+use anyhow::Result;
+use serde::{ser::SerializeSeq, Serialize, Serializer};
+
+use crate::response::FeroxResponse;
+
+use super::{
+    FeroxFilter, LinesFilter, RegexFilter, SimilarityFilter, SizeFilter, StatusCodeFilter,
+    WildcardFilter, WordsFilter,
+};
+use crate::{
+    event_handlers::Command::AddToUsizeField, statistics::StatField::WildcardsFiltered,
+    CommandSender,
+};
+/// Container around a collection of `FeroxFilters`s
+#[derive(Debug, Default)]
+pub struct FeroxFilters {
+    /// collection of `FeroxFilters`
+    pub filters: RwLock<Vec<Box<dyn FeroxFilter>>>,
+}
+
+/// implementation of FeroxFilter collection
+impl FeroxFilters {
+    /// add a single FeroxFilter to the collection
+    pub fn push(&self, filter: Box<dyn FeroxFilter>) -> Result<()> {
+        if let Ok(mut guard) = self.filters.write() {
+            if guard.contains(&filter) {
+                return Ok(());
+            }
+
+            guard.push(filter)
+        }
+        Ok(())
+    }
+
+    /// remove items from the underlying collection by their index
+    ///
+    /// note: indexes passed in should be index-to-remove+1. This is built for the scan mgt menu
+    ///       so indexes aren't 0-based whehn the user enters them.
+    ///       
+    pub fn remove(&self, indices: &mut [usize]) {
+        // since we're removing by index, indices must be sorted and then reversed.
+        // this allows us to iterate over the collection from the rear, allowing any shifting
+        // of the vector to happen on sections that we no longer care about, as we're moving
+        // in the opposite direction
+        indices.sort_unstable();
+        indices.reverse();
+
+        if let Ok(mut guard) = self.filters.write() {
+            for index in indices {
+                // numbering of the menu starts at 1, so we'll need to reduce the index by 1
+                // to account for that. if they've provided 0 as an offset, we'll set the
+                // result to a gigantic number and skip it in the loop with a bounds check
+                let reduced_idx = index.checked_sub(1).unwrap_or(usize::MAX);
+
+                // check if number provided is out of range
+                if reduced_idx >= guard.len() {
+                    // usize can't be negative, just need to handle exceeding bounds
+                    continue;
+                }
+
+                guard.remove(reduced_idx);
+            }
+        }
+    }
+
+    /// Simple helper to stay DRY; determines whether or not a given `FeroxResponse` should be reported
+    /// to the user or not.
+    pub fn should_filter_response(
+        &self,
+        response: &FeroxResponse,
+        tx_stats: CommandSender,
+    ) -> bool {
+        if let Ok(filters) = self.filters.read() {
+            for filter in filters.iter() {
+                // wildcard.should_filter goes here
+                if filter.should_filter_response(response) {
+                    log::debug!("filtering response due to: {filter:?}");
+                    if filter.as_any().downcast_ref::<WildcardFilter>().is_some() {
+                        tx_stats
+                            .send(AddToUsizeField(WildcardsFiltered, 1))
+                            .unwrap_or_default();
+                    }
+                    return true;
+                }
+            }
+        }
+        false
+    }
+}
+
+impl Serialize for FeroxFilters {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        if let Ok(guard) = self.filters.read() {
+            let mut seq = serializer.serialize_seq(Some(guard.len()))?;
+
+            for filter in &*guard {
+                if let Some(line_filter) = filter.as_any().downcast_ref::<LinesFilter>() {
+                    seq.serialize_element(line_filter).unwrap_or_default();
+                } else if let Some(word_filter) = filter.as_any().downcast_ref::<WordsFilter>() {
+                    seq.serialize_element(word_filter).unwrap_or_default();
+                } else if let Some(size_filter) = filter.as_any().downcast_ref::<SizeFilter>() {
+                    seq.serialize_element(size_filter).unwrap_or_default();
+                } else if let Some(wildcard_filter) =
+                    filter.as_any().downcast_ref::<WildcardFilter>()
+                {
+                    seq.serialize_element(wildcard_filter).unwrap_or_default();
+                } else if let Some(status_filter) =
+                    filter.as_any().downcast_ref::<StatusCodeFilter>()
+                {
+                    seq.serialize_element(status_filter).unwrap_or_default();
+                } else if let Some(regex_filter) = filter.as_any().downcast_ref::<RegexFilter>() {
+                    seq.serialize_element(regex_filter).unwrap_or_default();
+                } else if let Some(similarity_filter) =
+                    filter.as_any().downcast_ref::<SimilarityFilter>()
+                {
+                    seq.serialize_element(similarity_filter).unwrap_or_default();
+                }
+            }
+            seq.end()
+        } else {
+            // if for some reason we can't unlock the mutex, just write an empty list
+            let seq = serializer.serialize_seq(Some(0))?;
+            seq.end()
+        }
+    }
+}

src/filters/empty.rs

@@ -0,0 +1,22 @@
+use super::*;
+
+/// Dummy filter for internal shenanigans
+#[derive(Default, Debug, PartialEq, Eq)]
+pub struct EmptyFilter {}
+
+impl FeroxFilter for EmptyFilter {
+    /// `EmptyFilter` always returns false
+    fn should_filter_response(&self, _response: &FeroxResponse) -> bool {
+        false
+    }
+
+    /// Compare one EmptyFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>() == Some(self)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/filters/init.rs

@@ -0,0 +1,71 @@
+use super::{
+    utils::create_similarity_filter, LinesFilter, RegexFilter, SizeFilter, StatusCodeFilter,
+    WordsFilter,
+};
+use crate::{event_handlers::Handles, skip_fail, utils::fmt_err, Command::AddFilter};
+use anyhow::Result;
+use regex::Regex;
+use std::sync::Arc;
+
+/// add all user-supplied filters to the (already started) filters handler
+pub async fn initialize(handles: Arc<Handles>) -> Result<()> {
+    // add any status code filters to filters handler's FeroxFilters  (-C|--filter-status)
+    for code_filter in &handles.config.filter_status {
+        let filter = StatusCodeFilter {
+            filter_code: *code_filter,
+        };
+        let boxed_filter = Box::new(filter);
+        skip_fail!(handles.filters.send(AddFilter(boxed_filter)));
+    }
+
+    // add any line count filters to filters handler's FeroxFilters  (-N|--filter-lines)
+    for lines_filter in &handles.config.filter_line_count {
+        let filter = LinesFilter {
+            line_count: *lines_filter,
+        };
+        let boxed_filter = Box::new(filter);
+        skip_fail!(handles.filters.send(AddFilter(boxed_filter)));
+    }
+
+    // add any line count filters to filters handler's FeroxFilters  (-W|--filter-words)
+    for words_filter in &handles.config.filter_word_count {
+        let filter = WordsFilter {
+            word_count: *words_filter,
+        };
+        let boxed_filter = Box::new(filter);
+        skip_fail!(handles.filters.send(AddFilter(boxed_filter)));
+    }
+
+    // add any line count filters to filters handler's FeroxFilters  (-S|--filter-size)
+    for size_filter in &handles.config.filter_size {
+        let filter = SizeFilter {
+            content_length: *size_filter,
+        };
+        let boxed_filter = Box::new(filter);
+        skip_fail!(handles.filters.send(AddFilter(boxed_filter)));
+    }
+
+    // add any regex filters to filters handler's FeroxFilters  (-X|--filter-regex)
+    for regex_filter in &handles.config.filter_regex {
+        let raw = regex_filter;
+        let compiled = skip_fail!(Regex::new(raw));
+
+        let filter = RegexFilter {
+            raw_string: raw.to_owned(),
+            compiled,
+        };
+        let boxed_filter = Box::new(filter);
+        skip_fail!(handles.filters.send(AddFilter(boxed_filter)));
+    }
+
+    // add any similarity filters to filters handler's FeroxFilters  (--filter-similar-to)
+    for similarity_filter in &handles.config.filter_similar {
+        let filter = skip_fail!(create_similarity_filter(similarity_filter, handles.clone()).await);
+
+        let boxed_filter = Box::new(filter);
+        skip_fail!(handles.filters.send(AddFilter(boxed_filter)));
+    }
+
+    handles.filters.sync().await?;
+    Ok(())
+}

src/filters/lines.rs

@@ -0,0 +1,33 @@
+use super::*;
+
+/// Simple implementor of FeroxFilter; used to filter out responses based on the number of lines
+/// in a Response body; specified using -N|--filter-lines
+#[derive(Default, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct LinesFilter {
+    /// Number of lines in a Response's body that should be filtered
+    pub line_count: usize,
+}
+
+/// implementation of FeroxFilter for LinesFilter
+impl FeroxFilter for LinesFilter {
+    /// Check `line_count` against what was passed in via -N|--filter-lines
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({self:?} {response})");
+
+        let result = response.line_count() == self.line_count;
+
+        log::trace!("exit: should_filter_response -> {result}");
+
+        result
+    }
+
+    /// Compare one LinesFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>() == Some(self)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/filters/mod.rs

@@ -0,0 +1,33 @@
+//! contains all of feroxbuster's filters
+use serde::{Deserialize, Serialize};
+use std::any::Any;
+use std::fmt::Debug;
+
+use crate::response::FeroxResponse;
+use crate::traits::FeroxFilter;
+
+pub use self::container::FeroxFilters;
+pub(crate) use self::empty::EmptyFilter;
+pub use self::init::initialize;
+pub use self::lines::LinesFilter;
+pub use self::regex::RegexFilter;
+pub use self::similarity::{SimilarityFilter, SIM_HASHER};
+pub use self::size::SizeFilter;
+pub use self::status_code::StatusCodeFilter;
+pub(crate) use self::utils::{create_similarity_filter, filter_lookup};
+pub use self::wildcard::WildcardFilter;
+pub use self::words::WordsFilter;
+
+mod status_code;
+mod words;
+mod lines;
+mod size;
+mod regex;
+mod similarity;
+mod container;
+#[cfg(test)]
+mod tests;
+mod init;
+mod utils;
+mod wildcard;
+mod empty;

src/filters/regex.rs

@@ -0,0 +1,60 @@
+use super::*;
+use ::regex::Regex;
+
+/// Simple implementor of FeroxFilter; used to filter out responses based on a given regular
+/// expression; specified using -X|--filter-regex
+#[derive(Debug, Serialize, Deserialize)]
+pub struct RegexFilter {
+    /// Regular expression to be applied to the response body for filtering, compiled
+    #[serde(with = "serde_regex")]
+    pub compiled: Regex,
+
+    /// Regular expression as passed in on the command line, not compiled
+    pub raw_string: String,
+}
+
+impl Default for RegexFilter {
+    fn default() -> Self {
+        Self {
+            compiled: Regex::new("").unwrap(),
+            raw_string: String::new(),
+        }
+    }
+}
+
+/// implementation of FeroxFilter for RegexFilter
+impl FeroxFilter for RegexFilter {
+    /// Check `expression` against the response body, if the expression matches, the response
+    /// should be filtered out
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({self:?} {response})");
+
+        let result = self.compiled.is_match(response.text());
+        let other = response.headers().iter().any(|(k, v)| {
+            self.compiled.is_match(k.as_str()) || self.compiled.is_match(v.to_str().unwrap_or(""))
+        });
+
+        let final_result = result || other;
+        log::trace!("exit: should_filter_response -> {final_result}");
+
+        final_result
+    }
+
+    /// Compare one SizeFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>() == Some(self)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
+
+/// PartialEq implementation for RegexFilter
+impl PartialEq for RegexFilter {
+    /// Simple comparison of the raw string passed in via the command line
+    fn eq(&self, other: &RegexFilter) -> bool {
+        self.raw_string == other.raw_string
+    }
+}

src/filters/similarity.rs

@@ -0,0 +1,68 @@
+use super::*;
+use crate::nlp::preprocess;
+use crate::NEAR_DUPLICATE_DISTANCE;
+use gaoya::simhash::{SimHash, SimHashBits, SimSipHasher64};
+use lazy_static::lazy_static;
+
+lazy_static! {
+    /// single instance of the sip hasher used in similarity filtering
+    pub static ref SIM_HASHER: SimHash<SimSipHasher64, u64, 64> =
+        SimHash::<SimSipHasher64, u64, 64>::new(SimSipHasher64::new(1, 2));
+}
+
+/// Simple implementor of FeroxFilter; used to filter out responses based on the similarity of a
+/// Response body with a known response; specified using --filter-similar-to
+#[derive(Default, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct SimilarityFilter {
+    /// Hash of Response's body to be used during similarity comparison
+    pub hash: u64,
+
+    /// Url originally requested for the similarity filter
+    pub original_url: String,
+
+    /// Maximum hamming distance allowed between two signatures
+    pub cutoff: usize,
+}
+
+impl SimilarityFilter {
+    /// Create a new SimilarityFilter
+    pub fn new(hash: u64, original_url: String, cutoff: usize) -> Self {
+        Self {
+            hash,
+            original_url,
+            cutoff,
+        }
+    }
+}
+
+impl From<&FeroxResponse> for SimilarityFilter {
+    fn from(response: &FeroxResponse) -> Self {
+        Self::new(
+            SIM_HASHER.create_signature(preprocess(response.text()).iter()),
+            response.url().to_string(),
+            NEAR_DUPLICATE_DISTANCE,
+        )
+    }
+}
+
+/// implementation of FeroxFilter for SimilarityFilter
+impl FeroxFilter for SimilarityFilter {
+    /// Check `FeroxResponse::text` against what was requested from the site passed in via
+    /// --filter-similar-to
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        let other = SIM_HASHER.create_signature(preprocess(response.text()).iter());
+        self.hash.hamming_distance(&other) <= self.cutoff
+    }
+
+    /// Compare one SimilarityFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other
+            .downcast_ref::<Self>()
+            .is_some_and(|a| self.hash == a.hash)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/filters/size.rs

@@ -0,0 +1,33 @@
+use super::*;
+
+/// Simple implementor of FeroxFilter; used to filter out responses based on the length of a
+/// Response body; specified using -S|--filter-size
+#[derive(Default, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct SizeFilter {
+    /// Overall length of a Response's body that should be filtered
+    pub content_length: u64,
+}
+
+/// implementation of FeroxFilter for SizeFilter
+impl FeroxFilter for SizeFilter {
+    /// Check `content_length` against what was passed in via -S|--filter-size
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({self:?} {response})");
+
+        let result = response.content_length() == self.content_length;
+
+        log::trace!("exit: should_filter_response -> {result}");
+
+        result
+    }
+
+    /// Compare one SizeFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>() == Some(self)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/filters/status_code.rs

@@ -0,0 +1,40 @@
+use super::*;
+
+/// Simple implementor of FeroxFilter; used to filter out status codes specified using
+/// -C|--filter-status
+#[derive(Default, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct StatusCodeFilter {
+    /// Status code that should not be displayed to the user
+    pub filter_code: u16,
+}
+
+/// implementation of FeroxFilter for StatusCodeFilter
+impl FeroxFilter for StatusCodeFilter {
+    /// Check `filter_code` against what was passed in via -C|--filter-status
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({self:?} {response})");
+
+        if response.status().as_u16() == self.filter_code {
+            log::debug!(
+                "filtered out {} based on --filter-status of {}",
+                response.url(),
+                self.filter_code
+            );
+            log::trace!("exit: should_filter_response -> true");
+            return true;
+        }
+
+        log::trace!("exit: should_filter_response -> false");
+        false
+    }
+
+    /// Compare one StatusCodeFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>() == Some(self)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/filters/tests.rs

@@ -0,0 +1,290 @@
+use super::*;
+use crate::nlp::preprocess;
+use crate::DEFAULT_METHOD;
+use crate::NEAR_DUPLICATE_DISTANCE;
+use ::regex::Regex;
+
+#[test]
+/// simply test the default values for wildcardfilter
+fn wildcard_filter_default() {
+    let wcf = WildcardFilter::default();
+    assert_eq!(wcf.content_length, None);
+    assert_eq!(wcf.line_count, None);
+    assert_eq!(wcf.word_count, None);
+    assert_eq!(wcf.method, DEFAULT_METHOD.to_string());
+    assert_eq!(wcf.status_code, 0);
+    assert!(!wcf.dont_filter);
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn wildcard_filter_as_any() {
+    let mut filter = WildcardFilter::default();
+    let filter2 = WildcardFilter::default();
+
+    assert!(filter.box_eq(filter2.as_any()));
+
+    assert_eq!(
+        *filter.as_any().downcast_ref::<WildcardFilter>().unwrap(),
+        filter2
+    );
+
+    filter.content_length = Some(1);
+
+    assert_ne!(
+        *filter.as_any().downcast_ref::<WildcardFilter>().unwrap(),
+        filter2
+    );
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn lines_filter_as_any() {
+    let filter = LinesFilter { line_count: 1 };
+    let filter2 = LinesFilter { line_count: 1 };
+
+    assert!(filter.box_eq(filter2.as_any()));
+
+    assert_eq!(filter.line_count, 1);
+    assert_eq!(
+        *filter.as_any().downcast_ref::<LinesFilter>().unwrap(),
+        filter
+    );
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn words_filter_as_any() {
+    let filter = WordsFilter { word_count: 1 };
+    let filter2 = WordsFilter { word_count: 1 };
+
+    assert!(filter.box_eq(filter2.as_any()));
+
+    assert_eq!(filter.word_count, 1);
+    assert_eq!(
+        *filter.as_any().downcast_ref::<WordsFilter>().unwrap(),
+        filter
+    );
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn size_filter_as_any() {
+    let filter = SizeFilter { content_length: 1 };
+    let filter2 = SizeFilter { content_length: 1 };
+
+    assert!(filter.box_eq(filter2.as_any()));
+
+    assert_eq!(filter.content_length, 1);
+    assert_eq!(
+        *filter.as_any().downcast_ref::<SizeFilter>().unwrap(),
+        filter
+    );
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn status_code_filter_as_any() {
+    let filter = StatusCodeFilter { filter_code: 200 };
+    let filter2 = StatusCodeFilter { filter_code: 200 };
+
+    assert!(filter.box_eq(filter2.as_any()));
+
+    assert_eq!(filter.filter_code, 200);
+    assert_eq!(
+        *filter.as_any().downcast_ref::<StatusCodeFilter>().unwrap(),
+        filter
+    );
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn regex_filter_as_any() {
+    let raw = r".*\.txt$";
+    let compiled = Regex::new(raw).unwrap();
+    let compiled2 = Regex::new(raw).unwrap();
+    let filter = RegexFilter {
+        compiled,
+        raw_string: raw.to_string(),
+    };
+    let filter2 = RegexFilter {
+        compiled: compiled2,
+        raw_string: raw.to_string(),
+    };
+
+    assert!(filter.box_eq(filter2.as_any()));
+
+    assert_eq!(filter.raw_string, r".*\.txt$");
+    assert_eq!(
+        *filter.as_any().downcast_ref::<RegexFilter>().unwrap(),
+        filter
+    );
+}
+
+#[test]
+/// test should_filter on WilcardFilter where static logic matches
+fn wildcard_should_filter_when_static_wildcard_found() {
+    let body =
+        "pellentesque diam volutpat commodo sed egestas egestas fringilla phasellus faucibus";
+
+    let mut resp = FeroxResponse::default();
+    resp.set_wildcard(true);
+    resp.set_url("http://localhost");
+    resp.set_text(body);
+
+    let filter = WildcardFilter {
+        content_length: Some(body.len() as u64),
+        line_count: Some(1),
+        word_count: Some(10),
+        method: DEFAULT_METHOD.to_string(),
+        status_code: 200,
+        dont_filter: false,
+    };
+
+    assert!(filter.should_filter_response(&resp));
+}
+
+#[test]
+/// test should_filter on WilcardFilter where static logic matches but response length is 0
+fn wildcard_should_filter_when_static_wildcard_len_is_zero() {
+    let mut resp = FeroxResponse::default();
+    resp.set_wildcard(true);
+    resp.set_url("http://localhost");
+
+    // default WildcardFilter is used in the code that executes when response.content_length() == 0
+    let filter = WildcardFilter {
+        content_length: Some(0),
+        line_count: Some(0),
+        word_count: Some(0),
+        method: DEFAULT_METHOD.to_string(),
+        status_code: 200,
+        dont_filter: false,
+    };
+
+    assert!(filter.should_filter_response(&resp));
+}
+
+#[test]
+/// test should_filter on WilcardFilter where dynamic logic matches
+fn wildcard_should_filter_when_dynamic_wildcard_found() {
+    let mut resp = FeroxResponse::default();
+    resp.set_wildcard(true);
+    resp.set_url("http://localhost/stuff");
+    resp.set_text("pellentesque diam volutpat commodo sed egestas egestas fringilla");
+
+    let filter = WildcardFilter {
+        content_length: None,
+        line_count: None,
+        word_count: Some(8),
+        method: DEFAULT_METHOD.to_string(),
+        status_code: 200,
+        dont_filter: false,
+    };
+
+    assert!(filter.should_filter_response(&resp));
+}
+#[test]
+/// test should_filter on RegexFilter where regex matches body
+fn regexfilter_should_filter_when_regex_matches_on_response_body() {
+    let mut resp = FeroxResponse::default();
+    resp.set_url("http://localhost/stuff");
+    resp.set_text("im a body response hurr durr!");
+
+    let raw = r"response...rr";
+
+    let filter = RegexFilter {
+        raw_string: raw.to_string(),
+        compiled: Regex::new(raw).unwrap(),
+    };
+
+    assert!(filter.should_filter_response(&resp));
+}
+
+#[test]
+/// a few simple tests for similarity filter
+fn similarity_filter_is_accurate() {
+    let mut resp = FeroxResponse::default();
+    resp.set_url("http://localhost/stuff");
+    resp.set_text("sitting");
+
+    let mut filter = SimilarityFilter {
+        hash: SIM_HASHER.create_signature(["kitten"].iter()),
+        original_url: "".to_string(),
+        cutoff: NEAR_DUPLICATE_DISTANCE,
+    };
+
+    // kitten/sitting is 57% similar, so a threshold of 95 should not be filtered
+    assert!(!filter.should_filter_response(&resp));
+
+    resp.set_text("");
+    filter.hash = SIM_HASHER.create_signature([""].iter());
+
+    // two empty strings are the same
+    assert!(!filter.should_filter_response(&resp));
+
+    resp.set_text("some data hash purposes running test");
+    filter.hash = SIM_HASHER.create_signature(
+        preprocess("some data to hash for the purposes of running a test").iter(),
+    );
+
+    assert!(filter.should_filter_response(&resp));
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn similarity_filter_as_any() {
+    let filter = SimilarityFilter {
+        hash: 1,
+        original_url: "".to_string(),
+        cutoff: NEAR_DUPLICATE_DISTANCE,
+    };
+
+    let filter2 = SimilarityFilter {
+        hash: 1,
+        original_url: "".to_string(),
+        cutoff: NEAR_DUPLICATE_DISTANCE,
+    };
+
+    assert!(filter.box_eq(filter2.as_any()));
+
+    assert_eq!(
+        *filter.as_any().downcast_ref::<SimilarityFilter>().unwrap(),
+        filter
+    );
+}
+
+#[test]
+/// test correctness of FeroxFilters::remove
+fn remove_function_works_as_expected() {
+    let data = FeroxFilters::default();
+    assert!(data.filters.read().unwrap().is_empty());
+
+    (0..8).for_each(|i| {
+        data.push(Box::new(WordsFilter { word_count: i })).unwrap();
+    });
+
+    // remove removes index-1 from the vec, zero is skipped, and out-of-bounds indices are skipped
+    data.remove(&mut [0]);
+    assert_eq!(data.filters.read().unwrap().len(), 8);
+
+    data.remove(&mut [10000]);
+    assert_eq!(data.filters.read().unwrap().len(), 8);
+
+    // removing 0, 2, 4
+    data.remove(&mut [1, 3, 5]);
+
+    assert_eq!(data.filters.read().unwrap().len(), 5);
+
+    let expected = [
+        WordsFilter { word_count: 1 },
+        WordsFilter { word_count: 3 },
+        WordsFilter { word_count: 5 },
+        WordsFilter { word_count: 6 },
+        WordsFilter { word_count: 7 },
+    ];
+
+    for filter in data.filters.read().unwrap().iter() {
+        let downcast = filter.as_any().downcast_ref::<WordsFilter>().unwrap();
+        assert!(expected.contains(downcast));
+    }
+}

src/filters/utils.rs

@@ -0,0 +1,199 @@
+use super::FeroxFilter;
+use super::SimilarityFilter;
+use crate::event_handlers::Handles;
+use crate::response::FeroxResponse;
+use crate::utils::{logged_request, parse_url_with_raw_path};
+use crate::DEFAULT_METHOD;
+use crate::NEAR_DUPLICATE_DISTANCE;
+use anyhow::Result;
+use regex::Regex;
+use std::sync::Arc;
+
+/// wrapper around logic necessary to create a SimilarityFilter
+///
+/// - parses given url
+/// - makes request to the parsed url
+/// - gathers extensions from the url, if configured to do so
+/// - computes hash of response body
+/// - creates filter with hash
+pub(crate) async fn create_similarity_filter(
+    similarity_filter: &str,
+    handles: Arc<Handles>,
+) -> Result<SimilarityFilter> {
+    // url as-is based on input, ignores user-specified url manipulation options (add-slash etc)
+    let url = parse_url_with_raw_path(similarity_filter)?;
+
+    // attempt to request the given url
+    let resp = logged_request(&url, DEFAULT_METHOD, None, handles.clone()).await?;
+
+    // if successful, create a filter based on the response's body
+    let mut fr = FeroxResponse::from(
+        resp,
+        similarity_filter,
+        DEFAULT_METHOD,
+        handles.config.output_level,
+        handles.config.response_size_limit,
+    )
+    .await;
+
+    if handles.config.collect_extensions {
+        fr.parse_extension(handles.clone())?;
+    }
+
+    let filter = SimilarityFilter::from(&fr);
+
+    Ok(filter)
+}
+
+/// used in conjunction with the Scan Management Menu
+///
+/// when a user uses the n[ew-filter] command in the menu, the two params are passed here for
+/// processing.
+///
+/// an example command may be `new-filter lines 40`. `lines` and `40` are passed here as &str's
+///
+/// once here, the type and value are used to create an appropriate FeroxFilter. If anything
+/// goes wrong during creation, None is returned.
+pub(crate) fn filter_lookup(filter_type: &str, filter_value: &str) -> Option<Box<dyn FeroxFilter>> {
+    match filter_type {
+        "status" => {
+            if let Ok(parsed) = filter_value.parse() {
+                return Some(Box::new(super::StatusCodeFilter {
+                    filter_code: parsed,
+                }));
+            }
+        }
+        "lines" => {
+            if let Ok(parsed) = filter_value.parse() {
+                return Some(Box::new(super::LinesFilter { line_count: parsed }));
+            }
+        }
+        "size" => {
+            if let Ok(parsed) = filter_value.parse() {
+                return Some(Box::new(super::SizeFilter {
+                    content_length: parsed,
+                }));
+            }
+        }
+        "words" => {
+            if let Ok(parsed) = filter_value.parse() {
+                return Some(Box::new(super::WordsFilter { word_count: parsed }));
+            }
+        }
+        "regex" => {
+            if let Ok(parsed) = Regex::new(filter_value) {
+                return Some(Box::new(super::RegexFilter {
+                    compiled: parsed,
+                    raw_string: filter_value.to_string(),
+                }));
+            }
+        }
+        "similarity" => {
+            return Some(Box::new(SimilarityFilter::new(
+                0,
+                filter_value.to_string(),
+                NEAR_DUPLICATE_DISTANCE,
+            )));
+        }
+        _ => (),
+    }
+
+    None
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::config::Configuration;
+    use crate::filters::{LinesFilter, RegexFilter, SizeFilter, StatusCodeFilter, WordsFilter};
+    use crate::scan_manager::FeroxScans;
+    use httpmock::Method::GET;
+    use httpmock::MockServer;
+
+    #[test]
+    /// filter_lookup returns correct filters
+    fn filter_lookup_returns_correct_filters() {
+        let filter = filter_lookup("status", "200").unwrap();
+        assert_eq!(
+            filter.as_any().downcast_ref::<StatusCodeFilter>().unwrap(),
+            &StatusCodeFilter { filter_code: 200 }
+        );
+
+        let filter = filter_lookup("lines", "10").unwrap();
+        assert_eq!(
+            filter.as_any().downcast_ref::<LinesFilter>().unwrap(),
+            &LinesFilter { line_count: 10 }
+        );
+
+        let filter = filter_lookup("size", "20").unwrap();
+        assert_eq!(
+            filter.as_any().downcast_ref::<SizeFilter>().unwrap(),
+            &SizeFilter { content_length: 20 }
+        );
+
+        let filter = filter_lookup("words", "30").unwrap();
+        assert_eq!(
+            filter.as_any().downcast_ref::<WordsFilter>().unwrap(),
+            &WordsFilter { word_count: 30 }
+        );
+
+        let filter = filter_lookup("regex", "stuff.*").unwrap();
+        let compiled = Regex::new("stuff.*").unwrap();
+        let raw_string = String::from("stuff.*");
+        assert_eq!(
+            filter.as_any().downcast_ref::<RegexFilter>().unwrap(),
+            &RegexFilter {
+                compiled,
+                raw_string
+            }
+        );
+
+        let filter = filter_lookup("similarity", "http://localhost").unwrap();
+        assert_eq!(
+            filter.as_any().downcast_ref::<SimilarityFilter>().unwrap(),
+            &SimilarityFilter {
+                hash: 0,
+                original_url: "http://localhost".to_string(),
+                cutoff: NEAR_DUPLICATE_DISTANCE,
+            }
+        );
+
+        assert!(filter_lookup("non-existent", "").is_none());
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    /// ensure create_similarity_filter correctness of return value and side-effects
+    async fn create_similarity_filter_is_correct() {
+        let srv = MockServer::start();
+
+        let mock = srv.mock(|when, then| {
+            when.method(GET).path("/");
+            then.status(200).body("this is a test");
+        });
+
+        let scans = FeroxScans::default();
+        let config = Configuration {
+            collect_extensions: true,
+            ..Default::default()
+        };
+
+        let (test_handles, _) = Handles::for_testing(Some(Arc::new(scans)), Some(Arc::new(config)));
+
+        let handles = Arc::new(test_handles);
+
+        let filter = create_similarity_filter(&srv.url("/"), handles.clone())
+            .await
+            .unwrap();
+
+        assert_eq!(mock.hits(), 1);
+
+        assert_eq!(
+            filter,
+            SimilarityFilter {
+                hash: 14897447612059286329,
+                original_url: srv.url("/"),
+                cutoff: NEAR_DUPLICATE_DISTANCE,
+            }
+        );
+    }
+}

src/filters/wildcard.rs

@@ -0,0 +1,180 @@
+use console::style;
+
+use super::*;
+use crate::utils::create_report_string;
+use crate::{config::OutputLevel, DEFAULT_METHOD};
+
+/// Data holder for all relevant data needed when auto-filtering out wildcard responses
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct WildcardFilter {
+    /// The content-length of this response, if known
+    pub content_length: Option<u64>,
+
+    /// The number of lines contained in the body of this response, if known
+    pub line_count: Option<usize>,
+
+    /// The number of words contained in the body of this response, if known
+    pub word_count: Option<usize>,
+
+    /// method used in request that should be included with filters passed via runtime configuration
+    pub method: String,
+
+    /// the status code returned in the response
+    pub status_code: u16,
+
+    /// whether or not the user passed -D on the command line
+    pub dont_filter: bool,
+}
+
+/// implementation of WildcardFilter
+impl WildcardFilter {
+    /// given a boolean representing whether -D was used or not, create a new WildcardFilter
+    pub fn new(dont_filter: bool) -> Self {
+        Self {
+            dont_filter,
+            ..Default::default()
+        }
+    }
+}
+
+/// implement default that populates `method` with its default value
+impl Default for WildcardFilter {
+    fn default() -> Self {
+        Self {
+            content_length: None,
+            line_count: None,
+            word_count: None,
+            method: DEFAULT_METHOD.to_string(),
+            status_code: 0,
+            dont_filter: false,
+        }
+    }
+}
+
+/// implementation of FeroxFilter for WildcardFilter
+impl FeroxFilter for WildcardFilter {
+    /// Examine size/words/lines and method to determine whether or not the response received
+    /// is a wildcard response and therefore should be filtered out
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({self:?} {response})");
+
+        // quick return if dont_filter is set
+        if self.dont_filter {
+            // --dont-filter applies specifically to wildcard filters, it is not a 100% catch all
+            // for not filtering anything.  As such, it should live in the implementation of
+            // a wildcard filter
+            return false;
+        }
+
+        if self.method != response.method().as_str() {
+            // method's don't match, so this response should not be filtered out
+            log::trace!("exit: should_filter_response -> false");
+            return false;
+        }
+
+        if self.status_code != response.status().as_u16() {
+            // status codes don't match, so this response should not be filtered out
+            log::trace!("exit: should_filter_response -> false");
+            return false;
+        }
+
+        // methods and status codes match at this point, just need to check the other fields
+
+        match (self.content_length, self.word_count, self.line_count) {
+            (Some(cl), Some(wc), Some(lc)) => {
+                if cl == response.content_length()
+                    && wc == response.word_count()
+                    && lc == response.line_count()
+                {
+                    log::debug!("filtered out {}", response.url());
+                    log::trace!("exit: should_filter_response -> true");
+                    return true;
+                }
+            }
+            (Some(cl), Some(wc), None) => {
+                if cl == response.content_length() && wc == response.word_count() {
+                    log::debug!("filtered out {}", response.url());
+                    log::trace!("exit: should_filter_response -> true");
+                    return true;
+                }
+            }
+            (Some(cl), None, Some(lc)) => {
+                if cl == response.content_length() && lc == response.line_count() {
+                    log::debug!("filtered out {}", response.url());
+                    log::trace!("exit: should_filter_response -> true");
+                    return true;
+                }
+            }
+            (None, Some(wc), Some(lc)) => {
+                if wc == response.word_count() && lc == response.line_count() {
+                    log::debug!("filtered out {}", response.url());
+                    log::trace!("exit: should_filter_response -> true");
+                    return true;
+                }
+            }
+            (Some(cl), None, None) => {
+                if cl == response.content_length() {
+                    log::debug!("filtered out {}", response.url());
+                    log::trace!("exit: should_filter_response -> true");
+                    return true;
+                }
+            }
+            (None, Some(wc), None) => {
+                if wc == response.word_count() {
+                    log::debug!("filtered out {}", response.url());
+                    log::trace!("exit: should_filter_response -> true");
+                    return true;
+                }
+            }
+            (None, None, Some(lc)) => {
+                if lc == response.line_count() {
+                    log::debug!("filtered out {}", response.url());
+                    log::trace!("exit: should_filter_response -> true");
+                    return true;
+                }
+            }
+            (None, None, None) => {
+                unreachable!("wildcard filter without any filters set");
+            }
+        }
+
+        log::trace!("exit: should_filter_response -> false");
+        false
+    }
+
+    /// Compare one WildcardFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>() == Some(self)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
+
+impl std::fmt::Display for WildcardFilter {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let msg = create_report_string(
+            self.status_code.to_string().as_str(),
+            self.method.as_str(),
+            &self
+                .line_count
+                .map_or_else(|| "-".to_string(), |x| x.to_string()),
+            &self
+                .word_count
+                .map_or_else(|| "-".to_string(), |x| x.to_string()),
+            &self
+                .content_length
+                .map_or_else(|| "-".to_string(), |x| x.to_string()),
+            &format!(
+                "{} found {}-like response and created new filter; toggle off with {}",
+                style("Auto-filtering").bright().green(),
+                style("404").red(),
+                style("--dont-filter").yellow()
+            ),
+            OutputLevel::Default,
+        );
+        write!(f, "{msg}")
+    }
+}

src/filters/words.rs

@@ -0,0 +1,33 @@
+use super::*;
+
+/// Simple implementor of FeroxFilter; used to filter out responses based on the number of words
+/// in a Response body; specified using -W|--filter-words
+#[derive(Default, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub struct WordsFilter {
+    /// Number of words in a Response's body that should be filtered
+    pub word_count: usize,
+}
+
+/// implementation of FeroxFilter for WordsFilter
+impl FeroxFilter for WordsFilter {
+    /// Check `word_count` against what was passed in via -W|--filter-words
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({self:?} {response})");
+
+        let result = response.word_count() == self.word_count;
+
+        log::trace!("exit: should_filter_response -> {result}");
+
+        result
+    }
+
+    /// Compare one WordsFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>() == Some(self)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}

src/lib.rs

@@ -1,176 +1,361 @@
+#![deny(clippy::all)]
+#![allow(clippy::mutex_atomic)]
+use anyhow::Result;
+use reqwest::StatusCode;
+use std::collections::HashSet;
+use tokio::{
+    sync::mpsc::{UnboundedReceiver, UnboundedSender},
+    task::JoinHandle,
+};
+
+use crate::event_handlers::Command;
+
 pub mod banner;
-pub mod client;
 pub mod config;
-pub mod extractor;
+mod client;
+pub mod event_handlers;
+pub mod filters;
 pub mod heuristics;
 pub mod logger;
-pub mod parser;
+mod parser;
 pub mod progress;
-pub mod reporter;
+pub mod scan_manager;
 pub mod scanner;
+pub mod statistics;
+pub mod sync;
+mod traits;
 pub mod utils;
+mod extractor;
+mod macros;
+mod url;
+mod response;
+mod message;
+mod nlp;
 
-use reqwest::header::HeaderMap;
-use reqwest::{Response, StatusCode, Url};
-use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender};
+/// Alias for tokio::sync::mpsc::UnboundedSender<Command>
+pub(crate) type CommandSender = UnboundedSender<Command>;
 
-/// Generic Result type to ease error handling in async contexts
-pub type FeroxResult<T> =
-    std::result::Result<T, Box<dyn std::error::Error + Send + Sync + 'static>>;
+/// Alias for tokio::sync::mpsc::UnboundedSender<Command>
+pub(crate) type CommandReceiver = UnboundedReceiver<Command>;
+
+/// Alias for tokio::task::JoinHandle<anyhow::Result<()>>
+pub(crate) type Joiner = JoinHandle<Result<()>>;
 
 /// Generic mpsc::unbounded_channel type to tidy up some code
-pub type FeroxChannel<T> = (UnboundedSender<T>, UnboundedReceiver<T>);
+pub(crate) type FeroxChannel<T> = (UnboundedSender<T>, UnboundedReceiver<T>);
+
+/// Wrapper around the results of performing any kind of extraction against a target web page
+pub(crate) type ExtractionResult = HashSet<String>;
 
 /// Version pulled from Cargo.toml at compile time
-pub const VERSION: &str = env!("CARGO_PKG_VERSION");
+pub(crate) const VERSION: &str = env!("CARGO_PKG_VERSION");
+
+/// Maximum number of file descriptors that can be opened during a scan
+pub const DEFAULT_OPEN_FILE_LIMIT: u64 = 8192;
+
+/// Default set of extensions to Ignore when auto-collecting extensions during scans
+pub(crate) const DEFAULT_IGNORED_EXTENSIONS: [&str; 43] = [
+    "woff2", "woff", "ttf", "otf", "eot", "tif", "tiff", "ico", "cur", "bmp", "webp", "svg", "png",
+    "jpg", "jpeg", "jfif", "gif", "avif", "apng", "pjpeg", "pjp", "mov", "wav", "mpg", "mpeg",
+    "mp3", "mp4", "m4a", "m4p", "m4v", "ogg", "webm", "ogv", "oga", "flac", "aac", "3gp", "css",
+    "zip", "xls", "xml", "gz", "tgz",
+];
+
+/// Default set of extensions to search for when auto-collecting backups during scans
+pub(crate) const DEFAULT_BACKUP_EXTENSIONS: [&str; 5] = ["~", ".bak", ".bak2", ".old", ".1"];
+
+/// list of common file extensions for link density detection in directory listings
+/// based on https://www.computerhope.com/issues/ch001789.htm
+pub(crate) const COMMON_FILE_EXTENSIONS: [&str; 154] = [
+    // Web & Documents
+    ".html",
+    ".htm",
+    ".php",
+    ".asp",
+    ".aspx",
+    ".jsp",
+    ".jspx",
+    ".cgi",
+    ".pl",
+    ".py",
+    ".rb",
+    ".lua",
+    ".txt",
+    ".pdf",
+    ".doc",
+    ".docx",
+    ".xls",
+    ".xlsx",
+    ".ppt",
+    ".pptx",
+    ".odt",
+    ".ods",
+    ".odp",
+    ".rtf",
+    ".tex",
+    ".md",
+    ".csv",
+    // Programming & Scripts
+    ".js",
+    ".mjs",
+    ".ts",
+    ".jsx",
+    ".tsx",
+    ".css",
+    ".scss",
+    ".sass",
+    ".less",
+    ".java",
+    ".class",
+    ".jar",
+    ".c",
+    ".cpp",
+    ".h",
+    ".hpp",
+    ".cs",
+    ".vb",
+    ".go",
+    ".rs",
+    ".swift",
+    ".kt",
+    ".scala",
+    ".r",
+    ".m",
+    ".mm",
+    ".f",
+    ".f90",
+    ".pas",
+    ".asm",
+    ".sh",
+    ".bash",
+    ".zsh",
+    ".fish",
+    ".bat",
+    ".cmd",
+    ".ps1",
+    ".psm1",
+    // Data & Config
+    ".xml",
+    ".json",
+    ".yaml",
+    ".yml",
+    ".toml",
+    ".ini",
+    ".conf",
+    ".config",
+    ".cfg",
+    ".properties",
+    ".env",
+    ".sql",
+    ".db",
+    ".sqlite",
+    ".mdb",
+    ".accdb",
+    // Archives & Compressed
+    ".zip",
+    ".rar",
+    ".7z",
+    ".tar",
+    ".gz",
+    ".bz2",
+    ".xz",
+    ".tgz",
+    ".tbz2",
+    ".cab",
+    ".dmg",
+    ".iso",
+    ".img",
+    // Executables & Libraries
+    ".exe",
+    ".dll",
+    ".so",
+    ".dylib",
+    ".app",
+    ".deb",
+    ".rpm",
+    ".apk",
+    ".msi",
+    // Images
+    ".jpg",
+    ".jpeg",
+    ".png",
+    ".gif",
+    ".bmp",
+    ".svg",
+    ".webp",
+    ".ico",
+    ".tif",
+    ".tiff",
+    ".psd",
+    ".ai",
+    ".eps",
+    ".raw",
+    ".cr2",
+    ".nef",
+    // Audio
+    ".mp3",
+    ".wav",
+    ".flac",
+    ".aac",
+    ".ogg",
+    ".wma",
+    ".m4a",
+    ".opus",
+    ".aiff",
+    // Video
+    ".mp4",
+    ".avi",
+    ".mkv",
+    ".mov",
+    ".wmv",
+    ".flv",
+    ".webm",
+    ".m4v",
+    ".mpg",
+    ".mpeg",
+    ".3gp",
+    ".ogv",
+    // Fonts
+    ".ttf",
+    ".otf",
+    ".woff",
+    ".woff2",
+    ".eot",
+    // Backups & Logs
+    ".log",
+    ".bak",
+    ".tmp",
+    ".temp",
+    ".swp",
+    ".swo",
+    ".old",
+    ".orig",
+    ".backup",
+];
 
 /// Default wordlist to use when `-w|--wordlist` isn't specified and not `wordlist` isn't set
 /// in a [ferox-config.toml](constant.DEFAULT_CONFIG_NAME.html) config file.
 ///
-/// defaults to kali's default install location:
+/// defaults to kali's default install location on linux:
 /// - `/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt`
+///
+/// and to the current directory on windows
+/// - `.\seclists\Discovery\Web-Content\raft-medium-directories.txt`
+#[cfg(not(target_os = "windows"))]
 pub const DEFAULT_WORDLIST: &str =
     "/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt";
+#[cfg(target_os = "windows")]
+pub const DEFAULT_WORDLIST: &str =
+    ".\\SecLists\\Discovery\\Web-Content\\raft-medium-directories.txt";
+pub const SECONDARY_WORDLIST: &str =
+    "/usr/local/share/seclists/Discovery/Web-Content/raft-medium-directories.txt";
 
-/// Default list of status codes to report
-///
-/// * 200 Ok
-/// * 204 No Content
-/// * 301 Moved Permanently
-/// * 302 Found
-/// * 307 Temporary Redirect
-/// * 308 Permanent Redirect
-/// * 401 Unauthorized
-/// * 403 Forbidden
-/// * 405 Method Not Allowed
-pub const DEFAULT_STATUS_CODES: [StatusCode; 9] = [
+/// Number of milliseconds to wait between polls of `PAUSE_SCAN` when user pauses a scan
+pub(crate) const SLEEP_DURATION: u64 = 500;
+
+/// The percentage of requests as errors it takes to be deemed too high
+pub const HIGH_ERROR_RATIO: f64 = 0.90;
+
+/// Default list of status codes to report (all of them)
+pub const DEFAULT_STATUS_CODES: [StatusCode; 60] = [
+    // all 1XX response codes
+    StatusCode::CONTINUE,
+    StatusCode::SWITCHING_PROTOCOLS,
+    StatusCode::PROCESSING,
+    // all 2XX response codes
     StatusCode::OK,
+    StatusCode::CREATED,
+    StatusCode::ACCEPTED,
+    StatusCode::NON_AUTHORITATIVE_INFORMATION,
     StatusCode::NO_CONTENT,
+    StatusCode::RESET_CONTENT,
+    StatusCode::PARTIAL_CONTENT,
+    StatusCode::MULTI_STATUS,
+    StatusCode::ALREADY_REPORTED,
+    StatusCode::IM_USED,
+    // all 3XX response codes
+    StatusCode::MULTIPLE_CHOICES,
     StatusCode::MOVED_PERMANENTLY,
     StatusCode::FOUND,
+    StatusCode::SEE_OTHER,
+    StatusCode::NOT_MODIFIED,
+    StatusCode::USE_PROXY,
     StatusCode::TEMPORARY_REDIRECT,
     StatusCode::PERMANENT_REDIRECT,
+    // all 4XX response codes
+    StatusCode::BAD_REQUEST,
     StatusCode::UNAUTHORIZED,
+    StatusCode::PAYMENT_REQUIRED,
     StatusCode::FORBIDDEN,
+    StatusCode::NOT_FOUND,
     StatusCode::METHOD_NOT_ALLOWED,
+    StatusCode::NOT_ACCEPTABLE,
+    StatusCode::PROXY_AUTHENTICATION_REQUIRED,
+    StatusCode::REQUEST_TIMEOUT,
+    StatusCode::CONFLICT,
+    StatusCode::GONE,
+    StatusCode::LENGTH_REQUIRED,
+    StatusCode::PRECONDITION_FAILED,
+    StatusCode::PAYLOAD_TOO_LARGE,
+    StatusCode::URI_TOO_LONG,
+    StatusCode::UNSUPPORTED_MEDIA_TYPE,
+    StatusCode::RANGE_NOT_SATISFIABLE,
+    StatusCode::EXPECTATION_FAILED,
+    StatusCode::IM_A_TEAPOT,
+    StatusCode::MISDIRECTED_REQUEST,
+    StatusCode::UNPROCESSABLE_ENTITY,
+    StatusCode::LOCKED,
+    StatusCode::FAILED_DEPENDENCY,
+    StatusCode::UPGRADE_REQUIRED,
+    StatusCode::PRECONDITION_REQUIRED,
+    StatusCode::TOO_MANY_REQUESTS,
+    StatusCode::REQUEST_HEADER_FIELDS_TOO_LARGE,
+    StatusCode::UNAVAILABLE_FOR_LEGAL_REASONS,
+    // all 5XX response codes
+    StatusCode::INTERNAL_SERVER_ERROR,
+    StatusCode::NOT_IMPLEMENTED,
+    StatusCode::BAD_GATEWAY,
+    StatusCode::SERVICE_UNAVAILABLE,
+    StatusCode::GATEWAY_TIMEOUT,
+    StatusCode::HTTP_VERSION_NOT_SUPPORTED,
+    StatusCode::VARIANT_ALSO_NEGOTIATES,
+    StatusCode::INSUFFICIENT_STORAGE,
+    StatusCode::LOOP_DETECTED,
+    StatusCode::NOT_EXTENDED,
+    StatusCode::NETWORK_AUTHENTICATION_REQUIRED,
 ];
 
+/// Default method for requests
+pub(crate) const DEFAULT_METHOD: &str = "GET";
+
 /// Default filename for config file settings
 ///
 /// Expected location is in the same directory as the feroxbuster binary.
 pub const DEFAULT_CONFIG_NAME: &str = "ferox-config.toml";
+/// User agents to select from when random agent is being used
+pub const USER_AGENTS: [&str; 12] = [
+    "Mozilla/5.0 (Linux; Android 8.0.0; SM-G960F Build/R16NW) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.84 Mobile Safari/537.36",
+    "Mozilla/5.0 (iPhone; CPU iPhone OS 12_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1",
+    "Mozilla/5.0 (Windows Phone 10.0; Android 6.0.1; Microsoft; RM-1152) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Mobile Safari/537.36 Edge/15.15254",
+    "Mozilla/5.0 (Linux; Android 7.0; Pixel C Build/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/52.0.2743.98 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246",
+    "Mozilla/5.0 (X11; CrOS x86_64 8172.45.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.64 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/601.3.9 (KHTML, like Gecko) Version/9.0.2 Safari/601.3.9",
+    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.111 Safari/537.36",
+    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 Firefox/15.0.1",
+    "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
+    "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)",
+    "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)",
+];
 
-/// A `FeroxResponse`, derived from a `Response` to a submitted `Request`
-#[derive(Debug)]
-pub struct FeroxResponse {
-    /// The final `Url` of this `FeroxResponse`
-    url: Url,
+/// maximum hamming distance allowed between two simhash signatures when detecting near-duplicates
+///
+/// ref: https://static.googleusercontent.com/media/research.google.com/en//pubs/archive/33026.pdf
+/// section: 4.1 Choice of Parameters
+pub(crate) const NEAR_DUPLICATE_DISTANCE: usize = 3;
 
-    /// The `StatusCode` of this `FeroxResponse`
-    status: StatusCode,
-
-    /// The full response text
-    text: String,
-
-    /// The content-length of this response, if known
-    content_length: u64,
-
-    /// The `Headers` of this `FeroxResponse`
-    headers: HeaderMap,
-}
-
-/// `FeroxResponse` implementation
-impl FeroxResponse {
-    /// Get the `StatusCode` of this `FeroxResponse`
-    pub fn status(&self) -> &StatusCode {
-        &self.status
-    }
-
-    /// Get the final `Url` of this `FeroxResponse`.
-    pub fn url(&self) -> &Url {
-        &self.url
-    }
-
-    /// Get the full response text
-    pub fn text(&self) -> &str {
-        &self.text
-    }
-
-    /// Get the `Headers` of this `FeroxResponse`
-    pub fn headers(&self) -> &HeaderMap {
-        &self.headers
-    }
-
-    /// Get the content-length of this response, if known
-    pub fn content_length(&self) -> u64 {
-        self.content_length
-    }
-
-    /// Set `FeroxResponse`'s `url` attribute, has no affect if an error occurs
-    pub fn set_url(&mut self, url: &str) {
-        match Url::parse(&url) {
-            Ok(url) => {
-                self.url = url;
-            }
-            Err(e) => {
-                log::error!("Could not parse {} into a Url: {}", url, e);
-            }
-        };
-    }
-
-    /// Make a reasonable guess at whether the response is a file or not
-    ///
-    /// Examines the last part of a path to determine if it has an obvious extension
-    /// i.e. http://localhost/some/path/stuff.js where stuff.js indicates a file
-    ///
-    /// Additionally, inspects query parameters, as they're also often indicative of a file
-    pub fn is_file(&self) -> bool {
-        let has_extension = match self.url.path_segments() {
-            Some(path) => {
-                if let Some(last) = path.last() {
-                    last.contains('.') // last segment has some sort of extension, probably
-                } else {
-                    false
-                }
-            }
-            None => false,
-        };
-
-        self.url.query_pairs().count() > 0 || has_extension
-    }
-
-    /// Create a new `FeroxResponse` from the given `Response`
-    pub async fn from(response: Response, read_body: bool) -> Self {
-        let url = response.url().clone();
-        let status = response.status();
-        let headers = response.headers().clone();
-        let content_length = response.content_length().unwrap_or(0);
-
-        let text = if read_body {
-            // .text() consumes the response, must be called last
-            // additionally, --extract-links is currently the only place we use the body of the
-            // response, so we forego the processing if not performing extraction
-            match response.text().await {
-                // await the response's body
-                Ok(text) => text,
-                Err(e) => {
-                    log::error!("Could not parse body from response: {}", e);
-                    String::new()
-                }
-            }
-        } else {
-            String::new()
-        };
-
-        FeroxResponse {
-            url,
-            status,
-            content_length,
-            text,
-            headers,
-        }
-    }
-}
+/// maximum hamming distance allowed between two simhash signatures when unique'ifying responses
+pub(crate) const UNIQUE_DISTANCE: usize = 1;
 
 #[cfg(test)]
 mod tests {

src/logger.rs

@@ -1,26 +1,36 @@
-use crate::config::{CONFIGURATION, PROGRESS_PRINTER};
-use crate::reporter::{get_cached_file_handle, safe_file_write};
-use console::{style, Color};
-use env_logger::Builder;
 use std::env;
+use std::fs::OpenOptions;
+use std::io::BufWriter;
+use std::sync::{Arc, RwLock};
 use std::time::Instant;
 
+use anyhow::{Context, Result};
+use env_logger::Builder;
+
+use crate::{
+    config::Configuration,
+    message::FeroxMessage,
+    progress::PROGRESS_PRINTER,
+    traits::FeroxSerialize,
+    utils::{fmt_err, write_to},
+};
+
 /// Create a customized instance of
 /// [env_logger::Logger](https://docs.rs/env_logger/latest/env_logger/struct.Logger.html)
 /// with timer offset/color and set the log level based on `verbosity`
-pub fn initialize(verbosity: u8) {
+pub fn initialize(config: Arc<Configuration>) -> Result<()> {
     // use occurrences of -v on commandline to or verbosity = N in feroxconfig.toml to set
     // log level for the application; respects already specified RUST_LOG environment variable
     match env::var("RUST_LOG") {
         Ok(_) => {} // RUST_LOG found, don't override
         Err(_) => {
             // only set log level based on verbosity when RUST_LOG variable doesn't exist
-            match verbosity {
+            match config.verbosity {
                 0 => (),
                 1 => env::set_var("RUST_LOG", "warn"),
                 2 => env::set_var("RUST_LOG", "info"),
-                3 => env::set_var("RUST_LOG", "debug,hyper=info,reqwest=info"),
-                _ => env::set_var("RUST_LOG", "trace,hyper=info,reqwest=info"),
+                3 => env::set_var("RUST_LOG", "feroxbuster=debug,info"),
+                _ => env::set_var("RUST_LOG", "feroxbuster=trace,info"),
             }
         }
     }
@@ -28,46 +38,44 @@ pub fn initialize(verbosity: u8) {
     let start = Instant::now();
     let mut builder = Builder::from_default_env();
 
-    // I REALLY wanted the logger to also use the reporting channels found in the `reporter`
-    // module. However, in order to properly clean up the channels, all references to the
-    // transmitter side of a channel need to go out of scope, then you can await the future into
-    // which the receiver was moved.
-    //
-    // The problem was that putting a transmitter reference in this closure, which gets initialized
-    // as part of the global logger, made it so that I couldn't destroy/leak/take/swap the last
-    // reference to allow the channels to gracefully close.
-    //
-    // The workaround was to have a RwLock around the file and allow both the logger and the
-    // file handler to both write independent of each other.
-    let locked_file = get_cached_file_handle(&CONFIGURATION.output);
+    let file = if !config.debug_log.is_empty() {
+        let f = OpenOptions::new() // std fs
+            .create(true)
+            .append(true)
+            .open(&config.debug_log)
+            .with_context(|| fmt_err(&format!("Could not open {}", &config.debug_log)))?;
+
+        let mut writer = BufWriter::new(f);
+
+        // write out the configuration to the debug file if it exists
+        write_to(&*config, &mut writer, config.json)?;
+
+        Some(Arc::new(RwLock::new(writer)))
+    } else {
+        None
+    };
 
     builder
         .format(move |_, record| {
-            let t = start.elapsed().as_secs_f32();
-            let level = record.level();
-
-            let (level_name, level_color) = match level {
-                log::Level::Error => ("ERR", Color::Red),
-                log::Level::Warn => ("WRN", Color::Red),
-                log::Level::Info => ("INF", Color::Cyan),
-                log::Level::Debug => ("DBG", Color::Yellow),
-                log::Level::Trace => ("TRC", Color::Magenta),
+            let log_entry = FeroxMessage {
+                message: record.args().to_string(),
+                level: record.level().to_string(),
+                time_offset: start.elapsed().as_secs_f32(),
+                module: record.target().to_string(),
+                kind: "log".to_string(),
             };
 
-            let msg = format!(
-                "{} {:10.03} {}\n",
-                style(level_name).bg(level_color).black(),
-                style(t).dim(),
-                style(record.args()).dim(),
-            );
+            PROGRESS_PRINTER.println(log_entry.as_str());
 
-            PROGRESS_PRINTER.println(&msg);
-
-            if let Some(buffered_file) = locked_file.clone() {
-                safe_file_write(&msg, buffered_file);
+            if let Some(buffered_file) = file.clone() {
+                if let Ok(mut unlocked) = buffered_file.write() {
+                    let _ = write_to(&log_entry, &mut unlocked, config.json);
+                }
             }
 
             Ok(())
         })
         .init();
+
+    Ok(())
 }

src/macros.rs

@@ -0,0 +1,23 @@
+#![macro_use]
+
+#[macro_export]
+/// wrapper to improve code readability
+macro_rules! send_command {
+    ($tx:expr, $value:expr) => {
+        $tx.send($value).unwrap_or_default();
+    };
+}
+
+#[macro_export]
+/// while looping, check for a Result, if Ok return the value, if Err, continue
+macro_rules! skip_fail {
+    ($res:expr) => {
+        match $res {
+            Ok(val) => val,
+            Err(e) => {
+                log::warn!("{}", fmt_err(&format!("{}; skipping...", e)));
+                continue;
+            }
+        }
+    };
+}

src/main.rs

@@ -1,49 +1,84 @@
-use feroxbuster::config::{CONFIGURATION, PROGRESS_PRINTER};
-use feroxbuster::scanner::scan_url;
-use feroxbuster::utils::{ferox_print, get_current_depth, module_colorizer, status_colorizer};
-use feroxbuster::{banner, heuristics, logger, reporter, FeroxResponse, FeroxResult, VERSION};
+use std::{
+    env::{
+        args,
+        consts::{ARCH, OS},
+    },
+    fs::{create_dir, remove_file, File},
+    io::{stderr, BufRead, BufReader},
+    ops::Index,
+    path::Path,
+    process::{exit, Command, Stdio},
+    sync::{atomic::Ordering, Arc},
+};
+
+use anyhow::{bail, Context, Result};
 use futures::StreamExt;
-use std::collections::HashSet;
-use std::fs::File;
-use std::io::{stderr, BufRead, BufReader};
-use std::process;
-use std::sync::Arc;
-use tokio::io;
-use tokio::sync::mpsc::UnboundedSender;
+use tokio::{
+    io,
+    sync::{oneshot, Semaphore},
+};
 use tokio_util::codec::{FramedRead, LinesCodec};
 
-/// Create a HashSet of Strings from the given wordlist then stores it inside an Arc
-fn get_unique_words_from_wordlist(path: &str) -> FeroxResult<Arc<HashSet<String>>> {
-    log::trace!("enter: get_unique_words_from_wordlist({})", path);
+use feroxbuster::{
+    banner::{Banner, UPDATE_URL},
+    config::{Configuration, OutputLevel},
+    event_handlers::{
+        Command::{
+            AddHandles, CreateBar, Exit, JoinTasks, LoadStats, ScanInitialUrls, UpdateTargets,
+            UpdateWordlist,
+        },
+        FiltersHandler, Handles, ScanHandler, StatsHandler, Tasks, TermInputHandler,
+        TermOutHandler, SCAN_COMPLETE,
+    },
+    filters, heuristics, logger,
+    progress::PROGRESS_PRINTER,
+    scan_manager::{self, ScanType},
+    scanner,
+    utils::{fmt_err, slugify_filename},
+    SECONDARY_WORDLIST,
+};
+#[cfg(not(target_os = "windows"))]
+use feroxbuster::{utils::set_open_file_limit, DEFAULT_OPEN_FILE_LIMIT};
+use lazy_static::lazy_static;
+use regex::Regex;
+use self_update::cargo_crate_version;
 
-    let file = match File::open(&path) {
-        Ok(f) => f,
-        Err(e) => {
-            eprintln!(
-                "{} {} {}",
-                status_colorizer("ERROR"),
-                module_colorizer("main::get_unique_words_from_wordlist"),
-                e
-            );
-            log::error!("Could not open wordlist: {}", e);
-            log::trace!("exit: get_unique_words_from_wordlist -> {}", e);
+lazy_static! {
+    /// Limits the number of parallel scans active at any given time when using --parallel
+    static ref PARALLEL_LIMITER: Semaphore = Semaphore::new(0);
+}
 
-            return Err(Box::new(e));
-        }
-    };
+/// Create a Vec of Strings from the given wordlist then stores it inside an Arc
+fn get_unique_words_from_wordlist(path: &str) -> Result<Arc<Vec<String>>> {
+    log::trace!("enter: get_unique_words_from_wordlist({path})");
+    let mut trimmed_word = false;
+
+    let file = File::open(path).with_context(|| format!("Could not open {path}"))?;
 
     let reader = BufReader::new(file);
 
-    let mut words = HashSet::new();
+    // this empty string ensures that we call Requester::request with the base url, i.e.
+    // `http://localhost/` instead of going straight into `http://localhost/WORD.EXT`.
+    // for vanilla scans, it doesn't matter all that much, but it can be a significant difference
+    // when `-e` is used, depending on the content at the base url.
+    let mut words = vec![String::from("")];
 
     for line in reader.lines() {
-        let result = line?;
+        line.map(|result| {
+            if !result.starts_with('#') && !result.is_empty() {
+                if result.starts_with('/') {
+                    words.push(result.trim_start_matches('/').to_string());
+                    trimmed_word = true;
+                } else {
+                    words.push(result);
+                }
+            }
+        })
+        .ok();
+    }
 
-        if result.starts_with('#') || result.is_empty() {
-            continue;
-        }
-
-        words.insert(result);
+    if trimmed_word {
+        log::warn!("Some words in the wordlist started with a leading forward-slash; those words were trimmed (i.e. /word -> word)");
     }
 
     log::trace!(
@@ -55,57 +90,62 @@ fn get_unique_words_from_wordlist(path: &str) -> FeroxResult<Arc<HashSet<String>
 }
 
 /// Determine whether it's a single url scan or urls are coming from stdin, then scan as needed
-async fn scan(
-    targets: Vec<String>,
-    tx_term: UnboundedSender<FeroxResponse>,
-    tx_file: UnboundedSender<String>,
-) -> FeroxResult<()> {
-    log::trace!("enter: scan({:?}, {:?}, {:?})", targets, tx_term, tx_file);
-    // cloning an Arc is cheap (it's basically a pointer into the heap)
-    // so that will allow for cheap/safe sharing of a single wordlist across multi-target scans
-    // as well as additional directories found as part of recursion
-    let words =
-        tokio::spawn(async move { get_unique_words_from_wordlist(&CONFIGURATION.wordlist) })
-            .await??;
+async fn scan(targets: Vec<String>, handles: Arc<Handles>) -> Result<()> {
+    log::trace!("enter: scan({targets:?}, {handles:?})");
 
-    if words.len() == 0 {
-        eprintln!(
-            "{} {} Did not find any words in {}",
-            status_colorizer("ERROR"),
-            module_colorizer("main::scan"),
-            CONFIGURATION.wordlist
-        );
-        process::exit(1);
+    let scanned_urls = handles.ferox_scans()?;
+
+    handles.send_scan_command(UpdateWordlist(handles.wordlist.clone()))?;
+
+    scanner::initialize(handles.wordlist.len(), handles.clone()).await?;
+
+    // at this point, the stat thread's progress bar can be created; things that needed to happen
+    // first:
+    // - banner gets printed
+    // - scanner initialized (this sent expected requests per directory to the stats thread, which
+    //   having been set, makes it so the progress bar doesn't flash as full before anything has
+    //   even happened
+    if matches!(handles.config.output_level, OutputLevel::Default) {
+        let mut total_offset = 0;
+
+        if let Ok(guard) = handles.scans.read() {
+            if let Some(handle) = &*guard {
+                if let Ok(scans) = handle.data.scans.read() {
+                    for scan in scans.iter() {
+                        total_offset += scan.requests_made_so_far();
+                    }
+                }
+            }
+        }
+
+        // only create the bar if no --silent|--quiet
+        handles.stats.send(CreateBar(total_offset))?;
+
+        // blocks until the bar is created / avoids race condition in first two bars
+        handles.stats.sync().await?;
     }
 
-    let mut tasks = vec![];
-
-    for target in targets {
-        let word_clone = words.clone();
-        let term_clone = tx_term.clone();
-        let file_clone = tx_file.clone();
-
-        let task = tokio::spawn(async move {
-            let base_depth = get_current_depth(&target);
-            scan_url(&target, word_clone, base_depth, term_clone, file_clone).await;
-        });
-
-        tasks.push(task);
+    if handles.config.resumed {
+        // display what has already been completed
+        scanned_urls.print_known_responses();
+        scanned_urls.print_completed_bars(handles.wordlist.len())?;
     }
 
-    // drive execution of all accumulated futures
-    futures::future::join_all(tasks).await;
+    log::debug!("sending {targets:?} to be scanned as initial targets");
+    handles.send_scan_command(ScanInitialUrls(targets))?;
+
     log::trace!("exit: scan");
 
     Ok(())
 }
 
-async fn get_targets() -> FeroxResult<Vec<String>> {
-    log::trace!("enter: get_targets");
+/// Get targets from either commandline or stdin, pass them back to the caller as a Result<Vec>
+async fn get_targets(handles: Arc<Handles>) -> Result<Vec<String>> {
+    log::trace!("enter: get_targets({handles:?})");
 
     let mut targets = vec![];
 
-    if CONFIGURATION.stdin {
+    if handles.config.stdin && handles.config.cached_stdin.is_empty() {
         // got targets from stdin, i.e. cat sites | ./feroxbuster ...
         // just need to read the targets from stdin and spawn a future for each target found
         let stdin = io::stdin(); // tokio's stdin, not std
@@ -114,100 +154,545 @@ async fn get_targets() -> FeroxResult<Vec<String>> {
         while let Some(line) = reader.next().await {
             targets.push(line?);
         }
+    } else if !handles.config.cached_stdin.is_empty() {
+        // cached_stdin populated from config::container if --stdin was used
+        // keeping the if block above as a failsafe, but i dont think we'll hit it anymore
+        targets = handles.config.cached_stdin.clone();
+    } else if handles.config.resumed {
+        // resume-from can't be used with --url, and --stdin is marked false for every resumed
+        // scan, making it mutually exclusive from either of the other two options
+        let ferox_scans = handles.ferox_scans()?;
+
+        if let Ok(scans) = ferox_scans.scans.read() {
+            for scan in scans.iter() {
+                // ferox_scans gets deserialized scans added to it at program start if --resume-from
+                // is used, so scans that aren't marked complete still need to be scanned
+                if scan.is_complete() || matches!(scan.scan_type, ScanType::File) {
+                    // this one's already done, or it's not a directory, ignore it
+                    continue;
+                }
+
+                targets.push(scan.url().to_owned());
+            }
+        };
     } else {
-        targets.push(CONFIGURATION.target_url.clone());
+        targets.push(handles.config.target_url.clone());
     }
 
-    log::trace!("exit: get_targets -> {:?}", targets);
+    // remove footgun that arises if a --dont-scan value matches on a base url
+    for target in targets.iter_mut() {
+        for denier in &handles.config.regex_denylist {
+            if denier.is_match(target) {
+                bail!(
+                    "The regex '{}' matches {}; the scan will never start",
+                    denier,
+                    target
+                );
+            }
+        }
+        for denier in &handles.config.url_denylist {
+            if denier.as_str().trim_end_matches('/') == target.trim_end_matches('/') {
+                bail!(
+                    "The url '{}' matches {}; the scan will never start",
+                    denier,
+                    target
+                );
+            }
+        }
+
+        if !target.starts_with("http") {
+            // --url hackerone.com
+            // as of the 2.13.0 update, config::container handles both --url hackerone.com
+            // and urls coming in from --stdin. I think this is dead code now, but leaving
+            // it in just in case
+            *target = format!("{}://{target}", handles.config.protocol);
+        }
+    }
+
+    log::trace!("exit: get_targets -> {targets:?}");
 
     Ok(targets)
 }
 
-#[tokio::main]
-async fn main() {
-    // setup logging based on the number of -v's used
-    logger::initialize(CONFIGURATION.verbosity);
+/// async main called from real main, broken out in this way to allow for some synchronous code
+/// to be executed before bringing the tokio runtime online
+async fn wrapped_main(config: Arc<Configuration>) -> Result<()> {
+    // join can only be called once, otherwise it causes the thread to panic
+    tokio::task::spawn_blocking(move || {
+        // ok, lazy_static! uses (unsurprisingly in retrospect) a lazy loading model where the
+        // thing obtained through deref isn't actually created until it's used. This created a
+        // problem when initializing the logger as it relied on PROGRESS_PRINTER which may or may
+        // not have been created by the time it was needed for logging (really only occurred in
+        // heuristics / banner / main). In order to initialize logging properly, we need to ensure
+        // PROGRESS_PRINTER and PROGRESS_BAR have been used at least once.  This call satisfies
+        // that constraint
+        PROGRESS_PRINTER.println("");
+    });
 
-    // can't trace main until after logger is initialized
+    // check if update_app is true
+    if config.update_app {
+        match update_app().await {
+            Err(e) => eprintln!("\n[ERROR] {e}"),
+            Ok(self_update::Status::UpToDate(version)) => {
+                eprintln!("\nFeroxbuster {version} is up to date")
+            }
+            Ok(self_update::Status::Updated(version)) => {
+                eprintln!("\nFeroxbuster updated to {version} version")
+            }
+        }
+        exit(0);
+    }
+
+    let words = if config.wordlist.starts_with("http") {
+        // found a url scheme, attempt to download the wordlist
+        let response = config
+            .client
+            .get(&config.wordlist)
+            .send()
+            .await
+            .context(format!(
+                "Unable to download wordlist from remote url: {}",
+                config.wordlist
+            ))?;
+
+        if !response.status().is_success() {
+            // status code isn't a 200, bail
+            bail!(
+                "[{}] Unable to download wordlist from url: {}",
+                response.status().as_str(),
+                config.wordlist
+            );
+        }
+
+        // attempt to get the filename from the url's path
+        let Some(mut path_segments) = response.url().path_segments() else {
+            bail!("Unable to parse path from url: {}", response.url());
+        };
+
+        let Some(filename) = path_segments.next_back() else {
+            bail!(
+                "Unable to parse filename from url's path: {}",
+                response.url().path()
+            );
+        };
+
+        let filename = filename.to_string();
+
+        // read the body and write it to disk, then use existing code to read the wordlist
+        let body = response.text().await?;
+
+        std::fs::write(&filename, body)?;
+
+        get_unique_words_from_wordlist(&filename)?
+    } else {
+        match get_unique_words_from_wordlist(&config.wordlist) {
+            Ok(w) => w,
+            Err(err) => {
+                let secondary = Path::new(SECONDARY_WORDLIST);
+
+                if secondary.exists() {
+                    eprintln!("Found wordlist in secondary location");
+                    get_unique_words_from_wordlist(SECONDARY_WORDLIST)?
+                } else {
+                    return Err(err);
+                }
+            }
+        }
+    };
+
+    if words.len() <= 1 {
+        // the check is now <= 1 due to the initial empty string added in 2.6.0
+        // 1 -> empty wordlist
+        // 0 -> error
+        bail!("Did not find any words in {}", config.wordlist);
+    }
+
+    // spawn all event handlers, expect back a JoinHandle and a *Handle to the specific event
+    let (stats_task, stats_handle) = StatsHandler::initialize(config.clone());
+    let (filters_task, filters_handle) = FiltersHandler::initialize();
+    let (out_task, out_handle) =
+        TermOutHandler::initialize(config.clone(), stats_handle.tx.clone());
+
+    // bundle up all the disparate handles and JoinHandles (tasks)
+    let handles = Arc::new(Handles::new(
+        stats_handle,
+        filters_handle,
+        out_handle,
+        config.clone(),
+        words,
+    ));
+
+    let (scan_task, scan_handle) = ScanHandler::initialize(handles.clone());
+
+    handles.set_scan_handle(scan_handle); // must be done after Handles initialization
+    handles.output.send(AddHandles(handles.clone()))?;
+
+    filters::initialize(handles.clone()).await?; // send user-supplied filters to the handler
+
+    // create new Tasks object, each of these handles is one that will be joined on later
+    let tasks = Tasks::new(out_task, stats_task, filters_task, scan_task);
+
+    if !config.time_limit.is_empty() && config.parallel == 0 {
+        // --time-limit value not an empty string, need to kick off the thread that enforces
+        // the limit
+        //
+        // if --parallel is used, this branch won't execute in the main process, but will in the
+        // children. This is because --parallel is stripped from the children's command line
+        // arguments, so, when spawned, they won't have --parallel, the parallel value will be set
+        // to the default of 0, and will hit this branch. This makes it so that the time limit
+        // is enforced on each individual child process, instead of the main process
+        let time_handles = handles.clone();
+        tokio::spawn(async move { scan_manager::start_max_time_thread(time_handles).await });
+    }
+
+    // can't trace main until after logger is initialized and the above task is started
     log::trace!("enter: main");
-    log::debug!("{:#?}", *CONFIGURATION);
 
-    let save_output = !CONFIGURATION.output.is_empty(); // was -o used?
+    // spawn a thread that listens for keyboard input on stdin, when a user presses enter
+    // the input handler will toggle PAUSE_SCAN, which in turn is used to pause and resume
+    // scans that are already running
+    // also starts ctrl+c handler
+    TermInputHandler::initialize(handles.clone());
 
-    let (tx_term, tx_file, term_handle, file_handle) =
-        reporter::initialize(&CONFIGURATION.output, save_output);
+    if config.resumed {
+        let scanned_urls = handles.ferox_scans()?;
+        let from_here = config.resume_from.clone();
+
+        // populate FeroxScans object with previously seen scans
+        scanned_urls.add_serialized_scans(&from_here, handles.clone())?;
+
+        // populate Stats object with previously known statistics
+        handles.stats.send(LoadStats(from_here))?;
+    }
 
     // get targets from command line or stdin
-    let targets = match get_targets().await {
+    let targets = match get_targets(handles.clone()).await {
         Ok(t) => t,
         Err(e) => {
             // should only happen in the event that there was an error reading from stdin
-            log::error!("{}", e);
-            ferox_print(
-                &format!(
-                    "{} {} {}",
-                    status_colorizer("ERROR"),
-                    module_colorizer("main::get_targets"),
-                    e
-                ),
-                &PROGRESS_PRINTER,
-            );
-            process::exit(1);
+            clean_up(handles, tasks).await?;
+            bail!("Could not determine initial targets: {}", e);
         }
     };
 
-    if !CONFIGURATION.quiet {
-        // only print banner if -q isn't used
+    // --parallel branch
+    if config.parallel > 0 {
+        log::trace!("enter: parallel branch");
+
+        PARALLEL_LIMITER.add_permits(config.parallel);
+
+        let invocation = args();
+
+        let para_regex = Regex::new("--stdin").unwrap();
+
+        // remove stdin since only the original process will process targets
+        // remove quiet and silent so we can force silent later to normalize output
+        let mut original = invocation
+            .filter(|s| !para_regex.is_match(s))
+            .collect::<Vec<String>>();
+
+        // we need remove --parallel from command line so we don't hit this branch over and over
+        // but we must remove --parallel N manually; the filter above never sees --parallel and the
+        // value passed to it at the same time, so can't filter them out in one pass
+
+        // unwrap is fine, as it has to be in the args for us to be in this code branch
+        let parallel_index = original.iter().position(|s| *s == "--parallel").unwrap();
+
+        // remove --parallel
+        original.remove(parallel_index);
+
+        // remove N passed to --parallel (it's the same index again since everything shifts
+        // from removing --parallel)
+        original.remove(parallel_index);
+
+        // to log unique files to a shared folder, we need to first check for the presence
+        // of -o|--output.
+        let out_dir = if !config.output.is_empty() {
+            // -o|--output was used, so we'll attempt to create a directory to store the files
+            let output_path = Path::new(&handles.config.output);
+
+            // this only returns None if the path terminates in `..`. Since I don't want to
+            // hand-hold to that degree, we'll unwrap and fail if the output path ends in `..`
+            let base_name = output_path.file_name().unwrap();
+
+            let new_folder = slugify_filename(&base_name.to_string_lossy(), "", "logs");
+
+            let final_path = output_path.with_file_name(new_folder);
+
+            // create the directory or fail silently, assuming the reason for failure is that
+            // the path exists already
+            create_dir(&final_path).unwrap_or(());
+
+            final_path.to_string_lossy().to_string()
+        } else {
+            String::new()
+        };
+
+        // unvalidated targets fresh from stdin, just spawn children and let them do all checks
+        for target in targets {
+            // add the current target to the provided command
+            let mut cloned = original.clone();
+
+            if !out_dir.is_empty() {
+                // output directory value is not empty, need to join output directory with
+                // unique scan filename
+
+                // unwrap is ok, we already know -o was used
+                let out_idx = original
+                    .iter()
+                    .position(|s| *s == "--output" || *s == "-o")
+                    .unwrap();
+
+                let filename = slugify_filename(&target, "ferox", "log");
+
+                let full_path = Path::new(&out_dir)
+                    .join(filename)
+                    .to_string_lossy()
+                    .to_string();
+
+                // a +1 to the index is fine here, as clap has already validated that
+                // -o|--output has a value associated with it
+                cloned[out_idx + 1] = full_path;
+            }
+
+            cloned.push("-u".to_string());
+            cloned.push(target);
+
+            let bin = cloned.index(0).to_owned(); // user's path to feroxbuster
+            let args = cloned.index(1..).to_vec(); // and args
+
+            let permit = PARALLEL_LIMITER.acquire().await?;
+
+            log::debug!("parallel exec: {} {}", bin, args.join(" "));
+
+            tokio::task::spawn(async move {
+                let mut output = Command::new(bin)
+                    .args(&args)
+                    .stdout(Stdio::piped())
+                    .spawn()
+                    .expect("failed to spawn a child process");
+
+                let stdout = output.stdout.take().unwrap();
+
+                let mut bufread = BufReader::new(stdout);
+                // output for a single line is a minimum of 51 bytes, so we'll start with that
+                // + a little wiggle room, and grow as needed
+                let mut buf: String = String::with_capacity(128);
+
+                while let Ok(n) = bufread.read_line(&mut buf) {
+                    if n > 0 {
+                        let trimmed = buf.trim();
+                        if !trimmed.is_empty() {
+                            println!("{trimmed}");
+                        }
+                        buf.clear();
+                    } else {
+                        break;
+                    }
+                }
+                let _ = output.wait();
+                drop(permit);
+            });
+        }
+
+        // the output handler creates an empty file to which it will try to write, because
+        // this happens before we enter the --parallel branch, we need to remove that file
+        // if it's empty
+        let output = handles.config.output.to_owned();
+
+        clean_up(handles, tasks).await?;
+
+        let file = Path::new(&output);
+        if file.exists() {
+            // expectation is that this is always true for the first ferox process
+            if file.metadata()?.len() == 0 {
+                // empty file, attempt to remove it
+                remove_file(file)?;
+            }
+        }
+
+        log::trace!("exit: parallel branch && wrapped main");
+        return Ok(());
+    }
+
+    // in order for the Stats object to know about which targets are being scanned, we need to
+    // wait until the parallel branch has been handled before sending the UpdateTargets command
+    // this ensures that only the targets being scanned are sent to the Stats object
+    //
+    // if sent before the parallel branch is handled, the Stats object will have duplicate
+    // targets
+    handles.stats.send(UpdateTargets(targets.clone()))?;
+
+    if matches!(config.output_level, OutputLevel::Default) {
+        // only print banner if output level is default (no banner on --quiet|--silent)
         let std_stderr = stderr(); // std::io::stderr
-        banner::initialize(&targets, &CONFIGURATION, &VERSION, std_stderr).await;
+
+        let mut banner = Banner::new(&targets, &config);
+
+        // only interested in the side-effect that sets banner.update_status
+        let _ = banner.check_for_updates(UPDATE_URL, handles.clone()).await;
+
+        if banner.print_to(std_stderr, config.clone()).is_err() {
+            clean_up(handles, tasks).await?;
+            bail!(fmt_err("Could not print banner"));
+        }
+    }
+
+    {
+        let send_to_file = !config.output.is_empty();
+
+        // The TermOutHandler spawns a FileOutHandler, so errors in the FileOutHandler never bubble
+        // up due to the TermOutHandler never awaiting the result of FileOutHandler::start (that's
+        // done later here in main). sync checks that the tx/rx connection to the file handler works
+        if send_to_file && handles.output.sync(send_to_file).await.is_err() {
+            // output file specified and file handler could not initialize
+            clean_up(handles, tasks).await?;
+            let msg = format!("Couldn't start {} file handler", config.output);
+            bail!(fmt_err(&msg));
+        }
     }
 
     // discard non-responsive targets
-    let live_targets = heuristics::connectivity_test(&targets).await;
-
-    // kick off a scan against any targets determined to be responsive
-    match scan(live_targets, tx_term.clone(), tx_file.clone()).await {
-        Ok(_) => {
-            log::info!("All scans complete!");
+    let live_targets = {
+        let test = heuristics::HeuristicTests::new(handles.clone());
+        let result = test.connectivity(&targets).await;
+        if let Err(err) = result {
+            clean_up(handles, tasks).await?;
+            bail!(fmt_err(&err.to_string()));
         }
-        Err(e) => log::error!("An error occurred: {}", e),
+        result?
     };
 
-    // manually drop tx in order for the rx task's while loops to eval to false
-    drop(tx_term);
-    log::trace!("dropped terminal output handler's transmitter");
+    if live_targets.is_empty() {
+        clean_up(handles, tasks).await?;
+        bail!(fmt_err("Could not find any live targets to scan"));
+    }
 
-    log::trace!("awaiting terminal output handler's receiver");
-    // after dropping tx, we can await the future where rx lived
-    match term_handle.await {
+    // kick off a scan against any targets determined to be responsive
+    match scan(live_targets, handles.clone()).await {
         Ok(_) => {}
         Err(e) => {
-            log::error!("error awaiting terminal output handler's receiver: {}", e);
+            clean_up(handles, tasks).await?;
+            bail!(fmt_err(&format!("Failed while scanning: {e}")));
         }
     }
-    log::trace!("done awaiting terminal output handler's receiver");
 
-    log::trace!("tx_file: {:?}", tx_file);
-    // the same drop/await process used on the terminal handler is repeated for the file handler
-    // we drop the file transmitter every time, because it's created no matter what
-    drop(tx_file);
+    clean_up(handles, tasks).await?;
 
-    log::trace!("dropped file output handler's transmitter");
-    if save_output {
-        // but we only await if -o was specified
-        log::trace!("awaiting file output handler's receiver");
-        match file_handle.unwrap().await {
-            Ok(_) => {}
-            Err(e) => {
-                log::error!("error awaiting file output handler's receiver: {}", e);
+    log::trace!("exit: wrapped_main");
+    Ok(())
+}
+
+/// Single cleanup function that handles all the necessary drops/finishes etc required to gracefully
+/// shutdown the program
+async fn clean_up(handles: Arc<Handles>, tasks: Tasks) -> Result<()> {
+    log::trace!("enter: clean_up({handles:?}, {tasks:?})");
+
+    let (tx, rx) = oneshot::channel::<bool>();
+    handles.send_scan_command(JoinTasks(tx))?;
+    rx.await?;
+
+    log::info!("All scans complete!");
+
+    // terminal handler closes file handler if one is in use
+    handles.output.send(Exit)?;
+    tasks.terminal.await??;
+    log::trace!("terminal handler closed");
+
+    handles.filters.send(Exit)?;
+    tasks.filters.await??;
+    log::trace!("filters handler closed");
+
+    handles.stats.send(Exit)?;
+    tasks.stats.await??;
+    log::trace!("stats handler closed");
+
+    // mark all scans complete so the terminal input handler will exit cleanly
+    SCAN_COMPLETE.store(true, Ordering::Relaxed);
+
+    // clean-up function for the MultiProgress bar; must be called last in order to still see
+    // the final trace messages above
+    PROGRESS_PRINTER.finish();
+
+    log::trace!("exit: clean_up");
+    Ok(())
+}
+
+async fn update_app() -> Result<self_update::Status, Box<dyn ::std::error::Error>> {
+    let target_os = format!("{ARCH}-{OS}");
+    let status = tokio::task::spawn_blocking(move || {
+        self_update::backends::github::Update::configure()
+            .repo_owner("epi052")
+            .repo_name("feroxbuster")
+            .bin_name("feroxbuster")
+            .target(target_os.as_str())
+            .show_download_progress(true)
+            .current_version(cargo_crate_version!())
+            .build()?
+            .update()
+    })
+    .await??;
+
+    Ok(status)
+}
+
+fn main() -> Result<()> {
+    let config = Arc::new(Configuration::new().with_context(|| "Could not create Configuration")?);
+
+    // setup logging based on the number of -v's used
+    if matches!(
+        config.output_level,
+        OutputLevel::Default | OutputLevel::Quiet
+    ) {
+        // don't log on --silent
+        logger::initialize(config.clone())?;
+    }
+
+    // this function uses rlimit, which is not supported on windows
+    #[cfg(not(target_os = "windows"))]
+    set_open_file_limit(DEFAULT_OPEN_FILE_LIMIT);
+
+    if let Ok(runtime) = tokio::runtime::Builder::new_multi_thread()
+        .enable_all()
+        .build()
+    {
+        let future = wrapped_main(config.clone());
+        if let Err(e) = runtime.block_on(future) {
+            eprintln!("{e}");
+
+            // the code below is to facilitate testing tests/test_banner entries. Since it's an
+            // integration test, normal test detection (cfg!(test), etc...) won't work. So, in
+            // the tests themselves, we pass
+            // `--wordlist /definitely/doesnt/exist/0cd7fed0-47f4-4b18-a1b0-ac39708c1676`
+            // and look for that here to print the banner.
+            //
+            // this change became a necessity once we moved wordlist parsing out of `scan` and into
+            // `wrapped_main`.
+            if e.to_string()
+                .contains("/definitely/doesnt/exist/0cd7fed0-47f4-4b18-a1b0-ac39708c1676")
+            {
+                // support the handful of tests that use `--stdin`
+                let targets: Vec<_> = if config.cached_stdin.is_empty() {
+                    vec!["http://localhost".to_string()]
+                } else {
+                    config.cached_stdin.clone()
+                };
+
+                // print the banner to stderr
+                let std_stderr = stderr(); // std::io::stderr
+                let banner = Banner::new(&targets, &config);
+                if (!config.quiet && !config.silent) || config.parallel != 0 {
+                    banner.print_to(std_stderr, config).unwrap();
+                }
             }
-        }
-        log::trace!("done awaiting file output handler's receiver");
+
+            // if we've encountered an error before clean_up can be called (i.e. a wordlist error)
+            // we need to at least spin-down the progress bar
+            PROGRESS_PRINTER.finish();
+        };
     }
 
     log::trace!("exit: main");
 
-    // clean-up function for the MultiProgress bar; must be called last in order to still see
-    // the final trace message above
-    PROGRESS_PRINTER.finish();
+    Ok(())
 }

src/message.rs

@@ -0,0 +1,148 @@
+use anyhow::Context;
+use console::{style, Color};
+use serde::{Deserialize, Serialize};
+
+use crate::traits::FeroxSerialize;
+use crate::utils::fmt_err;
+
+#[derive(Serialize, Deserialize, Default, Debug)]
+/// Representation of a log entry, can be represented as a human readable string or JSON
+pub struct FeroxMessage {
+    #[serde(rename = "type")]
+    /// Name of this type of struct, used for serialization, i.e. `{"type":"log"}`
+    pub(crate) kind: String,
+
+    /// The log message
+    pub(crate) message: String,
+
+    /// The log level
+    pub(crate) level: String,
+
+    /// The number of seconds elapsed since the scan started
+    pub(crate) time_offset: f32,
+
+    /// The module from which log::* was called
+    pub(crate) module: String,
+}
+
+/// Implementation of FeroxMessage
+impl FeroxSerialize for FeroxMessage {
+    /// Create a string representation of the log message
+    ///
+    /// ex:  301       10l       16w      173c https://localhost/api
+    fn as_str(&self) -> String {
+        let (level_name, level_color) = match self.level.as_str() {
+            "ERROR" => ("ERR", Color::Red),
+            "WARN" => ("WRN", Color::Red),
+            "INFO" => ("INF", Color::Cyan),
+            "DEBUG" => ("DBG", Color::Yellow),
+            "TRACE" => ("TRC", Color::Magenta),
+            "WILDCARD" => ("WLD", Color::Cyan),
+            _ => ("MSG", Color::White),
+        };
+
+        format!(
+            "{} {:10.03} {} {}\n",
+            style(level_name).bg(level_color).black(),
+            style(self.time_offset).dim(),
+            self.module,
+            style(&self.message).dim(),
+        )
+    }
+
+    /// Create an NDJSON representation of the log message
+    ///
+    /// (expanded for clarity)
+    /// ex:
+    /// {
+    ///   "type": "log",
+    ///   "message": "Sent https://localhost/api to file handler",
+    ///   "level": "DEBUG",
+    ///   "time_offset": 0.86333454,
+    ///   "module": "feroxbuster::reporter"
+    /// }\n
+    fn as_json(&self) -> anyhow::Result<String> {
+        let mut json = serde_json::to_string(&self).with_context(|| {
+            fmt_err(&format!(
+                "Could not convert {}:{} to JSON",
+                self.level, self.message
+            ))
+        })?;
+        json.push('\n');
+        Ok(json)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    /// test as_str method of FeroxMessage
+    fn ferox_message_as_str_returns_string_with_newline() {
+        let message = FeroxMessage {
+            message: "message".to_string(),
+            module: "utils".to_string(),
+            time_offset: 1.0,
+            level: "INFO".to_string(),
+            kind: "log".to_string(),
+        };
+        let message_str = message.as_str();
+
+        assert!(message_str.contains("INF"));
+        assert!(message_str.contains("1.000"));
+        assert!(message_str.contains("utils"));
+        assert!(message_str.contains("message"));
+        assert!(message_str.ends_with('\n'));
+    }
+
+    #[test]
+    /// test as_json method of FeroxMessage
+    fn ferox_message_as_json_returns_json_representation_of_ferox_message_with_newline() {
+        let message = FeroxMessage {
+            message: "message".to_string(),
+            module: "utils".to_string(),
+            time_offset: 1.0,
+            level: "INFO".to_string(),
+            kind: "log".to_string(),
+        };
+
+        let message_str = message.as_json().unwrap();
+
+        let error_margin = f32::EPSILON;
+
+        let json: FeroxMessage = serde_json::from_str(&message_str).unwrap();
+        assert_eq!(json.module, message.module);
+        assert_eq!(json.message, message.message);
+        assert!((json.time_offset - message.time_offset).abs() < error_margin);
+        assert_eq!(json.level, message.level);
+        assert_eq!(json.kind, message.kind);
+    }
+
+    #[test]
+    /// test defaults for coverage
+    fn message_defaults() {
+        let msg = FeroxMessage::default();
+        assert_eq!(msg.level, String::new());
+        assert_eq!(msg.kind, String::new());
+        assert_eq!(msg.message, String::new());
+        assert_eq!(msg.module, String::new());
+        assert!(msg.time_offset < 0.1);
+    }
+
+    #[test]
+    /// ensure WILDCARD messages serialize to WLD and anything not known to UNK
+    fn message_as_str_edges() {
+        let mut msg = FeroxMessage {
+            message: "message".to_string(),
+            module: "utils".to_string(),
+            time_offset: 1.0,
+            level: "WILDCARD".to_string(),
+            kind: "log".to_string(),
+        };
+        assert!(console::strip_ansi_codes(&msg.as_str()).starts_with("WLD"));
+
+        msg.level = "UNKNOWN".to_string();
+        assert!(console::strip_ansi_codes(&msg.as_str()).starts_with("MSG"));
+    }
+}