updated deps

Add redirect messages to normal reports

.all-contributorsrc

@@ -0,0 +1,368 @@
+{
+  "files": [
+    "README.md"
+  ],
+  "imageSize": 100,
+  "commit": false,
+  "contributors": [
+    {
+      "login": "joohoi",
+      "name": "Joona Hoikkala",
+      "avatar_url": "https://avatars.githubusercontent.com/u/5235109?v=4",
+      "profile": "https://io.fi",
+      "contributions": [
+        "doc"
+      ]
+    },
+    {
+      "login": "jsav0",
+      "name": "J Savage",
+      "avatar_url": "https://avatars.githubusercontent.com/u/20546041?v=4",
+      "profile": "https://github.com/jsav0",
+      "contributions": [
+        "infra",
+        "doc"
+      ]
+    },
+    {
+      "login": "TGotwig",
+      "name": "Thomas Gotwig",
+      "avatar_url": "https://avatars.githubusercontent.com/u/30773779?v=4",
+      "profile": "http://www.tgotwig.dev",
+      "contributions": [
+        "infra",
+        "doc"
+      ]
+    },
+    {
+      "login": "spikecodes",
+      "name": "Spike",
+      "avatar_url": "https://avatars.githubusercontent.com/u/19519553?v=4",
+      "profile": "https://github.com/spikecodes",
+      "contributions": [
+        "infra",
+        "doc"
+      ]
+    },
+    {
+      "login": "evanrichter",
+      "name": "Evan Richter",
+      "avatar_url": "https://avatars.githubusercontent.com/u/330292?v=4",
+      "profile": "https://github.com/evanrichter",
+      "contributions": [
+        "code",
+        "doc"
+      ]
+    },
+    {
+      "login": "mzpqnxow",
+      "name": "AG",
+      "avatar_url": "https://avatars.githubusercontent.com/u/8016228?v=4",
+      "profile": "https://github.com/mzpqnxow",
+      "contributions": [
+        "ideas",
+        "doc"
+      ]
+    },
+    {
+      "login": "n-thumann",
+      "name": "Nicolas Thumann",
+      "avatar_url": "https://avatars.githubusercontent.com/u/46975855?v=4",
+      "profile": "https://n-thumann.de/",
+      "contributions": [
+        "code",
+        "doc"
+      ]
+    },
+    {
+      "login": "tomtastic",
+      "name": "Tom Matthews",
+      "avatar_url": "https://avatars.githubusercontent.com/u/302127?v=4",
+      "profile": "https://github.com/tomtastic",
+      "contributions": [
+        "doc"
+      ]
+    },
+    {
+      "login": "bsysop",
+      "name": "bsysop",
+      "avatar_url": "https://avatars.githubusercontent.com/u/9998303?v=4",
+      "profile": "https://github.com/bsysop",
+      "contributions": [
+        "doc"
+      ]
+    },
+    {
+      "login": "bpsizemore",
+      "name": "Brian Sizemore",
+      "avatar_url": "https://avatars.githubusercontent.com/u/11645898?v=4",
+      "profile": "http://bpsizemore.me",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "noraj",
+      "name": "Alexandre ZANNI",
+      "avatar_url": "https://avatars.githubusercontent.com/u/16578570?v=4",
+      "profile": "https://pwn.by/noraj",
+      "contributions": [
+        "infra",
+        "doc"
+      ]
+    },
+    {
+      "login": "craig",
+      "name": "Craig",
+      "avatar_url": "https://avatars.githubusercontent.com/u/99729?v=4",
+      "profile": "https://github.com/craig",
+      "contributions": [
+        "infra"
+      ]
+    },
+    {
+      "login": "EONRaider",
+      "name": "EONRaider",
+      "avatar_url": "https://avatars.githubusercontent.com/u/15611424?v=4",
+      "profile": "https://www.reddit.com/u/EONRaider",
+      "contributions": [
+        "infra"
+      ]
+    },
+    {
+      "login": "wtwver",
+      "name": "wtwver",
+      "avatar_url": "https://avatars.githubusercontent.com/u/53866088?v=4",
+      "profile": "https://github.com/wtwver",
+      "contributions": [
+        "infra"
+      ]
+    },
+    {
+      "login": "Tib3rius",
+      "name": "Tib3rius",
+      "avatar_url": "https://avatars.githubusercontent.com/u/48113936?v=4",
+      "profile": "https://tib3rius.com",
+      "contributions": [
+        "bug"
+      ]
+    },
+    {
+      "login": "0xdf",
+      "name": "0xdf",
+      "avatar_url": "https://avatars.githubusercontent.com/u/1489045?v=4",
+      "profile": "https://github.com/0xdf",
+      "contributions": [
+        "bug"
+      ]
+    },
+    {
+      "login": "secure-77",
+      "name": "secure-77",
+      "avatar_url": "https://avatars.githubusercontent.com/u/31564517?v=4",
+      "profile": "http://secure77.de",
+      "contributions": [
+        "bug"
+      ]
+    },
+    {
+      "login": "sbrun",
+      "name": "Sophie Brun",
+      "avatar_url": "https://avatars.githubusercontent.com/u/7712154?v=4",
+      "profile": "https://github.com/sbrun",
+      "contributions": [
+        "infra"
+      ]
+    },
+    {
+      "login": "black-A",
+      "name": "black-A",
+      "avatar_url": "https://avatars.githubusercontent.com/u/30686803?v=4",
+      "profile": "https://github.com/black-A",
+      "contributions": [
+        "ideas"
+      ]
+    },
+    {
+      "login": "dinosn",
+      "name": "Nicolas Krassas",
+      "avatar_url": "https://avatars.githubusercontent.com/u/3851678?v=4",
+      "profile": "https://github.com/dinosn",
+      "contributions": [
+        "ideas"
+      ]
+    },
+    {
+      "login": "N0ur5",
+      "name": "N0ur5",
+      "avatar_url": "https://avatars.githubusercontent.com/u/24260009?v=4",
+      "profile": "https://github.com/N0ur5",
+      "contributions": [
+        "ideas"
+      ]
+    },
+    {
+      "login": "moscowchill",
+      "name": "mchill",
+      "avatar_url": "https://avatars.githubusercontent.com/u/72578879?v=4",
+      "profile": "https://github.com/moscowchill",
+      "contributions": [
+        "bug"
+      ]
+    },
+    {
+      "login": "BitThr3at",
+      "name": "Naman",
+      "avatar_url": "https://avatars.githubusercontent.com/u/45028933?v=4",
+      "profile": "http://BitThr3at.github.io",
+      "contributions": [
+        "bug"
+      ]
+    },
+    {
+      "login": "sicks3c",
+      "name": "Ayoub Elaich",
+      "avatar_url": "https://avatars.githubusercontent.com/u/32225186?v=4",
+      "profile": "https://github.com/Sicks3c",
+      "contributions": [
+        "bug"
+      ]
+    },
+    {
+      "login": "HenryHoggard",
+      "name": "Henry",
+      "avatar_url": "https://avatars.githubusercontent.com/u/1208121?v=4",
+      "profile": "https://github.com/HenryHoggard",
+      "contributions": [
+        "bug"
+      ]
+    },
+    {
+      "login": "SleepiPanda",
+      "name": "SleepiPanda",
+      "avatar_url": "https://avatars.githubusercontent.com/u/6428561?v=4",
+      "profile": "https://github.com/SleepiPanda",
+      "contributions": [
+        "bug"
+      ]
+    },
+    {
+      "login": "uBadRequest",
+      "name": "Bad Requests",
+      "avatar_url": "https://avatars.githubusercontent.com/u/47282747?v=4",
+      "profile": "https://github.com/uBadRequest",
+      "contributions": [
+        "bug"
+      ]
+    },
+    {
+      "login": "dnaka91",
+      "name": "Dominik Nakamura",
+      "avatar_url": "https://avatars.githubusercontent.com/u/36804488?v=4",
+      "profile": "https://home.dnaka91.rocks",
+      "contributions": [
+        "infra"
+      ]
+    },
+    {
+      "login": "hunter0x8",
+      "name": "Muhammad Ahsan",
+      "avatar_url": "https://avatars.githubusercontent.com/u/46222314?v=4",
+      "profile": "https://github.com/hunter0x8",
+      "contributions": [
+        "bug"
+      ]
+    },
+    {
+      "login": "cortantief",
+      "name": "cortantief",
+      "avatar_url": "https://avatars.githubusercontent.com/u/34527333?v=4",
+      "profile": "https://github.com/cortantief",
+      "contributions": [
+        "bug",
+        "code"
+      ]
+    },
+    {
+      "login": "dsaxton",
+      "name": "Daniel Saxton",
+      "avatar_url": "https://avatars.githubusercontent.com/u/2658661?v=4",
+      "profile": "https://github.com/dsaxton",
+      "contributions": [
+        "ideas",
+        "code"
+      ]
+    },
+    {
+      "login": "narkopolo",
+      "name": "narkopolo",
+      "avatar_url": "https://avatars.githubusercontent.com/u/16690056?v=4",
+      "profile": "https://github.com/narkopolo",
+      "contributions": [
+        "ideas"
+      ]
+    },
+    {
+      "login": "justinsteven",
+      "name": "Justin Steven",
+      "avatar_url": "https://avatars.githubusercontent.com/u/1893909?v=4",
+      "profile": "https://ring0.lol",
+      "contributions": [
+        "ideas"
+      ]
+    },
+    {
+      "login": "7047payloads",
+      "name": "7047payloads",
+      "avatar_url": "https://avatars.githubusercontent.com/u/95562424?v=4",
+      "profile": "https://github.com/7047payloads",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "unkn0wnsyst3m",
+      "name": "unkn0wnsyst3m",
+      "avatar_url": "https://avatars.githubusercontent.com/u/21272239?v=4",
+      "profile": "https://github.com/unkn0wnsyst3m",
+      "contributions": [
+        "ideas"
+      ]
+    },
+    {
+      "login": "its0x08",
+      "name": "0x08",
+      "avatar_url": "https://avatars.githubusercontent.com/u/15280042?v=4",
+      "profile": "https://ironwort.me/",
+      "contributions": [
+        "ideas"
+      ]
+    },
+    {
+      "login": "MD-Levitan",
+      "name": "kusok",
+      "avatar_url": "https://avatars.githubusercontent.com/u/12116508?v=4",
+      "profile": "https://github.com/MD-Levitan",
+      "contributions": [
+        "ideas",
+        "code"
+      ]
+    },
+    {
+      "login": "godylockz",
+      "name": "godylockz",
+      "avatar_url": "https://avatars.githubusercontent.com/u/81207744?v=4",
+      "profile": "https://github.com/godylockz",
+      "contributions": [
+        "ideas",
+        "code"
+      ]
+    }
+  ],
+  "contributorsPerLine": 7,
+  "projectName": "feroxbuster",
+  "projectOwner": "epi052",
+  "repoType": "github",
+  "repoHost": "https://github.com",
+  "skipCi": true
+}

.github/pull_request_template.md

@@ -16,7 +16,11 @@ Long form explanations of most of the items below can be found in the [CONTRIBUT
 
 ## Documentation
 - [ ] New code is documented using [doc comments](https://doc.rust-lang.org/stable/rust-by-example/meta/doc.html)
-- [ ] Documentation about your PR is included in the README, as needed
+- [ ] Documentation about your PR is included in the `docs`, as needed. The docs live in a [separate repository](https://epi052.github.io/feroxbuster-docs/docs/). Update the appropriate pages at the links below.
+  - [ ] update [example config file section](https://epi052.github.io/feroxbuster-docs/docs/configuration/ferox-config-toml/)
+  - [ ] update [help output section](https://epi052.github.io/feroxbuster-docs/docs/configuration/command-line/)
+  - [ ] add an [example](https://epi052.github.io/feroxbuster-docs/docs/examples/)
+  - [ ] update [comparison table](https://epi052.github.io/feroxbuster-docs/docs/compare/)
 
 ## Additional Tests
 - [ ] New code is unit tested

.github/stale.yml

@@ -6,6 +6,7 @@ daysUntilClose: 7
 exemptLabels:
   - pinned
   - security
+  - confirmed
 # Label to use when marking an issue as stale
 staleLabel: stale
 # Comment to post when marking an issue as stale. Set to `false` to disable

.github/workflows/cicd-to-dockerhub.yml

@@ -0,0 +1,34 @@
+name: ci-to-dockerhub
+
+on:
+  push:
+    branches: [ main ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build and push
+        id: docker_build
+        uses: docker/build-push-action@v2
+        with:
+          context: ./
+          file: ./Dockerfile
+          push: true
+          tags: ${{ secrets.DOCKER_HUB_USERNAME }}/feroxbuster:latest
+
+      - name: Image digest
+        run: echo ${{ steps.docker_build.outputs.digest }}

.gitignore

@@ -9,6 +9,9 @@ target/
 # jetbrains metadata folder
 .idea/
 
+# vscode metadata folder
+.vscode/
+
 # personal feroxbuster config for testing
 ferox-config.toml
 
@@ -23,7 +26,7 @@ lcov_cobertura.py
 .dockerignore
 
 # state file created during tests
-ferox-http*
+ferox-*.state
 
 # python stuff cuz reasons
 Pipfile*

CONTRIBUTING.md

@@ -166,7 +166,7 @@ primarily related to continuous integration and release deployment.
 
 feroxbuster uses the [`clippy`](https://rust-lang.github.io/rust-clippy/) code linter.
 
-The command that will ultimately be used in the CI pipeline for linting is `cargo clippy --all-targets --all-features -- -D warnings -A clippy::unnecessary_unwrap`.  
+The command that will ultimately be used in the CI pipeline for linting is `cargo clippy --all-targets --all-features -- -D warnings -A clippy::mutex-atomic`.
 
 Before submitting a Pull Request, the above command should be run. Please do not ignore any linting errors in code you write or modify, as they are meant to **help** by ensuring a clean and simple code base.
 

Cargo.toml

@@ -1,9 +1,9 @@
 [package]
 name = "feroxbuster"
-version = "2.3.2"
-authors = ["Ben 'epi' Risher <epibar052@gmail.com>"]
+version = "2.5.0"
+authors = ["Ben 'epi' Risher (@epi052)"]
 license = "MIT"
-edition = "2018"
+edition = "2021"
 homepage = "https://github.com/epi052/feroxbuster"
 repository = "https://github.com/epi052/feroxbuster"
 description = "A fast, simple, recursive content discovery tool."
@@ -16,41 +16,45 @@ build = "build.rs"
 maintenance = { status = "actively-developed" }
 
 [build-dependencies]
-clap = "2.33"
+clap = {version = "3.0", features = ["cargo"]}
+clap_complete = "3.0"
 regex = "1"
 lazy_static = "1.4"
-dirs = "3.0"
+dirs = "4.0"
 
 [dependencies]
+scraper = "0.12"
 futures = { version = "0.3"}
-tokio = { version = "1.9", features = ["full"] }
-tokio-util = {version = "0.6.6", features = ["codec"]}
+tokio = { version = "1.15", features = ["full"] }
+tokio-util = {version = "0.6", features = ["codec"]}
 log = "0.4"
 env_logger = "0.9"
 reqwest = { version = "0.11", features = ["socks"] }
-clap = "2.33"
+url = { version = "2.2", features = ["serde"]}  # uses feature unification to add 'serde' to reqwest::Url
+serde_regex = "1.1"
+clap = {version = "3.0", features = ["wrap_help", "cargo"]}
 lazy_static = "1.4"
 toml = "0.5"
 serde = { version = "1.0", features = ["derive", "rc"] }
 serde_json = "1.0"
 uuid = { version = "0.8", features = ["v4"] }
 indicatif = "0.15"
-console = "0.14"
+console = "0.15"
 openssl = { version = "0.10", features = ["vendored"] }
-dirs = "3.0"
+dirs = "4.0"
 regex = "1"
 crossterm = "0.20"
 rlimit = "0.6"
-ctrlc = "3.1.9"
+ctrlc = "3.2"
 fuzzyhash = "0.2.1"
 anyhow = "1.0"
 leaky-bucket = "0.10.0"
 
 [dev-dependencies]
-tempfile = "3.1"
-httpmock = "0.5.8"
-assert_cmd = "1.0"
-predicates = "2.0"
+tempfile = "3.3"
+httpmock = "0.6"
+assert_cmd = "2.0"
+predicates = "2.1"
 
 [profile.release]
 lto = true

Dockerfile

@@ -1,14 +1,28 @@
-FROM alpine:latest
+# Image: alpine:3.14.2
+FROM alpine@sha256:69704ef328d05a9f806b6b8502915e6a0a4faa4d72018dc42343f511490daf8a as build
 LABEL maintainer="wfnintr@null.net"
 
-RUN sed -i -e 's/v[[:digit:]]\..*\//edge\//g' /etc/apk/repositories && apk upgrade --update-cache --available
+RUN sed -i -e 's/v[[:digit:]]\..*\//edge\//g' /etc/apk/repositories \
+    && apk upgrade --update-cache --available && apk add --update openssl
 
-# download default wordlists 
-RUN apk add --no-cache --virtual .depends subversion font-noto-emoji && \
-	svn export https://github.com/danielmiessler/SecLists/trunk/Discovery/Web-Content /usr/share/seclists/Discovery/Web-Content && \
-	apk del .depends
 
-# install latest release
-RUN wget https://github.com/epi052/feroxbuster/releases/latest/download/x86_64-linux-feroxbuster.zip -qO feroxbuster.zip && unzip -d /usr/local/bin/ feroxbuster.zip feroxbuster && rm feroxbuster.zip && chmod +x /usr/local/bin/feroxbuster
+# Download latest release
+RUN wget https://github.com/epi052/feroxbuster/releases/latest/download/x86_64-linux-feroxbuster.zip -qO feroxbuster.zip \
+    && unzip -d /tmp/ feroxbuster.zip feroxbuster \
+    && chmod +x /tmp/feroxbuster \
+    && wget https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/Web-Content/raft-medium-directories.txt -O /tmp/raft-medium-directories.txt
+
+# Image: alpine:3.14.2
+FROM alpine@sha256:69704ef328d05a9f806b6b8502915e6a0a4faa4d72018dc42343f511490daf8a as release
+
+COPY --from=build /tmp/raft-medium-directories.txt /usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt
+COPY --from=build /tmp/feroxbuster /usr/local/bin/feroxbuster
+
+RUN adduser \
+    --gecos "" \
+    --disabled-password \
+    feroxbuster
+
+USER feroxbuster
 
 ENTRYPOINT ["feroxbuster"]

build.rs

@@ -1,9 +1,7 @@
 use std::fs::{copy, create_dir_all, OpenOptions};
 use std::io::{Read, Seek, SeekFrom, Write};
-extern crate clap;
-extern crate dirs;
 
-use clap::Shell;
+use clap_complete::{generate_to, shells};
 
 include!("src/parser.rs");
 
@@ -18,11 +16,11 @@ fn main() {
 
     let mut app = initialize();
 
-    let shells: [Shell; 4] = [Shell::Bash, Shell::Fish, Shell::Zsh, Shell::PowerShell];
-
-    for shell in &shells {
-        app.gen_completions("feroxbuster", *shell, outdir);
-    }
+    generate_to(shells::Bash, &mut app, "feroxbuster", outdir).unwrap();
+    generate_to(shells::Zsh, &mut app, "feroxbuster", outdir).unwrap();
+    generate_to(shells::Zsh, &mut app, "feroxbuster", outdir).unwrap();
+    generate_to(shells::PowerShell, &mut app, "feroxbuster", outdir).unwrap();
+    generate_to(shells::Elvish, &mut app, "feroxbuster", outdir).unwrap();
 
     // 0xdf pointed out an oddity when tab-completing options that expect file paths, the fix we
     // landed on was to add -o plusdirs to the bash completion script. The following code aims to

docs/index.html

@@ -0,0 +1,11 @@
+<html>
+<head>
+   <meta http-equiv="refresh"
+   content="0; url=https://epi052.github.io/feroxbuster-docs/">
+</head>
+<body>
+   <p>The page has moved to:
+   <a href="https://epi052.github.io/feroxbuster-docs/">feroxbuster-docs</a></p>
+</body>
+</html>
+

ferox-config.toml.example

@@ -27,10 +27,14 @@
 # output = "/targets/ellingson_mineral_company/gibson.txt"
 # debug_log = "/var/log/find-the-derp.log"
 # user_agent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0"
+# random_agent = false
 # redirects = true
 # insecure = true
 # extensions = ["php", "html"]
+# methods = ["GET", "POST"]
+# data = [11, 12, 13, 14, 15]
 # url_denylist = ["http://dont-scan.me", "https://also-not.me"]
+# regex_denylist = ["/deny.*"]
 # no_recursion = true
 # add_slash = true
 # stdin = true

shell_completions/_feroxbuster

@@ -15,88 +15,92 @@ _feroxbuster() {
 
     local context curcontext="$curcontext" state line
     _arguments "${_arguments_options[@]}" \
-'-w+[Path to the wordlist]' \
-'--wordlist=[Path to the wordlist]' \
-'*-u+[The target URL(s) (required, unless --stdin used)]' \
-'*--url=[The target URL(s) (required, unless --stdin used)]' \
-'-t+[Number of concurrent threads (default: 50)]' \
-'--threads=[Number of concurrent threads (default: 50)]' \
-'-d+[Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)]' \
-'--depth=[Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)]' \
-'-T+[Number of seconds before a request times out (default: 7)]' \
-'--timeout=[Number of seconds before a request times out (default: 7)]' \
-'-p+[Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)]' \
-'--proxy=[Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)]' \
-'-P+[Send only unfiltered requests through a Replay Proxy, instead of all requests]' \
-'--replay-proxy=[Send only unfiltered requests through a Replay Proxy, instead of all requests]' \
-'*-R+[Status Codes to send through a Replay Proxy when found (default: --status-codes value)]' \
-'*--replay-codes=[Status Codes to send through a Replay Proxy when found (default: --status-codes value)]' \
-'*-s+[Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)]' \
-'*--status-codes=[Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)]' \
-'-o+[Output file to write results to (use w/ --json for JSON entries)]' \
-'--output=[Output file to write results to (use w/ --json for JSON entries)]' \
-'(-u --url)--resume-from=[State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)]' \
-'--debug-log=[Output file to write log entries (use w/ --json for JSON entries)]' \
-'-a+[Sets the User-Agent (default: feroxbuster/VERSION)]' \
-'--user-agent=[Sets the User-Agent (default: feroxbuster/VERSION)]' \
-'*-x+[File extension(s) to search for (ex: -x php -x pdf js)]' \
-'*--extensions=[File extension(s) to search for (ex: -x php -x pdf js)]' \
-'*--dont-scan=[URL(s) to exclude from recursion/scans]' \
-'*-H+[Specify HTTP headers (ex: -H Header:val '\''stuff: things'\'')]' \
-'*--headers=[Specify HTTP headers (ex: -H Header:val '\''stuff: things'\'')]' \
-'*-Q+[Specify URL query parameters (ex: -Q token=stuff -Q secret=key)]' \
-'*--query=[Specify URL query parameters (ex: -Q token=stuff -Q secret=key)]' \
-'*-S+[Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)]' \
-'*--filter-size=[Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)]' \
-'*-X+[Filter out messages via regular expression matching on the response'\''s body (ex: -X '\''^ignore me$'\'')]' \
-'*--filter-regex=[Filter out messages via regular expression matching on the response'\''s body (ex: -X '\''^ignore me$'\'')]' \
-'*-W+[Filter out messages of a particular word count (ex: -W 312 -W 91,82)]' \
-'*--filter-words=[Filter out messages of a particular word count (ex: -W 312 -W 91,82)]' \
-'*-N+[Filter out messages of a particular line count (ex: -N 20 -N 31,30)]' \
-'*--filter-lines=[Filter out messages of a particular line count (ex: -N 20 -N 31,30)]' \
-'*-C+[Filter out status codes (deny list) (ex: -C 200 -C 401)]' \
-'*--filter-status=[Filter out status codes (deny list) (ex: -C 200 -C 401)]' \
-'*--filter-similar-to=[Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)]' \
-'-L+[Limit total number of concurrent scans (default: 0, i.e. no limit)]' \
-'--scan-limit=[Limit total number of concurrent scans (default: 0, i.e. no limit)]' \
-'--parallel=[Run parallel feroxbuster instances (one child process per url passed via stdin)]' \
-'(--auto-tune)--rate-limit=[Limit number of requests per second (per directory) (default: 0, i.e. no limit)]' \
-'--time-limit=[Limit total run time of all scans (ex: --time-limit 10m)]' \
+'-u+[The target URL (required, unless \[--stdin || --resume-from\] used)]:URL:_urls' \
+'--url=[The target URL (required, unless \[--stdin || --resume-from\] used)]:URL:_urls' \
+'(-u --url)--resume-from=[State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)]:STATE_FILE:_files' \
+'-p+[Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)]:PROXY:_urls' \
+'--proxy=[Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)]:PROXY:_urls' \
+'-P+[Send only unfiltered requests through a Replay Proxy, instead of all requests]:REPLAY_PROXY:_urls' \
+'--replay-proxy=[Send only unfiltered requests through a Replay Proxy, instead of all requests]:REPLAY_PROXY:_urls' \
+'*-R+[Status Codes to send through a Replay Proxy when found (default: --status-codes value)]:REPLAY_CODE: ' \
+'*--replay-codes=[Status Codes to send through a Replay Proxy when found (default: --status-codes value)]:REPLAY_CODE: ' \
+'-a+[Sets the User-Agent (default: feroxbuster/2.5.0)]:USER_AGENT: ' \
+'--user-agent=[Sets the User-Agent (default: feroxbuster/2.5.0)]:USER_AGENT: ' \
+'*-x+[File extension(s) to search for (ex: -x php -x pdf js)]:FILE_EXTENSION: ' \
+'*--extensions=[File extension(s) to search for (ex: -x php -x pdf js)]:FILE_EXTENSION: ' \
+'*-m+[Which HTTP request method(s) should be sent (default: GET)]:HTTP_METHODS: ' \
+'*--methods=[Which HTTP request method(s) should be sent (default: GET)]:HTTP_METHODS: ' \
+'--data=[Request'\''s Body; can read data from a file if input starts with an @ (ex: @post.bin)]:DATA: ' \
+'*-H+[Specify HTTP headers to be used in each request (ex: -H Header:val -H '\''stuff: things'\'')]:HEADER: ' \
+'*--headers=[Specify HTTP headers to be used in each request (ex: -H Header:val -H '\''stuff: things'\'')]:HEADER: ' \
+'*-b+[Specify HTTP cookies to be used in each request (ex: -b stuff=things)]:COOKIE: ' \
+'*--cookies=[Specify HTTP cookies to be used in each request (ex: -b stuff=things)]:COOKIE: ' \
+'*-Q+[Request'\''s URL query parameters (ex: -Q token=stuff -Q secret=key)]:QUERY: ' \
+'*--query=[Request'\''s URL query parameters (ex: -Q token=stuff -Q secret=key)]:QUERY: ' \
+'*--dont-scan=[URL(s) or Regex Pattern(s) to exclude from recursion/scans]:URL: ' \
+'*-S+[Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)]:SIZE: ' \
+'*--filter-size=[Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)]:SIZE: ' \
+'*-X+[Filter out messages via regular expression matching on the response'\''s body (ex: -X '\''^ignore me$'\'')]:REGEX: ' \
+'*--filter-regex=[Filter out messages via regular expression matching on the response'\''s body (ex: -X '\''^ignore me$'\'')]:REGEX: ' \
+'*-W+[Filter out messages of a particular word count (ex: -W 312 -W 91,82)]:WORDS: ' \
+'*--filter-words=[Filter out messages of a particular word count (ex: -W 312 -W 91,82)]:WORDS: ' \
+'*-N+[Filter out messages of a particular line count (ex: -N 20 -N 31,30)]:LINES: ' \
+'*--filter-lines=[Filter out messages of a particular line count (ex: -N 20 -N 31,30)]:LINES: ' \
+'*-C+[Filter out status codes (deny list) (ex: -C 200 -C 401)]:STATUS_CODE: ' \
+'*--filter-status=[Filter out status codes (deny list) (ex: -C 200 -C 401)]:STATUS_CODE: ' \
+'*--filter-similar-to=[Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)]:UNWANTED_PAGE:_urls' \
+'*-s+[Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)]:STATUS_CODE: ' \
+'*--status-codes=[Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)]:STATUS_CODE: ' \
+'-T+[Number of seconds before a client'\''s request times out (default: 7)]:SECONDS: ' \
+'--timeout=[Number of seconds before a client'\''s request times out (default: 7)]:SECONDS: ' \
+'-t+[Number of concurrent threads (default: 50)]:THREADS: ' \
+'--threads=[Number of concurrent threads (default: 50)]:THREADS: ' \
+'-d+[Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)]:RECURSION_DEPTH: ' \
+'--depth=[Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)]:RECURSION_DEPTH: ' \
+'-L+[Limit total number of concurrent scans (default: 0, i.e. no limit)]:SCAN_LIMIT: ' \
+'--scan-limit=[Limit total number of concurrent scans (default: 0, i.e. no limit)]:SCAN_LIMIT: ' \
+'--parallel=[Run parallel feroxbuster instances (one child process per url passed via stdin)]:PARALLEL_SCANS: ' \
+'(--auto-tune)--rate-limit=[Limit number of requests per second (per directory) (default: 0, i.e. no limit)]:RATE_LIMIT: ' \
+'--time-limit=[Limit total run time of all scans (ex: --time-limit 10m)]:TIME_SPEC: ' \
+'-w+[Path to the wordlist]:FILE:_files' \
+'--wordlist=[Path to the wordlist]:FILE:_files' \
+'-o+[Output file to write results to (use w/ --json for JSON entries)]:FILE:_files' \
+'--output=[Output file to write results to (use w/ --json for JSON entries)]:FILE:_files' \
+'--debug-log=[Output file to write log entries (use w/ --json for JSON entries)]:FILE:_files' \
+'-h[Print help information]' \
+'--help[Print help information]' \
+'-V[Print version information]' \
+'--version[Print version information]' \
+'(-u --url)--stdin[Read url(s) from STDIN]' \
+'-A[Use a random User-Agent]' \
+'--random-agent[Use a random User-Agent]' \
+'-f[Append / to each request'\''s URL]' \
+'--add-slash[Append / to each request'\''s URL]' \
+'-r[Allow client to follow redirects]' \
+'--redirects[Allow client to follow redirects]' \
+'-k[Disables TLS certificate validation in the client]' \
+'--insecure[Disables TLS certificate validation in the client]' \
+'-n[Do not scan recursively]' \
+'--no-recursion[Do not scan recursively]' \
+'-e[Extract links from response body (html, javascript, etc...); make new requests based on findings]' \
+'--extract-links[Extract links from response body (html, javascript, etc...); make new requests based on findings]' \
+'(--auto-bail)--auto-tune[Automatically lower scan rate when an excessive amount of errors are encountered]' \
+'--auto-bail[Automatically stop scanning when an excessive amount of errors are encountered]' \
+'-D[Don'\''t auto-filter wildcard responses]' \
+'--dont-filter[Don'\''t auto-filter wildcard responses]' \
 '(--silent)*-v[Increase verbosity level (use -vv or more for greater effect. \[CAUTION\] 4 -v'\''s is probably too much)]' \
 '(--silent)*--verbosity[Increase verbosity level (use -vv or more for greater effect. \[CAUTION\] 4 -v'\''s is probably too much)]' \
 '(-q --quiet)--silent[Only print URLs + turn off logging (good for piping a list of urls to other commands)]' \
 '-q[Hide progress bars and banner (good for tmux windows w/ notifications)]' \
 '--quiet[Hide progress bars and banner (good for tmux windows w/ notifications)]' \
-'(--auto-bail)--auto-tune[Automatically lower scan rate when an excessive amount of errors are encountered]' \
-'--auto-bail[Automatically stop scanning when an excessive amount of errors are encountered]' \
 '--json[Emit JSON logs to --output and --debug-log instead of normal text]' \
-'-D[Don'\''t auto-filter wildcard responses]' \
-'--dont-filter[Don'\''t auto-filter wildcard responses]' \
-'-r[Follow redirects]' \
-'--redirects[Follow redirects]' \
-'-k[Disables TLS certificate validation]' \
-'--insecure[Disables TLS certificate validation]' \
-'-n[Do not scan recursively]' \
-'--no-recursion[Do not scan recursively]' \
-'(-x --extensions)-f[Append / to each request]' \
-'(-x --extensions)--add-slash[Append / to each request]' \
-'(-u --url)--stdin[Read url(s) from STDIN]' \
-'-e[Extract links from response body (html, javascript, etc...); make new requests based on findings (default: false)]' \
-'--extract-links[Extract links from response body (html, javascript, etc...); make new requests based on findings (default: false)]' \
-'-h[Prints help information]' \
-'--help[Prints help information]' \
-'-V[Prints version information]' \
-'--version[Prints version information]' \
 && ret=0
-    
 }
 
 (( $+functions[_feroxbuster_commands] )) ||
 _feroxbuster_commands() {
-    local commands; commands=(
-        
-    )
+    local commands; commands=()
     _describe -t commands 'feroxbuster commands' commands "$@"
 }
 
-_feroxbuster "$@"
+_feroxbuster "$@"

shell_completions/_feroxbuster.ps1

@@ -20,37 +20,29 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
 
     $completions = @(switch ($command) {
         'feroxbuster' {
-            [CompletionResult]::new('-w', 'w', [CompletionResultType]::ParameterName, 'Path to the wordlist')
-            [CompletionResult]::new('--wordlist', 'wordlist', [CompletionResultType]::ParameterName, 'Path to the wordlist')
-            [CompletionResult]::new('-u', 'u', [CompletionResultType]::ParameterName, 'The target URL(s) (required, unless --stdin used)')
-            [CompletionResult]::new('--url', 'url', [CompletionResultType]::ParameterName, 'The target URL(s) (required, unless --stdin used)')
-            [CompletionResult]::new('-t', 't', [CompletionResultType]::ParameterName, 'Number of concurrent threads (default: 50)')
-            [CompletionResult]::new('--threads', 'threads', [CompletionResultType]::ParameterName, 'Number of concurrent threads (default: 50)')
-            [CompletionResult]::new('-d', 'd', [CompletionResultType]::ParameterName, 'Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)')
-            [CompletionResult]::new('--depth', 'depth', [CompletionResultType]::ParameterName, 'Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)')
-            [CompletionResult]::new('-T', 'T', [CompletionResultType]::ParameterName, 'Number of seconds before a request times out (default: 7)')
-            [CompletionResult]::new('--timeout', 'timeout', [CompletionResultType]::ParameterName, 'Number of seconds before a request times out (default: 7)')
+            [CompletionResult]::new('-u', 'u', [CompletionResultType]::ParameterName, 'The target URL (required, unless [--stdin || --resume-from] used)')
+            [CompletionResult]::new('--url', 'url', [CompletionResultType]::ParameterName, 'The target URL (required, unless [--stdin || --resume-from] used)')
+            [CompletionResult]::new('--resume-from', 'resume-from', [CompletionResultType]::ParameterName, 'State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)')
             [CompletionResult]::new('-p', 'p', [CompletionResultType]::ParameterName, 'Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)')
             [CompletionResult]::new('--proxy', 'proxy', [CompletionResultType]::ParameterName, 'Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)')
             [CompletionResult]::new('-P', 'P', [CompletionResultType]::ParameterName, 'Send only unfiltered requests through a Replay Proxy, instead of all requests')
             [CompletionResult]::new('--replay-proxy', 'replay-proxy', [CompletionResultType]::ParameterName, 'Send only unfiltered requests through a Replay Proxy, instead of all requests')
             [CompletionResult]::new('-R', 'R', [CompletionResultType]::ParameterName, 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)')
             [CompletionResult]::new('--replay-codes', 'replay-codes', [CompletionResultType]::ParameterName, 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)')
-            [CompletionResult]::new('-s', 's', [CompletionResultType]::ParameterName, 'Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)')
-            [CompletionResult]::new('--status-codes', 'status-codes', [CompletionResultType]::ParameterName, 'Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)')
-            [CompletionResult]::new('-o', 'o', [CompletionResultType]::ParameterName, 'Output file to write results to (use w/ --json for JSON entries)')
-            [CompletionResult]::new('--output', 'output', [CompletionResultType]::ParameterName, 'Output file to write results to (use w/ --json for JSON entries)')
-            [CompletionResult]::new('--resume-from', 'resume-from', [CompletionResultType]::ParameterName, 'State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)')
-            [CompletionResult]::new('--debug-log', 'debug-log', [CompletionResultType]::ParameterName, 'Output file to write log entries (use w/ --json for JSON entries)')
-            [CompletionResult]::new('-a', 'a', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/VERSION)')
-            [CompletionResult]::new('--user-agent', 'user-agent', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/VERSION)')
+            [CompletionResult]::new('-a', 'a', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.5.0)')
+            [CompletionResult]::new('--user-agent', 'user-agent', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.5.0)')
             [CompletionResult]::new('-x', 'x', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js)')
             [CompletionResult]::new('--extensions', 'extensions', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js)')
-            [CompletionResult]::new('--dont-scan', 'dont-scan', [CompletionResultType]::ParameterName, 'URL(s) to exclude from recursion/scans')
-            [CompletionResult]::new('-H', 'H', [CompletionResultType]::ParameterName, 'Specify HTTP headers (ex: -H Header:val ''stuff: things'')')
-            [CompletionResult]::new('--headers', 'headers', [CompletionResultType]::ParameterName, 'Specify HTTP headers (ex: -H Header:val ''stuff: things'')')
-            [CompletionResult]::new('-Q', 'Q', [CompletionResultType]::ParameterName, 'Specify URL query parameters (ex: -Q token=stuff -Q secret=key)')
-            [CompletionResult]::new('--query', 'query', [CompletionResultType]::ParameterName, 'Specify URL query parameters (ex: -Q token=stuff -Q secret=key)')
+            [CompletionResult]::new('-m', 'm', [CompletionResultType]::ParameterName, 'Which HTTP request method(s) should be sent (default: GET)')
+            [CompletionResult]::new('--methods', 'methods', [CompletionResultType]::ParameterName, 'Which HTTP request method(s) should be sent (default: GET)')
+            [CompletionResult]::new('--data', 'data', [CompletionResultType]::ParameterName, 'Request''s Body; can read data from a file if input starts with an @ (ex: @post.bin)')
+            [CompletionResult]::new('-H', 'H', [CompletionResultType]::ParameterName, 'Specify HTTP headers to be used in each request (ex: -H Header:val -H ''stuff: things'')')
+            [CompletionResult]::new('--headers', 'headers', [CompletionResultType]::ParameterName, 'Specify HTTP headers to be used in each request (ex: -H Header:val -H ''stuff: things'')')
+            [CompletionResult]::new('-b', 'b', [CompletionResultType]::ParameterName, 'Specify HTTP cookies to be used in each request (ex: -b stuff=things)')
+            [CompletionResult]::new('--cookies', 'cookies', [CompletionResultType]::ParameterName, 'Specify HTTP cookies to be used in each request (ex: -b stuff=things)')
+            [CompletionResult]::new('-Q', 'Q', [CompletionResultType]::ParameterName, 'Request''s URL query parameters (ex: -Q token=stuff -Q secret=key)')
+            [CompletionResult]::new('--query', 'query', [CompletionResultType]::ParameterName, 'Request''s URL query parameters (ex: -Q token=stuff -Q secret=key)')
+            [CompletionResult]::new('--dont-scan', 'dont-scan', [CompletionResultType]::ParameterName, 'URL(s) or Regex Pattern(s) to exclude from recursion/scans')
             [CompletionResult]::new('-S', 'S', [CompletionResultType]::ParameterName, 'Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)')
             [CompletionResult]::new('--filter-size', 'filter-size', [CompletionResultType]::ParameterName, 'Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)')
             [CompletionResult]::new('-X', 'X', [CompletionResultType]::ParameterName, 'Filter out messages via regular expression matching on the response''s body (ex: -X ''^ignore me$'')')
@@ -62,36 +54,51 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
             [CompletionResult]::new('-C', 'C', [CompletionResultType]::ParameterName, 'Filter out status codes (deny list) (ex: -C 200 -C 401)')
             [CompletionResult]::new('--filter-status', 'filter-status', [CompletionResultType]::ParameterName, 'Filter out status codes (deny list) (ex: -C 200 -C 401)')
             [CompletionResult]::new('--filter-similar-to', 'filter-similar-to', [CompletionResultType]::ParameterName, 'Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)')
+            [CompletionResult]::new('-s', 's', [CompletionResultType]::ParameterName, 'Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)')
+            [CompletionResult]::new('--status-codes', 'status-codes', [CompletionResultType]::ParameterName, 'Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)')
+            [CompletionResult]::new('-T', 'T', [CompletionResultType]::ParameterName, 'Number of seconds before a client''s request times out (default: 7)')
+            [CompletionResult]::new('--timeout', 'timeout', [CompletionResultType]::ParameterName, 'Number of seconds before a client''s request times out (default: 7)')
+            [CompletionResult]::new('-t', 't', [CompletionResultType]::ParameterName, 'Number of concurrent threads (default: 50)')
+            [CompletionResult]::new('--threads', 'threads', [CompletionResultType]::ParameterName, 'Number of concurrent threads (default: 50)')
+            [CompletionResult]::new('-d', 'd', [CompletionResultType]::ParameterName, 'Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)')
+            [CompletionResult]::new('--depth', 'depth', [CompletionResultType]::ParameterName, 'Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)')
             [CompletionResult]::new('-L', 'L', [CompletionResultType]::ParameterName, 'Limit total number of concurrent scans (default: 0, i.e. no limit)')
             [CompletionResult]::new('--scan-limit', 'scan-limit', [CompletionResultType]::ParameterName, 'Limit total number of concurrent scans (default: 0, i.e. no limit)')
             [CompletionResult]::new('--parallel', 'parallel', [CompletionResultType]::ParameterName, 'Run parallel feroxbuster instances (one child process per url passed via stdin)')
             [CompletionResult]::new('--rate-limit', 'rate-limit', [CompletionResultType]::ParameterName, 'Limit number of requests per second (per directory) (default: 0, i.e. no limit)')
             [CompletionResult]::new('--time-limit', 'time-limit', [CompletionResultType]::ParameterName, 'Limit total run time of all scans (ex: --time-limit 10m)')
+            [CompletionResult]::new('-w', 'w', [CompletionResultType]::ParameterName, 'Path to the wordlist')
+            [CompletionResult]::new('--wordlist', 'wordlist', [CompletionResultType]::ParameterName, 'Path to the wordlist')
+            [CompletionResult]::new('-o', 'o', [CompletionResultType]::ParameterName, 'Output file to write results to (use w/ --json for JSON entries)')
+            [CompletionResult]::new('--output', 'output', [CompletionResultType]::ParameterName, 'Output file to write results to (use w/ --json for JSON entries)')
+            [CompletionResult]::new('--debug-log', 'debug-log', [CompletionResultType]::ParameterName, 'Output file to write log entries (use w/ --json for JSON entries)')
+            [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help information')
+            [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help information')
+            [CompletionResult]::new('-V', 'V', [CompletionResultType]::ParameterName, 'Print version information')
+            [CompletionResult]::new('--version', 'version', [CompletionResultType]::ParameterName, 'Print version information')
+            [CompletionResult]::new('--stdin', 'stdin', [CompletionResultType]::ParameterName, 'Read url(s) from STDIN')
+            [CompletionResult]::new('-A', 'A', [CompletionResultType]::ParameterName, 'Use a random User-Agent')
+            [CompletionResult]::new('--random-agent', 'random-agent', [CompletionResultType]::ParameterName, 'Use a random User-Agent')
+            [CompletionResult]::new('-f', 'f', [CompletionResultType]::ParameterName, 'Append / to each request''s URL')
+            [CompletionResult]::new('--add-slash', 'add-slash', [CompletionResultType]::ParameterName, 'Append / to each request''s URL')
+            [CompletionResult]::new('-r', 'r', [CompletionResultType]::ParameterName, 'Allow client to follow redirects')
+            [CompletionResult]::new('--redirects', 'redirects', [CompletionResultType]::ParameterName, 'Allow client to follow redirects')
+            [CompletionResult]::new('-k', 'k', [CompletionResultType]::ParameterName, 'Disables TLS certificate validation in the client')
+            [CompletionResult]::new('--insecure', 'insecure', [CompletionResultType]::ParameterName, 'Disables TLS certificate validation in the client')
+            [CompletionResult]::new('-n', 'n', [CompletionResultType]::ParameterName, 'Do not scan recursively')
+            [CompletionResult]::new('--no-recursion', 'no-recursion', [CompletionResultType]::ParameterName, 'Do not scan recursively')
+            [CompletionResult]::new('-e', 'e', [CompletionResultType]::ParameterName, 'Extract links from response body (html, javascript, etc...); make new requests based on findings')
+            [CompletionResult]::new('--extract-links', 'extract-links', [CompletionResultType]::ParameterName, 'Extract links from response body (html, javascript, etc...); make new requests based on findings')
+            [CompletionResult]::new('--auto-tune', 'auto-tune', [CompletionResultType]::ParameterName, 'Automatically lower scan rate when an excessive amount of errors are encountered')
+            [CompletionResult]::new('--auto-bail', 'auto-bail', [CompletionResultType]::ParameterName, 'Automatically stop scanning when an excessive amount of errors are encountered')
+            [CompletionResult]::new('-D', 'D', [CompletionResultType]::ParameterName, 'Don''t auto-filter wildcard responses')
+            [CompletionResult]::new('--dont-filter', 'dont-filter', [CompletionResultType]::ParameterName, 'Don''t auto-filter wildcard responses')
             [CompletionResult]::new('-v', 'v', [CompletionResultType]::ParameterName, 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v''s is probably too much)')
             [CompletionResult]::new('--verbosity', 'verbosity', [CompletionResultType]::ParameterName, 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v''s is probably too much)')
             [CompletionResult]::new('--silent', 'silent', [CompletionResultType]::ParameterName, 'Only print URLs + turn off logging (good for piping a list of urls to other commands)')
             [CompletionResult]::new('-q', 'q', [CompletionResultType]::ParameterName, 'Hide progress bars and banner (good for tmux windows w/ notifications)')
             [CompletionResult]::new('--quiet', 'quiet', [CompletionResultType]::ParameterName, 'Hide progress bars and banner (good for tmux windows w/ notifications)')
-            [CompletionResult]::new('--auto-tune', 'auto-tune', [CompletionResultType]::ParameterName, 'Automatically lower scan rate when an excessive amount of errors are encountered')
-            [CompletionResult]::new('--auto-bail', 'auto-bail', [CompletionResultType]::ParameterName, 'Automatically stop scanning when an excessive amount of errors are encountered')
             [CompletionResult]::new('--json', 'json', [CompletionResultType]::ParameterName, 'Emit JSON logs to --output and --debug-log instead of normal text')
-            [CompletionResult]::new('-D', 'D', [CompletionResultType]::ParameterName, 'Don''t auto-filter wildcard responses')
-            [CompletionResult]::new('--dont-filter', 'dont-filter', [CompletionResultType]::ParameterName, 'Don''t auto-filter wildcard responses')
-            [CompletionResult]::new('-r', 'r', [CompletionResultType]::ParameterName, 'Follow redirects')
-            [CompletionResult]::new('--redirects', 'redirects', [CompletionResultType]::ParameterName, 'Follow redirects')
-            [CompletionResult]::new('-k', 'k', [CompletionResultType]::ParameterName, 'Disables TLS certificate validation')
-            [CompletionResult]::new('--insecure', 'insecure', [CompletionResultType]::ParameterName, 'Disables TLS certificate validation')
-            [CompletionResult]::new('-n', 'n', [CompletionResultType]::ParameterName, 'Do not scan recursively')
-            [CompletionResult]::new('--no-recursion', 'no-recursion', [CompletionResultType]::ParameterName, 'Do not scan recursively')
-            [CompletionResult]::new('-f', 'f', [CompletionResultType]::ParameterName, 'Append / to each request')
-            [CompletionResult]::new('--add-slash', 'add-slash', [CompletionResultType]::ParameterName, 'Append / to each request')
-            [CompletionResult]::new('--stdin', 'stdin', [CompletionResultType]::ParameterName, 'Read url(s) from STDIN')
-            [CompletionResult]::new('-e', 'e', [CompletionResultType]::ParameterName, 'Extract links from response body (html, javascript, etc...); make new requests based on findings (default: false)')
-            [CompletionResult]::new('--extract-links', 'extract-links', [CompletionResultType]::ParameterName, 'Extract links from response body (html, javascript, etc...); make new requests based on findings (default: false)')
-            [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Prints help information')
-            [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Prints help information')
-            [CompletionResult]::new('-V', 'V', [CompletionResultType]::ParameterName, 'Prints version information')
-            [CompletionResult]::new('--version', 'version', [CompletionResultType]::ParameterName, 'Prints version information')
             break
         }
     })

shell_completions/feroxbuster.bash

@@ -9,10 +9,9 @@ _feroxbuster() {
     for i in ${COMP_WORDS[@]}
     do
         case "${i}" in
-            feroxbuster)
+            "$1")
                 cmd="feroxbuster"
                 ;;
-            
             *)
                 ;;
         esac
@@ -20,90 +19,17 @@ _feroxbuster() {
 
     case "${cmd}" in
         feroxbuster)
-            opts=" -v -q -D -r -k -n -f -e -h -V -w -u -t -d -T -p -P -R -s -o -a -x -H -Q -S -X -W -N -C -L  --verbosity --silent --quiet --auto-tune --auto-bail --json --dont-filter --redirects --insecure --no-recursion --add-slash --stdin --extract-links --help --version --wordlist --url --threads --depth --timeout --proxy --replay-proxy --replay-codes --status-codes --output --resume-from --debug-log --user-agent --extensions --dont-scan --headers --query --filter-size --filter-regex --filter-words --filter-lines --filter-status --filter-similar-to --scan-limit --parallel --rate-limit --time-limit  "
+            opts="-h -V -u -p -P -R -a -A -x -m -H -b -Q -f -S -X -W -N -C -s -T -r -k -t -n -d -e -L -w -D -v -q -o --help --version --url --stdin --resume-from --proxy --replay-proxy --replay-codes --user-agent --random-agent --extensions --methods --data --headers --cookies --query --add-slash --dont-scan --filter-size --filter-regex --filter-words --filter-lines --filter-status --filter-similar-to --status-codes --timeout --redirects --insecure --threads --no-recursion --depth --extract-links --scan-limit --parallel --rate-limit --time-limit --wordlist --auto-tune --auto-bail --dont-filter --verbosity --silent --quiet --json --output --debug-log"
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 1 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
             fi
             case "${prev}" in
-                
-                --wordlist)
-                    COMPREPLY=($(compgen -f "${cur}"))
-                    return 0
-                    ;;
-                    -w)
-                    COMPREPLY=($(compgen -f "${cur}"))
-                    return 0
-                    ;;
                 --url)
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
                     ;;
-                    -u)
-                    COMPREPLY=($(compgen -f "${cur}"))
-                    return 0
-                    ;;
-                --threads)
-                    COMPREPLY=($(compgen -f "${cur}"))
-                    return 0
-                    ;;
-                    -t)
-                    COMPREPLY=($(compgen -f "${cur}"))
-                    return 0
-                    ;;
-                --depth)
-                    COMPREPLY=($(compgen -f "${cur}"))
-                    return 0
-                    ;;
-                    -d)
-                    COMPREPLY=($(compgen -f "${cur}"))
-                    return 0
-                    ;;
-                --timeout)
-                    COMPREPLY=($(compgen -f "${cur}"))
-                    return 0
-                    ;;
-                    -T)
-                    COMPREPLY=($(compgen -f "${cur}"))
-                    return 0
-                    ;;
-                --proxy)
-                    COMPREPLY=($(compgen -f "${cur}"))
-                    return 0
-                    ;;
-                    -p)
-                    COMPREPLY=($(compgen -f "${cur}"))
-                    return 0
-                    ;;
-                --replay-proxy)
-                    COMPREPLY=($(compgen -f "${cur}"))
-                    return 0
-                    ;;
-                    -P)
-                    COMPREPLY=($(compgen -f "${cur}"))
-                    return 0
-                    ;;
-                --replay-codes)
-                    COMPREPLY=($(compgen -f "${cur}"))
-                    return 0
-                    ;;
-                    -R)
-                    COMPREPLY=($(compgen -f "${cur}"))
-                    return 0
-                    ;;
-                --status-codes)
-                    COMPREPLY=($(compgen -f "${cur}"))
-                    return 0
-                    ;;
-                    -s)
-                    COMPREPLY=($(compgen -f "${cur}"))
-                    return 0
-                    ;;
-                --output)
-                    COMPREPLY=($(compgen -f "${cur}"))
-                    return 0
-                    ;;
-                    -o)
+                -u)
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
                     ;;
@@ -111,7 +37,27 @@ _feroxbuster() {
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
                     ;;
-                --debug-log)
+                --proxy)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -p)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --replay-proxy)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -P)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --replay-codes)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -R)
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
                     ;;
@@ -119,7 +65,7 @@ _feroxbuster() {
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
                     ;;
-                    -a)
+                -a)
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
                     ;;
@@ -127,11 +73,19 @@ _feroxbuster() {
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
                     ;;
-                    -x)
+                -x)
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
                     ;;
-                --dont-scan)
+                --methods)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -m)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --data)
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
                     ;;
@@ -139,7 +93,15 @@ _feroxbuster() {
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
                     ;;
-                    -H)
+                -H)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --cookies)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -b)
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
                     ;;
@@ -147,7 +109,11 @@ _feroxbuster() {
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
                     ;;
-                    -Q)
+                -Q)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --dont-scan)
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
                     ;;
@@ -155,7 +121,7 @@ _feroxbuster() {
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
                     ;;
-                    -S)
+                -S)
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
                     ;;
@@ -163,7 +129,7 @@ _feroxbuster() {
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
                     ;;
-                    -X)
+                -X)
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
                     ;;
@@ -171,7 +137,7 @@ _feroxbuster() {
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
                     ;;
-                    -W)
+                -W)
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
                     ;;
@@ -179,7 +145,7 @@ _feroxbuster() {
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
                     ;;
-                    -N)
+                -N)
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
                     ;;
@@ -187,7 +153,7 @@ _feroxbuster() {
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
                     ;;
-                    -C)
+                -C)
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
                     ;;
@@ -195,11 +161,43 @@ _feroxbuster() {
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
                     ;;
+                --status-codes)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -s)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --timeout)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -T)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --threads)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -t)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --depth)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -d)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
                 --scan-limit)
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
                     ;;
-                    -L)
+                -L)
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
                     ;;
@@ -215,6 +213,26 @@ _feroxbuster() {
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
                     ;;
+                --wordlist)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -w)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --output)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -o)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --debug-log)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
                 *)
                     COMPREPLY=()
                     ;;
@@ -222,7 +240,6 @@ _feroxbuster() {
             COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
             return 0
             ;;
-        
     esac
 }
 

shell_completions/feroxbuster.elv

@@ -0,0 +1,103 @@
+
+use builtin;
+use str;
+
+set edit:completion:arg-completer[feroxbuster] = {|@words|
+    fn spaces {|n|
+        builtin:repeat $n ' ' | str:join ''
+    }
+    fn cand {|text desc|
+        edit:complex-candidate $text &display=$text' '(spaces (- 14 (wcswidth $text)))$desc
+    }
+    var command = 'feroxbuster'
+    for word $words[1..-1] {
+        if (str:has-prefix $word '-') {
+            break
+        }
+        set command = $command';'$word
+    }
+    var completions = [
+        &'feroxbuster'= {
+            cand -u 'The target URL (required, unless [--stdin || --resume-from] used)'
+            cand --url 'The target URL (required, unless [--stdin || --resume-from] used)'
+            cand --resume-from 'State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)'
+            cand -p 'Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)'
+            cand --proxy 'Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)'
+            cand -P 'Send only unfiltered requests through a Replay Proxy, instead of all requests'
+            cand --replay-proxy 'Send only unfiltered requests through a Replay Proxy, instead of all requests'
+            cand -R 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)'
+            cand --replay-codes 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)'
+            cand -a 'Sets the User-Agent (default: feroxbuster/2.5.0)'
+            cand --user-agent 'Sets the User-Agent (default: feroxbuster/2.5.0)'
+            cand -x 'File extension(s) to search for (ex: -x php -x pdf js)'
+            cand --extensions 'File extension(s) to search for (ex: -x php -x pdf js)'
+            cand -m 'Which HTTP request method(s) should be sent (default: GET)'
+            cand --methods 'Which HTTP request method(s) should be sent (default: GET)'
+            cand --data 'Request''s Body; can read data from a file if input starts with an @ (ex: @post.bin)'
+            cand -H 'Specify HTTP headers to be used in each request (ex: -H Header:val -H ''stuff: things'')'
+            cand --headers 'Specify HTTP headers to be used in each request (ex: -H Header:val -H ''stuff: things'')'
+            cand -b 'Specify HTTP cookies to be used in each request (ex: -b stuff=things)'
+            cand --cookies 'Specify HTTP cookies to be used in each request (ex: -b stuff=things)'
+            cand -Q 'Request''s URL query parameters (ex: -Q token=stuff -Q secret=key)'
+            cand --query 'Request''s URL query parameters (ex: -Q token=stuff -Q secret=key)'
+            cand --dont-scan 'URL(s) or Regex Pattern(s) to exclude from recursion/scans'
+            cand -S 'Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)'
+            cand --filter-size 'Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)'
+            cand -X 'Filter out messages via regular expression matching on the response''s body (ex: -X ''^ignore me$'')'
+            cand --filter-regex 'Filter out messages via regular expression matching on the response''s body (ex: -X ''^ignore me$'')'
+            cand -W 'Filter out messages of a particular word count (ex: -W 312 -W 91,82)'
+            cand --filter-words 'Filter out messages of a particular word count (ex: -W 312 -W 91,82)'
+            cand -N 'Filter out messages of a particular line count (ex: -N 20 -N 31,30)'
+            cand --filter-lines 'Filter out messages of a particular line count (ex: -N 20 -N 31,30)'
+            cand -C 'Filter out status codes (deny list) (ex: -C 200 -C 401)'
+            cand --filter-status 'Filter out status codes (deny list) (ex: -C 200 -C 401)'
+            cand --filter-similar-to 'Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)'
+            cand -s 'Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)'
+            cand --status-codes 'Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)'
+            cand -T 'Number of seconds before a client''s request times out (default: 7)'
+            cand --timeout 'Number of seconds before a client''s request times out (default: 7)'
+            cand -t 'Number of concurrent threads (default: 50)'
+            cand --threads 'Number of concurrent threads (default: 50)'
+            cand -d 'Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)'
+            cand --depth 'Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)'
+            cand -L 'Limit total number of concurrent scans (default: 0, i.e. no limit)'
+            cand --scan-limit 'Limit total number of concurrent scans (default: 0, i.e. no limit)'
+            cand --parallel 'Run parallel feroxbuster instances (one child process per url passed via stdin)'
+            cand --rate-limit 'Limit number of requests per second (per directory) (default: 0, i.e. no limit)'
+            cand --time-limit 'Limit total run time of all scans (ex: --time-limit 10m)'
+            cand -w 'Path to the wordlist'
+            cand --wordlist 'Path to the wordlist'
+            cand -o 'Output file to write results to (use w/ --json for JSON entries)'
+            cand --output 'Output file to write results to (use w/ --json for JSON entries)'
+            cand --debug-log 'Output file to write log entries (use w/ --json for JSON entries)'
+            cand -h 'Print help information'
+            cand --help 'Print help information'
+            cand -V 'Print version information'
+            cand --version 'Print version information'
+            cand --stdin 'Read url(s) from STDIN'
+            cand -A 'Use a random User-Agent'
+            cand --random-agent 'Use a random User-Agent'
+            cand -f 'Append / to each request''s URL'
+            cand --add-slash 'Append / to each request''s URL'
+            cand -r 'Allow client to follow redirects'
+            cand --redirects 'Allow client to follow redirects'
+            cand -k 'Disables TLS certificate validation in the client'
+            cand --insecure 'Disables TLS certificate validation in the client'
+            cand -n 'Do not scan recursively'
+            cand --no-recursion 'Do not scan recursively'
+            cand -e 'Extract links from response body (html, javascript, etc...); make new requests based on findings'
+            cand --extract-links 'Extract links from response body (html, javascript, etc...); make new requests based on findings'
+            cand --auto-tune 'Automatically lower scan rate when an excessive amount of errors are encountered'
+            cand --auto-bail 'Automatically stop scanning when an excessive amount of errors are encountered'
+            cand -D 'Don''t auto-filter wildcard responses'
+            cand --dont-filter 'Don''t auto-filter wildcard responses'
+            cand -v 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v''s is probably too much)'
+            cand --verbosity 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v''s is probably too much)'
+            cand --silent 'Only print URLs + turn off logging (good for piping a list of urls to other commands)'
+            cand -q 'Hide progress bars and banner (good for tmux windows w/ notifications)'
+            cand --quiet 'Hide progress bars and banner (good for tmux windows w/ notifications)'
+            cand --json 'Emit JSON logs to --output and --debug-log instead of normal text'
+        }
+    ]
+    $completions[$command]
+}

shell_completions/feroxbuster.fish

@@ -12,8 +12,11 @@ complete -c feroxbuster -n "__fish_use_subcommand" -l resume-from -d 'State file
 complete -c feroxbuster -n "__fish_use_subcommand" -l debug-log -d 'Output file to write log entries (use w/ --json for JSON entries)'
 complete -c feroxbuster -n "__fish_use_subcommand" -s a -l user-agent -d 'Sets the User-Agent (default: feroxbuster/VERSION)'
 complete -c feroxbuster -n "__fish_use_subcommand" -s x -l extensions -d 'File extension(s) to search for (ex: -x php -x pdf js)'
-complete -c feroxbuster -n "__fish_use_subcommand" -l dont-scan -d 'URL(s) to exclude from recursion/scans'
+complete -c feroxbuster -n "__fish_use_subcommand" -s m -l methods -d 'HTTP request method(s) (default: GET)'
+complete -c feroxbuster -n "__fish_use_subcommand" -l data -d 'HTTP Body data; can read data from a file if input starts with an @ (ex: @post.bin)'
+complete -c feroxbuster -n "__fish_use_subcommand" -l dont-scan -d 'URL(s) or Regex Pattern(s) to exclude from recursion/scans'
 complete -c feroxbuster -n "__fish_use_subcommand" -s H -l headers -d 'Specify HTTP headers (ex: -H Header:val \'stuff: things\')'
+complete -c feroxbuster -n "__fish_use_subcommand" -s b -l cookies -d 'Specify HTTP cookies (ex: -b stuff=things)'
 complete -c feroxbuster -n "__fish_use_subcommand" -s Q -l query -d 'Specify URL query parameters (ex: -Q token=stuff -Q secret=key)'
 complete -c feroxbuster -n "__fish_use_subcommand" -s S -l filter-size -d 'Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)'
 complete -c feroxbuster -n "__fish_use_subcommand" -s X -l filter-regex -d 'Filter out messages via regular expression matching on the response\'s body (ex: -X \'^ignore me$\')'
@@ -32,6 +35,7 @@ complete -c feroxbuster -n "__fish_use_subcommand" -l auto-tune -d 'Automaticall
 complete -c feroxbuster -n "__fish_use_subcommand" -l auto-bail -d 'Automatically stop scanning when an excessive amount of errors are encountered'
 complete -c feroxbuster -n "__fish_use_subcommand" -l json -d 'Emit JSON logs to --output and --debug-log instead of normal text'
 complete -c feroxbuster -n "__fish_use_subcommand" -s D -l dont-filter -d 'Don\'t auto-filter wildcard responses'
+complete -c feroxbuster -n "__fish_use_subcommand" -s A -l random-agent -d 'Use a random User-Agent'
 complete -c feroxbuster -n "__fish_use_subcommand" -s r -l redirects -d 'Follow redirects'
 complete -c feroxbuster -n "__fish_use_subcommand" -s k -l insecure -d 'Disables TLS certificate validation'
 complete -c feroxbuster -n "__fish_use_subcommand" -s n -l no-recursion -d 'Do not scan recursively'

src/banner/container.rs

@@ -3,7 +3,7 @@ use crate::{
     config::Configuration,
     event_handlers::Handles,
     utils::{logged_request, status_colorizer},
-    VERSION,
+    DEFAULT_METHOD, VERSION,
 };
 use anyhow::{bail, Result};
 use console::{style, Emoji};
@@ -50,6 +50,9 @@ pub struct Banner {
     /// represents Configuration.user_agent
     user_agent: BannerEntry,
 
+    /// represents Configuration.random_agent
+    random_agent: BannerEntry,
+
     /// represents Configuration.config
     config: BannerEntry,
 
@@ -95,6 +98,12 @@ pub struct Banner {
     /// represents Configuration.extensions
     extensions: BannerEntry,
 
+    /// represents Configuration.methods
+    methods: BannerEntry,
+
+    /// represents Configuration.data
+    data: BannerEntry,
+
     /// represents Configuration.insecure
     insecure: BannerEntry,
 
@@ -165,7 +174,19 @@ impl Banner {
         }
 
         for denied_url in &config.url_denylist {
-            url_denylist.push(BannerEntry::new("🚫", "Don't Scan", denied_url));
+            url_denylist.push(BannerEntry::new(
+                "🚫",
+                "Don't Scan Url",
+                denied_url.as_str(),
+            ));
+        }
+
+        for denied_regex in &config.regex_denylist {
+            url_denylist.push(BannerEntry::new(
+                "🚫",
+                "Don't Scan Regex",
+                denied_regex.as_str(),
+            ));
         }
 
         let mut codes = vec![];
@@ -276,6 +297,7 @@ impl Banner {
         let wordlist = BannerEntry::new("📖", "Wordlist", &config.wordlist);
         let timeout = BannerEntry::new("💥", "Timeout (secs)", &config.timeout.to_string());
         let user_agent = BannerEntry::new("🦡", "User-Agent", &config.user_agent);
+        let random_agent = BannerEntry::new("🦡", "User-Agent", "Random");
         let extract_links =
             BannerEntry::new("🔎", "Extract Links", &config.extract_links.to_string());
         let json = BannerEntry::new("🧔", "JSON Output", &config.json.to_string());
@@ -286,6 +308,23 @@ impl Banner {
             "Extensions",
             &format!("[{}]", config.extensions.join(", ")),
         );
+        let methods = BannerEntry::new(
+            "🏁",
+            "HTTP methods",
+            &format!("[{}]", config.methods.join(", ")),
+        );
+
+        let offset = std::cmp::min(config.data.len(), 30);
+        let data = String::from_utf8(config.data[..offset].to_vec())
+            .unwrap_or_else(|_err| {
+                format!(
+                    "{:x?} ...",
+                    &config.data[..std::cmp::min(config.data.len(), 13)]
+                )
+            })
+            .replace("\n", " ")
+            .replace("\r", "");
+        let data = BannerEntry::new("💣", "HTTP Body", &data);
         let insecure = BannerEntry::new("🔓", "Insecure", &config.insecure.to_string());
         let redirects = BannerEntry::new("📍", "Follow Redirects", &config.redirects.to_string());
         let dont_filter =
@@ -304,6 +343,7 @@ impl Banner {
             filter_status,
             timeout,
             user_agent,
+            random_agent,
             auto_bail,
             auto_tune,
             proxy,
@@ -322,6 +362,8 @@ impl Banner {
             output,
             debug_log,
             extensions,
+            methods,
+            data,
             insecure,
             dont_filter,
             redirects,
@@ -363,7 +405,7 @@ by Ben "epi" Risher {}                 ver: {}"#,
         let instructions = format!(
             " 🏁  Press [{}] to use the {}™",
             style("ENTER").yellow(),
-            style("Scan Cancel Menu").bright().yellow(),
+            style("Scan Management Menu").bright().yellow(),
         );
 
         format!("{}\n{}\n{}", bottom, instructions, addl_section)
@@ -378,7 +420,7 @@ by Ben "epi" Risher {}                 ver: {}"#,
 
         let api_url = Url::parse(url)?;
 
-        let result = logged_request(&api_url, handles.clone()).await?;
+        let result = logged_request(&api_url, DEFAULT_METHOD, None, handles.clone()).await?;
         let body = result.text().await?;
 
         let json_response: Value = serde_json::from_str(&body)?;
@@ -437,7 +479,12 @@ by Ben "epi" Risher {}                 ver: {}"#,
         }
 
         writeln!(&mut writer, "{}", self.timeout)?;
-        writeln!(&mut writer, "{}", self.user_agent)?;
+
+        if config.random_agent {
+            writeln!(&mut writer, "{}", self.random_agent)?;
+        } else {
+            writeln!(&mut writer, "{}", self.user_agent)?;
+        }
 
         // followed by the maybe printed or variably displayed values
         if !config.config.is_empty() {
@@ -503,6 +550,14 @@ by Ben "epi" Risher {}                 ver: {}"#,
             writeln!(&mut writer, "{}", self.extensions)?;
         }
 
+        if !config.methods.is_empty() {
+            writeln!(&mut writer, "{}", self.methods)?;
+        }
+
+        if !config.data.is_empty() {
+            writeln!(&mut writer, "{}", self.data)?;
+        }
+
         if config.insecure {
             writeln!(&mut writer, "{}", self.insecure)?;
         }

src/config/container.rs

@@ -1,5 +1,5 @@
 use super::utils::{
-    depth, report_and_exit, save_state, serialized_type, status_codes, threads, timeout,
+    depth, methods, report_and_exit, save_state, serialized_type, status_codes, threads, timeout,
     user_agent, wordlist, OutputLevel, RequesterPolicy,
 };
 use crate::config::determine_output_level;
@@ -9,8 +9,9 @@ use crate::{
     DEFAULT_CONFIG_NAME,
 };
 use anyhow::{anyhow, Context, Result};
-use clap::{value_t, ArgMatches};
-use reqwest::{Client, StatusCode};
+use clap::ArgMatches;
+use regex::Regex;
+use reqwest::{Client, Method, StatusCode, Url};
 use serde::{Deserialize, Serialize};
 use std::{
     collections::HashMap,
@@ -21,17 +22,15 @@ use std::{
 
 /// macro helper to abstract away repetitive configuration updates
 macro_rules! update_config_if_present {
-    ($c:expr, $m:ident, $v:expr, $t:ty) => {
-        match value_t!($m, $v, $t) {
-            Ok(value) => *$c = value, // Update value
-            Err(clap::Error {
-                kind: clap::ErrorKind::ArgumentNotFound,
-                message: _,
-                info: _,
-            }) => {
-                // Do nothing if argument not found
+    ($conf_val:expr, $matches:ident, $arg_name:expr) => {
+        match $matches.value_of_t($arg_name) {
+            Ok(value) => *$conf_val = value, // Update value
+            Err(err) => {
+                if !matches!(err.kind, clap::ErrorKind::ArgumentNotFound) {
+                    // Do nothing if argument not found
+                    err.exit() // Exit with error on any other parse error
+                }
             }
-            Err(e) => e.exit(), // Exit with error on parse error
         }
     };
 }
@@ -154,6 +153,10 @@ pub struct Configuration {
     #[serde(default = "user_agent")]
     pub user_agent: String,
 
+    /// Use random User-Agent
+    #[serde(default)]
+    pub random_agent: bool,
+
     /// Follow redirects
     #[serde(default)]
     pub redirects: bool,
@@ -166,6 +169,14 @@ pub struct Configuration {
     #[serde(default)]
     pub extensions: Vec<String>,
 
+    /// HTTP requests methods(s) to search for
+    #[serde(default = "methods")]
+    pub methods: Vec<String>,
+
+    /// HTTP Body data to send during request
+    #[serde(default)]
+    pub data: Vec<u8>,
+
     /// HTTP headers to be used in each request
     #[serde(default)]
     pub headers: HashMap<String, String>,
@@ -251,7 +262,10 @@ pub struct Configuration {
 
     /// URLs that should never be scanned/recursed into
     #[serde(default)]
-    pub url_denylist: Vec<String>,
+    pub url_denylist: Vec<Url>,
+
+    #[serde(with = "serde_regex", default)]
+    pub regex_denylist: Vec<Regex>,
 }
 
 impl Default for Configuration {
@@ -295,6 +309,7 @@ impl Default for Configuration {
             redirects: false,
             no_recursion: false,
             extract_links: false,
+            random_agent: false,
             save_state: true,
             proxy: String::new(),
             config: String::new(),
@@ -306,9 +321,12 @@ impl Default for Configuration {
             replay_proxy: String::new(),
             queries: Vec::new(),
             extensions: Vec::new(),
+            methods: methods(),
+            data: Vec::new(),
             filter_size: Vec::new(),
             filter_regex: Vec::new(),
             url_denylist: Vec::new(),
+            regex_denylist: Vec::new(),
             filter_line_count: Vec::new(),
             filter_word_count: Vec::new(),
             filter_status: Vec::new(),
@@ -344,9 +362,13 @@ impl Configuration {
     /// - **auto_bail**: `false`
     /// - **save_state**: `true`
     /// - **user_agent**: `feroxbuster/VERSION`
+    /// - **random_agent**: `false`
     /// - **insecure**: `false` (don't be insecure, i.e. don't allow invalid certs)
     /// - **extensions**: `None`
+    /// - **methods**: [`DEFAULT_METHOD`]
+    /// - **data**: `None`
     /// - **url_denylist**: `None`
+    /// - **regex_denylist**: `None`
     /// - **filter_size**: `None`
     /// - **filter_similar**: `None`
     /// - **filter_regex**: `None`
@@ -447,7 +469,7 @@ impl Configuration {
 
     /// Parse all possible versions of the ferox-config.toml file, adhering to the order of
     /// precedence outlined above
-    fn parse_config_files(mut config: &mut Self) -> Result<()> {
+    fn parse_config_files(config: &mut Self) -> Result<()> {
         // Next, we parse the ferox-config.toml file, if present and set the values
         // therein to overwrite our default values. Deserialized defaults are specified
         // in the Configuration struct so that we don't change anything that isn't
@@ -463,7 +485,7 @@ impl Configuration {
         let config_file = PathBuf::new()
             .join("/etc/feroxbuster")
             .join(DEFAULT_CONFIG_NAME);
-        Self::parse_and_merge_config(config_file, &mut config)?;
+        Self::parse_and_merge_config(config_file, config)?;
 
         // merge a config found at ~/.config/feroxbuster/ferox-config.toml
         // config_dir() resolves to one of the following
@@ -472,7 +494,7 @@ impl Configuration {
         //   - windows: {FOLDERID_RoamingAppData}
         let config_dir = dirs::config_dir().ok_or_else(|| anyhow!("Couldn't load config"))?;
         let config_file = config_dir.join("feroxbuster").join(DEFAULT_CONFIG_NAME);
-        Self::parse_and_merge_config(config_file, &mut config)?;
+        Self::parse_and_merge_config(config_file, config)?;
 
         // merge a config found in same the directory as feroxbuster executable
         let exe_path = current_exe()?;
@@ -480,12 +502,12 @@ impl Configuration {
             .parent()
             .ok_or_else(|| anyhow!("Couldn't load config"))?;
         let config_file = bin_dir.join(DEFAULT_CONFIG_NAME);
-        Self::parse_and_merge_config(config_file, &mut config)?;
+        Self::parse_and_merge_config(config_file, config)?;
 
         // merge a config found in the user's current working directory
         let cwd = current_dir()?;
         let config_file = cwd.join(DEFAULT_CONFIG_NAME);
-        Self::parse_and_merge_config(config_file, &mut config)?;
+        Self::parse_and_merge_config(config_file, config)?;
 
         Ok(())
     }
@@ -495,16 +517,16 @@ impl Configuration {
     fn parse_cli_args(args: &ArgMatches) -> Self {
         let mut config = Configuration::default();
 
-        update_config_if_present!(&mut config.threads, args, "threads", usize);
-        update_config_if_present!(&mut config.depth, args, "depth", usize);
-        update_config_if_present!(&mut config.scan_limit, args, "scan_limit", usize);
-        update_config_if_present!(&mut config.parallel, args, "parallel", usize);
-        update_config_if_present!(&mut config.rate_limit, args, "rate_limit", usize);
-        update_config_if_present!(&mut config.wordlist, args, "wordlist", String);
-        update_config_if_present!(&mut config.output, args, "output", String);
-        update_config_if_present!(&mut config.debug_log, args, "debug_log", String);
-        update_config_if_present!(&mut config.time_limit, args, "time_limit", String);
-        update_config_if_present!(&mut config.resume_from, args, "resume_from", String);
+        update_config_if_present!(&mut config.threads, args, "threads");
+        update_config_if_present!(&mut config.depth, args, "depth");
+        update_config_if_present!(&mut config.scan_limit, args, "scan_limit");
+        update_config_if_present!(&mut config.parallel, args, "parallel");
+        update_config_if_present!(&mut config.rate_limit, args, "rate_limit");
+        update_config_if_present!(&mut config.wordlist, args, "wordlist");
+        update_config_if_present!(&mut config.output, args, "output");
+        update_config_if_present!(&mut config.debug_log, args, "debug_log");
+        update_config_if_present!(&mut config.time_limit, args, "time_limit");
+        update_config_if_present!(&mut config.resume_from, args, "resume_from");
 
         if let Some(arg) = args.values_of("status_codes") {
             config.status_codes = arg
@@ -544,8 +566,75 @@ impl Configuration {
             config.extensions = arg.map(|val| val.to_string()).collect();
         }
 
+        if let Some(arg) = args.values_of("methods") {
+            config.methods = arg
+                .map(|val| {
+                    // Check methods if they are correct
+                    Method::from_bytes(val.as_bytes())
+                        .unwrap_or_else(|e| report_and_exit(&e.to_string()))
+                        .as_str()
+                        .to_string()
+                })
+                .collect();
+        }
+
+        if let Some(arg) = args.value_of("data") {
+            if let Some(stripped) = arg.strip_prefix('@') {
+                config.data =
+                    std::fs::read(stripped).unwrap_or_else(|e| report_and_exit(&e.to_string()));
+            } else {
+                config.data = arg.as_bytes().to_vec();
+            }
+        }
+
+        if args.is_present("stdin") {
+            config.stdin = true;
+        } else if let Some(url) = args.value_of("url") {
+            config.target_url = String::from(url);
+        }
+
         if let Some(arg) = args.values_of("url_denylist") {
-            config.url_denylist = arg.map(|val| val.to_string()).collect();
+            // compile all regular expressions and absolute urls used for --dont-scan
+            //
+            // when --dont-scan is used, the should_deny_url function is called at least once per
+            // url to be scanned. With the addition of regex support, I want to move parsing
+            // out of should_deny_url and into here, so it's performed once instead of thousands
+            // of times
+            for denier in arg {
+                // could be an absolute url or a regex, need to determine which and populate the
+                // appropriate vector
+                match Url::parse(denier.trim_end_matches('/')) {
+                    Ok(absolute) => {
+                        // denier is an absolute url and can be parsed as such
+                        config.url_denylist.push(absolute);
+                    }
+                    Err(err) => {
+                        // there are some expected errors that happen when we try to parse a url
+                        //     ex: Url::parse("/login") -> Err("relative URL without a base")
+                        //     ex: Url::parse("http:") -> Err("empty host")
+                        //
+                        // these are known errors and are used to determine a valid value to
+                        // --dont-scan, when it's not an absolute url
+                        //
+                        // when expected errors are encountered, we're going to assume
+                        // that the input is a regular expression to be parsed. The possibility
+                        // exists that the user rolled their face across the keyboard and we're
+                        // dealing with the results, in which case we'll report it as an error and
+                        // give up
+                        if err.to_string().contains("relative URL without a base")
+                            || err.to_string().contains("empty host")
+                        {
+                            let regex = Regex::new(denier)
+                                .unwrap_or_else(|e| report_and_exit(&e.to_string()));
+
+                            config.regex_denylist.push(regex);
+                        } else {
+                            // unexpected error has occurred; bail
+                            report_and_exit(&err.to_string());
+                        }
+                    }
+                }
+            }
         }
 
         if let Some(arg) = args.values_of("filter_regex") {
@@ -633,19 +722,17 @@ impl Configuration {
             config.json = true;
         }
 
-        if args.is_present("stdin") {
-            config.stdin = true;
-        } else if let Some(url) = args.value_of("url") {
-            config.target_url = String::from(url);
-        }
-
         ////
         // organizational breakpoint; all options below alter the Client configuration
         ////
-        update_config_if_present!(&mut config.proxy, args, "proxy", String);
-        update_config_if_present!(&mut config.replay_proxy, args, "replay_proxy", String);
-        update_config_if_present!(&mut config.user_agent, args, "user_agent", String);
-        update_config_if_present!(&mut config.timeout, args, "timeout", u64);
+        update_config_if_present!(&mut config.proxy, args, "proxy");
+        update_config_if_present!(&mut config.replay_proxy, args, "replay_proxy");
+        update_config_if_present!(&mut config.user_agent, args, "user_agent");
+        update_config_if_present!(&mut config.timeout, args, "timeout");
+
+        if args.is_present("random_agent") {
+            config.random_agent = true;
+        }
 
         if args.is_present("redirects") {
             config.redirects = true;
@@ -669,6 +756,22 @@ impl Configuration {
             }
         }
 
+        if let Some(cookies) = args.values_of("cookies") {
+            config.headers.insert(
+                // we know the header name is always "cookie"
+                "Cookie".to_string(),
+                // on splitting, there should be only two elements,
+                // a key and a value
+                cookies
+                    .map(|cookie| cookie.split('=').collect::<Vec<&str>>()[..].to_owned())
+                    .filter(|parts| parts.len() == 2)
+                    .map(|parts| format!("{}={}", parts[0].trim(), parts[1].trim()))
+                    // trim the spaces, join with an equals sign
+                    .collect::<Vec<String>>()
+                    .join("; "), // join all the cookies with semicolons for the final header
+            );
+        }
+
         if let Some(queries) = args.values_of("queries") {
             for val in queries {
                 // same basic logic used as reading in the headers HashMap above
@@ -748,7 +851,7 @@ impl Configuration {
             config.config = conf_str;
 
             // update the settings
-            Self::merge_config(&mut config, settings);
+            Self::merge_config(config, settings);
         }
         Ok(())
     }
@@ -777,11 +880,17 @@ impl Configuration {
         update_if_not_default!(&mut conf.insecure, new.insecure, false);
         update_if_not_default!(&mut conf.extract_links, new.extract_links, false);
         update_if_not_default!(&mut conf.extensions, new.extensions, Vec::<String>::new());
-        update_if_not_default!(
-            &mut conf.url_denylist,
-            new.url_denylist,
-            Vec::<String>::new()
-        );
+        update_if_not_default!(&mut conf.methods, new.methods, Vec::<String>::new());
+        update_if_not_default!(&mut conf.data, new.data, Vec::<u8>::new());
+        update_if_not_default!(&mut conf.url_denylist, new.url_denylist, Vec::<Url>::new());
+        if !new.regex_denylist.is_empty() {
+            // cant use the update_if_not_default macro due to the following error
+            //
+            //    binary operation `!=` cannot be applied to type `Vec<regex::Regex>`
+            //
+            // if we get a non-empty list of regex in the new config, override the old
+            conf.regex_denylist = new.regex_denylist;
+        }
         update_if_not_default!(&mut conf.headers, new.headers, HashMap::new());
         update_if_not_default!(&mut conf.queries, new.queries, Vec::new());
         update_if_not_default!(&mut conf.no_recursion, new.no_recursion, false);
@@ -824,6 +933,7 @@ impl Configuration {
 
         update_if_not_default!(&mut conf.timeout, new.timeout, timeout());
         update_if_not_default!(&mut conf.user_agent, new.user_agent, user_agent());
+        update_if_not_default!(&mut conf.random_agent, new.random_agent, false);
         update_if_not_default!(&mut conf.threads, new.threads, threads());
         update_if_not_default!(&mut conf.depth, new.depth, depth());
         update_if_not_default!(&mut conf.wordlist, new.wordlist, wordlist());

src/config/tests.rs

@@ -1,6 +1,8 @@
 use super::utils::*;
 use super::*;
 use crate::{traits::FeroxSerialize, DEFAULT_CONFIG_NAME};
+use regex::Regex;
+use reqwest::Url;
 use std::{collections::HashMap, fs::write};
 use tempfile::TempDir;
 
@@ -29,7 +31,10 @@ fn setup_config_test() -> Configuration {
             redirects = true
             insecure = true
             extensions = ["html", "php", "js"]
+            methods = ["GET", "PUT", "DELETE"]
+            data = [31, 32, 33, 34]
             url_denylist = ["http://dont-scan.me", "https://also-not.me"]
+            regex_denylist = ["/deny.*"]
             headers = {stuff = "things", mostuff = "mothings"}
             queries = [["name","value"], ["rick", "astley"]]
             no_recursion = true
@@ -81,6 +86,7 @@ fn default_configuration() {
     assert!(!config.auto_bail);
     assert_eq!(config.requester_policy, RequesterPolicy::Default);
     assert!(!config.no_recursion);
+    assert!(!config.random_agent);
     assert!(!config.json);
     assert!(config.save_state);
     assert!(!config.stdin);
@@ -88,10 +94,13 @@ fn default_configuration() {
     assert!(!config.redirects);
     assert!(!config.extract_links);
     assert!(!config.insecure);
+    assert!(config.regex_denylist.is_empty());
     assert_eq!(config.queries, Vec::new());
     assert_eq!(config.filter_size, Vec::<u64>::new());
     assert_eq!(config.extensions, Vec::<String>::new());
-    assert_eq!(config.url_denylist, Vec::<String>::new());
+    assert_eq!(config.methods, vec!["GET"]);
+    assert_eq!(config.data, Vec::<u8>::new());
+    assert_eq!(config.url_denylist, Vec::<Url>::new());
     assert_eq!(config.filter_regex, Vec::<String>::new());
     assert_eq!(config.filter_similar, Vec::<String>::new());
     assert_eq!(config.filter_word_count, Vec::<usize>::new());
@@ -289,13 +298,40 @@ fn config_reads_extensions() {
     assert_eq!(config.extensions, vec!["html", "php", "js"]);
 }
 
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_methods() {
+    let config = setup_config_test();
+    assert_eq!(config.methods, vec!["GET", "PUT", "DELETE"]);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_data() {
+    let config = setup_config_test();
+    assert_eq!(config.data, vec![31, 32, 33, 34]);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_regex_denylist() {
+    let config = setup_config_test();
+    assert_eq!(
+        config.regex_denylist[0].as_str(),
+        Regex::new("/deny.*").unwrap().as_str()
+    );
+}
+
 #[test]
 /// parse the test config and see that the value parsed is correct
 fn config_reads_url_denylist() {
     let config = setup_config_test();
     assert_eq!(
         config.url_denylist,
-        vec!["http://dont-scan.me", "https://also-not.me"]
+        vec![
+            Url::parse("http://dont-scan.me").unwrap(),
+            Url::parse("https://also-not.me").unwrap(),
+        ]
     );
 }
 
@@ -383,6 +419,12 @@ fn config_reads_queries() {
     assert_eq!(config.queries, queries);
 }
 
+#[test]
+fn config_default_not_random_agent() {
+    let config = setup_config_test();
+    assert!(!config.random_agent);
+}
+
 #[test]
 #[should_panic]
 /// test that an error message is printed and panic is called when report_and_exit is called

src/config/utils.rs

@@ -1,6 +1,6 @@
 use crate::{
     utils::{module_colorizer, status_colorizer},
-    DEFAULT_STATUS_CODES, DEFAULT_WORDLIST, VERSION,
+    DEFAULT_METHOD, DEFAULT_STATUS_CODES, DEFAULT_WORDLIST, VERSION,
 };
 #[cfg(not(test))]
 use std::process::exit;
@@ -52,6 +52,11 @@ pub(super) fn status_codes() -> Vec<u16> {
         .collect()
 }
 
+/// default HTTP Method
+pub(super) fn methods() -> Vec<String> {
+    vec![DEFAULT_METHOD.to_owned()]
+}
+
 /// default wordlist
 pub(super) fn wordlist() -> String {
     String::from(DEFAULT_WORDLIST)

src/event_handlers/command.rs

@@ -48,6 +48,9 @@ pub enum Command {
     /// Send a group of urls to be scanned (only used for the urls passed in explicitly by the user)
     ScanInitialUrls(Vec<String>),
 
+    /// Send a single url to be scanned (presumably added from the interactive menu)
+    ScanNewUrl(String),
+
     /// Determine whether or not recursion is appropriate, given a FeroxResponse, if so start a scan
     TryRecursion(Box<FeroxResponse>),
 

src/event_handlers/container.rs

@@ -85,13 +85,7 @@ impl Handles {
         let configuration = config.unwrap_or_else(|| Arc::new(Configuration::new().unwrap()));
         let (tx, rx) = mpsc::unbounded_channel::<Command>();
         let terminal_handle = TermOutHandle::new(tx.clone(), tx.clone());
-        let stats_handle = StatsHandle::new(
-            Arc::new(Stats::new(
-                configuration.extensions.len(),
-                configuration.json,
-            )),
-            tx.clone(),
-        );
+        let stats_handle = StatsHandle::new(Arc::new(Stats::new(configuration.json)), tx.clone());
         let filters_handle = FiltersHandle::new(Arc::new(FeroxFilters::default()), tx.clone());
         let handles = Self::new(stats_handle, filters_handle, terminal_handle, configuration);
         if let Some(sh) = scanned_urls {

src/event_handlers/inputs.rs

@@ -33,7 +33,7 @@ pub struct TermInputHandler {
 ///
 /// kicks off the following handlers related to terminal input:
 ///     ctrl+c handler that saves scan state to disk
-///     enter handler that listens for enter during scans to drop into interactive scan cancel menu
+///     enter handler that listens for enter during scans to drop into interactive scan management menu
 impl TermInputHandler {
     /// Create new event handler
     pub fn new(handles: Arc<Handles>) -> Self {

src/event_handlers/outputs.rs

@@ -214,7 +214,10 @@ impl TermOutHandler {
                         make_request(
                             self.config.replay_client.as_ref().unwrap(),
                             resp.url(),
+                            resp.method().as_str(),
+                            None,
                             self.config.output_level,
+                            &self.config,
                             tx_stats.clone(),
                         )
                         .await

src/event_handlers/scans.rs

@@ -146,6 +146,15 @@ impl ScanHandler {
                 Command::ScanInitialUrls(targets) => {
                     self.ordered_scan_url(targets, ScanOrder::Initial).await?;
                 }
+                Command::ScanNewUrl(target) => {
+                    // added as part of interactive menu ability (2.4.1) to add a new scan.
+                    // we don't have a way of knowing if they're adding a new url entirely (i.e.
+                    // new base url), or simply adding a new sub-directory found some other way.
+                    // Since we can't know, we'll start a scan as though we received the scan
+                    // from -u | --stdin
+                    self.ordered_scan_url(vec![target], ScanOrder::Initial)
+                        .await?;
+                }
                 Command::UpdateWordlist(wordlist) => {
                     self.wordlist(wordlist);
                 }
@@ -189,7 +198,8 @@ impl ScanHandler {
     /// wrapper around scanning a url to stay DRY
     async fn ordered_scan_url(&mut self, targets: Vec<String>, order: ScanOrder) -> Result<()> {
         log::trace!("enter: ordered_scan_url({:?}, {:?})", targets, order);
-        let should_test_deny = !self.handles.config.url_denylist.is_empty();
+        let should_test_deny = !self.handles.config.url_denylist.is_empty()
+            || !self.handles.config.regex_denylist.is_empty();
 
         for target in targets {
             if self.data.contains(&target) && matches!(order, ScanOrder::Latest) {

src/event_handlers/statistics.rs

@@ -155,7 +155,7 @@ impl StatsHandler {
     pub fn initialize(config: Arc<Configuration>) -> (Joiner, StatsHandle) {
         log::trace!("enter: initialize");
 
-        let data = Arc::new(Stats::new(config.extensions.len(), config.json));
+        let data = Arc::new(Stats::new(config.json));
         let (tx, rx): FeroxChannel<Command> = mpsc::unbounded_channel();
 
         let mut handler = StatsHandler::new(data.clone(), rx);

src/extractor/builder.rs

@@ -21,6 +21,9 @@ pub enum ExtractionTarget {
 
     /// Examine robots.txt (specifically) and extract links
     RobotsTxt,
+
+    // Parse HTML and extract links
+    ParseHtml,
 }
 
 /// responsible for building an `Extractor`
@@ -28,7 +31,7 @@ pub struct ExtractorBuilder<'a> {
     /// Response from which to extract links
     response: Option<&'a FeroxResponse>,
 
-    /// Response from which to extract links
+    /// URL of where to extract links
     url: String,
 
     /// Handles object to house the underlying mpsc transmitters

src/extractor/container.rs

@@ -14,9 +14,11 @@ use crate::{
     },
     url::FeroxUrl,
     utils::{logged_request, make_request},
+    DEFAULT_METHOD,
 };
 use anyhow::{bail, Context, Result};
 use reqwest::{StatusCode, Url};
+use scraper::{Html, Selector};
 use std::collections::HashSet;
 use tokio::sync::oneshot;
 
@@ -42,7 +44,7 @@ pub struct Extractor<'a> {
     /// Response from which to extract links
     pub(super) response: Option<&'a FeroxResponse>,
 
-    /// Response from which to extract links
+    /// URL of where to extract links
     pub(super) url: String,
 
     /// Handles object to house the underlying mpsc transmitters
@@ -55,11 +57,12 @@ pub struct Extractor<'a> {
 /// Extractor implementation
 impl<'a> Extractor<'a> {
     /// perform extraction from the given target and return any links found
-    pub async fn extract(&self) -> Result<HashSet<String>> {
+    pub async fn extract(&self) -> Result<(HashSet<String>, bool)> {
         log::trace!("enter: extract (this fn has associated trace exit msg)");
         match self.target {
             ExtractionTarget::ResponseBody => Ok(self.extract_from_body().await?),
             ExtractionTarget::RobotsTxt => Ok(self.extract_from_robots().await?),
+            ExtractionTarget::ParseHtml => Ok(self.parse_html().await?),
         }
     }
 
@@ -91,11 +94,11 @@ impl<'a> Extractor<'a> {
                 continue;
             }
 
-            if resp.is_file() {
-                // very likely a file, simply request and report
-                log::debug!("Extracted file: {}", resp);
+            // request and report assumed file
+            if resp.is_file() || !resp.is_directory() {
+                log::debug!("Extracted File: {}", resp);
 
-                scanned_urls.add_file_scan(&resp.url().to_string(), ScanOrder::Latest);
+                scanned_urls.add_file_scan(resp.url().as_str(), ScanOrder::Latest);
 
                 if let Err(e) = resp.send_report(self.handles.output.tx.clone()) {
                     log::warn!("Could not send FeroxResponse to output handler: {}", e);
@@ -142,12 +145,28 @@ impl<'a> Extractor<'a> {
     ///         - homepage/assets/img/
     ///         - homepage/assets/
     ///         - homepage/
-    pub(super) async fn extract_from_body(&self) -> Result<HashSet<String>> {
-        log::trace!("enter: get_links");
+    pub(super) async fn extract_from_body(&self) -> Result<(HashSet<String>, bool)> {
+        log::trace!("enter: extract_from_body");
 
         let mut links = HashSet::<String>::new();
+        let dirlist_flag = false;
 
-        let body = self.response.unwrap().text();
+        // Response
+        let response = self.response.unwrap();
+        let resp_url = response.url();
+        let body = response.text();
+        let html = Html::parse_document(body);
+
+        // Extract Links
+        self.extract_links_by_attr(resp_url, &mut links, &html, "a", "href");
+        self.extract_links_by_attr(resp_url, &mut links, &html, "img", "src");
+        self.extract_links_by_attr(resp_url, &mut links, &html, "form", "action");
+        self.extract_links_by_attr(resp_url, &mut links, &html, "script", "src");
+        self.extract_links_by_attr(resp_url, &mut links, &html, "iframe", "src");
+        self.extract_links_by_attr(resp_url, &mut links, &html, "div", "src");
+        self.extract_links_by_attr(resp_url, &mut links, &html, "frame", "src");
+        self.extract_links_by_attr(resp_url, &mut links, &html, "embed", "src");
+        self.extract_links_by_attr(resp_url, &mut links, &html, "script", "src");
 
         for capture in self.links_regex.captures_iter(body) {
             // remove single & double quotes from both ends of the capture
@@ -187,22 +206,21 @@ impl<'a> Extractor<'a> {
 
         self.update_stats(links.len())?;
 
-        log::trace!("exit: get_links -> {:?}", links);
-
-        Ok(links)
+        log::trace!("exit: extract_from_body -> {:?} {}", links, dirlist_flag);
+        Ok((links, dirlist_flag))
     }
 
     /// take a url fragment like homepage/assets/img/icons/handshake.svg and
     /// incrementally add
-    ///     - homepage/assets/img/icons/
-    ///     - homepage/assets/img/
-    ///     - homepage/assets/
-    ///     - homepage/
-    fn add_all_sub_paths(&self, url_path: &str, mut links: &mut HashSet<String>) -> Result<()> {
+    ///   - homepage/assets/img/icons/
+    ///   - homepage/assets/img/
+    ///   - homepage/assets/
+    ///   - homepage/
+    fn add_all_sub_paths(&self, url_path: &str, links: &mut HashSet<String>) -> Result<()> {
         log::trace!("enter: add_all_sub_paths({}, {:?})", url_path, links);
 
         for sub_path in self.get_sub_paths_from_path(url_path) {
-            self.add_link_to_set_of_links(&sub_path, &mut links)?;
+            self.add_link_to_set_of_links(&sub_path, links)?;
         }
 
         log::trace!("exit: add_all_sub_paths");
@@ -266,12 +284,14 @@ impl<'a> Extractor<'a> {
 
         let old_url = match self.target {
             ExtractionTarget::ResponseBody => self.response.unwrap().url().clone(),
-            ExtractionTarget::RobotsTxt => match Url::parse(&self.url) {
-                Ok(u) => u,
-                Err(e) => {
-                    bail!("Could not parse {}: {}", self.url, e);
+            ExtractionTarget::ParseHtml | ExtractionTarget::RobotsTxt => {
+                match Url::parse(&self.url) {
+                    Ok(u) => u,
+                    Err(e) => {
+                        bail!("Could not parse {}: {}", self.url, e);
+                    }
                 }
-            },
+            }
         };
 
         let new_url = old_url
@@ -286,11 +306,6 @@ impl<'a> Extractor<'a> {
     }
 
     /// Wrapper around link extraction logic
-    /// currently used in two places:
-    ///   - links from response bodies
-    ///   - links from robots.txt responses
-    ///
-    /// general steps taken:
     ///   - create a new Url object based on cli options/args
     ///   - check if the new Url has already been seen/scanned -> None
     ///   - make a request to the new Url ? -> Some(response) : None
@@ -310,22 +325,31 @@ impl<'a> Extractor<'a> {
             bail!("previously seen url");
         }
 
-        if !self.handles.config.url_denylist.is_empty()
+        if (!self.handles.config.url_denylist.is_empty()
+            || !self.handles.config.regex_denylist.is_empty())
             && should_deny_url(&new_url, self.handles.clone())?
         {
             // can't allow a denied url to be requested
             bail!(
-                "prevented request to {} due to {:?}",
+                "prevented request to {} due to {:?} || {:?}",
                 url,
-                self.handles.config.url_denylist
+                self.handles.config.url_denylist,
+                self.handles.config.regex_denylist,
             );
         }
 
         // make the request and store the response
-        let new_response = logged_request(&new_url, self.handles.clone()).await?;
+        let new_response =
+            logged_request(&new_url, DEFAULT_METHOD, None, self.handles.clone()).await?;
 
-        let new_ferox_response =
-            FeroxResponse::from(new_response, true, self.handles.config.output_level).await;
+        let new_ferox_response = FeroxResponse::from(
+            new_response,
+            url,
+            DEFAULT_METHOD,
+            true,
+            self.handles.config.output_level,
+        )
+        .await;
 
         log::trace!("exit: request_link -> {:?}", new_ferox_response);
 
@@ -340,14 +364,17 @@ impl<'a> Extractor<'a> {
     ///     http://localhost/stuff/things
     /// this function requests:
     ///     http://localhost/robots.txt
-    pub(super) async fn extract_from_robots(&self) -> Result<HashSet<String>> {
+    pub(super) async fn extract_from_robots(&self) -> Result<(HashSet<String>, bool)> {
         log::trace!("enter: extract_robots_txt");
 
         let mut links: HashSet<String> = HashSet::new();
+        let dirlist_flag = false;
 
-        let response = self.request_robots_txt().await?;
+        // request
+        let response = self.make_extract_request("/robots.txt").await?;
+        let body = response.text();
 
-        for capture in self.robots_regex.captures_iter(response.text()) {
+        for capture in self.robots_regex.captures_iter(body) {
             if let Some(new_path) = capture.name("url_path") {
                 let mut new_url = Url::parse(&self.url)?;
                 new_url.set_path(new_path.as_str());
@@ -359,19 +386,126 @@ impl<'a> Extractor<'a> {
 
         self.update_stats(links.len())?;
 
-        log::trace!("exit: extract_robots_txt -> {:?}", links);
-        Ok(links)
+        log::trace!("exit: extract_robots_txt -> {:?} {}", links, dirlist_flag);
+        Ok((links, dirlist_flag))
     }
 
-    /// helper function that simply requests /robots.txt on the given url's base url
+    /// Entry point to parse html for links (i.e. webscraping, directory listings)
+    /// this function requests:
+    ///     http://localhost/<location>
+    pub(super) async fn parse_html(&self) -> Result<(HashSet<String>, bool)> {
+        log::trace!("enter: parse_html");
+
+        let mut links: HashSet<String> = HashSet::new();
+        let mut dirlist_flag = false;
+
+        // Response
+        let url = Url::parse(&self.url)?;
+        let response = self.make_extract_request(url.path()).await?;
+        let resp_url = response.url();
+        let body = response.text();
+        let html = Html::parse_document(body);
+
+        // Directory listing heuristic detection to not continue scanning
+        // Index of /: apache
+        // Directory Listing for /: tomcat,
+        // Directory Listing -- /: ASP.NET
+        // <host> - /: iis, azure, skipping due to loose heuristic
+        let title_selector = Selector::parse("title").unwrap();
+        for t in html.select(&title_selector) {
+            let title = t.inner_html().to_lowercase();
+            if title.contains("directory listing for /")
+                || title.contains("index of /")
+                || title.contains("directory listing -- /")
+            {
+                log::debug!("Directory listing heuristic detection from \"{}\"", title);
+                dirlist_flag = true;
+
+                self.extract_links_by_attr(resp_url, &mut links, &html, "a", "href");
+                self.update_stats(links.len())?;
+
+                log::trace!("exit: parse_html -> {:?} {}", links, dirlist_flag);
+                return Ok((links, dirlist_flag));
+            }
+        }
+
+        // Extract Links
+        self.extract_links_by_attr(resp_url, &mut links, &html, "a", "href");
+        self.extract_links_by_attr(resp_url, &mut links, &html, "img", "src");
+        self.extract_links_by_attr(resp_url, &mut links, &html, "form", "action");
+        self.extract_links_by_attr(resp_url, &mut links, &html, "script", "src");
+        self.extract_links_by_attr(resp_url, &mut links, &html, "iframe", "src");
+        self.extract_links_by_attr(resp_url, &mut links, &html, "div", "src");
+        self.extract_links_by_attr(resp_url, &mut links, &html, "frame", "src");
+        self.extract_links_by_attr(resp_url, &mut links, &html, "embed", "src");
+        self.extract_links_by_attr(resp_url, &mut links, &html, "script", "src");
+
+        self.update_stats(links.len())?;
+
+        log::trace!("exit: parse_html -> {:?} {}", links, dirlist_flag);
+        Ok((links, dirlist_flag))
+    }
+
+    /// simple helper to get html links by tag/attribute and add it to the `links` HashSet
+    fn extract_links_by_attr(
+        &self,
+        resp_url: &Url,
+        links: &mut HashSet<String>,
+        html: &Html,
+        html_tag: &str,
+        html_attr: &str,
+    ) {
+        log::trace!("enter: extract_links_by_attr");
+
+        let selector = Selector::parse(html_tag).unwrap();
+        let tags = html
+            .select(&selector)
+            .filter(|a| a.value().attrs().any(|attr| attr.0 == html_attr));
+        for t in tags {
+            if let Some(link) = t.value().attr(html_attr) {
+                log::debug!("Parsed link \"{}\" from {}", link, resp_url.as_str());
+
+                match Url::parse(link) {
+                    Ok(absolute) => {
+                        if absolute.domain() != resp_url.domain()
+                            || absolute.host() != resp_url.host()
+                        {
+                            // domains/ips are not the same, don't scan things that aren't part of the original
+                            // target url
+                            continue;
+                        }
+
+                        if self.add_all_sub_paths(absolute.path(), links).is_err() {
+                            log::warn!("could not add sub-paths from {} to {:?}", absolute, links);
+                        }
+                    }
+                    Err(e) => {
+                        // this is the expected error that happens when we try to parse a url fragment
+                        //     ex: Url::parse("/login") -> Err("relative URL without a base")
+                        // while this is technically an error, these are good results for us
+                        if e.to_string().contains("relative URL without a base") {
+                            if self.add_all_sub_paths(link, links).is_err() {
+                                log::warn!("could not add sub-paths from {} to {:?}", link, links);
+                            }
+                        } else {
+                            // unexpected error has occurred
+                            log::warn!("Could not parse given url: {}", e);
+                            self.handles.stats.send(AddError(Other)).unwrap_or_default();
+                        }
+                    }
+                }
+            }
+        }
+
+        log::trace!("exit: extract_links_by_attr");
+    }
+
+    /// helper function that simply requests at <location> on the given url's base url
     ///
     /// example:
-    ///     http://localhost/api/users -> http://localhost/robots.txt
-    ///     
-    /// The length of the given path has no effect on what's requested; it's always
-    /// base url + /robots.txt
-    pub(super) async fn request_robots_txt(&self) -> Result<FeroxResponse> {
-        log::trace!("enter: get_robots_file");
+    ///     http://localhost/api/users -> http://localhost/<location>
+    pub(super) async fn make_extract_request(&self, location: &str) -> Result<FeroxResponse> {
+        log::trace!("enter: make_extract_request");
 
         // more often than not, domain/robots.txt will redirect to www.domain/robots.txt or something
         // similar; to account for that, create a client that will follow redirects, regardless of
@@ -395,21 +529,30 @@ impl<'a> Extractor<'a> {
         )?;
 
         let mut url = Url::parse(&self.url)?;
-        url.set_path("/robots.txt"); // overwrite existing path with /robots.txt
+        url.set_path(location); // overwrite existing path
 
         // purposefully not using logged_request here due to using the special client
         let response = make_request(
             &client,
             &url,
+            DEFAULT_METHOD,
+            None,
             self.handles.config.output_level,
+            &self.handles.config,
             self.handles.stats.tx.clone(),
         )
         .await?;
 
-        let ferox_response =
-            FeroxResponse::from(response, true, self.handles.config.output_level).await;
+        let ferox_response = FeroxResponse::from(
+            response,
+            &self.url,
+            DEFAULT_METHOD,
+            true,
+            self.handles.config.output_level,
+        )
+        .await;
 
-        log::trace!("exit: get_robots_file -> {}", ferox_response);
+        log::trace!("exit: make_extract_request -> {}", ferox_response);
         Ok(ferox_response)
     }
 

src/extractor/tests.rs

@@ -4,6 +4,7 @@ use crate::config::{Configuration, OutputLevel};
 use crate::scan_manager::ScanOrder;
 use crate::{
     event_handlers::Handles, scan_manager::FeroxScans, utils::make_request, Command, FeroxChannel,
+    DEFAULT_METHOD,
 };
 use anyhow::Result;
 use httpmock::{Method::GET, MockServer};
@@ -19,6 +20,9 @@ lazy_static! {
     /// Extractor for testing response bodies
     static ref BODY_EXT: Extractor<'static> = setup_extractor(ExtractionTarget::ResponseBody, Arc::new(FeroxScans::default()));
 
+    /// Extractor for testing paring html
+    static ref PARSEHTML_EXT: Extractor<'static> = setup_extractor(ExtractionTarget::ParseHtml, Arc::new(FeroxScans::default()));
+
     /// FeroxResponse for Extractor
     static ref RESPONSE: FeroxResponse = get_test_response();
 }
@@ -41,6 +45,9 @@ fn setup_extractor(target: ExtractionTarget, scanned_urls: Arc<FeroxScans>) -> E
         ExtractionTarget::RobotsTxt => builder
             .url("http://localhost")
             .target(ExtractionTarget::RobotsTxt),
+        ExtractionTarget::ParseHtml => builder
+            .url("http://localhost")
+            .target(ExtractionTarget::ParseHtml),
     };
 
     let config = Arc::new(Configuration::new().unwrap());
@@ -211,20 +218,36 @@ async fn extractor_get_links_with_absolute_url_that_differs_from_target_domain()
     let mock = srv.mock(|when, then| {
         when.method(GET).path("/some-path");
         then.status(200).body(
-            "\"http://defintely.not.a.thing.probably.com/homepage/assets/img/icons/handshake.svg\"",
+            "\"http://definitely.not.a.thing.probably.com/homepage/assets/img/icons/handshake.svg\"",
         );
     });
 
     let client = Client::new();
     let url = Url::parse(&srv.url("/some-path")).unwrap();
+    let config = Configuration::new().unwrap();
 
-    let response = make_request(&client, &url, OutputLevel::Default, tx_stats.clone())
-        .await
-        .unwrap();
+    let response = make_request(
+        &client,
+        &url,
+        DEFAULT_METHOD,
+        None,
+        OutputLevel::Default,
+        &config,
+        tx_stats.clone(),
+    )
+    .await
+    .unwrap();
     let (handles, _rx) = Handles::for_testing(None, None);
 
     let handles = Arc::new(handles);
-    let ferox_response = FeroxResponse::from(response, true, OutputLevel::Default).await;
+    let ferox_response = FeroxResponse::from(
+        response,
+        &srv.url(""),
+        DEFAULT_METHOD,
+        true,
+        OutputLevel::Default,
+    )
+    .await;
 
     let extractor = Extractor {
         links_regex: Regex::new(LINKFINDER_REGEX).unwrap(),
@@ -235,7 +258,7 @@ async fn extractor_get_links_with_absolute_url_that_differs_from_target_domain()
         handles: handles.clone(),
     };
 
-    let links = extractor.extract_from_body().await?;
+    let links = (extractor.extract_from_body().await?).0;
 
     assert!(links.is_empty());
     assert_eq!(mock.hits(), 1);
@@ -263,7 +286,7 @@ async fn request_robots_txt_without_proxy() -> Result<()> {
         handles,
     };
 
-    let resp = extractor.request_robots_txt().await?;
+    let resp = extractor.make_extract_request("/robots.txt").await?;
 
     assert!(matches!(resp.status(), &StatusCode::OK));
     println!("{}", resp);
@@ -296,7 +319,7 @@ async fn request_robots_txt_with_proxy() -> Result<()> {
         .handles(handles)
         .build()?;
 
-    let resp = extractor.request_robots_txt().await?;
+    let resp = extractor.make_extract_request("/robots.txt").await?;
 
     assert!(matches!(resp.status(), &StatusCode::OK));
     assert_eq!(resp.content_length(), 19);

src/filters/init.rs

@@ -7,7 +7,7 @@ use crate::{
     skip_fail,
     utils::{fmt_err, logged_request},
     Command::AddFilter,
-    SIMILARITY_THRESHOLD,
+    DEFAULT_METHOD, SIMILARITY_THRESHOLD,
 };
 use anyhow::Result;
 use fuzzyhash::FuzzyHash;
@@ -72,10 +72,17 @@ pub async fn initialize(handles: Arc<Handles>) -> Result<()> {
         let url = skip_fail!(Url::parse(similarity_filter));
 
         // attempt to request the given url
-        let resp = skip_fail!(logged_request(&url, handles.clone()).await);
+        let resp = skip_fail!(logged_request(&url, DEFAULT_METHOD, None, handles.clone()).await);
 
         // if successful, create a filter based on the response's body
-        let fr = FeroxResponse::from(resp, true, handles.config.output_level).await;
+        let fr = FeroxResponse::from(
+            resp,
+            similarity_filter,
+            DEFAULT_METHOD,
+            true,
+            handles.config.output_level,
+        )
+        .await;
 
         // hash the response body and store the resulting hash in the filter object
         let hash = FuzzyHash::new(&fr.text()).to_string();

src/filters/tests.rs

@@ -122,11 +122,25 @@ fn wildcard_should_filter_when_static_wildcard_found() {
         size: 83,
         dynamic: 0,
         dont_filter: false,
+        method: "GET".to_owned(),
     };
 
     assert!(filter.should_filter_response(&resp));
 }
 
+#[test]
+/// test should_filter on WilcardFilter where static logic matches but response length is 0
+fn wildcard_should_filter_when_static_wildcard_len_is_zero() {
+    let mut resp = FeroxResponse::default();
+    resp.set_wildcard(true);
+    resp.set_url("http://localhost");
+
+    // default WildcardFilter is used in the code that executes when response.content_length() == 0
+    let filter = WildcardFilter::new(false);
+
+    assert!(filter.should_filter_response(&resp));
+}
+
 #[test]
 /// test should_filter on WilcardFilter where dynamic logic matches
 fn wildcard_should_filter_when_dynamic_wildcard_found() {
@@ -139,6 +153,7 @@ fn wildcard_should_filter_when_dynamic_wildcard_found() {
         size: 0,
         dynamic: 59, // content-length - 5 (len('stuff'))
         dont_filter: false,
+        method: "GET".to_owned(),
     };
 
     println!("resp: {:?}: filter: {:?}", resp, filter);

src/filters/wildcard.rs

@@ -1,5 +1,5 @@
 use super::*;
-use crate::url::FeroxUrl;
+use crate::{url::FeroxUrl, DEFAULT_METHOD};
 
 /// Data holder for two pieces of data needed when auto-filtering out wildcard responses
 ///
@@ -18,6 +18,9 @@ pub struct WildcardFilter {
     /// size of the response that should be included with filters passed via runtime configuration
     pub size: u64,
 
+    /// method used in request that should be included with filters passed via runtime configuration
+    pub method: String,
+
     /// whether or not the user passed -D on the command line
     pub(super) dont_filter: bool,
 }
@@ -40,6 +43,7 @@ impl Default for WildcardFilter {
         Self {
             dont_filter: false,
             size: u64::MAX,
+            method: DEFAULT_METHOD.to_owned(),
             dynamic: u64::MAX,
         }
     }
@@ -60,7 +64,10 @@ impl FeroxFilter for WildcardFilter {
             return false;
         }
 
-        if self.size != u64::MAX && self.size == response.content_length() {
+        if self.size != u64::MAX
+            && self.size == response.content_length()
+            && self.method == response.method().as_str()
+        {
             // static wildcard size found during testing
             // size isn't default, size equals response length, and auto-filter is on
             log::debug!("static wildcard: filtered out {}", response.url());
@@ -68,6 +75,17 @@ impl FeroxFilter for WildcardFilter {
             return true;
         }
 
+        if self.size == u64::MAX
+            && response.content_length() == 0
+            && self.method == response.method().as_str()
+        {
+            // static wildcard size found during testing
+            // but response length was zero; pointed out by @Tib3rius
+            log::debug!("static wildcard: filtered out {}", response.url());
+            log::trace!("exit: should_filter_response -> true");
+            return true;
+        }
+
         if self.dynamic != u64::MAX {
             // dynamic wildcard offset found during testing
 

src/heuristics.rs

@@ -13,6 +13,7 @@ use crate::{
     skip_fail,
     url::FeroxUrl,
     utils::{ferox_print, fmt_err, logged_request, status_colorizer},
+    DEFAULT_METHOD,
 };
 
 /// length of a standard UUID, used when determining wildcard responses
@@ -20,10 +21,11 @@ const UUID_LENGTH: u64 = 32;
 
 /// wrapper around ugly string formatting
 macro_rules! format_template {
-    ($template:expr, $length:expr) => {
+    ($template:expr, $method:expr, $length:expr) => {
         format!(
             $template,
             status_colorizer("WLD"),
+            $method,
             "-",
             "-",
             "-",
@@ -89,55 +91,68 @@ impl HeuristicTests {
             return Ok(0);
         }
 
+        let data = match self.handles.config.data.is_empty() {
+            true => None,
+            false => Some(self.handles.config.data.as_slice()),
+        };
+
         let ferox_url = FeroxUrl::from_string(target_url, self.handles.clone());
 
-        let ferox_response = self.make_wildcard_request(&ferox_url, 1).await?;
+        for method in self.handles.config.methods.iter() {
+            let ferox_response = self
+                .make_wildcard_request(&ferox_url, method.as_str(), data, 1)
+                .await?;
 
-        // found a wildcard response
-        let mut wildcard = WildcardFilter::new(self.handles.config.dont_filter);
+            // found a wildcard response
+            let mut wildcard = WildcardFilter::new(self.handles.config.dont_filter);
 
-        let wc_length = ferox_response.content_length();
+            let wc_length = ferox_response.content_length();
+
+            if wc_length == 0 {
+                log::trace!("exit: wildcard_test -> 1");
+                self.send_filter(wildcard)?;
+                return Ok(1);
+            }
+
+            // content length of wildcard is non-zero, perform additional tests:
+            //   make a second request, with a known-sized (64) longer request
+            let resp_two = self
+                .make_wildcard_request(&ferox_url, method.as_str(), data, 3)
+                .await?;
+
+            let wc2_length = resp_two.content_length();
+
+            wildcard.method = resp_two.method().as_str().to_owned();
+
+            if wc2_length == wc_length + (UUID_LENGTH * 2) {
+                // second length is what we'd expect to see if the requested url is
+                // reflected in the response along with some static content; aka custom 404
+                let url_len = ferox_url.path_length()?;
+
+                wildcard.dynamic = wc_length - url_len;
+
+                if matches!(
+                    self.handles.config.output_level,
+                    OutputLevel::Default | OutputLevel::Quiet
+                ) {
+                    let msg = format_template!("{} {:>8} {:>9} {:>9} {:>9} Wildcard response is dynamic; {} ({} + url length) responses; toggle this behavior by using {}\n", method, wildcard.dynamic);
+                    ferox_print(&msg, &PROGRESS_PRINTER);
+                }
+            } else if wc_length == wc2_length {
+                wildcard.size = wc_length;
+
+                if matches!(
+                    self.handles.config.output_level,
+                    OutputLevel::Default | OutputLevel::Quiet
+                ) {
+                    let msg = format_template!("{} {:>8} {:>9} {:>9} {:>9} Wildcard response is static; {} {} responses; toggle this behavior by using {}\n", method, wildcard.size);
+                    ferox_print(&msg, &PROGRESS_PRINTER);
+                }
+            }
 
-        if wc_length == 0 {
-            log::trace!("exit: wildcard_test -> 1");
             self.send_filter(wildcard)?;
-            return Ok(1);
         }
 
-        // content length of wildcard is non-zero, perform additional tests:
-        //   make a second request, with a known-sized (64) longer request
-        let resp_two = self.make_wildcard_request(&ferox_url, 3).await?;
-
-        let wc2_length = resp_two.content_length();
-
-        if wc2_length == wc_length + (UUID_LENGTH * 2) {
-            // second length is what we'd expect to see if the requested url is
-            // reflected in the response along with some static content; aka custom 404
-            let url_len = ferox_url.path_length()?;
-
-            wildcard.dynamic = wc_length - url_len;
-
-            if matches!(
-                self.handles.config.output_level,
-                OutputLevel::Default | OutputLevel::Quiet
-            ) {
-                let msg = format_template!("{} {:>9} {:>9} {:>9} Wildcard response is dynamic; {} ({} + url length) responses; toggle this behavior by using {}\n", wildcard.dynamic);
-                ferox_print(&msg, &PROGRESS_PRINTER);
-            }
-        } else if wc_length == wc2_length {
-            wildcard.size = wc_length;
-
-            if matches!(
-                self.handles.config.output_level,
-                OutputLevel::Default | OutputLevel::Quiet
-            ) {
-                let msg = format_template!("{} {:>9} {:>9} {:>9} Wildcard response is static; {} {} responses; toggle this behavior by using {}\n", wildcard.size);
-                ferox_print(&msg, &PROGRESS_PRINTER);
-            }
-        }
-
-        self.send_filter(wildcard)?;
-
         log::trace!("exit: wildcard_test");
         Ok(2)
     }
@@ -151,14 +166,30 @@ impl HeuristicTests {
     async fn make_wildcard_request(
         &self,
         target: &FeroxUrl,
+        method: &str,
+        data: Option<&[u8]>,
         length: usize,
     ) -> Result<FeroxResponse> {
         log::trace!("enter: make_wildcard_request({}, {})", target, length);
 
         let unique_str = self.unique_string(length);
-        let nonexistent_url = target.format(&unique_str, None)?;
 
-        let response = logged_request(&nonexistent_url.to_owned(), self.handles.clone()).await?;
+        // To take care of slash when needed
+        let slash = if self.handles.config.add_slash {
+            Some("/")
+        } else {
+            None
+        };
+
+        let nonexistent_url = target.format(&unique_str, slash)?;
+
+        let response = logged_request(
+            &nonexistent_url.to_owned(),
+            method,
+            data,
+            self.handles.clone(),
+        )
+        .await?;
 
         if self
             .handles
@@ -167,8 +198,14 @@ impl HeuristicTests {
             .contains(&response.status().as_u16())
         {
             // found a wildcard response
-            let mut ferox_response =
-                FeroxResponse::from(response, true, self.handles.config.output_level).await;
+            let mut ferox_response = FeroxResponse::from(
+                response,
+                &target.target,
+                method,
+                true,
+                self.handles.config.output_level,
+            )
+            .await;
             ferox_response.set_wildcard(true);
 
             if self
@@ -210,7 +247,7 @@ impl HeuristicTests {
             let url = FeroxUrl::from_string(target_url, self.handles.clone());
             let request = skip_fail!(url.format("", None));
 
-            let result = logged_request(&request, self.handles.clone()).await;
+            let result = logged_request(&request, DEFAULT_METHOD, None, self.handles.clone()).await;
 
             match result {
                 Ok(_) => {

src/lib.rs

@@ -87,10 +87,28 @@ pub const DEFAULT_STATUS_CODES: [StatusCode; 10] = [
     StatusCode::INTERNAL_SERVER_ERROR,
 ];
 
+/// Default method for requests
+pub(crate) const DEFAULT_METHOD: &str = "GET";
+
 /// Default filename for config file settings
 ///
 /// Expected location is in the same directory as the feroxbuster binary.
 pub const DEFAULT_CONFIG_NAME: &str = "ferox-config.toml";
+/// User agents to select from when random agent is being used
+pub const USER_AGENTS: [&str; 12] = [
+    "Mozilla/5.0 (Linux; Android 8.0.0; SM-G960F Build/R16NW) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.84 Mobile Safari/537.36",
+    "Mozilla/5.0 (iPhone; CPU iPhone OS 12_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1",
+    "Mozilla/5.0 (Windows Phone 10.0; Android 6.0.1; Microsoft; RM-1152) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Mobile Safari/537.36 Edge/15.15254",
+    "Mozilla/5.0 (Linux; Android 7.0; Pixel C Build/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/52.0.2743.98 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246",
+    "Mozilla/5.0 (X11; CrOS x86_64 8172.45.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.64 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/601.3.9 (KHTML, like Gecko) Version/9.0.2 Safari/601.3.9",
+    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.111 Safari/537.36",
+    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 Firefox/15.0.1",
+    "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
+    "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)",
+    "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)",
+];
 
 #[cfg(test)]
 mod tests {

src/main.rs

@@ -51,16 +51,12 @@ fn get_unique_words_from_wordlist(path: &str) -> Result<Arc<Vec<String>>> {
     let mut words = Vec::new();
 
     for line in reader.lines() {
-        let result = match line {
-            Ok(read_line) => read_line,
-            Err(_) => continue,
-        };
-
-        if result.starts_with('#') || result.is_empty() {
-            continue;
-        }
-
-        words.push(result);
+        line.map(|result| {
+            if !result.starts_with('#') && !result.is_empty() {
+                words.push(result);
+            }
+        })
+        .ok();
     }
 
     log::trace!(
@@ -154,6 +150,28 @@ async fn get_targets(handles: Arc<Handles>) -> Result<Vec<String>> {
         targets.push(handles.config.target_url.clone());
     }
 
+    // remove footgun that arises if a --dont-scan value matches on a base url
+    for target in &targets {
+        for denier in &handles.config.regex_denylist {
+            if denier.is_match(target) {
+                bail!(
+                    "The regex '{}' matches {}; the scan will never start",
+                    denier,
+                    target
+                );
+            }
+        }
+        for denier in &handles.config.url_denylist {
+            if denier.as_str().trim_end_matches('/') == target.trim_end_matches('/') {
+                bail!(
+                    "The url '{}' matches {}; the scan will never start",
+                    denier,
+                    target
+                );
+            }
+        }
+    }
+
     log::trace!("exit: get_targets -> {:?}", targets);
 
     Ok(targets)
@@ -231,7 +249,7 @@ async fn wrapped_main(config: Arc<Configuration>) -> Result<()> {
         Err(e) => {
             // should only happen in the event that there was an error reading from stdin
             clean_up(handles, tasks).await?;
-            bail!("Could not get determine initial targets: {}", e);
+            bail!("Could not determine initial targets: {}", e);
         }
     };
 

src/parser.rs

@@ -1,4 +1,6 @@
-use clap::{App, Arg, ArgGroup};
+use clap::{
+    crate_authors, crate_description, crate_name, crate_version, App, Arg, ArgGroup, ValueHint,
+};
 use lazy_static::lazy_static;
 use regex::Regex;
 use std::env;
@@ -14,382 +16,567 @@ lazy_static! {
     /// - 1d
     pub static ref TIMESPEC_REGEX: Regex =
         Regex::new(r"^(?i)(?P<n>\d+)(?P<m>[smdh])$").expect("Could not compile regex");
+
+    /// help string for user agent, your guess is as good as mine as to why this is required...
+    static ref DEFAULT_USER_AGENT: String = format!(
+        "Sets the User-Agent (default: feroxbuster/{})",
+        crate_version!()
+    );
 }
 
 /// Create and return an instance of [clap::App](https://docs.rs/clap/latest/clap/struct.App.html), i.e. the Command Line Interface's configuration
-pub fn initialize() -> App<'static, 'static> {
-    let mut app = App::new("feroxbuster")
-        .version(env!("CARGO_PKG_VERSION"))
-        .author("Ben 'epi' Risher (@epi052)")
-        .about("A fast, simple, recursive content discovery tool written in Rust")
+pub fn initialize() -> App<'static> {
+    let app = App::new(crate_name!())
+        .version(crate_version!())
+        .author(crate_authors!())
+        .about(crate_description!());
+
+    /////////////////////////////////////////////////////////////////////
+    // group - target selection
+    /////////////////////////////////////////////////////////////////////
+    let app = app
         .arg(
-            Arg::with_name("wordlist")
-                .short("w")
-                .long("wordlist")
-                .value_name("FILE")
-                .help("Path to the wordlist")
-                .takes_value(true),
-        )
-        .arg(
-            Arg::with_name("url")
-                .short("u")
+            Arg::new("url")
+                .short('u')
                 .long("url")
-                .required_unless_one(&["stdin", "resume_from"])
+                .required_unless_present_any(&["stdin", "resume_from"])
+                .help_heading("Target selection")
                 .value_name("URL")
-                .multiple(true)
                 .use_delimiter(true)
-                .help("The target URL(s) (required, unless --stdin used)"),
+                .value_hint(ValueHint::Url)
+                .help("The target URL (required, unless [--stdin || --resume-from] used)"),
         )
         .arg(
-            Arg::with_name("threads")
-                .short("t")
-                .long("threads")
-                .value_name("THREADS")
-                .takes_value(true)
-                .help("Number of concurrent threads (default: 50)"),
-        )
-        .arg(
-            Arg::with_name("depth")
-                .short("d")
-                .long("depth")
-                .value_name("RECURSION_DEPTH")
-                .takes_value(true)
-                .help("Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)"),
-        )
-        .arg(
-            Arg::with_name("timeout")
-                .short("T")
-                .long("timeout")
-                .value_name("SECONDS")
-                .takes_value(true)
-                .help("Number of seconds before a request times out (default: 7)"),
-        )
-        .arg(
-            Arg::with_name("verbosity")
-                .short("v")
-                .long("verbosity")
-                .takes_value(false)
-                .multiple(true)
-                .conflicts_with("silent")
-                .help("Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v's is probably too much)"),
-        )
-        .arg(
-            Arg::with_name("proxy")
-                .short("p")
-                .long("proxy")
-                .takes_value(true)
-                .value_name("PROXY")
-                .help(
-                    "Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)",
-                ),
-        )
-        .arg(
-            Arg::with_name("replay_proxy")
-                .short("P")
-                .long("replay-proxy")
-                .takes_value(true)
-                .value_name("REPLAY_PROXY")
-                .help(
-                    "Send only unfiltered requests through a Replay Proxy, instead of all requests",
-                ),
-        )
-        .arg(
-            Arg::with_name("replay_codes")
-                .short("R")
-                .long("replay-codes")
-                .value_name("REPLAY_CODE")
-                .takes_value(true)
-                .multiple(true)
-                .use_delimiter(true)
-                .requires("replay_proxy")
-                .help(
-                    "Status Codes to send through a Replay Proxy when found (default: --status-codes value)",
-                ),
-        )
-        .arg(
-            Arg::with_name("status_codes")
-                .short("s")
-                .long("status-codes")
-                .value_name("STATUS_CODE")
-                .takes_value(true)
-                .multiple(true)
-                .use_delimiter(true)
-                .help(
-                    "Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)",
-                ),
-        )
-        .arg(
-            Arg::with_name("silent")
-                .long("silent")
-                .takes_value(false)
-                .conflicts_with("quiet")
-                .help("Only print URLs + turn off logging (good for piping a list of urls to other commands)")
-        )
-        .arg(
-            Arg::with_name("quiet")
-                .short("q")
-                .long("quiet")
-                .takes_value(false)
-                .help("Hide progress bars and banner (good for tmux windows w/ notifications)")
-        )
-        .arg(
-            Arg::with_name("auto_tune")
-                .long("auto-tune")
-                .takes_value(false)
-                .conflicts_with("auto_bail")
-                .help("Automatically lower scan rate when an excessive amount of errors are encountered")
-        )
-        .arg(
-            Arg::with_name("auto_bail")
-                .long("auto-bail")
-                .takes_value(false)
-                .help("Automatically stop scanning when an excessive amount of errors are encountered")
-        )
-        .arg(
-            Arg::with_name("json")
-                .long("json")
-                .takes_value(false)
-                .requires("output_files")
-                .help("Emit JSON logs to --output and --debug-log instead of normal text")
-        )
-        .arg(
-            Arg::with_name("dont_filter")
-                .short("D")
-                .long("dont-filter")
-                .takes_value(false)
-                .help("Don't auto-filter wildcard responses")
-        )
-        .arg(
-            Arg::with_name("output")
-                .short("o")
-                .long("output")
-                .value_name("FILE")
-                .help("Output file to write results to (use w/ --json for JSON entries)")
-                .takes_value(true),
-        )
-        .arg(
-            Arg::with_name("resume_from")
-                .long("resume-from")
-                .value_name("STATE_FILE")
-                .help("State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)")
-                .conflicts_with("url")
-                .takes_value(true),
-        )
-        .arg(
-            Arg::with_name("debug_log")
-                .long("debug-log")
-                .value_name("FILE")
-                .help("Output file to write log entries (use w/ --json for JSON entries)")
-                .takes_value(true),
-        )
-        .arg(
-            Arg::with_name("user_agent")
-                .short("a")
-                .long("user-agent")
-                .value_name("USER_AGENT")
-                .takes_value(true)
-                .help(
-                    "Sets the User-Agent (default: feroxbuster/VERSION)"
-                ),
-        )
-        .arg(
-            Arg::with_name("redirects")
-                .short("r")
-                .long("redirects")
-                .takes_value(false)
-                .help("Follow redirects")
-        )
-        .arg(
-            Arg::with_name("insecure")
-                .short("k")
-                .long("insecure")
-                .takes_value(false)
-                .help("Disables TLS certificate validation")
-        )
-        .arg(
-            Arg::with_name("extensions")
-                .short("x")
-                .long("extensions")
-                .value_name("FILE_EXTENSION")
-                .takes_value(true)
-                .multiple(true)
-                .use_delimiter(true)
-                .help(
-                    "File extension(s) to search for (ex: -x php -x pdf js)",
-                ),
-        )
-        .arg(
-            Arg::with_name("url_denylist")
-                .long("dont-scan")
-                .value_name("URL")
-                .takes_value(true)
-                .multiple(true)
-                .use_delimiter(true)
-                .help(
-                    "URL(s) to exclude from recursion/scans",
-                ),
-        )
-        .arg(
-            Arg::with_name("headers")
-                .short("H")
-                .long("headers")
-                .value_name("HEADER")
-                .takes_value(true)
-                .multiple(true)
-                .use_delimiter(true)
-                .help(
-                    "Specify HTTP headers (ex: -H Header:val 'stuff: things')",
-                ),
-        )
-        .arg(
-            Arg::with_name("queries")
-                .short("Q")
-                .long("query")
-                .value_name("QUERY")
-                .takes_value(true)
-                .multiple(true)
-                .use_delimiter(true)
-                .help(
-                    "Specify URL query parameters (ex: -Q token=stuff -Q secret=key)",
-                ),
-        )
-        .arg(
-            Arg::with_name("no_recursion")
-                .short("n")
-                .long("no-recursion")
-                .takes_value(false)
-                .help("Do not scan recursively")
-        )
-        .arg(
-            Arg::with_name("add_slash")
-                .short("f")
-                .long("add-slash")
-                .takes_value(false)
-                .conflicts_with("extensions")
-                .help("Append / to each request")
-        )
-        .arg(
-            Arg::with_name("stdin")
+            Arg::new("stdin")
                 .long("stdin")
+                .help_heading("Target selection")
                 .takes_value(false)
                 .help("Read url(s) from STDIN")
                 .conflicts_with("url")
         )
         .arg(
-            Arg::with_name("filter_size")
-                .short("S")
+            Arg::new("resume_from")
+                .long("resume-from")
+                .value_hint(ValueHint::FilePath)
+                .value_name("STATE_FILE")
+                .help_heading("Target selection")
+                .help("State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)")
+                .conflicts_with("url")
+                .takes_value(true),
+        );
+
+    /////////////////////////////////////////////////////////////////////
+    // group - proxy settings
+    /////////////////////////////////////////////////////////////////////
+    let app = app
+        .arg(
+            Arg::new("proxy")
+                .short('p')
+                .long("proxy")
+                .takes_value(true)
+                .value_name("PROXY")
+                .value_hint(ValueHint::Url)
+                .help_heading("Proxy settings")
+                .help(
+                    "Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)",
+                ),
+        )
+        .arg(
+            Arg::new("replay_proxy")
+                .short('P')
+                .long("replay-proxy")
+                .takes_value(true)
+                .value_hint(ValueHint::Url)
+                .value_name("REPLAY_PROXY")
+                .help_heading("Proxy settings")
+                .help(
+                    "Send only unfiltered requests through a Replay Proxy, instead of all requests",
+                ),
+        )
+        .arg(
+            Arg::new("replay_codes")
+                .short('R')
+                .long("replay-codes")
+                .value_name("REPLAY_CODE")
+                .takes_value(true)
+                .multiple_values(true)
+                .multiple_occurrences(true)
+                .use_delimiter(true)
+                .requires("replay_proxy")
+                .help_heading("Proxy settings")
+                .help(
+                    "Status Codes to send through a Replay Proxy when found (default: --status-codes value)",
+                ),
+        );
+
+    /////////////////////////////////////////////////////////////////////
+    // group - request settings
+    /////////////////////////////////////////////////////////////////////
+    let app = app
+        .arg(
+            Arg::new("user_agent")
+                .short('a')
+                .long("user-agent")
+                .value_name("USER_AGENT")
+                .takes_value(true)
+                .help_heading("Request settings")
+                .help(&**DEFAULT_USER_AGENT),
+        )
+        .arg(
+            Arg::new("random_agent")
+                .short('A')
+                .long("random-agent")
+                .takes_value(false)
+                .help_heading("Request settings")
+                .help("Use a random User-Agent"),
+        )
+        .arg(
+            Arg::new("extensions")
+                .short('x')
+                .long("extensions")
+                .value_name("FILE_EXTENSION")
+                .takes_value(true)
+                .multiple_values(true)
+                .multiple_occurrences(true)
+                .use_delimiter(true)
+                .help_heading("Request settings")
+                .help(
+                    "File extension(s) to search for (ex: -x php -x pdf js)",
+                ),
+        )
+        .arg(
+            Arg::new("methods")
+                .short('m')
+                .long("methods")
+                .value_name("HTTP_METHODS")
+                .takes_value(true)
+                .multiple_values(true)
+                .multiple_occurrences(true)
+                .use_delimiter(true)
+                .help_heading("Request settings")
+                .help(
+                    "Which HTTP request method(s) should be sent (default: GET)",
+                ),
+        )
+        .arg(
+            Arg::new("data")
+                .long("data")
+                .value_name("DATA")
+                .takes_value(true)
+                .help_heading("Request settings")
+                .help(
+                    "Request's Body; can read data from a file if input starts with an @ (ex: @post.bin)",
+                ),
+        )
+        .arg(
+            Arg::new("headers")
+                .short('H')
+                .long("headers")
+                .value_name("HEADER")
+                .takes_value(true)
+                .help_heading("Request settings")
+                .multiple_values(true)
+                .multiple_occurrences(true)
+                .use_delimiter(true)
+                .help(
+                    "Specify HTTP headers to be used in each request (ex: -H Header:val -H 'stuff: things')",
+                ),
+        )
+        .arg(
+            Arg::new("cookies")
+                .short('b')
+                .long("cookies")
+                .value_name("COOKIE")
+                .takes_value(true)
+                .multiple_values(true)
+                .multiple_occurrences(true)
+                .use_delimiter(true)
+                .help_heading("Request settings")
+                .help(
+                    "Specify HTTP cookies to be used in each request (ex: -b stuff=things)",
+                ),
+        )
+        .arg(
+            Arg::new("queries")
+                .short('Q')
+                .long("query")
+                .value_name("QUERY")
+                .takes_value(true)
+                .multiple_values(true)
+                .multiple_occurrences(true)
+                .use_delimiter(true)
+                .help_heading("Request settings")
+                .help(
+                    "Request's URL query parameters (ex: -Q token=stuff -Q secret=key)",
+                ),
+        )
+        .arg(
+            Arg::new("add_slash")
+                .short('f')
+                .long("add-slash")
+                .help_heading("Request settings")
+                .takes_value(false)
+                .help("Append / to each request's URL")
+        );
+
+    /////////////////////////////////////////////////////////////////////
+    // group - request filters
+    /////////////////////////////////////////////////////////////////////
+    let app = app.arg(
+        Arg::new("url_denylist")
+            .long("dont-scan")
+            .value_name("URL")
+            .takes_value(true)
+            .multiple_values(true)
+            .multiple_occurrences(true)
+            .use_delimiter(true)
+            .help_heading("Request filters")
+            .help("URL(s) or Regex Pattern(s) to exclude from recursion/scans"),
+    );
+
+    /////////////////////////////////////////////////////////////////////
+    // group - response filters
+    /////////////////////////////////////////////////////////////////////
+    let app = app
+        .arg(
+            Arg::new("filter_size")
+                .short('S')
                 .long("filter-size")
                 .value_name("SIZE")
                 .takes_value(true)
-                .multiple(true)
+                .multiple_values(true)
+                .multiple_occurrences(true)
                 .use_delimiter(true)
+                .help_heading("Response filters")
                 .help(
                     "Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)",
                 ),
         )
         .arg(
-            Arg::with_name("filter_regex")
-                .short("X")
+            Arg::new("filter_regex")
+                .short('X')
                 .long("filter-regex")
                 .value_name("REGEX")
                 .takes_value(true)
-                .multiple(true)
+                .multiple_values(true)
+                .multiple_occurrences(true)
                 .use_delimiter(true)
+                .help_heading("Response filters")
                 .help(
                     "Filter out messages via regular expression matching on the response's body (ex: -X '^ignore me$')",
                 ),
         )
         .arg(
-            Arg::with_name("filter_words")
-                .short("W")
+            Arg::new("filter_words")
+                .short('W')
                 .long("filter-words")
                 .value_name("WORDS")
                 .takes_value(true)
-                .multiple(true)
+                .multiple_values(true)
+                .multiple_occurrences(true)
                 .use_delimiter(true)
+                .help_heading("Response filters")
                 .help(
                     "Filter out messages of a particular word count (ex: -W 312 -W 91,82)",
                 ),
         )
         .arg(
-            Arg::with_name("filter_lines")
-                .short("N")
+            Arg::new("filter_lines")
+                .short('N')
                 .long("filter-lines")
                 .value_name("LINES")
                 .takes_value(true)
-                .multiple(true)
+                .multiple_values(true)
+                .multiple_occurrences(true)
                 .use_delimiter(true)
+                .help_heading("Response filters")
                 .help(
                     "Filter out messages of a particular line count (ex: -N 20 -N 31,30)",
                 ),
         )
         .arg(
-            Arg::with_name("filter_status")
-                .short("C")
+            Arg::new("filter_status")
+                .short('C')
                 .long("filter-status")
                 .value_name("STATUS_CODE")
                 .takes_value(true)
-                .multiple(true)
+                .multiple_values(true)
+                .multiple_occurrences(true)
                 .use_delimiter(true)
+                .help_heading("Response filters")
                 .help(
                     "Filter out status codes (deny list) (ex: -C 200 -C 401)",
                 ),
         )
         .arg(
-            Arg::with_name("filter_similar")
+            Arg::new("filter_similar")
                 .long("filter-similar-to")
                 .value_name("UNWANTED_PAGE")
                 .takes_value(true)
-                .multiple(true)
+                .multiple_values(true)
+                .multiple_occurrences(true)
+                .value_hint(ValueHint::Url)
                 .use_delimiter(true)
+                .help_heading("Response filters")
                 .help(
                     "Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)",
                 ),
         )
         .arg(
-            Arg::with_name("extract_links")
-                .short("e")
-                .long("extract-links")
-                .takes_value(false)
-                .help("Extract links from response body (html, javascript, etc...); make new requests based on findings (default: false)")
+            Arg::new("status_codes")
+                .short('s')
+                .long("status-codes")
+                .value_name("STATUS_CODE")
+                .takes_value(true)
+                .multiple_values(true)
+                .multiple_occurrences(true)
+                .use_delimiter(true)
+                .help_heading("Response filters")
+                .help(
+                    "Status Codes to include (allow list) (default: 200 204 301 302 307 308 401 403 405)",
+                ),
+        );
+
+    /////////////////////////////////////////////////////////////////////
+    // group - client settings
+    /////////////////////////////////////////////////////////////////////
+    let app = app
+        .arg(
+            Arg::new("timeout")
+                .short('T')
+                .long("timeout")
+                .value_name("SECONDS")
+                .takes_value(true)
+                .help_heading("Client settings")
+                .help("Number of seconds before a client's request times out (default: 7)"),
         )
         .arg(
-            Arg::with_name("scan_limit")
-                .short("L")
+            Arg::new("redirects")
+                .short('r')
+                .long("redirects")
+                .takes_value(false)
+                .help_heading("Client settings")
+                .help("Allow client to follow redirects"),
+        )
+        .arg(
+            Arg::new("insecure")
+                .short('k')
+                .long("insecure")
+                .takes_value(false)
+                .help_heading("Client settings")
+                .help("Disables TLS certificate validation in the client"),
+        );
+
+    /////////////////////////////////////////////////////////////////////
+    // group - scan settings
+    /////////////////////////////////////////////////////////////////////
+    let app = app
+        .arg(
+            Arg::new("threads")
+                .short('t')
+                .long("threads")
+                .value_name("THREADS")
+                .takes_value(true)
+                .help_heading("Scan settings")
+                .help("Number of concurrent threads (default: 50)"),
+        )
+        .arg(
+            Arg::new("no_recursion")
+                .short('n')
+                .long("no-recursion")
+                .takes_value(false)
+                .help_heading("Scan settings")
+                .help("Do not scan recursively"),
+        )
+        .arg(
+            Arg::new("depth")
+                .short('d')
+                .long("depth")
+                .value_name("RECURSION_DEPTH")
+                .takes_value(true)
+                .help_heading("Scan settings")
+                .help("Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)"),
+        ).arg(
+            Arg::new("extract_links")
+                .short('e')
+                .long("extract-links")
+                .takes_value(false)
+                .help_heading("Scan settings")
+                .help("Extract links from response body (html, javascript, etc...); make new requests based on findings")
+        )
+        .arg(
+            Arg::new("scan_limit")
+                .short('L')
                 .long("scan-limit")
                 .value_name("SCAN_LIMIT")
                 .takes_value(true)
+                .help_heading("Scan settings")
                 .help("Limit total number of concurrent scans (default: 0, i.e. no limit)")
         )
         .arg(
-            Arg::with_name("parallel")
+            Arg::new("parallel")
                 .long("parallel")
                 .value_name("PARALLEL_SCANS")
                 .takes_value(true)
                 .requires("stdin")
+                .help_heading("Scan settings")
                 .help("Run parallel feroxbuster instances (one child process per url passed via stdin)")
         )
         .arg(
-            Arg::with_name("rate_limit")
+            Arg::new("rate_limit")
                 .long("rate-limit")
                 .value_name("RATE_LIMIT")
                 .takes_value(true)
                 .conflicts_with("auto_tune")
+                .help_heading("Scan settings")
                 .help("Limit number of requests per second (per directory) (default: 0, i.e. no limit)")
         )
         .arg(
-            Arg::with_name("time_limit")
+            Arg::new("time_limit")
                 .long("time-limit")
                 .value_name("TIME_SPEC")
                 .takes_value(true)
                 .validator(valid_time_spec)
+                .help_heading("Scan settings")
                 .help("Limit total run time of all scans (ex: --time-limit 10m)")
         )
-        .group(ArgGroup::with_name("output_files")
-            .args(&["debug_log", "output"])
-            .multiple(true)
+        .arg(
+            Arg::new("wordlist")
+                .short('w')
+                .long("wordlist")
+                .value_hint(ValueHint::FilePath)
+                .value_name("FILE")
+                .help("Path to the wordlist")
+                .help_heading("Scan settings")
+                .takes_value(true),
+        ).arg(
+            Arg::new("auto_tune")
+                .long("auto-tune")
+                .takes_value(false)
+                .conflicts_with("auto_bail")
+                .help_heading("Scan settings")
+                .help("Automatically lower scan rate when an excessive amount of errors are encountered")
         )
-        .after_help(r#"NOTE:
+        .arg(
+            Arg::new("auto_bail")
+                .long("auto-bail")
+                .takes_value(false)
+                .help_heading("Scan settings")
+                .help("Automatically stop scanning when an excessive amount of errors are encountered")
+        ).arg(
+            Arg::new("dont_filter")
+                .short('D')
+                .long("dont-filter")
+                .takes_value(false)
+                .help_heading("Scan settings")
+                .help("Don't auto-filter wildcard responses")
+        );
+
+    /////////////////////////////////////////////////////////////////////
+    // group - output settings
+    /////////////////////////////////////////////////////////////////////
+    let app = app
+        .arg(
+            Arg::new("verbosity")
+                .short('v')
+                .long("verbosity")
+                .takes_value(false)
+                .multiple_occurrences(true)
+                .conflicts_with("silent")
+                .help_heading("Output settings")
+                .help("Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v's is probably too much)"),
+        ).arg(
+            Arg::new("silent")
+                .long("silent")
+                .takes_value(false)
+                .conflicts_with("quiet")
+                .help_heading("Output settings")
+                .help("Only print URLs + turn off logging (good for piping a list of urls to other commands)")
+        )
+        .arg(
+            Arg::new("quiet")
+                .short('q')
+                .long("quiet")
+                .takes_value(false)
+                .help_heading("Output settings")
+                .help("Hide progress bars and banner (good for tmux windows w/ notifications)")
+        )
+
+        .arg(
+            Arg::new("json")
+                .long("json")
+                .takes_value(false)
+                .requires("output_files")
+                .help_heading("Output settings")
+                .help("Emit JSON logs to --output and --debug-log instead of normal text")
+        ).arg(
+            Arg::new("output")
+                .short('o')
+                .long("output")
+                .value_hint(ValueHint::FilePath)
+                .value_name("FILE")
+                .help_heading("Output settings")
+                .help("Output file to write results to (use w/ --json for JSON entries)")
+                .takes_value(true),
+        )
+        .arg(
+            Arg::new("debug_log")
+                .long("debug-log")
+                .value_name("FILE")
+                .value_hint(ValueHint::FilePath)
+                .help_heading("Output settings")
+                .help("Output file to write log entries (use w/ --json for JSON entries)")
+                .takes_value(true),
+        );
+
+    /////////////////////////////////////////////////////////////////////
+    // group - miscellaneous
+    /////////////////////////////////////////////////////////////////////
+    let mut app = app
+        .group(
+            ArgGroup::new("output_files")
+                .args(&["debug_log", "output"])
+                .multiple(true),
+        )
+        .after_long_help(EPILOGUE);
+
+    /////////////////////////////////////////////////////////////////////
+    // end parser
+    /////////////////////////////////////////////////////////////////////
+    for arg in env::args() {
+        // secure-77 noticed that when an incorrect flag/option is used, the short help message is printed
+        // which is fine, but if you add -h|--help, it still errors out on the bad flag/option,
+        // never showing the full help message. This code addresses that behavior
+        if arg == "--help" {
+            app.print_long_help().unwrap();
+            println!(); // just a newline to mirror original --help output
+            process::exit(0);
+        } else if arg == "-h" {
+            // same for -h, just shorter
+            app.print_help().unwrap();
+            println!();
+            process::exit(0);
+        }
+    }
+
+    app
+}
+
+/// Validate that a string is formatted as a number followed by s, m, h, or d (10d, 30s, etc...)
+fn valid_time_spec(time_spec: &str) -> Result<(), String> {
+    match TIMESPEC_REGEX.is_match(time_spec) {
+        true => Ok(()),
+        false => {
+            let msg = format!(
+                "Expected a non-negative, whole number followed by s, m, h, or d (case insensitive); received {}",
+                time_spec
+            );
+            Err(msg)
+        }
+    }
+}
+
+const EPILOGUE: &str = r#"NOTE:
     Options that take multiple values are very flexible.  Consider the following ways of specifying
     extensions:
         ./feroxbuster -u http://127.1 -x pdf -x js,html -x php txt json,docx
@@ -422,43 +609,28 @@ EXAMPLES:
         ./feroxbuster -u http://127.1 --extract-links
 
     Ludicrous speed... go!
-        ./feroxbuster -u http://127.1 -t 200
-    "#);
-
-    for arg in env::args() {
-        // secure-77 noticed that when an incorrect flag/option is used, the short help message is printed
-        // which is fine, but if you add -h|--help, it still errors out on the bad flag/option,
-        // never showing the full help message. This code addresses that behavior
-        if arg == "--help" || arg == "-h" {
-            app.print_long_help().unwrap();
-            println!(); // just a newline to mirror original --help output
-            process::exit(0);
-        }
-    }
-
-    app
-}
-
-/// Validate that a string is formatted as a number followed by s, m, h, or d (10d, 30s, etc...)
-fn valid_time_spec(time_spec: String) -> Result<(), String> {
-    match TIMESPEC_REGEX.is_match(&time_spec) {
-        true => Ok(()),
-        false => {
-            let msg = format!(
-                "Expected a non-negative, whole number followed by s, m, h, or d (case insensitive); received {}",
-                time_spec
-            );
-            Err(msg)
-        }
-    }
-}
+        ./feroxbuster -u http://127.1 -threads 200
+        
+    Limit to a total of 60 active requests at any given time (threads * scan limit)
+        ./feroxbuster -u http://127.1 --threads 30 --scan-limit 2
+    
+    Send all 200/302 responses to a proxy (only proxy requests/responses you care about)
+        ./feroxbuster -u http://127.1 --replay-proxy http://localhost:8080 --replay-codes 200 302 --insecure
+        
+    Abort or reduce scan speed to individual directory scans when too many errors have occurred
+        ./feroxbuster -u http://127.1 --auto-bail
+        ./feroxbuster -u http://127.1 --auto-tune
+        
+    Examples and demonstrations of all features
+        https://epi052.github.io/feroxbuster-docs/docs/examples/
+    "#;
 
 #[cfg(test)]
 mod tests {
     use super::*;
 
     #[test]
-    /// initalize parser, expect a clap::App returned
+    /// initialize parser, expect a clap::App returned
     fn parser_initialize_gives_defaults() {
         let app = initialize();
         assert_eq!(app.get_name(), "feroxbuster");
@@ -471,29 +643,29 @@ mod tests {
     /// that i didn't hose up the regex.  Going to consolidate them into a single test
     fn validate_valid_time_spec_validation() {
         let float_rejected = "1.4m";
-        assert!(valid_time_spec(float_rejected.into()).is_err());
+        assert!(valid_time_spec(float_rejected).is_err());
 
         let negative_rejected = "-1m";
-        assert!(valid_time_spec(negative_rejected.into()).is_err());
+        assert!(valid_time_spec(negative_rejected).is_err());
 
         let only_number_rejected = "1";
-        assert!(valid_time_spec(only_number_rejected.into()).is_err());
+        assert!(valid_time_spec(only_number_rejected).is_err());
 
         let only_measurement_rejected = "m";
-        assert!(valid_time_spec(only_measurement_rejected.into()).is_err());
+        assert!(valid_time_spec(only_measurement_rejected).is_err());
 
         for accepted_measurement in &["s", "m", "h", "d", "S", "M", "H", "D"] {
             // all upper/lowercase should be good
-            assert!(valid_time_spec(format!("1{}", *accepted_measurement)).is_ok());
+            assert!(valid_time_spec(&format!("1{}", *accepted_measurement)).is_ok());
         }
 
         let leading_space_rejected = " 14m";
-        assert!(valid_time_spec(leading_space_rejected.into()).is_err());
+        assert!(valid_time_spec(leading_space_rejected).is_err());
 
         let trailing_space_rejected = "14m ";
-        assert!(valid_time_spec(trailing_space_rejected.into()).is_err());
+        assert!(valid_time_spec(trailing_space_rejected).is_err());
 
         let space_between_rejected = "1 4m";
-        assert!(valid_time_spec(space_between_rejected.into()).is_err());
+        assert!(valid_time_spec(space_between_rejected).is_err());
     }
 }

src/progress.rs

@@ -35,10 +35,11 @@ pub fn add_bar(prefix: &str, length: u64, bar_type: BarType) -> ProgressBar {
 
     style = match bar_type {
         BarType::Hidden => style.template(""),
-        BarType::Default => style
-            .template("[{bar:.cyan/blue}] - {elapsed:<4} {pos:>7}/{len:7} {per_sec:7} {prefix}"),
+        BarType::Default => style.template(
+            "[{bar:.cyan/blue}] - {elapsed:<4} {pos:>7}/{len:7} {per_sec:7} {prefix} {msg}",
+        ),
         BarType::Message => style.template(&format!(
-            "[{{bar:.cyan/blue}}] - {{elapsed:<4}} {{pos:>7}}/{{len:7}} {:7} {{prefix}}",
+            "[{{bar:.cyan/blue}}] - {{elapsed:<4}} {{pos:>7}}/{{len:7}} {:7} {{prefix}} {{msg}}",
             "-"
         )),
         BarType::Total => {

src/response.rs

@@ -7,9 +7,10 @@ use std::{
 };
 
 use anyhow::{Context, Result};
+use console::style;
 use reqwest::{
     header::{HeaderMap, HeaderName, HeaderValue},
-    Response, StatusCode, Url,
+    Method, Response, StatusCode, Url,
 };
 use serde::ser::SerializeStruct;
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
@@ -30,9 +31,15 @@ pub struct FeroxResponse {
     /// The final `Url` of this `FeroxResponse`
     url: Url,
 
+    /// The original url from which the final `Url` was derived
+    original_url: String,
+
     /// The `StatusCode` of this `FeroxResponse`
     status: StatusCode,
 
+    /// The HTTP Request `Method` of this `FeroxResponse`
+    method: Method,
+
     /// The full response text
     text: String,
 
@@ -61,7 +68,9 @@ impl Default for FeroxResponse {
     fn default() -> Self {
         Self {
             url: Url::parse("http://localhost").unwrap(),
+            original_url: "".to_string(),
             status: Default::default(),
+            method: Method::default(),
             text: "".to_string(),
             content_length: 0,
             line_count: 0,
@@ -79,8 +88,9 @@ impl fmt::Display for FeroxResponse {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         write!(
             f,
-            "FeroxResponse {{ url: {}, status: {}, content-length: {} }}",
+            "FeroxResponse {{ url: {}, method: {}, status: {}, content-length: {} }}",
             self.url(),
+            self.method(),
             self.status(),
             self.content_length()
         )
@@ -94,6 +104,11 @@ impl FeroxResponse {
         &self.status
     }
 
+    /// Get the `Method` of this `FeroxResponse`
+    pub fn method(&self) -> &Method {
+        &self.method
+    }
+
     /// Get the `wildcard` of this `FeroxResponse`
     pub fn wildcard(&self) -> bool {
         self.wildcard
@@ -186,7 +201,13 @@ impl FeroxResponse {
     }
 
     /// Create a new `FeroxResponse` from the given `Response`
-    pub async fn from(response: Response, read_body: bool, output_level: OutputLevel) -> Self {
+    pub async fn from(
+        response: Response,
+        original_url: &str,
+        method: &str,
+        read_body: bool,
+        output_level: OutputLevel,
+    ) -> Self {
         let url = response.url().clone();
         let status = response.status();
         let headers = response.headers().clone();
@@ -213,7 +234,9 @@ impl FeroxResponse {
 
         FeroxResponse {
             url,
+            original_url: original_url.to_string(),
             status,
+            method: Method::from_bytes(method.as_bytes()).unwrap_or(Method::GET),
             content_length,
             text,
             headers,
@@ -326,16 +349,42 @@ impl FeroxSerialize for FeroxResponse {
         let words = self.word_count().to_string();
         let chars = self.content_length().to_string();
         let status = self.status().as_str();
+        let method = self.method().as_str();
         let wild_status = status_colorizer("WLD");
 
+        let mut url_with_redirect = match (
+            self.status().is_redirection(),
+            self.headers().get("Location").is_some(),
+        ) {
+            (true, true) => {
+                // redirect with Location header, show where it goes if possible
+                let loc = self
+                    .headers()
+                    .get("Location")
+                    .unwrap() // known Some() already
+                    .to_str()
+                    .unwrap_or("Unknown");
+
+                // prettify the redirect target
+                let loc = style(loc).yellow();
+
+                format!("{} => {loc}", self.url())
+            }
+            _ => {
+                // no redirect, just use the normal url
+                self.url().to_string()
+            }
+        };
+
         if self.wildcard && matches!(self.output_level, OutputLevel::Default | OutputLevel::Quiet) {
             // --silent was not used and response is a wildcard, special messages abound when
             // this is the case...
 
             // create the base message
             let mut message = format!(
-                "{} {:>8}l {:>8}w {:>8}c Got {} for {} (url length: {})\n",
+                "{} {:>8} {:>8}l {:>8}w {:>8}c Got {} for {} (url length: {})\n",
                 wild_status,
+                method,
                 lines,
                 words,
                 chars,
@@ -345,34 +394,26 @@ impl FeroxSerialize for FeroxResponse {
             );
 
             if self.status().is_redirection() {
-                // when it's a redirect, show where it goes, if possible
-                if let Some(next_loc) = self.headers().get("Location") {
-                    let next_loc_str = next_loc.to_str().unwrap_or("Unknown");
+                // initial wildcard messages are wordy enough, put the redirect by itself
+                url_with_redirect = format!(
+                    "{} {:>9} {:>9} {:>9} {}\n",
+                    wild_status, "-", "-", "-", url_with_redirect
+                );
 
-                    let redirect_msg = format!(
-                        "{} {:>9} {:>9} {:>9} {} redirects to => {}\n",
-                        wild_status,
-                        "-",
-                        "-",
-                        "-",
-                        self.url(),
-                        next_loc_str
-                    );
-
-                    message.push_str(&redirect_msg);
-                }
+                // base message + redirection message (either empty string or redir msg)
+                message.push_str(&url_with_redirect);
             }
 
-            // base message + redirection message (if appropriate)
             message
         } else {
             // not a wildcard, just create a normal entry
             utils::create_report_string(
                 self.status.as_str(),
+                method,
                 &lines,
                 &words,
                 &chars,
-                self.url().as_str(),
+                &url_with_redirect,
                 self.output_level,
             )
         }
@@ -423,7 +464,7 @@ impl Serialize for FeroxResponse {
         S: Serializer,
     {
         let mut headers = HashMap::new();
-        let mut state = serializer.serialize_struct("FeroxResponse", 7)?;
+        let mut state = serializer.serialize_struct("FeroxResponse", 8)?;
 
         // need to convert the HeaderMap to a HashMap in order to pass it to the serializer
         for (key, value) in &self.headers {
@@ -434,9 +475,11 @@ impl Serialize for FeroxResponse {
 
         state.serialize_field("type", "response")?;
         state.serialize_field("url", self.url.as_str())?;
+        state.serialize_field("original_url", self.original_url.as_str())?;
         state.serialize_field("path", self.url.path())?;
         state.serialize_field("wildcard", &self.wildcard)?;
         state.serialize_field("status", &self.status.as_u16())?;
+        state.serialize_field("method", &self.method.as_str())?;
         state.serialize_field("content_length", &self.content_length)?;
         state.serialize_field("line_count", &self.line_count)?;
         state.serialize_field("word_count", &self.word_count)?;
@@ -455,7 +498,9 @@ impl<'de> Deserialize<'de> for FeroxResponse {
     {
         let mut response = Self {
             url: Url::parse("http://localhost").unwrap(),
+            original_url: String::new(),
             status: StatusCode::OK,
+            method: Method::GET,
             text: String::new(),
             content_length: 0,
             headers: HeaderMap::new(),
@@ -476,6 +521,11 @@ impl<'de> Deserialize<'de> for FeroxResponse {
                         }
                     }
                 }
+                "original_url" => {
+                    if let Some(og_url) = value.as_str() {
+                        response.original_url = String::from(og_url);
+                    }
+                }
                 "status" => {
                     if let Some(num) = value.as_u64() {
                         if let Ok(smaller) = u16::try_from(num) {
@@ -485,6 +535,11 @@ impl<'de> Deserialize<'de> for FeroxResponse {
                         }
                     }
                 }
+                "method" => {
+                    if let Some(method) = value.as_str() {
+                        response.method = Method::from_bytes(method.as_bytes()).unwrap_or_default();
+                    }
+                }
                 "content_length" => {
                     if let Some(num) = value.as_u64() {
                         response.content_length = num;
@@ -540,7 +595,9 @@ mod tests {
         let url = Url::parse("http://localhost").unwrap();
         let response = FeroxResponse {
             url,
+            original_url: String::new(),
             status: Default::default(),
+            method: Default::default(),
             text: "".to_string(),
             content_length: 0,
             line_count: 0,
@@ -561,7 +618,9 @@ mod tests {
         let url = Url::parse("http://localhost/one/two").unwrap();
         let response = FeroxResponse {
             url,
+            original_url: String::new(),
             status: Default::default(),
+            method: Default::default(),
             text: "".to_string(),
             content_length: 0,
             line_count: 0,
@@ -582,7 +641,9 @@ mod tests {
         let url = Url::parse("http://localhost").unwrap();
         let response = FeroxResponse {
             url,
+            original_url: String::new(),
             status: Default::default(),
+            method: Default::default(),
             text: "".to_string(),
             content_length: 0,
             line_count: 0,
@@ -603,7 +664,9 @@ mod tests {
         let url = Url::parse("http://localhost/one/two").unwrap();
         let response = FeroxResponse {
             url,
+            original_url: String::new(),
             status: Default::default(),
+            method: Default::default(),
             text: "".to_string(),
             content_length: 0,
             line_count: 0,
@@ -624,7 +687,9 @@ mod tests {
         let url = Url::parse("http://localhost/one/two/three").unwrap();
         let response = FeroxResponse {
             url,
+            original_url: String::new(),
             status: Default::default(),
+            method: Default::default(),
             text: "".to_string(),
             content_length: 0,
             line_count: 0,

src/scan_manager/menu.rs

@@ -1,27 +1,39 @@
 use crate::progress::PROGRESS_BAR;
 use console::{measure_text_width, pad_str, style, Alignment, Term};
 use indicatif::ProgressDrawTarget;
+use regex::Regex;
+
+/// Data container for a command entered by the user interactively
+#[derive(Debug)]
+pub enum MenuCmd {
+    /// user wants to add a url to be scanned
+    Add(String),
+
+    /// user wants to cancel one or more active scans
+    Cancel(Vec<usize>, bool),
+}
+
+/// Data container for a command result to be used internally by the ferox_scanner
+#[derive(Debug)]
+pub enum MenuCmdResult {
+    /// Url to be added to the scan queue
+    Url(String),
+
+    /// Number of scans that were actually cancelled, can be 0
+    NumCancelled(usize),
+}
 
 /// Interactive scan cancellation menu
 #[derive(Debug)]
 pub(super) struct Menu {
-    /// character to use as visual separator of lines
-    separator: String,
-
-    /// name of menu
-    name: String,
-
     /// header: name surrounded by separators
     header: String,
 
-    /// instructions
-    instructions: String,
-
     /// footer: instructions surrounded by separators
     footer: String,
 
     /// target for output
-    term: Term,
+    pub(super) term: Term,
 }
 
 /// Implementation of Menu
@@ -30,42 +42,43 @@ impl Menu {
     pub(super) fn new() -> Self {
         let separator = "─".to_string();
 
-        let instructions = format!(
-            "Enter a {} list of indexes/ranges to {} ({}: 1-4,8,9-13)",
-            style("comma-separated").yellow(),
-            style("cancel").red(),
-            style("ex").cyan(),
-        );
-
         let name = format!(
             "{} {} {}",
             "💀",
-            style("Scan Cancel Menu").bright().yellow(),
+            style("Scan Management Menu").bright().yellow(),
             "💀"
         );
 
-        let force_msg = format!(
-            "Add {} to {} confirmation ({}: 3-5 -f)",
-            style("-f").yellow(),
-            style("skip").yellow(),
-            style("ex").cyan(),
+        let add_cmd = format!(
+            "  {}[{}] NEW_URL (ex: {} http://localhost)\n",
+            style("a").green(),
+            style("dd").green(),
+            style("add").green()
         );
 
-        let longest = measure_text_width(&instructions).max(measure_text_width(&name));
+        let canx_cmd = format!(
+            "  {}[{}] [-f] SCAN_ID[-SCAN_ID[,...]] (ex: {} 1-4,8,9-13 or {} -f 3)",
+            style("c").red(),
+            style("ancel").red(),
+            style("cancel").red(),
+            style("c").red(),
+        );
+
+        let mut commands = String::from("Commands:\n");
+        commands.push_str(&add_cmd);
+        commands.push_str(&canx_cmd);
+
+        let longest = measure_text_width(&canx_cmd).max(measure_text_width(&name));
 
         let border = separator.repeat(longest);
 
         let padded_name = pad_str(&name, longest, Alignment::Center, None);
-        let padded_force = pad_str(&force_msg, longest, Alignment::Center, None);
 
         let header = format!("{}\n{}\n{}", border, padded_name, border);
-        let footer = format!("{}\n{}\n{}\n{}", border, instructions, padded_force, border);
+        let footer = format!("{}\n{}\n{}", border, commands, border);
 
         Self {
-            separator,
-            name,
             header,
-            instructions,
             footer,
             term: Term::stderr(),
         }
@@ -161,14 +174,36 @@ impl Menu {
         nums
     }
 
-    /// get comma-separated list of scan indexes from the user
-    pub(super) fn get_scans_from_user(&self) -> Option<(Vec<usize>, bool)> {
-        if let Ok(line) = self.term.read_line() {
-            let force = line.contains("-f");
-            let line = line.replace("-f", "");
-            Some((self.split_to_nums(&line), force))
-        } else {
-            None
+    /// get input from the user and translate it to a `MenuCmd`
+    pub(super) fn get_command_input_from_user(&self, line: &str) -> Option<MenuCmd> {
+        let line = line.trim(); // normalize input if there are leading spaces
+
+        match line.chars().next().unwrap_or('_').to_ascii_lowercase() {
+            'c' => {
+                // cancel command; start by determining if -f was used
+                let force = line.contains("-f");
+
+                // then remove c[ancel] from the command so it can be passed to the number
+                // splitter
+                let re = Regex::new(r"^[cC][ancelANCEL]*").unwrap();
+                let line = line.replace("-f", "");
+                let line = re.replace(&line, "").to_string();
+
+                Some(MenuCmd::Cancel(self.split_to_nums(&line), force))
+            }
+            'a' => {
+                // add command
+                // similar to cancel, we need to remove the a[dd] substring, the rest should be
+                // a url
+                let re = Regex::new(r"^[aA][dD]*").unwrap();
+                let line = re.replace(line, "").to_string().trim().to_string();
+
+                Some(MenuCmd::Add(line))
+            }
+            _ => {
+                // invalid input
+                None
+            }
         }
     }
 

src/scan_manager/mod.rs

@@ -9,6 +9,7 @@ mod state;
 mod tests;
 
 pub(self) use menu::Menu;
+pub use menu::{MenuCmd, MenuCmdResult};
 pub use order::ScanOrder;
 pub use response_container::FeroxResponses;
 pub use scan::{FeroxScan, ScanStatus, ScanType};

src/scan_manager/scan_container.rs

@@ -4,6 +4,7 @@ use crate::{
     config::OutputLevel,
     progress::PROGRESS_PRINTER,
     progress::{add_bar, BarType},
+    scan_manager::{MenuCmd, MenuCmdResult},
     scanner::RESPONSES,
     traits::FeroxSerialize,
     SLEEP_DURATION,
@@ -300,23 +301,33 @@ impl FeroxScans {
     }
 
     /// CLI menu that allows for interactive cancellation of recursed-into directories
-    async fn interactive_menu(&self) -> usize {
+    async fn interactive_menu(&self) -> Option<MenuCmdResult> {
         self.menu.hide_progress_bars();
         self.menu.clear_screen();
         self.menu.print_header();
         self.display_scans().await;
         self.menu.print_footer();
 
-        let mut num_cancelled = 0_usize;
+        let menu_cmd = if let Ok(line) = self.menu.term.read_line() {
+            self.menu.get_command_input_from_user(&line)
+        } else {
+            None
+        };
 
-        if let Some((input, force)) = self.menu.get_scans_from_user() {
-            num_cancelled += self.cancel_scans(input, force).await;
+        let result = match menu_cmd {
+            Some(MenuCmd::Cancel(indices, should_force)) => {
+                // cancel the things
+                let num_cancelled = self.cancel_scans(indices, should_force).await;
+                Some(MenuCmdResult::NumCancelled(num_cancelled))
+            }
+            Some(MenuCmd::Add(url)) => Some(MenuCmdResult::Url(url)),
+            None => None,
         };
 
         self.menu.clear_screen();
         self.menu.show_progress_bars();
 
-        num_cancelled
+        result
     }
 
     /// prints all known responses that the scanner has already seen
@@ -364,19 +375,19 @@ impl FeroxScans {
     ///
     /// When the value stored in `PAUSE_SCAN` becomes `false`, the function returns, exiting the busy
     /// loop
-    pub async fn pause(&self, get_user_input: bool) -> usize {
+    pub async fn pause(&self, get_user_input: bool) -> Option<MenuCmdResult> {
         // function uses tokio::time, not std
 
         // local testing showed a pretty slow increase (less than linear) in CPU usage as # of
         // concurrent scans rose when SLEEP_DURATION was set to 500, using that as the default for now
         let mut interval = time::interval(time::Duration::from_millis(SLEEP_DURATION));
-        let mut num_cancelled = 0_usize;
+        let mut command_result = None;
 
         if INTERACTIVE_BARRIER.load(Ordering::Relaxed) == 0 {
             INTERACTIVE_BARRIER.fetch_add(1, Ordering::Relaxed);
 
             if get_user_input {
-                num_cancelled += self.interactive_menu().await;
+                command_result = self.interactive_menu().await;
                 PAUSE_SCAN.store(false, Ordering::Relaxed);
                 self.print_known_responses();
             }
@@ -393,8 +404,8 @@ impl FeroxScans {
                     INTERACTIVE_BARRIER.fetch_sub(1, Ordering::Relaxed);
                 }
 
-                log::trace!("exit: pause_scan -> {}", num_cancelled);
-                return num_cancelled;
+                log::trace!("exit: pause_scan -> {:?}", command_result);
+                return command_result;
             }
         }
     }

src/scan_manager/tests.rs

@@ -303,7 +303,7 @@ fn ferox_scans_serialize() {
 #[test]
 /// given a FeroxResponses, test that it serializes into the proper JSON entry
 fn ferox_responses_serialize() {
-    let json_response = r#"{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{"server":"nginx/1.16.1"}}"#;
+    let json_response = r#"{"type":"response","url":"https://nerdcore.com/css","original_url":"https://nerdcore.com","path":"/css","wildcard":true,"status":301,"method":"GET","content_length":173,"line_count":10,"word_count":16,"headers":{"server":"nginx/1.16.1"}}"#;
     let response: FeroxResponse = serde_json::from_str(json_response).unwrap();
 
     let responses = FeroxResponses::default();
@@ -321,7 +321,7 @@ fn ferox_responses_serialize() {
 /// given a FeroxResponse, test that it serializes into the proper JSON entry
 fn ferox_response_serialize_and_deserialize() {
     // deserialize
-    let json_response = r#"{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{"server":"nginx/1.16.1"}}"#;
+    let json_response = r#"{"type":"response","url":"https://nerdcore.com/css","original_url":"https://nerdcore.com","path":"/css","wildcard":true,"status":301,"method":"GET","content_length":173,"line_count":10,"word_count":16,"headers":{"server":"nginx/1.16.1"}}"#;
     let response: FeroxResponse = serde_json::from_str(json_response).unwrap();
 
     assert_eq!(response.url().as_str(), "https://nerdcore.com/css");
@@ -354,7 +354,7 @@ fn feroxstates_feroxserialize_implementation() {
     ferox_scans.insert(ferox_scan);
 
     let config = Configuration::new().unwrap();
-    let stats = Arc::new(Stats::new(config.extensions.len(), config.json));
+    let stats = Arc::new(Stats::new(config.json));
 
     let json_response = r#"{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{"server":"nginx/1.16.1"}}"#;
     let response: FeroxResponse = serde_json::from_str(json_response).unwrap();
@@ -378,12 +378,82 @@ fn feroxstates_feroxserialize_implementation() {
     assert!(expected_strs.eval(&ferox_state.as_str()));
 
     let json_state = ferox_state.as_json().unwrap();
-    let expected = format!(
-        r#"{{"scans":[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","status":"NotStarted","num_requests":0}}],"config":{{"type":"configuration","wordlist":"/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt","config":"","proxy":"","replay_proxy":"","target_url":"","status_codes":[200,204,301,302,307,308,401,403,405,500],"replay_codes":[200,204,301,302,307,308,401,403,405,500],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"silent":false,"quiet":false,"auto_bail":false,"auto_tune":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/{}","redirects":false,"insecure":false,"extensions":[],"headers":{{}},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":false,"depth":4,"scan_limit":0,"parallel":0,"rate_limit":0,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false,"resumed":false,"resume_from":"","save_state":false,"time_limit":"","filter_similar":[],"url_denylist":[]}},"responses":[{{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{{"server":"nginx/1.16.1"}}}}]"#,
-        saved_id, VERSION
-    );
-    println!("{}\n{}", expected, json_state);
-    assert!(predicates::str::contains(expected).eval(&json_state));
+    for expected in [
+        r#""scans""#,
+        &format!(r#""id":"{}""#, saved_id),
+        r#""url":"https://spiritanimal.com""#,
+        r#""scan_type":"Directory""#,
+        r#""status":"NotStarted""#,
+        r#""num_requests":0"#,
+        r#""config""#,
+        r#""type":"configuration""#,
+        r#""wordlist":"/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt""#,
+        r#""config""#,
+        r#""proxy":"""#,
+        r#""replay_proxy":"""#,
+        r#""target_url":"""#,
+        r#""status_codes":[200,204,301,302,307,308,401,403,405,500]"#,
+        r#""replay_codes":[200,204,301,302,307,308,401,403,405,500]"#,
+        r#""filter_status":[]"#,
+        r#""threads":50"#,
+        r#""timeout":7"#,
+        r#""verbosity":0"#,
+        r#""silent":false"#,
+        r#""quiet":false"#,
+        r#""auto_bail":false"#,
+        r#""auto_tune":false"#,
+        r#""json":false"#,
+        r#""output":"""#,
+        r#""debug_log":"""#,
+        &format!(r#""user_agent":"feroxbuster/{}""#, VERSION),
+        r#""random_agent":false"#,
+        r#""redirects":false"#,
+        r#""insecure":false"#,
+        r#""extensions":[]"#,
+        r#""methods":["GET"],"#,
+        r#""data":[]"#,
+        r#""headers""#,
+        r#""queries":[]"#,
+        r#""no_recursion":false"#,
+        r#""extract_links":false"#,
+        r#""add_slash":false"#,
+        r#""stdin":false"#,
+        r#""depth":4"#,
+        r#""scan_limit":0"#,
+        r#""parallel":0"#,
+        r#""rate_limit":0"#,
+        r#""filter_size":[]"#,
+        r#""filter_line_count":[]"#,
+        r#""filter_word_count":[]"#,
+        r#""filter_regex":[]"#,
+        r#""dont_filter":false"#,
+        r#""resumed":false"#,
+        r#""resume_from":"""#,
+        r#""save_state":false"#,
+        r#""time_limit":"""#,
+        r#""filter_similar":[]"#,
+        r#""url_denylist":[]"#,
+        r#""responses""#,
+        r#""type":"response""#,
+        r#""url":"https://nerdcore.com/css""#,
+        r#""path":"/css""#,
+        r#""wildcard":true"#,
+        r#""status":301"#,
+        r#""method":"GET""#,
+        r#""content_length":173"#,
+        r#""line_count":10"#,
+        r#""word_count":16"#,
+        r#""headers""#,
+        r#""server":"nginx/1.16.1"#,
+    ]
+    .iter()
+    {
+        assert!(
+            predicates::str::contains(*expected).eval(&json_state),
+            "{}",
+            expected
+        )
+    }
 }
 
 #[should_panic]
@@ -505,6 +575,14 @@ async fn ferox_scan_abort() {
 /// and their correctness can be verified easily manually; just calling for now
 fn menu_print_header_and_footer() {
     let menu = Menu::new();
+    let menu_cmd_1 = MenuCmd::Add(String::from("http://localhost"));
+    let menu_cmd_2 = MenuCmd::Cancel(vec![0], false);
+    let menu_cmd_res_1 = MenuCmdResult::Url(String::from("http://localhost"));
+    let menu_cmd_res_2 = MenuCmdResult::NumCancelled(2);
+    println!(
+        "{:?}{:?}{:?}{:?}",
+        menu_cmd_1, menu_cmd_2, menu_cmd_res_1, menu_cmd_res_2
+    );
     menu.clear_screen();
     menu.print_header();
     menu.print_footer();
@@ -512,6 +590,57 @@ fn menu_print_header_and_footer() {
     menu.show_progress_bars();
 }
 
+#[test]
+/// ensure command parsing from user input results int he correct MenuCmd returned
+fn menu_get_command_input_from_user_returns_cancel() {
+    let menu = Menu::new();
+
+    for (idx, cmd) in ["cancel", "Cancel", "c", "C"].iter().enumerate() {
+        let force = idx % 2 == 0;
+
+        let full_cmd = if force {
+            format!("{} -f {}\n", cmd, idx)
+        } else {
+            format!("{} {}\n", cmd, idx)
+        };
+
+        let result = menu.get_command_input_from_user(&full_cmd).unwrap();
+
+        assert!(matches!(result, MenuCmd::Cancel(_, _)));
+
+        if let MenuCmd::Cancel(canx_list, ret_force) = result {
+            if idx == 0 {
+                assert!(canx_list.is_empty());
+            } else {
+                assert_eq!(canx_list, vec![idx]);
+            }
+            assert_eq!(force, ret_force);
+        }
+    }
+}
+
+#[test]
+/// ensure command parsing from user input results int he correct MenuCmd returned
+fn menu_get_command_input_from_user_returns_add() {
+    let menu = Menu::new();
+
+    for cmd in ["add", "Addd", "a", "A", "None"] {
+        let test_url = "http://happyfuntimes.commmm";
+        let full_cmd = format!("{} {}\n", cmd, test_url);
+
+        if cmd != "None" {
+            let result = menu.get_command_input_from_user(&full_cmd).unwrap();
+            assert!(matches!(result, MenuCmd::Add(_)));
+
+            if let MenuCmd::Add(url) = result {
+                assert_eq!(url, test_url);
+            }
+        } else {
+            assert!(menu.get_command_input_from_user(&full_cmd).is_none());
+        };
+    }
+}
+
 #[test]
 /// ensure spaces are trimmed and numbers are returned from split_to_nums
 fn split_to_nums_is_correct() {

src/scanner/ferox_scanner.rs

@@ -1,6 +1,7 @@
 use std::{ops::Deref, sync::atomic::Ordering, sync::Arc, time::Instant};
 
 use anyhow::{bail, Result};
+use console::style;
 use futures::{stream, StreamExt};
 use lazy_static::lazy_static;
 use tokio::sync::Semaphore;
@@ -10,14 +11,15 @@ use crate::{
         Command::{AddError, AddToF64Field, SubtractFromUsizeField},
         Handles,
     },
-    extractor::{ExtractionTarget::RobotsTxt, ExtractorBuilder},
+    extractor::{ExtractionTarget, ExtractorBuilder},
     heuristics,
-    scan_manager::{FeroxResponses, ScanOrder, ScanStatus, PAUSE_SCAN},
+    scan_manager::{FeroxResponses, MenuCmdResult, ScanOrder, ScanStatus, PAUSE_SCAN},
     statistics::{
         StatError::Other,
         StatField::{DirScanTimes, TotalExpected},
     },
     utils::fmt_err,
+    Command,
 };
 
 use super::requester::Requester;
@@ -42,7 +44,7 @@ pub struct FeroxScanner {
     /// wordlist that's already been read from disk
     wordlist: Arc<Vec<String>>,
 
-    /// limiter that restricts the number of active FeroxScanners  
+    /// limiter that restricts the number of active FeroxScanners
     scan_limiter: Arc<Semaphore>,
 }
 
@@ -72,23 +74,34 @@ impl FeroxScanner {
         log::trace!("enter: scan_url");
         log::info!("Starting scan against: {}", self.target_url);
 
-        let scan_timer = Instant::now();
+        let mut scan_timer = Instant::now();
+        let mut dirlist_flag = false;
 
-        if matches!(self.order, ScanOrder::Initial) && self.handles.config.extract_links {
-            // only grab robots.txt on the initial scan_url calls. all fresh dirs will be passed
-            // to try_recursion
+        if self.handles.config.extract_links {
+            // parse html for links (i.e. web scraping)
             let extractor = ExtractorBuilder::default()
+                .target(ExtractionTarget::ParseHtml)
                 .url(&self.target_url)
                 .handles(self.handles.clone())
-                .target(RobotsTxt)
                 .build()?;
-
-            let links = extractor.extract().await?;
+            let extract_out = extractor.extract().await?;
+            let links = extract_out.0;
+            dirlist_flag = extract_out.1;
             extractor.request_links(links).await?;
+
+            if matches!(self.order, ScanOrder::Initial) {
+                // check for robots.txt (cannot be in subdirs)
+                let extractor = ExtractorBuilder::default()
+                    .target(ExtractionTarget::RobotsTxt)
+                    .url(&self.target_url)
+                    .handles(self.handles.clone())
+                    .build()?;
+                let links = (extractor.extract().await?).0;
+                extractor.request_links(links).await?;
+            }
         }
 
         let scanned_urls = self.handles.ferox_scans()?;
-
         let ferox_scan = match scanned_urls.get_scan_by_url(&self.target_url) {
             Some(scan) => {
                 scan.set_status(ScanStatus::Running)?;
@@ -105,11 +118,40 @@ impl FeroxScanner {
 
         let progress_bar = ferox_scan.progress_bar();
 
+        // Directory listing heuristic detection to not continue scanning
+        if dirlist_flag {
+            log::trace!("exit: scan_url -> Directory listing heuristic");
+
+            self.handles.stats.send(AddToF64Field(
+                DirScanTimes,
+                scan_timer.elapsed().as_secs_f64(),
+            ))?;
+
+            self.handles.stats.send(SubtractFromUsizeField(
+                TotalExpected,
+                progress_bar.length() as usize,
+            ))?;
+
+            progress_bar.reset_eta();
+            progress_bar.finish_with_message(&format!(
+                "=> {}",
+                style("Directory listing").blue().bright()
+            ));
+
+            ferox_scan.finish()?;
+
+            return Ok(());
+        }
+
         // When acquire is called and the semaphore has remaining permits, the function immediately
         // returns a permit. However, if no remaining permits are available, acquire (asynchronously)
         // waits until an outstanding permit is dropped, at which point, the freed permit is assigned
         // to the caller.
         let _permit = self.scan_limiter.acquire().await;
+        if self.handles.config.scan_limit > 0 {
+            scan_timer = Instant::now();
+            progress_bar.reset();
+        }
 
         // Arc clones to be passed around to the various scans
         let looping_words = self.wordlist.clone();
@@ -122,7 +164,8 @@ impl FeroxScanner {
         }
 
         let requester = Arc::new(Requester::from(self, ferox_scan.clone())?);
-        let increment_len = (self.handles.config.extensions.len() + 1) as u64;
+        let increment_len =
+            ((self.handles.config.extensions.len() + 1) * self.handles.config.methods.len()) as u64;
 
         // producer tasks (mp of mpsc); responsible for making requests
         let producers = stream::iter(looping_words.deref().to_owned())
@@ -137,14 +180,30 @@ impl FeroxScanner {
                             // for every word in the wordlist, check to see if PAUSE_SCAN is set to true
                             // when true; enter a busy loop that only exits by setting PAUSE_SCAN back
                             // to false
-                            let num_cancelled = scanned_urls_clone.pause(true).await;
-                            if num_cancelled > 0 {
-                                handles_clone
-                                    .stats
-                                    .send(SubtractFromUsizeField(TotalExpected, num_cancelled))
-                                    .unwrap_or_else(|e| {
-                                        log::warn!("Could not update overall scan bar: {}", e)
-                                    });
+                            match scanned_urls_clone.pause(true).await {
+                                Some(MenuCmdResult::Url(url)) => {
+                                    // user wants to add a new url to be scanned, need to send
+                                    // it over to the event handler for processing
+                                    handles_clone
+                                        .send_scan_command(Command::ScanNewUrl(url))
+                                        .unwrap_or_else(|e| {
+                                            log::warn!("Could not add scan to scan queue: {}", e)
+                                        })
+                                }
+                                Some(MenuCmdResult::NumCancelled(num_canx)) => {
+                                    if num_canx > 0 {
+                                        handles_clone
+                                            .stats
+                                            .send(SubtractFromUsizeField(TotalExpected, num_canx))
+                                            .unwrap_or_else(|e| {
+                                                log::warn!(
+                                                    "Could not update overall scan bar: {}",
+                                                    e
+                                                )
+                                            });
+                                    }
+                                }
+                                _ => {}
                             }
                         }
                         requester_clone

src/scanner/init.rs

@@ -11,12 +11,9 @@ pub async fn initialize(num_words: usize, handles: Arc<Handles>) -> Result<()> {
     log::trace!("enter: initialize({}, {:?})", num_words, handles);
 
     // number of requests only needs to be calculated once, and then can be reused
-    let num_reqs_expected: u64 = if handles.config.extensions.is_empty() {
-        num_words.try_into()?
-    } else {
-        let total = num_words * (handles.config.extensions.len() + 1);
-        total.try_into()?
-    };
+    let num_reqs_expected: u64 =
+        (num_words * (handles.config.extensions.len() + 1) * (handles.config.methods.len()))
+            .try_into()?;
 
     {
         // no real reason to keep the arc around beyond this call

src/scanner/requester.rs

@@ -17,7 +17,7 @@ use crate::{
         Command::{self, AddError, SubtractFromUsizeField},
         Handles,
     },
-    extractor::{ExtractionTarget::ResponseBody, ExtractorBuilder},
+    extractor::{ExtractionTarget, ExtractorBuilder},
     response::FeroxResponse,
     scan_manager::{FeroxScan, ScanStatus},
     statistics::{StatError::Other, StatField::TotalExpected},
@@ -306,107 +306,124 @@ impl Requester {
         let urls =
             FeroxUrl::from_string(&self.target_url, self.handles.clone()).formatted_urls(word)?;
 
-        let should_test_deny = !self.handles.config.url_denylist.is_empty();
+        let should_test_deny = !self.handles.config.url_denylist.is_empty()
+            || !self.handles.config.regex_denylist.is_empty();
 
         for url in urls {
-            // auto_tune is true, or rate_limit was set (mutually exclusive to user)
-            // and a rate_limiter has been created
-            // short-circuiting the lock access behind the first boolean check
-            let should_tune = self.handles.config.auto_tune || self.handles.config.rate_limit > 0;
-            let should_limit = should_tune && self.rate_limiter.read().await.is_some();
+            for method in self.handles.config.methods.iter() {
+                // auto_tune is true, or rate_limit was set (mutually exclusive to user)
+                // and a rate_limiter has been created
+                // short-circuiting the lock access behind the first boolean check
+                let should_tune =
+                    self.handles.config.auto_tune || self.handles.config.rate_limit > 0;
+                let should_limit = should_tune && self.rate_limiter.read().await.is_some();
 
-            if should_limit {
-                // found a rate limiter, limit that junk!
-                if let Err(e) = self.limit().await {
-                    log::warn!("Could not rate limit scan: {}", e);
-                    self.handles.stats.send(AddError(Other)).unwrap_or_default();
-                }
-            }
-
-            if should_test_deny && should_deny_url(&url, self.handles.clone())? {
-                // can't allow a denied url to be requested
-                continue;
-            }
-
-            let response = logged_request(&url, self.handles.clone()).await?;
-
-            if (should_tune || self.handles.config.auto_bail)
-                && !atomic_load!(self.policy_data.cooling_down, Ordering::SeqCst)
-            {
-                // only check for policy enforcement when the trigger isn't on cooldown and tuning
-                // or bailing is in place (should_tune used here because when auto-tune is on, we'll
-                // reach this without a rate_limiter in place)
-                match self.policy_data.policy {
-                    RequesterPolicy::AutoTune => {
-                        if let Some(trigger) = self.should_enforce_policy() {
-                            self.tune(trigger).await?;
-                        }
+                if should_limit {
+                    // found a rate limiter, limit that junk!
+                    if let Err(e) = self.limit().await {
+                        log::warn!("Could not rate limit scan: {}", e);
+                        self.handles.stats.send(AddError(Other)).unwrap_or_default();
                     }
-                    RequesterPolicy::AutoBail => {
-                        if let Some(trigger) = self.should_enforce_policy() {
-                            self.bail(trigger).await?;
-                        }
-                    }
-                    RequesterPolicy::Default => {}
                 }
-            }
 
-            // response came back without error, convert it to FeroxResponse
-            let ferox_response =
-                FeroxResponse::from(response, true, self.handles.config.output_level).await;
+                if should_test_deny && should_deny_url(&url, self.handles.clone())? {
+                    // can't allow a denied url to be requested
+                    continue;
+                }
 
-            // do recursion if appropriate
-            if !self.handles.config.no_recursion {
-                self.handles
-                    .send_scan_command(Command::TryRecursion(Box::new(ferox_response.clone())))?;
-                let (tx, rx) = oneshot::channel::<bool>();
-                self.handles.send_scan_command(Command::Sync(tx))?;
-                rx.await?;
-            }
-
-            // purposefully doing recursion before filtering. the thought process is that
-            // even though this particular url is filtered, subsequent urls may not
-            if self
-                .handles
-                .filters
-                .data
-                .should_filter_response(&ferox_response, self.handles.stats.tx.clone())
-            {
-                continue;
-            }
-
-            if self.handles.config.extract_links && !ferox_response.status().is_redirection() {
-                let extractor = ExtractorBuilder::default()
-                    .target(ResponseBody)
-                    .response(&ferox_response)
-                    .handles(self.handles.clone())
-                    .build()?;
-
-                let new_links: HashSet<_>;
-                let extracted = extractor.extract().await?;
+                let response = logged_request(
+                    &url,
+                    method.as_str(),
+                    Some(self.handles.config.data.as_slice()),
+                    self.handles.clone(),
+                )
+                .await?;
 
+                if (should_tune || self.handles.config.auto_bail)
+                    && !atomic_load!(self.policy_data.cooling_down, Ordering::SeqCst)
                 {
-                    // gain and quickly drop the read lock on seen_links, using it while unlocked
-                    // to determine if there are any new links to process
-                    let read_links = self.seen_links.read().await;
-                    new_links = extracted.difference(&read_links).cloned().collect();
-                }
-
-                if !new_links.is_empty() {
-                    // using is_empty instead of direct iteration to acquire the write lock behind
-                    // some kind of less expensive gate (and not in a loop, obv)
-                    let mut write_links = self.seen_links.write().await;
-                    for new_link in &new_links {
-                        write_links.insert(new_link.to_owned());
+                    // only check for policy enforcement when the trigger isn't on cooldown and tuning
+                    // or bailing is in place (should_tune used here because when auto-tune is on, we'll
+                    // reach this without a rate_limiter in place)
+                    match self.policy_data.policy {
+                        RequesterPolicy::AutoTune => {
+                            if let Some(trigger) = self.should_enforce_policy() {
+                                self.tune(trigger).await?;
+                            }
+                        }
+                        RequesterPolicy::AutoBail => {
+                            if let Some(trigger) = self.should_enforce_policy() {
+                                self.bail(trigger).await?;
+                            }
+                        }
+                        RequesterPolicy::Default => {}
                     }
                 }
 
-                extractor.request_links(new_links).await?;
-            }
+                // response came back without error, convert it to FeroxResponse
+                let ferox_response = FeroxResponse::from(
+                    response,
+                    &self.target_url,
+                    method,
+                    true,
+                    self.handles.config.output_level,
+                )
+                .await;
 
-            // everything else should be reported
-            if let Err(e) = ferox_response.send_report(self.handles.output.tx.clone()) {
-                log::warn!("Could not send FeroxResponse to output handler: {}", e);
+                // do recursion if appropriate
+                if !self.handles.config.no_recursion {
+                    self.handles
+                        .send_scan_command(Command::TryRecursion(Box::new(
+                            ferox_response.clone(),
+                        )))?;
+                    let (tx, rx) = oneshot::channel::<bool>();
+                    self.handles.send_scan_command(Command::Sync(tx))?;
+                    rx.await?;
+                }
+
+                // purposefully doing recursion before filtering. the thought process is that
+                // even though this particular url is filtered, subsequent urls may not
+                if self
+                    .handles
+                    .filters
+                    .data
+                    .should_filter_response(&ferox_response, self.handles.stats.tx.clone())
+                {
+                    continue;
+                }
+
+                if self.handles.config.extract_links && !ferox_response.status().is_redirection() {
+                    let extractor = ExtractorBuilder::default()
+                        .target(ExtractionTarget::ResponseBody)
+                        .response(&ferox_response)
+                        .handles(self.handles.clone())
+                        .build()?;
+                    let new_links: HashSet<_>;
+                    let extracted = (extractor.extract().await?).0;
+
+                    {
+                        // gain and quickly drop the read lock on seen_links, using it while unlocked
+                        // to determine if there are any new links to process
+                        let read_links = self.seen_links.read().await;
+                        new_links = extracted.difference(&read_links).cloned().collect();
+                    }
+
+                    if !new_links.is_empty() {
+                        // using is_empty instead of direct iteration to acquire the write lock behind
+                        // some kind of less expensive gate (and not in a loop, obv)
+                        let mut write_links = self.seen_links.write().await;
+                        for new_link in &new_links {
+                            write_links.insert(new_link.to_owned());
+                        }
+                    }
+
+                    extractor.request_links(new_links).await?;
+                }
+
+                // everything else should be reported
+                if let Err(e) = ferox_response.send_report(self.handles.output.tx.clone()) {
+                    log::warn!("Could not send FeroxResponse to output handler: {}", e);
+                }
             }
         }
 

src/statistics/container.rs

@@ -126,9 +126,6 @@ pub struct Stats {
     /// tracker for total runtime
     total_runtime: Mutex<Vec<f64>>,
 
-    /// tracker for the number of extensions the user specified
-    num_extensions: usize,
-
     /// tracker for whether to use json during serialization or not
     json: bool,
 }
@@ -203,7 +200,7 @@ impl<'a> Deserialize<'a> for Stats {
     where
         D: Deserializer<'a>,
     {
-        let stats = Self::new(0, false);
+        let stats = Self::new(false);
 
         let map: HashMap<String, Value> = HashMap::deserialize(deserializer)?;
 
@@ -446,9 +443,8 @@ impl<'a> Deserialize<'a> for Stats {
 impl Stats {
     /// Small wrapper for default to set `kind` to "statistics" and `total_runtime` to have at least
     /// one value
-    pub fn new(num_extensions: usize, is_json: bool) -> Self {
+    pub fn new(is_json: bool) -> Self {
         Self {
-            num_extensions,
             json: is_json,
             kind: String::from("statistics"),
             total_runtime: Mutex::new(vec![0.0]),
@@ -620,12 +616,10 @@ impl Stats {
                 atomic_increment!(self.expected_per_scan, value);
             }
             StatField::TotalScans => {
-                let multiplier = self.num_extensions.max(1);
-
                 atomic_increment!(self.total_scans, value);
                 atomic_increment!(
                     self.total_expected,
-                    value * self.expected_per_scan.load(Ordering::Relaxed) * multiplier
+                    value * self.expected_per_scan.load(Ordering::Relaxed)
                 );
             }
             StatField::TotalExpected => {
@@ -792,7 +786,7 @@ mod tests {
     ///     - errors
     fn stats_increments_timeouts() {
         let config = Configuration::new().unwrap();
-        let stats = Stats::new(config.extensions.len(), config.json);
+        let stats = Stats::new(config.json);
 
         stats.add_error(StatError::Timeout);
         stats.add_error(StatError::Timeout);
@@ -810,7 +804,7 @@ mod tests {
     ///     - responses_filtered
     fn stats_increments_wildcards() {
         let config = Configuration::new().unwrap();
-        let stats = Stats::new(config.extensions.len(), config.json);
+        let stats = Stats::new(config.json);
 
         assert_eq!(stats.responses_filtered.load(Ordering::Relaxed), 0);
         assert_eq!(stats.wildcards_filtered.load(Ordering::Relaxed), 0);
@@ -826,7 +820,7 @@ mod tests {
     /// when Stats::update_usize_field receives StatField::ResponsesFiltered, it should increment
     fn stats_increments_responses_filtered() {
         let config = Configuration::new().unwrap();
-        let stats = Stats::new(config.extensions.len(), config.json);
+        let stats = Stats::new(config.json);
 
         assert_eq!(stats.responses_filtered.load(Ordering::Relaxed), 0);
 
@@ -842,7 +836,7 @@ mod tests {
     fn stats_merge_from_alters_correct_fields() {
         let contents = r#"{"statistics":{"type":"statistics","timeouts":1,"requests":9207,"expected_per_scan":707,"total_expected":9191,"errors":3,"successes":720,"redirects":13,"client_errors":8474,"server_errors":2,"total_scans":13,"initial_targets":1,"links_extracted":51,"status_403s":3,"status_200s":720,"status_301s":12,"status_302s":1,"status_401s":4,"status_429s":2,"status_500s":5,"status_503s":9,"status_504s":6,"status_508s":7,"wildcards_filtered":707,"responses_filtered":707,"resources_discovered":27,"directory_scan_times":[2.211973078,1.989015505,1.898675839,3.9714468910000003,4.938152838,5.256073528,6.021986595,6.065740734,6.42633762,7.095142125,7.336982137,5.319785619,4.843649778],"total_runtime":[11.556575456000001],"url_format_errors":17,"redirection_errors":12,"connection_errors":21,"request_errors":4}}"#;
         let config = Configuration::new().unwrap();
-        let stats = Stats::new(config.extensions.len(), config.json);
+        let stats = Stats::new(config.json);
 
         let tfile = NamedTempFile::new().unwrap();
         write(&tfile, contents).unwrap();
@@ -893,7 +887,7 @@ mod tests {
     /// ensure update runtime overwrites the default 0th entry
     fn update_runtime_works() {
         let config = Configuration::new().unwrap();
-        let stats = Stats::new(config.extensions.len(), config.json);
+        let stats = Stats::new(config.json);
 
         assert!((stats.total_runtime.lock().unwrap()[0] - 0.0).abs() < f64::EPSILON);
         stats.update_runtime(20.2);
@@ -904,7 +898,7 @@ mod tests {
     /// ensure status_403s returns the correct value
     fn status_403s_returns_correct_value() {
         let config = Configuration::new().unwrap();
-        let stats = Stats::new(config.extensions.len(), config.json);
+        let stats = Stats::new(config.json);
         stats.status_403s.store(12, Ordering::Relaxed);
         assert_eq!(stats.status_403s(), 12);
     }
@@ -913,7 +907,7 @@ mod tests {
     /// ensure status_403s returns the correct value
     fn status_429s_returns_correct_value() {
         let config = Configuration::new().unwrap();
-        let stats = Stats::new(config.extensions.len(), config.json);
+        let stats = Stats::new(config.json);
         stats.status_429s.store(141, Ordering::Relaxed);
         assert_eq!(stats.status_429s(), 141);
     }

src/statistics/macros.rs

@@ -18,10 +18,10 @@ macro_rules! atomic_increment {
 #[macro_export]
 macro_rules! atomic_load {
     ($metric:expr) => {
-        $metric.load(Ordering::Relaxed);
+        $metric.load(Ordering::Relaxed)
     };
     ($metric:expr, $ordering:expr) => {
-        $metric.load($ordering);
+        $metric.load($ordering)
     };
 }
 

src/statistics/tests.rs

@@ -41,7 +41,7 @@ async fn statistics_handler_exits() -> Result<()> {
 /// Stats::save should write contents of Stats to disk
 fn save_writes_stats_object_to_disk() {
     let config = Configuration::new().unwrap();
-    let stats = Stats::new(config.extensions.len(), config.json);
+    let stats = Stats::new(config.json);
 
     stats.add_request();
     stats.add_request();

src/url.rs

@@ -7,7 +7,7 @@ use std::{convert::TryInto, fmt, sync::Arc};
 #[derive(Debug)]
 pub struct FeroxUrl {
     /// string representation of the target url
-    target: String,
+    pub target: String,
 
     /// Handles object for grabbing config values
     handles: Arc<Handles>,
@@ -42,7 +42,13 @@ impl FeroxUrl {
 
         let mut urls = vec![];
 
-        match self.format(word, None) {
+        let slash = if self.handles.config.add_slash {
+            Some("/")
+        } else {
+            None
+        };
+
+        match self.format(word, slash) {
             // default request, i.e. no extension
             Ok(url) => urls.push(url),
             Err(_) => self.handles.stats.send(AddError(UrlFormat))?,
@@ -55,7 +61,6 @@ impl FeroxUrl {
                 Err(_) => self.handles.stats.send(AddError(UrlFormat))?,
             }
         }
-
         log::trace!("exit: formatted_urls -> {:?}", urls);
         Ok(urls)
     }
@@ -97,13 +102,25 @@ impl FeroxUrl {
             self.target.to_string()
         };
 
-        // extensions and slashes are mutually exclusive cases
-        let word = if extension.is_some() {
-            format!("{}.{}", word, extension.unwrap())
-        } else if self.handles.config.add_slash && !word.ends_with('/') {
-            // -f used, and word doesn't already end with a /
-            format!("{}/", word)
-        } else if word.starts_with("//") {
+        // As of version 2.3.4, extensions and trailing slashes are no longer mutually exclusive.
+        // Trailing slashes are now treated as just another extension, which is pretty clever.
+        //
+        // In addition to the change above, @cortantief ID'd a bug here that incorrectly handled
+        // 2 leading forward slashes when extensions were used. This block addresses the bugfix.
+        let mut word = if let Some(ext) = extension {
+            // We handle the special case of forward slash
+            // That allow us to treat it as an extension with a particular format
+            if ext == "/" {
+                format!("{}/", word)
+            } else {
+                format!("{}.{}", word, ext)
+            }
+        } else {
+            String::from(word)
+        };
+
+        // We check separately if the current word begins with 2 forward slashes
+        if word.starts_with("//") {
             // bug ID'd by @Sicks3c, when a wordlist contains words that begin with 2 forward slashes
             // i.e. //1_40_0/static/js, it gets joined onto the base url in a surprising way
             // ex: https://localhost/ + //1_40_0/static/js -> https://1_40_0/static/js
@@ -111,9 +128,7 @@ impl FeroxUrl {
             // and simply removes prefixed forward slashes if there are two of them. Additionally,
             // trim_start_matches will trim the pattern until it's gone, so even if there are more than
             // 2 /'s, they'll still be trimmed
-            word.trim_start_matches('/').to_string()
-        } else {
-            String::from(word)
+            word = word.trim_start_matches('/').to_string();
         };
 
         let base_url = Url::parse(&url)?;
@@ -451,6 +466,20 @@ mod tests {
         );
     }
 
+    #[test]
+    /// word with two prepended slashes and extensions doesn't discard the entire domain
+    fn format_url_word_with_two_prepended_slashes_and_extensions() {
+        let handles = Arc::new(Handles::for_testing(None, None).0);
+        let url = FeroxUrl::from_string("http://localhost", handles);
+        for ext in ["rocks", "fun"] {
+            let to_check = format!("http://localhost/upload/ferox.{}", ext);
+            assert_eq!(
+                url.format("//upload/ferox", Some(ext)).unwrap(),
+                reqwest::Url::parse(&to_check[..]).unwrap()
+            );
+        }
+    }
+
     #[test]
     /// word that is a fully formed url, should return an error
     fn format_url_word_that_is_a_url() {
@@ -460,4 +489,33 @@ mod tests {
 
         assert!(formatted.is_err());
     }
+
+    #[test]
+    /// sending url + word with both an extension and add-slash should get back
+    /// two urls, one with '/' appended to the word, and the other with the extension
+    /// appended
+    fn formatted_urls_with_postslash_and_extensions() {
+        let config = Configuration {
+            add_slash: true,
+            extensions: vec!["rocks".to_string(), "fun".to_string()],
+            ..Default::default()
+        };
+        let handles = Arc::new(Handles::for_testing(None, Some(Arc::new(config))).0);
+        let url = FeroxUrl::from_string("http://localhost", handles);
+        match url.formatted_urls("ferox") {
+            Ok(urls) => {
+                // 3 = One for the main word + slash and for the two extensions
+                assert_eq!(urls.len(), 3);
+                assert_eq!(
+                    urls,
+                    [
+                        Url::parse("http://localhost/ferox/").unwrap(),
+                        Url::parse("http://localhost/ferox.rocks").unwrap(),
+                        Url::parse("http://localhost/ferox.fun").unwrap(),
+                    ]
+                )
+            }
+            Err(err) => panic!("{}", err.to_string()),
+        }
+    }
 }

src/utils.rs

@@ -1,7 +1,8 @@
 use anyhow::{bail, Context, Result};
 use console::{strip_ansi_codes, style, user_attended};
 use indicatif::ProgressBar;
-use reqwest::{Client, Response, StatusCode, Url};
+use regex::Regex;
+use reqwest::{Client, Method, Response, StatusCode, Url};
 #[cfg(not(target_os = "windows"))]
 use rlimit::{getrlimit, setrlimit, Resource};
 use std::{
@@ -13,6 +14,7 @@ use std::{
 };
 use tokio::sync::mpsc::UnboundedSender;
 
+use crate::config::Configuration;
 use crate::{
     config::OutputLevel,
     event_handlers::{
@@ -23,8 +25,12 @@ use crate::{
     send_command,
     statistics::StatError::{Connection, Other, Redirection, Request, Timeout},
     traits::FeroxSerialize,
+    USER_AGENTS,
 };
 
+/// simple counter for grabbing 'random' user agents
+static mut USER_AGENT_CTR: usize = 0;
+
 /// Given the path to a file, open the file in append mode (create it if it doesn't exist) and
 /// return a reference to the buffered file
 pub fn open_file(filename: &str) -> Result<BufWriter<fs::File>> {
@@ -89,15 +95,19 @@ pub fn ferox_print(msg: &str, bar: &ProgressBar) {
 
 /// wrapper for make_request used to pass error/response codes to FeroxScans for per-scan stats
 /// tracking of information related to auto-tune/bail
-pub async fn logged_request(url: &Url, handles: Arc<Handles>) -> Result<Response> {
+pub async fn logged_request(
+    url: &Url,
+    method: &str,
+    data: Option<&[u8]>,
+    handles: Arc<Handles>,
+) -> Result<Response> {
     let client = &handles.config.client;
     let level = handles.config.output_level;
     let tx_stats = handles.stats.tx.clone();
 
-    let response = make_request(client, url, level, tx_stats).await;
+    let response = make_request(client, url, method, data, level, &handles.config, tx_stats).await;
 
     let scans = handles.ferox_scans()?;
-
     match response {
         Ok(resp) => {
             match resp.status() {
@@ -120,7 +130,10 @@ pub async fn logged_request(url: &Url, handles: Arc<Handles>) -> Result<Response
 pub async fn make_request(
     client: &Client,
     url: &Url,
+    method: &str,
+    data: Option<&[u8]>,
     output_level: OutputLevel,
+    config: &Configuration,
     tx_stats: UnboundedSender<Command>,
 ) -> Result<Response> {
     log::trace!(
@@ -130,7 +143,23 @@ pub async fn make_request(
         tx_stats
     );
 
-    match client.get(url.to_owned()).send().await {
+    let mut request = client.request(Method::from_bytes(method.as_bytes())?, url.to_owned());
+    if let Some(body_data) = data {
+        request = request.body(body_data.to_vec());
+    }
+
+    if config.random_agent {
+        let index = unsafe {
+            USER_AGENT_CTR += 1;
+            USER_AGENT_CTR % USER_AGENTS.len()
+        };
+
+        let user_agent = USER_AGENTS[index];
+
+        request = request.header("User-Agent", user_agent);
+    }
+
+    match request.send().await {
         Err(e) => {
             log::trace!("exit: make_request -> {}", e);
 
@@ -139,22 +168,28 @@ pub async fn make_request(
             } else if e.is_redirect() {
                 if let Some(last_redirect) = e.url() {
                     // get where we were headed (last_redirect) and where we came from (url)
-                    let fancy_message = format!("{} !=> {}", url, last_redirect);
+                    let fancy_message = format!(
+                        "{} !=> {} ({})",
+                        url,
+                        last_redirect,
+                        style("too many redirects").red(),
+                    );
 
-                    let report = if let Some(msg_status) = e.status() {
-                        send_command!(tx_stats, AddStatus(msg_status));
-                        create_report_string(
-                            msg_status.as_str(),
-                            "-1",
-                            "-1",
-                            "-1",
-                            &fancy_message,
-                            output_level,
-                        )
-                    } else {
-                        create_report_string("UNK", "-1", "-1", "-1", &fancy_message, output_level)
+                    let msg_status = match e.status() {
+                        Some(status) => status.to_string(),
+                        None => "ERR".to_string(),
                     };
 
+                    let report = create_report_string(
+                        &msg_status,
+                        method,
+                        "-1",
+                        "-1",
+                        "-1",
+                        &fancy_message,
+                        output_level,
+                    );
+
                     send_command!(tx_stats, AddError(Redirection));
 
                     ferox_print(&report, &PROGRESS_PRINTER)
@@ -184,6 +219,7 @@ pub async fn make_request(
 /// 200      127l      283w     4134c http://localhost/faq
 pub fn create_report_string(
     status: &str,
+    method: &str,
     line_count: &str,
     word_count: &str,
     content_length: &str,
@@ -197,8 +233,8 @@ pub fn create_report_string(
         // normal printing with status and sizes
         let color_status = status_colorizer(status);
         format!(
-            "{} {:>8}l {:>8}w {:>8}c {}\n",
-            color_status, line_count, word_count, content_length, url
+            "{} {:>8} {:>8}l {:>8}w {:>8}c {}\n",
+            color_status, method, line_count, word_count, content_length, url
         )
     }
 }
@@ -288,6 +324,110 @@ where
     Ok(())
 }
 
+/// determine if a url should be denied based on the given absolute url
+fn should_deny_absolute(url_to_test: &Url, denier: &Url, handles: Arc<Handles>) -> Result<bool> {
+    log::trace!(
+        "enter: should_deny_absolute({}, {:?})",
+        url_to_test.as_str(),
+        denier.as_str(),
+    );
+
+    // simplest case is an exact match, check for it first
+    if url_to_test == denier {
+        log::trace!("exit: should_deny_absolute -> true");
+        return Ok(true);
+    }
+
+    match (url_to_test.host(), denier.host()) {
+        // .host() will return an enum with ipv4|6 or domain and is comparable
+        // whereas .domain() returns None for ip addresses
+        (Some(normed_host), Some(denier_host)) => {
+            if normed_host != denier_host {
+                // domains don't even match
+                return Ok(false);
+            }
+        }
+        _ => {
+            // one or the other couldn't determine the host value, which probably means
+            // it's not suitable for further comparison
+            return Ok(false);
+        }
+    }
+
+    let tested_host = url_to_test.host().unwrap(); // match above will catch errors
+
+    // at this point, we have a matching set of ips or domain names. now we can process the
+    // url path. The goal is to determine whether the given url's path is a subpath of any
+    // url in the deny list, for example
+    //    GIVEN URL                        URL DENY LIST               USER-SPECIFIED URLS TO SCAN
+    //    http://some.domain/stuff/things, [http://some.domain/stuff], [http://some.domain] => true
+    //    http://some.domain/stuff/things, [http://some.domain/stuff/things], [http://some.domain] => true
+    //    http://some.domain/stuff/things, [http://some.domain/api], [http://some.domain] => false
+    // the examples above are all pretty obvious, the kicker comes when the blocking url's
+    // path is a parent to a scanned url
+    //    http://some.domain/stuff/things, [http://some.domain/], [http://some.domain/stuff] => false
+    //    http://some.domain/api, [http://some.domain/], [http://some.domain/stuff] => true
+    // we want to deny all children of the parent, unless that child is a child of a scan
+    // we specified through -u(s) or --stdin
+
+    let deny_path = denier.path();
+    let tested_path = url_to_test.path();
+
+    if tested_path.starts_with(deny_path) {
+        // at this point, we know that the given normalized path is a sub-path of the
+        // current deny-url, now we just need to check to see if this deny-url is a parent
+        // to a scanned url that is also a parent of the given url
+        for ferox_scan in handles.ferox_scans()?.get_active_scans() {
+            let scanner = Url::parse(ferox_scan.url().trim_end_matches('/'))
+                .with_context(|| format!("Could not parse {} as a url", ferox_scan))?;
+
+            if let Some(scan_host) = scanner.host() {
+                // same domain/ip check we perform on the denier above
+                if tested_host != scan_host {
+                    // domains don't even match, keep on keepin' on...
+                    continue;
+                }
+            } else {
+                // couldn't process .host from scanner
+                continue;
+            };
+
+            let scan_path = scanner.path();
+
+            if scan_path.starts_with(deny_path) && tested_path.starts_with(scan_path) {
+                // user-specified scan url is a sub-path of the deny-urls's path AND the
+                // url to check is a sub-path of the user-specified scan url
+                //
+                // the assumption is the user knew what they wanted and we're going to give
+                // the scanned url precedence, even though it's a sub-path
+                log::trace!("exit: should_deny_absolute -> false");
+                return Ok(false);
+            }
+        }
+        log::trace!("exit: should_deny_absolute -> true");
+        return Ok(true);
+    }
+
+    log::trace!("exit: should_deny_absolute -> false");
+    Ok(false)
+}
+
+/// determine if a url should be denied based on the given regular expression
+///
+/// the regex ONLY matches against the PATH of the url (not the scheme, host, port, etc)
+fn should_deny_regex(url_to_test: &Url, denier: &Regex) -> bool {
+    log::trace!(
+        "enter: should_deny_regex({}, {})",
+        url_to_test.as_str(),
+        denier,
+    );
+
+    let result = denier.is_match(url_to_test.as_str());
+
+    log::trace!("exit: should_deny_regex -> {}", result);
+    result
+}
+
 /// determines whether or not a given url should be denied based on the user-supplied --dont-scan
 /// flag
 pub fn should_deny_url(url: &Url, handles: Arc<Handles>) -> Result<bool> {
@@ -297,91 +437,29 @@ pub fn should_deny_url(url: &Url, handles: Arc<Handles>) -> Result<bool> {
         handles.config.url_denylist,
         handles.ferox_scans()?
     );
+
     // normalization for comparison is to remove the trailing / if one exists, this is done for
     // the given url and any url to which it's compared
     let normed_url = Url::parse(url.to_string().trim_end_matches('/'))?;
 
-    for deny_url in &handles.config.url_denylist {
-        // parse the denying url for easier comparison
-        let denier = Url::parse(deny_url.trim_end_matches('/'))
-            .with_context(|| format!("Could not parse {} as a url", deny_url))?;
-
-        // simplest case is an exact match, check for it first
-        if normed_url == denier {
-            log::trace!("exit: should_deny_url -> true");
-            return Ok(true);
-        }
-
-        match (normed_url.host(), denier.host()) {
-            // .host() will return an enum with ipv4|6 or domain and is comparable
-            // whereas .domain() returns None for ip addresses
-            (Some(normed_host), Some(denier_host)) => {
-                if normed_host != denier_host {
-                    // domains don't even match, keep on keepin' on...
-                    continue;
-                }
-            }
-            _ => {
-                // one or the other couldn't determine the host value, which probably means
-                // it's not suitable for further comparison
-                continue;
+    for denier in &handles.config.url_denylist {
+        // note to self: it may seem as though we can use regex only for --dont-scan, however, in
+        // doing so, we lose the ability to block a parent directory while scanning a child
+        if let Ok(should_deny) = should_deny_absolute(&normed_url, denier, handles.clone()) {
+            if should_deny {
+                return Ok(true);
             }
         }
+    }
 
-        let normed_host = normed_url.host().unwrap(); // match above will catch errors
-
-        // at this point, we have a matching set of ips or domain names. now we can process the
-        // url path. The goal is to determine whether the given url's path is a subpath of any
-        // url in the deny list, for example
-        //    GIVEN URL                        URL DENY LIST               USER-SPECIFIED URLS TO SCAN
-        //    http://some.domain/stuff/things, [http://some.domain/stuff], [http://some.domain] => true
-        //    http://some.domain/stuff/things, [http://some.domain/stuff/things], [http://some.domain] => true
-        //    http://some.domain/stuff/things, [http://some.domain/api], [http://some.domain] => false
-        // the examples above are all pretty obvious, the kicker comes when the blocking url's
-        // path is a parent to a scanned url
-        //    http://some.domain/stuff/things, [http://some.domain/], [http://some.domain/stuff] => false
-        //    http://some.domain/api, [http://some.domain/], [http://some.domain/stuff] => true
-        // we want to deny all children of the parent, unless that child is a child of a scan
-        // we specified through -u(s) or --stdin
-
-        let deny_path = denier.path();
-        let norm_path = normed_url.path();
-
-        if norm_path.starts_with(deny_path) {
-            // at this point, we know that the given normalized path is a sub-path of the
-            // current deny-url, now we just need to check to see if this deny-url is a parent
-            // to a scanned url that is also a parent of the given url
-            for ferox_scan in handles.ferox_scans()?.get_active_scans() {
-                let scanner = Url::parse(ferox_scan.url().trim_end_matches('/'))
-                    .with_context(|| format!("Could not parse {} as a url", ferox_scan))?;
-
-                if let Some(scan_host) = scanner.host() {
-                    // same domain/ip check we perform on the denier above
-                    if normed_host != scan_host {
-                        // domains don't even match, keep on keepin' on...
-                        continue;
-                    }
-                } else {
-                    // couldn't process .host from scanner
-                    continue;
-                };
-
-                let scan_path = scanner.path();
-
-                if scan_path.starts_with(deny_path) && norm_path.starts_with(scan_path) {
-                    // user-specified scan url is a sub-path of the deny-urls's path AND the
-                    // url to check is a sub-path of the user-specified scan url
-                    //
-                    // the assumption is the user knew what they wanted and we're going to give
-                    // the scanned url precedence, even though it's a sub-path
-                    log::trace!("exit: should_deny_url -> false");
-                    return Ok(false);
-                }
-            }
-            log::trace!("exit: should_deny_url -> true");
+    for denier in &handles.config.regex_denylist {
+        if should_deny_regex(&normed_url, denier) {
             return Ok(true);
         }
     }
+
+    // made it to the end of the deny lists unscathed, return false, indicating we should not deny
+    // this particular url
     log::trace!("exit: should_deny_url -> false");
     Ok(false)
 }
@@ -506,7 +584,7 @@ mod tests {
         scans.add_directory_scan(scan_url, ScanOrder::Initial);
 
         let mut config = Configuration::new().unwrap();
-        config.url_denylist = vec![String::from(deny_url)];
+        config.url_denylist = vec![Url::parse(deny_url).unwrap()];
         let config = Arc::new(config);
 
         let handles = Arc::new(Handles::for_testing(Some(scans), Some(config)).0);
@@ -525,7 +603,7 @@ mod tests {
         scans.add_directory_scan(scan_url, ScanOrder::Initial);
 
         let mut config = Configuration::new().unwrap();
-        config.url_denylist = vec![String::from(deny_url)];
+        config.url_denylist = vec![Url::parse(deny_url).unwrap()];
         let config = Arc::new(config);
 
         let handles = Arc::new(Handles::for_testing(Some(scans), Some(config)).0);
@@ -544,7 +622,7 @@ mod tests {
         scans.add_directory_scan(scan_url, ScanOrder::Initial);
 
         let mut config = Configuration::new().unwrap();
-        config.url_denylist = vec![String::from(deny_url)];
+        config.url_denylist = vec![Url::parse(deny_url).unwrap()];
         let config = Arc::new(config);
 
         let handles = Arc::new(Handles::for_testing(Some(scans), Some(config)).0);
@@ -565,7 +643,7 @@ mod tests {
         scans.add_directory_scan(scan_url, ScanOrder::Initial);
 
         let mut config = Configuration::new().unwrap();
-        config.url_denylist = vec![String::from(deny_url)];
+        config.url_denylist = vec![Url::parse(deny_url).unwrap()];
         let config = Arc::new(config);
 
         let handles = Arc::new(Handles::for_testing(Some(scans), Some(config)).0);
@@ -586,7 +664,7 @@ mod tests {
         scans.add_directory_scan(scan_url, ScanOrder::Initial);
 
         let mut config = Configuration::new().unwrap();
-        config.url_denylist = vec![String::from(deny_url)];
+        config.url_denylist = vec![Url::parse(deny_url).unwrap()];
         let config = Arc::new(config);
 
         let handles = Arc::new(Handles::for_testing(Some(scans), Some(config)).0);
@@ -605,7 +683,7 @@ mod tests {
         scans.add_directory_scan(scan_url, ScanOrder::Initial);
 
         let mut config = Configuration::new().unwrap();
-        config.url_denylist = vec![String::from(deny_url)];
+        config.url_denylist = vec![Url::parse(deny_url).unwrap()];
         let config = Arc::new(config);
 
         let handles = Arc::new(Handles::for_testing(Some(scans), Some(config)).0);
@@ -624,7 +702,7 @@ mod tests {
         scans.add_directory_scan(scan_url, ScanOrder::Initial);
 
         let mut config = Configuration::new().unwrap();
-        config.url_denylist = vec![String::from(deny_url)];
+        config.url_denylist = vec![Url::parse(deny_url).unwrap()];
         let config = Arc::new(config);
 
         let handles = Arc::new(Handles::for_testing(Some(scans), Some(config)).0);
@@ -643,7 +721,7 @@ mod tests {
         scans.add_directory_scan(scan_url, ScanOrder::Initial);
 
         let mut config = Configuration::new().unwrap();
-        config.url_denylist = vec![String::from(deny_url)];
+        config.url_denylist = vec![Url::parse(deny_url).unwrap()];
         let config = Arc::new(config);
 
         let handles = Arc::new(Handles::for_testing(Some(scans), Some(config)).0);
@@ -662,11 +740,53 @@ mod tests {
         scans.add_directory_scan(scan_url, ScanOrder::Initial);
 
         let mut config = Configuration::new().unwrap();
-        config.url_denylist = vec![String::from(deny_url)];
+        config.url_denylist = vec![Url::parse(deny_url).unwrap()];
         let config = Arc::new(config);
 
         let handles = Arc::new(Handles::for_testing(Some(scans), Some(config)).0);
 
         assert!(!should_deny_url(&tested_url, handles).unwrap());
     }
+
+    #[test]
+    /// provide a denier where the tested url is matched against a regular expression in the path
+    /// of the url
+    fn should_deny_url_blocks_urls_based_on_regex_in_path() {
+        let scan_url = "https://testdomain.com/";
+        let deny_pattern = "/deni.*";
+        let tested_url = Url::parse("https://testdomain.com/denied/").unwrap();
+
+        let scans = Arc::new(FeroxScans::default());
+        scans.add_directory_scan(scan_url, ScanOrder::Initial);
+
+        let mut config = Configuration::new().unwrap();
+        config.regex_denylist = vec![Regex::new(deny_pattern).unwrap()];
+        let config = Arc::new(config);
+
+        let handles = Arc::new(Handles::for_testing(Some(scans), Some(config)).0);
+
+        assert!(should_deny_url(&tested_url, handles).unwrap());
+    }
+
+    #[test]
+    /// provide a denier where the tested url is matched against a regular expression in the scheme
+    /// of the url
+    fn should_deny_url_blocks_urls_based_on_regex_in_scheme() {
+        let scan_url = "https://testdomain.com/";
+        let deny_pattern = "http:";
+        let tested_http_url = Url::parse("http://testdomain.com/denied/").unwrap();
+        let tested_https_url = Url::parse("https://testdomain.com/denied/").unwrap();
+
+        let scans = Arc::new(FeroxScans::default());
+        scans.add_directory_scan(scan_url, ScanOrder::Initial);
+
+        let mut config = Configuration::new().unwrap();
+        config.regex_denylist = vec![Regex::new(deny_pattern).unwrap()];
+        let config = Arc::new(config);
+
+        let handles = Arc::new(Handles::for_testing(Some(scans), Some(config)).0);
+
+        assert!(!should_deny_url(&tested_https_url, handles.clone()).unwrap());
+        assert!(should_deny_url(&tested_http_url, handles).unwrap());
+    }
 }

tests/test_banner.rs

@@ -115,7 +115,7 @@ fn banner_prints_headers() {
 
 #[test]
 /// test allows non-existent wordlist to trigger the banner printing to stderr
-/// expect to see all mandatory prints + multiple dont scan entries
+/// expect to see all mandatory prints + multiple dont scan url & regex entries
 fn banner_prints_denied_urls() {
     Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -123,8 +123,9 @@ fn banner_prints_denied_urls() {
         .arg("http://localhost")
         .arg("--dont-scan")
         .arg("http://dont-scan.me")
-        .arg("--dont-scan")
         .arg("https://also-not.me")
+        .arg("https:")
+        .arg("/deny.*")
         .assert()
         .success()
         .stderr(
@@ -136,9 +137,37 @@ fn banner_prints_denied_urls() {
                 .and(predicate::str::contains("Status Codes"))
                 .and(predicate::str::contains("Timeout (secs)"))
                 .and(predicate::str::contains("User-Agent"))
-                .and(predicate::str::contains("Don't Scan"))
+                .and(predicate::str::contains("Don't Scan Url"))
+                .and(predicate::str::contains("Don't Scan Regex"))
                 .and(predicate::str::contains("http://dont-scan.me"))
                 .and(predicate::str::contains("https://also-not.me"))
+                .and(predicate::str::contains("https:"))
+                .and(predicate::str::contains("/deny.*"))
+                .and(predicate::str::contains("─┴─")),
+        );
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + multiple headers
+fn banner_prints_random_agent() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--random-agent")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Random"))
                 .and(predicate::str::contains("─┴─")),
         );
 }
@@ -1002,3 +1031,63 @@ fn banner_prints_parallel() {
                 .and(predicate::str::contains("User-Agent").not()),
         );
 }
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + methods
+fn banner_prints_methods() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-m")
+        .arg("PUT")
+        .arg("--methods")
+        .arg("OPTIONS")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("HTTP methods"))
+                .and(predicate::str::contains("[PUT, OPTIONS]"))
+                .and(predicate::str::contains("─┴─")),
+        );
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + data body
+fn banner_prints_data() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-m")
+        .arg("PUT")
+        .arg("--methods")
+        .arg("POST")
+        .arg("--data")
+        .arg("some_data")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("HTTP Body"))
+                .and(predicate::str::contains("some_data"))
+                .and(predicate::str::contains("─┴─")),
+        );
+}

tests/test_deny_list.rs

@@ -210,3 +210,73 @@ fn deny_list_works_during_recursion_with_inverted_parents() {
 
     teardown_tmp_directory(tmp_dir);
 }
+
+#[test]
+/// test that a regex that prevents the base url from being scanned results in an early exit
+fn deny_list_prevents_regex_that_denies_base_url() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--dont-scan")
+        .arg("/")
+        .unwrap();
+
+    teardown_tmp_directory(tmp_dir);
+
+    let err_msg = format!(
+        "Could not determine initial targets: The regex '/' matches {}/; the scan will never start",
+        srv.base_url()
+    );
+    cmd.assert()
+        .success()
+        .stderr(predicate::str::contains(err_msg));
+
+    assert_eq!(mock.hits(), 0);
+}
+
+#[test]
+/// test that a url that prevents the base url from being scanned results in an early exit
+fn deny_list_prevents_url_that_denies_base_url() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--dont-scan")
+        .arg(srv.base_url())
+        .unwrap();
+
+    teardown_tmp_directory(tmp_dir);
+
+    let err_msg = format!(
+        "Could not determine initial targets: The url '{}/' matches {}/; the scan will never start",
+        srv.base_url(),
+        srv.base_url()
+    );
+
+    cmd.assert()
+        .success()
+        .stderr(predicate::str::contains(err_msg));
+
+    assert_eq!(mock.hits(), 0);
+}

tests/test_extractor.rs

@@ -288,7 +288,7 @@ fn extractor_finds_robots_txt_links_and_displays_files_or_scans_directories() {
     );
 
     assert_eq!(mock.hits(), 1);
-    assert_eq!(mock_dir.hits(), 1);
+    assert_eq!(mock_dir.hits(), 2);
     assert_eq!(mock_two.hits(), 1);
     assert_eq!(mock_file.hits(), 1);
     assert_eq!(mock_disallowed.hits(), 1);
@@ -370,6 +370,226 @@ fn extractor_finds_robots_txt_links_and_displays_files_non_recursive() {
     teardown_tmp_directory(tmp_dir);
 }
 
+#[test]
+/// serve a directory listing with a file and and a folder contained within it. ferox should
+/// find both links and request each one.
+fn extractor_finds_directory_listing_links_and_displays_files() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["invalid".to_string()], "wordlist").unwrap();
+
+    let mock_root = srv.mock(|when, then| {
+        when.method(GET).path("/");
+        then.status(200).body(
+            r#"
+            <html>
+            <head>
+            <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+            <title>Directory listing for /</title>
+            </head>
+            <body>
+            <h1>Directory listing for /</h1>
+            <hr>
+            <ul>
+            <li><a href="disallowed-subdir/">disallowed-subdir/</a></li>
+            <li><a href="LICENSE">LICENSE</a></li>
+            <li><a href="misc/">misc/</a></li>
+            </ul>
+            <hr>
+            </body>
+            </html>
+            "#,
+        );
+    });
+
+    let mock_root_file = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("im a little teapot"); // 18
+    });
+
+    let mock_dir_disallowed = srv.mock(|when, then| {
+        when.method(GET).path("/disallowed-subdir");
+        then.status(404);
+    });
+
+    let mock_dir_redir = srv.mock(|when, then| {
+        when.method(GET).path("/misc");
+        then.status(301).header("Location", &srv.url("/misc/"));
+    });
+    let mock_dir = srv.mock(|when, then| {
+        when.method(GET).path("/misc/");
+        then.status(200).body(
+            r#"
+            <html>
+            <head>
+            <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+            <title>Directory listing for /misc</title>
+            </head>
+            <body>
+            <h1>Directory listing for /misc</h1>
+            <hr>
+            <ul>
+            <li><a href="LICENSE">LICENSE</a></li>
+            <li><a href="stupidfile.php">stupidfile.php</a></li>
+            </ul>
+            <hr>
+            </body>
+            </html>
+            "#,
+        );
+    });
+
+    let mock_dir_file = srv.mock(|when, then| {
+        when.method(GET).path("/misc/LICENSE");
+        then.status(200).body("i too, am a container for tea"); // 29
+    });
+
+    let mock_dir_file_ext = srv.mock(|when, then| {
+        when.method(GET).path("/misc/stupidfile.php");
+        then.status(200).body("im a little teapot too"); // 22
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--extract-links")
+        .arg("--redirects")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE") // 2 directories contain LICENSE
+            .count(2)
+            .and(predicate::str::contains("18c"))
+            .and(predicate::str::contains("/misc/stupidfile.php"))
+            .and(predicate::str::contains("22c"))
+            .and(predicate::str::contains("/misc/LICENSE"))
+            .and(predicate::str::contains("29c"))
+            .and(predicate::str::contains("200").count(3)),
+    );
+
+    assert_eq!(mock_root.hits(), 2);
+    assert_eq!(mock_root_file.hits(), 1);
+    assert_eq!(mock_dir_disallowed.hits(), 1);
+    assert_eq!(mock_dir_redir.hits(), 1);
+    assert_eq!(mock_dir.hits(), 2);
+    assert_eq!(mock_dir_file.hits(), 1);
+    assert_eq!(mock_dir_file_ext.hits(), 1);
+    teardown_tmp_directory(tmp_dir);
+}
+
+#[test]
+/// serve a directory listing with a file and and a folder contained within it. ferox should
+/// find both links and request each one. This is the non-recursive version of the test above
+fn extractor_finds_directory_listing_links_and_displays_files_non_recursive() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["invalid".to_string()], "wordlist").unwrap();
+
+    let mock_root = srv.mock(|when, then| {
+        when.method(GET).path("/");
+        then.status(200).body(
+            r#"
+            <html>
+            <head>
+            <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+            <title>Directory listing for /</title>
+            </head>
+            <body>
+            <h1>Directory listing for /</h1>
+            <hr>
+            <ul>
+            <li><a href="disallowed-subdir/">disallowed-subdir/</a></li>
+            <li><a href="LICENSE">LICENSE</a></li>
+            <li><a href="misc/">misc/</a></li>
+            </ul>
+            <hr>
+            </body>
+            </html>
+            "#,
+        );
+    });
+
+    let mock_root_file = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("im a little teapot"); // 18
+    });
+
+    let mock_dir_disallowed = srv.mock(|when, then| {
+        when.method(GET).path("/disallowed-subdir");
+        then.status(404);
+    });
+
+    let mock_dir_redir = srv.mock(|when, then| {
+        when.method(GET).path("/misc");
+        then.status(301).header("Location", &srv.url("/misc/"));
+    });
+    let mock_dir = srv.mock(|when, then| {
+        when.method(GET).path("/misc/");
+        then.status(200).body(
+            r#"
+            <html>
+            <head>
+            <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+            <title>Directory listing for /misc</title>
+            </head>
+            <body>
+            <h1>Directory listing for /misc</h1>
+            <hr>
+            <ul>
+            <li><a href="LICENSE">LICENSE</a></li>
+            <li><a href="stupidfile.php">stupidfile.php</a></li>
+            </ul>
+            <hr>
+            </body>
+            </html>
+            "#,
+        );
+    });
+
+    let mock_dir_file = srv.mock(|when, then| {
+        when.method(GET).path("/misc/LICENSE");
+        then.status(200).body("i too, am a container for tea"); // 29
+    });
+
+    let mock_dir_file_ext = srv.mock(|when, then| {
+        when.method(GET).path("/misc/stupidfile.php");
+        then.status(200).body("im a little teapot too"); // 22
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--extract-links")
+        .arg("--redirects")
+        .arg("--no-recursion")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .and(predicate::str::contains("18c"))
+            .and(predicate::str::contains("/misc/stupidfile.php"))
+            .not()
+            .and(predicate::str::contains("22c"))
+            .not()
+            .and(predicate::str::contains("/misc/LICENSE").not())
+            .and(predicate::str::contains("29c").not())
+            .and(predicate::str::contains("200").count(1)),
+    );
+
+    assert_eq!(mock_root.hits(), 2);
+    assert_eq!(mock_root_file.hits(), 1);
+    assert_eq!(mock_dir_disallowed.hits(), 1);
+    assert_eq!(mock_dir_redir.hits(), 1);
+    assert_eq!(mock_dir.hits(), 1);
+    assert_eq!(mock_dir_file.hits(), 0);
+    assert_eq!(mock_dir_file_ext.hits(), 0);
+    teardown_tmp_directory(tmp_dir);
+}
+
 #[test]
 /// send a request to a page that contains a link that contains a directory that returns a 403
 /// --extract-links should find the link and make recurse into the 403 directory, finding LICENSE
@@ -416,7 +636,7 @@ fn extractor_recurses_into_403_directories() -> Result<(), Box<dyn std::error::E
 
     assert_eq!(mock.hits(), 1);
     assert_eq!(mock_two.hits(), 1);
-    assert_eq!(forbidden_dir.hits(), 1);
+    assert_eq!(forbidden_dir.hits(), 2);
     teardown_tmp_directory(tmp_dir);
     Ok(())
 }

tests/test_heuristics.rs

@@ -454,12 +454,12 @@ fn heuristics_wildcard_test_with_redirect_as_response_code(
     assert!(contents.contains("WLD"));
     assert!(contents.contains("301"));
     assert!(contents.contains("/some-redirect"));
-    assert!(contents.contains("redirects to => "));
+    assert!(contents.contains(" => "));
     assert!(contents.contains(&srv.url("/")));
     assert!(contents.contains("(url length: 32)"));
 
     cmd.assert().success().stdout(
-        predicate::str::contains("redirects to => ")
+        predicate::str::contains(" => ")
             .and(predicate::str::contains("/some-redirect"))
             .and(predicate::str::contains("301"))
             .and(predicate::str::contains(srv.url("/")))

tests/test_parser.rs

@@ -33,8 +33,8 @@ fn parser_incorrect_param_with_tack_tack_help() {
 ///
 /// For more information try --help
 ///
-/// the new behavior we expect to see is to print the long form help message, of which
-/// Ludicrous speed... go! is near the bottom of that output, so we can test for that
+/// the new behavior we expect to see is to print the short form help message, of which
+/// "[CAUTION] 4 -v's is probably too much" is near the bottom of that output, so we can test for that
 fn parser_incorrect_param_with_tack_h() {
     Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -42,5 +42,7 @@ fn parser_incorrect_param_with_tack_h() {
         .arg("-h")
         .assert()
         .success()
-        .stdout(predicate::str::contains("Ludicrous speed... go!"));
+        .stdout(predicate::str::contains(
+            "[CAUTION] 4 -v's is probably too much",
+        ));
 }

tests/test_scanner.rs

@@ -363,7 +363,7 @@ fn scanner_single_request_replayed_to_proxy() -> Result<(), Box<dyn std::error::
         .arg("--wordlist")
         .arg(file.as_os_str())
         .arg("--replay-proxy")
-        .arg(format!("http://{}", proxy.address().to_string()))
+        .arg(format!("http://{}", proxy.address()))
         .arg("--replay-codes")
         .arg("200")
         .unwrap();