Merge pull request #869 from epi052/auto-filtering-account-for-extensions

added extensions and status codes into auto filtering decision calculus

.all-contributorsrc

@@ -542,6 +542,35 @@
       "contributions": [
         "ideas"
       ]
+    },
+    {
+      "login": "aancw",
+      "name": "Aan",
+      "avatar_url": "https://avatars.githubusercontent.com/u/6284204?v=4",
+      "profile": "https://petruknisme.com",
+      "contributions": [
+        "code",
+        "infra",
+        "ideas"
+      ]
+    },
+    {
+      "login": "imBigo",
+      "name": "Simon",
+      "avatar_url": "https://avatars.githubusercontent.com/u/54672433?v=4",
+      "profile": "https://github.com/imBigo",
+      "contributions": [
+        "bug"
+      ]
+    },
+    {
+      "login": "acut3",
+      "name": "Nicolas Christin",
+      "avatar_url": "https://avatars.githubusercontent.com/u/17295243?v=4",
+      "profile": "https://acut3.github.io/",
+      "contributions": [
+        "bug"
+      ]
     }
   ],
   "contributorsPerLine": 7,

.gitignore

@@ -30,3 +30,6 @@ ferox-*.state
 
 # python stuff cuz reasons
 Pipfile*
+
+# ignore choco_package generated nupkg
+/choco_package/*.nupkg

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "feroxbuster"
-version = "2.8.0"
+version = "2.9.3"
 authors = ["Ben 'epi' Risher (@epi052)"]
 license = "MIT"
 edition = "2021"
@@ -22,16 +22,16 @@ build = "build.rs"
 maintenance = { status = "actively-developed" }
 
 [build-dependencies]
-clap = { version = "4.1.6", features = ["wrap_help", "cargo"] }
-clap_complete = "4.1.3"
+clap = { version = "4.1.8", features = ["wrap_help", "cargo"] }
+clap_complete = "4.1.4"
 regex = "1.5.5"
 lazy_static = "1.4.0"
-dirs = "4.0.0"
+dirs = "5.0.0"
 
 [dependencies]
-scraper = "0.14.0"
+scraper = "0.16.0"
 futures = "0.3.26"
-tokio = { version = "1.25.0", features = ["full"] }
+tokio = { version = "1.26.0", features = ["full"] }
 tokio-util = { version = "0.7.7", features = ["codec"] }
 log = "0.4.17"
 env_logger = "0.10.0"
@@ -39,16 +39,16 @@ reqwest = { version = "0.11.10", features = ["socks"] }
 # uses feature unification to add 'serde' to reqwest::Url
 url = { version = "2.2.2", features = ["serde"] }
 serde_regex = "1.1.0"
-clap = { version = "4.1.6", features = ["wrap_help", "cargo"] }
+clap = { version = "4.1.8", features = ["wrap_help", "cargo"] }
 lazy_static = "1.4.0"
 toml = "0.7.2"
 serde = { version = "1.0.137", features = ["derive", "rc"] }
-serde_json = "1.0.93"
+serde_json = "1.0.94"
 uuid = { version = "1.3.0", features = ["v4"] }
 indicatif = "0.15"
 console = "0.15.2"
 openssl = { version = "0.10", features = ["vendored"] }
-dirs = "4.0.0"
+dirs = "5.0.0"
 regex = "1.5.5"
 crossterm = "0.26.0"
 rlimit = "0.9.1"
@@ -56,12 +56,18 @@ ctrlc = "3.2.2"
 anyhow = "1.0.69"
 leaky-bucket = "0.12.1"
 gaoya = "0.1.2"
+self_update = { version = "0.36.0", features = [
+    "archive-tar",
+    "compression-flate2",
+    "archive-zip",
+    "compression-zip-deflate",
+] }
 
 [dev-dependencies]
 tempfile = "3.3.0"
 httpmock = "0.6.6"
 assert_cmd = "2.0.4"
-predicates = "2.1.1"
+predicates = "3.0.1"
 
 [profile.release]
 lto = true

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2020 epi
+Copyright (c) 2020-2023 epi
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile.toml

@@ -23,3 +23,10 @@ clear = true
 script = """
 cargo clippy --all-targets --all-features -- -D warnings
 """
+
+# tests
+[tasks.test]
+clear = true
+script = """
+cargo nextest run --all-features --all-targets --retries 10
+"""

README.md

@@ -101,6 +101,11 @@ sudo apt update && sudo apt install -y feroxbuster
 curl -sL https://raw.githubusercontent.com/epi052/feroxbuster/master/install-nix.sh | bash
 ```
 
+#### MacOS via Homebrew 
+
+```
+brew install feroxbuster
+```
 
 #### Windows x86_64
 
@@ -110,10 +115,22 @@ Expand-Archive .\feroxbuster.zip
 .\feroxbuster\feroxbuster.exe -V
 ```
 
+#### Windows via Chocolatey
+
+```
+choco install feroxbuster
+```
+
 #### All others 
 
 Please refer the the [documentation](https://epi052.github.io/feroxbuster-docs/docs/).
 
+### Updating feroxbuster (new in v2.9.1)
+
+```
+./feroxbuster --update
+```
+
 ## 🧰 Example Usage
 
 Here are a few brief examples to get you started.  Please note, feroxbuster can do a **lot more** than what's listed below.  As a result, there are **many more** examples, with **demonstration gifs** that highlight specific features, in the [documentation](https://epi052.github.io/feroxbuster-docs/docs/).
@@ -167,6 +184,8 @@ cat targets | ./feroxbuster --stdin --silent -s 200 301 302 --redirects -x js |
 ./feroxbuster -u http://127.1 --query token=0123456789ABCDEF
 ```
 
+
+
 ## 🚀 Documentation has **moved** 🚀  
 
 For realsies, there used to be over 1300 lines in this README, but it's all been moved to the [new documentation site](https://epi052.github.io/feroxbuster-docs/docs/). Go check it out! 
@@ -257,6 +276,9 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
     <tr>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/xaeroborg"><img src="https://avatars.githubusercontent.com/u/33274680?v=4?s=100" width="100px;" alt="xaeroborg"/><br /><sub><b>xaeroborg</b></sub></a><br /><a href="#ideas-xaeroborg" title="Ideas, Planning, & Feedback">🤔</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/Luoooio"><img src="https://avatars.githubusercontent.com/u/26653157?v=4?s=100" width="100px;" alt="Luoooio"/><br /><sub><b>Luoooio</b></sub></a><br /><a href="#ideas-Luoooio" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://petruknisme.com"><img src="https://avatars.githubusercontent.com/u/6284204?v=4?s=100" width="100px;" alt="Aan"/><br /><sub><b>Aan</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/commits?author=aancw" title="Code">💻</a> <a href="#infra-aancw" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#ideas-aancw" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/imBigo"><img src="https://avatars.githubusercontent.com/u/54672433?v=4?s=100" width="100px;" alt="Simon"/><br /><sub><b>Simon</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3AimBigo" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://acut3.github.io/"><img src="https://avatars.githubusercontent.com/u/17295243?v=4?s=100" width="100px;" alt="Nicolas Christin"/><br /><sub><b>Nicolas Christin</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3Aacut3" title="Bug reports">🐛</a></td>
     </tr>
   </tbody>
 </table>

choco_package/feroxbuster.nuspec

@@ -0,0 +1,79 @@
+<?xml version="1.0" encoding="utf-8"?>
+<package xmlns="http://schemas.microsoft.com/packaging/2015/06/nuspec.xsd">
+  <metadata>
+    <id>feroxbuster</id>
+    <version>2.8.0</version>
+    <packageSourceUrl>https://github.com/epi052/feroxbuster/releases/</packageSourceUrl>
+    <owners>epi052</owners>
+    <title>feroxbuster (Install)</title>
+    <authors>epi052</authors>
+    <projectUrl>https://github.com/epi052/feroxbuster</projectUrl>
+    <iconUrl>https://rawcdn.githack.com/epi052/feroxbuster/2d381e7e057ce60c580b324dd36c9abaf30c2ec7/img/logo/logo.png</iconUrl>
+    <copyright>2020-2023</copyright>
+    <licenseUrl>https://github.com/epi052/feroxbuster/blob/main/LICENSE</licenseUrl>
+    <requireLicenseAcceptance>true</requireLicenseAcceptance>
+    <projectSourceUrl>https://github.com/epi052/feroxbuster</projectSourceUrl>
+    <docsUrl>https://epi052.github.io/feroxbuster-docs/docs/</docsUrl>
+    <!--<mailingListUrl></mailingListUrl>-->
+    <bugTrackerUrl>https://github.com/epi052/feroxbuster/issues</bugTrackerUrl>
+    <tags>content-discovery pentesting-tool url-bruteforcer</tags>
+    <summary>A simple, fast, recursive content discovery tool written in Rust</summary>
+    <description>
+    A simple, fast, recursive content discovery tool written in Rust
+[![Feroxbuster](https://github.com/epi052/feroxbuster/raw/main/img/logo/default-cropped.png)](https://github.com/epi052/feroxbuster)
+
+## What the heck is a ferox anyway?
+
+Ferox is short for Ferric Oxide. Ferric Oxide, simply put, is rust. The name rustbuster was taken, so I decided on a
+variation. 
+
+## What's it do tho?
+
+`feroxbuster` is a tool designed to perform [Forced Browsing](https://owasp.org/www-community/attacks/Forced_browsing).
+
+Forced browsing is an attack where the aim is to enumerate and access resources that are not referenced by the web
+application, but are still accessible by an attacker.
+
+`feroxbuster` uses brute force combined with a wordlist to search for unlinked content in target directories. These
+resources may store sensitive information about web applications and operational systems, such as source code,
+credentials, internal network addressing, etc...
+
+This attack is also known as Predictable Resource Location, File Enumeration, Directory Enumeration, and Resource
+Enumeration.
+
+## Quick Start
+
+This section will cover the minimum amount of information to get up and running with feroxbuster. Please refer the the [documentation](https://epi052.github.io/feroxbuster-docs/docs/), as it's much more comprehensive.
+
+### Installation
+
+There are quite a few other [installation methods](https://epi052.github.io/feroxbuster-docs/docs/installation/), but these snippets should cover the majority of users. 
+
+#### All others Docs
+
+Please refer the the [documentation](https://epi052.github.io/feroxbuster-docs/docs/).
+
+## Example Usage
+
+Here are a few brief examples to get you started.  Please note, feroxbuster can do a **lot more** than what's listed below.  As a result, there are **many more** examples, with **demonstration gifs** that highlight specific features, in the [documentation](https://epi052.github.io/feroxbuster-docs/docs/).
+
+### Multiple Values
+
+Options that take multiple values are very flexible. Consider the following ways of specifying extensions:
+
+```
+./feroxbuster -u http://127.1 -x pdf -x js,html -x php txt json,docx
+```
+
+The command above adds .pdf, .js, .html, .php, .txt, .json, and .docx to each url
+
+All of the methods above (multiple flags, space separated, comma separated, etc...) are valid and interchangeable. The
+same goes for urls, headers, status codes, queries, and size filters.
+</description>
+    <!-- <releaseNotes>__REPLACE_OR_REMOVE__MarkDown_Okay</releaseNotes> -->
+  </metadata>
+  <files>
+    <!-- this section controls what actually gets packaged into the Chocolatey package -->
+    <file src="tools\**" target="tools" />
+  </files>
+</package>

choco_package/legal/LICENSE.txt

@@ -0,0 +1,26 @@
+
+From: https://github.com/epi052/feroxbuster/blob/main/LICENSE
+
+LICENSE
+
+MIT License
+
+Copyright (c) 2020-2023 epi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

choco_package/legal/VERIFICATION.txt

@@ -0,0 +1,5 @@
+
+VERIFICATION
+
+checksum -t sha512 -f .\x86-windows-feroxbuster.exe.zip
+checksum -t sha512 -f .\x86_64-windows-feroxbuster.exe.zip

choco_package/tools/chocolateyinstall.ps1

@@ -0,0 +1,27 @@
+$ErrorActionPreference = 'Stop' 
+
+$toolsDir   = "$(Split-Path -parent $MyInvocation.MyCommand.Definition)"
+$version    = '2.8.0'
+$url        = "https://github.com/epi052/feroxbuster/releases/download/v$version/x86-windows-feroxbuster.exe.zip"
+$url64      = "https://github.com/epi052/feroxbuster/releases/download/v$version/x86_64-windows-feroxbuster.exe.zip"
+
+$packageArgs = @{
+  packageName   = $env:ChocolateyPackageName
+  unzipLocation = $toolsDir
+  fileType      = 'exe' #only one of these: exe, msi, msu
+  url           = $url
+  url64bit      = $url64
+  #file         = $fileLocation
+
+  softwareName  = 'feroxbuster*' 
+
+  # Checksums are now required as of 0.10.0.
+  # To determine checksums, you can get that from the original site if provided.
+  # You can also use checksum.exe (choco install checksum) and use it
+  # e.g. checksum -t sha256 -f path\to\file
+  checksum      = 'e5cac59c737260233903a17706a68bac11fe0d7a15169e1c5a9637cc221e7230fd6ddbfc1a7243833dde6472ad053c033449ca8338164654f7354363da54ba88'
+  checksumType  = 'sha512'
+  checksum64    = 'cce58d6eacef7e12c31076f5a00fee9742a4e3fdfc69d807d98736200e50469f77359978e137ecafd87b14460845c65c6808d1f8b23ae561f7e7c637e355dee3'
+  checksumType64= 'sha512'
+}
+Install-ChocolateyZipPackage @packageArgs # https://docs.chocolatey.org/en-us/create/functions/install-chocolateyzippackage

choco_package/tools/chocolateyuninstall.ps1

@@ -0,0 +1,47 @@
+$ErrorActionPreference = 'Stop' # stop on all errors
+$packageArgs = @{
+  packageName   = $env:ChocolateyPackageName
+  softwareName  = 'feroxbuster*'  #part or all of the Display Name as you see it in Programs and Features. It should be enough to be unique
+  fileType      = 'exe' #only one of these: MSI or EXE (ignore MSU for now)
+}
+
+# Get-UninstallRegistryKey is new to 0.9.10, if supporting 0.9.9.x and below,
+# take a dependency on "chocolatey-core.extension" in your nuspec file.
+# This is only a fuzzy search if $softwareName includes '*'. Otherwise it is
+# exact. In the case of versions in key names, we recommend removing the version
+# and using '*'.
+[array]$key = Get-UninstallRegistryKey -SoftwareName $packageArgs['softwareName']
+
+if ($key.Count -eq 1) {
+  $key | % {
+    $packageArgs['file'] = "$($_.UninstallString)" #NOTE: You may need to split this if it contains spaces, see below
+
+    if ($packageArgs['fileType'] -eq 'MSI') {
+      # The Product Code GUID is all that should be passed for MSI, and very
+      # FIRST, because it comes directly after /x, which is already set in the
+      # Uninstall-ChocolateyPackage msiargs (facepalm).
+      $packageArgs['silentArgs'] = "$($_.PSChildName) $($packageArgs['silentArgs'])"
+
+      # Don't pass anything for file, it is ignored for msi (facepalm number 2)
+      # Alternatively if you need to pass a path to an msi, determine that and
+      # use it instead of the above in silentArgs, still very first
+      $packageArgs['file'] = ''
+    } else {
+      # NOTES:
+      # - You probably will need to sanitize $packageArgs['file'] as it comes from the registry and could be in a variety of fun but unusable formats
+      # - Split args from exe in $packageArgs['file'] and pass those args through $packageArgs['silentArgs'] or ignore them
+      # - Ensure you don't pass double quotes in $file (aka $packageArgs['file']) - otherwise you will get "Illegal characters in path when you attempt to run this"
+      # - Review the code for auto-uninstaller for all of the fun things it does in sanitizing - https://github.com/chocolatey/choco/blob/bfe351b7d10c798014efe4bfbb100b171db25099/src/chocolatey/infrastructure.app/services/AutomaticUninstallerService.cs#L142-L192
+    }
+
+    Uninstall-ChocolateyPackage @packageArgs
+  }
+} elseif ($key.Count -eq 0) {
+  Write-Warning "$packageName has already been uninstalled by other means."
+} elseif ($key.Count -gt 1) {
+  Write-Warning "$($key.Count) matches found!"
+  Write-Warning "To prevent accidental data loss, no programs will be uninstalled."
+  Write-Warning "Please alert package maintainer the following keys were matched:"
+  $key | % {Write-Warning "- $($_.DisplayName)"}
+}
+

shell_completions/_feroxbuster

@@ -24,8 +24,8 @@ _feroxbuster() {
 '--replay-proxy=[Send only unfiltered requests through a Replay Proxy, instead of all requests]:REPLAY_PROXY:_urls' \
 '*-R+[Status Codes to send through a Replay Proxy when found (default: --status-codes value)]:REPLAY_CODE: ' \
 '*--replay-codes=[Status Codes to send through a Replay Proxy when found (default: --status-codes value)]:REPLAY_CODE: ' \
-'-a+[Sets the User-Agent (default: feroxbuster/2.8.0)]:USER_AGENT: ' \
-'--user-agent=[Sets the User-Agent (default: feroxbuster/2.8.0)]:USER_AGENT: ' \
+'-a+[Sets the User-Agent (default: feroxbuster/2.9.3)]:USER_AGENT: ' \
+'--user-agent=[Sets the User-Agent (default: feroxbuster/2.9.3)]:USER_AGENT: ' \
 '*-x+[File extension(s) to search for (ex: -x php -x pdf js)]:FILE_EXTENSION: ' \
 '*--extensions=[File extension(s) to search for (ex: -x php -x pdf js)]:FILE_EXTENSION: ' \
 '*-m+[Which HTTP request method(s) should be sent (default: GET)]:HTTP_METHODS: ' \
@@ -62,8 +62,8 @@ _feroxbuster() {
 '--parallel=[Run parallel feroxbuster instances (one child process per url passed via stdin)]:PARALLEL_SCANS: ' \
 '(--auto-tune)--rate-limit=[Limit number of requests per second (per directory) (default: 0, i.e. no limit)]:RATE_LIMIT: ' \
 '--time-limit=[Limit total run time of all scans (ex: --time-limit 10m)]:TIME_SPEC: ' \
-'-w+[Path to the wordlist]:FILE:_files' \
-'--wordlist=[Path to the wordlist]:FILE:_files' \
+'-w+[Path or URL of the wordlist]:FILE:_files' \
+'--wordlist=[Path or URL of the wordlist]:FILE:_files' \
 '*-I+[File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)]:FILE_EXTENSION: ' \
 '*--dont-collect=[File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)]:FILE_EXTENSION: ' \
 '-o+[Output file to write results to (use w/ --json for JSON entries)]:FILE:_files' \
@@ -72,8 +72,8 @@ _feroxbuster() {
 '(-u --url)--stdin[Read url(s) from STDIN]' \
 '(-p --proxy -k --insecure --burp-replay)--burp[Set --proxy to http://127.0.0.1:8080 and set --insecure to true]' \
 '(-P --replay-proxy -k --insecure)--burp-replay[Set --replay-proxy to http://127.0.0.1:8080 and set --insecure to true]' \
-'--smart[Set --extract-links, --auto-tune, --collect-words, and --collect-backups to true]' \
-'--thorough[Use the same settings as --smart and set --collect-extensions to true]' \
+'(--rate-limit --auto-bail)--smart[Set --auto-tune, --collect-words, and --collect-backups to true]' \
+'(--rate-limit --auto-bail)--thorough[Use the same settings as --smart and set --collect-extensions to true]' \
 '-A[Use a random User-Agent]' \
 '--random-agent[Use a random User-Agent]' \
 '-f[Append / to each request'\''s URL]' \
@@ -85,8 +85,9 @@ _feroxbuster() {
 '-n[Do not scan recursively]' \
 '--no-recursion[Do not scan recursively]' \
 '(-n --no-recursion)--force-recursion[Force recursion attempts on all '\''found'\'' endpoints (still respects recursion depth)]' \
-'-e[Extract links from response body (html, javascript, etc...); make new requests based on findings]' \
-'--extract-links[Extract links from response body (html, javascript, etc...); make new requests based on findings]' \
+'-e[Extract links from response body (html, javascript, etc...); make new requests based on findings (default: true)]' \
+'--extract-links[Extract links from response body (html, javascript, etc...); make new requests based on findings (default: true)]' \
+'--dont-extract-links[Don'\''t extract links from response body (html, javascript, etc...)]' \
 '(--auto-bail)--auto-tune[Automatically lower scan rate when an excessive amount of errors are encountered]' \
 '--auto-bail[Automatically stop scanning when an excessive amount of errors are encountered]' \
 '-D[Don'\''t auto-filter wildcard responses]' \
@@ -104,6 +105,8 @@ _feroxbuster() {
 '--quiet[Hide progress bars and banner (good for tmux windows w/ notifications)]' \
 '--json[Emit JSON logs to --output and --debug-log instead of normal text]' \
 '--no-state[Disable state output file (*.state)]' \
+'-U[Update feroxbuster to the latest version]' \
+'--update[Update feroxbuster to the latest version]' \
 '-h[Print help (see more with '\''--help'\'')]' \
 '--help[Print help (see more with '\''--help'\'')]' \
 '-V[Print version]' \
@@ -117,4 +120,8 @@ _feroxbuster_commands() {
     _describe -t commands 'feroxbuster commands' commands "$@"
 }
 
-_feroxbuster "$@"
+if [ "$funcstack[1]" = "_feroxbuster" ]; then
+    _feroxbuster "$@"
+else
+    compdef _feroxbuster feroxbuster
+fi

shell_completions/_feroxbuster.ps1

@@ -30,8 +30,8 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
             [CompletionResult]::new('--replay-proxy', 'replay-proxy', [CompletionResultType]::ParameterName, 'Send only unfiltered requests through a Replay Proxy, instead of all requests')
             [CompletionResult]::new('-R', 'R', [CompletionResultType]::ParameterName, 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)')
             [CompletionResult]::new('--replay-codes', 'replay-codes', [CompletionResultType]::ParameterName, 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)')
-            [CompletionResult]::new('-a', 'a', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.8.0)')
-            [CompletionResult]::new('--user-agent', 'user-agent', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.8.0)')
+            [CompletionResult]::new('-a', 'a', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.9.3)')
+            [CompletionResult]::new('--user-agent', 'user-agent', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.9.3)')
             [CompletionResult]::new('-x', 'x', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js)')
             [CompletionResult]::new('--extensions', 'extensions', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js)')
             [CompletionResult]::new('-m', 'm', [CompletionResultType]::ParameterName, 'Which HTTP request method(s) should be sent (default: GET)')
@@ -68,8 +68,8 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
             [CompletionResult]::new('--parallel', 'parallel', [CompletionResultType]::ParameterName, 'Run parallel feroxbuster instances (one child process per url passed via stdin)')
             [CompletionResult]::new('--rate-limit', 'rate-limit', [CompletionResultType]::ParameterName, 'Limit number of requests per second (per directory) (default: 0, i.e. no limit)')
             [CompletionResult]::new('--time-limit', 'time-limit', [CompletionResultType]::ParameterName, 'Limit total run time of all scans (ex: --time-limit 10m)')
-            [CompletionResult]::new('-w', 'w', [CompletionResultType]::ParameterName, 'Path to the wordlist')
-            [CompletionResult]::new('--wordlist', 'wordlist', [CompletionResultType]::ParameterName, 'Path to the wordlist')
+            [CompletionResult]::new('-w', 'w', [CompletionResultType]::ParameterName, 'Path or URL of the wordlist')
+            [CompletionResult]::new('--wordlist', 'wordlist', [CompletionResultType]::ParameterName, 'Path or URL of the wordlist')
             [CompletionResult]::new('-I', 'I', [CompletionResultType]::ParameterName, 'File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)')
             [CompletionResult]::new('--dont-collect', 'dont-collect', [CompletionResultType]::ParameterName, 'File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)')
             [CompletionResult]::new('-o', 'o', [CompletionResultType]::ParameterName, 'Output file to write results to (use w/ --json for JSON entries)')
@@ -78,7 +78,7 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
             [CompletionResult]::new('--stdin', 'stdin', [CompletionResultType]::ParameterName, 'Read url(s) from STDIN')
             [CompletionResult]::new('--burp', 'burp', [CompletionResultType]::ParameterName, 'Set --proxy to http://127.0.0.1:8080 and set --insecure to true')
             [CompletionResult]::new('--burp-replay', 'burp-replay', [CompletionResultType]::ParameterName, 'Set --replay-proxy to http://127.0.0.1:8080 and set --insecure to true')
-            [CompletionResult]::new('--smart', 'smart', [CompletionResultType]::ParameterName, 'Set --extract-links, --auto-tune, --collect-words, and --collect-backups to true')
+            [CompletionResult]::new('--smart', 'smart', [CompletionResultType]::ParameterName, 'Set --auto-tune, --collect-words, and --collect-backups to true')
             [CompletionResult]::new('--thorough', 'thorough', [CompletionResultType]::ParameterName, 'Use the same settings as --smart and set --collect-extensions to true')
             [CompletionResult]::new('-A', 'A', [CompletionResultType]::ParameterName, 'Use a random User-Agent')
             [CompletionResult]::new('--random-agent', 'random-agent', [CompletionResultType]::ParameterName, 'Use a random User-Agent')
@@ -91,8 +91,9 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
             [CompletionResult]::new('-n', 'n', [CompletionResultType]::ParameterName, 'Do not scan recursively')
             [CompletionResult]::new('--no-recursion', 'no-recursion', [CompletionResultType]::ParameterName, 'Do not scan recursively')
             [CompletionResult]::new('--force-recursion', 'force-recursion', [CompletionResultType]::ParameterName, 'Force recursion attempts on all ''found'' endpoints (still respects recursion depth)')
-            [CompletionResult]::new('-e', 'e', [CompletionResultType]::ParameterName, 'Extract links from response body (html, javascript, etc...); make new requests based on findings')
-            [CompletionResult]::new('--extract-links', 'extract-links', [CompletionResultType]::ParameterName, 'Extract links from response body (html, javascript, etc...); make new requests based on findings')
+            [CompletionResult]::new('-e', 'e', [CompletionResultType]::ParameterName, 'Extract links from response body (html, javascript, etc...); make new requests based on findings (default: true)')
+            [CompletionResult]::new('--extract-links', 'extract-links', [CompletionResultType]::ParameterName, 'Extract links from response body (html, javascript, etc...); make new requests based on findings (default: true)')
+            [CompletionResult]::new('--dont-extract-links', 'dont-extract-links', [CompletionResultType]::ParameterName, 'Don''t extract links from response body (html, javascript, etc...)')
             [CompletionResult]::new('--auto-tune', 'auto-tune', [CompletionResultType]::ParameterName, 'Automatically lower scan rate when an excessive amount of errors are encountered')
             [CompletionResult]::new('--auto-bail', 'auto-bail', [CompletionResultType]::ParameterName, 'Automatically stop scanning when an excessive amount of errors are encountered')
             [CompletionResult]::new('-D', 'D', [CompletionResultType]::ParameterName, 'Don''t auto-filter wildcard responses')
@@ -110,6 +111,8 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
             [CompletionResult]::new('--quiet', 'quiet', [CompletionResultType]::ParameterName, 'Hide progress bars and banner (good for tmux windows w/ notifications)')
             [CompletionResult]::new('--json', 'json', [CompletionResultType]::ParameterName, 'Emit JSON logs to --output and --debug-log instead of normal text')
             [CompletionResult]::new('--no-state', 'no-state', [CompletionResultType]::ParameterName, 'Disable state output file (*.state)')
+            [CompletionResult]::new('-U', 'U', [CompletionResultType]::ParameterName, 'Update feroxbuster to the latest version')
+            [CompletionResult]::new('--update', 'update', [CompletionResultType]::ParameterName, 'Update feroxbuster to the latest version')
             [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
             [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
             [CompletionResult]::new('-V', 'V', [CompletionResultType]::ParameterName, 'Print version')

shell_completions/feroxbuster.bash

@@ -19,7 +19,7 @@ _feroxbuster() {
 
     case "${cmd}" in
         feroxbuster)
-            opts="-u -p -P -R -a -A -x -m -H -b -Q -f -S -X -W -N -C -s -T -r -k -t -n -d -e -L -w -D -E -B -g -I -v -q -o -h -V --url --stdin --resume-from --burp --burp-replay --smart --thorough --proxy --replay-proxy --replay-codes --user-agent --random-agent --extensions --methods --data --headers --cookies --query --add-slash --dont-scan --filter-size --filter-regex --filter-words --filter-lines --filter-status --filter-similar-to --status-codes --timeout --redirects --insecure --threads --no-recursion --depth --force-recursion --extract-links --scan-limit --parallel --rate-limit --time-limit --wordlist --auto-tune --auto-bail --dont-filter --collect-extensions --collect-backups --collect-words --dont-collect --verbosity --silent --quiet --json --output --debug-log --no-state --help --version"
+            opts="-u -p -P -R -a -A -x -m -H -b -Q -f -S -X -W -N -C -s -T -r -k -t -n -d -e -L -w -D -E -B -g -I -v -q -o -U -h -V --url --stdin --resume-from --burp --burp-replay --smart --thorough --proxy --replay-proxy --replay-codes --user-agent --random-agent --extensions --methods --data --headers --cookies --query --add-slash --dont-scan --filter-size --filter-regex --filter-words --filter-lines --filter-status --filter-similar-to --status-codes --timeout --redirects --insecure --threads --no-recursion --depth --force-recursion --extract-links --dont-extract-links --scan-limit --parallel --rate-limit --time-limit --wordlist --auto-tune --auto-bail --dont-filter --collect-extensions --collect-backups --collect-words --dont-collect --verbosity --silent --quiet --json --output --debug-log --no-state --update --help --version"
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 1 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0

shell_completions/feroxbuster.elv

@@ -27,8 +27,8 @@ set edit:completion:arg-completer[feroxbuster] = {|@words|
             cand --replay-proxy 'Send only unfiltered requests through a Replay Proxy, instead of all requests'
             cand -R 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)'
             cand --replay-codes 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)'
-            cand -a 'Sets the User-Agent (default: feroxbuster/2.8.0)'
-            cand --user-agent 'Sets the User-Agent (default: feroxbuster/2.8.0)'
+            cand -a 'Sets the User-Agent (default: feroxbuster/2.9.3)'
+            cand --user-agent 'Sets the User-Agent (default: feroxbuster/2.9.3)'
             cand -x 'File extension(s) to search for (ex: -x php -x pdf js)'
             cand --extensions 'File extension(s) to search for (ex: -x php -x pdf js)'
             cand -m 'Which HTTP request method(s) should be sent (default: GET)'
@@ -65,8 +65,8 @@ set edit:completion:arg-completer[feroxbuster] = {|@words|
             cand --parallel 'Run parallel feroxbuster instances (one child process per url passed via stdin)'
             cand --rate-limit 'Limit number of requests per second (per directory) (default: 0, i.e. no limit)'
             cand --time-limit 'Limit total run time of all scans (ex: --time-limit 10m)'
-            cand -w 'Path to the wordlist'
-            cand --wordlist 'Path to the wordlist'
+            cand -w 'Path or URL of the wordlist'
+            cand --wordlist 'Path or URL of the wordlist'
             cand -I 'File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)'
             cand --dont-collect 'File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)'
             cand -o 'Output file to write results to (use w/ --json for JSON entries)'
@@ -75,7 +75,7 @@ set edit:completion:arg-completer[feroxbuster] = {|@words|
             cand --stdin 'Read url(s) from STDIN'
             cand --burp 'Set --proxy to http://127.0.0.1:8080 and set --insecure to true'
             cand --burp-replay 'Set --replay-proxy to http://127.0.0.1:8080 and set --insecure to true'
-            cand --smart 'Set --extract-links, --auto-tune, --collect-words, and --collect-backups to true'
+            cand --smart 'Set --auto-tune, --collect-words, and --collect-backups to true'
             cand --thorough 'Use the same settings as --smart and set --collect-extensions to true'
             cand -A 'Use a random User-Agent'
             cand --random-agent 'Use a random User-Agent'
@@ -88,8 +88,9 @@ set edit:completion:arg-completer[feroxbuster] = {|@words|
             cand -n 'Do not scan recursively'
             cand --no-recursion 'Do not scan recursively'
             cand --force-recursion 'Force recursion attempts on all ''found'' endpoints (still respects recursion depth)'
-            cand -e 'Extract links from response body (html, javascript, etc...); make new requests based on findings'
-            cand --extract-links 'Extract links from response body (html, javascript, etc...); make new requests based on findings'
+            cand -e 'Extract links from response body (html, javascript, etc...); make new requests based on findings (default: true)'
+            cand --extract-links 'Extract links from response body (html, javascript, etc...); make new requests based on findings (default: true)'
+            cand --dont-extract-links 'Don''t extract links from response body (html, javascript, etc...)'
             cand --auto-tune 'Automatically lower scan rate when an excessive amount of errors are encountered'
             cand --auto-bail 'Automatically stop scanning when an excessive amount of errors are encountered'
             cand -D 'Don''t auto-filter wildcard responses'
@@ -107,6 +108,8 @@ set edit:completion:arg-completer[feroxbuster] = {|@words|
             cand --quiet 'Hide progress bars and banner (good for tmux windows w/ notifications)'
             cand --json 'Emit JSON logs to --output and --debug-log instead of normal text'
             cand --no-state 'Disable state output file (*.state)'
+            cand -U 'Update feroxbuster to the latest version'
+            cand --update 'Update feroxbuster to the latest version'
             cand -h 'Print help (see more with ''--help'')'
             cand --help 'Print help (see more with ''--help'')'
             cand -V 'Print version'

src/config/container.rs

@@ -1,6 +1,7 @@
 use super::utils::{
-    depth, ignored_extensions, methods, report_and_exit, save_state, serialized_type, status_codes,
-    threads, timeout, user_agent, wordlist, OutputLevel, RequesterPolicy,
+    depth, extract_links, ignored_extensions, methods, report_and_exit, save_state,
+    serialized_type, status_codes, threads, timeout, user_agent, wordlist, OutputLevel,
+    RequesterPolicy,
 };
 use crate::config::determine_output_level;
 use crate::config::utils::determine_requester_policy;
@@ -214,7 +215,7 @@ pub struct Configuration {
     pub no_recursion: bool,
 
     /// Extract links from html/javscript
-    #[serde(default)]
+    #[serde(default = "extract_links")]
     pub extract_links: bool,
 
     /// Append / to each request
@@ -309,6 +310,10 @@ pub struct Configuration {
     /// override recursion logic to always attempt recursion, still respects --depth
     #[serde(default)]
     pub force_recursion: bool,
+
+    /// Auto update app feature
+    #[serde(skip)]
+    pub update_app: bool,
 }
 
 impl Default for Configuration {
@@ -324,6 +329,7 @@ impl Default for Configuration {
         let kind = serialized_type();
         let output_level = OutputLevel::Default;
         let requester_policy = RequesterPolicy::Default;
+        let extract_links = extract_links();
 
         Configuration {
             kind,
@@ -332,6 +338,7 @@ impl Default for Configuration {
             user_agent,
             replay_codes,
             status_codes,
+            extract_links,
             replay_client,
             requester_policy,
             dont_filter: false,
@@ -351,13 +358,13 @@ impl Default for Configuration {
             insecure: false,
             redirects: false,
             no_recursion: false,
-            extract_links: false,
             random_agent: false,
             collect_extensions: false,
             collect_backups: false,
             collect_words: false,
             save_state: true,
             force_recursion: false,
+            update_app: false,
             proxy: String::new(),
             config: String::new(),
             output: String::new(),
@@ -393,7 +400,7 @@ impl Configuration {
     ///
     /// - **timeout**: `5` seconds
     /// - **redirects**: `false`
-    /// - **extract-links**: `false`
+    /// - **extract_links**: `true`
     /// - **wordlist**: [`DEFAULT_WORDLIST`](constant.DEFAULT_WORDLIST.html)
     /// - **config**: `None`
     /// - **threads**: `50`
@@ -441,6 +448,7 @@ impl Configuration {
     /// - **time_limit**: `None` (no limit on length of scan imposed)
     /// - **replay_proxy**: `None` (no limit on concurrent scans imposed)
     /// - **replay_codes**: [`DEFAULT_RESPONSE_CODES`](constant.DEFAULT_RESPONSE_CODES.html)
+    /// - **update_app**: `false`
     ///
     /// After which, any values defined in a
     /// [ferox-config.toml](constant.DEFAULT_CONFIG_NAME.html) config file will override the
@@ -801,11 +809,8 @@ impl Configuration {
             config.add_slash = true;
         }
 
-        if came_from_cli!(args, "extract_links")
-            || came_from_cli!(args, "smart")
-            || came_from_cli!(args, "thorough")
-        {
-            config.extract_links = true;
+        if came_from_cli!(args, "dont_extract_links") {
+            config.extract_links = false;
         }
 
         if came_from_cli!(args, "json") {
@@ -816,6 +821,10 @@ impl Configuration {
             config.force_recursion = true;
         }
 
+        if came_from_cli!(args, "update_app") {
+            config.update_app = true;
+        }
+
         ////
         // organizational breakpoint; all options below alter the Client configuration
         ////
@@ -987,11 +996,12 @@ impl Configuration {
         update_if_not_default!(&mut conf.redirects, new.redirects, false);
         update_if_not_default!(&mut conf.insecure, new.insecure, false);
         update_if_not_default!(&mut conf.force_recursion, new.force_recursion, false);
-        update_if_not_default!(&mut conf.extract_links, new.extract_links, false);
+        update_if_not_default!(&mut conf.extract_links, new.extract_links, extract_links());
         update_if_not_default!(&mut conf.extensions, new.extensions, Vec::<String>::new());
         update_if_not_default!(&mut conf.methods, new.methods, methods());
         update_if_not_default!(&mut conf.data, new.data, Vec::<u8>::new());
         update_if_not_default!(&mut conf.url_denylist, new.url_denylist, Vec::<Url>::new());
+        update_if_not_default!(&mut conf.update_app, new.update_app, false);
         if !new.regex_denylist.is_empty() {
             // cant use the update_if_not_default macro due to the following error
             //

src/config/tests.rs

@@ -45,7 +45,7 @@ fn setup_config_test() -> Configuration {
             add_slash = true
             stdin = true
             dont_filter = true
-            extract_links = true
+            extract_links = false
             json = true
             save_state = false
             depth = 1
@@ -98,7 +98,7 @@ fn default_configuration() {
     assert!(!config.add_slash);
     assert!(!config.force_recursion);
     assert!(!config.redirects);
-    assert!(!config.extract_links);
+    assert!(config.extract_links);
     assert!(!config.insecure);
     assert!(!config.collect_extensions);
     assert!(!config.collect_backups);
@@ -305,7 +305,7 @@ fn config_reads_add_slash() {
 /// parse the test config and see that the value parsed is correct
 fn config_reads_extract_links() {
     let config = setup_config_test();
-    assert!(config.extract_links);
+    assert!(!config.extract_links);
 }
 
 #[test]

src/config/utils.rs

@@ -84,6 +84,11 @@ pub(super) fn depth() -> usize {
     4
 }
 
+/// default extract links
+pub(super) fn extract_links() -> bool {
+    true
+}
+
 /// enum representing the three possible states for informational output (not logging verbosity)
 #[derive(Debug, Copy, Clone, PartialEq, Eq)]
 pub enum OutputLevel {

src/event_handlers/container.rs

@@ -160,13 +160,17 @@ impl Handles {
     /// number of extensions plus the number of request method types plus any dynamically collected
     /// extensions
     pub fn expected_num_requests_multiplier(&self) -> usize {
-        let multiplier = self.config.extensions.len()
-            + self.config.methods.len()
-            + self.num_collected_extensions();
+        let mut multiplier = self.config.extensions.len().max(1);
 
-        // methods should always have at least 1 member, likely making this .max call unneeded
-        // but leaving it for 'just in case' reasons
-        multiplier.max(1)
+        if multiplier > 1 {
+            // when we have more than one extension, we need to account for the fact that we'll
+            // be making a request for each extension and the base word (e.g. /foo.html and /foo)
+            multiplier += 1;
+        }
+
+        multiplier *= self.config.methods.len().max(1) * self.num_collected_extensions().max(1);
+
+        multiplier
     }
 
     /// Helper to easily get the (locked) underlying FeroxScans object

src/event_handlers/outputs.rs

@@ -248,7 +248,7 @@ impl TermOutHandler {
                 .unwrap()
                 .filters
                 .data
-                .should_filter_response(&resp);
+                .should_filter_response(&resp, tx_stats.clone());
 
             let contains_sentry = if !self.config.filter_status.is_empty() {
                 // -C was used, meaning -s was not and we should ignore the defaults

src/event_handlers/scans.rs

@@ -6,7 +6,7 @@ use tokio::sync::{mpsc, Semaphore};
 use crate::{
     response::FeroxResponse,
     scan_manager::{FeroxScan, FeroxScans, ScanOrder},
-    scanner::FeroxScanner,
+    scanner::{FeroxScanner, RESPONSES},
     statistics::StatField::TotalScans,
     url::FeroxUrl,
     utils::should_deny_url,
@@ -222,7 +222,7 @@ impl ScanHandler {
         let current_expectation = self.handles.expected_num_requests_per_dir() as u64;
 
         // used in the calculation of bar width down below, see explanation there
-        let divisor = self.handles.expected_num_requests_multiplier() as u64 - 1;
+        let divisor = (self.handles.expected_num_requests_multiplier() as u64 - 1).max(1);
 
         // add another `wordlist.len` to the expected per scan tracker in the statistics handler
         self.handles
@@ -294,12 +294,7 @@ impl ScanHandler {
         if let Ok(guard) = self.wordlist.lock().as_ref() {
             if let Some(list) = guard.as_ref() {
                 return if offset > 0 {
-                    // the offset could be off a bit, so we'll adjust it backwards by 10%
-                    // of the overall wordlist size to ensure we don't miss any words
-                    // (hopefully)
-                    let adjusted_offset = offset - ((offset as f64 * 0.10) as usize);
-
-                    Ok(Arc::new(list[adjusted_offset..].to_vec()))
+                    Ok(Arc::new(list[offset..].to_vec()))
                 } else {
                     Ok(list.clone())
                 };
@@ -337,7 +332,18 @@ impl ScanHandler {
                 continue;
             }
 
-            let list = self.get_wordlist(scan.requests() as usize)?;
+            let divisor = self.handles.expected_num_requests_multiplier();
+
+            let list = if divisor > 1 && scan.requests() > 0 {
+                // if there were extensions provided and/or more than a single method used, and some
+                // number of requests have already been sent, we need to adjust the offset into the
+                // wordlist to ensure we don't index out of bounds
+
+                let adjusted = scan.requests_made_so_far() as f64 / (divisor as f64 - 1.0).max(1.0);
+                self.get_wordlist(adjusted as usize)?
+            } else {
+                self.get_wordlist(scan.requests_made_so_far() as usize)?
+            };
 
             log::info!("scan handler received {} - beginning scan", target);
 
@@ -395,6 +401,58 @@ impl ScanHandler {
             return Ok(());
         }
 
+        if let Ok(responses) = RESPONSES.responses.read() {
+            for maybe_wild in responses.iter() {
+                if !maybe_wild.wildcard() || !maybe_wild.is_directory() {
+                    // if the stored response isn't a wildcard, skip it
+                    // if the stored response isn't a directory, skip it
+                    // we're only interested in preventing recursion into wildcard directories
+                    continue;
+                }
+
+                if maybe_wild.method() != response.method() {
+                    // methods don't match, skip it
+                    continue;
+                }
+
+                // methods match and is a directory wildcard
+                // need to check the wildcard's parent directory
+                // for equality with the incoming response's parent directory
+                //
+                // if the parent directories match, we need to prevent recursion
+                // into the wildcard directory
+
+                match (
+                    maybe_wild.url().path_segments(),
+                    response.url().path_segments(),
+                ) {
+                    // both urls must have path segments
+                    (Some(mut maybe_wild_segments), Some(mut response_segments)) => {
+                        match (
+                            maybe_wild_segments.nth_back(1),
+                            response_segments.nth_back(1),
+                        ) {
+                            // both urls must have at least 2 path segments, the next to last being the parent
+                            (Some(maybe_wild_parent), Some(response_parent)) => {
+                                if maybe_wild_parent == response_parent {
+                                    // the parent directories match, so we need to prevent recursion
+                                    return Ok(());
+                                }
+                            }
+                            _ => {
+                                // we couldn't get the parent directory, so we'll skip this
+                                continue;
+                            }
+                        }
+                    }
+                    _ => {
+                        // we couldn't get the path segments, so we'll skip this
+                        continue;
+                    }
+                }
+            }
+        }
+
         let targets = vec![response.url().to_string()];
         self.ordered_scan_url(targets, ScanOrder::Latest).await?;
 

src/event_handlers/statistics.rs

@@ -148,7 +148,12 @@ impl StatsHandler {
         );
 
         self.bar.set_message(&msg);
-        self.bar.inc(1);
+
+        if self.bar.position() < self.stats.total_expected() as u64 {
+            // don't run off the end when we're a few requests over the expected total
+            // due to the heuristics tests
+            self.bar.inc(1);
+        }
     }
 
     /// Initialize new `Stats` object and the sc side of an mpsc channel that is responsible for

src/extractor/container.rs

@@ -144,7 +144,12 @@ impl<'a> Extractor<'a> {
             };
 
             // filter if necessary
-            if self.handles.filters.data.should_filter_response(&resp) {
+            if self
+                .handles
+                .filters
+                .data
+                .should_filter_response(&resp, self.handles.stats.tx.clone())
+            {
                 continue;
             }
 

src/filters/container.rs

@@ -7,9 +7,12 @@ use crate::response::FeroxResponse;
 
 use super::{
     FeroxFilter, LinesFilter, RegexFilter, SimilarityFilter, SizeFilter, StatusCodeFilter,
-    WordsFilter,
+    WildcardFilter, WordsFilter,
+};
+use crate::{
+    event_handlers::Command::AddToUsizeField, statistics::StatField::WildcardsFiltered,
+    CommandSender,
 };
-
 /// Container around a collection of `FeroxFilters`s
 #[derive(Debug, Default)]
 pub struct FeroxFilters {
@@ -64,12 +67,21 @@ impl FeroxFilters {
 
     /// Simple helper to stay DRY; determines whether or not a given `FeroxResponse` should be reported
     /// to the user or not.
-    pub fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+    pub fn should_filter_response(
+        &self,
+        response: &FeroxResponse,
+        tx_stats: CommandSender,
+    ) -> bool {
         if let Ok(filters) = self.filters.read() {
             for filter in filters.iter() {
                 // wildcard.should_filter goes here
                 if filter.should_filter_response(response) {
                     log::debug!("filtering response due to: {:?}", filter);
+                    if filter.as_any().downcast_ref::<WildcardFilter>().is_some() {
+                        tx_stats
+                            .send(AddToUsizeField(WildcardsFiltered, 1))
+                            .unwrap_or_default();
+                    }
                     return true;
                 }
             }
@@ -93,6 +105,10 @@ impl Serialize for FeroxFilters {
                     seq.serialize_element(word_filter).unwrap_or_default();
                 } else if let Some(size_filter) = filter.as_any().downcast_ref::<SizeFilter>() {
                     seq.serialize_element(size_filter).unwrap_or_default();
+                } else if let Some(wildcard_filter) =
+                    filter.as_any().downcast_ref::<WildcardFilter>()
+                {
+                    seq.serialize_element(wildcard_filter).unwrap_or_default();
                 } else if let Some(status_filter) =
                     filter.as_any().downcast_ref::<StatusCodeFilter>()
                 {

src/filters/mod.rs

@@ -15,6 +15,7 @@ pub use self::similarity::{SimilarityFilter, SIM_HASHER};
 pub use self::size::SizeFilter;
 pub use self::status_code::StatusCodeFilter;
 pub(crate) use self::utils::{create_similarity_filter, filter_lookup};
+pub use self::wildcard::WildcardFilter;
 pub use self::words::WordsFilter;
 
 mod status_code;
@@ -28,4 +29,5 @@ mod container;
 mod tests;
 mod init;
 mod utils;
+mod wildcard;
 mod empty;

src/filters/similarity.rs

@@ -37,7 +37,9 @@ impl FeroxFilter for SimilarityFilter {
 
     /// Compare one SimilarityFilter to another
     fn box_eq(&self, other: &dyn Any) -> bool {
-        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+        other
+            .downcast_ref::<Self>()
+            .map_or(false, |a| self.hash == a.hash)
     }
 
     /// Return self as Any for dynamic dispatch purposes

src/filters/tests.rs

@@ -1,7 +1,41 @@
 use super::*;
 use crate::nlp::preprocess;
+use crate::DEFAULT_METHOD;
 use ::regex::Regex;
 
+#[test]
+/// simply test the default values for wildcardfilter
+fn wildcard_filter_default() {
+    let wcf = WildcardFilter::default();
+    assert_eq!(wcf.content_length, None);
+    assert_eq!(wcf.line_count, None);
+    assert_eq!(wcf.word_count, None);
+    assert_eq!(wcf.method, DEFAULT_METHOD.to_string());
+    assert_eq!(wcf.status_code, 0);
+    assert!(!wcf.dont_filter);
+}
+
+#[test]
+/// just a simple test to increase code coverage by hitting as_any and the inner value
+fn wildcard_filter_as_any() {
+    let mut filter = WildcardFilter::default();
+    let filter2 = WildcardFilter::default();
+
+    assert!(filter.box_eq(filter2.as_any()));
+
+    assert_eq!(
+        *filter.as_any().downcast_ref::<WildcardFilter>().unwrap(),
+        filter2
+    );
+
+    filter.content_length = Some(1);
+
+    assert_ne!(
+        *filter.as_any().downcast_ref::<WildcardFilter>().unwrap(),
+        filter2
+    );
+}
+
 #[test]
 /// just a simple test to increase code coverage by hitting as_any and the inner value
 fn lines_filter_as_any() {
@@ -86,6 +120,68 @@ fn regex_filter_as_any() {
     );
 }
 
+#[test]
+/// test should_filter on WilcardFilter where static logic matches
+fn wildcard_should_filter_when_static_wildcard_found() {
+    let body =
+        "pellentesque diam volutpat commodo sed egestas egestas fringilla phasellus faucibus";
+
+    let mut resp = FeroxResponse::default();
+    resp.set_wildcard(true);
+    resp.set_url("http://localhost");
+    resp.set_text(body);
+
+    let filter = WildcardFilter {
+        content_length: Some(body.len() as u64),
+        line_count: Some(1),
+        word_count: Some(10),
+        method: DEFAULT_METHOD.to_string(),
+        status_code: 200,
+        dont_filter: false,
+    };
+
+    assert!(filter.should_filter_response(&resp));
+}
+
+#[test]
+/// test should_filter on WilcardFilter where static logic matches but response length is 0
+fn wildcard_should_filter_when_static_wildcard_len_is_zero() {
+    let mut resp = FeroxResponse::default();
+    resp.set_wildcard(true);
+    resp.set_url("http://localhost");
+
+    // default WildcardFilter is used in the code that executes when response.content_length() == 0
+    let filter = WildcardFilter {
+        content_length: Some(0),
+        line_count: Some(0),
+        word_count: Some(0),
+        method: DEFAULT_METHOD.to_string(),
+        status_code: 200,
+        dont_filter: false,
+    };
+
+    assert!(filter.should_filter_response(&resp));
+}
+
+#[test]
+/// test should_filter on WilcardFilter where dynamic logic matches
+fn wildcard_should_filter_when_dynamic_wildcard_found() {
+    let mut resp = FeroxResponse::default();
+    resp.set_wildcard(true);
+    resp.set_url("http://localhost/stuff");
+    resp.set_text("pellentesque diam volutpat commodo sed egestas egestas fringilla");
+
+    let filter = WildcardFilter {
+        content_length: None,
+        line_count: None,
+        word_count: Some(8),
+        method: DEFAULT_METHOD.to_string(),
+        status_code: 200,
+        dont_filter: false,
+    };
+
+    assert!(filter.should_filter_response(&resp));
+}
 #[test]
 /// test should_filter on RegexFilter where regex matches body
 fn regexfilter_should_filter_when_regex_matches_on_response_body() {

src/filters/wildcard.rs

@@ -0,0 +1,180 @@
+use console::style;
+
+use super::*;
+use crate::utils::create_report_string;
+use crate::{config::OutputLevel, DEFAULT_METHOD};
+
+/// Data holder for all relevant data needed when auto-filtering out wildcard responses
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct WildcardFilter {
+    /// The content-length of this response, if known
+    pub content_length: Option<u64>,
+
+    /// The number of lines contained in the body of this response, if known
+    pub line_count: Option<usize>,
+
+    /// The number of words contained in the body of this response, if known
+    pub word_count: Option<usize>,
+
+    /// method used in request that should be included with filters passed via runtime configuration
+    pub method: String,
+
+    /// the status code returned in the response
+    pub status_code: u16,
+
+    /// whether or not the user passed -D on the command line
+    pub dont_filter: bool,
+}
+
+/// implementation of WildcardFilter
+impl WildcardFilter {
+    /// given a boolean representing whether -D was used or not, create a new WildcardFilter
+    pub fn new(dont_filter: bool) -> Self {
+        Self {
+            dont_filter,
+            ..Default::default()
+        }
+    }
+}
+
+/// implement default that populates `method` with its default value
+impl Default for WildcardFilter {
+    fn default() -> Self {
+        Self {
+            content_length: None,
+            line_count: None,
+            word_count: None,
+            method: DEFAULT_METHOD.to_string(),
+            status_code: 0,
+            dont_filter: false,
+        }
+    }
+}
+
+/// implementation of FeroxFilter for WildcardFilter
+impl FeroxFilter for WildcardFilter {
+    /// Examine size/words/lines and method to determine whether or not the response received
+    /// is a wildcard response and therefore should be filtered out
+    fn should_filter_response(&self, response: &FeroxResponse) -> bool {
+        log::trace!("enter: should_filter_response({:?} {})", self, response);
+
+        // quick return if dont_filter is set
+        if self.dont_filter {
+            // --dont-filter applies specifically to wildcard filters, it is not a 100% catch all
+            // for not filtering anything.  As such, it should live in the implementation of
+            // a wildcard filter
+            return false;
+        }
+
+        if self.method != response.method().as_str() {
+            // method's don't match, so this response should not be filtered out
+            log::trace!("exit: should_filter_response -> false");
+            return false;
+        }
+
+        if self.status_code != response.status().as_u16() {
+            // status codes don't match, so this response should not be filtered out
+            log::trace!("exit: should_filter_response -> false");
+            return false;
+        }
+
+        // methods and status codes match at this point, just need to check the other fields
+
+        match (self.content_length, self.word_count, self.line_count) {
+            (Some(cl), Some(wc), Some(lc)) => {
+                if cl == response.content_length()
+                    && wc == response.word_count()
+                    && lc == response.line_count()
+                {
+                    log::debug!("filtered out {}", response.url());
+                    log::trace!("exit: should_filter_response -> true");
+                    return true;
+                }
+            }
+            (Some(cl), Some(wc), None) => {
+                if cl == response.content_length() && wc == response.word_count() {
+                    log::debug!("filtered out {}", response.url());
+                    log::trace!("exit: should_filter_response -> true");
+                    return true;
+                }
+            }
+            (Some(cl), None, Some(lc)) => {
+                if cl == response.content_length() && lc == response.line_count() {
+                    log::debug!("filtered out {}", response.url());
+                    log::trace!("exit: should_filter_response -> true");
+                    return true;
+                }
+            }
+            (None, Some(wc), Some(lc)) => {
+                if wc == response.word_count() && lc == response.line_count() {
+                    log::debug!("filtered out {}", response.url());
+                    log::trace!("exit: should_filter_response -> true");
+                    return true;
+                }
+            }
+            (Some(cl), None, None) => {
+                if cl == response.content_length() {
+                    log::debug!("filtered out {}", response.url());
+                    log::trace!("exit: should_filter_response -> true");
+                    return true;
+                }
+            }
+            (None, Some(wc), None) => {
+                if wc == response.word_count() {
+                    log::debug!("filtered out {}", response.url());
+                    log::trace!("exit: should_filter_response -> true");
+                    return true;
+                }
+            }
+            (None, None, Some(lc)) => {
+                if lc == response.line_count() {
+                    log::debug!("filtered out {}", response.url());
+                    log::trace!("exit: should_filter_response -> true");
+                    return true;
+                }
+            }
+            (None, None, None) => {
+                unreachable!("wildcard filter without any filters set");
+            }
+        }
+
+        log::trace!("exit: should_filter_response -> false");
+        false
+    }
+
+    /// Compare one WildcardFilter to another
+    fn box_eq(&self, other: &dyn Any) -> bool {
+        other.downcast_ref::<Self>().map_or(false, |a| self == a)
+    }
+
+    /// Return self as Any for dynamic dispatch purposes
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+}
+
+impl std::fmt::Display for WildcardFilter {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let msg = create_report_string(
+            self.status_code.to_string().as_str(),
+            self.method.as_str(),
+            &self
+                .line_count
+                .map_or_else(|| "-".to_string(), |x| x.to_string()),
+            &self
+                .word_count
+                .map_or_else(|| "-".to_string(), |x| x.to_string()),
+            &self
+                .content_length
+                .map_or_else(|| "-".to_string(), |x| x.to_string()),
+            &format!(
+                "{} found {}-like response and created new filter; toggle off with {}",
+                style("Auto-filtering").bright().green(),
+                style("404").red(),
+                style("--dont-filter").yellow()
+            ),
+            OutputLevel::Default,
+        );
+        write!(f, "{}", msg)
+    }
+}

src/heuristics.rs

@@ -1,22 +1,22 @@
+use std::collections::HashMap;
 use std::sync::Arc;
 
 use anyhow::{bail, Result};
-use console::style;
 use scraper::{Html, Selector};
 use uuid::Uuid;
 
-use crate::filters::{SimilarityFilter, SIM_HASHER};
+use crate::filters::{SimilarityFilter, WildcardFilter, SIM_HASHER};
 use crate::message::FeroxMessage;
 use crate::nlp::preprocess;
+use crate::scanner::RESPONSES;
 use crate::{
     config::OutputLevel,
     event_handlers::{Command, Handles},
-    filters::{LinesFilter, SizeFilter, WordsFilter},
     progress::PROGRESS_PRINTER,
     response::FeroxResponse,
     skip_fail,
     url::FeroxUrl,
-    utils::{ferox_print, fmt_err, logged_request, status_colorizer},
+    utils::{ferox_print, fmt_err, logged_request},
     DEFAULT_METHOD,
 };
 
@@ -50,6 +50,16 @@ pub struct DirListingResult {
     pub response: FeroxResponse,
 }
 
+/// wrapper around the results of running a wildcard detection against a target web page
+#[derive(Copy, Debug, Clone)]
+pub enum WildcardResult {
+    /// variant that represents a wildcard directory
+    WildcardDirectory(usize),
+
+    /// variant that represents the presence of a 404-like response
+    FourOhFourLike(usize),
+}
+
 /// container for heuristics related info
 pub struct HeuristicTests {
     /// Handles object for event handler interaction
@@ -240,13 +250,16 @@ impl HeuristicTests {
     /// given a target's base url, attempt to automatically detect its 404 response
     /// pattern(s), and then set filters that will exclude those patterns from future
     /// responses
-    pub async fn detect_404_like_responses(&self, target_url: &str) -> Result<u64> {
+    pub async fn detect_404_like_responses(
+        &self,
+        target_url: &str,
+    ) -> Result<Option<WildcardResult>> {
         log::trace!("enter: detect_404_like_responses({:?})", target_url);
 
         if self.handles.config.dont_filter {
             // early return, dont_filter scans don't need tested
             log::trace!("exit: detect_404_like_responses -> dont_filter is true");
-            return Ok(0);
+            return Ok(None);
         }
 
         let mut req_counter = 0;
@@ -264,183 +277,323 @@ impl HeuristicTests {
             None
         };
 
-        // 4 is due to the array in the nested for loop below
-        let mut responses = Vec::with_capacity(4);
+        // 6 is due to the array in the nested for loop below
+        let mut responses = Vec::with_capacity(6);
+
+        // no matter what, we want an empty extension for the base case
+        let mut extensions = vec!["".to_string()];
+
+        // and then we want to add any extensions that was specified
+        // or has since been added to the running config
+        for ext in &self.handles.config.extensions {
+            extensions.push(format!(".{}", ext));
+        }
 
         // for every method, attempt to id its 404 response
         //
         // a good example of one where the GET/POST differ is on hackthebox:
         // - http://prd.m.rendering-api.interface.htb/api
+        //
+        // a good example of one where the heuristics return a 403 and a 404 (apache)
+        // as well as return two different types of 404s based on the file extension
+        // - http://10.10.11.198 (Encoding box in normal labs)
+        //
+        // both methods and extensions can elicit different responses from a given
+        // server, so both are considered when building auto-filter rules
         for method in self.handles.config.methods.iter() {
-            for (prefix, length) in [("", 1), ("", 3), (".htaccess", 1), ("admin", 1)] {
-                let path = format!("{prefix}{}", self.unique_string(length));
+            for extension in extensions.iter() {
+                for (prefix, length) in [
+                    ("", 1),
+                    ("", 3),
+                    (".htaccess", 1),
+                    (".htaccess", 3),
+                    ("admin", 1),
+                    ("admin", 3),
+                ] {
+                    let path = format!("{prefix}{}{extension}", self.unique_string(length));
 
-                let ferox_url = FeroxUrl::from_string(target_url, self.handles.clone());
+                    let ferox_url = FeroxUrl::from_string(target_url, self.handles.clone());
 
-                let nonexistent_url = ferox_url.format(&path, slash)?;
+                    let nonexistent_url = ferox_url.format(&path, slash)?;
 
-                // example requests:
-                // - http://localhost/2fc1077836ad43ab98b7a31c2ca28fea
-                // - http://localhost/92969beae6bf4beb855d1622406d87e395c87387a9ad432e8a11245002b709b03cf609d471004154b83bcc1c6ec49f6f
-                // - http://localhost/.htaccessa005a2131e68449aa26e99029c914c09
-                // - http://localhost/adminf1d2541e73c44dcb9d1fb7d93334b280
-                let response =
-                    logged_request(&nonexistent_url, method, data, self.handles.clone()).await;
+                    // example requests:
+                    // - http://localhost/2fc1077836ad43ab98b7a31c2ca28fea
+                    // - http://localhost/92969beae6bf4beb855d1622406d87e395c87387a9ad432e8a11245002b709b03cf609d471004154b83bcc1c6ec49f6f
+                    // - http://localhost/.htaccessa005a2131e68449aa26e99029c914c09
+                    // - http://localhost/.htaccess92969beae6bf4beb855d1622406d87e395c87387a9ad432e8a11245002b709b03cf609d471004154b83bcc1c6ec49f6f
+                    // - http://localhost/adminf1d2541e73c44dcb9d1fb7d93334b280
+                    // - http://localhost/admin92969beae6bf4beb855d1622406d87e395c87387a9ad432e8a11245002b709b03cf609d471004154b83bcc1c6ec49f6f
+                    let response =
+                        logged_request(&nonexistent_url, method, data, self.handles.clone()).await;
 
-                req_counter += 1;
+                    req_counter += 1;
 
-                // continue to next on error
-                let response = skip_fail!(response);
+                    // continue to next on error
+                    let response = skip_fail!(response);
 
-                if !self
-                    .handles
-                    .config
-                    .status_codes
-                    .contains(&response.status().as_u16())
-                {
-                    // if the response code isn't one that's accepted via -s values, then skip to the next
-                    //
-                    // the default value for -s is all status codes, so unless the user says otherwise
-                    // this won't fire
+                    if !self
+                        .handles
+                        .config
+                        .status_codes
+                        .contains(&response.status().as_u16())
+                    {
+                        // if the response code isn't one that's accepted via -s values, then skip to the next
+                        //
+                        // the default value for -s is all status codes, so unless the user says otherwise
+                        // this won't fire
+                        continue;
+                    }
+
+                    let ferox_response = FeroxResponse::from(
+                        response,
+                        &ferox_url.target,
+                        method,
+                        self.handles.config.output_level,
+                    )
+                    .await;
+
+                    responses.push(ferox_response);
+                }
+
+                if responses.len() < 2 {
+                    // don't have enough responses to make a determination, continue to next method
+                    responses.clear();
                     continue;
                 }
 
-                let ferox_response = FeroxResponse::from(
-                    response,
-                    &ferox_url.target,
-                    method,
+                // check the responses for similarities on which we can filter, multiple may be returned
+                let Some((wildcard_filters, wildcard_responses)) = self.examine_404_like_responses(&responses) else {
+                    // no match was found during analysis of responses
+                    responses.clear();
+                    log::warn!("no match found for 404 responses");
+                    continue;
+                };
+
+                // report to the user, if appropriate
+                if matches!(
                     self.handles.config.output_level,
-                )
-                .await;
+                    OutputLevel::Default | OutputLevel::Quiet
+                ) {
+                    // sentry value to control whether or not to print the filter
+                    // used because we only want to print the same filter once
+                    let mut print_sentry;
 
-                responses.push(ferox_response);
-            }
+                    if let Ok(filters) = self.handles.filters.data.filters.read() {
+                        for new_wildcard in &wildcard_filters {
+                            // reset the sentry for every new wildcard produced by examine_404_like_responses
+                            print_sentry = true;
 
-            if responses.len() < 2 {
-                // don't have enough responses to make a determination, continue to next method
-                responses.clear();
-                continue;
-            }
+                            for other in filters.iter() {
+                                if let Some(other_wildcard) =
+                                    other.as_any().downcast_ref::<WildcardFilter>()
+                                {
+                                    // check the new wildcard against all existing wildcards, if it was added
+                                    // on the cli or by a previous directory, don't print it
+                                    if new_wildcard.as_ref() == other_wildcard {
+                                        print_sentry = false;
+                                        break;
+                                    }
+                                }
+                            }
 
-            // Command::AddFilter, &str (bytes/words/lines), usize (i.e. length associated with the type)
-            let Some((command, filter_type, filter_length)) = self.examine_404_like_responses(&responses) else {
-                // no match was found during analysis of responses
-                responses.clear();
-                continue;
-            };
-
-            // check whether we already know about this filter
-            match command {
-                Command::AddFilter(ref filter) => {
-                    if let Ok(guard) = self.handles.filters.data.filters.read() {
-                        if guard.contains(filter) {
-                            // match was found, but already known; clear the vec and continue to the next
-                            responses.clear();
-                            continue;
+                            // if we're here, we've found a new wildcard that we didn't previously display, print it
+                            if print_sentry {
+                                ferox_print(&format!("{}", new_wildcard), &PROGRESS_PRINTER);
+                            }
                         }
                     }
                 }
-                _ => unreachable!(),
-            }
 
-            // create the new filter
-            self.handles.filters.send(command)?;
+                // create the new filter
+                for wildcard in wildcard_filters {
+                    self.handles.filters.send(Command::AddFilter(wildcard))?;
+                }
 
-            // if we're here, we've detected a 404-like response pattern, and we're already filtering for size/word/line
-            //
-            // in addition, we'll create a similarity filter as a fallback
-            let hash = SIM_HASHER.create_signature(preprocess(responses[0].text()).iter());
+                // if we're here, we've detected a 404-like response pattern, and we're already filtering for size/word/line
+                //
+                // in addition, we'll create a similarity filter as a fallback
+                for resp in wildcard_responses {
+                    let hash = SIM_HASHER.create_signature(preprocess(resp.text()).iter());
 
-            let sim_filter = SimilarityFilter {
-                hash,
-                original_url: responses[0].url().to_string(),
-            };
+                    let sim_filter = SimilarityFilter {
+                        hash,
+                        original_url: resp.url().to_string(),
+                    };
 
-            self.handles
-                .filters
-                .send(Command::AddFilter(Box::new(sim_filter)))?;
+                    self.handles
+                        .filters
+                        .send(Command::AddFilter(Box::new(sim_filter)))?;
 
-            // reset the responses for the next method, if it exists
-            responses.clear();
+                    if resp.is_directory() {
+                        // response is either a 3XX with a Location header that matches url + '/'
+                        // or it's a 2XX that ends with a '/'
+                        // or it's a 403 that ends with a '/'
 
-            // report to the user, if appropriate
-            if matches!(
-                self.handles.config.output_level,
-                OutputLevel::Default | OutputLevel::Quiet
-            ) {
-                let msg = format!("{} {:>8} {:>9} {:>9} {:>9} {} => {} {}-like response ({} {}); toggle this behavior by using {}\n", status_colorizer("WLD"), "-", "-", "-", "-", style(target_url).cyan(), style("auto-filtering").bright().green(), style("404").red(), style(filter_length).cyan(), filter_type, style("--dont-filter").yellow());
-                ferox_print(&msg, &PROGRESS_PRINTER);
+                        // set the wildcard flag to true, so we can check it when preventing
+                        // recursion in event_handlers/scans.rs
+
+                        // we'd need to clone the response to give ownership to the global list anyway
+                        // so we'll also use that clone to set the wildcard flag
+                        let mut cloned_resp = resp.clone();
+
+                        cloned_resp.set_wildcard(true);
+
+                        // add the response to the global list of responses
+                        RESPONSES.insert(cloned_resp);
+
+                        // function-internal magic number, indicates that we've detected a wildcard directory
+                        req_counter += 100;
+                    }
+                }
+
+                // reset the responses for the next method, if it exists
+                responses.clear();
             }
         }
 
         log::trace!("exit: detect_404_like_responses");
 
-        Ok(req_counter)
+        let retval = if req_counter > 100 {
+            WildcardResult::WildcardDirectory(req_counter)
+        } else {
+            WildcardResult::FourOhFourLike(req_counter)
+        };
+
+        Ok(Some(retval))
     }
 
-    /// for all responses, examine chars/words/lines
-    /// if all responses respective lengths match each other, we can assume
-    /// that will remain true for subsequent non-existent urls
+    /// for all responses, group them by status code, then examine chars/words/lines.
+    /// if all responses' respective lengths within a status code grouping match
+    /// each other, we can assume that will remain true for subsequent non-existent urls
     ///
-    /// values are examined from most to least specific (content length, word count, line count)
-    fn examine_404_like_responses(
+    /// within a status code grouping, values are examined from most to
+    /// least specific (content length, word count, line count)
+    #[allow(clippy::vec_box)] // the box is needed in the caller and i dont feel like changing it
+    fn examine_404_like_responses<'a>(
         &self,
-        responses: &[FeroxResponse],
-    ) -> Option<(Command, &'static str, usize)> {
+        responses: &'a [FeroxResponse],
+    ) -> Option<(Vec<Box<WildcardFilter>>, Vec<&'a FeroxResponse>)> {
+        // aside from word/line/byte counts, additional discriminators are status code
+        // extension, and request method. The request method and extension are handled by
+        // the caller, since they're part of the request and make up the nested for loops
+        // in detect_404_like_responses.
+        //
+        // The status code is handled here, since it's part of the response to catch cases
+        // where we have something like a 403 and a 404
+
         let mut size_sentry = true;
         let mut word_sentry = true;
         let mut line_sentry = true;
 
-        let content_length = responses[0].content_length();
-        let word_count = responses[0].word_count();
-        let line_count = responses[0].line_count();
+        // returned vec of boxed wildcard filters
+        let mut wildcards = Vec::new();
 
-        for response in &responses[1..] {
-            // if any of the responses differ in length, that particular
-            // response length type is no longer a candidate for filtering
-            if response.content_length() != content_length {
-                size_sentry = false;
-            }
+        // returned vec of ferox responses that are needed for additional
+        // analysis
+        let mut wild_responses = Vec::new();
 
-            if response.word_count() != word_count {
-                word_sentry = false;
-            }
+        // mapping of grouped responses to status code
+        let mut grouped_responses = HashMap::new();
 
-            if response.line_count() != line_count {
-                line_sentry = false;
-            }
+        // iterate over all responses and add each response to its
+        // corresponding status code group
+        for response in responses {
+            grouped_responses
+                .entry(response.status())
+                .or_insert_with(Vec::new)
+                .push(response);
         }
 
-        // the if/else-if/else nature of the block means that we'll get the most
-        // specific match, if one is to be had
-        //
-        // each block returns the information needed to send the filter away and
-        // display a message to the user
-        if size_sentry {
-            // - command to send to the filters handler
-            // - the unit-type we're filtering on (bytes/words/lines)
-            // - the value associated with the unit-type on which we're filtering
-            Some((
-                Command::AddFilter(Box::new(SizeFilter { content_length })),
-                "bytes",
-                content_length as usize,
-            ))
-        } else if word_sentry {
-            Some((
-                Command::AddFilter(Box::new(WordsFilter { word_count })),
-                "words",
-                word_count,
-            ))
-        } else if line_sentry {
-            Some((
-                Command::AddFilter(Box::new(LinesFilter { line_count })),
-                "lines",
-                line_count,
-            ))
-        } else {
-            // no match was found; clear the vec and continue to the next
-            None
+        // iterate over each grouped response and determine the most specific
+        // filter that can be applied to all responses in the group, i.e.
+        // start from byte count and work 'out' to line count
+        for response_group in grouped_responses.values() {
+            if response_group.len() < 2 {
+                // not enough responses to make a determination
+                continue;
+            }
+
+            let method = response_group[0].method();
+            let status_code = response_group[0].status();
+            let content_length = response_group[0].content_length();
+            let word_count = response_group[0].word_count();
+            let line_count = response_group[0].line_count();
+
+            for response in &response_group[1..] {
+                // if any of the responses differ in length, that particular
+                // response length type is no longer a candidate for filtering
+                if response.content_length() != content_length {
+                    size_sentry = false;
+                }
+
+                if response.word_count() != word_count {
+                    word_sentry = false;
+                }
+
+                if response.line_count() != line_count {
+                    line_sentry = false;
+                }
+            }
+
+            if !size_sentry && !word_sentry && !line_sentry {
+                // none of the response lengths match, so we can't filter on any of them
+                continue;
+            }
+
+            let mut wildcard = WildcardFilter {
+                content_length: None,
+                line_count: None,
+                word_count: None,
+                method: method.to_string(),
+                status_code: status_code.as_u16(),
+                dont_filter: self.handles.config.dont_filter,
+            };
+
+            match (size_sentry, word_sentry, line_sentry) {
+                (true, true, true) => {
+                    // all three types of length match, so we can't filter on any of them
+                    wildcard.content_length = Some(content_length);
+                    wildcard.word_count = Some(word_count);
+                    wildcard.line_count = Some(line_count);
+                }
+                (true, true, false) => {
+                    // content length and word count match, so we can filter on either
+                    wildcard.content_length = Some(content_length);
+                    wildcard.word_count = Some(word_count);
+                }
+                (true, false, true) => {
+                    // content length and line count match, so we can filter on either
+                    wildcard.content_length = Some(content_length);
+                    wildcard.line_count = Some(line_count);
+                }
+                (false, true, true) => {
+                    // word count and line count match, so we can filter on either
+                    wildcard.word_count = Some(word_count);
+                    wildcard.line_count = Some(line_count);
+                }
+                (true, false, false) => {
+                    // content length matches, so we can filter on that
+                    wildcard.content_length = Some(content_length);
+                }
+                (false, true, false) => {
+                    // word count matches, so we can filter on that
+                    wildcard.word_count = Some(word_count);
+                }
+                (false, false, true) => {
+                    // line count matches, so we can filter on that
+                    wildcard.line_count = Some(line_count);
+                }
+                (false, false, false) => {
+                    // none of the length types match, so we can't filter on any of them
+                    unreachable!("no wildcard size matches; handled by the if statement above");
+                }
+            };
+
+            wild_responses.push(response_group[0]);
+            wildcards.push(Box::new(wildcard));
         }
+
+        Some((wildcards, wild_responses))
     }
 }
 

src/main.rs

@@ -1,11 +1,14 @@
 use std::io::stdin;
 use std::{
-    env::args,
+    env::{
+        args,
+        consts::{ARCH, OS},
+    },
     fs::{create_dir, remove_file, File},
     io::{stderr, BufRead, BufReader},
     ops::Index,
     path::Path,
-    process::Command,
+    process::{exit, Command},
     sync::{atomic::Ordering, Arc},
 };
 
@@ -38,6 +41,7 @@ use feroxbuster::{
 use feroxbuster::{utils::set_open_file_limit, DEFAULT_OPEN_FILE_LIMIT};
 use lazy_static::lazy_static;
 use regex::Regex;
+use self_update::cargo_crate_version;
 
 lazy_static! {
     /// Limits the number of parallel scans active at any given time when using --parallel
@@ -219,19 +223,64 @@ async fn wrapped_main(config: Arc<Configuration>) -> Result<()> {
         PROGRESS_BAR.join().unwrap();
     });
 
-    // cloning an Arc is cheap (it's basically a pointer into the heap)
-    // so that will allow for cheap/safe sharing of a single wordlist across multi-target scans
-    // as well as additional directories found as part of recursion
-    let words = match get_unique_words_from_wordlist(&config.wordlist) {
-        Ok(w) => w,
-        Err(err) => {
-            let secondary = Path::new(SECONDARY_WORDLIST);
+    // check if update_app is true
+    if config.update_app {
+        match update_app().await {
+            Err(e) => eprintln!("\n[ERROR] {}", e),
+            Ok(self_update::Status::UpToDate(version)) => {
+                eprintln!("\nFeroxbuster {} is up to date", version)
+            }
+            Ok(self_update::Status::Updated(version)) => {
+                eprintln!("\nFeroxbuster updated to {} version", version)
+            }
+        }
+        exit(0);
+    }
 
-            if secondary.exists() {
-                eprintln!("Found wordlist in secondary location");
-                get_unique_words_from_wordlist(SECONDARY_WORDLIST)?
-            } else {
-                return Err(err);
+    let words = if config.wordlist.starts_with("http") {
+        // found a url scheme, attempt to download the wordlist
+        let response = config.client.get(&config.wordlist).send().await?;
+
+        if !response.status().is_success() {
+            // status code isn't a 200, bail
+            bail!(
+                "[{}] Unable to download wordlist from url: {}",
+                response.status().as_str(),
+                config.wordlist
+            );
+        }
+
+        // attempt to get the filename from the url's path
+        let Some(path_segments) = response
+            .url()
+            .path_segments() else {
+                bail!("Unable to parse path from url: {}", response.url());
+            };
+
+        let Some(filename) = path_segments.last() else {
+            bail!("Unable to parse filename from url's path: {}", response.url().path());
+        };
+
+        let filename = filename.to_string();
+
+        // read the body and write it to disk, then use existing code to read the wordlist
+        let body = response.text().await?;
+
+        std::fs::write(&filename, body)?;
+
+        get_unique_words_from_wordlist(&filename)?
+    } else {
+        match get_unique_words_from_wordlist(&config.wordlist) {
+            Ok(w) => w,
+            Err(err) => {
+                let secondary = Path::new(SECONDARY_WORDLIST);
+
+                if secondary.exists() {
+                    eprintln!("Found wordlist in secondary location");
+                    get_unique_words_from_wordlist(SECONDARY_WORDLIST)?
+                } else {
+                    return Err(err);
+                }
             }
         }
     };
@@ -526,6 +575,24 @@ async fn clean_up(handles: Arc<Handles>, tasks: Tasks) -> Result<()> {
     Ok(())
 }
 
+async fn update_app() -> Result<self_update::Status, Box<dyn ::std::error::Error>> {
+    let target_os = format!("{}-{}", ARCH, OS);
+    let status = tokio::task::spawn_blocking(move || {
+        self_update::backends::github::Update::configure()
+            .repo_owner("epi052")
+            .repo_name("feroxbuster")
+            .bin_name("feroxbuster")
+            .target(target_os.as_str())
+            .show_download_progress(true)
+            .current_version(cargo_crate_version!())
+            .build()?
+            .update()
+    })
+    .await??;
+
+    Ok(status)
+}
+
 fn main() -> Result<()> {
     let config = Arc::new(Configuration::new().with_context(|| "Could not create Configuration")?);
 

src/parser.rs

@@ -40,7 +40,7 @@ pub fn initialize() -> Command {
             Arg::new("url")
                 .short('u')
                 .long("url")
-                .required_unless_present_any(["stdin", "resume_from"])
+                .required_unless_present_any(["stdin", "resume_from", "update_app"])
                 .help_heading("Target selection")
                 .value_name("URL")
                 .use_value_delimiter(true)
@@ -91,12 +91,15 @@ pub fn initialize() -> Command {
                 .long("smart")
                 .num_args(0)
                 .help_heading("Composite settings")
-                .help("Set --extract-links, --auto-tune, --collect-words, and --collect-backups to true"),
-        ).arg(
+                .conflicts_with_all(["rate_limit", "auto_bail"])
+                .help("Set --auto-tune, --collect-words, and --collect-backups to true"),
+        )
+        .arg(
             Arg::new("thorough")
                 .long("thorough")
                 .num_args(0)
                 .help_heading("Composite settings")
+                .conflicts_with_all(["rate_limit", "auto_bail"])
                 .help("Use the same settings as --smart and set --collect-extensions to true"),
         );
 
@@ -431,7 +434,15 @@ pub fn initialize() -> Command {
                 .long("extract-links")
                 .num_args(0)
                 .help_heading("Scan settings")
-                .help("Extract links from response body (html, javascript, etc...); make new requests based on findings")
+                .hide(true)
+                .help("Extract links from response body (html, javascript, etc...); make new requests based on findings (default: true)")
+        )
+        .arg(
+            Arg::new("dont_extract_links")
+                .long("dont-extract-links")
+                .num_args(0)
+                .help_heading("Scan settings")
+                .help("Don't extract links from response body (html, javascript, etc...)")
         )
         .arg(
             Arg::new("scan_limit")
@@ -475,7 +486,7 @@ pub fn initialize() -> Command {
                 .long("wordlist")
                 .value_hint(ValueHint::FilePath)
                 .value_name("FILE")
-                .help("Path to the wordlist")
+                .help("Path or URL of the wordlist")
                 .help_heading("Scan settings")
                 .num_args(1),
         ).arg(
@@ -513,7 +524,8 @@ pub fn initialize() -> Command {
                 .num_args(0)
                 .help_heading("Dynamic collection settings")
                 .help("Automatically request likely backup extensions for \"found\" urls")
-        ).arg(
+        )
+        .arg(
             Arg::new("collect_words")
                 .short('g')
                 .long("collect-words")
@@ -607,6 +619,15 @@ pub fn initialize() -> Command {
                 .args(["debug_log", "output"])
                 .multiple(true),
         )
+        .arg(
+            Arg::new("update_app")
+                .short('U')
+                .long("update")
+                .exclusive(true)
+                .num_args(0)
+                .help_heading("Update settings")
+                .help("Update feroxbuster to the latest version"),
+        )
         .after_long_help(EPILOGUE);
 
     /////////////////////////////////////////////////////////////////////
@@ -673,9 +694,6 @@ EXAMPLES:
     Pass auth token via query parameter
         ./feroxbuster -u http://127.1 --query token=0123456789ABCDEF
 
-    Find links in javascript/html and make additional requests based on results
-        ./feroxbuster -u http://127.1 --extract-links
-
     Ludicrous speed... go!
         ./feroxbuster -u http://127.1 --threads 200
         

src/scan_manager/menu.rs

@@ -210,10 +210,14 @@ impl Menu {
                     }
                 });
             } else {
+                if value.is_empty() {
+                    continue;
+                }
+
                 let value = self.str_to_usize(value);
 
-                if value != 0 && !nums.contains(&value) {
-                    // the zeroth scan is always skipped, skip already known values
+                if !nums.contains(&value) {
+                    // skip already known values
                     nums.push(value);
                 }
             }

src/scan_manager/scan.rs

@@ -39,6 +39,7 @@ pub struct FeroxScan {
     pub scan_type: ScanType,
 
     /// The order in which the scan was received
+    #[allow(dead_code)] // not entirely sure this isn't used somewhere
     pub(crate) scan_order: ScanOrder,
 
     /// Number of requests to populate the progress bar with
@@ -153,7 +154,13 @@ impl FeroxScan {
     pub(super) fn stop_progress_bar(&self) {
         if let Ok(guard) = self.progress_bar.lock() {
             if guard.is_some() {
-                (*guard).as_ref().unwrap().finish_at_current_pos()
+                let pb = (*guard).as_ref().unwrap();
+
+                if pb.position() > self.num_requests {
+                    pb.finish()
+                } else {
+                    pb.finish_at_current_pos()
+                }
             }
         }
     }

src/scan_manager/scan_container.rs

@@ -3,11 +3,12 @@ use super::*;
 use crate::event_handlers::Handles;
 use crate::filters::{
     EmptyFilter, LinesFilter, RegexFilter, SimilarityFilter, SizeFilter, StatusCodeFilter,
-    WordsFilter,
+    WildcardFilter, WordsFilter,
 };
 use crate::traits::FeroxFilter;
 use crate::Command::AddFilter;
 use crate::{
+    banner::Banner,
     config::OutputLevel,
     progress::PROGRESS_PRINTER,
     progress::{add_bar, BarType},
@@ -182,6 +183,10 @@ impl FeroxScans {
                         serde_json::from_value::<WordsFilter>(filter.clone())
                     {
                         Box::new(deserialized)
+                    } else if let Ok(deserialized) =
+                        serde_json::from_value::<WildcardFilter>(filter.clone())
+                    {
+                        Box::new(deserialized)
                     } else if let Ok(deserialized) =
                         serde_json::from_value::<SizeFilter>(filter.clone())
                     {
@@ -320,11 +325,6 @@ impl FeroxScans {
         let mut printed = 0;
 
         for (i, scan) in scans.iter().enumerate() {
-            if matches!(scan.scan_order, ScanOrder::Initial) || scan.task.try_lock().is_err() {
-                // original target passed in via either -u or --stdin
-                continue;
-            }
-
             if matches!(scan.scan_type, ScanType::Directory) {
                 if printed == 0 {
                     self.menu
@@ -373,14 +373,13 @@ impl FeroxScans {
 
             if input == 'y' || input == '\n' {
                 self.menu.println(&format!("Stopping {}...", selected.url));
-
                 selected
                     .abort()
                     .await
                     .unwrap_or_else(|e| log::warn!("Could not cancel task: {}", e));
 
                 let pb = selected.progress_bar();
-                num_cancelled += pb.length() as usize - pb.position() as usize
+                num_cancelled += pb.length() as usize - pb.position() as usize;
             } else {
                 self.menu.println("Ok, doing nothing...");
             }
@@ -446,8 +445,40 @@ impl FeroxScans {
         };
 
         self.menu.clear_screen();
+
+        let banner = Banner::new(&[handles.config.target_url.clone()], &handles.config);
+        banner
+            .print_to(&self.menu.term, handles.config.clone())
+            .unwrap_or_default();
+
         self.menu.show_progress_bars();
 
+        let has_active_scans = if let Ok(guard) = self.scans.read() {
+            guard.iter().any(|s| s.is_active())
+        } else {
+            // if we can't tell for sure, we'll let it ride
+            //
+            // i'm not sure which is the better option here:
+            // either return true and let it potentially hang, or
+            // return false and exit, so just going with not
+            // abruptly exiting for maybe no reason
+            true
+        };
+
+        if !has_active_scans {
+            // the last active scan was cancelled, so we can exit
+            self.menu.println(&format!(
+                " 😱 no more active scans... {}",
+                style("exiting").red()
+            ));
+
+            let (tx, rx) = tokio::sync::oneshot::channel::<bool>();
+            handles
+                .send_scan_command(Command::JoinTasks(tx))
+                .unwrap_or_default();
+            rx.await.unwrap_or_default();
+        }
+
         result
     }
 

src/scan_manager/tests.rs

@@ -469,7 +469,7 @@ fn feroxstates_feroxserialize_implementation() {
         r#""headers""#,
         r#""queries":[]"#,
         r#""no_recursion":false"#,
-        r#""extract_links":false"#,
+        r#""extract_links":true"#,
         r#""add_slash":false"#,
         r#""stdin":false"#,
         r#""depth":4"#,
@@ -668,11 +668,7 @@ fn menu_get_command_input_from_user_returns_cancel() {
         assert!(matches!(result, MenuCmd::Cancel(_, _)));
 
         if let MenuCmd::Cancel(canx_list, ret_force) = result {
-            if idx == 0 {
-                assert!(canx_list.is_empty());
-            } else {
-                assert_eq!(canx_list, vec![idx]);
-            }
+            assert_eq!(canx_list, vec![idx]);
             assert_eq!(force, ret_force);
         }
     }

src/scanner/ferox_scanner.rs

@@ -10,6 +10,7 @@ use lazy_static::lazy_static;
 use tokio::sync::Semaphore;
 
 use crate::filters::{create_similarity_filter, EmptyFilter, SimilarityFilter};
+use crate::heuristics::WildcardResult;
 use crate::Command::AddFilter;
 use crate::{
     event_handlers::{
@@ -282,15 +283,21 @@ impl FeroxScanner {
                     let mut message = format!("=> {}", style("Directory listing").blue().bright());
 
                     if !self.handles.config.extract_links {
-                        write!(message, " (add {} to scan)", style("-e").bright().yellow())?;
+                        write!(
+                            message,
+                            " (remove {} to scan)",
+                            style("--dont-extract-links").bright().yellow()
+                        )?;
                     }
 
-                    progress_bar.reset_eta();
-                    progress_bar.finish_with_message(&message);
+                    if !self.handles.config.force_recursion {
+                        progress_bar.reset_eta();
+                        progress_bar.finish_with_message(&message);
 
-                    ferox_scan.finish()?;
+                        ferox_scan.finish()?;
 
-                    return Ok(()); // nothing left to do if we found a dir listing
+                        return Ok(()); // nothing left to do if we found a dir listing
+                    }
                 }
             }
 
@@ -303,7 +310,21 @@ impl FeroxScanner {
             // wildcard test
             let num_reqs_made = test.detect_404_like_responses(&self.target_url).await?;
 
-            progress_bar.inc(num_reqs_made);
+            match num_reqs_made {
+                Some(WildcardResult::WildcardDirectory(num_reqs)) => {
+                    let message = format!(
+                        "=> {} dir! {} recursion",
+                        style("Wildcard").blue().bright(),
+                        style("stopped").red()
+                    );
+                    progress_bar.set_message(&message);
+                    progress_bar.inc(num_reqs as u64);
+                }
+                Some(WildcardResult::FourOhFourLike(num_reqs)) => {
+                    progress_bar.inc(num_reqs as u64);
+                }
+                _ => {}
+            }
         }
 
         // Arc clones to be passed around to the various scans

src/scanner/requester.rs

@@ -437,7 +437,7 @@ impl Requester {
                     .handles
                     .filters
                     .data
-                    .should_filter_response(&ferox_response)
+                    .should_filter_response(&ferox_response, self.handles.stats.tx.clone())
                 {
                     continue;
                 }

src/traits.rs

@@ -1,8 +1,10 @@
 //! collection of all traits used
 use crate::filters::{
-    LinesFilter, RegexFilter, SimilarityFilter, SizeFilter, StatusCodeFilter, WordsFilter,
+    LinesFilter, RegexFilter, SimilarityFilter, SizeFilter, StatusCodeFilter, WildcardFilter,
+    WordsFilter,
 };
 use crate::response::FeroxResponse;
+use crate::utils::status_colorizer;
 use anyhow::Result;
 use crossterm::style::{style, Stylize};
 use serde::Serialize;
@@ -36,6 +38,44 @@ impl Display for dyn FeroxFilter {
             write!(f, "Response size: {}", style(filter.content_length).cyan())
         } else if let Some(filter) = self.as_any().downcast_ref::<RegexFilter>() {
             write!(f, "Regex: {}", style(&filter.raw_string).cyan())
+        } else if let Some(filter) = self.as_any().downcast_ref::<WildcardFilter>() {
+            let mut msg = format!(
+                "{} requests with {} responses ",
+                style(&filter.method).cyan(),
+                status_colorizer(&filter.status_code.to_string())
+            );
+
+            match (filter.content_length, filter.word_count, filter.line_count) {
+                (None, None, None) => {
+                    unreachable!("wildcard filter without any filters set");
+                }
+                (None, None, Some(lc)) => {
+                    msg.push_str(&format!("containing {} lines", lc));
+                }
+                (None, Some(wc), None) => {
+                    msg.push_str(&format!("containing {} words", wc));
+                }
+                (None, Some(wc), Some(lc)) => {
+                    msg.push_str(&format!("containing {} words and {} lines", wc, lc));
+                }
+                (Some(cl), None, None) => {
+                    msg.push_str(&format!("containing {} bytes", cl));
+                }
+                (Some(cl), None, Some(lc)) => {
+                    msg.push_str(&format!("containing {} bytes and {} lines", cl, lc));
+                }
+                (Some(cl), Some(wc), None) => {
+                    msg.push_str(&format!("containing {} bytes and {} words", cl, wc));
+                }
+                (Some(cl), Some(wc), Some(lc)) => {
+                    msg.push_str(&format!(
+                        "containing {} bytes, {} words, and {} lines",
+                        cl, wc, lc
+                    ));
+                }
+            }
+
+            write!(f, "{}", msg)
         } else if let Some(filter) = self.as_any().downcast_ref::<StatusCodeFilter>() {
             write!(f, "Status code: {}", style(filter.filter_code).cyan())
         } else if let Some(filter) = self.as_any().downcast_ref::<SimilarityFilter>() {

tests/test_banner.rs

@@ -1420,3 +1420,15 @@ fn banner_prints_force_recursion() {
                 .and(predicate::str::contains("─┴─")),
         );
 }
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + force recursion
+fn banner_prints_update_app() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--update")
+        .assert()
+        .success()
+        .stdout(predicate::str::contains("Checking target-arch..."));
+}

tests/test_heuristics.rs

@@ -164,7 +164,7 @@ fn test_static_wildcard_request_found() -> Result<(), Box<dyn std::error::Error>
 
     let mock = srv.mock(|when, then| {
         when.method(GET)
-            .path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap());
+            .path_matches(Regex::new("/[.a-zA-Z0-9]{32,}/").unwrap());
         then.status(200).body("this is a test");
     });
 
@@ -180,12 +180,16 @@ fn test_static_wildcard_request_found() -> Result<(), Box<dyn std::error::Error>
     teardown_tmp_directory(tmp_dir);
 
     cmd.assert().success().stdout(
-        predicate::str::contains("WLD").and(predicate::str::contains(
-            "auto-filtering 404-like response (1 lines);",
-        )),
+        predicate::str::contains("GET")
+            .and(predicate::str::contains(
+                "Auto-filtering found 404-like response and created new filter",
+            ))
+            .and(predicate::str::contains("200"))
+            .and(predicate::str::contains("1l")),
     );
 
-    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock.hits(), 6);
+
     Ok(())
 }
 
@@ -273,14 +277,14 @@ fn heuristics_wildcard_test_with_two_static_wildcards_with_silent_enabled(
 
     let mock = srv.mock(|when, then| {
         when.method(GET)
-            .path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap());
+            .path_matches(Regex::new("/.?[a-zA-Z0-9]{32,}").unwrap());
         then.status(200)
             .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
     });
 
     let mock2 = srv.mock(|when, then| {
         when.method(GET)
-            .path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap());
+            .path_matches(Regex::new("/LICENSE").unwrap());
         then.status(200)
             .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
     });
@@ -291,7 +295,6 @@ fn heuristics_wildcard_test_with_two_static_wildcards_with_silent_enabled(
         .arg(srv.url("/"))
         .arg("--wordlist")
         .arg(file.as_os_str())
-        .arg("--add-slash")
         .arg("--silent")
         .arg("--threads")
         .arg("1")
@@ -299,13 +302,71 @@ fn heuristics_wildcard_test_with_two_static_wildcards_with_silent_enabled(
 
     teardown_tmp_directory(tmp_dir);
 
-    cmd.assert().success().stdout(predicate::str::is_empty());
+    cmd.assert()
+        .success()
+        .stdout(predicate::str::contains(srv.url("/")));
 
-    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock.hits(), 6);
     assert_eq!(mock2.hits(), 1);
     Ok(())
 }
 
+#[test]
+/// test finds a 404-like response that returns a 403 and a 403 directory should still be allowed
+/// to be tested for recrusion
+fn heuristics_wildcard_test_that_auto_filtering_403s_still_allows_for_recursion_into_403_directories(
+) -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+
+    let super_long = String::from("92969beae6bf4beb855d1622406d87e395c87387a9ad432e8a11245002b709b03cf609d471004154b83bcc1c6ec49f6f09d471004154b83bcc1c6ec49f6f");
+
+    let (tmp_dir, file) =
+        setup_tmp_directory(&["LICENSE".to_string(), super_long.clone()], "wordlist")?;
+
+    srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/.?[a-zA-Z0-9]{32,103}").unwrap());
+        then.status(403)
+            .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+    });
+
+    srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE/");
+        then.status(403)
+            .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+    });
+
+    srv.mock(|when, then| {
+        when.method(GET).path(format!("/LICENSE/{}", super_long));
+        then.status(200);
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--add-slash")
+        .unwrap();
+
+    teardown_tmp_directory(tmp_dir);
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("GET")
+            .and(predicate::str::contains(
+                "Auto-filtering found 404-like response and created new filter",
+            ))
+            .and(predicate::str::contains("403"))
+            .and(predicate::str::contains("1l"))
+            .and(predicate::str::contains("4w"))
+            .and(predicate::str::contains("46c"))
+            .and(predicate::str::contains(srv.url("/LICENSE/LICENSE/"))),
+    );
+
+    Ok(())
+}
+
 // #[test]
 // /// test finds a static wildcard and reports as much to stdout and a file
 // fn heuristics_wildcard_test_with_two_static_wildcards_and_output_to_file() {

tests/test_main.rs

@@ -218,3 +218,46 @@ fn main_parallel_creates_output_directory() -> Result<(), Box<dyn std::error::Er
 
     Ok(())
 }
+
+#[test]
+/// download a wordlist from a url
+fn main_download_wordlist_from_url() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+
+    let (tmp_dir, _) = setup_tmp_directory(&["a".to_string()], "wordlist")?;
+
+    let mock1 = srv.mock(|when, then| {
+        when.method(GET).path("/derp");
+        then.status(200).body("stuff\nthings");
+    });
+
+    // serve endpoints stuff and things
+    let mock2 = srv.mock(|when, then| {
+        when.method(GET).path("/stuff");
+        then.status(200);
+    });
+
+    let mock3 = srv.mock(|when, then| {
+        when.method(GET).path("/things");
+        then.status(200);
+    });
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .current_dir(&tmp_dir)
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(srv.url("/derp"))
+        .assert()
+        .success()
+        .stderr(predicate::str::contains(srv.url("/derp")));
+
+    teardown_tmp_directory(tmp_dir);
+
+    assert_eq!(mock1.hits(), 1); // downloaded wordlist
+    assert_eq!(mock2.hits(), 1); // found stuff from wordlist
+    assert_eq!(mock3.hits(), 1); // found things from wordlist
+
+    Ok(())
+}

tests/test_scanner.rs

@@ -573,7 +573,7 @@ fn scanner_recursion_works_with_403_directories() {
     let found_anyway = srv.mock(|when, then| {
         when.method(GET).path("/ignored/LICENSE");
         then.status(200)
-            .body("this is a test\nThat rugf really tied the room together");
+            .body("this is a test\nThat rug really tied the room together");
     });
 
     let cmd = Command::cargo_bin("feroxbuster")
@@ -588,9 +588,10 @@ fn scanner_recursion_works_with_403_directories() {
         predicate::str::contains("/LICENSE")
             .count(2)
             .and(predicate::str::contains("200"))
-            .and(predicate::str::contains("WLD"))
+            .and(predicate::str::contains("404"))
+            .and(predicate::str::contains("53c Auto-filtering"))
             .and(predicate::str::contains(
-                "auto-filtering 404-like response (53 bytes);",
+                "Auto-filtering found 404-like response and created new filter;",
             ))
             .and(predicate::str::contains("14c"))
             .and(predicate::str::contains("0c"))