docs: add RavySena as a contributor for ideas (#1025 )

* docs: update README.md [skip ci] * docs: update .all-contributorsrc [skip ci] --------- Co-authored-by: allcontributors[bot] <46447321+allcontributors[bot]@users.noreply.github.com>
docs: add ocervell as a contributor for ideas (#1024 )
2026-05-22 20:31:13 -03:00 · 2023-11-08 21:24:35 -05:00 · 2023-11-08 21:23:26 -05:00 · 2023-11-08 21:22:40 -05:00 · 2023-11-08 21:18:53 -05:00 · 2023-11-08 06:22:00 -05:00
67 changed files with 3581 additions and 1381 deletions
--- a/.all-contributorsrc
+++ b/.all-contributorsrc
@@ -198,7 +198,8 @@
      "avatar_url": "https://avatars.githubusercontent.com/u/24260009?v=4",
      "profile": "https://github.com/N0ur5",
      "contributions": [
-        "ideas"
+        "ideas",
+        "bug"
      ]
    },
    {
@@ -550,7 +551,127 @@
      "profile": "https://petruknisme.com",
      "contributions": [
        "code",
-        "infra"
+        "infra",
+        "ideas"
+      ]
+    },
+    {
+      "login": "imBigo",
+      "name": "Simon",
+      "avatar_url": "https://avatars.githubusercontent.com/u/54672433?v=4",
+      "profile": "https://github.com/imBigo",
+      "contributions": [
+        "bug"
+      ]
+    },
+    {
+      "login": "acut3",
+      "name": "Nicolas Christin",
+      "avatar_url": "https://avatars.githubusercontent.com/u/17295243?v=4",
+      "profile": "https://acut3.github.io/",
+      "contributions": [
+        "bug"
+      ]
+    },
+    {
+      "login": "DrorDvash",
+      "name": "DrDv",
+      "avatar_url": "https://avatars.githubusercontent.com/u/8413651?v=4",
+      "profile": "https://github.com/DrorDvash",
+      "contributions": [
+        "bug"
+      ]
+    },
+    {
+      "login": "aroly",
+      "name": "Antoine Roly",
+      "avatar_url": "https://avatars.githubusercontent.com/u/1257705?v=4",
+      "profile": "https://github.com/aroly",
+      "contributions": [
+        "ideas"
+      ]
+    },
+    {
+      "login": "lavafroth",
+      "name": "Himadri Bhattacharjee",
+      "avatar_url": "https://avatars.githubusercontent.com/u/107522312?v=4",
+      "profile": "http://lavafroth.is-a.dev",
+      "contributions": [
+        "code",
+        "ideas"
+      ]
+    },
+    {
+      "login": "AkechiShiro",
+      "name": "Samy Lahfa",
+      "avatar_url": "https://avatars.githubusercontent.com/u/14914796?v=4",
+      "profile": "https://github.com/AkechiShiro",
+      "contributions": [
+        "ideas"
+      ]
+    },
+    {
+      "login": "sectroyer",
+      "name": "sectroyer",
+      "avatar_url": "https://avatars.githubusercontent.com/u/6706818?v=4",
+      "profile": "https://github.com/sectroyer",
+      "contributions": [
+        "bug",
+        "ideas"
+      ]
+    },
+    {
+      "login": "ktecv2000",
+      "name": "ktecv2000",
+      "avatar_url": "https://avatars.githubusercontent.com/u/19836003?v=4",
+      "profile": "https://medium.com/@b3rm1nG",
+      "contributions": [
+        "bug"
+      ]
+    },
+    {
+      "login": "andreademurtas",
+      "name": "Andrea De Murtas",
+      "avatar_url": "https://avatars.githubusercontent.com/u/56048157?v=4",
+      "profile": "http://untrue.me",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "sawmj",
+      "name": "sawmj",
+      "avatar_url": "https://avatars.githubusercontent.com/u/30024085?v=4",
+      "profile": "https://github.com/sawmj",
+      "contributions": [
+        "bug"
+      ]
+    },
+    {
+      "login": "devx00",
+      "name": "Zach Hanson",
+      "avatar_url": "https://avatars.githubusercontent.com/u/6897405?v=4",
+      "profile": "https://github.com/devx00",
+      "contributions": [
+        "bug"
+      ]
+    },
+    {
+      "login": "ocervell",
+      "name": "Olivier Cervello",
+      "avatar_url": "https://avatars.githubusercontent.com/u/9629314?v=4",
+      "profile": "https://github.com/ocervell",
+      "contributions": [
+        "ideas"
+      ]
+    },
+    {
+      "login": "RavySena",
+      "name": "RavySena",
+      "avatar_url": "https://avatars.githubusercontent.com/u/67729597?v=4",
+      "profile": "https://github.com/RavySena",
+      "contributions": [
+        "ideas"
      ]
    }
  ],
@@ -560,5 +681,6 @@
  "repoType": "github",
  "repoHost": "https://github.com",
  "skipCi": true,
-  "commitConvention": "angular"
+  "commitConvention": "angular",
+  "commitType": "docs"
 }
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -27,13 +27,13 @@ jobs:
          - type: armv7
            os: ubuntu-latest
            target: armv7-unknown-linux-gnueabihf
-            name: armv7-feroxbuster
+            name: armv7-linux-feroxbuster
            path: target/armv7-unknown-linux-gnueabihf/release/feroxbuster
            pkg_config_path: /usr/lib/x86_64-linux-gnu/pkgconfig
          - type: aarch64
            os: ubuntu-latest
            target: aarch64-unknown-linux-gnu
-            name: aarch64-feroxbuster
+            name: aarch64-linux-feroxbuster
            path: target/aarch64-unknown-linux-gnu/release/feroxbuster
            pkg_config_path: /usr/lib/x86_64-linux-gnu/pkgconfig
    steps:
--- a/.gitignore
+++ b/.gitignore
@@ -30,3 +30,6 @@ ferox-*.state

 # python stuff cuz reasons
 Pipfile*
+
+# ignore choco_package generated nupkg
+/choco_package/*.nupkg
--- a/Cargo.lock
+++ b/Cargo.lock
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "feroxbuster"
-version = "2.9.0"
+version = "2.10.1"
 authors = ["Ben 'epi' Risher (@epi052)"]
 license = "MIT"
 edition = "2021"
@@ -22,46 +22,57 @@ build = "build.rs"
 maintenance = { status = "actively-developed" }

 [build-dependencies]
-clap = { version = "4.1.8", features = ["wrap_help", "cargo"] }
-clap_complete = "4.1.4"
-regex = "1.5.5"
-lazy_static = "1.4.0"
-dirs = "4.0.0"
+clap = { version = "4.3", features = ["wrap_help", "cargo"] }
+clap_complete = "4.3"
+regex = "1.9"
+lazy_static = "1.4"
+dirs = "5.0"

 [dependencies]
-scraper = "0.15.0"
-futures = "0.3.26"
-tokio = { version = "1.26.0", features = ["full"] }
-tokio-util = { version = "0.7.7", features = ["codec"] }
-log = "0.4.17"
-env_logger = "0.10.0"
-reqwest = { version = "0.11.10", features = ["socks"] }
+scraper = "0.18"
+futures = "0.3"
+tokio = { version = "1.29", features = ["full"] }
+tokio-util = { version = "0.7", features = ["codec"] }
+log = "0.4"
+env_logger = "0.10"
+reqwest = { version = "0.11", features = ["socks", "native-tls-alpn"] }
 # uses feature unification to add 'serde' to reqwest::Url
-url = { version = "2.2.2", features = ["serde"] }
-serde_regex = "1.1.0"
-clap = { version = "4.1.8", features = ["wrap_help", "cargo"] }
-lazy_static = "1.4.0"
-toml = "0.7.2"
-serde = { version = "1.0.137", features = ["derive", "rc"] }
-serde_json = "1.0.94"
-uuid = { version = "1.3.0", features = ["v4"] }
-indicatif = "0.15"
-console = "0.15.2"
+url = { version = "2.4", features = ["serde"] }
+serde_regex = "1.1"
+clap = { version = "4.3", features = ["wrap_help", "cargo"] }
+lazy_static = "1.4"
+toml = "0.8"
+serde = { version = "1.0", features = ["derive", "rc"] }
+serde_json = "1.0"
+uuid = { version = "1.4", features = ["v4"] }
+# last known working version of indicatif; 0.17.5 has a bug that causes the 
+# scan menu to fail spectacularly 
+indicatif = { version = "0.17.3" }
+console = "0.15"
 openssl = { version = "0.10", features = ["vendored"] }
-dirs = "4.0.0"
-regex = "1.5.5"
-crossterm = "0.26.0"
-rlimit = "0.9.1"
-ctrlc = "3.2.2"
-anyhow = "1.0.69"
-leaky-bucket = "0.12.1"
-gaoya = "0.1.2"
+dirs = "5.0"
+regex = "1.9"
+crossterm = "0.27"
+rlimit = "0.10"
+ctrlc = "3.4"
+anyhow = "1.0"
+leaky-bucket = "1.0"
+gaoya = "0.2"
+# 0.37+ relies on the broken version of indicatif and forces
+# the broken version to be used regardless of the version
+# specified above 
+self_update = { version = "0.36", features = [
+    "archive-tar",
+    "compression-flate2",
+    "archive-zip",
+    "compression-zip-deflate",
+] }

 [dev-dependencies]
-tempfile = "3.3.0"
-httpmock = "0.6.6"
-assert_cmd = "2.0.4"
-predicates = "2.1.1"
+tempfile = "3.6"
+httpmock = "0.6"
+assert_cmd = "2.0"
+predicates = "3.0"

 [profile.release]
 lto = true
--- a/2
+++ b/2
@@ -1,6 +1,6 @@
 MIT License

-Copyright (c) 2020 epi
+Copyright (c) 2020-2023 epi

 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
--- a/Makefile.toml
+++ b/Makefile.toml
@@ -11,7 +11,7 @@ rm ferox-*.state
 # dependency management
 [tasks.upgrade-deps]
 command = "cargo"
-args = ["upgrade", "--exclude", "indicatif"]
+args = ["upgrade", "--to-lockfile", "--exclude", "indicatif", "self_update"]

 [tasks.update]
 command = "cargo"
--- a/README.md
+++ b/README.md
@@ -97,10 +97,21 @@ sudo apt update && sudo apt install -y feroxbuster

 #### Linux (32 and 64-bit) & MacOS

+Install to a particular directory
 ```
-curl -sL https://raw.githubusercontent.com/epi052/feroxbuster/master/install-nix.sh | bash
+curl -sL https://raw.githubusercontent.com/epi052/feroxbuster/main/install-nix.sh | bash -s $HOME/.local/bin
 ```

+Install to current working directory
+```
+curl -sL https://raw.githubusercontent.com/epi052/feroxbuster/main/install-nix.sh | bash
+```
+
+#### MacOS via Homebrew 
+
+```
+brew install feroxbuster
+```

 #### Windows x86_64

@@ -110,10 +121,22 @@ Expand-Archive .\feroxbuster.zip
 .\feroxbuster\feroxbuster.exe -V
 ```

+#### Windows via Chocolatey
+
+```
+choco install feroxbuster
+```
+
 #### All others 

 Please refer the the [documentation](https://epi052.github.io/feroxbuster-docs/docs/).

+### Updating feroxbuster (new in v2.9.1)
+
+```
+./feroxbuster --update
+```
+
 ## 🧰 Example Usage

 Here are a few brief examples to get you started.  Please note, feroxbuster can do a **lot more** than what's listed below.  As a result, there are **many more** examples, with **demonstration gifs** that highlight specific features, in the [documentation](https://epi052.github.io/feroxbuster-docs/docs/).
@@ -167,6 +190,8 @@ cat targets | ./feroxbuster --stdin --silent -s 200 301 302 --redirects -x js |
 ./feroxbuster -u http://127.1 --query token=0123456789ABCDEF
 ```

+
+
 ## 🚀 Documentation has **moved** 🚀  

 For realsies, there used to be over 1300 lines in this README, but it's all been moved to the [new documentation site](https://epi052.github.io/feroxbuster-docs/docs/). Go check it out! 
@@ -207,7 +232,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
      <td align="center" valign="top" width="14.28%"><a href="https://github.com/sbrun"><img src="https://avatars.githubusercontent.com/u/7712154?v=4?s=100" width="100px;" alt="Sophie Brun"/><br /><sub><b>Sophie Brun</b></sub></a><br /><a href="#infra-sbrun" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a></td>
      <td align="center" valign="top" width="14.28%"><a href="https://github.com/black-A"><img src="https://avatars.githubusercontent.com/u/30686803?v=4?s=100" width="100px;" alt="black-A"/><br /><sub><b>black-A</b></sub></a><br /><a href="#ideas-black-A" title="Ideas, Planning, & Feedback">🤔</a></td>
      <td align="center" valign="top" width="14.28%"><a href="https://github.com/dinosn"><img src="https://avatars.githubusercontent.com/u/3851678?v=4?s=100" width="100px;" alt="Nicolas Krassas"/><br /><sub><b>Nicolas Krassas</b></sub></a><br /><a href="#ideas-dinosn" title="Ideas, Planning, & Feedback">🤔</a></td>
-      <td align="center" valign="top" width="14.28%"><a href="https://github.com/N0ur5"><img src="https://avatars.githubusercontent.com/u/24260009?v=4?s=100" width="100px;" alt="N0ur5"/><br /><sub><b>N0ur5</b></sub></a><br /><a href="#ideas-N0ur5" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/N0ur5"><img src="https://avatars.githubusercontent.com/u/24260009?v=4?s=100" width="100px;" alt="N0ur5"/><br /><sub><b>N0ur5</b></sub></a><br /><a href="#ideas-N0ur5" title="Ideas, Planning, & Feedback">🤔</a> <a href="https://github.com/epi052/feroxbuster/issues?q=author%3AN0ur5" title="Bug reports">🐛</a></td>
    </tr>
    <tr>
      <td align="center" valign="top" width="14.28%"><a href="https://github.com/moscowchill"><img src="https://avatars.githubusercontent.com/u/72578879?v=4?s=100" width="100px;" alt="mchill"/><br /><sub><b>mchill</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3Amoscowchill" title="Bug reports">🐛</a></td>
@@ -257,7 +282,24 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
    <tr>
      <td align="center" valign="top" width="14.28%"><a href="https://github.com/xaeroborg"><img src="https://avatars.githubusercontent.com/u/33274680?v=4?s=100" width="100px;" alt="xaeroborg"/><br /><sub><b>xaeroborg</b></sub></a><br /><a href="#ideas-xaeroborg" title="Ideas, Planning, & Feedback">🤔</a></td>
      <td align="center" valign="top" width="14.28%"><a href="https://github.com/Luoooio"><img src="https://avatars.githubusercontent.com/u/26653157?v=4?s=100" width="100px;" alt="Luoooio"/><br /><sub><b>Luoooio</b></sub></a><br /><a href="#ideas-Luoooio" title="Ideas, Planning, & Feedback">🤔</a></td>
-      <td align="center" valign="top" width="14.28%"><a href="https://petruknisme.com"><img src="https://avatars.githubusercontent.com/u/6284204?v=4?s=100" width="100px;" alt="Aan"/><br /><sub><b>Aan</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/commits?author=aancw" title="Code">💻</a> <a href="#infra-aancw" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://petruknisme.com"><img src="https://avatars.githubusercontent.com/u/6284204?v=4?s=100" width="100px;" alt="Aan"/><br /><sub><b>Aan</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/commits?author=aancw" title="Code">💻</a> <a href="#infra-aancw" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#ideas-aancw" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/imBigo"><img src="https://avatars.githubusercontent.com/u/54672433?v=4?s=100" width="100px;" alt="Simon"/><br /><sub><b>Simon</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3AimBigo" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://acut3.github.io/"><img src="https://avatars.githubusercontent.com/u/17295243?v=4?s=100" width="100px;" alt="Nicolas Christin"/><br /><sub><b>Nicolas Christin</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3Aacut3" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/DrorDvash"><img src="https://avatars.githubusercontent.com/u/8413651?v=4?s=100" width="100px;" alt="DrDv"/><br /><sub><b>DrDv</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3ADrorDvash" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/aroly"><img src="https://avatars.githubusercontent.com/u/1257705?v=4?s=100" width="100px;" alt="Antoine Roly"/><br /><sub><b>Antoine Roly</b></sub></a><br /><a href="#ideas-aroly" title="Ideas, Planning, & Feedback">🤔</a></td>
+    </tr>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="http://lavafroth.is-a.dev"><img src="https://avatars.githubusercontent.com/u/107522312?v=4?s=100" width="100px;" alt="Himadri Bhattacharjee"/><br /><sub><b>Himadri Bhattacharjee</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/commits?author=lavafroth" title="Code">💻</a> <a href="#ideas-lavafroth" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/AkechiShiro"><img src="https://avatars.githubusercontent.com/u/14914796?v=4?s=100" width="100px;" alt="Samy Lahfa"/><br /><sub><b>Samy Lahfa</b></sub></a><br /><a href="#ideas-AkechiShiro" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/sectroyer"><img src="https://avatars.githubusercontent.com/u/6706818?v=4?s=100" width="100px;" alt="sectroyer"/><br /><sub><b>sectroyer</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3Asectroyer" title="Bug reports">🐛</a> <a href="#ideas-sectroyer" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://medium.com/@b3rm1nG"><img src="https://avatars.githubusercontent.com/u/19836003?v=4?s=100" width="100px;" alt="ktecv2000"/><br /><sub><b>ktecv2000</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3Aktecv2000" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="http://untrue.me"><img src="https://avatars.githubusercontent.com/u/56048157?v=4?s=100" width="100px;" alt="Andrea De Murtas"/><br /><sub><b>Andrea De Murtas</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/commits?author=andreademurtas" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/sawmj"><img src="https://avatars.githubusercontent.com/u/30024085?v=4?s=100" width="100px;" alt="sawmj"/><br /><sub><b>sawmj</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3Asawmj" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/devx00"><img src="https://avatars.githubusercontent.com/u/6897405?v=4?s=100" width="100px;" alt="Zach Hanson"/><br /><sub><b>Zach Hanson</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3Adevx00" title="Bug reports">🐛</a></td>
+    </tr>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/ocervell"><img src="https://avatars.githubusercontent.com/u/9629314?v=4?s=100" width="100px;" alt="Olivier Cervello"/><br /><sub><b>Olivier Cervello</b></sub></a><br /><a href="#ideas-ocervell" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/RavySena"><img src="https://avatars.githubusercontent.com/u/67729597?v=4?s=100" width="100px;" alt="RavySena"/><br /><sub><b>RavySena</b></sub></a><br /><a href="#ideas-RavySena" title="Ideas, Planning, & Feedback">🤔</a></td>
    </tr>
  </tbody>
 </table>
--- a/choco_package/feroxbuster.nuspec
+++ b/choco_package/feroxbuster.nuspec
@@ -0,0 +1,79 @@
+<?xml version="1.0" encoding="utf-8"?>
+<package xmlns="http://schemas.microsoft.com/packaging/2015/06/nuspec.xsd">
+  <metadata>
+    <id>feroxbuster</id>
+    <version>2.8.0</version>
+    <packageSourceUrl>https://github.com/epi052/feroxbuster/releases/</packageSourceUrl>
+    <owners>epi052</owners>
+    <title>feroxbuster (Install)</title>
+    <authors>epi052</authors>
+    <projectUrl>https://github.com/epi052/feroxbuster</projectUrl>
+    <iconUrl>https://rawcdn.githack.com/epi052/feroxbuster/2d381e7e057ce60c580b324dd36c9abaf30c2ec7/img/logo/logo.png</iconUrl>
+    <copyright>2020-2023</copyright>
+    <licenseUrl>https://github.com/epi052/feroxbuster/blob/main/LICENSE</licenseUrl>
+    <requireLicenseAcceptance>true</requireLicenseAcceptance>
+    <projectSourceUrl>https://github.com/epi052/feroxbuster</projectSourceUrl>
+    <docsUrl>https://epi052.github.io/feroxbuster-docs/docs/</docsUrl>
+    <!--<mailingListUrl></mailingListUrl>-->
+    <bugTrackerUrl>https://github.com/epi052/feroxbuster/issues</bugTrackerUrl>
+    <tags>content-discovery pentesting-tool url-bruteforcer</tags>
+    <summary>A simple, fast, recursive content discovery tool written in Rust</summary>
+    <description>
+    A simple, fast, recursive content discovery tool written in Rust
+[![Feroxbuster](https://github.com/epi052/feroxbuster/raw/main/img/logo/default-cropped.png)](https://github.com/epi052/feroxbuster)
+
+## What the heck is a ferox anyway?
+
+Ferox is short for Ferric Oxide. Ferric Oxide, simply put, is rust. The name rustbuster was taken, so I decided on a
+variation. 
+
+## What's it do tho?
+
+`feroxbuster` is a tool designed to perform [Forced Browsing](https://owasp.org/www-community/attacks/Forced_browsing).
+
+Forced browsing is an attack where the aim is to enumerate and access resources that are not referenced by the web
+application, but are still accessible by an attacker.
+
+`feroxbuster` uses brute force combined with a wordlist to search for unlinked content in target directories. These
+resources may store sensitive information about web applications and operational systems, such as source code,
+credentials, internal network addressing, etc...
+
+This attack is also known as Predictable Resource Location, File Enumeration, Directory Enumeration, and Resource
+Enumeration.
+
+## Quick Start
+
+This section will cover the minimum amount of information to get up and running with feroxbuster. Please refer the the [documentation](https://epi052.github.io/feroxbuster-docs/docs/), as it's much more comprehensive.
+
+### Installation
+
+There are quite a few other [installation methods](https://epi052.github.io/feroxbuster-docs/docs/installation/), but these snippets should cover the majority of users. 
+
+#### All others Docs
+
+Please refer the the [documentation](https://epi052.github.io/feroxbuster-docs/docs/).
+
+## Example Usage
+
+Here are a few brief examples to get you started.  Please note, feroxbuster can do a **lot more** than what's listed below.  As a result, there are **many more** examples, with **demonstration gifs** that highlight specific features, in the [documentation](https://epi052.github.io/feroxbuster-docs/docs/).
+
+### Multiple Values
+
+Options that take multiple values are very flexible. Consider the following ways of specifying extensions:
+
+```
+./feroxbuster -u http://127.1 -x pdf -x js,html -x php txt json,docx
+```
+
+The command above adds .pdf, .js, .html, .php, .txt, .json, and .docx to each url
+
+All of the methods above (multiple flags, space separated, comma separated, etc...) are valid and interchangeable. The
+same goes for urls, headers, status codes, queries, and size filters.
+</description>
+    <!-- <releaseNotes>__REPLACE_OR_REMOVE__MarkDown_Okay</releaseNotes> -->
+  </metadata>
+  <files>
+    <!-- this section controls what actually gets packaged into the Chocolatey package -->
+    <file src="tools\**" target="tools" />
+  </files>
+</package>
--- a/choco_package/legal/LICENSE.txt
+++ b/choco_package/legal/LICENSE.txt
@@ -0,0 +1,26 @@
+
+From: https://github.com/epi052/feroxbuster/blob/main/LICENSE
+
+LICENSE
+
+MIT License
+
+Copyright (c) 2020-2023 epi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
--- a/choco_package/legal/VERIFICATION.txt
+++ b/choco_package/legal/VERIFICATION.txt
@@ -0,0 +1,5 @@
+
+VERIFICATION
+
+checksum -t sha512 -f .\x86-windows-feroxbuster.exe.zip
+checksum -t sha512 -f .\x86_64-windows-feroxbuster.exe.zip
--- a/choco_package/tools/chocolateyinstall.ps1
+++ b/choco_package/tools/chocolateyinstall.ps1
@@ -0,0 +1,27 @@
+$ErrorActionPreference = 'Stop' 
+
+$toolsDir   = "$(Split-Path -parent $MyInvocation.MyCommand.Definition)"
+$version    = '2.8.0'
+$url        = "https://github.com/epi052/feroxbuster/releases/download/v$version/x86-windows-feroxbuster.exe.zip"
+$url64      = "https://github.com/epi052/feroxbuster/releases/download/v$version/x86_64-windows-feroxbuster.exe.zip"
+
+$packageArgs = @{
+  packageName   = $env:ChocolateyPackageName
+  unzipLocation = $toolsDir
+  fileType      = 'exe' #only one of these: exe, msi, msu
+  url           = $url
+  url64bit      = $url64
+  #file         = $fileLocation
+
+  softwareName  = 'feroxbuster*' 
+
+  # Checksums are now required as of 0.10.0.
+  # To determine checksums, you can get that from the original site if provided.
+  # You can also use checksum.exe (choco install checksum) and use it
+  # e.g. checksum -t sha256 -f path\to\file
+  checksum      = 'e5cac59c737260233903a17706a68bac11fe0d7a15169e1c5a9637cc221e7230fd6ddbfc1a7243833dde6472ad053c033449ca8338164654f7354363da54ba88'
+  checksumType  = 'sha512'
+  checksum64    = 'cce58d6eacef7e12c31076f5a00fee9742a4e3fdfc69d807d98736200e50469f77359978e137ecafd87b14460845c65c6808d1f8b23ae561f7e7c637e355dee3'
+  checksumType64= 'sha512'
+}
+Install-ChocolateyZipPackage @packageArgs # https://docs.chocolatey.org/en-us/create/functions/install-chocolateyzippackage
--- a/choco_package/tools/chocolateyuninstall.ps1
+++ b/choco_package/tools/chocolateyuninstall.ps1
@@ -0,0 +1,47 @@
+$ErrorActionPreference = 'Stop' # stop on all errors
+$packageArgs = @{
+  packageName   = $env:ChocolateyPackageName
+  softwareName  = 'feroxbuster*'  #part or all of the Display Name as you see it in Programs and Features. It should be enough to be unique
+  fileType      = 'exe' #only one of these: MSI or EXE (ignore MSU for now)
+}
+
+# Get-UninstallRegistryKey is new to 0.9.10, if supporting 0.9.9.x and below,
+# take a dependency on "chocolatey-core.extension" in your nuspec file.
+# This is only a fuzzy search if $softwareName includes '*'. Otherwise it is
+# exact. In the case of versions in key names, we recommend removing the version
+# and using '*'.
+[array]$key = Get-UninstallRegistryKey -SoftwareName $packageArgs['softwareName']
+
+if ($key.Count -eq 1) {
+  $key | % {
+    $packageArgs['file'] = "$($_.UninstallString)" #NOTE: You may need to split this if it contains spaces, see below
+
+    if ($packageArgs['fileType'] -eq 'MSI') {
+      # The Product Code GUID is all that should be passed for MSI, and very
+      # FIRST, because it comes directly after /x, which is already set in the
+      # Uninstall-ChocolateyPackage msiargs (facepalm).
+      $packageArgs['silentArgs'] = "$($_.PSChildName) $($packageArgs['silentArgs'])"
+
+      # Don't pass anything for file, it is ignored for msi (facepalm number 2)
+      # Alternatively if you need to pass a path to an msi, determine that and
+      # use it instead of the above in silentArgs, still very first
+      $packageArgs['file'] = ''
+    } else {
+      # NOTES:
+      # - You probably will need to sanitize $packageArgs['file'] as it comes from the registry and could be in a variety of fun but unusable formats
+      # - Split args from exe in $packageArgs['file'] and pass those args through $packageArgs['silentArgs'] or ignore them
+      # - Ensure you don't pass double quotes in $file (aka $packageArgs['file']) - otherwise you will get "Illegal characters in path when you attempt to run this"
+      # - Review the code for auto-uninstaller for all of the fun things it does in sanitizing - https://github.com/chocolatey/choco/blob/bfe351b7d10c798014efe4bfbb100b171db25099/src/chocolatey/infrastructure.app/services/AutomaticUninstallerService.cs#L142-L192
+    }
+
+    Uninstall-ChocolateyPackage @packageArgs
+  }
+} elseif ($key.Count -eq 0) {
+  Write-Warning "$packageName has already been uninstalled by other means."
+} elseif ($key.Count -gt 1) {
+  Write-Warning "$($key.Count) matches found!"
+  Write-Warning "To prevent accidental data loss, no programs will be uninstalled."
+  Write-Warning "Please alert package maintainer the following keys were matched:"
+  $key | % {Write-Warning "- $($_.DisplayName)"}
+}
+
--- a/ferox-config.toml.example
+++ b/ferox-config.toml.example
@@ -54,6 +54,9 @@
 # queries = [["name","value"], ["rick", "astley"]]
 # save_state = false
 # time_limit = "10m"
+# server_certs = ["/some/cert.pem", "/some/other/cert.pem"]
+# client_cert = "/some/client/cert.pem"
+# client_key = "/some/client/key.pem"

 # headers can be specified on multiple lines or as an inline table
 #
--- a/install-nix.sh
+++ b/install-nix.sh
@@ -13,13 +13,13 @@ LIN64_URL="$BASE_URL/$LIN64_ZIP"

 EMOJI_URL=https://gist.github.com/epi052/8196b550ea51d0907ad4b93751b1b57d/raw/6112c9f32ae07922983fdc549c54fd3fb9a38e4c/NotoColorEmoji.ttf

-echo "[+] Installing feroxbuster!"
+INSTALL_DIR="${1:-$(pwd)}"
+
+echo "[+] Installing feroxbuster to ${INSTALL_DIR}!"

 which unzip &>/dev/null
-if [ "$?" = "0" ]; then
-  echo "[+] unzip found"
-else
-  echo "[ ] unzip not found, exiting. "
+if [ "$?" != "0" ]; then
+  echo "[!] unzip not found, exiting. "
  exit -1
 fi

@@ -27,20 +27,20 @@ if [[ "$(uname)" == "Darwin" ]]; then
  echo "[=] Found MacOS, downloading from $MAC_URL"

  curl -sLO "$MAC_URL"
-  unzip -o "$MAC_ZIP" >/dev/null
+  unzip -o "$MAC_ZIP" -d "${INSTALL_DIR}" >/dev/null
  rm "$MAC_ZIP"
 elif [[ "$(expr substr $(uname -s) 1 5)" == "Linux" ]]; then
  if [[ $(getconf LONG_BIT) == 32 ]]; then
    echo "[=] Found 32-bit Linux, downloading from $LIN32_URL"

    curl -sLO "$LIN32_URL"
-    unzip -o "$LIN32_ZIP" >/dev/null
+    unzip -o "$LIN32_ZIP" -d "${INSTALL_DIR}" >/dev/null
    rm "$LIN32_ZIP"
  else
    echo "[=] Found 64-bit Linux, downloading from $LIN64_URL"

    curl -sLO "$LIN64_URL"
-    unzip -o "$LIN64_ZIP" >/dev/null
+    unzip -o "$LIN64_ZIP" -d "${INSTALL_DIR}" >/dev/null
    rm "$LIN64_ZIP"
  fi

@@ -60,6 +60,8 @@ elif [[ "$(expr substr $(uname -s) 1 5)" == "Linux" ]]; then
  fi
 fi

-chmod +x ./feroxbuster
+chmod +x "${INSTALL_DIR}/feroxbuster"

-echo "[+] Installed feroxbuster version $(./feroxbuster -V)"
+echo "[+] Installed feroxbuster"
+echo "  [-] path: ${INSTALL_DIR}/feroxbuster"
+echo "  [-] version: $(${INSTALL_DIR}/feroxbuster -V | awk '{print $2}')"
--- a/shell_completions/_feroxbuster
+++ b/shell_completions/_feroxbuster
@@ -18,61 +18,64 @@ _feroxbuster() {
 '-u+[The target URL (required, unless \[--stdin || --resume-from\] used)]:URL:_urls' \
 '--url=[The target URL (required, unless \[--stdin || --resume-from\] used)]:URL:_urls' \
 '(-u --url)--resume-from=[State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)]:STATE_FILE:_files' \
-'-p+[Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)]:PROXY:_urls' \
-'--proxy=[Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)]:PROXY:_urls' \
+'-p+[Proxy to use for requests (ex\: http(s)\://host\:port, socks5(h)\://host\:port)]:PROXY:_urls' \
+'--proxy=[Proxy to use for requests (ex\: http(s)\://host\:port, socks5(h)\://host\:port)]:PROXY:_urls' \
 '-P+[Send only unfiltered requests through a Replay Proxy, instead of all requests]:REPLAY_PROXY:_urls' \
 '--replay-proxy=[Send only unfiltered requests through a Replay Proxy, instead of all requests]:REPLAY_PROXY:_urls' \
-'*-R+[Status Codes to send through a Replay Proxy when found (default: --status-codes value)]:REPLAY_CODE: ' \
-'*--replay-codes=[Status Codes to send through a Replay Proxy when found (default: --status-codes value)]:REPLAY_CODE: ' \
-'-a+[Sets the User-Agent (default: feroxbuster/2.9.0)]:USER_AGENT: ' \
-'--user-agent=[Sets the User-Agent (default: feroxbuster/2.9.0)]:USER_AGENT: ' \
-'*-x+[File extension(s) to search for (ex: -x php -x pdf js)]:FILE_EXTENSION: ' \
-'*--extensions=[File extension(s) to search for (ex: -x php -x pdf js)]:FILE_EXTENSION: ' \
-'*-m+[Which HTTP request method(s) should be sent (default: GET)]:HTTP_METHODS: ' \
-'*--methods=[Which HTTP request method(s) should be sent (default: GET)]:HTTP_METHODS: ' \
-'--data=[Request'\''s Body; can read data from a file if input starts with an @ (ex: @post.bin)]:DATA: ' \
-'*-H+[Specify HTTP headers to be used in each request (ex: -H Header:val -H '\''stuff: things'\'')]:HEADER: ' \
-'*--headers=[Specify HTTP headers to be used in each request (ex: -H Header:val -H '\''stuff: things'\'')]:HEADER: ' \
-'*-b+[Specify HTTP cookies to be used in each request (ex: -b stuff=things)]:COOKIE: ' \
-'*--cookies=[Specify HTTP cookies to be used in each request (ex: -b stuff=things)]:COOKIE: ' \
-'*-Q+[Request'\''s URL query parameters (ex: -Q token=stuff -Q secret=key)]:QUERY: ' \
-'*--query=[Request'\''s URL query parameters (ex: -Q token=stuff -Q secret=key)]:QUERY: ' \
+'*-R+[Status Codes to send through a Replay Proxy when found (default\: --status-codes value)]:REPLAY_CODE: ' \
+'*--replay-codes=[Status Codes to send through a Replay Proxy when found (default\: --status-codes value)]:REPLAY_CODE: ' \
+'-a+[Sets the User-Agent (default\: feroxbuster/2.10.1)]:USER_AGENT: ' \
+'--user-agent=[Sets the User-Agent (default\: feroxbuster/2.10.1)]:USER_AGENT: ' \
+'*-x+[File extension(s) to search for (ex\: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex\: @ext.txt)]:FILE_EXTENSION: ' \
+'*--extensions=[File extension(s) to search for (ex\: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex\: @ext.txt)]:FILE_EXTENSION: ' \
+'*-m+[Which HTTP request method(s) should be sent (default\: GET)]:HTTP_METHODS: ' \
+'*--methods=[Which HTTP request method(s) should be sent (default\: GET)]:HTTP_METHODS: ' \
+'--data=[Request'\''s Body; can read data from a file if input starts with an @ (ex\: @post.bin)]:DATA: ' \
+'*-H+[Specify HTTP headers to be used in each request (ex\: -H Header\:val -H '\''stuff\: things'\'')]:HEADER: ' \
+'*--headers=[Specify HTTP headers to be used in each request (ex\: -H Header\:val -H '\''stuff\: things'\'')]:HEADER: ' \
+'*-b+[Specify HTTP cookies to be used in each request (ex\: -b stuff=things)]:COOKIE: ' \
+'*--cookies=[Specify HTTP cookies to be used in each request (ex\: -b stuff=things)]:COOKIE: ' \
+'*-Q+[Request'\''s URL query parameters (ex\: -Q token=stuff -Q secret=key)]:QUERY: ' \
+'*--query=[Request'\''s URL query parameters (ex\: -Q token=stuff -Q secret=key)]:QUERY: ' \
 '*--dont-scan=[URL(s) or Regex Pattern(s) to exclude from recursion/scans]:URL: ' \
-'*-S+[Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)]:SIZE: ' \
-'*--filter-size=[Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)]:SIZE: ' \
-'*-X+[Filter out messages via regular expression matching on the response'\''s body (ex: -X '\''^ignore me$'\'')]:REGEX: ' \
-'*--filter-regex=[Filter out messages via regular expression matching on the response'\''s body (ex: -X '\''^ignore me$'\'')]:REGEX: ' \
-'*-W+[Filter out messages of a particular word count (ex: -W 312 -W 91,82)]:WORDS: ' \
-'*--filter-words=[Filter out messages of a particular word count (ex: -W 312 -W 91,82)]:WORDS: ' \
-'*-N+[Filter out messages of a particular line count (ex: -N 20 -N 31,30)]:LINES: ' \
-'*--filter-lines=[Filter out messages of a particular line count (ex: -N 20 -N 31,30)]:LINES: ' \
-'(-s --status-codes)*-C+[Filter out status codes (deny list) (ex: -C 200 -C 401)]:STATUS_CODE: ' \
-'(-s --status-codes)*--filter-status=[Filter out status codes (deny list) (ex: -C 200 -C 401)]:STATUS_CODE: ' \
-'*--filter-similar-to=[Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)]:UNWANTED_PAGE:_urls' \
-'*-s+[Status Codes to include (allow list) (default: All Status Codes)]:STATUS_CODE: ' \
-'*--status-codes=[Status Codes to include (allow list) (default: All Status Codes)]:STATUS_CODE: ' \
-'-T+[Number of seconds before a client'\''s request times out (default: 7)]:SECONDS: ' \
-'--timeout=[Number of seconds before a client'\''s request times out (default: 7)]:SECONDS: ' \
-'-t+[Number of concurrent threads (default: 50)]:THREADS: ' \
-'--threads=[Number of concurrent threads (default: 50)]:THREADS: ' \
-'-d+[Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)]:RECURSION_DEPTH: ' \
-'--depth=[Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)]:RECURSION_DEPTH: ' \
-'-L+[Limit total number of concurrent scans (default: 0, i.e. no limit)]:SCAN_LIMIT: ' \
-'--scan-limit=[Limit total number of concurrent scans (default: 0, i.e. no limit)]:SCAN_LIMIT: ' \
+'*-S+[Filter out messages of a particular size (ex\: -S 5120 -S 4927,1970)]:SIZE: ' \
+'*--filter-size=[Filter out messages of a particular size (ex\: -S 5120 -S 4927,1970)]:SIZE: ' \
+'*-X+[Filter out messages via regular expression matching on the response'\''s body (ex\: -X '\''^ignore me\$'\'')]:REGEX: ' \
+'*--filter-regex=[Filter out messages via regular expression matching on the response'\''s body (ex\: -X '\''^ignore me\$'\'')]:REGEX: ' \
+'*-W+[Filter out messages of a particular word count (ex\: -W 312 -W 91,82)]:WORDS: ' \
+'*--filter-words=[Filter out messages of a particular word count (ex\: -W 312 -W 91,82)]:WORDS: ' \
+'*-N+[Filter out messages of a particular line count (ex\: -N 20 -N 31,30)]:LINES: ' \
+'*--filter-lines=[Filter out messages of a particular line count (ex\: -N 20 -N 31,30)]:LINES: ' \
+'(-s --status-codes)*-C+[Filter out status codes (deny list) (ex\: -C 200 -C 401)]:STATUS_CODE: ' \
+'(-s --status-codes)*--filter-status=[Filter out status codes (deny list) (ex\: -C 200 -C 401)]:STATUS_CODE: ' \
+'*--filter-similar-to=[Filter out pages that are similar to the given page (ex. --filter-similar-to http\://site.xyz/soft404)]:UNWANTED_PAGE:_urls' \
+'*-s+[Status Codes to include (allow list) (default\: All Status Codes)]:STATUS_CODE: ' \
+'*--status-codes=[Status Codes to include (allow list) (default\: All Status Codes)]:STATUS_CODE: ' \
+'-T+[Number of seconds before a client'\''s request times out (default\: 7)]:SECONDS: ' \
+'--timeout=[Number of seconds before a client'\''s request times out (default\: 7)]:SECONDS: ' \
+'--server-certs=[Add custom root certificate(s) for servers with unknown certificates]:PEM|DER:_files' \
+'--client-cert=[Add a PEM encoded certificate for mutual authentication (mTLS)]:PEM:_files' \
+'--client-key=[Add a PEM encoded private key for mutual authentication (mTLS)]:PEM:_files' \
+'-t+[Number of concurrent threads (default\: 50)]:THREADS: ' \
+'--threads=[Number of concurrent threads (default\: 50)]:THREADS: ' \
+'-d+[Maximum recursion depth, a depth of 0 is infinite recursion (default\: 4)]:RECURSION_DEPTH: ' \
+'--depth=[Maximum recursion depth, a depth of 0 is infinite recursion (default\: 4)]:RECURSION_DEPTH: ' \
+'-L+[Limit total number of concurrent scans (default\: 0, i.e. no limit)]:SCAN_LIMIT: ' \
+'--scan-limit=[Limit total number of concurrent scans (default\: 0, i.e. no limit)]:SCAN_LIMIT: ' \
 '--parallel=[Run parallel feroxbuster instances (one child process per url passed via stdin)]:PARALLEL_SCANS: ' \
-'(--auto-tune)--rate-limit=[Limit number of requests per second (per directory) (default: 0, i.e. no limit)]:RATE_LIMIT: ' \
-'--time-limit=[Limit total run time of all scans (ex: --time-limit 10m)]:TIME_SPEC: ' \
-'-w+[Path to the wordlist]:FILE:_files' \
-'--wordlist=[Path to the wordlist]:FILE:_files' \
+'(--auto-tune)--rate-limit=[Limit number of requests per second (per directory) (default\: 0, i.e. no limit)]:RATE_LIMIT: ' \
+'--time-limit=[Limit total run time of all scans (ex\: --time-limit 10m)]:TIME_SPEC: ' \
+'-w+[Path or URL of the wordlist]:FILE:_files' \
+'--wordlist=[Path or URL of the wordlist]:FILE:_files' \
 '*-I+[File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)]:FILE_EXTENSION: ' \
 '*--dont-collect=[File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)]:FILE_EXTENSION: ' \
 '-o+[Output file to write results to (use w/ --json for JSON entries)]:FILE:_files' \
 '--output=[Output file to write results to (use w/ --json for JSON entries)]:FILE:_files' \
 '--debug-log=[Output file to write log entries (use w/ --json for JSON entries)]:FILE:_files' \
 '(-u --url)--stdin[Read url(s) from STDIN]' \
-'(-p --proxy -k --insecure --burp-replay)--burp[Set --proxy to http://127.0.0.1:8080 and set --insecure to true]' \
-'(-P --replay-proxy -k --insecure)--burp-replay[Set --replay-proxy to http://127.0.0.1:8080 and set --insecure to true]' \
-'(--rate-limit --auto-bail)--smart[Set --extract-links, --auto-tune, --collect-words, and --collect-backups to true]' \
+'(-p --proxy -k --insecure --burp-replay)--burp[Set --proxy to http\://127.0.0.1\:8080 and set --insecure to true]' \
+'(-P --replay-proxy -k --insecure)--burp-replay[Set --replay-proxy to http\://127.0.0.1\:8080 and set --insecure to true]' \
+'(--rate-limit --auto-bail)--smart[Set --auto-tune, --collect-words, and --collect-backups to true]' \
 '(--rate-limit --auto-bail)--thorough[Use the same settings as --smart and set --collect-extensions to true]' \
 '-A[Use a random User-Agent]' \
 '--random-agent[Use a random User-Agent]' \
@@ -85,8 +88,9 @@ _feroxbuster() {
 '-n[Do not scan recursively]' \
 '--no-recursion[Do not scan recursively]' \
 '(-n --no-recursion)--force-recursion[Force recursion attempts on all '\''found'\'' endpoints (still respects recursion depth)]' \
-'-e[Extract links from response body (html, javascript, etc...); make new requests based on findings]' \
-'--extract-links[Extract links from response body (html, javascript, etc...); make new requests based on findings]' \
+'-e[Extract links from response body (html, javascript, etc...); make new requests based on findings (default\: true)]' \
+'--extract-links[Extract links from response body (html, javascript, etc...); make new requests based on findings (default\: true)]' \
+'--dont-extract-links[Don'\''t extract links from response body (html, javascript, etc...)]' \
 '(--auto-bail)--auto-tune[Automatically lower scan rate when an excessive amount of errors are encountered]' \
 '--auto-bail[Automatically stop scanning when an excessive amount of errors are encountered]' \
 '-D[Don'\''t auto-filter wildcard responses]' \
@@ -99,11 +103,13 @@ _feroxbuster() {
 '--collect-words[Automatically discover important words from within responses and add them to the wordlist]' \
 '(--silent)*-v[Increase verbosity level (use -vv or more for greater effect. \[CAUTION\] 4 -v'\''s is probably too much)]' \
 '(--silent)*--verbosity[Increase verbosity level (use -vv or more for greater effect. \[CAUTION\] 4 -v'\''s is probably too much)]' \
-'(-q --quiet)--silent[Only print URLs + turn off logging (good for piping a list of urls to other commands)]' \
+'(-q --quiet)--silent[Only print URLs (or JSON w/ --json) + turn off logging (good for piping a list of urls to other commands)]' \
 '-q[Hide progress bars and banner (good for tmux windows w/ notifications)]' \
 '--quiet[Hide progress bars and banner (good for tmux windows w/ notifications)]' \
 '--json[Emit JSON logs to --output and --debug-log instead of normal text]' \
 '--no-state[Disable state output file (*.state)]' \
+'-U[Update feroxbuster to the latest version]' \
+'--update[Update feroxbuster to the latest version]' \
 '-h[Print help (see more with '\''--help'\'')]' \
 '--help[Print help (see more with '\''--help'\'')]' \
 '-V[Print version]' \
--- a/shell_completions/_feroxbuster.ps1
+++ b/shell_completions/_feroxbuster.ps1
@@ -26,51 +26,54 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
            [CompletionResult]::new('--resume-from', 'resume-from', [CompletionResultType]::ParameterName, 'State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)')
            [CompletionResult]::new('-p', 'p', [CompletionResultType]::ParameterName, 'Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)')
            [CompletionResult]::new('--proxy', 'proxy', [CompletionResultType]::ParameterName, 'Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)')
-            [CompletionResult]::new('-P', 'P', [CompletionResultType]::ParameterName, 'Send only unfiltered requests through a Replay Proxy, instead of all requests')
+            [CompletionResult]::new('-P', 'P ', [CompletionResultType]::ParameterName, 'Send only unfiltered requests through a Replay Proxy, instead of all requests')
            [CompletionResult]::new('--replay-proxy', 'replay-proxy', [CompletionResultType]::ParameterName, 'Send only unfiltered requests through a Replay Proxy, instead of all requests')
-            [CompletionResult]::new('-R', 'R', [CompletionResultType]::ParameterName, 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)')
+            [CompletionResult]::new('-R', 'R ', [CompletionResultType]::ParameterName, 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)')
            [CompletionResult]::new('--replay-codes', 'replay-codes', [CompletionResultType]::ParameterName, 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)')
-            [CompletionResult]::new('-a', 'a', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.9.0)')
-            [CompletionResult]::new('--user-agent', 'user-agent', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.9.0)')
-            [CompletionResult]::new('-x', 'x', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js)')
-            [CompletionResult]::new('--extensions', 'extensions', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js)')
+            [CompletionResult]::new('-a', 'a', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.10.1)')
+            [CompletionResult]::new('--user-agent', 'user-agent', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.10.1)')
+            [CompletionResult]::new('-x', 'x', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex: @ext.txt)')
+            [CompletionResult]::new('--extensions', 'extensions', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex: @ext.txt)')
            [CompletionResult]::new('-m', 'm', [CompletionResultType]::ParameterName, 'Which HTTP request method(s) should be sent (default: GET)')
            [CompletionResult]::new('--methods', 'methods', [CompletionResultType]::ParameterName, 'Which HTTP request method(s) should be sent (default: GET)')
            [CompletionResult]::new('--data', 'data', [CompletionResultType]::ParameterName, 'Request''s Body; can read data from a file if input starts with an @ (ex: @post.bin)')
-            [CompletionResult]::new('-H', 'H', [CompletionResultType]::ParameterName, 'Specify HTTP headers to be used in each request (ex: -H Header:val -H ''stuff: things'')')
+            [CompletionResult]::new('-H', 'H ', [CompletionResultType]::ParameterName, 'Specify HTTP headers to be used in each request (ex: -H Header:val -H ''stuff: things'')')
            [CompletionResult]::new('--headers', 'headers', [CompletionResultType]::ParameterName, 'Specify HTTP headers to be used in each request (ex: -H Header:val -H ''stuff: things'')')
            [CompletionResult]::new('-b', 'b', [CompletionResultType]::ParameterName, 'Specify HTTP cookies to be used in each request (ex: -b stuff=things)')
            [CompletionResult]::new('--cookies', 'cookies', [CompletionResultType]::ParameterName, 'Specify HTTP cookies to be used in each request (ex: -b stuff=things)')
-            [CompletionResult]::new('-Q', 'Q', [CompletionResultType]::ParameterName, 'Request''s URL query parameters (ex: -Q token=stuff -Q secret=key)')
+            [CompletionResult]::new('-Q', 'Q ', [CompletionResultType]::ParameterName, 'Request''s URL query parameters (ex: -Q token=stuff -Q secret=key)')
            [CompletionResult]::new('--query', 'query', [CompletionResultType]::ParameterName, 'Request''s URL query parameters (ex: -Q token=stuff -Q secret=key)')
            [CompletionResult]::new('--dont-scan', 'dont-scan', [CompletionResultType]::ParameterName, 'URL(s) or Regex Pattern(s) to exclude from recursion/scans')
-            [CompletionResult]::new('-S', 'S', [CompletionResultType]::ParameterName, 'Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)')
+            [CompletionResult]::new('-S', 'S ', [CompletionResultType]::ParameterName, 'Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)')
            [CompletionResult]::new('--filter-size', 'filter-size', [CompletionResultType]::ParameterName, 'Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)')
-            [CompletionResult]::new('-X', 'X', [CompletionResultType]::ParameterName, 'Filter out messages via regular expression matching on the response''s body (ex: -X ''^ignore me$'')')
+            [CompletionResult]::new('-X', 'X ', [CompletionResultType]::ParameterName, 'Filter out messages via regular expression matching on the response''s body (ex: -X ''^ignore me$'')')
            [CompletionResult]::new('--filter-regex', 'filter-regex', [CompletionResultType]::ParameterName, 'Filter out messages via regular expression matching on the response''s body (ex: -X ''^ignore me$'')')
-            [CompletionResult]::new('-W', 'W', [CompletionResultType]::ParameterName, 'Filter out messages of a particular word count (ex: -W 312 -W 91,82)')
+            [CompletionResult]::new('-W', 'W ', [CompletionResultType]::ParameterName, 'Filter out messages of a particular word count (ex: -W 312 -W 91,82)')
            [CompletionResult]::new('--filter-words', 'filter-words', [CompletionResultType]::ParameterName, 'Filter out messages of a particular word count (ex: -W 312 -W 91,82)')
-            [CompletionResult]::new('-N', 'N', [CompletionResultType]::ParameterName, 'Filter out messages of a particular line count (ex: -N 20 -N 31,30)')
+            [CompletionResult]::new('-N', 'N ', [CompletionResultType]::ParameterName, 'Filter out messages of a particular line count (ex: -N 20 -N 31,30)')
            [CompletionResult]::new('--filter-lines', 'filter-lines', [CompletionResultType]::ParameterName, 'Filter out messages of a particular line count (ex: -N 20 -N 31,30)')
-            [CompletionResult]::new('-C', 'C', [CompletionResultType]::ParameterName, 'Filter out status codes (deny list) (ex: -C 200 -C 401)')
+            [CompletionResult]::new('-C', 'C ', [CompletionResultType]::ParameterName, 'Filter out status codes (deny list) (ex: -C 200 -C 401)')
            [CompletionResult]::new('--filter-status', 'filter-status', [CompletionResultType]::ParameterName, 'Filter out status codes (deny list) (ex: -C 200 -C 401)')
            [CompletionResult]::new('--filter-similar-to', 'filter-similar-to', [CompletionResultType]::ParameterName, 'Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)')
            [CompletionResult]::new('-s', 's', [CompletionResultType]::ParameterName, 'Status Codes to include (allow list) (default: All Status Codes)')
            [CompletionResult]::new('--status-codes', 'status-codes', [CompletionResultType]::ParameterName, 'Status Codes to include (allow list) (default: All Status Codes)')
-            [CompletionResult]::new('-T', 'T', [CompletionResultType]::ParameterName, 'Number of seconds before a client''s request times out (default: 7)')
+            [CompletionResult]::new('-T', 'T ', [CompletionResultType]::ParameterName, 'Number of seconds before a client''s request times out (default: 7)')
            [CompletionResult]::new('--timeout', 'timeout', [CompletionResultType]::ParameterName, 'Number of seconds before a client''s request times out (default: 7)')
+            [CompletionResult]::new('--server-certs', 'server-certs', [CompletionResultType]::ParameterName, 'Add custom root certificate(s) for servers with unknown certificates')
+            [CompletionResult]::new('--client-cert', 'client-cert', [CompletionResultType]::ParameterName, 'Add a PEM encoded certificate for mutual authentication (mTLS)')
+            [CompletionResult]::new('--client-key', 'client-key', [CompletionResultType]::ParameterName, 'Add a PEM encoded private key for mutual authentication (mTLS)')
            [CompletionResult]::new('-t', 't', [CompletionResultType]::ParameterName, 'Number of concurrent threads (default: 50)')
            [CompletionResult]::new('--threads', 'threads', [CompletionResultType]::ParameterName, 'Number of concurrent threads (default: 50)')
            [CompletionResult]::new('-d', 'd', [CompletionResultType]::ParameterName, 'Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)')
            [CompletionResult]::new('--depth', 'depth', [CompletionResultType]::ParameterName, 'Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)')
-            [CompletionResult]::new('-L', 'L', [CompletionResultType]::ParameterName, 'Limit total number of concurrent scans (default: 0, i.e. no limit)')
+            [CompletionResult]::new('-L', 'L ', [CompletionResultType]::ParameterName, 'Limit total number of concurrent scans (default: 0, i.e. no limit)')
            [CompletionResult]::new('--scan-limit', 'scan-limit', [CompletionResultType]::ParameterName, 'Limit total number of concurrent scans (default: 0, i.e. no limit)')
            [CompletionResult]::new('--parallel', 'parallel', [CompletionResultType]::ParameterName, 'Run parallel feroxbuster instances (one child process per url passed via stdin)')
            [CompletionResult]::new('--rate-limit', 'rate-limit', [CompletionResultType]::ParameterName, 'Limit number of requests per second (per directory) (default: 0, i.e. no limit)')
            [CompletionResult]::new('--time-limit', 'time-limit', [CompletionResultType]::ParameterName, 'Limit total run time of all scans (ex: --time-limit 10m)')
-            [CompletionResult]::new('-w', 'w', [CompletionResultType]::ParameterName, 'Path to the wordlist')
-            [CompletionResult]::new('--wordlist', 'wordlist', [CompletionResultType]::ParameterName, 'Path to the wordlist')
-            [CompletionResult]::new('-I', 'I', [CompletionResultType]::ParameterName, 'File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)')
+            [CompletionResult]::new('-w', 'w', [CompletionResultType]::ParameterName, 'Path or URL of the wordlist')
+            [CompletionResult]::new('--wordlist', 'wordlist', [CompletionResultType]::ParameterName, 'Path or URL of the wordlist')
+            [CompletionResult]::new('-I', 'I ', [CompletionResultType]::ParameterName, 'File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)')
            [CompletionResult]::new('--dont-collect', 'dont-collect', [CompletionResultType]::ParameterName, 'File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)')
            [CompletionResult]::new('-o', 'o', [CompletionResultType]::ParameterName, 'Output file to write results to (use w/ --json for JSON entries)')
            [CompletionResult]::new('--output', 'output', [CompletionResultType]::ParameterName, 'Output file to write results to (use w/ --json for JSON entries)')
@@ -78,9 +81,9 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
            [CompletionResult]::new('--stdin', 'stdin', [CompletionResultType]::ParameterName, 'Read url(s) from STDIN')
            [CompletionResult]::new('--burp', 'burp', [CompletionResultType]::ParameterName, 'Set --proxy to http://127.0.0.1:8080 and set --insecure to true')
            [CompletionResult]::new('--burp-replay', 'burp-replay', [CompletionResultType]::ParameterName, 'Set --replay-proxy to http://127.0.0.1:8080 and set --insecure to true')
-            [CompletionResult]::new('--smart', 'smart', [CompletionResultType]::ParameterName, 'Set --extract-links, --auto-tune, --collect-words, and --collect-backups to true')
+            [CompletionResult]::new('--smart', 'smart', [CompletionResultType]::ParameterName, 'Set --auto-tune, --collect-words, and --collect-backups to true')
            [CompletionResult]::new('--thorough', 'thorough', [CompletionResultType]::ParameterName, 'Use the same settings as --smart and set --collect-extensions to true')
-            [CompletionResult]::new('-A', 'A', [CompletionResultType]::ParameterName, 'Use a random User-Agent')
+            [CompletionResult]::new('-A', 'A ', [CompletionResultType]::ParameterName, 'Use a random User-Agent')
            [CompletionResult]::new('--random-agent', 'random-agent', [CompletionResultType]::ParameterName, 'Use a random User-Agent')
            [CompletionResult]::new('-f', 'f', [CompletionResultType]::ParameterName, 'Append / to each request''s URL')
            [CompletionResult]::new('--add-slash', 'add-slash', [CompletionResultType]::ParameterName, 'Append / to each request''s URL')
@@ -91,28 +94,31 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
            [CompletionResult]::new('-n', 'n', [CompletionResultType]::ParameterName, 'Do not scan recursively')
            [CompletionResult]::new('--no-recursion', 'no-recursion', [CompletionResultType]::ParameterName, 'Do not scan recursively')
            [CompletionResult]::new('--force-recursion', 'force-recursion', [CompletionResultType]::ParameterName, 'Force recursion attempts on all ''found'' endpoints (still respects recursion depth)')
-            [CompletionResult]::new('-e', 'e', [CompletionResultType]::ParameterName, 'Extract links from response body (html, javascript, etc...); make new requests based on findings')
-            [CompletionResult]::new('--extract-links', 'extract-links', [CompletionResultType]::ParameterName, 'Extract links from response body (html, javascript, etc...); make new requests based on findings')
+            [CompletionResult]::new('-e', 'e', [CompletionResultType]::ParameterName, 'Extract links from response body (html, javascript, etc...); make new requests based on findings (default: true)')
+            [CompletionResult]::new('--extract-links', 'extract-links', [CompletionResultType]::ParameterName, 'Extract links from response body (html, javascript, etc...); make new requests based on findings (default: true)')
+            [CompletionResult]::new('--dont-extract-links', 'dont-extract-links', [CompletionResultType]::ParameterName, 'Don''t extract links from response body (html, javascript, etc...)')
            [CompletionResult]::new('--auto-tune', 'auto-tune', [CompletionResultType]::ParameterName, 'Automatically lower scan rate when an excessive amount of errors are encountered')
            [CompletionResult]::new('--auto-bail', 'auto-bail', [CompletionResultType]::ParameterName, 'Automatically stop scanning when an excessive amount of errors are encountered')
-            [CompletionResult]::new('-D', 'D', [CompletionResultType]::ParameterName, 'Don''t auto-filter wildcard responses')
+            [CompletionResult]::new('-D', 'D ', [CompletionResultType]::ParameterName, 'Don''t auto-filter wildcard responses')
            [CompletionResult]::new('--dont-filter', 'dont-filter', [CompletionResultType]::ParameterName, 'Don''t auto-filter wildcard responses')
-            [CompletionResult]::new('-E', 'E', [CompletionResultType]::ParameterName, 'Automatically discover extensions and add them to --extensions (unless they''re in --dont-collect)')
+            [CompletionResult]::new('-E', 'E ', [CompletionResultType]::ParameterName, 'Automatically discover extensions and add them to --extensions (unless they''re in --dont-collect)')
            [CompletionResult]::new('--collect-extensions', 'collect-extensions', [CompletionResultType]::ParameterName, 'Automatically discover extensions and add them to --extensions (unless they''re in --dont-collect)')
-            [CompletionResult]::new('-B', 'B', [CompletionResultType]::ParameterName, 'Automatically request likely backup extensions for "found" urls')
+            [CompletionResult]::new('-B', 'B ', [CompletionResultType]::ParameterName, 'Automatically request likely backup extensions for "found" urls')
            [CompletionResult]::new('--collect-backups', 'collect-backups', [CompletionResultType]::ParameterName, 'Automatically request likely backup extensions for "found" urls')
            [CompletionResult]::new('-g', 'g', [CompletionResultType]::ParameterName, 'Automatically discover important words from within responses and add them to the wordlist')
            [CompletionResult]::new('--collect-words', 'collect-words', [CompletionResultType]::ParameterName, 'Automatically discover important words from within responses and add them to the wordlist')
            [CompletionResult]::new('-v', 'v', [CompletionResultType]::ParameterName, 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v''s is probably too much)')
            [CompletionResult]::new('--verbosity', 'verbosity', [CompletionResultType]::ParameterName, 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v''s is probably too much)')
-            [CompletionResult]::new('--silent', 'silent', [CompletionResultType]::ParameterName, 'Only print URLs + turn off logging (good for piping a list of urls to other commands)')
+            [CompletionResult]::new('--silent', 'silent', [CompletionResultType]::ParameterName, 'Only print URLs (or JSON w/ --json) + turn off logging (good for piping a list of urls to other commands)')
            [CompletionResult]::new('-q', 'q', [CompletionResultType]::ParameterName, 'Hide progress bars and banner (good for tmux windows w/ notifications)')
            [CompletionResult]::new('--quiet', 'quiet', [CompletionResultType]::ParameterName, 'Hide progress bars and banner (good for tmux windows w/ notifications)')
            [CompletionResult]::new('--json', 'json', [CompletionResultType]::ParameterName, 'Emit JSON logs to --output and --debug-log instead of normal text')
            [CompletionResult]::new('--no-state', 'no-state', [CompletionResultType]::ParameterName, 'Disable state output file (*.state)')
+            [CompletionResult]::new('-U', 'U ', [CompletionResultType]::ParameterName, 'Update feroxbuster to the latest version')
+            [CompletionResult]::new('--update', 'update', [CompletionResultType]::ParameterName, 'Update feroxbuster to the latest version')
            [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
            [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
-            [CompletionResult]::new('-V', 'V', [CompletionResultType]::ParameterName, 'Print version')
+            [CompletionResult]::new('-V', 'V ', [CompletionResultType]::ParameterName, 'Print version')
            [CompletionResult]::new('--version', 'version', [CompletionResultType]::ParameterName, 'Print version')
            break
        }
--- a/shell_completions/feroxbuster.bash
+++ b/shell_completions/feroxbuster.bash
@@ -19,7 +19,7 @@ _feroxbuster() {

    case "${cmd}" in
        feroxbuster)
-            opts="-u -p -P -R -a -A -x -m -H -b -Q -f -S -X -W -N -C -s -T -r -k -t -n -d -e -L -w -D -E -B -g -I -v -q -o -h -V --url --stdin --resume-from --burp --burp-replay --smart --thorough --proxy --replay-proxy --replay-codes --user-agent --random-agent --extensions --methods --data --headers --cookies --query --add-slash --dont-scan --filter-size --filter-regex --filter-words --filter-lines --filter-status --filter-similar-to --status-codes --timeout --redirects --insecure --threads --no-recursion --depth --force-recursion --extract-links --scan-limit --parallel --rate-limit --time-limit --wordlist --auto-tune --auto-bail --dont-filter --collect-extensions --collect-backups --collect-words --dont-collect --verbosity --silent --quiet --json --output --debug-log --no-state --help --version"
+            opts="-u -p -P -R -a -A -x -m -H -b -Q -f -S -X -W -N -C -s -T -r -k -t -n -d -e -L -w -D -E -B -g -I -v -q -o -U -h -V --url --stdin --resume-from --burp --burp-replay --smart --thorough --proxy --replay-proxy --replay-codes --user-agent --random-agent --extensions --methods --data --headers --cookies --query --add-slash --dont-scan --filter-size --filter-regex --filter-words --filter-lines --filter-status --filter-similar-to --status-codes --timeout --redirects --insecure --server-certs --client-cert --client-key --threads --no-recursion --depth --force-recursion --extract-links --dont-extract-links --scan-limit --parallel --rate-limit --time-limit --wordlist --auto-tune --auto-bail --dont-filter --collect-extensions --collect-backups --collect-words --dont-collect --verbosity --silent --quiet --json --output --debug-log --no-state --update --help --version"
            if [[ ${cur} == -* || ${COMP_CWORD} -eq 1 ]] ; then
                COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                return 0
@@ -177,6 +177,18 @@ _feroxbuster() {
                    COMPREPLY=($(compgen -f "${cur}"))
                    return 0
                    ;;
+                --server-certs)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --client-cert)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                --client-key)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
                --threads)
                    COMPREPLY=($(compgen -f "${cur}"))
                    return 0
--- a/shell_completions/feroxbuster.elv
+++ b/shell_completions/feroxbuster.elv
@@ -27,10 +27,10 @@ set edit:completion:arg-completer[feroxbuster] = {|@words|
            cand --replay-proxy 'Send only unfiltered requests through a Replay Proxy, instead of all requests'
            cand -R 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)'
            cand --replay-codes 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)'
-            cand -a 'Sets the User-Agent (default: feroxbuster/2.9.0)'
-            cand --user-agent 'Sets the User-Agent (default: feroxbuster/2.9.0)'
-            cand -x 'File extension(s) to search for (ex: -x php -x pdf js)'
-            cand --extensions 'File extension(s) to search for (ex: -x php -x pdf js)'
+            cand -a 'Sets the User-Agent (default: feroxbuster/2.10.1)'
+            cand --user-agent 'Sets the User-Agent (default: feroxbuster/2.10.1)'
+            cand -x 'File extension(s) to search for (ex: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex: @ext.txt)'
+            cand --extensions 'File extension(s) to search for (ex: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex: @ext.txt)'
            cand -m 'Which HTTP request method(s) should be sent (default: GET)'
            cand --methods 'Which HTTP request method(s) should be sent (default: GET)'
            cand --data 'Request''s Body; can read data from a file if input starts with an @ (ex: @post.bin)'
@@ -56,6 +56,9 @@ set edit:completion:arg-completer[feroxbuster] = {|@words|
            cand --status-codes 'Status Codes to include (allow list) (default: All Status Codes)'
            cand -T 'Number of seconds before a client''s request times out (default: 7)'
            cand --timeout 'Number of seconds before a client''s request times out (default: 7)'
+            cand --server-certs 'Add custom root certificate(s) for servers with unknown certificates'
+            cand --client-cert 'Add a PEM encoded certificate for mutual authentication (mTLS)'
+            cand --client-key 'Add a PEM encoded private key for mutual authentication (mTLS)'
            cand -t 'Number of concurrent threads (default: 50)'
            cand --threads 'Number of concurrent threads (default: 50)'
            cand -d 'Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)'
@@ -65,8 +68,8 @@ set edit:completion:arg-completer[feroxbuster] = {|@words|
            cand --parallel 'Run parallel feroxbuster instances (one child process per url passed via stdin)'
            cand --rate-limit 'Limit number of requests per second (per directory) (default: 0, i.e. no limit)'
            cand --time-limit 'Limit total run time of all scans (ex: --time-limit 10m)'
-            cand -w 'Path to the wordlist'
-            cand --wordlist 'Path to the wordlist'
+            cand -w 'Path or URL of the wordlist'
+            cand --wordlist 'Path or URL of the wordlist'
            cand -I 'File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)'
            cand --dont-collect 'File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)'
            cand -o 'Output file to write results to (use w/ --json for JSON entries)'
@@ -75,7 +78,7 @@ set edit:completion:arg-completer[feroxbuster] = {|@words|
            cand --stdin 'Read url(s) from STDIN'
            cand --burp 'Set --proxy to http://127.0.0.1:8080 and set --insecure to true'
            cand --burp-replay 'Set --replay-proxy to http://127.0.0.1:8080 and set --insecure to true'
-            cand --smart 'Set --extract-links, --auto-tune, --collect-words, and --collect-backups to true'
+            cand --smart 'Set --auto-tune, --collect-words, and --collect-backups to true'
            cand --thorough 'Use the same settings as --smart and set --collect-extensions to true'
            cand -A 'Use a random User-Agent'
            cand --random-agent 'Use a random User-Agent'
@@ -88,8 +91,9 @@ set edit:completion:arg-completer[feroxbuster] = {|@words|
            cand -n 'Do not scan recursively'
            cand --no-recursion 'Do not scan recursively'
            cand --force-recursion 'Force recursion attempts on all ''found'' endpoints (still respects recursion depth)'
-            cand -e 'Extract links from response body (html, javascript, etc...); make new requests based on findings'
-            cand --extract-links 'Extract links from response body (html, javascript, etc...); make new requests based on findings'
+            cand -e 'Extract links from response body (html, javascript, etc...); make new requests based on findings (default: true)'
+            cand --extract-links 'Extract links from response body (html, javascript, etc...); make new requests based on findings (default: true)'
+            cand --dont-extract-links 'Don''t extract links from response body (html, javascript, etc...)'
            cand --auto-tune 'Automatically lower scan rate when an excessive amount of errors are encountered'
            cand --auto-bail 'Automatically stop scanning when an excessive amount of errors are encountered'
            cand -D 'Don''t auto-filter wildcard responses'
@@ -102,11 +106,13 @@ set edit:completion:arg-completer[feroxbuster] = {|@words|
            cand --collect-words 'Automatically discover important words from within responses and add them to the wordlist'
            cand -v 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v''s is probably too much)'
            cand --verbosity 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v''s is probably too much)'
-            cand --silent 'Only print URLs + turn off logging (good for piping a list of urls to other commands)'
+            cand --silent 'Only print URLs (or JSON w/ --json) + turn off logging (good for piping a list of urls to other commands)'
            cand -q 'Hide progress bars and banner (good for tmux windows w/ notifications)'
            cand --quiet 'Hide progress bars and banner (good for tmux windows w/ notifications)'
            cand --json 'Emit JSON logs to --output and --debug-log instead of normal text'
            cand --no-state 'Disable state output file (*.state)'
+            cand -U 'Update feroxbuster to the latest version'
+            cand --update 'Update feroxbuster to the latest version'
            cand -h 'Print help (see more with ''--help'')'
            cand --help 'Print help (see more with ''--help'')'
            cand -V 'Print version'
--- a/src/banner/container.rs
+++ b/src/banner/container.rs
@@ -2,12 +2,11 @@ use super::entry::BannerEntry;
 use crate::{
    config::Configuration,
    event_handlers::Handles,
-    utils::{logged_request, status_colorizer},
+    utils::{logged_request, parse_url_with_raw_path, status_colorizer},
    DEFAULT_IGNORED_EXTENSIONS, DEFAULT_METHOD, DEFAULT_STATUS_CODES, VERSION,
 };
 use anyhow::{bail, Result};
 use console::{style, Emoji};
-use reqwest::Url;
 use serde_json::Value;
 use std::{io::Write, sync::Arc};

@@ -59,6 +58,15 @@ pub struct Banner {
    /// represents Configuration.proxy
    proxy: BannerEntry,

+    /// represents Configuration.client_key
+    client_key: BannerEntry,
+
+    /// represents Configuration.client_cert
+    client_cert: BannerEntry,
+
+    /// represents Configuration.server_certs
+    server_certs: BannerEntry,
+
    /// represents Configuration.replay_proxy
    replay_proxy: BannerEntry,

@@ -323,6 +331,13 @@ impl Banner {
        let auto_bail = BannerEntry::new("🙅", "Auto Bail", &config.auto_bail.to_string());
        let cfg = BannerEntry::new("💉", "Config File", &config.config);
        let proxy = BannerEntry::new("💎", "Proxy", &config.proxy);
+        let server_certs = BannerEntry::new(
+            "🏅",
+            "Server Certificates",
+            &format!("[{}]", config.server_certs.join(", ")),
+        );
+        let client_cert = BannerEntry::new("🏅", "Client Certificate", &config.client_cert);
+        let client_key = BannerEntry::new("🔑", "Client Key", &config.client_key);
        let threads = BannerEntry::new("🚀", "Threads", &config.threads.to_string());
        let wordlist = BannerEntry::new("📖", "Wordlist", &config.wordlist);
        let timeout = BannerEntry::new("💥", "Timeout (secs)", &config.timeout.to_string());
@@ -402,6 +417,9 @@ impl Banner {
            auto_bail,
            auto_tune,
            proxy,
+            client_cert,
+            client_key,
+            server_certs,
            replay_codes,
            replay_proxy,
            headers,
@@ -478,7 +496,7 @@ by Ben "epi" Risher {}                 ver: {}"#,
    pub async fn check_for_updates(&mut self, url: &str, handles: Arc<Handles>) -> Result<()> {
        log::trace!("enter: needs_update({}, {:?})", url, handles);

-        let api_url = Url::parse(url)?;
+        let api_url = parse_url_with_raw_path(url)?;

        let result = logged_request(&api_url, DEFAULT_METHOD, None, handles.clone()).await?;
        let body = result.text().await?;
@@ -556,6 +574,18 @@ by Ben "epi" Risher {}                 ver: {}"#,
            writeln!(&mut writer, "{}", self.proxy)?;
        }

+        if !config.client_cert.is_empty() {
+            writeln!(&mut writer, "{}", self.client_cert)?;
+        }
+
+        if !config.client_key.is_empty() {
+            writeln!(&mut writer, "{}", self.client_key)?;
+        }
+
+        if !config.server_certs.is_empty() {
+            writeln!(&mut writer, "{}", self.server_certs)?;
+        }
+
        if !config.replay_proxy.is_empty() {
            // i include replay codes logic here because in config.rs, replay codes are set to the
            // value in status codes, meaning it's never empty
--- a/src/client.rs
+++ b/src/client.rs
@@ -1,19 +1,29 @@
-use anyhow::Result;
+use anyhow::{Context, Result};
 use reqwest::header::HeaderMap;
 use reqwest::{redirect::Policy, Client, Proxy};
 use std::collections::HashMap;
 use std::convert::TryInto;
+use std::path::Path;
 use std::time::Duration;

 /// Create and return an instance of [reqwest::Client](https://docs.rs/reqwest/latest/reqwest/struct.Client.html)
-pub fn initialize(
+/// For now, silence clippy for this one
+#[allow(clippy::too_many_arguments)]
+pub fn initialize<I>(
    timeout: u64,
    user_agent: &str,
    redirects: bool,
    insecure: bool,
    headers: &HashMap<String, String>,
    proxy: Option<&str>,
-) -> Result<Client> {
+    server_certs: I,
+    client_cert: Option<&str>,
+    client_key: Option<&str>,
+) -> Result<Client>
+where
+    I: IntoIterator,
+    I::Item: AsRef<Path> + std::fmt::Debug,
+{
    let policy = if redirects {
        Policy::limited(10)
    } else {
@@ -22,7 +32,7 @@ pub fn initialize(

    let header_map: HeaderMap = headers.try_into()?;

-    let client = Client::builder()
+    let mut client = Client::builder()
        .timeout(Duration::new(timeout, 0))
        .user_agent(user_agent)
        .danger_accept_invalid_certs(insecure)
@@ -34,10 +44,42 @@ pub fn initialize(
        if !some_proxy.is_empty() {
            // it's not an empty string; set the proxy
            let proxy_obj = Proxy::all(some_proxy)?;
-            return Ok(client.proxy(proxy_obj).build()?);
+            // just add the proxy to the client
+            // don't build and return it just yet
+            client = client.proxy(proxy_obj);
        }
    }

+    for cert_path in server_certs {
+        let buf = std::fs::read(&cert_path)?;
+
+        let cert = match reqwest::Certificate::from_pem(&buf) {
+            Ok(cert) => cert,
+            Err(err) => reqwest::Certificate::from_der(&buf).with_context(|| {
+                format!(
+                    "{:?} does not contain a valid PEM or DER certificate\n{}",
+                    &cert_path, err
+                )
+            })?,
+        };
+
+        client = client.add_root_certificate(cert);
+    }
+
+    if let (Some(cert_path), Some(key_path)) = (client_cert, client_key) {
+        let cert = std::fs::read(cert_path)?;
+        let key = std::fs::read(key_path)?;
+
+        let identity = reqwest::Identity::from_pkcs8_pem(&cert, &key).with_context(|| {
+            format!(
+                "either {} or {} are invalid; expecting PEM encoded certificate and key",
+                cert_path, key_path
+            )
+        })?;
+
+        client = client.identity(identity);
+    }
+
    Ok(client.build()?)
 }

@@ -50,7 +92,18 @@ mod tests {
    /// create client with a bad proxy, expect panic
    fn client_with_bad_proxy() {
        let headers = HashMap::new();
-        initialize(0, "stuff", true, false, &headers, Some("not a valid proxy")).unwrap();
+        initialize(
+            0,
+            "stuff",
+            true,
+            false,
+            &headers,
+            Some("not a valid proxy"),
+            Vec::<String>::new(),
+            None,
+            None,
+        )
+        .unwrap();
    }

    #[test]
@@ -58,6 +111,99 @@ mod tests {
    fn client_with_good_proxy() {
        let headers = HashMap::new();
        let proxy = "http://127.0.0.1:8080";
-        initialize(0, "stuff", true, true, &headers, Some(proxy)).unwrap();
+        initialize(
+            0,
+            "stuff",
+            true,
+            true,
+            &headers,
+            Some(proxy),
+            Vec::<String>::new(),
+            None,
+            None,
+        )
+        .unwrap();
+    }
+
+    #[test]
+    /// create client with a server cert in pem format, expect no error
+    fn client_with_valid_server_pem() {
+        let headers = HashMap::new();
+
+        initialize(
+            0,
+            "stuff",
+            true,
+            true,
+            &headers,
+            None,
+            vec!["tests/mutual-auth/certs/server/server.crt.1".to_string()],
+            None,
+            None,
+        )
+        .unwrap();
+    }
+
+    #[test]
+    /// create client with a server cert in der format, expect no error
+    fn client_with_valid_server_der() {
+        let headers = HashMap::new();
+
+        initialize(
+            0,
+            "stuff",
+            true,
+            true,
+            &headers,
+            None,
+            vec!["tests/mutual-auth/certs/server/server.der".to_string()],
+            None,
+            None,
+        )
+        .unwrap();
+    }
+
+    #[test]
+    /// create client with two server certs (pem and der), expect no error
+    fn client_with_valid_server_pem_and_der() {
+        let headers = HashMap::new();
+
+        println!("{}", std::env::current_dir().unwrap().display());
+
+        initialize(
+            0,
+            "stuff",
+            true,
+            true,
+            &headers,
+            None,
+            vec![
+                "tests/mutual-auth/certs/server/server.crt.1".to_string(),
+                "tests/mutual-auth/certs/server/server.der".to_string(),
+            ],
+            None,
+            None,
+        )
+        .unwrap();
+    }
+
+    /// create client with invalid certificate, expect panic
+    #[test]
+    #[should_panic]
+    fn client_with_invalid_server_cert() {
+        let headers = HashMap::new();
+
+        initialize(
+            0,
+            "stuff",
+            true,
+            true,
+            &headers,
+            None,
+            vec!["tests/mutual-auth/certs/client/client.key".to_string()],
+            None,
+            None,
+        )
+        .unwrap();
    }
 }
--- a/src/config/container.rs
+++ b/src/config/container.rs
@@ -1,11 +1,15 @@
 use super::utils::{
-    depth, ignored_extensions, methods, report_and_exit, save_state, serialized_type, status_codes,
-    threads, timeout, user_agent, wordlist, OutputLevel, RequesterPolicy,
+    depth, extract_links, ignored_extensions, methods, report_and_exit, save_state,
+    serialized_type, status_codes, threads, timeout, user_agent, wordlist, OutputLevel,
+    RequesterPolicy,
 };
 use crate::config::determine_output_level;
 use crate::config::utils::determine_requester_policy;
 use crate::{
-    client, parser, scan_manager::resume_scan, traits::FeroxSerialize, utils::fmt_err,
+    client, parser,
+    scan_manager::resume_scan,
+    traits::FeroxSerialize,
+    utils::{fmt_err, parse_url_with_raw_path},
    DEFAULT_CONFIG_NAME,
 };
 use anyhow::{anyhow, Context, Result};
@@ -100,6 +104,18 @@ pub struct Configuration {
    #[serde(default)]
    pub replay_proxy: String,

+    /// Path to a custom root certificate for connecting to servers with a self-signed certificate
+    #[serde(default)]
+    pub server_certs: Vec<String>,
+
+    /// Path to a client's PEM encoded X509 certificate used during mutual authentication
+    #[serde(default)]
+    pub client_cert: String,
+
+    /// Path to a client's PEM encoded PKSC #8 private key used during mutual authentication
+    #[serde(default)]
+    pub client_key: String,
+
    /// The target URL
    #[serde(default)]
    pub target_url: String,
@@ -214,7 +230,7 @@ pub struct Configuration {
    pub no_recursion: bool,

    /// Extract links from html/javscript
-    #[serde(default)]
+    #[serde(default = "extract_links")]
    pub extract_links: bool,

    /// Append / to each request
@@ -309,6 +325,10 @@ pub struct Configuration {
    /// override recursion logic to always attempt recursion, still respects --depth
    #[serde(default)]
    pub force_recursion: bool,
+
+    /// Auto update app feature
+    #[serde(skip)]
+    pub update_app: bool,
 }

 impl Default for Configuration {
@@ -316,14 +336,25 @@ impl Default for Configuration {
    fn default() -> Self {
        let timeout = timeout();
        let user_agent = user_agent();
-        let client = client::initialize(timeout, &user_agent, false, false, &HashMap::new(), None)
-            .expect("Could not build client");
+        let client = client::initialize(
+            timeout,
+            &user_agent,
+            false,
+            false,
+            &HashMap::new(),
+            None,
+            Vec::<String>::new(),
+            None,
+            None,
+        )
+        .expect("Could not build client");
        let replay_client = None;
        let status_codes = status_codes();
        let replay_codes = status_codes.clone();
        let kind = serialized_type();
        let output_level = OutputLevel::Default;
        let requester_policy = RequesterPolicy::Default;
+        let extract_links = extract_links();

        Configuration {
            kind,
@@ -332,6 +363,7 @@ impl Default for Configuration {
            user_agent,
            replay_codes,
            status_codes,
+            extract_links,
            replay_client,
            requester_policy,
            dont_filter: false,
@@ -351,14 +383,16 @@ impl Default for Configuration {
            insecure: false,
            redirects: false,
            no_recursion: false,
-            extract_links: false,
            random_agent: false,
            collect_extensions: false,
            collect_backups: false,
            collect_words: false,
            save_state: true,
            force_recursion: false,
+            update_app: false,
            proxy: String::new(),
+            client_cert: String::new(),
+            client_key: String::new(),
            config: String::new(),
            output: String::new(),
            debug_log: String::new(),
@@ -366,6 +400,7 @@ impl Default for Configuration {
            time_limit: String::new(),
            resume_from: String::new(),
            replay_proxy: String::new(),
+            server_certs: Vec::new(),
            queries: Vec::new(),
            extensions: Vec::new(),
            methods: methods(),
@@ -393,7 +428,7 @@ impl Configuration {
    ///
    /// - **timeout**: `5` seconds
    /// - **redirects**: `false`
-    /// - **extract-links**: `false`
+    /// - **extract_links**: `true`
    /// - **wordlist**: [`DEFAULT_WORDLIST`](constant.DEFAULT_WORDLIST.html)
    /// - **config**: `None`
    /// - **threads**: `50`
@@ -441,6 +476,7 @@ impl Configuration {
    /// - **time_limit**: `None` (no limit on length of scan imposed)
    /// - **replay_proxy**: `None` (no limit on concurrent scans imposed)
    /// - **replay_codes**: [`DEFAULT_RESPONSE_CODES`](constant.DEFAULT_RESPONSE_CODES.html)
+    /// - **update_app**: `false`
    ///
    /// After which, any values defined in a
    /// [ferox-config.toml](constant.DEFAULT_CONFIG_NAME.html) config file will override the
@@ -619,9 +655,27 @@ impl Configuration {
        }

        if let Some(arg) = args.get_many::<String>("extensions") {
-            config.extensions = arg
-                .map(|val| val.trim_start_matches('.').to_string())
-                .collect();
+            let mut extensions = Vec::<String>::new();
+            for ext in arg {
+                if let Some(stripped) = ext.strip_prefix('@') {
+                    let contents = read_to_string(stripped)
+                        .unwrap_or_else(|e| report_and_exit(&e.to_string()));
+                    let exts_from_file = contents.split('\n').filter_map(|s| {
+                        let trimmed = s.trim().trim_start_matches('.');
+
+                        if trimmed.is_empty() || trimmed.starts_with('#') {
+                            None
+                        } else {
+                            Some(trimmed.to_string())
+                        }
+                    });
+
+                    extensions.extend(exts_from_file);
+                } else {
+                    extensions.push(ext.trim().trim_start_matches('.').to_string());
+                }
+            }
+            config.extensions = extensions;
        }

        if let Some(arg) = args.get_many::<String>("dont_collect") {
@@ -665,7 +719,7 @@ impl Configuration {
            for denier in arg {
                // could be an absolute url or a regex, need to determine which and populate the
                // appropriate vector
-                match Url::parse(denier.trim_end_matches('/')) {
+                match parse_url_with_raw_path(denier.trim_end_matches('/')) {
                    Ok(absolute) => {
                        // denier is an absolute url and can be parsed as such
                        config.url_denylist.push(absolute);
@@ -740,7 +794,11 @@ impl Configuration {
            // if the line below is outside of the if, we'd overwrite true with
            // false if no --silent is used on the command line
            config.silent = true;
-            config.output_level = OutputLevel::Silent;
+            config.output_level = if config.json {
+                OutputLevel::SilentJSON
+            } else {
+                OutputLevel::Silent
+            };
        }

        if came_from_cli!(args, "quiet") {
@@ -801,11 +859,8 @@ impl Configuration {
            config.add_slash = true;
        }

-        if came_from_cli!(args, "extract_links")
-            || came_from_cli!(args, "smart")
-            || came_from_cli!(args, "thorough")
-        {
-            config.extract_links = true;
+        if came_from_cli!(args, "dont_extract_links") {
+            config.extract_links = false;
        }

        if came_from_cli!(args, "json") {
@@ -816,10 +871,16 @@ impl Configuration {
            config.force_recursion = true;
        }

+        if came_from_cli!(args, "update_app") {
+            config.update_app = true;
+        }
+
        ////
        // organizational breakpoint; all options below alter the Client configuration
        ////
        update_config_if_present!(&mut config.proxy, args, "proxy", String);
+        update_config_if_present!(&mut config.client_cert, args, "client_cert", String);
+        update_config_if_present!(&mut config.client_key, args, "client_key", String);
        update_config_if_present!(&mut config.replay_proxy, args, "replay_proxy", String);
        update_config_if_present!(&mut config.user_agent, args, "user_agent", String);
        update_config_with_num_type_if_present!(&mut config.timeout, args, "timeout", u64);
@@ -890,6 +951,12 @@ impl Configuration {
            }
        }

+        if let Some(certs) = args.get_many::<String>("server_certs") {
+            for val in certs {
+                config.server_certs.push(val.to_string());
+            }
+        }
+
        config
    }

@@ -897,35 +964,53 @@ impl Configuration {
    /// either the config file or command line arguments; if we have, we need to rebuild
    /// the client and store it in the config struct
    fn try_rebuild_clients(configuration: &mut Configuration) {
-        if !configuration.proxy.is_empty()
+        // check if the proxy and certificate fields are empty
+        // and parse them into Some or None variants ahead of time
+        // so we may use the is_some method on them instead of
+        // multiple initializations
+        let proxy = if configuration.proxy.is_empty() {
+            None
+        } else {
+            Some(configuration.proxy.as_str())
+        };
+
+        let server_certs = &configuration.server_certs;
+
+        let client_cert = if configuration.client_cert.is_empty() {
+            None
+        } else {
+            Some(configuration.client_cert.as_str())
+        };
+
+        let client_key = if configuration.client_key.is_empty() {
+            None
+        } else {
+            Some(configuration.client_key.as_str())
+        };
+
+        if proxy.is_some()
            || configuration.timeout != timeout()
            || configuration.user_agent != user_agent()
            || configuration.redirects
            || configuration.insecure
            || !configuration.headers.is_empty()
            || configuration.resumed
+            || !server_certs.is_empty()
+            || client_cert.is_some()
+            || client_key.is_some()
        {
-            if configuration.proxy.is_empty() {
-                configuration.client = client::initialize(
-                    configuration.timeout,
-                    &configuration.user_agent,
-                    configuration.redirects,
-                    configuration.insecure,
-                    &configuration.headers,
-                    None,
-                )
-                .expect("Could not rebuild client")
-            } else {
-                configuration.client = client::initialize(
-                    configuration.timeout,
-                    &configuration.user_agent,
-                    configuration.redirects,
-                    configuration.insecure,
-                    &configuration.headers,
-                    Some(&configuration.proxy),
-                )
-                .expect("Could not rebuild client")
-            }
+            configuration.client = client::initialize(
+                configuration.timeout,
+                &configuration.user_agent,
+                configuration.redirects,
+                configuration.insecure,
+                &configuration.headers,
+                proxy,
+                server_certs,
+                client_cert,
+                client_key,
+            )
+            .expect("Could not rebuild client");
        }

        if !configuration.replay_proxy.is_empty() {
@@ -938,6 +1023,9 @@ impl Configuration {
                    configuration.insecure,
                    &configuration.headers,
                    Some(&configuration.replay_proxy),
+                    server_certs,
+                    client_cert,
+                    client_key,
                )
                .expect("Could not rebuild client"),
            );
@@ -946,7 +1034,7 @@ impl Configuration {

    /// Given a configuration file's location and an instance of `Configuration`, read in
    /// the config file if found and update the current settings with the settings found therein
-    fn parse_and_merge_config(config_file: PathBuf, mut config: &mut Self) -> Result<()> {
+    fn parse_and_merge_config(config_file: PathBuf, config: &mut Self) -> Result<()> {
        if config_file.exists() {
            // save off a string version of the path before it goes out of scope
            let conf_str = config_file.to_str().unwrap_or("").to_string();
@@ -972,6 +1060,14 @@ impl Configuration {
        update_if_not_default!(&mut conf.target_url, new.target_url, "");
        update_if_not_default!(&mut conf.time_limit, new.time_limit, "");
        update_if_not_default!(&mut conf.proxy, new.proxy, "");
+        update_if_not_default!(
+            &mut conf.server_certs,
+            new.server_certs,
+            Vec::<String>::new()
+        );
+        update_if_not_default!(&mut conf.json, new.json, false);
+        update_if_not_default!(&mut conf.client_cert, new.client_cert, "");
+        update_if_not_default!(&mut conf.client_key, new.client_key, "");
        update_if_not_default!(&mut conf.verbosity, new.verbosity, 0);
        update_if_not_default!(&mut conf.silent, new.silent, false);
        update_if_not_default!(&mut conf.quiet, new.quiet, false);
@@ -981,17 +1077,18 @@ impl Configuration {
        update_if_not_default!(&mut conf.collect_backups, new.collect_backups, false);
        update_if_not_default!(&mut conf.collect_words, new.collect_words, false);
        // use updated quiet/silent values to determine output level; same for requester policy
-        conf.output_level = determine_output_level(conf.quiet, conf.silent);
+        conf.output_level = determine_output_level(conf.quiet, conf.silent, conf.json);
        conf.requester_policy = determine_requester_policy(conf.auto_tune, conf.auto_bail);
        update_if_not_default!(&mut conf.output, new.output, "");
        update_if_not_default!(&mut conf.redirects, new.redirects, false);
        update_if_not_default!(&mut conf.insecure, new.insecure, false);
        update_if_not_default!(&mut conf.force_recursion, new.force_recursion, false);
-        update_if_not_default!(&mut conf.extract_links, new.extract_links, false);
+        update_if_not_default!(&mut conf.extract_links, new.extract_links, extract_links());
        update_if_not_default!(&mut conf.extensions, new.extensions, Vec::<String>::new());
        update_if_not_default!(&mut conf.methods, new.methods, methods());
        update_if_not_default!(&mut conf.data, new.data, Vec::<u8>::new());
        update_if_not_default!(&mut conf.url_denylist, new.url_denylist, Vec::<Url>::new());
+        update_if_not_default!(&mut conf.update_app, new.update_app, false);
        if !new.regex_denylist.is_empty() {
            // cant use the update_if_not_default macro due to the following error
            //
@@ -1038,7 +1135,6 @@ impl Configuration {
        update_if_not_default!(&mut conf.replay_proxy, new.replay_proxy, "");
        update_if_not_default!(&mut conf.debug_log, new.debug_log, "");
        update_if_not_default!(&mut conf.resume_from, new.resume_from, "");
-        update_if_not_default!(&mut conf.json, new.json, false);

        update_if_not_default!(&mut conf.timeout, new.timeout, timeout());
        update_if_not_default!(&mut conf.user_agent, new.user_agent, user_agent());
--- a/src/config/tests.rs
+++ b/src/config/tests.rs
@@ -45,7 +45,7 @@ fn setup_config_test() -> Configuration {
            add_slash = true
            stdin = true
            dont_filter = true
-            extract_links = true
+            extract_links = false
            json = true
            save_state = false
            depth = 1
@@ -56,6 +56,9 @@ fn setup_config_test() -> Configuration {
            filter_word_count = [994, 992]
            filter_line_count = [34]
            filter_status = [201]
+            server_certs = ["/some/cert.pem", "/some/other/cert.pem"]
+            client_cert = "/some/client/cert.pem"
+            client_key = "/some/client/key.pem"
        "#;
    let tmp_dir = TempDir::new().unwrap();
    let file = tmp_dir.path().join(DEFAULT_CONFIG_NAME);
@@ -98,7 +101,7 @@ fn default_configuration() {
    assert!(!config.add_slash);
    assert!(!config.force_recursion);
    assert!(!config.redirects);
-    assert!(!config.extract_links);
+    assert!(config.extract_links);
    assert!(!config.insecure);
    assert!(!config.collect_extensions);
    assert!(!config.collect_backups);
@@ -117,6 +120,9 @@ fn default_configuration() {
    assert_eq!(config.filter_line_count, Vec::<usize>::new());
    assert_eq!(config.filter_status, Vec::<u16>::new());
    assert_eq!(config.headers, HashMap::new());
+    assert_eq!(config.server_certs, Vec::<String>::new());
+    assert_eq!(config.client_cert, String::new());
+    assert_eq!(config.client_key, String::new());
 }

 #[test]
@@ -305,7 +311,7 @@ fn config_reads_add_slash() {
 /// parse the test config and see that the value parsed is correct
 fn config_reads_extract_links() {
    let config = setup_config_test();
-    assert!(config.extract_links);
+    assert!(!config.extract_links);
 }

 #[test]
@@ -443,6 +449,30 @@ fn config_reads_resume_from() {
    assert_eq!(config.resume_from, "/some/state/file");
 }

+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_server_certs() {
+    let config = setup_config_test();
+    assert_eq!(
+        config.server_certs,
+        ["/some/cert.pem", "/some/other/cert.pem"]
+    );
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_client_cert() {
+    let config = setup_config_test();
+    assert_eq!(config.client_cert, "/some/client/cert.pem");
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_client_key() {
+    let config = setup_config_test();
+    assert_eq!(config.client_key, "/some/client/key.pem");
+}
+
 #[test]
 /// parse the test config and see that the values parsed are correct
 fn config_reads_headers() {
--- a/src/config/utils.rs
+++ b/src/config/utils.rs
@@ -84,6 +84,11 @@ pub(super) fn depth() -> usize {
    4
 }

+/// default extract links
+pub(super) fn extract_links() -> bool {
+    true
+}
+
 /// enum representing the three possible states for informational output (not logging verbosity)
 #[derive(Debug, Copy, Clone, PartialEq, Eq)]
 pub enum OutputLevel {
@@ -95,6 +100,9 @@ pub enum OutputLevel {

    /// silent scan, only print urls (used to be --quiet in versions 1.x.x)
    Silent,
+
+    /// silent scan, but with JSON output
+    SilentJSON,
 }

 /// implement a default for OutputLevel
@@ -106,14 +114,22 @@ impl Default for OutputLevel {
 }

 /// given the current settings for quiet and silent, determine output_level (DRY helper)
-pub fn determine_output_level(quiet: bool, silent: bool) -> OutputLevel {
+pub fn determine_output_level(quiet: bool, silent: bool, json: bool) -> OutputLevel {
    if quiet && silent {
        // user COULD have both as true in config file, take the more quiet of the two
-        OutputLevel::Silent
+        if json {
+            OutputLevel::SilentJSON
+        } else {
+            OutputLevel::Silent
+        }
    } else if quiet {
        OutputLevel::Quiet
    } else if silent {
-        OutputLevel::Silent
+        if json {
+            OutputLevel::SilentJSON
+        } else {
+            OutputLevel::Silent
+        }
    } else {
        OutputLevel::Default
    }
@@ -161,16 +177,28 @@ mod tests {
    #[test]
    /// test determine_output_level returns higher of the two levels if both given values are true
    fn determine_output_level_returns_correct_results() {
-        let mut level = determine_output_level(true, true);
+        let mut level = determine_output_level(true, true, false);
        assert_eq!(level, OutputLevel::Silent);

-        level = determine_output_level(false, true);
+        level = determine_output_level(false, true, false);
        assert_eq!(level, OutputLevel::Silent);

-        level = determine_output_level(false, false);
+        let mut level = determine_output_level(true, true, true);
+        assert_eq!(level, OutputLevel::SilentJSON);
+
+        level = determine_output_level(false, true, true);
+        assert_eq!(level, OutputLevel::SilentJSON);
+
+        level = determine_output_level(false, false, false);
        assert_eq!(level, OutputLevel::Default);

-        level = determine_output_level(true, false);
+        level = determine_output_level(true, false, false);
+        assert_eq!(level, OutputLevel::Quiet);
+
+        level = determine_output_level(false, false, true);
+        assert_eq!(level, OutputLevel::Default);
+
+        level = determine_output_level(true, false, true);
        assert_eq!(level, OutputLevel::Quiet);
    }

--- a/src/event_handlers/container.rs
+++ b/src/event_handlers/container.rs
@@ -160,13 +160,17 @@ impl Handles {
    /// number of extensions plus the number of request method types plus any dynamically collected
    /// extensions
    pub fn expected_num_requests_multiplier(&self) -> usize {
-        let multiplier = self.config.extensions.len()
-            + self.config.methods.len()
-            + self.num_collected_extensions();
+        let mut multiplier = self.config.extensions.len().max(1);

-        // methods should always have at least 1 member, likely making this .max call unneeded
-        // but leaving it for 'just in case' reasons
-        multiplier.max(1)
+        if multiplier > 1 {
+            // when we have more than one extension, we need to account for the fact that we'll
+            // be making a request for each extension and the base word (e.g. /foo.html and /foo)
+            multiplier += 1;
+        }
+
+        multiplier *= self.config.methods.len().max(1) * self.num_collected_extensions().max(1);
+
+        multiplier
    }

    /// Helper to easily get the (locked) underlying FeroxScans object
--- a/src/event_handlers/inputs.rs
+++ b/src/event_handlers/inputs.rs
@@ -12,6 +12,7 @@ use anyhow::Result;
 use console::style;
 use crossterm::event::{self, Event, KeyCode};
 use std::{
+    env::temp_dir,
    sync::{
        atomic::{AtomicBool, Ordering},
        Arc,
@@ -103,10 +104,36 @@ impl TermInputHandler {

        // User didn't set the --no-state flag (so saved_state is still the default true)
        if handles.config.save_state {
-            let state_file = open_file(&filename);
+            let Ok(mut state_file) = open_file(&filename) else {
+                // couldn't open the file, let the user know we're going to try again
+                let error = format!(
+                    "❌ Could not save {}, falling back to {}",
+                    filename,
+                    temp_dir().to_string_lossy()
+                );
+                PROGRESS_PRINTER.println(error);

-            let mut buffered_file = state_file?;
-            write_to(&state, &mut buffered_file, true)?;
+                let temp_filename = temp_dir().join(&filename);
+
+                let Ok(mut state_file) = open_file(&temp_filename.to_string_lossy()) else {
+                    // couldn't open the fallback file, let the user know
+                    let error = format!("❌❌ Could not save {:?}, giving up...", temp_filename);
+                    PROGRESS_PRINTER.println(error);
+
+                    log::trace!("exit: sigint_handler (failed to write)");
+                    std::process::exit(1);
+                };
+
+                write_to(&state, &mut state_file, true)?;
+
+                let msg = format!("✅ Saved scan state to {:?}", temp_filename);
+                PROGRESS_PRINTER.println(msg);
+
+                log::trace!("exit: sigint_handler (saved to temp folder)");
+                std::process::exit(1);
+            };
+
+            write_to(&state, &mut state_file, true)?;
        }

        log::trace!("exit: sigint_handler (end of program)");
--- a/src/event_handlers/outputs.rs
+++ b/src/event_handlers/outputs.rs
@@ -242,14 +242,6 @@ impl TermOutHandler {
        log::trace!("enter: process_response({:?}, {:?})", resp, call_type);

        async move {
-            let should_filter = self
-                .handles
-                .as_ref()
-                .unwrap()
-                .filters
-                .data
-                .should_filter_response(&resp, tx_stats.clone());
-
            let contains_sentry = if !self.config.filter_status.is_empty() {
                // -C was used, meaning -s was not and we should ignore the defaults
                // https://github.com/epi052/feroxbuster/issues/535
@@ -261,7 +253,7 @@ impl TermOutHandler {
            };

            let unknown_sentry = !RESPONSES.contains(&resp); // !contains == unknown
-            let should_process_response = contains_sentry && unknown_sentry && !should_filter;
+            let should_process_response = contains_sentry && unknown_sentry;

            if should_process_response {
                // print to stdout
@@ -336,6 +328,21 @@ impl TermOutHandler {
                    )
                    .await;

+                    let Some(handles) = self.handles.as_ref() else {
+                        // shouldn't ever happen, but we'll log and return early if it does
+                        log::error!("handles were unexpectedly None, this shouldn't happen");
+                        return Ok(());
+                    };
+
+                    if handles
+                        .filters
+                        .data
+                        .should_filter_response(&ferox_response, tx_stats.clone())
+                    {
+                        // response was filtered for one reason or another, don't process it
+                        continue;
+                    }
+
                    self.process_response(
                        tx_stats.clone(),
                        Box::new(ferox_response),
--- a/src/event_handlers/scans.rs
+++ b/src/event_handlers/scans.rs
@@ -16,7 +16,7 @@ use crate::{
 use super::command::Command::AddToUsizeField;
 use super::*;
 use crate::statistics::StatField;
-use reqwest::Url;
+use crate::utils::parse_url_with_raw_path;
 use tokio::time::Duration;

 #[derive(Debug)]
@@ -222,7 +222,7 @@ impl ScanHandler {
        let current_expectation = self.handles.expected_num_requests_per_dir() as u64;

        // used in the calculation of bar width down below, see explanation there
-        let divisor = self.handles.expected_num_requests_multiplier() as u64 - 1;
+        let divisor = (self.handles.expected_num_requests_multiplier() as u64 - 1).max(1);

        // add another `wordlist.len` to the expected per scan tracker in the statistics handler
        self.handles
@@ -266,7 +266,7 @@ impl ScanHandler {
                        let bar = scan.progress_bar();

                        // (4000 - 3000) / 2 => 500 words left to send
-                        let length = bar.length();
+                        let length = bar.length().unwrap_or(1);
                        let num_words_left = (length - bar.position()) / divisor;

                        // accumulate each bar's increment value for incrementing the total bar
@@ -294,12 +294,7 @@ impl ScanHandler {
        if let Ok(guard) = self.wordlist.lock().as_ref() {
            if let Some(list) = guard.as_ref() {
                return if offset > 0 {
-                    // the offset could be off a bit, so we'll adjust it backwards by 10%
-                    // of the overall wordlist size to ensure we don't miss any words
-                    // (hopefully)
-                    let adjusted_offset = offset - ((offset as f64 * 0.10) as usize);
-
-                    Ok(Arc::new(list[adjusted_offset..].to_vec()))
+                    Ok(Arc::new(list[offset..].to_vec()))
                } else {
                    Ok(list.clone())
                };
@@ -330,14 +325,27 @@ impl ScanHandler {
                self.data.add_directory_scan(&target, order).1 // add the new target; return FeroxScan
            };

-            if should_test_deny && should_deny_url(&Url::parse(&target)?, self.handles.clone())? {
+            if should_test_deny
+                && should_deny_url(&parse_url_with_raw_path(&target)?, self.handles.clone())?
+            {
                // response was caught by a user-provided deny list
                // checking this last, since it's most susceptible to longer runtimes due to what
                // input is received
                continue;
            }

-            let list = self.get_wordlist(scan.requests() as usize)?;
+            let divisor = self.handles.expected_num_requests_multiplier();
+
+            let list = if divisor > 1 && scan.requests() > 0 {
+                // if there were extensions provided and/or more than a single method used, and some
+                // number of requests have already been sent, we need to adjust the offset into the
+                // wordlist to ensure we don't index out of bounds
+
+                let adjusted = scan.requests_made_so_far() as f64 / (divisor as f64 - 1.0).max(1.0);
+                self.get_wordlist(adjusted as usize)?
+            } else {
+                self.get_wordlist(scan.requests_made_so_far() as usize)?
+            };

            log::info!("scan handler received {} - beginning scan", target);

--- a/src/event_handlers/statistics.rs
+++ b/src/event_handlers/statistics.rs
@@ -147,8 +147,13 @@ impl StatsHandler {
            self.stats.errors(),
        );

-        self.bar.set_message(&msg);
-        self.bar.inc(1);
+        self.bar.set_message(msg);
+
+        if self.bar.position() < self.stats.total_expected() as u64 {
+            // don't run off the end when we're a few requests over the expected total
+            // due to the heuristics tests
+            self.bar.inc(1);
+        }
    }

    /// Initialize new `Stats` object and the sc side of an mpsc channel that is responsible for
--- a/src/extractor/container.rs
+++ b/src/extractor/container.rs
@@ -11,16 +11,60 @@ use crate::{
        StatField::{LinksExtracted, TotalExpected},
    },
    url::FeroxUrl,
-    utils::{logged_request, make_request, send_try_recursion_command, should_deny_url},
+    utils::{
+        logged_request, make_request, parse_url_with_raw_path, send_try_recursion_command,
+        should_deny_url,
+    },
    ExtractionResult, DEFAULT_METHOD,
 };
 use anyhow::{bail, Context, Result};
-use reqwest::{Client, StatusCode, Url};
+use futures::StreamExt;
+use reqwest::{Client, Response, StatusCode, Url};
 use scraper::{Html, Selector};
 use std::{borrow::Cow, collections::HashSet};

+/// Wrapper around link extraction logic
+///   - create a new Url object based on cli options/args
+///   - check if the new Url has already been seen/scanned -> None
+///   - make a request to the new Url ? -> Some(response) : None
+pub(super) async fn request_link(url: &str, handles: Arc<Handles>) -> Result<Response> {
+    log::trace!("enter: request_link({})", url);
+
+    let ferox_url = FeroxUrl::from_string(url, handles.clone());
+
+    // create a url based on the given command line options
+    let new_url = ferox_url.format("", None)?;
+
+    let scanned_urls = handles.ferox_scans()?;
+
+    if scanned_urls.get_scan_by_url(new_url.as_ref()).is_some() {
+        //we've seen the url before and don't need to scan again
+        log::trace!("exit: request_link -> None");
+        bail!("previously seen url");
+    }
+
+    if (!handles.config.url_denylist.is_empty() || !handles.config.regex_denylist.is_empty())
+        && should_deny_url(&new_url, handles.clone())?
+    {
+        // can't allow a denied url to be requested
+        bail!(
+            "prevented request to {} due to {:?} || {:?}",
+            url,
+            handles.config.url_denylist,
+            handles.config.regex_denylist,
+        );
+    }
+
+    // make the request and store the response
+    let new_response = logged_request(&new_url, DEFAULT_METHOD, None, handles.clone()).await?;
+
+    log::trace!("exit: request_link -> {:?}", new_response);
+
+    Ok(new_response)
+}
+
 /// Whether an active scan is recursive or not
-#[derive(Debug)]
+#[derive(Debug, Copy, Clone)]
 enum RecursionStatus {
    /// Scan is recursive
    Recursive,
@@ -81,7 +125,7 @@ impl<'a> Extractor<'a> {
    ) -> Result<()> {
        log::trace!("enter: parse_url_and_add_subpaths({:?})", links);

-        match Url::parse(url_to_parse) {
+        match parse_url_with_raw_path(url_to_parse) {
            Ok(absolute) => {
                if absolute.domain() != original_url.domain()
                    || absolute.host() != original_url.host()
@@ -121,91 +165,140 @@ impl<'a> Extractor<'a> {

    /// given a set of links from a normal http body response, task the request handler to make
    /// the requests
-    pub async fn request_links(&mut self, links: HashSet<String>) -> Result<()> {
+    pub async fn request_links(
+        &mut self,
+        links: HashSet<String>,
+    ) -> Result<Option<tokio::task::JoinHandle<()>>> {
        log::trace!("enter: request_links({:?})", links);

        if links.is_empty() {
-            return Ok(());
+            return Ok(None);
        }

+        self.update_stats(links.len())?;
+
+        // create clones/remove use of self of/from everything the async move block will need to function
+        let cloned_scanned_urls = self.handles.ferox_scans()?;
+        let cloned_handles = self.handles.clone();
+        let cloned_url = self.url.clone();
+        let threads = self.handles.config.threads;
        let recursive = if self.handles.config.no_recursion {
            RecursionStatus::NotRecursive
        } else {
            RecursionStatus::Recursive
        };

-        let scanned_urls = self.handles.ferox_scans()?;
-        self.update_stats(links.len())?;
+        let link_request_task = tokio::spawn(async move {
+            let producers = futures::stream::iter(links.into_iter())
+                .map(|link| {
+                    // another clone to satisfy the async move block
+                    let inner_clone = cloned_handles.clone();

-        for link in links {
-            let mut resp = match self.request_link(&link).await {
-                Ok(resp) => resp,
-                Err(_) => continue,
-            };
+                    (
+                        tokio::spawn(async move { request_link(&link, inner_clone).await }),
+                        cloned_handles.clone(),
+                        cloned_scanned_urls.clone(),
+                        recursive,
+                        cloned_url.clone(),
+                    )
+                })
+                .for_each_concurrent(
+                    threads,
+                    |(join_handle, c_handles, c_scanned_urls, c_recursive, og_url)| async move {
+                        match join_handle.await {
+                            Ok(Ok(reqwest_response)) => {
+                                let mut resp = FeroxResponse::from(
+                                    reqwest_response,
+                                    &og_url,
+                                    DEFAULT_METHOD,
+                                    c_handles.config.output_level,
+                                )
+                                .await;

-            // filter if necessary
-            if self
-                .handles
-                .filters
-                .data
-                .should_filter_response(&resp, self.handles.stats.tx.clone())
-            {
-                continue;
-            }
+                                // filter if necessary
+                                if c_handles
+                                    .filters
+                                    .data
+                                    .should_filter_response(&resp, c_handles.stats.tx.clone())
+                                {
+                                    return;
+                                }

-            // request and report assumed file
-            if resp.is_file() || !resp.is_directory() {
-                log::debug!("Extracted File: {}", resp);
+                                // request and report assumed file
+                                if resp.is_file() || !resp.is_directory() {
+                                    log::debug!("Extracted File: {}", resp);

-                scanned_urls.add_file_scan(resp.url().as_str(), ScanOrder::Latest);
+                                    c_scanned_urls
+                                        .add_file_scan(resp.url().as_str(), ScanOrder::Latest);

-                if self.handles.config.collect_extensions {
-                    resp.parse_extension(self.handles.clone())?;
-                }
+                                    if c_handles.config.collect_extensions {
+                                        // no real reason this should fail
+                                        resp.parse_extension(c_handles.clone()).unwrap();
+                                    }

-                if let Err(e) = resp.send_report(self.handles.output.tx.clone()) {
-                    log::warn!("Could not send FeroxResponse to output handler: {}", e);
-                }
+                                    if let Err(e) = resp.send_report(c_handles.output.tx.clone()) {
+                                        log::warn!(
+                                            "Could not send FeroxResponse to output handler: {}",
+                                            e
+                                        );
+                                    }

-                continue;
-            }
+                                    return;
+                                }

-            if matches!(recursive, RecursionStatus::Recursive) {
-                log::debug!("Extracted Directory: {}", resp);
+                                if matches!(c_recursive, RecursionStatus::Recursive) {
+                                    log::debug!("Extracted Directory: {}", resp);

-                if !resp.url().as_str().ends_with('/')
-                    && (resp.status().is_success()
-                        || matches!(resp.status(), &StatusCode::FORBIDDEN))
-                {
-                    // if the url doesn't end with a /
-                    // and the response code is either a 2xx or 403
+                                    if !resp.url().as_str().ends_with('/')
+                                        && (resp.status().is_success()
+                                            || matches!(resp.status(), &StatusCode::FORBIDDEN))
+                                    {
+                                        // if the url doesn't end with a /
+                                        // and the response code is either a 2xx or 403

-                    // since all of these are 2xx or 403, recursion is only attempted if the
-                    // url ends in a /. I am actually ok with adding the slash and not
-                    // adding it, as both have merit.  Leaving it in for now to see how
-                    // things turn out (current as of: v1.1.0)
-                    resp.set_url(&format!("{}/", resp.url()));
-                }
+                                        // since all of these are 2xx or 403, recursion is only attempted if the
+                                        // url ends in a /. I am actually ok with adding the slash and not
+                                        // adding it, as both have merit.  Leaving it in for now to see how
+                                        // things turn out (current as of: v1.1.0)
+                                        resp.set_url(&format!("{}/", resp.url()));
+                                    }
+
+                                    if c_handles.config.filter_status.is_empty() {
+                                        // -C wasn't used, so -s is the only 'filter' left to account for
+                                        if c_handles
+                                            .config
+                                            .status_codes
+                                            .contains(&resp.status().as_u16())
+                                        {
+                                            send_try_recursion_command(c_handles.clone(), resp)
+                                                .await
+                                                .unwrap_or_default();
+                                        }
+                                    } else {
+                                        // -C was used, that means the filters above would have removed
+                                        // those responses, and anything else should be let through
+                                        send_try_recursion_command(c_handles.clone(), resp)
+                                            .await
+                                            .unwrap_or_default();
+                                    }
+                                }
+                            }
+                            Ok(Err(err)) => {
+                                log::warn!("Error during link extraction: {}", err);
+                            }
+                            Err(err) => {
+                                log::warn!("JoinError during link extraction: {}", err);
+                            }
+                        }
+                    },
+                );
+
+            // wait for the requests to finish
+            producers.await;
+        });

-                if self.handles.config.filter_status.is_empty() {
-                    // -C wasn't used, so -s is the only 'filter' left to account for
-                    if self
-                        .handles
-                        .config
-                        .status_codes
-                        .contains(&resp.status().as_u16())
-                    {
-                        send_try_recursion_command(self.handles.clone(), resp).await?;
-                    }
-                } else {
-                    // -C was used, that means the filters above would have removed
-                    // those responses, and anything else should be let through
-                    send_try_recursion_command(self.handles.clone(), resp).await?;
-                }
-            }
-        }
        log::trace!("exit: request_links");
-        Ok(())
+        Ok(Some(link_request_task))
    }

    /// wrapper around link extraction via html attributes
@@ -385,7 +478,7 @@ impl<'a> Extractor<'a> {
            ExtractionTarget::ResponseBody | ExtractionTarget::DirectoryListing => {
                self.response.unwrap().url().clone()
            }
-            ExtractionTarget::RobotsTxt => match Url::parse(&self.url) {
+            ExtractionTarget::RobotsTxt => match parse_url_with_raw_path(&self.url) {
                Ok(u) => u,
                Err(e) => {
                    bail!("Could not parse {}: {}", self.url, e);
@@ -415,56 +508,6 @@ impl<'a> Extractor<'a> {
        Ok(())
    }

-    /// Wrapper around link extraction logic
-    ///   - create a new Url object based on cli options/args
-    ///   - check if the new Url has already been seen/scanned -> None
-    ///   - make a request to the new Url ? -> Some(response) : None
-    pub(super) async fn request_link(&self, url: &str) -> Result<FeroxResponse> {
-        log::trace!("enter: request_link({})", url);
-
-        let ferox_url = FeroxUrl::from_string(url, self.handles.clone());
-
-        // create a url based on the given command line options
-        let new_url = ferox_url.format("", None)?;
-
-        let scanned_urls = self.handles.ferox_scans()?;
-
-        if scanned_urls.get_scan_by_url(new_url.as_ref()).is_some() {
-            //we've seen the url before and don't need to scan again
-            log::trace!("exit: request_link -> None");
-            bail!("previously seen url");
-        }
-
-        if (!self.handles.config.url_denylist.is_empty()
-            || !self.handles.config.regex_denylist.is_empty())
-            && should_deny_url(&new_url, self.handles.clone())?
-        {
-            // can't allow a denied url to be requested
-            bail!(
-                "prevented request to {} due to {:?} || {:?}",
-                url,
-                self.handles.config.url_denylist,
-                self.handles.config.regex_denylist,
-            );
-        }
-
-        // make the request and store the response
-        let new_response =
-            logged_request(&new_url, DEFAULT_METHOD, None, self.handles.clone()).await?;
-
-        let new_ferox_response = FeroxResponse::from(
-            new_response,
-            url,
-            DEFAULT_METHOD,
-            self.handles.config.output_level,
-        )
-        .await;
-
-        log::trace!("exit: request_link -> {:?}", new_ferox_response);
-
-        Ok(new_ferox_response)
-    }
-
    /// Entry point to perform link extraction from robots.txt
    ///
    /// `base_url` can have paths and subpaths, however robots.txt will be requested from the
@@ -484,7 +527,7 @@ impl<'a> Extractor<'a> {

        for capture in self.robots_regex.captures_iter(body) {
            if let Some(new_path) = capture.name("url_path") {
-                let mut new_url = Url::parse(&self.url)?;
+                let mut new_url = parse_url_with_raw_path(&self.url)?;

                new_url.set_path(new_path.as_str());

@@ -598,6 +641,20 @@ impl<'a> Extractor<'a> {
                Some(self.handles.config.proxy.as_str())
            };

+            let server_certs = &self.handles.config.server_certs;
+
+            let client_cert = if self.handles.config.client_cert.is_empty() {
+                None
+            } else {
+                Some(self.handles.config.client_cert.as_str())
+            };
+
+            let client_key = if self.handles.config.client_key.is_empty() {
+                None
+            } else {
+                Some(self.handles.config.client_key.as_str())
+            };
+
            client = client::initialize(
                self.handles.config.timeout,
                &self.handles.config.user_agent,
@@ -605,6 +662,9 @@ impl<'a> Extractor<'a> {
                self.handles.config.insecure,
                &self.handles.config.headers,
                proxy,
+                server_certs,
+                client_cert,
+                client_key,
            )?;
        }

@@ -614,7 +674,7 @@ impl<'a> Extractor<'a> {
            &client
        };

-        let mut url = Url::parse(&self.url)?;
+        let mut url = parse_url_with_raw_path(&self.url)?;
        url.set_path(location); // overwrite existing path

        // purposefully not using logged_request here due to using the special client
--- a/src/extractor/tests.rs
+++ b/src/extractor/tests.rs
@@ -1,4 +1,5 @@
 use super::builder::{LINKFINDER_REGEX, ROBOTS_TXT_REGEX, URL_CHARS_REGEX};
+use super::container::request_link;
 use super::*;
 use crate::config::{Configuration, OutputLevel};
 use crate::scan_manager::ScanOrder;
@@ -360,13 +361,13 @@ async fn request_link_happy_path() -> Result<()> {
        then.status(200).body("this is a test");
    });

-    let r_resp = ROBOTS_EXT.request_link(&srv.url("/login.php")).await?;
-    let b_resp = BODY_EXT.request_link(&srv.url("/login.php")).await?;
+    let r_resp = request_link(&srv.url("/login.php"), ROBOTS_EXT.handles.clone()).await?;
+    let b_resp = request_link(&srv.url("/login.php"), BODY_EXT.handles.clone()).await?;

-    assert!(matches!(r_resp.status(), &StatusCode::OK));
-    assert!(matches!(b_resp.status(), &StatusCode::OK));
-    assert_eq!(r_resp.content_length(), 14);
-    assert_eq!(b_resp.content_length(), 14);
+    assert!(matches!(r_resp.status(), StatusCode::OK));
+    assert!(matches!(b_resp.status(), StatusCode::OK));
+    assert_eq!(r_resp.content_length().unwrap(), 14);
+    assert_eq!(b_resp.content_length().unwrap(), 14);
    assert_eq!(mock.hits(), 2);
    Ok(())
 }
@@ -390,8 +391,8 @@ async fn request_link_bails_on_seen_url() -> Result<()> {
    let robots = setup_extractor(ExtractionTarget::RobotsTxt, scans.clone());
    let body = setup_extractor(ExtractionTarget::ResponseBody, scans);

-    let r_resp = robots.request_link(&served).await;
-    let b_resp = body.request_link(&served).await;
+    let r_resp = request_link(&served, robots.handles.clone()).await;
+    let b_resp = request_link(&served, body.handles.clone()).await;

    assert!(r_resp.is_err());
    assert!(b_resp.is_err());
--- a/src/filters/utils.rs
+++ b/src/filters/utils.rs
@@ -4,11 +4,10 @@ use crate::event_handlers::Handles;
 use crate::filters::similarity::SIM_HASHER;
 use crate::nlp::preprocess;
 use crate::response::FeroxResponse;
-use crate::utils::logged_request;
+use crate::utils::{logged_request, parse_url_with_raw_path};
 use crate::DEFAULT_METHOD;
 use anyhow::Result;
 use regex::Regex;
-use reqwest::Url;
 use std::sync::Arc;

 /// wrapper around logic necessary to create a SimilarityFilter
@@ -23,7 +22,7 @@ pub(crate) async fn create_similarity_filter(
    handles: Arc<Handles>,
 ) -> Result<SimilarityFilter> {
    // url as-is based on input, ignores user-specified url manipulation options (add-slash etc)
-    let url = Url::parse(similarity_filter)?;
+    let url = parse_url_with_raw_path(similarity_filter)?;

    // attempt to request the given url
    let resp = logged_request(&url, DEFAULT_METHOD, None, handles.clone()).await?;
--- a/src/heuristics.rs
+++ b/src/heuristics.rs
@@ -1,6 +1,8 @@
+use std::collections::HashMap;
 use std::sync::Arc;

 use anyhow::{bail, Result};
+use futures::future;
 use scraper::{Html, Selector};
 use uuid::Uuid;

@@ -118,12 +120,15 @@ impl HeuristicTests {
                    ) {
                        if e.to_string().contains(":SSL") {
                            ferox_print(
-                                &format!("Could not connect to {target_url} due to SSL errors (run with -k to ignore), skipping..."),
+                                &format!("Could not connect to {target_url} due to SSL errors (run with -k to ignore), skipping...\n  => {}\n", e.root_cause()),
                                &PROGRESS_PRINTER,
                            );
                        } else {
                            ferox_print(
-                                &format!("Could not connect to {target_url}, skipping..."),
+                                &format!(
+                                    "Could not connect to {target_url}, skipping...\n  => {}\n",
+                                    e.root_cause()
+                                ),
                                &PROGRESS_PRINTER,
                            );
                        }
@@ -276,138 +281,187 @@ impl HeuristicTests {
            None
        };

-        // 4 is due to the array in the nested for loop below
-        let mut responses = Vec::with_capacity(4);
+        // no matter what, we want an empty extension for the base case
+        let mut extensions = vec!["".to_string()];
+
+        // and then we want to add any extensions that was specified
+        // or has since been added to the running config
+        for ext in &self.handles.config.extensions {
+            extensions.push(format!(".{}", ext));
+        }

        // for every method, attempt to id its 404 response
        //
        // a good example of one where the GET/POST differ is on hackthebox:
        // - http://prd.m.rendering-api.interface.htb/api
+        //
+        // a good example of one where the heuristics return a 403 and a 404 (apache)
+        // as well as return two different types of 404s based on the file extension
+        // - http://10.10.11.198 (Encoding box in normal labs)
+        //
+        // both methods and extensions can elicit different responses from a given
+        // server, so both are considered when building auto-filter rules
        for method in self.handles.config.methods.iter() {
-            for (prefix, length) in [("", 1), ("", 3), (".htaccess", 1), ("admin", 1)] {
-                let path = format!("{prefix}{}", self.unique_string(length));
+            for extension in extensions.iter() {
+                // build out the 6 paths we'll use
+                let paths = [
+                    ("", 1),
+                    ("", 3),
+                    (".htaccess", 1),
+                    (".htaccess", 3),
+                    ("admin", 1),
+                    ("admin", 3),
+                ]
+                .map(|(prefix, length)| {
+                    format!("{prefix}{}{extension}", self.unique_string(length))
+                });

-                let ferox_url = FeroxUrl::from_string(target_url, self.handles.clone());
+                // allow all 6 requests to fly asynchronously
+                let responses = future::join_all(paths.into_iter().map(|path| async move {
+                    let ferox_url = FeroxUrl::from_string(target_url, self.handles.clone());

-                let nonexistent_url = ferox_url.format(&path, slash)?;
+                    let Ok(nonexistent_url) = ferox_url.format(&path, slash) else {
+                        return None;
+                    };

-                // example requests:
-                // - http://localhost/2fc1077836ad43ab98b7a31c2ca28fea
-                // - http://localhost/92969beae6bf4beb855d1622406d87e395c87387a9ad432e8a11245002b709b03cf609d471004154b83bcc1c6ec49f6f
-                // - http://localhost/.htaccessa005a2131e68449aa26e99029c914c09
-                // - http://localhost/adminf1d2541e73c44dcb9d1fb7d93334b280
-                let response =
-                    logged_request(&nonexistent_url, method, data, self.handles.clone()).await;
+                    // example requests:
+                    // - http://localhost/2fc1077836ad43ab98b7a31c2ca28fea
+                    // - http://localhost/92969beae6bf4beb855d1622406d87e395c87387a9ad432e8a11245002b709b03cf609d471004154b83bcc1c6ec49f6f
+                    // - http://localhost/.htaccessa005a2131e68449aa26e99029c914c09
+                    // - http://localhost/.htaccess92969beae6bf4beb855d1622406d87e395c87387a9ad432e8a11245002b709b03cf609d471004154b83bcc1c6ec49f6f
+                    // - http://localhost/adminf1d2541e73c44dcb9d1fb7d93334b280
+                    // - http://localhost/admin92969beae6bf4beb855d1622406d87e395c87387a9ad432e8a11245002b709b03cf609d471004154b83bcc1c6ec49f6f
+                    let Ok(response) =
+                        logged_request(&nonexistent_url, method, data, self.handles.clone()).await
+                    else {
+                        return None;
+                    };

-                req_counter += 1;
+                    if !self
+                        .handles
+                        .config
+                        .status_codes
+                        .contains(&response.status().as_u16())
+                    {
+                        // if the response code isn't one that's accepted via -s values, then skip to the next
+                        //
+                        // the default value for -s is all status codes, so unless the user says otherwise
+                        // this won't fire
+                        return None;
+                    }

-                // continue to next on error
-                let response = skip_fail!(response);
+                    Some(
+                        FeroxResponse::from(
+                            response,
+                            &ferox_url.target,
+                            method,
+                            self.handles.config.output_level,
+                        )
+                        .await,
+                    )
+                }))
+                .await // await gives vector of options containing feroxresponses
+                .into_iter()
+                .flatten() // strip out the none values
+                .collect::<Vec<_>>();

-                if !self
-                    .handles
-                    .config
-                    .status_codes
-                    .contains(&response.status().as_u16())
-                {
-                    // if the response code isn't one that's accepted via -s values, then skip to the next
-                    //
-                    // the default value for -s is all status codes, so unless the user says otherwise
-                    // this won't fire
+                if responses.len() < 2 {
+                    // don't have enough responses to make a determination, continue to next method
+                    log::debug!("not enough responses to make a determination");
                    continue;
                }

-                let ferox_response = FeroxResponse::from(
-                    response,
-                    &ferox_url.target,
-                    method,
+                // check the responses for similarities on which we can filter, multiple may be returned
+                let Some((wildcard_filters, wildcard_responses)) =
+                    self.examine_404_like_responses(&responses)
+                else {
+                    // no match was found during analysis of responses
+                    log::warn!("no match found for 404 responses");
+                    continue;
+                };
+
+                // report to the user, if appropriate
+                if matches!(
                    self.handles.config.output_level,
-                )
-                .await;
+                    OutputLevel::Default | OutputLevel::Quiet
+                ) {
+                    // sentry value to control whether or not to print the filter
+                    // used because we only want to print the same filter once
+                    let mut print_sentry;

-                responses.push(ferox_response);
-            }
+                    if let Ok(filters) = self.handles.filters.data.filters.read() {
+                        for new_wildcard in &wildcard_filters {
+                            // reset the sentry for every new wildcard produced by examine_404_like_responses
+                            print_sentry = true;

-            if responses.len() < 2 {
-                // don't have enough responses to make a determination, continue to next method
-                responses.clear();
-                continue;
-            }
+                            for other in filters.iter() {
+                                if let Some(other_wildcard) =
+                                    other.as_any().downcast_ref::<WildcardFilter>()
+                                {
+                                    // check the new wildcard against all existing wildcards, if it was added
+                                    // on the cli or by a previous directory, don't print it
+                                    if new_wildcard.as_ref() == other_wildcard {
+                                        print_sentry = false;
+                                        break;
+                                    }
+                                }
+                            }

-            // Command::AddFilter, &str (bytes/words/lines), usize (i.e. length associated with the type)
-            let Some(filter) = self.examine_404_like_responses(&responses) else {
-                // no match was found during analysis of responses
-                responses.clear();
-                continue;
-            };
-
-            // report to the user, if appropriate
-            if matches!(
-                self.handles.config.output_level,
-                OutputLevel::Default | OutputLevel::Quiet
-            ) {
-                // sentry value to control whether or not to print the filter
-                // used because we only want to print the same filter once
-                let mut print_sentry = true;
-
-                if let Ok(filters) = self.handles.filters.data.filters.read() {
-                    for other in filters.iter() {
-                        if let Some(other_wildcard) =
-                            other.as_any().downcast_ref::<WildcardFilter>()
-                        {
-                            if &*filter == other_wildcard {
-                                print_sentry = false;
-                                break;
+                            // if we're here, we've found a new wildcard that we didn't previously display, print it
+                            if print_sentry {
+                                ferox_print(&format!("{}", new_wildcard), &PROGRESS_PRINTER);
                            }
                        }
                    }
                }

-                if print_sentry {
-                    ferox_print(&format!("{}", filter), &PROGRESS_PRINTER);
+                // create the new filter
+                for wildcard in wildcard_filters {
+                    self.handles.filters.send(Command::AddFilter(wildcard))?;
+                }
+
+                // if we're here, we've detected a 404-like response pattern, and we're already filtering for size/word/line
+                //
+                // in addition, we'll create a similarity filter as a fallback
+                for resp in wildcard_responses {
+                    let hash = SIM_HASHER.create_signature(preprocess(resp.text()).iter());
+
+                    let sim_filter = SimilarityFilter {
+                        hash,
+                        original_url: resp.url().to_string(),
+                    };
+
+                    self.handles
+                        .filters
+                        .send(Command::AddFilter(Box::new(sim_filter)))?;
+
+                    if resp.is_directory() {
+                        // response is either a 3XX with a Location header that matches url + '/'
+                        // or it's a 2XX that ends with a '/'
+                        // or it's a 403 that ends with a '/'
+
+                        // set the wildcard flag to true, so we can check it when preventing
+                        // recursion in event_handlers/scans.rs
+
+                        // we'd need to clone the response to give ownership to the global list anyway
+                        // so we'll also use that clone to set the wildcard flag
+                        let mut cloned_resp = resp.clone();
+
+                        cloned_resp.set_wildcard(true);
+
+                        // add the response to the global list of responses
+                        RESPONSES.insert(cloned_resp);
+
+                        // function-internal magic number, indicates that we've detected a wildcard directory
+                        req_counter += 100;
+                    }
                }
            }
-
-            // create the new filter
-            self.handles.filters.send(Command::AddFilter(filter))?;
-
-            // if we're here, we've detected a 404-like response pattern, and we're already filtering for size/word/line
-            //
-            // in addition, we'll create a similarity filter as a fallback
-            let hash = SIM_HASHER.create_signature(preprocess(responses[0].text()).iter());
-
-            let sim_filter = SimilarityFilter {
-                hash,
-                original_url: responses[0].url().to_string(),
-            };
-
-            self.handles
-                .filters
-                .send(Command::AddFilter(Box::new(sim_filter)))?;
-
-            if responses[0].is_directory() {
-                // response is either a 3XX with a Location header that matches url + '/'
-                // or it's a 2XX that ends with a '/'
-                // or it's a 403 that ends with a '/'
-
-                // set the wildcard flag to true, so we can check it when preventing
-                // recursion in event_handlers/scans.rs
-                responses[0].set_wildcard(true);
-
-                // add the response to the global list of responses
-                RESPONSES.insert(responses[0].clone());
-
-                // function-internal magic number, indicates that we've detected a wildcard directory
-                req_counter += 100;
-            }
-
-            // reset the responses for the next method, if it exists
-            responses.clear();
        }

        log::trace!("exit: detect_404_like_responses");

-        let retval = if req_counter > 100 {
+        let retval = if req_counter >= 100 {
            WildcardResult::WildcardDirectory(req_counter)
        } else {
            WildcardResult::FourOhFourLike(req_counter)
@@ -416,96 +470,138 @@ impl HeuristicTests {
        Ok(Some(retval))
    }

-    /// for all responses, examine chars/words/lines
-    /// if all responses respective lengths match each other, we can assume
-    /// that will remain true for subsequent non-existent urls
+    /// for all responses, group them by status code, then examine chars/words/lines.
+    /// if all responses' respective lengths within a status code grouping match
+    /// each other, we can assume that will remain true for subsequent non-existent urls
    ///
-    /// values are examined from most to least specific (content length, word count, line count)
-    fn examine_404_like_responses(
+    /// within a status code grouping, values are examined from most to
+    /// least specific (content length, word count, line count)
+    #[allow(clippy::vec_box)] // the box is needed in the caller and i dont feel like changing it
+    fn examine_404_like_responses<'a>(
        &self,
-        responses: &[FeroxResponse],
-    ) -> Option<Box<WildcardFilter>> {
+        responses: &'a [FeroxResponse],
+    ) -> Option<(Vec<Box<WildcardFilter>>, Vec<&'a FeroxResponse>)> {
+        // aside from word/line/byte counts, additional discriminators are status code
+        // extension, and request method. The request method and extension are handled by
+        // the caller, since they're part of the request and make up the nested for loops
+        // in detect_404_like_responses.
+        //
+        // The status code is handled here, since it's part of the response to catch cases
+        // where we have something like a 403 and a 404
+
        let mut size_sentry = true;
        let mut word_sentry = true;
        let mut line_sentry = true;

-        let method = responses[0].method();
-        let status_code = responses[0].status();
-        let content_length = responses[0].content_length();
-        let word_count = responses[0].word_count();
-        let line_count = responses[0].line_count();
+        // returned vec of boxed wildcard filters
+        let mut wildcards = Vec::new();

-        for response in &responses[1..] {
-            // if any of the responses differ in length, that particular
-            // response length type is no longer a candidate for filtering
-            if response.content_length() != content_length {
-                size_sentry = false;
-            }
+        // returned vec of ferox responses that are needed for additional
+        // analysis
+        let mut wild_responses = Vec::new();

-            if response.word_count() != word_count {
-                word_sentry = false;
-            }
+        // mapping of grouped responses to status code
+        let mut grouped_responses = HashMap::new();

-            if response.line_count() != line_count {
-                line_sentry = false;
-            }
+        // iterate over all responses and add each response to its
+        // corresponding status code group
+        for response in responses {
+            grouped_responses
+                .entry(response.status())
+                .or_insert_with(Vec::new)
+                .push(response);
        }

-        if !size_sentry && !word_sentry && !line_sentry {
-            // none of the response lengths match, so we can't filter on any of them
-            return None;
+        // iterate over each grouped response and determine the most specific
+        // filter that can be applied to all responses in the group, i.e.
+        // start from byte count and work 'out' to line count
+        for response_group in grouped_responses.values() {
+            if response_group.len() < 2 {
+                // not enough responses to make a determination
+                continue;
+            }
+
+            let method = response_group[0].method();
+            let status_code = response_group[0].status();
+            let content_length = response_group[0].content_length();
+            let word_count = response_group[0].word_count();
+            let line_count = response_group[0].line_count();
+
+            for response in &response_group[1..] {
+                // if any of the responses differ in length, that particular
+                // response length type is no longer a candidate for filtering
+                if response.content_length() != content_length {
+                    size_sentry = false;
+                }
+
+                if response.word_count() != word_count {
+                    word_sentry = false;
+                }
+
+                if response.line_count() != line_count {
+                    line_sentry = false;
+                }
+            }
+
+            if !size_sentry && !word_sentry && !line_sentry {
+                // none of the response lengths match, so we can't filter on any of them
+                continue;
+            }
+
+            let mut wildcard = WildcardFilter {
+                content_length: None,
+                line_count: None,
+                word_count: None,
+                method: method.to_string(),
+                status_code: status_code.as_u16(),
+                dont_filter: self.handles.config.dont_filter,
+            };
+
+            match (size_sentry, word_sentry, line_sentry) {
+                (true, true, true) => {
+                    // all three types of length match, so we can't filter on any of them
+                    wildcard.content_length = Some(content_length);
+                    wildcard.word_count = Some(word_count);
+                    wildcard.line_count = Some(line_count);
+                }
+                (true, true, false) => {
+                    // content length and word count match, so we can filter on either
+                    wildcard.content_length = Some(content_length);
+                    wildcard.word_count = Some(word_count);
+                }
+                (true, false, true) => {
+                    // content length and line count match, so we can filter on either
+                    wildcard.content_length = Some(content_length);
+                    wildcard.line_count = Some(line_count);
+                }
+                (false, true, true) => {
+                    // word count and line count match, so we can filter on either
+                    wildcard.word_count = Some(word_count);
+                    wildcard.line_count = Some(line_count);
+                }
+                (true, false, false) => {
+                    // content length matches, so we can filter on that
+                    wildcard.content_length = Some(content_length);
+                }
+                (false, true, false) => {
+                    // word count matches, so we can filter on that
+                    wildcard.word_count = Some(word_count);
+                }
+                (false, false, true) => {
+                    // line count matches, so we can filter on that
+                    wildcard.line_count = Some(line_count);
+                }
+                (false, false, false) => {
+                    // none of the length types match, so we can't filter on any of them
+                    unreachable!("no wildcard size matches; handled by the if statement above");
+                }
+            };
+
+            wild_responses.push(response_group[0]);
+            wildcards.push(Box::new(wildcard));
        }

-        let mut wildcard = WildcardFilter {
-            content_length: None,
-            line_count: None,
-            word_count: None,
-            method: method.to_string(),
-            status_code: status_code.as_u16(),
-            dont_filter: self.handles.config.dont_filter,
-        };
-
-        match (size_sentry, word_sentry, line_sentry) {
-            (true, true, true) => {
-                // all three types of length match, so we can't filter on any of them
-                wildcard.content_length = Some(content_length);
-                wildcard.word_count = Some(word_count);
-                wildcard.line_count = Some(line_count);
-            }
-            (true, true, false) => {
-                // content length and word count match, so we can filter on either
-                wildcard.content_length = Some(content_length);
-                wildcard.word_count = Some(word_count);
-            }
-            (true, false, true) => {
-                // content length and line count match, so we can filter on either
-                wildcard.content_length = Some(content_length);
-                wildcard.line_count = Some(line_count);
-            }
-            (false, true, true) => {
-                // word count and line count match, so we can filter on either
-                wildcard.word_count = Some(word_count);
-                wildcard.line_count = Some(line_count);
-            }
-            (true, false, false) => {
-                // content length matches, so we can filter on that
-                wildcard.content_length = Some(content_length);
-            }
-            (false, true, false) => {
-                // word count matches, so we can filter on that
-                wildcard.word_count = Some(word_count);
-            }
-            (false, false, true) => {
-                // line count matches, so we can filter on that
-                wildcard.line_count = Some(line_count);
-            }
-            (false, false, false) => {
-                // none of the length types match, so we can't filter on any of them
-                unreachable!("no wildcard size matches; handled by the if statement above");
-            }
-        };
-
-        Some(Box::new(wildcard))
+        Some((wildcards, wild_responses))
    }
 }

--- a/src/main.rs
+++ b/src/main.rs
@@ -1,11 +1,14 @@
 use std::io::stdin;
 use std::{
-    env::args,
+    env::{
+        args,
+        consts::{ARCH, OS},
+    },
    fs::{create_dir, remove_file, File},
    io::{stderr, BufRead, BufReader},
    ops::Index,
    path::Path,
-    process::Command,
+    process::{exit, Command},
    sync::{atomic::Ordering, Arc},
 };

@@ -28,7 +31,7 @@ use feroxbuster::{
        TermOutHandler, SCAN_COMPLETE,
    },
    filters, heuristics, logger,
-    progress::{PROGRESS_BAR, PROGRESS_PRINTER},
+    progress::PROGRESS_PRINTER,
    scan_manager::{self, ScanType},
    scanner,
    utils::{fmt_err, slugify_filename},
@@ -38,6 +41,7 @@ use feroxbuster::{
 use feroxbuster::{utils::set_open_file_limit, DEFAULT_OPEN_FILE_LIMIT};
 use lazy_static::lazy_static;
 use regex::Regex;
+use self_update::cargo_crate_version;

 lazy_static! {
    /// Limits the number of parallel scans active at any given time when using --parallel
@@ -216,22 +220,75 @@ async fn wrapped_main(config: Arc<Configuration>) -> Result<()> {
        // PROGRESS_PRINTER and PROGRESS_BAR have been used at least once.  This call satisfies
        // that constraint
        PROGRESS_PRINTER.println("");
-        PROGRESS_BAR.join().unwrap();
    });

-    // cloning an Arc is cheap (it's basically a pointer into the heap)
-    // so that will allow for cheap/safe sharing of a single wordlist across multi-target scans
-    // as well as additional directories found as part of recursion
-    let words = match get_unique_words_from_wordlist(&config.wordlist) {
-        Ok(w) => w,
-        Err(err) => {
-            let secondary = Path::new(SECONDARY_WORDLIST);
+    // check if update_app is true
+    if config.update_app {
+        match update_app().await {
+            Err(e) => eprintln!("\n[ERROR] {}", e),
+            Ok(self_update::Status::UpToDate(version)) => {
+                eprintln!("\nFeroxbuster {} is up to date", version)
+            }
+            Ok(self_update::Status::Updated(version)) => {
+                eprintln!("\nFeroxbuster updated to {} version", version)
+            }
+        }
+        exit(0);
+    }

-            if secondary.exists() {
-                eprintln!("Found wordlist in secondary location");
-                get_unique_words_from_wordlist(SECONDARY_WORDLIST)?
-            } else {
-                return Err(err);
+    let words = if config.wordlist.starts_with("http") {
+        // found a url scheme, attempt to download the wordlist
+        let response = config
+            .client
+            .get(&config.wordlist)
+            .send()
+            .await
+            .context(format!(
+                "Unable to download wordlist from remote url: {}",
+                config.wordlist
+            ))?;
+
+        if !response.status().is_success() {
+            // status code isn't a 200, bail
+            bail!(
+                "[{}] Unable to download wordlist from url: {}",
+                response.status().as_str(),
+                config.wordlist
+            );
+        }
+
+        // attempt to get the filename from the url's path
+        let Some(path_segments) = response.url().path_segments() else {
+            bail!("Unable to parse path from url: {}", response.url());
+        };
+
+        let Some(filename) = path_segments.last() else {
+            bail!(
+                "Unable to parse filename from url's path: {}",
+                response.url().path()
+            );
+        };
+
+        let filename = filename.to_string();
+
+        // read the body and write it to disk, then use existing code to read the wordlist
+        let body = response.text().await?;
+
+        std::fs::write(&filename, body)?;
+
+        get_unique_words_from_wordlist(&filename)?
+    } else {
+        match get_unique_words_from_wordlist(&config.wordlist) {
+            Ok(w) => w,
+            Err(err) => {
+                let secondary = Path::new(SECONDARY_WORDLIST);
+
+                if secondary.exists() {
+                    eprintln!("Found wordlist in secondary location");
+                    get_unique_words_from_wordlist(SECONDARY_WORDLIST)?
+                } else {
+                    return Err(err);
+                }
            }
        }
    };
@@ -526,6 +583,24 @@ async fn clean_up(handles: Arc<Handles>, tasks: Tasks) -> Result<()> {
    Ok(())
 }

+async fn update_app() -> Result<self_update::Status, Box<dyn ::std::error::Error>> {
+    let target_os = format!("{}-{}", ARCH, OS);
+    let status = tokio::task::spawn_blocking(move || {
+        self_update::backends::github::Update::configure()
+            .repo_owner("epi052")
+            .repo_name("feroxbuster")
+            .bin_name("feroxbuster")
+            .target(target_os.as_str())
+            .show_download_progress(true)
+            .current_version(cargo_crate_version!())
+            .build()?
+            .update()
+    })
+    .await??;
+
+    Ok(status)
+}
+
 fn main() -> Result<()> {
    let config = Arc::new(Configuration::new().with_context(|| "Could not create Configuration")?);

--- a/src/nlp/document.rs
+++ b/src/nlp/document.rs
@@ -35,20 +35,21 @@ impl Document {
    fn add_term(&mut self, word: &str) {
        let term = Term::new(word);

-        let metadata = self.terms.entry(term).or_insert_with(TermMetaData::new);
+        let metadata = self.terms.entry(term).or_default();
        *metadata.count_mut() += 1;
    }

    /// create a new `Document` from the given HTML string
-    pub(crate) fn from_html(raw_html: &str) -> Self {
+    pub(crate) fn from_html(raw_html: &str) -> Option<Self> {
        let selector = Selector::parse("body").unwrap();

        let html = Html::parse_document(raw_html);

-        let text = html
-            .select(&selector)
-            .next()
-            .unwrap()
+        let Some(element) = html.select(&selector).next() else {
+            return None;
+        };
+
+        let text = element
            .descendants()
            .filter_map(|node| {
                if !node.value().is_text() && !node.value().is_comment() {
@@ -95,7 +96,7 @@ impl Document {

        // call `new` to push the parsed html through the pre-processing pipeline and process all
        // the words
-        Self::new(&text)
+        Some(Self::new(&text))
    }

    /// Log normalized weighting scheme for term frequency
@@ -146,19 +147,20 @@ mod tests {
    #[test]
    /// `Document::new` should preprocess html and generate a hashmap of `Term, TermMetadata`
    fn nlp_document_creation_from_html() {
-        let empty = Document::from_html("<html></html>");
+        let empty = Document::from_html("<html></html>").unwrap();
        assert_eq!(empty.number_of_terms, 0);

-        let other_empty = Document::from_html("<html><body><p></p></body></html>");
+        let other_empty = Document::from_html("<html><body><p></p></body></html>").unwrap();
        assert_eq!(other_empty.number_of_terms, 0);

-        let third_empty = Document::from_html("<!DOCTYPE html><html><!DOCTYPE html><p></p></html>");
+        let third_empty =
+            Document::from_html("<!DOCTYPE html><html><!DOCTYPE html><p></p></html>").unwrap();
        assert_eq!(third_empty.number_of_terms, 0);

        // p tag for is_text check and comment for is_comment
        let doc = Document::from_html(
            "<html><body><p>The air quality in Singapore.</p><!--got worse on Wednesday--></body></html>",
-        );
+        ).unwrap();

        let expected_terms = ["air", "quality", "singapore", "worse", "wednesday"];

@@ -209,7 +211,7 @@ mod tests {
    /// ensure words in script/style tags aren't processed
    fn document_creation_skips_script_and_style_tags() {
        let html = "<body><script>The air quality</script><style>in Singapore</style><p>got worse on Wednesday.</p></body>";
-        let doc = Document::from_html(html);
+        let doc = Document::from_html(html).unwrap();
        let keys = doc.terms().keys().map(|key| key.raw()).collect::<Vec<_>>();

        let expected = ["worse", "wednesday"];
--- a/src/nlp/term.rs
+++ b/src/nlp/term.rs
@@ -35,11 +35,6 @@ pub(super) struct TermMetaData {
 }

 impl TermMetaData {
-    /// create a new metadata container
-    pub(super) fn new() -> Self {
-        Self::default()
-    }
-
    /// number of times a `Term` has appeared in any `Document` within the corpus
    pub(super) fn document_frequency(&self) -> usize {
        self.term_frequencies().len()
@@ -90,7 +85,7 @@ mod tests {
    #[test]
    /// test accessors for correctness
    fn nlp_term_metadata_accessor_test() {
-        let mut metadata = TermMetaData::new();
+        let mut metadata = TermMetaData::default();

        *metadata.count_mut() += 1;
        assert_eq!(metadata.count(), 1);
--- a/src/parser.rs
+++ b/src/parser.rs
@@ -40,7 +40,7 @@ pub fn initialize() -> Command {
            Arg::new("url")
                .short('u')
                .long("url")
-                .required_unless_present_any(["stdin", "resume_from"])
+                .required_unless_present_any(["stdin", "resume_from", "update_app"])
                .help_heading("Target selection")
                .value_name("URL")
                .use_value_delimiter(true)
@@ -92,8 +92,9 @@ pub fn initialize() -> Command {
                .num_args(0)
                .help_heading("Composite settings")
                .conflicts_with_all(["rate_limit", "auto_bail"])
-                .help("Set --extract-links, --auto-tune, --collect-words, and --collect-backups to true"),
-        ).arg(
+                .help("Set --auto-tune, --collect-words, and --collect-backups to true"),
+        )
+        .arg(
            Arg::new("thorough")
                .long("thorough")
                .num_args(0)
@@ -176,7 +177,7 @@ pub fn initialize() -> Command {
                .use_value_delimiter(true)
                .help_heading("Request settings")
                .help(
-                    "File extension(s) to search for (ex: -x php -x pdf js)",
+                    "File extension(s) to search for (ex: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex: @ext.txt)",
                ),
        )
        .arg(
@@ -389,6 +390,35 @@ pub fn initialize() -> Command {
                .num_args(0)
                .help_heading("Client settings")
                .help("Disables TLS certificate validation in the client"),
+        )
+        .arg(
+            Arg::new("server_certs")
+                .long("server-certs")
+                .value_name("PEM|DER")
+                .value_hint(ValueHint::FilePath)
+                .num_args(1..)
+                .help_heading("Client settings")
+                .help("Add custom root certificate(s) for servers with unknown certificates"),
+        )
+        .arg(
+            Arg::new("client_cert")
+                .long("client-cert")
+                .value_name("PEM")
+                .value_hint(ValueHint::FilePath)
+                .num_args(1)
+                .requires("client_key")
+                .help_heading("Client settings")
+                .help("Add a PEM encoded certificate for mutual authentication (mTLS)"),
+        )
+        .arg(
+            Arg::new("client_key")
+                .long("client-key")
+                .value_name("PEM")
+                .value_hint(ValueHint::FilePath)
+                .num_args(1)
+                .requires("client_cert")
+                .help_heading("Client settings")
+                .help("Add a PEM encoded private key for mutual authentication (mTLS)"),
        );

    /////////////////////////////////////////////////////////////////////
@@ -433,7 +463,15 @@ pub fn initialize() -> Command {
                .long("extract-links")
                .num_args(0)
                .help_heading("Scan settings")
-                .help("Extract links from response body (html, javascript, etc...); make new requests based on findings")
+                .hide(true)
+                .help("Extract links from response body (html, javascript, etc...); make new requests based on findings (default: true)")
+        )
+        .arg(
+            Arg::new("dont_extract_links")
+                .long("dont-extract-links")
+                .num_args(0)
+                .help_heading("Scan settings")
+                .help("Don't extract links from response body (html, javascript, etc...)")
        )
        .arg(
            Arg::new("scan_limit")
@@ -477,7 +515,7 @@ pub fn initialize() -> Command {
                .long("wordlist")
                .value_hint(ValueHint::FilePath)
                .value_name("FILE")
-                .help("Path to the wordlist")
+                .help("Path or URL of the wordlist")
                .help_heading("Scan settings")
                .num_args(1),
        ).arg(
@@ -515,7 +553,8 @@ pub fn initialize() -> Command {
                .num_args(0)
                .help_heading("Dynamic collection settings")
                .help("Automatically request likely backup extensions for \"found\" urls")
-        ).arg(
+        )
+        .arg(
            Arg::new("collect_words")
                .short('g')
                .long("collect-words")
@@ -555,7 +594,7 @@ pub fn initialize() -> Command {
                .num_args(0)
                .conflicts_with("quiet")
                .help_heading("Output settings")
-                .help("Only print URLs + turn off logging (good for piping a list of urls to other commands)")
+                .help("Only print URLs (or JSON w/ --json) + turn off logging (good for piping a list of urls to other commands)")
        )
        .arg(
            Arg::new("quiet")
@@ -606,9 +645,18 @@ pub fn initialize() -> Command {
    let mut app = app
        .group(
            ArgGroup::new("output_files")
-                .args(["debug_log", "output"])
+                .args(["debug_log", "output", "silent"])
                .multiple(true),
        )
+        .arg(
+            Arg::new("update_app")
+                .short('U')
+                .long("update")
+                .exclusive(true)
+                .num_args(0)
+                .help_heading("Update settings")
+                .help("Update feroxbuster to the latest version"),
+        )
        .after_long_help(EPILOGUE);

    /////////////////////////////////////////////////////////////////////
@@ -675,9 +723,6 @@ EXAMPLES:
    Pass auth token via query parameter
        ./feroxbuster -u http://127.1 --query token=0123456789ABCDEF

-    Find links in javascript/html and make additional requests based on results
-        ./feroxbuster -u http://127.1 --extract-links
-
    Ludicrous speed... go!
        ./feroxbuster -u http://127.1 --threads 200
        
--- a/src/progress.rs
+++ b/src/progress.rs
@@ -1,4 +1,6 @@
-use indicatif::{MultiProgress, ProgressBar, ProgressDrawTarget, ProgressStyle};
+use std::time::Duration;
+
+use indicatif::{HumanDuration, MultiProgress, ProgressBar, ProgressDrawTarget, ProgressStyle};
 use lazy_static::lazy_static;

 lazy_static! {
@@ -31,30 +33,68 @@ pub enum BarType {
 /// Add an [indicatif::ProgressBar](https://docs.rs/indicatif/latest/indicatif/struct.ProgressBar.html)
 /// to the global [PROGRESS_BAR](../config/struct.PROGRESS_BAR.html)
 pub fn add_bar(prefix: &str, length: u64, bar_type: BarType) -> ProgressBar {
-    let mut style = ProgressStyle::default_bar().progress_chars("#>-");
+    let mut style = ProgressStyle::default_bar()
+        .progress_chars("#>-")
+        .with_key(
+            "smoothed_per_sec",
+            |state: &indicatif::ProgressState, w: &mut dyn std::fmt::Write| match (
+                state.pos(),
+                state.elapsed().as_millis(),
+            ) {
+                // https://github.com/console-rs/indicatif/issues/394#issuecomment-1309971049
+                //
+                // indicatif released a change to how they reported eta/per_sec
+                // and the results looked really weird based on how we use the progress
+                // bars. this fixes that
+                (pos, elapsed_ms) if elapsed_ms > 0 => {
+                    write!(w, "{:.0}/s", pos as f64 * 1000_f64 / elapsed_ms as f64).unwrap()
+                }
+                _ => write!(w, "-").unwrap(),
+            },
+        )
+        .with_key(
+            "smoothed_eta",
+            |state: &indicatif::ProgressState, w: &mut dyn std::fmt::Write| match (
+                state.pos(),
+                state.len(),
+            ) {
+                (pos, Some(len)) => write!(
+                    w,
+                    "{:#}",
+                    HumanDuration(Duration::from_millis(
+                        (state.elapsed().as_millis()
+                            * ((len as u128).checked_sub(pos as u128).unwrap_or(1))
+                                .checked_div(pos as u128)
+                                .unwrap_or(1)) as u64
+                    ))
+                )
+                .unwrap(),
+                _ => write!(w, "-").unwrap(),
+            },
+        );

    style = match bar_type {
-        BarType::Hidden => style.template(""),
-        BarType::Default => style.template(
-            "[{bar:.cyan/blue}] - {elapsed:<4} {pos:>7}/{len:7} {per_sec:7} {prefix} {msg}",
-        ),
-        BarType::Message => style.template(&format!(
+        BarType::Hidden => style.template("").unwrap(),
+        BarType::Default => style
+            .template("[{bar:.cyan/blue}] - {elapsed:<4} {pos:>7}/{len:7} {smoothed_per_sec:7} {prefix} {msg}")
+            .unwrap(),
+        BarType::Message => style
+            .template(&format!(
            "[{{bar:.cyan/blue}}] - {{elapsed:<4}} {{pos:>7}}/{{len:7}} {:7} {{prefix}} {{msg}}",
            "-"
-        )),
-        BarType::Total => {
-            style.template("[{bar:.yellow/blue}] - {elapsed:<4} {pos:>7}/{len:7} {eta:7} {msg}")
-        }
-        BarType::Quiet => style.template("Scanning: {prefix}"),
+        ))
+            .unwrap(),
+        BarType::Total => style
+            .template("[{bar:.yellow/blue}] - {elapsed:<4} {pos:>7}/{len:7} {smoothed_eta:7} {msg}")
+            .unwrap(),
+        BarType::Quiet => style.template("Scanning: {prefix}").unwrap(),
    };

-    let progress_bar = PROGRESS_BAR.add(ProgressBar::new(length));
-
-    progress_bar.set_style(style);
-
-    progress_bar.set_prefix(prefix);
-
-    progress_bar
+    PROGRESS_BAR.add(
+        ProgressBar::new(length)
+            .with_style(style)
+            .with_prefix(prefix.to_string()),
+    )
 }

 #[cfg(test)]
--- a/src/response.rs
+++ b/src/response.rs
@@ -21,7 +21,7 @@ use crate::{
    event_handlers::{Command, Handles},
    traits::FeroxSerialize,
    url::FeroxUrl,
-    utils::{self, fmt_err, status_colorizer},
+    utils::{self, fmt_err, parse_url_with_raw_path, status_colorizer},
    CommandSender,
 };

@@ -140,7 +140,7 @@ impl FeroxResponse {

    /// Set `FeroxResponse`'s `url` attribute, has no affect if an error occurs
    pub fn set_url(&mut self, url: &str) {
-        match Url::parse(url) {
+        match parse_url_with_raw_path(url) {
            Ok(url) => {
                self.url = url;
            }
@@ -170,7 +170,8 @@ impl FeroxResponse {

    /// free the `text` data, reducing memory usage
    pub fn drop_text(&mut self) {
-        self.text = String::new();
+        self.text.clear(); // length is set to 0
+        self.text.shrink_to_fit(); // allocated capacity shrinks to reflect the new size
    }

    /// Make a reasonable guess at whether the response is a file or not
@@ -394,7 +395,14 @@ impl FeroxResponse {
    pub fn send_report(self, report_sender: CommandSender) -> Result<()> {
        log::trace!("enter: send_report({:?}", report_sender);

-        report_sender.send(Command::Report(Box::new(self)))?;
+        // there's no reason to send the response body across the mpsc
+        //
+        // the only possible reason is for filtering on the body, but both `send_report`
+        // calls are gated behind checks for `should_filter_response`
+        let mut me = self;
+        me.drop_text();
+
+        report_sender.send(Command::Report(Box::new(me)))?;

        log::trace!("exit: send_report");
        Ok(())
@@ -477,15 +485,19 @@ impl FeroxSerialize for FeroxResponse {
            message
        } else {
            // not a wildcard, just create a normal entry
-            utils::create_report_string(
-                self.status.as_str(),
-                method,
-                &lines,
-                &words,
-                &chars,
-                &url_with_redirect,
-                self.output_level,
-            )
+            if matches!(self.output_level, OutputLevel::SilentJSON) {
+                self.as_json().unwrap_or_default()
+            } else {
+                utils::create_report_string(
+                    self.status.as_str(),
+                    method,
+                    &lines,
+                    &words,
+                    &chars,
+                    &url_with_redirect,
+                    self.output_level,
+                )
+            }
        }
    }

@@ -591,7 +603,7 @@ impl<'de> Deserialize<'de> for FeroxResponse {
            match key.as_str() {
                "url" => {
                    if let Some(url) = value.as_str() {
-                        if let Ok(parsed) = Url::parse(url) {
+                        if let Ok(parsed) = parse_url_with_raw_path(url) {
                            response.url = parsed;
                        }
                    }
--- a/src/scan_manager/menu.rs
+++ b/src/scan_manager/menu.rs
@@ -210,10 +210,14 @@ impl Menu {
                    }
                });
            } else {
+                if value.is_empty() {
+                    continue;
+                }
+
                let value = self.str_to_usize(value);

-                if value != 0 && !nums.contains(&value) {
-                    // the zeroth scan is always skipped, skip already known values
+                if !nums.contains(&value) {
+                    // skip already known values
                    nums.push(value);
                }
            }
--- a/src/scan_manager/mod.rs
+++ b/src/scan_manager/mod.rs
@@ -8,7 +8,7 @@ mod state;
 #[cfg(test)]
 mod tests;

-pub(self) use menu::Menu;
+use menu::Menu;
 pub use menu::{MenuCmd, MenuCmdResult};
 pub use order::ScanOrder;
 pub use response_container::FeroxResponses;
--- a/src/scan_manager/scan.rs
+++ b/src/scan_manager/scan.rs
@@ -39,6 +39,7 @@ pub struct FeroxScan {
    pub scan_type: ScanType,

    /// The order in which the scan was received
+    #[allow(dead_code)] // not entirely sure this isn't used somewhere
    pub(crate) scan_order: ScanOrder,

    /// Number of requests to populate the progress bar with
@@ -109,7 +110,7 @@ impl FeroxScan {

        match self.task.try_lock() {
            Ok(mut guard) => {
-                if let Some(task) = std::mem::replace(&mut *guard, None) {
+                if let Some(task) = guard.take() {
                    log::trace!("aborting {:?}", self);
                    task.abort();
                    self.set_status(ScanStatus::Cancelled)?;
@@ -153,7 +154,13 @@ impl FeroxScan {
    pub(super) fn stop_progress_bar(&self) {
        if let Ok(guard) = self.progress_bar.lock() {
            if guard.is_some() {
-                (*guard).as_ref().unwrap().finish_at_current_pos()
+                let pb = (*guard).as_ref().unwrap();
+
+                if pb.position() > self.num_requests {
+                    pb.finish()
+                } else {
+                    pb.abandon()
+                }
            }
        }
    }
@@ -168,7 +175,7 @@ impl FeroxScan {
                    let bar_type = match self.output_level {
                        OutputLevel::Default => BarType::Default,
                        OutputLevel::Quiet => BarType::Quiet,
-                        OutputLevel::Silent => BarType::Hidden,
+                        OutputLevel::Silent | OutputLevel::SilentJSON => BarType::Hidden,
                    };

                    let pb = add_bar(&self.url, self.num_requests, bar_type);
@@ -187,7 +194,7 @@ impl FeroxScan {
                let bar_type = match self.output_level {
                    OutputLevel::Default => BarType::Default,
                    OutputLevel::Quiet => BarType::Quiet,
-                    OutputLevel::Silent => BarType::Hidden,
+                    OutputLevel::Silent | OutputLevel::SilentJSON => BarType::Hidden,
                };

                let pb = add_bar(&self.url, self.num_requests, bar_type);
@@ -261,7 +268,7 @@ impl FeroxScan {
        let mut guard = self.task.lock().await;

        if guard.is_some() {
-            if let Some(task) = std::mem::replace(&mut *guard, None) {
+            if let Some(task) = guard.take() {
                task.await.unwrap();
                self.set_status(ScanStatus::Complete)
                    .unwrap_or_else(|e| log::warn!("Could not mark scan complete: {}", e))
--- a/src/scan_manager/scan_container.rs
+++ b/src/scan_manager/scan_container.rs
@@ -325,16 +325,18 @@ impl FeroxScans {
        let mut printed = 0;

        for (i, scan) in scans.iter().enumerate() {
-            if matches!(scan.scan_order, ScanOrder::Initial) || scan.task.try_lock().is_err() {
-                // original target passed in via either -u or --stdin
-                continue;
-            }
-
            if matches!(scan.scan_type, ScanType::Directory) {
                if printed == 0 {
                    self.menu
                        .println(&format!("{}:", style("Scans").bright().blue()));
                }
+
+                if let Ok(guard) = scan.status.lock() {
+                    if matches!(*guard, ScanStatus::Cancelled) {
+                        continue;
+                    }
+                }
+
                // we're only interested in displaying directory scans, as those are
                // the only ones that make sense to be stopped
                let scan_msg = format!("{i:3}: {scan}");
@@ -365,7 +367,14 @@ impl FeroxScans {
                        sleep(menu_pause_duration);
                        continue;
                    }
-                    u_scans.index(num).clone()
+
+                    let selected = u_scans.index(num);
+
+                    if matches!(selected.scan_type, ScanType::File) {
+                        continue;
+                    }
+
+                    selected.clone()
                }
                Err(..) => continue,
            };
@@ -378,14 +387,13 @@ impl FeroxScans {

            if input == 'y' || input == '\n' {
                self.menu.println(&format!("Stopping {}...", selected.url));
-
                selected
                    .abort()
                    .await
                    .unwrap_or_else(|e| log::warn!("Could not cancel task: {}", e));

                let pb = selected.progress_bar();
-                num_cancelled += pb.length() as usize - pb.position() as usize
+                num_cancelled += pb.length().unwrap_or(0) as usize - pb.position() as usize;
            } else {
                self.menu.println("Ok, doing nothing...");
            }
@@ -459,6 +467,32 @@ impl FeroxScans {

        self.menu.show_progress_bars();

+        let has_active_scans = if let Ok(guard) = self.scans.read() {
+            guard.iter().any(|s| s.is_active())
+        } else {
+            // if we can't tell for sure, we'll let it ride
+            //
+            // i'm not sure which is the better option here:
+            // either return true and let it potentially hang, or
+            // return false and exit, so just going with not
+            // abruptly exiting for maybe no reason
+            true
+        };
+
+        if !has_active_scans {
+            // the last active scan was cancelled, so we can exit
+            self.menu.println(&format!(
+                " 😱 no more active scans... {}",
+                style("exiting").red()
+            ));
+
+            let (tx, rx) = tokio::sync::oneshot::channel::<bool>();
+            handles
+                .send_scan_command(Command::JoinTasks(tx))
+                .unwrap_or_default();
+            rx.await.unwrap_or_default();
+        }
+
        result
    }

@@ -482,7 +516,7 @@ impl FeroxScans {
        let bar_type = match self.output_level {
            OutputLevel::Default => BarType::Message,
            OutputLevel::Quiet => BarType::Quiet,
-            OutputLevel::Silent => return Ok(()), // fast exit when --silent was used
+            OutputLevel::Silent | OutputLevel::SilentJSON => return Ok(()), // fast exit when --silent was used
        };

        if let Ok(scans) = self.scans.read() {
@@ -575,7 +609,7 @@ impl FeroxScans {
                let bar_type = match self.output_level {
                    OutputLevel::Default => BarType::Default,
                    OutputLevel::Quiet => BarType::Quiet,
-                    OutputLevel::Silent => BarType::Hidden,
+                    OutputLevel::Silent | OutputLevel::SilentJSON => BarType::Hidden,
                };

                let progress_bar = add_bar(url, bar_length, bar_type);
--- a/src/scan_manager/tests.rs
+++ b/src/scan_manager/tests.rs
@@ -72,7 +72,7 @@ fn add_url_to_list_of_scanned_urls_with_known_url() {
        url,
        ScanType::Directory,
        ScanOrder::Latest,
-        pb.length(),
+        pb.length().unwrap(),
        OutputLevel::Default,
        Some(pb),
    );
@@ -94,7 +94,7 @@ fn stop_progress_bar_stops_bar() {
        url,
        ScanType::Directory,
        ScanOrder::Latest,
-        pb.length(),
+        pb.length().unwrap(),
        OutputLevel::Default,
        Some(pb),
    );
@@ -152,7 +152,7 @@ async fn call_display_scans() {
        url,
        ScanType::Directory,
        ScanOrder::Latest,
-        pb.length(),
+        pb.length().unwrap(),
        OutputLevel::Default,
        Some(pb),
    );
@@ -160,7 +160,7 @@ async fn call_display_scans() {
        url_two,
        ScanType::Directory,
        ScanOrder::Latest,
-        pb_two.length(),
+        pb_two.length().unwrap(),
        OutputLevel::Default,
        Some(pb_two),
    );
@@ -202,6 +202,7 @@ fn partial_eq_compares_the_id_field() {

    assert!(!scan.eq(&scan_two));

+    #[allow(clippy::redundant_clone)]
    let scan_two = scan.clone();

    assert!(scan.eq(&scan_two));
@@ -469,7 +470,7 @@ fn feroxstates_feroxserialize_implementation() {
        r#""headers""#,
        r#""queries":[]"#,
        r#""no_recursion":false"#,
-        r#""extract_links":false"#,
+        r#""extract_links":true"#,
        r#""add_slash":false"#,
        r#""stdin":false"#,
        r#""depth":4"#,
@@ -489,6 +490,9 @@ fn feroxstates_feroxserialize_implementation() {
        r#""url_denylist":[]"#,
        r#""responses""#,
        r#""type":"response""#,
+        r#""client_cert":"""#,
+        r#""client_key":"""#,
+        r#""server_certs":[]"#,
        r#""url":"https://nerdcore.com/css""#,
        r#""path":"/css""#,
        r#""wildcard":true"#,
@@ -668,11 +672,7 @@ fn menu_get_command_input_from_user_returns_cancel() {
        assert!(matches!(result, MenuCmd::Cancel(_, _)));

        if let MenuCmd::Cancel(canx_list, ret_force) = result {
-            if idx == 0 {
-                assert!(canx_list.is_empty());
-            } else {
-                assert_eq!(canx_list, vec![idx]);
-            }
+            assert_eq!(canx_list, vec![idx]);
            assert_eq!(force, ret_force);
        }
    }
--- a/src/scanner/ferox_scanner.rs
+++ b/src/scanner/ferox_scanner.rs
@@ -203,6 +203,9 @@ impl FeroxScanner {
        log::info!("Starting scan against: {}", self.target_url);

        let mut scan_timer = Instant::now();
+        // every time we extract links we'll need to await the task to make sure
+        // it completes before the scan ends
+        let mut extraction_tasks = Vec::new();

        if self.handles.config.extract_links && matches!(self.order, ScanOrder::Initial) {
            // check for robots.txt (cannot be in sub-directories, so limited to Initial)
@@ -213,7 +216,7 @@ impl FeroxScanner {
                .build()?;

            let result = extractor.extract().await?;
-            extractor.request_links(result).await?;
+            extraction_tasks.push(extractor.request_links(result).await?)
        }

        let scanned_urls = self.handles.ferox_scans()?;
@@ -248,46 +251,53 @@ impl FeroxScanner {
            // heuristics test block:
            let test = heuristics::HeuristicTests::new(self.handles.clone());

-            if let Ok(dirlist_result) = test.directory_listing(&self.target_url).await {
-                if dirlist_result.is_some() {
-                    let dirlist_result = dirlist_result.unwrap();
-                    // at this point, we have a DirListingType, and it's not the None variant
-                    // which means we found directory listing based on the heuristic; now we need
-                    // to process the links that are available if --extract-links was used
+            if let Ok(Some(dirlist_result)) = test.directory_listing(&self.target_url).await {
+                // at this point, we have a DirListingType, and it's not the None variant
+                // which means we found directory listing based on the heuristic; now we need
+                // to process the links that are available if --extract-links was used

-                    if self.handles.config.extract_links {
-                        let mut extractor = ExtractorBuilder::default()
-                            .response(&dirlist_result.response)
-                            .target(ExtractionTarget::DirectoryListing)
-                            .url(&self.target_url)
-                            .handles(self.handles.clone())
-                            .build()?;
+                if self.handles.config.extract_links {
+                    let mut extractor = ExtractorBuilder::default()
+                        .response(&dirlist_result.response)
+                        .target(ExtractionTarget::DirectoryListing)
+                        .url(&self.target_url)
+                        .handles(self.handles.clone())
+                        .build()?;

-                        let result = extractor.extract_from_dir_listing().await?;
+                    let result = extractor.extract_from_dir_listing().await?;

-                        extractor.request_links(result).await?;
+                    extraction_tasks.push(extractor.request_links(result).await?);

-                        log::trace!("exit: scan_url -> Directory listing heuristic");
+                    log::trace!("exit: scan_url -> Directory listing heuristic");

-                        self.handles.stats.send(AddToF64Field(
-                            DirScanTimes,
-                            scan_timer.elapsed().as_secs_f64(),
-                        ))?;
+                    self.handles.stats.send(AddToF64Field(
+                        DirScanTimes,
+                        scan_timer.elapsed().as_secs_f64(),
+                    ))?;

-                        self.handles.stats.send(SubtractFromUsizeField(
-                            TotalExpected,
-                            progress_bar.length() as usize,
-                        ))?;
-                    }
+                    self.handles.stats.send(SubtractFromUsizeField(
+                        TotalExpected,
+                        progress_bar.length().unwrap_or(0) as usize,
+                    ))?;
+                }

-                    let mut message = format!("=> {}", style("Directory listing").blue().bright());
+                let mut message = format!("=> {}", style("Directory listing").blue().bright());

-                    if !self.handles.config.extract_links {
-                        write!(message, " (add {} to scan)", style("-e").bright().yellow())?;
+                if !self.handles.config.extract_links {
+                    write!(
+                        message,
+                        " (remove {} to scan)",
+                        style("--dont-extract-links").bright().yellow()
+                    )?;
+                }
+
+                if !self.handles.config.force_recursion {
+                    for handle in extraction_tasks.into_iter().flatten() {
+                        _ = handle.await;
                    }

                    progress_bar.reset_eta();
-                    progress_bar.finish_with_message(&message);
+                    progress_bar.finish_with_message(message);

                    ferox_scan.finish()?;

@@ -311,7 +321,7 @@ impl FeroxScanner {
                        style("Wildcard").blue().bright(),
                        style("stopped").red()
                    );
-                    progress_bar.set_message(&message);
+                    progress_bar.set_message(message);
                    progress_bar.inc(num_reqs as u64);
                }
                Some(WildcardResult::FourOhFourLike(num_reqs)) => {
@@ -338,7 +348,7 @@ impl FeroxScanner {
            let new_words = TF_IDF.read().unwrap().all_words();
            let new_words_len = new_words.len();

-            let cur_length = progress_bar.length();
+            let cur_length = progress_bar.length().unwrap_or(0);
            let new_length = cur_length + new_words_len as u64;

            progress_bar.set_length(new_length);
@@ -368,6 +378,10 @@ impl FeroxScanner {
            scan_timer.elapsed().as_secs_f64(),
        ))?;

+        for handle in extraction_tasks.into_iter().flatten() {
+            _ = handle.await;
+        }
+
        ferox_scan.finish()?;

        log::trace!("exit: scan_url");
--- a/src/scanner/requester.rs
+++ b/src/scanner/requester.rs
@@ -217,7 +217,7 @@ impl Requester {

                    self.ferox_scan
                        .progress_bar()
-                        .set_message(&format!("=> 🚦 {styled_direction} scan speed",));
+                        .set_message(format!("=> 🚦 {styled_direction} scan speed",));
                }
                self.policy_data.set_errors(scan_errors);
            } else {
@@ -230,7 +230,7 @@ impl Requester {

                self.ferox_scan
                    .progress_bar()
-                    .set_message(&format!("=> 🚦 {styled_direction} scan speed",));
+                    .set_message(format!("=> 🚦 {styled_direction} scan speed",));
            }
        }

@@ -286,7 +286,7 @@ impl Requester {
            self.set_rate_limiter(Some(new_limit)).await?;
            self.ferox_scan
                .progress_bar()
-                .set_message(&format!("=> 🚦 set rate limit ({new_limit}/s)"));
+                .set_message(format!("=> 🚦 set rate limit ({new_limit}/s)"));
        }

        self.adjust_limit(trigger, true).await?;
@@ -321,11 +321,11 @@ impl Requester {

            // figure out how many requests are skipped as a result
            let pb = self.ferox_scan.progress_bar();
-            let num_skipped = pb.length().saturating_sub(pb.position()) as usize;
+            let num_skipped = pb.length().unwrap_or(0).saturating_sub(pb.position()) as usize;

            let styled_trigger = style(format!("{trigger:?}")).red();

-            pb.set_message(&format!(
+            pb.set_message(format!(
                "=> 💀 too many {} ({}) 💀 bailing",
                styled_trigger,
                self.ferox_scan.num_errors(trigger),
@@ -475,12 +475,13 @@ impl Requester {

                if self.handles.config.collect_words {
                    if let Ok(mut guard) = TF_IDF.write() {
-                        let doc = Document::from_html(ferox_response.text());
-                        guard.add_document(doc);
-                        if guard.num_documents() % 12 == 0
-                            || (guard.num_documents() < 5 && guard.num_documents() % 2 == 0)
-                        {
-                            guard.calculate_tf_idf_scores();
+                        if let Some(doc) = Document::from_html(ferox_response.text()) {
+                            guard.add_document(doc);
+                            if guard.num_documents() % 12 == 0
+                                || (guard.num_documents() < 5 && guard.num_documents() % 2 == 0)
+                            {
+                                guard.calculate_tf_idf_scores();
+                            }
                        }
                    }
                }
@@ -490,6 +491,7 @@ impl Requester {
                        .target(ExtractionTarget::ResponseBody)
                        .response(&ferox_response)
                        .handles(self.handles.clone())
+                        .url(self.ferox_scan.url())
                        .build()?;

                    let new_links: HashSet<_>;
@@ -513,7 +515,11 @@ impl Requester {
                    }

                    if !new_links.is_empty() {
-                        extractor.request_links(new_links).await?;
+                        let extraction_task = extractor.request_links(new_links).await?;
+
+                        if let Some(task) = extraction_task {
+                            _ = task.await;
+                        }
                    }
                }

--- a/src/statistics/tests.rs
+++ b/src/statistics/tests.rs
@@ -55,7 +55,7 @@ fn save_writes_stats_object_to_disk() {
    stats.add_status_code(StatusCode::OK);
    stats.add_status_code(StatusCode::OK);
    let outfile = NamedTempFile::new().unwrap();
-    if stats.save(174.33, outfile.path().to_str().unwrap()).is_ok() {}
+    assert!(stats.save(174.33, outfile.path().to_str().unwrap()).is_ok());

    assert!(stats.as_json().unwrap().contains("statistics"));
    assert!(stats.as_json().unwrap().contains("11")); // requests made
--- a/src/url.rs
+++ b/src/url.rs
@@ -1,3 +1,4 @@
+use crate::utils::parse_url_with_raw_path;
 use crate::{event_handlers::Handles, statistics::StatError::UrlFormat, Command::AddError};
 use anyhow::{anyhow, bail, Result};
 use reqwest::Url;
@@ -142,19 +143,19 @@ impl FeroxUrl {
            word = word.trim_start_matches('/').to_string();
        };

-        let base_url = Url::parse(&url)?;
-        let joined = base_url.join(&word)?;
+        let base_url = parse_url_with_raw_path(&url)?;
+        let mut joined = base_url.join(&word)?;

-        if self.handles.config.queries.is_empty() {
-            // no query params to process
-            log::trace!("exit: format -> {}", joined);
-            Ok(joined)
-        } else {
-            let with_params =
-                Url::parse_with_params(joined.as_str(), &self.handles.config.queries)?;
-            log::trace!("exit: format_url -> {}", with_params);
-            Ok(with_params) // request with params attached
+        if !self.handles.config.queries.is_empty() {
+            // if called, this adds a '?' to the url, whether or not there are queries to be added
+            // so we need to check if there are queries to be added before blindly adding the '?'
+            joined
+                .query_pairs_mut()
+                .extend_pairs(self.handles.config.queries.iter());
        }
+
+        log::trace!("exit: format_url -> {}", joined);
+        Ok(joined)
    }

    /// Simple helper to abstract away adding a forward-slash to a url if not present
@@ -189,7 +190,7 @@ impl FeroxUrl {

        let target = self.normalize();

-        let parsed = Url::parse(&target)?;
+        let parsed = parse_url_with_raw_path(&target)?;
        let parts = parsed
            .path_segments()
            .ok_or_else(|| anyhow!("No path segments found"))?;
--- a/src/utils.rs
+++ b/src/utils.rs
@@ -75,7 +75,12 @@ pub(crate) async fn send_try_recursion_command(
    handles: Arc<Handles>,
    response: FeroxResponse,
 ) -> Result<()> {
-    handles.send_scan_command(Command::TryRecursion(Box::new(response.clone())))?;
+    // make the response mutable so we can drop the body before
+    // sending it over the mpsc
+    let mut response = response;
+    response.drop_text();
+
+    handles.send_scan_command(Command::TryRecursion(Box::new(response)))?;
    let (tx, rx) = oneshot::channel::<bool>();
    handles.send_scan_command(Command::Sync(tx))?;
    rx.await?;
@@ -420,9 +425,14 @@ fn should_deny_absolute(url_to_test: &Url, denier: &Url, handles: Arc<Handles>)
        // current deny-url, now we just need to check to see if this deny-url is a parent
        // to a scanned url that is also a parent of the given url
        for ferox_scan in handles.ferox_scans()?.get_active_scans() {
-            let scanner = Url::parse(ferox_scan.url().trim_end_matches('/'))
+            let scanner = parse_url_with_raw_path(ferox_scan.url().trim_end_matches('/'))
                .with_context(|| format!("Could not parse {ferox_scan} as a url"))?;

+            // by calling the new parse_url_with_raw_path, and reaching this point without an
+            // error, we know we have an authority and therefore a host. leaving the code
+            // below, but we should never hit the else condition. leaving it in so if we find
+            // a case where i'm mistaken, we'll know about it and can address it
+
            if let Some(scan_host) = scanner.host() {
                // same domain/ip check we perform on the denier above
                if tested_host != scan_host {
@@ -431,7 +441,7 @@ fn should_deny_absolute(url_to_test: &Url, denier: &Url, handles: Arc<Handles>)
                }
            } else {
                // couldn't process .host from scanner
-                continue;
+                unreachable!("should_deny_absolute: scanner.host() returned None, which shouldn't be possible");
            };

            let scan_path = scanner.path();
@@ -482,7 +492,7 @@ pub fn should_deny_url(url: &Url, handles: Arc<Handles>) -> Result<bool> {

    // normalization for comparison is to remove the trailing / if one exists, this is done for
    // the given url and any url to which it's compared
-    let normed_url = Url::parse(url.to_string().trim_end_matches('/'))?;
+    let normed_url = parse_url_with_raw_path(url.to_string().trim_end_matches('/'))?;

    for denier in &handles.config.url_denylist {
        // note to self: it may seem as though we can use regex only for --dont-scan, however, in
@@ -532,6 +542,187 @@ pub fn slugify_filename(url: &str, prefix: &str, suffix: &str) -> String {
    filename
 }

+/// This function takes a url string and returns a `url::Url`
+///
+/// It is primarily used to detect url paths that `url::Url::parse` will
+/// silently transform, such as /path/../file.html -> /file.html
+///
+/// # Warning
+///
+/// In the instance of a url with encoded path traversal strings, such as
+/// /path/%2e%2e/file.html, the underlying `url::Url::parse` will
+/// further encode the %-signs and return /path/%252e%252e/file.html
+pub fn parse_url_with_raw_path(url: &str) -> Result<Url> {
+    log::trace!("enter: parse_url_with_raw_path({})", url);
+
+    let parsed = Url::parse(url)?;
+
+    if !parsed.has_authority() {
+        // parsed correctly, but no authority, meaning mailto: or tel: or
+        // some other url that we don't care about
+        bail!("url to parse has no authority and is therefore invalid");
+    }
+
+    // we have a valid url, the next step is to check the path and see if it's
+    // something that url::Url::parse would silently transform
+    //
+    // i.e. if the path is /path/../file.html, url::Url::parse will transform it
+    // to /file.html, which is not what we want
+
+    let farthest_right_authority_part;
+
+    // we want to find the farthest right authority component, which is the
+    // component that is the furthest right in the url that is part of the
+    // authority
+    //
+    // per RFC 3986, the authority is defined as:
+    // - authority = [ userinfo "@" ] host [ ":" port ]
+    //
+    // so the farthest right authority component is either the port or the host
+    //
+    // i.e. in http://example.com:80/path/file.html, the farthest right authority
+    // component is :80
+    //
+    // in http://example.com/path/file.html, the farthest right authority component
+    // is example.com
+    //
+    // the farthest right authority component is used to split the url into two
+    // parts: the part before the authority and the part after the authority
+    if let Some(port) = parsed.port() {
+        // if the url has a port, then the farthest right authority component is
+        // the port
+        farthest_right_authority_part = format!(":{}", port);
+    } else if parsed.has_host() {
+        // if the url has a host, then the farthest right authority component is
+        // the host
+        farthest_right_authority_part = parsed.host_str().unwrap().to_owned();
+    } else {
+        // if the url has neither a port nor a host, then the url is invalid
+        // and we can't do anything with it, but i don't think this is possible
+        unreachable!("url has an authority, but has neither a port nor a host");
+    }
+
+    // split the original url string into two parts: the part before the authority and the part
+    // after the authority (i.e. the path + query + fragment)
+
+    let Some((_, after_authority)) = url.split_once(&farthest_right_authority_part) else {
+        // if we can't split the url string into two parts, then the url doesn't conform to our
+        // expectations, and we can't continue processing it, so we'll return the parsed url
+        return Ok(parsed);
+    };
+
+    // when there is a port, but it matches the default port for the scheme,
+    // url::Url::parse will mark the port as None, giving us a
+    // `after_authority` that looks something like this:
+    // - :80/path/file.html
+    let after_authority = after_authority
+        .replacen(":80", "", 1)
+        .replacen(":443", "", 1);
+
+    // snippets from rfc-3986:
+    //
+    //          foo://example.com:8042/over/there?name=ferret#nose
+    //          \_/   \______________/\_________/ \_________/ \__/
+    //           |           |            |            |        |
+    //        scheme     authority       path        query   fragment
+    //
+    // The path component is terminated
+    //    by the first question mark ("?") or number sign ("#") character, or
+    //    by the end of the URI.
+    //
+    // The query component is indicated by the first question
+    //    mark ("?") character and terminated by a number sign ("#") character
+    //    or by the end of the URI.
+    let (path, _discarded) = after_authority
+        .split_once('?')
+        // if there isn't a '?', try to remove a fragment
+        .unwrap_or_else(|| {
+            // if there isn't a '#', return (original, empty)
+            after_authority
+                .split_once('#')
+                .unwrap_or((&after_authority, ""))
+        });
+
+    // at this point, we have the path, all by itself
+
+    // each of the following is a string that we can expect url::Url::parse to
+    // transform. The variety is to ensure we cover most common path traversal
+    // encodings
+    let transformation_detectors = [
+        // ascii
+        "..",
+        // single url encoded
+        "%2e%2e",
+        // double url encoded
+        "%25%32%65%25%32%65",
+        // utf-8 encoded
+        "%c0%ae%c0%ae",
+        "%e0%40%ae%e0%40%ae",
+        "%c0ae%c0ae",
+        // 16 bit shenanigans
+        "%uff0e%uff0e",
+        "%u002e%u002e",
+    ];
+
+    let parsing_will_transform_path = transformation_detectors
+        .iter()
+        .any(|detector| path.to_lowercase().contains(detector));
+
+    if !parsing_will_transform_path {
+        // there's no string in the path of the url that will trigger a transformation
+        // so, we can return it as-is
+        return Ok(parsed);
+    }
+
+    // if we reach this point, the path contains a string that will trigger a transformation
+    // so we need to manually create a Url that doesn't have the transformation
+    // and return that
+    //
+    // special thanks to github user @lavafroth for this workaround
+
+    let mut hacked_url = if path.ends_with('/') {
+        // from_file_path silently strips trailing slashes, and
+        // from_directory_path adds them, so we'll choose the appropriate
+        // constructor based on the presence of a path's trailing slash
+
+        // according to from_file_path docs:
+        //   from_file_path returns `Err` if the given path is not absolute or,
+        //   on Windows, if the prefix is not a disk prefix (e.g. `C:`) or a UNC prefix (`\\`).
+        //
+        // since we parsed out a valid url path, we know it is absolute, so on non-windows
+        // platforms, we can safely unwrap. On windows, we need to fix up the path
+        #[cfg(target_os = "windows")]
+        {
+            let path = format!("\\/IGNOREME{path}");
+            Url::from_directory_path(path).unwrap()
+        }
+        #[cfg(not(target_os = "windows"))]
+        Url::from_directory_path(path).unwrap()
+    } else {
+        #[cfg(target_os = "windows")]
+        {
+            let path = format!("\\/IGNOREME{path}");
+            Url::from_file_path(path).unwrap()
+        }
+        #[cfg(not(target_os = "windows"))]
+        Url::from_file_path(path).unwrap()
+    };
+
+    // host must be set first, otherwise multiple components may return Err
+    hacked_url.set_host(parsed.host_str())?;
+    // scheme/port/username/password can fail, but in this instance, we know they won't
+    hacked_url.set_scheme(parsed.scheme()).unwrap();
+    hacked_url.set_port(parsed.port()).unwrap();
+    hacked_url.set_username(parsed.username()).unwrap();
+    hacked_url.set_password(parsed.password()).unwrap();
+    // query/fragment can't fail
+    hacked_url.set_query(parsed.query());
+    hacked_url.set_fragment(parsed.fragment());
+
+    log::trace!("exit: parse_url_with_raw_path -> {}", hacked_url);
+    Ok(hacked_url)
+}
+
 #[cfg(test)]
 mod tests {
    use super::*;
@@ -539,31 +730,159 @@ mod tests {
    use crate::scan_manager::{FeroxScans, ScanOrder};

    #[test]
-    /// set_open_file_limit with a low requested limit succeeds
-    fn utils_set_open_file_limit_with_low_requested_limit() {
-        let (_, hard) = getrlimit(Resource::NOFILE).unwrap();
-        let lower_limit = hard - 1;
-        assert!(set_open_file_limit(lower_limit));
+    /// multiple tests for parse_url_with_raw_path
+    fn utils_parse_url_with_raw_path() {
+        // ../.. is preserved
+        let url = "https://www.google.com/../../stuff";
+        let parsed = parse_url_with_raw_path(url).unwrap();
+        assert_eq!(parsed.as_str(), url);
+
+        // ../.. is preserved as well as the trailing slash
+        let url = "https://www.google.com/../../stuff/";
+        let parsed = parse_url_with_raw_path(url).unwrap();
+        assert_eq!(parsed.as_str(), url);
+
+        // no trailing slash is preserved
+        let url = "https://www.google.com/stuff";
+        let parsed = parse_url_with_raw_path(url).unwrap();
+        assert_eq!(parsed.as_str(), url);
+
+        // trailing slash is preserved
+        let url = "https://www.google.com/stuff/";
+        let parsed: Url = parse_url_with_raw_path(url).unwrap();
+        assert_eq!(parsed.as_str(), url);
+
+        // mailto is an error
+        let url = "mailto:user@example.com";
+        let parsed = parse_url_with_raw_path(url);
+        assert!(parsed.is_err());
+
+        // relative url is an error
+        let url = "../../stuff";
+        let parsed = parse_url_with_raw_path(url);
+        assert!(parsed.is_err());
+
+        // absolute without host is an error
+        let url = "/../../stuff";
+        let parsed = parse_url_with_raw_path(url);
+        assert!(parsed.is_err());
+
+        // default ports are parsed correctly
+        for url in [
+            "http://example.com:80/path/file.html",
+            "https://example.com:443/path/file.html",
+        ] {
+            let parsed = parse_url_with_raw_path(url).unwrap();
+            assert!(parsed.port().is_none());
+            assert_eq!(parsed.host().unwrap().to_string().as_str(), "example.com");
+        }
+
+        // non-default ports are parsed correctly
+        for url in [
+            "http://example.com:8080/path/file.html",
+            "https://example.com:4433/path/file.html",
+        ] {
+            let parsed = parse_url_with_raw_path(url).unwrap();
+            assert!(parsed.port().is_some());
+            assert_eq!(parsed.as_str(), url);
+        }
+
+        // different encodings are respected if found in doubles
+        //
+        // note that the % sign is encoded as %25...
+        let url = "http://user:pass@example.com/%2e%2e/stuff.php";
+        let parsed = parse_url_with_raw_path(url).unwrap();
+        assert_eq!(
+            parsed.as_str(),
+            "http://user:pass@example.com/%252e%252e/stuff.php"
+        );
+
+        let url = "http://user:pass@example.com/%25%32%65%25%32%65/stuff.php";
+        let parsed = parse_url_with_raw_path(url).unwrap();
+        assert_eq!(parsed.username(), "user");
+        assert_eq!(parsed.password().unwrap(), "pass");
+        assert_eq!(
+            parsed.as_str(),
+            "http://user:pass@example.com/%2525%2532%2565%2525%2532%2565/stuff.php"
+        );
+
+        let url = "http://user:pass@example.com/%c0%ae%c0%ae/stuff.php";
+        let parsed = parse_url_with_raw_path(url).unwrap();
+        assert_eq!(parsed.username(), "user");
+        assert_eq!(parsed.password().unwrap(), "pass");
+        assert_eq!(
+            parsed.as_str(),
+            "http://user:pass@example.com/%25c0%25ae%25c0%25ae/stuff.php"
+        );
+
+        let url = "http://user:pass@example.com/%e0%40%ae%e0%40%ae/stuff.php";
+        let parsed = parse_url_with_raw_path(url).unwrap();
+        assert_eq!(parsed.username(), "user");
+        assert_eq!(parsed.password().unwrap(), "pass");
+        assert_eq!(
+            parsed.as_str(),
+            "http://user:pass@example.com/%25e0%2540%25ae%25e0%2540%25ae/stuff.php"
+        );
+
+        let url = "http://user:pass@example.com/%c0ae%c0ae/stuff.php";
+        let parsed = parse_url_with_raw_path(url).unwrap();
+        assert_eq!(parsed.username(), "user");
+        assert_eq!(parsed.password().unwrap(), "pass");
+        assert_eq!(
+            parsed.as_str(),
+            "http://user:pass@example.com/%25c0ae%25c0ae/stuff.php"
+        );
+
+        let url = "http://user:pass@example.com/%uff0e%uff0e/stuff.php";
+        let parsed = parse_url_with_raw_path(url).unwrap();
+        assert_eq!(parsed.username(), "user");
+        assert_eq!(parsed.password().unwrap(), "pass");
+        assert_eq!(
+            parsed.as_str(),
+            "http://user:pass@example.com/%25uff0e%25uff0e/stuff.php"
+        );
+
+        let url = "http://user:pass@example.com/%u002e%u002e/stuff.php";
+        let parsed = parse_url_with_raw_path(url).unwrap();
+        assert_eq!(parsed.username(), "user");
+        assert_eq!(parsed.password().unwrap(), "pass");
+        assert_eq!(
+            parsed.as_str(),
+            "http://user:pass@example.com/%25u002e%25u002e/stuff.php"
+        );
    }

-    #[test]
-    /// set_open_file_limit with a high requested limit succeeds
-    fn utils_set_open_file_limit_with_high_requested_limit() {
-        let (_, hard) = getrlimit(Resource::NOFILE).unwrap();
-        let higher_limit = hard + 1;
-        // calculate a new soft to ensure soft != hard and hit that logic branch
-        let new_soft = hard - 1;
-        setrlimit(Resource::NOFILE, new_soft, hard).unwrap();
-        assert!(set_open_file_limit(higher_limit));
-    }
+    #[cfg(not(target_os = "windows"))]
+    mod nix_only_tests {
+        use super::*;

-    #[test]
-    /// set_open_file_limit should fail when hard == soft
-    fn utils_set_open_file_limit_with_fails_when_both_limits_are_equal() {
-        let (_, hard) = getrlimit(Resource::NOFILE).unwrap();
-        // calculate a new soft to ensure soft == hard and hit the failure logic branch
-        setrlimit(Resource::NOFILE, hard, hard).unwrap();
-        assert!(!set_open_file_limit(hard)); // returns false
+        #[test]
+        /// set_open_file_limit with a low requested limit succeeds
+        fn utils_set_open_file_limit_with_low_requested_limit() {
+            let (_, hard) = getrlimit(Resource::NOFILE).unwrap();
+            let lower_limit = hard - 1;
+            assert!(set_open_file_limit(lower_limit));
+        }
+
+        #[test]
+        /// set_open_file_limit with a high requested limit succeeds
+        fn utils_set_open_file_limit_with_high_requested_limit() {
+            let (_, hard) = getrlimit(Resource::NOFILE).unwrap();
+            let higher_limit = hard + 1;
+            // calculate a new soft to ensure soft != hard and hit that logic branch
+            let new_soft = hard - 1;
+            setrlimit(Resource::NOFILE, new_soft, hard).unwrap();
+            assert!(set_open_file_limit(higher_limit));
+        }
+
+        #[test]
+        /// set_open_file_limit should fail when hard == soft
+        fn utils_set_open_file_limit_with_fails_when_both_limits_are_equal() {
+            let (_, hard) = getrlimit(Resource::NOFILE).unwrap();
+            // calculate a new soft to ensure soft == hard and hit the failure logic branch
+            setrlimit(Resource::NOFILE, hard, hard).unwrap();
+            assert!(!set_open_file_limit(hard)); // returns false
+        }
    }

    #[test]
@@ -697,6 +1016,13 @@ mod tests {
    /// provide a denier from which we can't check a host, which results in no comparison, expect false
    /// because the denier is a parent to the tested, even tho the scanned doesn't compare, it
    /// still returns true
+    ///
+    /// note: adding parse_url_with_raw_path changed the behavior of this test, it used to return
+    /// true, now it returns false. see my note in should_deny_absolute and the unreachable!
+    /// call block to see why
+    ///
+    /// leaving this test here to document the behavior change and to catch regressions in the
+    /// new expected behavior
    fn should_deny_url_doesnt_compare_non_domains_in_scanned() {
        let deny_url = "https://testdomain.com/";
        let scan_url = "unix:/run/foo.socket";
@@ -710,8 +1036,7 @@ mod tests {
        let config = Arc::new(config);

        let handles = Arc::new(Handles::for_testing(Some(scans), Some(config)).0);
-
-        assert!(should_deny_url(&tested_url, handles).unwrap());
+        assert!(!should_deny_url(&tested_url, handles).unwrap());
    }

    #[test]
--- a/tests/mutual-auth/Caddyfile
+++ b/tests/mutual-auth/Caddyfile
@@ -0,0 +1,17 @@
+(mTLS) {
+	tls {
+		client_auth {
+			mode require_and_verify
+			trusted_ca_cert_file certs/server/ca.crt
+		}
+	}
+}
+
+https://localhost:8001 {
+	import mTLS
+	log
+
+	handle / {
+		file_server browse
+	}
+}
--- a/tests/mutual-auth/README.md
+++ b/tests/mutual-auth/README.md
@@ -0,0 +1,6 @@
+# Testing mTLS
+
+- run `gen-certs.sh`
+- run `sudo /path/to/caddy run`
+- expect listener on port 8001
+- run `feroxbuster -u https://localhost:8001 --client-key certs/client/client.key --client-cert certs/client/client.crt`
--- a/tests/mutual-auth/certs/client/client.crt
+++ b/tests/mutual-auth/certs/client/client.crt
@@ -0,0 +1,17 @@
+-----BEGIN CERTIFICATE-----
+MIICqzCCAZMCFE22XDzrLwkJIkb3EdP333d4HoXQMA0GCSqGSIb3DQEBCwUAMBMx
+ETAPBgNVBAMMCFNlcnZlckNBMB4XDTIzMDUwNjExMDYyM1oXDTI0MDUwNTExMDYy
+M1owETEPMA0GA1UEAwwGQ2xpZW50MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB
+CgKCAQEAz3EPWMsh+dfPdbHtpNhizZZs+r0djzdHHgkbnNQ1PodWDnv0Rf1YgNEa
+umQuUvIgjMtorRqbz9HLG4+H2aR5KHgPwBNHyKS4PEiQvWDV88aJxdMbgL/IfzAt
+di85UcBUkyqUe1r6vIS0smJo1wVwxLEmD6kdt1BEI3LaK1j99JeG8TAS8f+/xf4s
+ouE4lA+y3bJQP18wUGuyudntFQBKgjY2Tx+RWbBcx0zW68M7IMQ5bDz0oK9MYw8G
+q2vwcRyMLuoyNpbDT5mI2wsQu/r2O0CCNbtkg5JxasdYR7Llw9YTl74st3dshM9e
+4V5uuVotcWXW6U518nWHOQy9qiBSOQIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQB4
+xOVvWrRZ4SBqzaen32COXpjddX28Q7YmNB/UKl3ZT7R1dIjUMfJz2le0mj2UpSAr
+rDT7PCsXnDP0KswGiJC3IVTa/hnkUk798jwUvp221jvebyy8/NMWfWPoIKfhELdb
+3uJfrGyQuB8Zf9Q1hc9jYDX27EbGaDSpOrpE9Ej2riVnbgBKZsS5jcfY8JDrkv+F
+4cP2pTu6mVRuU1Bzx3SB0Vg2uGi1QTJuuA905Y3zpoRfTtybKlRRkMQk+46xrdyV
+x64wq9zcL6Kq4D/UE3EjLnjbRw6H6g8jbnBjT5KRfP2tmbF9RTZs44Dl0hYvXber
+HrvWtxHG8OJ8BLQg1rQd
+-----END CERTIFICATE-----
--- a/tests/mutual-auth/certs/client/client.key
+++ b/tests/mutual-auth/certs/client/client.key
@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDPcQ9YyyH51891
+se2k2GLNlmz6vR2PN0ceCRuc1DU+h1YOe/RF/ViA0Rq6ZC5S8iCMy2itGpvP0csb
+j4fZpHkoeA/AE0fIpLg8SJC9YNXzxonF0xuAv8h/MC12LzlRwFSTKpR7Wvq8hLSy
+YmjXBXDEsSYPqR23UEQjctorWP30l4bxMBLx/7/F/iyi4TiUD7LdslA/XzBQa7K5
+2e0VAEqCNjZPH5FZsFzHTNbrwzsgxDlsPPSgr0xjDwara/BxHIwu6jI2lsNPmYjb
+CxC7+vY7QII1u2SDknFqx1hHsuXD1hOXviy3d2yEz17hXm65Wi1xZdbpTnXydYc5
+DL2qIFI5AgMBAAECggEAC8XVeoM1w4uITDxLucMnkVYgC3dj5/K5zCY1bVg8SNcO
+rt4BSh8TkKT9ZLZmjCHOb9sj7s4PqXLVOXRTAAq17xJoR2z4shYKGC7AmyTVo6MB
+AuuFGDCaMQCzlc1ejgmRqzP7jwgl6oDIDgcofsqB4MHSgIlHJNYO9emQ4OypJgJA
+xd8KT5S/hThJG1VqJ6P0oiB/WBlzcJ5wX4GSVE25RlpRX8ogqCyI9V+SRq2CrG7U
+Jqv3Kbag7derTfqmsKyjv/kckOgfKH/rm61HMrYshcPfgxL2fZe2Q8wCTexvhZwZ
+8vD8bvR++SxOxbigCIB7ReYgmoj4bocjqDX4vUhe8QKBgQD35oDdOa2uiOs7NWVf
+IV1ZwPWxxwnYFIEA8paQwsYGIxHrYNdGSsGBzwvLDPpTeOO0VdoC+sP5zytTv547
+djeOzGf9Hj6swa5tPdzkYjZV/85mnmGKaEmmCN4AvpYol5l2BTetFtX0v6QEaqvU
+uZbV5X2UcuClExA0frNUJDVHkQKBgQDWOCZq1r9X3iEkcFSBhironVNj80jFqIum
+rMbGUUcOI05U2hkmMDluSW1NNL2k+SNJXq7fmkjIQEXffqcbsXUSIQB0MU6yddt5
+7+c19ioZChx91Kl049rKQ21kPTh7D0TCUvDQLapt2xbUNg6rGCLSrkkVlWwxLnDU
+pNk/c4QcKQKBgEreedLWhabtwSV7pecKO5hM16dedpGk96UinuiPeqEF3HabI8kd
+8L1Um7oybDPjkdm4CATYWXHL6Mj9WTuaI4NkJo/in4krYZOqmFj9dG2auWpysQDN
+KFkV2n6dENqnlnh3cO48tFebvVx8HvM7Ldvh2ICKBWC1ljJUhbKG0PSRAoGBAJVy
+fNLCWKEbVbHPMBVgnaTExT2Qp29F4493MAGBCHpDhU1LDoqG0DoxvbBEIB3stYJl
+LMjQIQCbXmPKPxjh15O7NE7ba1SzRleuV3Zc8wee9zuN1l6265d6LOHml/W6NDUB
+mgESKrkTRLztrZQNdZXXgyMsqFszVAH1s55Bn6PpAoGBAN13Ev7Ynysdvkc3aHO6
+qM0hH6mAlEOAyCTk5r/0cyz9rGyYWXiVXen0ftSaBcISdzhrVkRDs3rLrHwEXdu1
+Y2Z1HhZkILw/C4t+Eaa6FOWfwwPAdOpaxYpxKxCEeCBKmkd1z0Dx0vDEDrt+AaHa
+UYIQ9wAbZpuKGfFQceyr1lBO
+-----END PRIVATE KEY-----
--- a/tests/mutual-auth/certs/server/ca.crt
+++ b/tests/mutual-auth/certs/server/ca.crt
@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDBzCCAe+gAwIBAgIUG3vb4pIbvaI/+LzOpu6Z4b6s4iIwDQYJKoZIhvcNAQEL
+BQAwEzERMA8GA1UEAwwIU2VydmVyQ0EwHhcNMjMwNTA2MTEwNjIzWhcNMzMwNTAz
+MTEwNjIzWjATMREwDwYDVQQDDAhTZXJ2ZXJDQTCCASIwDQYJKoZIhvcNAQEBBQAD
+ggEPADCCAQoCggEBAKazNKPaH8LDzcaZRvBLrDNJkL1pukmB36mbczj07hZVbPmS
+/hyBvAdBFom0ZTw5dIpsUtRSZbDPrsCVpdY9O1jxwhrDi6mfvyJtKLEbTW4PvARq
+WwDhpa2SYwBMI+0ilXWTAzwJuWT1NhuUsAcB6SGwkNm3iKqZUDxn3V2L2AHRcKEJ
+9Zn9ePP4BsvtAS8ZBLxTnoo7R2SHiWwjDwuTtS4fQ5bWzGkmdmeuJ7JJt4ZzQV+m
+MBqrK3XVi+MXayvt5affGvHj/KuhlVBXHnUgSvEgFpuhK9elsds2iRho8mp1d0iH
+EIMp9LHVftsIpUxbKt/Pa/JL7oG9LBIvPj/SIjsCAwEAAaNTMFEwHQYDVR0OBBYE
+FLlRKsDb/ducVIBirME0VJZ3TwfkMB8GA1UdIwQYMBaAFLlRKsDb/ducVIBirME0
+VJZ3TwfkMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAD7lqNzU
+wxuyO60Gn2q6DUBb1Kseq6bSndNHeagdfMfKManKl1YObnB0ciTO3bnmNXiXktSu
+BsQzlmr3O+H6X39Vpdyqq4SoOcOt0I+bvBykk1UZqEoc7jGXdZVmnk9Q0uoKtWxJ
+rV9CHEhyPNnEh4W07y05UUn9S6EiKy5232yi4USdmk44GXhFblS5inhTTxca2vEq
+9h+FH+QZ7ehaAaWR+EaQjXNwm2mN7gWxM3Q6RfK9N67MHD9ggmfdyZmnyt5gCidC
+ys4W4stEh6d6fXZT77dcGaHKdXW3GwP3ZcAlRFYPqpAvWzndC9kDCgIULeSP1ALy
+cILcb0HQvNS0t60=
+-----END CERTIFICATE-----
--- a/tests/mutual-auth/certs/server/server.crt
+++ b/tests/mutual-auth/certs/server/server.crt
@@ -0,0 +1,17 @@
+-----BEGIN CERTIFICATE-----
+MIICrzCCAZcCFGMKRtmMLuut+sxC+TbWQfum7oXZMA0GCSqGSIb3DQEBCwUAMBQx
+EjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0yMzA1MDYxMTA2MjNaFw0zMzA1MDMxMTA2
+MjNaMBQxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBAKazNKPaH8LDzcaZRvBLrDNJkL1pukmB36mbczj07hZVbPmS/hyB
+vAdBFom0ZTw5dIpsUtRSZbDPrsCVpdY9O1jxwhrDi6mfvyJtKLEbTW4PvARqWwDh
+pa2SYwBMI+0ilXWTAzwJuWT1NhuUsAcB6SGwkNm3iKqZUDxn3V2L2AHRcKEJ9Zn9
+ePP4BsvtAS8ZBLxTnoo7R2SHiWwjDwuTtS4fQ5bWzGkmdmeuJ7JJt4ZzQV+mMBqr
+K3XVi+MXayvt5affGvHj/KuhlVBXHnUgSvEgFpuhK9elsds2iRho8mp1d0iHEIMp
+9LHVftsIpUxbKt/Pa/JL7oG9LBIvPj/SIjsCAwEAATANBgkqhkiG9w0BAQsFAAOC
+AQEAjPAtZs1by2h/1fr/ypojw16llzbReT8J+T8YHSTf6YwjoE83I0QDOLEo1ax+
+e/8qyQLs0EnlfdomNyA4Z/ECbY5c1nY0Dp//u6WH7AwLUx5HiwUw4Fmxu9Q/oB1o
+3vhIPl5Vd/VpdxDzuO8q8WvagwjVaxsZP3PVaBDRzZZPldPgTakfk+w5XnjNfgJi
+RDRutTRe6KBOxt7PAzAVV71FtOIq0b4xCNJGNurYBhRgZ5iQ7yMw+I5Vte1TakWr
+9gfE/yoKbU1W+y0QxSDTsnTCO4i3mXmBTuceTVWELwqZcr34W7n3vD8UtZQfanML
+cHCZaLPSMDuDtS74FSamP3i+oQ==
+-----END CERTIFICATE-----
--- a/tests/mutual-auth/certs/server/server.crt.1
+++ b/tests/mutual-auth/certs/server/server.crt.1
@@ -0,0 +1,17 @@
+-----BEGIN CERTIFICATE-----
+MIICrzCCAZcCFGMKRtmMLuut+sxC+TbWQfum7oXZMA0GCSqGSIb3DQEBCwUAMBQx
+EjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0yMzA1MDYxMTA2MjNaFw0zMzA1MDMxMTA2
+MjNaMBQxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBAKazNKPaH8LDzcaZRvBLrDNJkL1pukmB36mbczj07hZVbPmS/hyB
+vAdBFom0ZTw5dIpsUtRSZbDPrsCVpdY9O1jxwhrDi6mfvyJtKLEbTW4PvARqWwDh
+pa2SYwBMI+0ilXWTAzwJuWT1NhuUsAcB6SGwkNm3iKqZUDxn3V2L2AHRcKEJ9Zn9
+ePP4BsvtAS8ZBLxTnoo7R2SHiWwjDwuTtS4fQ5bWzGkmdmeuJ7JJt4ZzQV+mMBqr
+K3XVi+MXayvt5affGvHj/KuhlVBXHnUgSvEgFpuhK9elsds2iRho8mp1d0iHEIMp
+9LHVftsIpUxbKt/Pa/JL7oG9LBIvPj/SIjsCAwEAATANBgkqhkiG9w0BAQsFAAOC
+AQEAjPAtZs1by2h/1fr/ypojw16llzbReT8J+T8YHSTf6YwjoE83I0QDOLEo1ax+
+e/8qyQLs0EnlfdomNyA4Z/ECbY5c1nY0Dp//u6WH7AwLUx5HiwUw4Fmxu9Q/oB1o
+3vhIPl5Vd/VpdxDzuO8q8WvagwjVaxsZP3PVaBDRzZZPldPgTakfk+w5XnjNfgJi
+RDRutTRe6KBOxt7PAzAVV71FtOIq0b4xCNJGNurYBhRgZ5iQ7yMw+I5Vte1TakWr
+9gfE/yoKbU1W+y0QxSDTsnTCO4i3mXmBTuceTVWELwqZcr34W7n3vD8UtZQfanML
+cHCZaLPSMDuDtS74FSamP3i+oQ==
+-----END CERTIFICATE-----
--- a/tests/mutual-auth/certs/server/server.crt.2
+++ b/tests/mutual-auth/certs/server/server.crt.2
@@ -0,0 +1,17 @@
+-----BEGIN CERTIFICATE-----
+MIICrzCCAZcCFGMKRtmMLuut+sxC+TbWQfum7oXZMA0GCSqGSIb3DQEBCwUAMBQx
+EjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0yMzA1MDYxMTA2MjNaFw0zMzA1MDMxMTA2
+MjNaMBQxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBAKazNKPaH8LDzcaZRvBLrDNJkL1pukmB36mbczj07hZVbPmS/hyB
+vAdBFom0ZTw5dIpsUtRSZbDPrsCVpdY9O1jxwhrDi6mfvyJtKLEbTW4PvARqWwDh
+pa2SYwBMI+0ilXWTAzwJuWT1NhuUsAcB6SGwkNm3iKqZUDxn3V2L2AHRcKEJ9Zn9
+ePP4BsvtAS8ZBLxTnoo7R2SHiWwjDwuTtS4fQ5bWzGkmdmeuJ7JJt4ZzQV+mMBqr
+K3XVi+MXayvt5affGvHj/KuhlVBXHnUgSvEgFpuhK9elsds2iRho8mp1d0iHEIMp
+9LHVftsIpUxbKt/Pa/JL7oG9LBIvPj/SIjsCAwEAATANBgkqhkiG9w0BAQsFAAOC
+AQEAjPAtZs1by2h/1fr/ypojw16llzbReT8J+T8YHSTf6YwjoE83I0QDOLEo1ax+
+e/8qyQLs0EnlfdomNyA4Z/ECbY5c1nY0Dp//u6WH7AwLUx5HiwUw4Fmxu9Q/oB1o
+3vhIPl5Vd/VpdxDzuO8q8WvagwjVaxsZP3PVaBDRzZZPldPgTakfk+w5XnjNfgJi
+RDRutTRe6KBOxt7PAzAVV71FtOIq0b4xCNJGNurYBhRgZ5iQ7yMw+I5Vte1TakWr
+9gfE/yoKbU1W+y0QxSDTsnTCO4i3mXmBTuceTVWELwqZcr34W7n3vD8UtZQfanML
+cHCZaLPSMDuDtS74FSamP3i+oQ==
+-----END CERTIFICATE-----
--- a/tests/mutual-auth/certs/server/server.der
+++ b/tests/mutual-auth/certs/server/server.der
--- a/tests/mutual-auth/certs/server/server.key
+++ b/tests/mutual-auth/certs/server/server.key
@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCmszSj2h/Cw83G
+mUbwS6wzSZC9abpJgd+pm3M49O4WVWz5kv4cgbwHQRaJtGU8OXSKbFLUUmWwz67A
+laXWPTtY8cIaw4upn78ibSixG01uD7wEalsA4aWtkmMATCPtIpV1kwM8Cblk9TYb
+lLAHAekhsJDZt4iqmVA8Z91di9gB0XChCfWZ/Xjz+AbL7QEvGQS8U56KO0dkh4ls
+Iw8Lk7UuH0OW1sxpJnZnrieySbeGc0FfpjAaqyt11YvjF2sr7eWn3xrx4/yroZVQ
+Vx51IErxIBaboSvXpbHbNokYaPJqdXdIhxCDKfSx1X7bCKVMWyrfz2vyS+6BvSwS
+Lz4/0iI7AgMBAAECggEAKvz2u7Rh0WOWGrtnQEt7bkRv13C+8frUd1QXnB4JkefY
+sOmXrzlDiGlgCwXiv2ufopy5pXhUMgr0qUROHlfvCIpbwHQh/Y2tCA83WajNSG81
+ULwumKUYCRFBh4+bCimLemT9hguJ7D+SAv3OgRgciywRxpteWoQr3U/5lYidHSZ/
+gv13lVKbn72zD5opeGA2hS1MlLZV/xueSvhT3lzbv61hqdersACj2Tvi8O2/imVy
+XjEZnPRQhlFPtiAN5J3on6Xo+MqieuhA3yxhBrYoLsMrTCK6ePThdXfcgmpEjQ8l
+6HxNmnPri5KbxCTbGgCjMiSidnRim2IpBMEP32eN8QKBgQDLr2CoMdyliFU6Gm1P
+rxWTMnvzdVbXUp4B8YEdyNyKdWt50cqbB3UvnFX2gpELYdy3uYcXTKm+Nynruam1
+Z/Ya1HXwN+wdgQhjq9n4izLvEfUkXWDNNikQmts8Uxkp+uEK+OOp1/NjZlA6YdS3
+crq5wPxLoAP2JxiaoIJF74QMqwKBgQDRhABgZbWVHwPcLVVqZ5+MJYvQORqIqapf
+kGe/jR/CMC0Tkop2O1tY3f68bMNKkXfj7QEtDlppMswZ9MOqBBr56yGZzrQa+cB3
+lF4+hP06OvyIkdmZlP4NHm/DtF9gt1KjWPF2VcIfD3VfZO8E/XJn2n1KnKCU+4cb
+lyJYi9AgsQKBgQCWkgPy8kE5QSo3tJeAI17gnJ5SoDhdHp7dsukO2pBl7l1QBY0v
+w3iWhIxrmaOddW+ThZve1nZYvjDIKEzTZJHizZKNzNlICj3oaH7OpCA36N9+TWUk
+7le3BbLxykA870/zK4Ao6xHqNhUyw2VbY32zmX0obpbfHZGrpOIIzwGf1wKBgDlM
+U1oJls5QbBrT3w85hZ2rSwBIDaSgWfLGqEjvjGbsC/fVVL6e3w1/sMHRMNt8yv/v
+einbSgiJFt5mXPhrJQGCN28742+ZK/TIA7ovXp2FMjkbQhpJb+0gjMpF0uu9VwFL
+OsX1ECC0dpH/JYsE0TvrueYkzZnQ7BM0kvUKT4IRAoGAOPVed0zkDh3iobQ3A3IG
+JepRygabC68iOHlrD6sVxST0HdyP9pxwMe9gnz5TDAZkWvhJV0UUmaMCpbShsc+n
+ymKSNnXAxt+G6XHH3Mg9aDNi70og4HhhT6dU2579xUOBY2057ZgpWXK3rf+JKls4
+XlkplyHw0UqkEhCw+FMa3Gs=
+-----END PRIVATE KEY-----
--- a/tests/mutual-auth/gen-certs.sh
+++ b/tests/mutual-auth/gen-certs.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# Create server and client certificate directories
+mkdir -p certs/server
+mkdir -p certs/client
+
+# Generate server key
+openssl genrsa -out certs/server/server.key 2048
+
+# Generate a Certificate Signing Request (CSR) for the server key
+openssl req -new -key certs/server/server.key -out certs/server/server.csr -subj "/CN=localhost"
+
+# Self-sign the server CSR to create the server certificate
+openssl x509 -req -in certs/server/server.csr -signkey certs/server/server.key -out certs/server/server.crt -days 3650
+
+# Generate server-side Certificate Authority (CA) file
+openssl req -x509 -nodes -new -key certs/server/server.key -sha256 -days 3650 -out certs/server/ca.crt -subj "/CN=ServerCA"
+
+# Generate client key
+openssl genrsa -out certs/client/client.key 2048
+
+# Generate a Certificate Signing Request (CSR) for the client key
+openssl req -new -key certs/client/client.key -out certs/client/client.csr -subj "/CN=Client"
+
+# Sign the client CSR with the server CA to create the client certificate
+openssl x509 -req -in certs/client/client.csr -CA certs/server/ca.crt -CAkey certs/server/server.key -CAcreateserial -out certs/client/client.crt -days 365
+
+# Cleanup
+rm -f certs/server/server.csr
+rm -f certs/client/client.csr
+
--- a/tests/test_banner.rs
+++ b/tests/test_banner.rs
@@ -661,6 +661,70 @@ fn banner_prints_recursion_depth() {
        );
 }

+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + server certs
+fn banner_prints_server_certs() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--server-certs")
+        .arg("tests/mutual-auth/certs/server/server.crt.1")
+        .arg("tests/mutual-auth/certs/server/server.crt.2")
+        .arg("--wordlist")
+        .arg("/definitely/doesnt/exist/0cd7fed0-47f4-4b18-a1b0-ac39708c1676")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Server Certificates"))
+                .and(predicate::str::contains("server.crt.1"))
+                .and(predicate::str::contains("server.crt.2"))
+                .and(predicate::str::contains("─┴─")),
+        );
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + server certs
+fn banner_prints_client_cert_and_key() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--client-cert")
+        .arg("tests/mutual-auth/certs/client/client.crt")
+        .arg("--client-key")
+        .arg("tests/mutual-auth/certs/client/client.key")
+        .arg("--wordlist")
+        .arg("/definitely/doesnt/exist/0cd7fed0-47f4-4b18-a1b0-ac39708c1676")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Client Certificate"))
+                .and(predicate::str::contains("Client Key"))
+                .and(predicate::str::contains("certs/client/client.crt"))
+                .and(predicate::str::contains("certs/client/client.key"))
+                .and(predicate::str::contains("─┴─")),
+        );
+}
+
 #[test]
 /// test allows non-existent wordlist to trigger the banner printing to stderr
 /// expect to see all mandatory prints + no recursion
@@ -1366,6 +1430,7 @@ fn banner_prints_all_composite_settings_burp() {
                .and(predicate::str::contains("─┴─")),
        );
 }
+
 #[test]
 /// test allows non-existent wordlist to trigger the banner printing to stderr
 /// expect to see all mandatory prints + collect words
@@ -1420,3 +1485,15 @@ fn banner_prints_force_recursion() {
                .and(predicate::str::contains("─┴─")),
        );
 }
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + force recursion
+fn banner_prints_update_app() {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--update")
+        .assert()
+        .success()
+        .stdout(predicate::str::contains("Checking target-arch..."));
+}
--- a/tests/test_config.rs
+++ b/tests/test_config.rs
@@ -1,5 +1,6 @@
 mod utils;
 use assert_cmd::prelude::*;
+use httpmock::MockServer;
 use predicates::prelude::*;
 use std::process::Command;
 use utils::{setup_tmp_directory, teardown_tmp_directory};
@@ -7,13 +8,15 @@ use utils::{setup_tmp_directory, teardown_tmp_directory};
 #[test]
 /// send a single valid request, expect a 200 response
 fn read_in_config_file_for_settings() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+
    let (tmp_dir, file) = setup_tmp_directory(&["threads = 37".to_string()], "ferox-config.toml")?;

    Command::cargo_bin("feroxbuster")
        .unwrap()
        .current_dir(&tmp_dir)
        .arg("--url")
-        .arg("http://localhost")
+        .arg(srv.url("/"))
        .arg("--wordlist")
        .arg(file.as_os_str())
        .arg("-vvvv")
--- a/tests/test_extractor.rs
+++ b/tests/test_extractor.rs
@@ -640,3 +640,73 @@ fn extractor_recurses_into_403_directories() -> Result<(), Box<dyn std::error::E
    teardown_tmp_directory(tmp_dir);
    Ok(())
 }
+
+#[test]
+/// robots.txt requests shouldn't fire when --dont-extract-links is used
+fn robots_text_extraction_doesnt_run_with_dont_extract_links() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("im a little teapot"); // 18
+    });
+
+    let mock_two = srv.mock(|when, then| {
+        when.method(GET).path("/robots.txt");
+        then.status(200).body(
+            r#"
+            User-agent: *
+            Crawl-delay: 10
+            # CSS, JS, Images
+            Allow: /misc/*.css$
+            Disallow: /misc/stupidfile.php
+               Disallow: /disallowed-subdir/
+            "#,
+        );
+    });
+
+    let mock_file = srv.mock(|when, then| {
+        when.method(GET).path("/misc/stupidfile.php");
+        then.status(200).body("im a little teapot too"); // 22
+    });
+
+    let mock_scanned_file = srv.mock(|when, then| {
+        when.method(GET).path("/misc/LICENSE");
+        then.status(200).body("i too, am a container for tea"); // 29
+    });
+
+    let mock_dir = srv.mock(|when, _| {
+        when.method(GET).path("/misc/");
+    });
+
+    let mock_disallowed = srv.mock(|when, then| {
+        when.method(GET).path("/disallowed-subdir");
+        then.status(404);
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--dont-extract-links")
+        .arg("--no-recursion")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .and(predicate::str::contains("18c"))
+            .and(predicate::str::contains("/misc/stupidfile.php"))
+            .not(),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_dir.hits(), 0);
+    assert_eq!(mock_two.hits(), 0);
+    assert_eq!(mock_file.hits(), 0);
+    assert_eq!(mock_disallowed.hits(), 0);
+    assert_eq!(mock_scanned_file.hits(), 0);
+    teardown_tmp_directory(tmp_dir);
+}
--- a/tests/test_filters.rs
+++ b/tests/test_filters.rs
@@ -247,3 +247,46 @@ fn filters_similar_should_filter_response() {
    assert_eq!(not_similar.hits(), 1);
    teardown_tmp_directory(tmp_dir);
 }
+
+#[test]
+/// when using --collect-backups, should only see results in output
+/// when the response shouldn't be otherwise filtered
+fn collect_backups_should_be_filtered() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
+
+    let mock = srv.mock(|when: httpmock::When, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
+
+    let mock_two = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE.bak");
+        then.status(201)
+            .body("im a backup file, but filtered out because im not 200");
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--status-codes")
+        .arg("200")
+        .arg("--collect-backups")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .and(predicate::str::contains("200"))
+            .and(predicate::str::contains("/LICENSE.bak"))
+            .not()
+            .and(predicate::str::contains("201"))
+            .not(),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_two.hits(), 1);
+    teardown_tmp_directory(tmp_dir);
+}
--- a/tests/test_heuristics.rs
+++ b/tests/test_heuristics.rs
@@ -164,7 +164,7 @@ fn test_static_wildcard_request_found() -> Result<(), Box<dyn std::error::Error>

    let mock = srv.mock(|when, then| {
        when.method(GET)
-            .path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap());
+            .path_matches(Regex::new("/[.a-zA-Z0-9]{32,}/").unwrap());
        then.status(200).body("this is a test");
    });

@@ -188,7 +188,8 @@ fn test_static_wildcard_request_found() -> Result<(), Box<dyn std::error::Error>
            .and(predicate::str::contains("1l")),
    );

-    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock.hits(), 6);
+
    Ok(())
 }

@@ -305,11 +306,67 @@ fn heuristics_wildcard_test_with_two_static_wildcards_with_silent_enabled(
        .success()
        .stdout(predicate::str::contains(srv.url("/")));

-    assert_eq!(mock.hits(), 4);
+    assert_eq!(mock.hits(), 6);
    assert_eq!(mock2.hits(), 1);
    Ok(())
 }

+#[test]
+/// test finds a 404-like response that returns a 403 and a 403 directory should still be allowed
+/// to be tested for recrusion
+fn heuristics_wildcard_test_that_auto_filtering_403s_still_allows_for_recursion_into_403_directories(
+) -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+
+    let super_long = String::from("92969beae6bf4beb855d1622406d87e395c87387a9ad432e8a11245002b709b03cf609d471004154b83bcc1c6ec49f6f09d471004154b83bcc1c6ec49f6f");
+
+    let (tmp_dir, file) =
+        setup_tmp_directory(&["LICENSE".to_string(), super_long.clone()], "wordlist")?;
+
+    srv.mock(|when, then| {
+        when.method(GET)
+            .path_matches(Regex::new("/.?[a-zA-Z0-9]{32,103}").unwrap());
+        then.status(403)
+            .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+    });
+
+    srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE/");
+        then.status(403)
+            .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+    });
+
+    srv.mock(|when, then| {
+        when.method(GET).path(format!("/LICENSE/{}", super_long));
+        then.status(200);
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--add-slash")
+        .unwrap();
+
+    teardown_tmp_directory(tmp_dir);
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("GET")
+            .and(predicate::str::contains(
+                "Auto-filtering found 404-like response and created new filter",
+            ))
+            .and(predicate::str::contains("403"))
+            .and(predicate::str::contains("1l"))
+            .and(predicate::str::contains("4w"))
+            .and(predicate::str::contains("46c"))
+            .and(predicate::str::contains(srv.url("/LICENSE/LICENSE/"))),
+    );
+
+    Ok(())
+}
+
 // #[test]
 // /// test finds a static wildcard and reports as much to stdout and a file
 // fn heuristics_wildcard_test_with_two_static_wildcards_and_output_to_file() {
--- a/tests/test_main.rs
+++ b/tests/test_main.rs
@@ -218,3 +218,46 @@ fn main_parallel_creates_output_directory() -> Result<(), Box<dyn std::error::Er

    Ok(())
 }
+
+#[test]
+/// download a wordlist from a url
+fn main_download_wordlist_from_url() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+
+    let (tmp_dir, _) = setup_tmp_directory(&["a".to_string()], "wordlist")?;
+
+    let mock1 = srv.mock(|when, then| {
+        when.method(GET).path("/derp");
+        then.status(200).body("stuff\nthings");
+    });
+
+    // serve endpoints stuff and things
+    let mock2 = srv.mock(|when, then| {
+        when.method(GET).path("/stuff");
+        then.status(200);
+    });
+
+    let mock3 = srv.mock(|when, then| {
+        when.method(GET).path("/things");
+        then.status(200);
+    });
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .current_dir(&tmp_dir)
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(srv.url("/derp"))
+        .assert()
+        .success()
+        .stderr(predicate::str::contains(srv.url("/derp")));
+
+    teardown_tmp_directory(tmp_dir);
+
+    assert_eq!(mock1.hits(), 1); // downloaded wordlist
+    assert_eq!(mock2.hits(), 1); // found stuff from wordlist
+    assert_eq!(mock3.hits(), 1); // found things from wordlist
+
+    Ok(())
+}
--- a/tests/test_scanner.rs
+++ b/tests/test_scanner.rs
@@ -693,7 +693,7 @@ fn collect_backups_makes_appropriate_requests() {
    let srv = MockServer::start();
    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE.txt".to_string()], "wordlist").unwrap();

-    let valid_paths = vec![
+    let valid_paths = [
        "/LICENSE.txt",
        "/LICENSE.txt~",
        "/LICENSE.txt.bak",
Author	SHA1	Message	Date
allcontributors[bot]	fe71f288e3	docs: add RavySena as a contributor for ideas (#1025 ) * docs: update README.md [skip ci] * docs: update .all-contributorsrc [skip ci] --------- Co-authored-by: allcontributors[bot] <46447321+allcontributors[bot]@users.noreply.github.com>	2023-11-08 21:24:35 -05:00
allcontributors[bot]	a38a0444fe	docs: add ocervell as a contributor for ideas (#1024 ) * docs: update README.md [skip ci] * docs: update .all-contributorsrc [skip ci] --------- Co-authored-by: allcontributors[bot] <46447321+allcontributors[bot]@users.noreply.github.com>	2023-11-08 21:23:26 -05:00
allcontributors[bot]	2938094c73	docs: add devx00 as a contributor for bug (#1023 ) * docs: update README.md [skip ci] * docs: update .all-contributorsrc [skip ci] --------- Co-authored-by: allcontributors[bot] <46447321+allcontributors[bot]@users.noreply.github.com>	2023-11-08 21:22:40 -05:00
epi	55c67358d6	allow --json in conjunction with --silent (#1022 ) * updated parser to allow --silent with --json * updated config parsing with new silentjson output variant * added new silentjson output variant * updated outputlevel usage to include new variant	2023-11-08 21:18:53 -05:00
epi	c3c6fc6753	add http/2 support (#1020 ) * added http/2 support * updated deps	2023-11-08 06:22:00 -05:00
epi	a28ff857ca	added test for robots/--dont-extract-links	2023-11-03 06:38:55 -04:00
epi	6c0fe90909	fixed collect backups filtering (#1016 ) * fixed collect backups filtering and clippy * added test for filtered backups	2023-11-03 06:28:09 -04:00
epi	bc486ac8d3	nitpickery; added success msg	2023-10-04 21:43:17 -04:00
epi	fa9d42554f	Merge pull request #1001 from epi052/all-contributors/add-N0ur5 docs: add N0ur5 as a contributor for bug	2023-10-04 21:45:11 -04:00
allcontributors[bot]	b78dbe6cc4	docs: update .all-contributorsrc [skip ci]	2023-10-05 01:45:01 +00:00
allcontributors[bot]	29f616f51a	docs: update README.md [skip ci]	2023-10-05 01:45:00 +00:00
epi	c1ba5cf942	fixed unwritable cwd bug	2023-10-04 20:53:49 -04:00
epi	e3ec3aee3a	Merge pull request #980 from epi052/all-contributors/add-sawmj docs: add sawmj as a contributor for bug	2023-09-11 21:18:35 -04:00
allcontributors[bot]	52db396aa9	docs: update .all-contributorsrc [skip ci]	2023-09-12 01:17:22 +00:00
allcontributors[bot]	e1066cd5c7	docs: update README.md [skip ci]	2023-09-12 01:17:21 +00:00
epi	d90ee38aad	added error message in response to issue #977	2023-09-11 21:10:42 -04:00
epi	a3501ac494	clippy	2023-09-11 08:02:35 -04:00
epi	23827a1d45	fmt	2023-09-11 07:57:54 -04:00
epi	a2b04b2b5e	Merge pull request #978 from epi052/all-contributors/add-andreademurtas docs: add andreademurtas as a contributor for code	2023-09-11 08:00:45 -04:00
epi	362633bc63	Merge pull request #976 from andreademurtas/extensions-from-file Enable reading extensions from file	2023-09-11 08:00:26 -04:00
allcontributors[bot]	08c5b2bf67	docs: update .all-contributorsrc [skip ci]	2023-09-11 12:00:21 +00:00
allcontributors[bot]	ccef4fd713	docs: update README.md [skip ci]	2023-09-11 12:00:20 +00:00
Andrea De Murtas	4afe0cf95c	Update src/config/container.rs Co-authored-by: epi <43392618+epi052@users.noreply.github.com>	2023-09-10 18:28:48 +02:00
Andrea De Murtas	564686bc5a	Update src/config/container.rs Co-authored-by: epi <43392618+epi052@users.noreply.github.com>	2023-09-10 18:28:42 +02:00
Andrea De Murtas	83f90529e9	Update src/config/container.rs Co-authored-by: epi <43392618+epi052@users.noreply.github.com>	2023-09-10 18:28:35 +02:00
Andrea De Murtas	ad49320968	enable reading extensions from file	2023-09-07 19:27:31 +02:00
epi	70946ad916	Merge pull request #938 from epi052/all-contributors/add-ktecv2000 docs: add ktecv2000 as a contributor for bug	2023-07-11 05:35:21 -05:00
allcontributors[bot]	fd5c5af5fa	docs: update .all-contributorsrc [skip ci]	2023-07-11 10:35:12 +00:00
allcontributors[bot]	ff32aba1db	docs: update README.md [skip ci]	2023-07-11 10:35:11 +00:00
epi	cbf028a8ac	Merge pull request #937 from epi052/all-contributors/add-sectroyer docs: add sectroyer as a contributor for bug, and ideas	2023-07-11 05:34:39 -05:00
allcontributors[bot]	8bf80f4eda	docs: update .all-contributorsrc [skip ci]	2023-07-11 10:34:19 +00:00
allcontributors[bot]	7c2d09cc22	docs: update README.md [skip ci]	2023-07-11 10:34:18 +00:00
epi	0fb682c121	Merge pull request #936 from epi052/935-fix-scan-menu-range-issue 935 fix scan menu range issue	2023-07-11 05:32:59 -05:00
epi	bcfd8b6eef	fixed unwrap in nlp::document	2023-07-11 06:23:18 -04:00
epi	1c9235a56b	dont show cancelled scans in scan menu	2023-07-11 06:04:34 -04:00
epi	4d787f08d0	kept indicatif pinned to 17.3 due to bug crashing scan menu	2023-07-11 06:04:09 -04:00
epi	0c7520f5ee	clippy	2023-07-11 05:25:53 -04:00
epi	83b55aaf10	updated deps	2023-07-11 05:22:11 -04:00
epi	aea64324f7	cancel scans ignores scantypes of file	2023-07-11 05:14:33 -04:00
epi	8d0614b1a5	Merge pull request #898 from epi052/all-contributors/add-AkechiShiro docs: add AkechiShiro as a contributor for ideas	2023-05-06 06:47:16 -05:00
allcontributors[bot]	d19cf58af3	docs: update .all-contributorsrc [skip ci]	2023-05-06 11:46:58 +00:00
allcontributors[bot]	bd44bacf95	docs: update README.md [skip ci]	2023-05-06 11:46:57 +00:00
epi	2bf5dc5e6f	updated deps	2023-05-06 06:44:18 -05:00
epi	e5fadde073	Merge pull request #892 from lavafroth/client_ssl_cert Adds certificate management options (unknown servers & mTLS)	2023-05-06 06:38:32 -05:00
epi	ac3fdb1975	added mutual auth testing server and cert generating script	2023-05-06 06:21:08 -05:00
epi	ff40549140	added a little more context to connection errors	2023-05-06 05:54:05 -05:00
epi	0cd25eedfc	added client init tests; removed extension requirement	2023-05-06 05:47:22 -05:00
epi	328d1d2ec9	bumped version to 2.10.0	2023-05-06 04:54:17 -05:00
epi	68cc6bc748	nitpickery	2023-05-06 04:53:28 -05:00
epi	f44f320a49	added banner tests	2023-05-06 04:53:10 -05:00
Himadri Bhattacharjee	0965379b9a	cargo fmt --all	2023-05-06 09:24:31 +05:30
Himadri Bhattacharjee	4afbf77631	do not use option for server_certs since next() on an empty iterator yields None	2023-05-06 09:14:19 +05:30
epi	5385ce5e99	bumped version to 2.9.6	2023-05-05 18:48:58 -05:00
epi	c8a50d9c0c	added new config values to ferox-config example	2023-05-05 18:47:56 -05:00
epi	a3c887f2d7	added config tests for new values	2023-05-05 18:47:01 -05:00
epi	e094dab4a4	key requires cert; accept multiple server certs	2023-05-05 07:07:06 -05:00
epi	c307e6d56d	fixed missing client cert issues	2023-05-05 06:32:46 -05:00
Himadri Bhattacharjee	d27cb57d66	simplify certificate and key parsing logic	2023-05-04 08:52:19 +05:30
Himadri Bhattacharjee	372f7c5cd4	add client_key flag to separated PEM key file for identity from_pkcs8_pem	2023-05-04 08:35:25 +05:30
Himadri Bhattacharjee	4986ebdaae	cargo clippy	2023-05-02 12:47:47 +05:30
Himadri Bhattacharjee	4198a019d3	added functionality to use SSL key as client identity for mutual authentication	2023-05-02 12:43:45 +05:30
Himadri Bhattacharjee	3b8c6f6ba9	added doc comments for certificate field in Configuration struct	2023-04-30 08:36:47 +05:30
Himadri Bhattacharjee	39f8259f31	documentation for changes in client	2023-04-30 08:36:40 +05:30
Himadri Bhattacharjee	3f5ff1ad3e	create Some or None variants for proxy and certificate ahead of time to avoid multiple initializations.	2023-04-30 08:30:17 +05:30
Himadri Bhattacharjee	12206e668f	added basic functionality to add root cert to client	2023-04-29 21:07:08 +05:30
Himadri Bhattacharjee	1796e4eeb2	added cert flag	2023-04-29 18:20:04 +05:30
epi	70a8d0f5df	Merge pull request #889 from epi052/all-contributors/add-lavafroth docs: add lavafroth as a contributor for code, and ideas	2023-04-26 19:35:58 -05:00
allcontributors[bot]	11831a3ab5	docs: update .all-contributorsrc [skip ci]	2023-04-27 00:35:40 +00:00
allcontributors[bot]	9356b058eb	docs: update README.md [skip ci]	2023-04-27 00:35:39 +00:00
epi	df490f6224	update	2023-04-26 19:35:28 -05:00
epi	4079551c96	update	2023-04-26 19:34:18 -05:00
epi	6d0658a635	update	2023-04-26 19:33:14 -05:00
epi	890519f39c	Merge pull request #888 from epi052/all-contributors/add-aroly docs: add aroly as a contributor for ideas, and code	2023-04-26 19:32:09 -05:00
allcontributors[bot]	abf18b0481	docs: update .all-contributorsrc [skip ci]	2023-04-27 00:29:40 +00:00
allcontributors[bot]	409844ed05	docs: update README.md [skip ci]	2023-04-27 00:29:39 +00:00
epi	1cf37e38a2	Merge pull request #884 from epi052/878-support-raw-urls 878 support raw urls	2023-04-26 06:59:04 -05:00
epi	9876759606	nitpickery	2023-04-26 06:45:13 -05:00
epi	4150b61a42	fixed windows logic	2023-04-26 06:33:43 -05:00
epi	16d34bbee0	bumped version to 2.9.5	2023-04-25 07:10:48 -05:00
epi	f1fd2fc379	updated Url::parse callsites to use the new utility function	2023-04-25 07:09:56 -05:00
epi	3dd070a0db	fmt	2023-04-24 06:20:14 -05:00
epi	a3dc6c97a0	added workaround to add partial support for raw urls	2023-04-24 06:19:21 -05:00
epi	ec78ec3049	added ability to specify install directory for install-nix.sh	2023-04-19 17:15:50 -05:00
epi	960536e918	Merge pull request #879 from epi052/all-contributors/add-DrorDvash docs: add DrorDvash as a contributor for bug	2023-04-19 08:05:15 -05:00
allcontributors[bot]	fdae9aa9d6	docs: update .all-contributorsrc [skip ci]	2023-04-19 13:03:50 +00:00
allcontributors[bot]	5c73c3fb23	docs: update README.md [skip ci]	2023-04-19 13:03:49 +00:00
epi	02ef6d7e3f	Merge pull request #877 from epi052/update-indicatif-finally Random improvements	2023-04-19 07:59:47 -05:00
epi	3378246820	updated arm release names for --update fix	2023-04-19 07:46:43 -05:00
epi	692db93048	clippy/tests and added logic to wait for link extraction if done	2023-04-19 06:57:36 -05:00
epi	233cf99907	made link extraction req/resp async	2023-04-19 06:56:52 -05:00
epi	8cd9918b76	upgraded deps	2023-04-19 06:55:23 -05:00
epi	66bcbfc2f2	bumped version to 2.9.4	2023-04-19 06:51:35 -05:00
epi	8b127c0093	made 404-like req/resp async	2023-04-17 06:37:28 -05:00
epi	94de58d855	removed response body from mpsc traversal	2023-04-17 06:36:47 -05:00
epi	2b95b7be69	updated indicatif to 0.17.3	2023-04-17 06:26:59 -05:00
epi	e77c1314b1	Merge pull request #869 from epi052/auto-filtering-account-for-extensions added extensions and status codes into auto filtering decision calculus	2023-04-11 19:07:53 -05:00
epi	1ced3b5d77	modified msg when dir listing is found with dont-extract	2023-04-11 18:48:18 -05:00
epi	b5472f5341	updated deps	2023-04-11 18:39:28 -05:00
epi	ea81600850	clippy	2023-04-11 18:36:37 -05:00
epi	4f679592b8	bumped version to 2.9.3	2023-04-11 18:34:02 -05:00
epi	b375893461	nitpickery	2023-04-11 18:32:56 -05:00
epi	e110f86f39	added extensions and status codes into auto filtering decision calculus	2023-04-11 18:29:12 -05:00
epi	c7498a7695	Merge pull request #839 from epi052/all-contributors/add-acut3 docs: add acut3 as a contributor for bug	2023-03-18 12:23:34 -05:00
allcontributors[bot]	f973baaba8	docs: update .all-contributorsrc [skip ci]	2023-03-18 17:23:25 +00:00
allcontributors[bot]	148982cdc4	docs: update README.md [skip ci]	2023-03-18 17:23:24 +00:00
epi	5d96658c79	Merge pull request #834 from epi052/827-load-wordlist-from-url load wordlist from url; change some defaults/fix some bugs	2023-03-18 11:59:23 -05:00
epi	46d00507b0	removed cruft	2023-03-18 11:52:58 -05:00
epi	d561e59ec9	added test	2023-03-18 11:44:45 -05:00
epi	b786578c03	Merge pull request #824 from aancw/docs-package Update alternative installation method for brew and chocolatey	2023-03-18 07:13:38 -05:00
epi	bd54ad0087	Merge branch 'main' into 827-load-wordlist-from-url	2023-03-18 07:09:14 -05:00
epi	d98c6a7457	bumped deps	2023-03-18 07:07:40 -05:00
epi	c493d001b5	fmt clippy etc	2023-03-18 07:02:45 -05:00
epi	bd4566fa7b	updated parser text	2023-03-18 07:01:07 -05:00
epi	8fbf9d0274	-w accepts http/https urls	2023-03-18 06:59:19 -05:00
epi	d6b10c6476	reverted collect-backups change	2023-03-18 06:07:38 -05:00
epi	a5e845864c	Merge branch 'main' of github.com:epi052/feroxbuster	2023-03-17 06:47:19 -05:00
epi	b02358678b	added check for force-recursion to dirlisting check	2023-03-17 06:47:13 -05:00
epi	1b8fdcec17	hid old false defaults; added dont-* flags	2023-03-17 06:32:28 -05:00
epi	92cc2ab448	fixed test	2023-03-17 06:31:23 -05:00
epi	0b0e08ae4f	updated extract-links and collect-backups default to true	2023-03-17 05:45:19 -05:00
epi	25762395b1	Merge pull request #833 from epi052/all-contributors/add-imBigo docs: add imBigo as a contributor for bug	2023-03-16 21:30:12 -05:00
allcontributors[bot]	55b4034bd0	docs: update .all-contributorsrc [skip ci]	2023-03-17 02:29:52 +00:00
allcontributors[bot]	ffa409ca3d	docs: update README.md [skip ci]	2023-03-17 02:29:51 +00:00
epi	bb4a335299	fixed divide by zero error	2023-03-16 21:23:39 -05:00
epi	1e0ec5c833	fixed divide by zero error	2023-03-16 21:21:05 -05:00
Aan	b5fa6b149e	Update alternative installation method for brew and chocolatey	2023-03-12 22:05:05 +07:00
epi	04a43a0892	Merge pull request #823 from epi052/819-fix-resume-with-offset fix resume with offset	2023-03-12 07:02:30 -05:00
epi	8a72e498e6	updated deps	2023-03-12 06:41:47 -05:00
epi	2987a84776	cleaned up another prog bar logic issue	2023-03-12 06:28:59 -05:00
epi	8add5599fb	fixed the prog bar # issue	2023-03-12 06:28:22 -05:00
epi	9f557329eb	fixed indexing out of bounds w/ extensions/methods on resume	2023-03-11 07:34:17 -06:00
epi	c04bf4a703	Merge pull request #807 from aancw/chocolatey Adding feroxbuster as chocolatey package	2023-03-11 06:19:26 -06:00
epi	03e8625c6e	Merge pull request #821 from epi052/816-fix-scan-mgt-menu-things fix scan mgt menu things	2023-03-10 21:20:50 -06:00
epi	5d6b85fe12	clippy/fmt	2023-03-10 21:10:26 -06:00
epi	771041d225	added ability to stop previously unstoppable scans	2023-03-10 20:43:12 -06:00
epi	b5debed322	merged main	2023-03-10 19:42:44 -06:00
epi	30407cd338	fixed broken test	2023-03-10 16:19:52 -06:00
epi	ba4b26f2cd	Update README.md	2023-03-10 16:15:23 -06:00
epi	4fdf558936	Merge pull request #820 from epi052/all-contributors/add-aancw docs: add aancw as a contributor for ideas	2023-03-10 16:14:24 -06:00
allcontributors[bot]	2ffb0df516	docs: update .all-contributorsrc [skip ci]	2023-03-10 22:14:04 +00:00
allcontributors[bot]	10260f9db7	docs: update README.md [skip ci]	2023-03-10 22:14:03 +00:00
epi	4067be2f82	Merge pull request #813 from aancw/update-package Implement auto update feature	2023-03-10 16:13:45 -06:00
Aan	7cb9c1c914	remove old commented code	2023-03-10 20:47:08 +07:00
Aan	99cbd657a5	Update parser, banner & test, exception handling, etc	2023-03-10 20:44:34 +07:00
Aan	703da383a7	Fix for fmt, clippy and nextest	2023-03-09 11:28:11 +07:00
Aan	aa83e40c4f	Update README.md	2023-03-09 10:55:09 +07:00
Aan	a77c436e04	New feature checklist	2023-03-09 10:49:25 +07:00
Aan	c3455d123e	Implement auto update feature	2023-03-09 10:06:17 +07:00
Aan	6431f01f12	Update iconUrl and copyright year in nuspec	2023-03-09 07:02:14 +07:00
Aan	fd0f31705d	Update Copyright year in license	2023-03-08 14:49:35 +07:00
Aan	5252587e65	Adding feroxbuster as chocolatey package	2023-03-06 21:56:44 +07:00