updated dependencies

* docs: update README.md [skip ci]

* docs: update .all-contributorsrc [skip ci]

---------

Co-authored-by: allcontributors[bot] <46447321+allcontributors[bot]@users.noreply.github.com>

.all-contributorsrc

@@ -198,7 +198,8 @@
       "avatar_url": "https://avatars.githubusercontent.com/u/24260009?v=4",
       "profile": "https://github.com/N0ur5",
       "contributions": [
-        "ideas"
+        "ideas",
+        "bug"
       ]
     },
     {
@@ -608,6 +609,88 @@
       "contributions": [
         "ideas"
       ]
+    },
+    {
+      "login": "sectroyer",
+      "name": "sectroyer",
+      "avatar_url": "https://avatars.githubusercontent.com/u/6706818?v=4",
+      "profile": "https://github.com/sectroyer",
+      "contributions": [
+        "bug",
+        "ideas"
+      ]
+    },
+    {
+      "login": "ktecv2000",
+      "name": "ktecv2000",
+      "avatar_url": "https://avatars.githubusercontent.com/u/19836003?v=4",
+      "profile": "https://medium.com/@b3rm1nG",
+      "contributions": [
+        "bug"
+      ]
+    },
+    {
+      "login": "andreademurtas",
+      "name": "Andrea De Murtas",
+      "avatar_url": "https://avatars.githubusercontent.com/u/56048157?v=4",
+      "profile": "http://untrue.me",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "sawmj",
+      "name": "sawmj",
+      "avatar_url": "https://avatars.githubusercontent.com/u/30024085?v=4",
+      "profile": "https://github.com/sawmj",
+      "contributions": [
+        "bug"
+      ]
+    },
+    {
+      "login": "devx00",
+      "name": "Zach Hanson",
+      "avatar_url": "https://avatars.githubusercontent.com/u/6897405?v=4",
+      "profile": "https://github.com/devx00",
+      "contributions": [
+        "bug"
+      ]
+    },
+    {
+      "login": "ocervell",
+      "name": "Olivier Cervello",
+      "avatar_url": "https://avatars.githubusercontent.com/u/9629314?v=4",
+      "profile": "https://github.com/ocervell",
+      "contributions": [
+        "ideas"
+      ]
+    },
+    {
+      "login": "RavySena",
+      "name": "RavySena",
+      "avatar_url": "https://avatars.githubusercontent.com/u/67729597?v=4",
+      "profile": "https://github.com/RavySena",
+      "contributions": [
+        "ideas"
+      ]
+    },
+    {
+      "login": "stuhlmann",
+      "name": "Florian Stuhlmann",
+      "avatar_url": "https://avatars.githubusercontent.com/u/11061864?v=4",
+      "profile": "https://github.com/stuhlmann",
+      "contributions": [
+        "bug"
+      ]
+    },
+    {
+      "login": "Mister7F",
+      "name": "Mister7F",
+      "avatar_url": "https://avatars.githubusercontent.com/u/35213773?v=4",
+      "profile": "https://github.com/Mister7F",
+      "contributions": [
+        "ideas"
+      ]
     }
   ],
   "contributorsPerLine": 7,
@@ -616,5 +699,6 @@
   "repoType": "github",
   "repoHost": "https://github.com",
   "skipCi": true,
-  "commitConvention": "angular"
+  "commitConvention": "angular",
+  "commitType": "docs"
 }

.github/workflows/build.yml

@@ -73,6 +73,35 @@ jobs:
           name: ${{ matrix.name }}.tar.gz
           path: ${{ matrix.name }}.tar.gz
 
+  build-debug:
+    env:
+      IN_PIPELINE: true
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install System Dependencies
+        run: |
+          env
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends libssl-dev pkg-config
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          target: x86_64-unknown-linux-musl
+          override: true
+      - uses: actions-rs/cargo@v1
+        env:
+          PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig
+          OPENSSL_DIR: /usr/lib/ssl
+        with:
+          use-cross: true
+          command: build
+          args: --target=x86_64-unknown-linux-musl
+      - uses: actions/upload-artifact@v2
+        with:
+          name: x86_64-linux-debug-feroxbuster
+          path: target/x86_64-unknown-linux-musl/debug/feroxbuster
+
   # build-deb:
   #   needs: [build-nix]
   #   runs-on: ubuntu-latest

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "feroxbuster"
-version = "2.10.0"
+version = "2.10.2"
 authors = ["Ben 'epi' Risher (@epi052)"]
 license = "MIT"
 edition = "2021"
@@ -22,40 +22,45 @@ build = "build.rs"
 maintenance = { status = "actively-developed" }
 
 [build-dependencies]
-clap = { version = "4.2", features = ["wrap_help", "cargo"] }
-clap_complete = "4.2"
-regex = "1.8"
+clap = { version = "4.4", features = ["wrap_help", "cargo"] }
+clap_complete = "4.4"
+regex = "1.10"
 lazy_static = "1.4"
 dirs = "5.0"
 
 [dependencies]
-scraper = "0.16"
+scraper = "0.18"
 futures = "0.3"
-tokio = { version = "1.28", features = ["full"] }
+tokio = { version = "1.35", features = ["full"] }
 tokio-util = { version = "0.7", features = ["codec"] }
 log = "0.4"
 env_logger = "0.10"
-reqwest = { version = "0.11", features = ["socks", "native-tls"] }
+reqwest = { version = "0.11", features = ["socks", "native-tls-alpn"] }
 # uses feature unification to add 'serde' to reqwest::Url
-url = { version = "2.3", features = ["serde"] }
+url = { version = "2.5", features = ["serde"] }
 serde_regex = "1.1"
-clap = { version = "4.2", features = ["wrap_help", "cargo"] }
+clap = { version = "4.4", features = ["wrap_help", "cargo"] }
 lazy_static = "1.4"
-toml = "0.7"
+toml = "0.8"
 serde = { version = "1.0", features = ["derive", "rc"] }
 serde_json = "1.0"
-uuid = { version = "1.3", features = ["v4"] }
-indicatif = "0.17"
+uuid = { version = "1.6", features = ["v4"] }
+# last known working version of indicatif; 0.17.5 has a bug that causes the 
+# scan menu to fail spectacularly 
+indicatif = { version = "0.17.3" }
 console = "0.15"
 openssl = { version = "0.10", features = ["vendored"] }
 dirs = "5.0"
-regex = "1.8"
-crossterm = "0.26"
-rlimit = "0.9"
-ctrlc = "3.2"
+regex = "1.10"
+crossterm = "0.27"
+rlimit = "0.10"
+ctrlc = "3.4"
 anyhow = "1.0"
-leaky-bucket = "0.12"
-gaoya = "0.1"
+leaky-bucket = "1.0"
+gaoya = "0.2"
+# 0.37+ relies on the broken version of indicatif and forces
+# the broken version to be used regardless of the version
+# specified above 
 self_update = { version = "0.36", features = [
     "archive-tar",
     "compression-flate2",
@@ -64,10 +69,10 @@ self_update = { version = "0.36", features = [
 ] }
 
 [dev-dependencies]
-tempfile = "3.5"
+tempfile = "3.9"
 httpmock = "0.6"
 assert_cmd = "2.0"
-predicates = "3.0"
+predicates = "3.1"
 
 [profile.release]
 lto = true

Makefile.toml

@@ -11,7 +11,7 @@ rm ferox-*.state
 # dependency management
 [tasks.upgrade-deps]
 command = "cargo"
-args = ["upgrade", "--to-lockfile"]
+args = ["upgrade", "--exclude", "indicatif", "--exclude", "self_update"]
 
 [tasks.update]
 command = "cargo"

README.md

@@ -232,7 +232,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/sbrun"><img src="https://avatars.githubusercontent.com/u/7712154?v=4?s=100" width="100px;" alt="Sophie Brun"/><br /><sub><b>Sophie Brun</b></sub></a><br /><a href="#infra-sbrun" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/black-A"><img src="https://avatars.githubusercontent.com/u/30686803?v=4?s=100" width="100px;" alt="black-A"/><br /><sub><b>black-A</b></sub></a><br /><a href="#ideas-black-A" title="Ideas, Planning, & Feedback">🤔</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/dinosn"><img src="https://avatars.githubusercontent.com/u/3851678?v=4?s=100" width="100px;" alt="Nicolas Krassas"/><br /><sub><b>Nicolas Krassas</b></sub></a><br /><a href="#ideas-dinosn" title="Ideas, Planning, & Feedback">🤔</a></td>
-      <td align="center" valign="top" width="14.28%"><a href="https://github.com/N0ur5"><img src="https://avatars.githubusercontent.com/u/24260009?v=4?s=100" width="100px;" alt="N0ur5"/><br /><sub><b>N0ur5</b></sub></a><br /><a href="#ideas-N0ur5" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/N0ur5"><img src="https://avatars.githubusercontent.com/u/24260009?v=4?s=100" width="100px;" alt="N0ur5"/><br /><sub><b>N0ur5</b></sub></a><br /><a href="#ideas-N0ur5" title="Ideas, Planning, & Feedback">🤔</a> <a href="https://github.com/epi052/feroxbuster/issues?q=author%3AN0ur5" title="Bug reports">🐛</a></td>
     </tr>
     <tr>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/moscowchill"><img src="https://avatars.githubusercontent.com/u/72578879?v=4?s=100" width="100px;" alt="mchill"/><br /><sub><b>mchill</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3Amoscowchill" title="Bug reports">🐛</a></td>
@@ -291,6 +291,17 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
     <tr>
       <td align="center" valign="top" width="14.28%"><a href="http://lavafroth.is-a.dev"><img src="https://avatars.githubusercontent.com/u/107522312?v=4?s=100" width="100px;" alt="Himadri Bhattacharjee"/><br /><sub><b>Himadri Bhattacharjee</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/commits?author=lavafroth" title="Code">💻</a> <a href="#ideas-lavafroth" title="Ideas, Planning, & Feedback">🤔</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/AkechiShiro"><img src="https://avatars.githubusercontent.com/u/14914796?v=4?s=100" width="100px;" alt="Samy Lahfa"/><br /><sub><b>Samy Lahfa</b></sub></a><br /><a href="#ideas-AkechiShiro" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/sectroyer"><img src="https://avatars.githubusercontent.com/u/6706818?v=4?s=100" width="100px;" alt="sectroyer"/><br /><sub><b>sectroyer</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3Asectroyer" title="Bug reports">🐛</a> <a href="#ideas-sectroyer" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://medium.com/@b3rm1nG"><img src="https://avatars.githubusercontent.com/u/19836003?v=4?s=100" width="100px;" alt="ktecv2000"/><br /><sub><b>ktecv2000</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3Aktecv2000" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="http://untrue.me"><img src="https://avatars.githubusercontent.com/u/56048157?v=4?s=100" width="100px;" alt="Andrea De Murtas"/><br /><sub><b>Andrea De Murtas</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/commits?author=andreademurtas" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/sawmj"><img src="https://avatars.githubusercontent.com/u/30024085?v=4?s=100" width="100px;" alt="sawmj"/><br /><sub><b>sawmj</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3Asawmj" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/devx00"><img src="https://avatars.githubusercontent.com/u/6897405?v=4?s=100" width="100px;" alt="Zach Hanson"/><br /><sub><b>Zach Hanson</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3Adevx00" title="Bug reports">🐛</a></td>
+    </tr>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/ocervell"><img src="https://avatars.githubusercontent.com/u/9629314?v=4?s=100" width="100px;" alt="Olivier Cervello"/><br /><sub><b>Olivier Cervello</b></sub></a><br /><a href="#ideas-ocervell" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/RavySena"><img src="https://avatars.githubusercontent.com/u/67729597?v=4?s=100" width="100px;" alt="RavySena"/><br /><sub><b>RavySena</b></sub></a><br /><a href="#ideas-RavySena" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/stuhlmann"><img src="https://avatars.githubusercontent.com/u/11061864?v=4?s=100" width="100px;" alt="Florian Stuhlmann"/><br /><sub><b>Florian Stuhlmann</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3Astuhlmann" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/Mister7F"><img src="https://avatars.githubusercontent.com/u/35213773?v=4?s=100" width="100px;" alt="Mister7F"/><br /><sub><b>Mister7F</b></sub></a><br /><a href="#ideas-Mister7F" title="Ideas, Planning, & Feedback">🤔</a></td>
     </tr>
   </tbody>
 </table>

shell_completions/_feroxbuster

@@ -24,10 +24,10 @@ _feroxbuster() {
 '--replay-proxy=[Send only unfiltered requests through a Replay Proxy, instead of all requests]:REPLAY_PROXY:_urls' \
 '*-R+[Status Codes to send through a Replay Proxy when found (default\: --status-codes value)]:REPLAY_CODE: ' \
 '*--replay-codes=[Status Codes to send through a Replay Proxy when found (default\: --status-codes value)]:REPLAY_CODE: ' \
-'-a+[Sets the User-Agent (default\: feroxbuster/2.10.0)]:USER_AGENT: ' \
-'--user-agent=[Sets the User-Agent (default\: feroxbuster/2.10.0)]:USER_AGENT: ' \
-'*-x+[File extension(s) to search for (ex\: -x php -x pdf js)]:FILE_EXTENSION: ' \
-'*--extensions=[File extension(s) to search for (ex\: -x php -x pdf js)]:FILE_EXTENSION: ' \
+'-a+[Sets the User-Agent (default\: feroxbuster/2.10.2)]:USER_AGENT: ' \
+'--user-agent=[Sets the User-Agent (default\: feroxbuster/2.10.2)]:USER_AGENT: ' \
+'*-x+[File extension(s) to search for (ex\: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex\: @ext.txt)]:FILE_EXTENSION: ' \
+'*--extensions=[File extension(s) to search for (ex\: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex\: @ext.txt)]:FILE_EXTENSION: ' \
 '*-m+[Which HTTP request method(s) should be sent (default\: GET)]:HTTP_METHODS: ' \
 '*--methods=[Which HTTP request method(s) should be sent (default\: GET)]:HTTP_METHODS: ' \
 '--data=[Request'\''s Body; can read data from a file if input starts with an @ (ex\: @post.bin)]:DATA: ' \
@@ -67,6 +67,8 @@ _feroxbuster() {
 '--time-limit=[Limit total run time of all scans (ex\: --time-limit 10m)]:TIME_SPEC: ' \
 '-w+[Path or URL of the wordlist]:FILE:_files' \
 '--wordlist=[Path or URL of the wordlist]:FILE:_files' \
+'-B+[Automatically request likely backup extensions for "found" urls (default\: ~, .bak, .bak2, .old, .1)]' \
+'--collect-backups=[Automatically request likely backup extensions for "found" urls (default\: ~, .bak, .bak2, .old, .1)]' \
 '*-I+[File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)]:FILE_EXTENSION: ' \
 '*--dont-collect=[File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)]:FILE_EXTENSION: ' \
 '-o+[Output file to write results to (use w/ --json for JSON entries)]:FILE:_files' \
@@ -97,13 +99,11 @@ _feroxbuster() {
 '--dont-filter[Don'\''t auto-filter wildcard responses]' \
 '-E[Automatically discover extensions and add them to --extensions (unless they'\''re in --dont-collect)]' \
 '--collect-extensions[Automatically discover extensions and add them to --extensions (unless they'\''re in --dont-collect)]' \
-'-B[Automatically request likely backup extensions for "found" urls]' \
-'--collect-backups[Automatically request likely backup extensions for "found" urls]' \
 '-g[Automatically discover important words from within responses and add them to the wordlist]' \
 '--collect-words[Automatically discover important words from within responses and add them to the wordlist]' \
 '(--silent)*-v[Increase verbosity level (use -vv or more for greater effect. \[CAUTION\] 4 -v'\''s is probably too much)]' \
 '(--silent)*--verbosity[Increase verbosity level (use -vv or more for greater effect. \[CAUTION\] 4 -v'\''s is probably too much)]' \
-'(-q --quiet)--silent[Only print URLs + turn off logging (good for piping a list of urls to other commands)]' \
+'(-q --quiet)--silent[Only print URLs (or JSON w/ --json) + turn off logging (good for piping a list of urls to other commands)]' \
 '-q[Hide progress bars and banner (good for tmux windows w/ notifications)]' \
 '--quiet[Hide progress bars and banner (good for tmux windows w/ notifications)]' \
 '--json[Emit JSON logs to --output and --debug-log instead of normal text]' \

shell_completions/_feroxbuster.ps1

@@ -26,38 +26,38 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
             [CompletionResult]::new('--resume-from', 'resume-from', [CompletionResultType]::ParameterName, 'State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)')
             [CompletionResult]::new('-p', 'p', [CompletionResultType]::ParameterName, 'Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)')
             [CompletionResult]::new('--proxy', 'proxy', [CompletionResultType]::ParameterName, 'Proxy to use for requests (ex: http(s)://host:port, socks5(h)://host:port)')
-            [CompletionResult]::new('-P', 'P', [CompletionResultType]::ParameterName, 'Send only unfiltered requests through a Replay Proxy, instead of all requests')
+            [CompletionResult]::new('-P', 'P ', [CompletionResultType]::ParameterName, 'Send only unfiltered requests through a Replay Proxy, instead of all requests')
             [CompletionResult]::new('--replay-proxy', 'replay-proxy', [CompletionResultType]::ParameterName, 'Send only unfiltered requests through a Replay Proxy, instead of all requests')
-            [CompletionResult]::new('-R', 'R', [CompletionResultType]::ParameterName, 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)')
+            [CompletionResult]::new('-R', 'R ', [CompletionResultType]::ParameterName, 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)')
             [CompletionResult]::new('--replay-codes', 'replay-codes', [CompletionResultType]::ParameterName, 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)')
-            [CompletionResult]::new('-a', 'a', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.10.0)')
-            [CompletionResult]::new('--user-agent', 'user-agent', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.10.0)')
-            [CompletionResult]::new('-x', 'x', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js)')
-            [CompletionResult]::new('--extensions', 'extensions', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js)')
+            [CompletionResult]::new('-a', 'a', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.10.2)')
+            [CompletionResult]::new('--user-agent', 'user-agent', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.10.2)')
+            [CompletionResult]::new('-x', 'x', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex: @ext.txt)')
+            [CompletionResult]::new('--extensions', 'extensions', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex: @ext.txt)')
             [CompletionResult]::new('-m', 'm', [CompletionResultType]::ParameterName, 'Which HTTP request method(s) should be sent (default: GET)')
             [CompletionResult]::new('--methods', 'methods', [CompletionResultType]::ParameterName, 'Which HTTP request method(s) should be sent (default: GET)')
             [CompletionResult]::new('--data', 'data', [CompletionResultType]::ParameterName, 'Request''s Body; can read data from a file if input starts with an @ (ex: @post.bin)')
-            [CompletionResult]::new('-H', 'H', [CompletionResultType]::ParameterName, 'Specify HTTP headers to be used in each request (ex: -H Header:val -H ''stuff: things'')')
+            [CompletionResult]::new('-H', 'H ', [CompletionResultType]::ParameterName, 'Specify HTTP headers to be used in each request (ex: -H Header:val -H ''stuff: things'')')
             [CompletionResult]::new('--headers', 'headers', [CompletionResultType]::ParameterName, 'Specify HTTP headers to be used in each request (ex: -H Header:val -H ''stuff: things'')')
             [CompletionResult]::new('-b', 'b', [CompletionResultType]::ParameterName, 'Specify HTTP cookies to be used in each request (ex: -b stuff=things)')
             [CompletionResult]::new('--cookies', 'cookies', [CompletionResultType]::ParameterName, 'Specify HTTP cookies to be used in each request (ex: -b stuff=things)')
-            [CompletionResult]::new('-Q', 'Q', [CompletionResultType]::ParameterName, 'Request''s URL query parameters (ex: -Q token=stuff -Q secret=key)')
+            [CompletionResult]::new('-Q', 'Q ', [CompletionResultType]::ParameterName, 'Request''s URL query parameters (ex: -Q token=stuff -Q secret=key)')
             [CompletionResult]::new('--query', 'query', [CompletionResultType]::ParameterName, 'Request''s URL query parameters (ex: -Q token=stuff -Q secret=key)')
             [CompletionResult]::new('--dont-scan', 'dont-scan', [CompletionResultType]::ParameterName, 'URL(s) or Regex Pattern(s) to exclude from recursion/scans')
-            [CompletionResult]::new('-S', 'S', [CompletionResultType]::ParameterName, 'Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)')
+            [CompletionResult]::new('-S', 'S ', [CompletionResultType]::ParameterName, 'Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)')
             [CompletionResult]::new('--filter-size', 'filter-size', [CompletionResultType]::ParameterName, 'Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)')
-            [CompletionResult]::new('-X', 'X', [CompletionResultType]::ParameterName, 'Filter out messages via regular expression matching on the response''s body (ex: -X ''^ignore me$'')')
+            [CompletionResult]::new('-X', 'X ', [CompletionResultType]::ParameterName, 'Filter out messages via regular expression matching on the response''s body (ex: -X ''^ignore me$'')')
             [CompletionResult]::new('--filter-regex', 'filter-regex', [CompletionResultType]::ParameterName, 'Filter out messages via regular expression matching on the response''s body (ex: -X ''^ignore me$'')')
-            [CompletionResult]::new('-W', 'W', [CompletionResultType]::ParameterName, 'Filter out messages of a particular word count (ex: -W 312 -W 91,82)')
+            [CompletionResult]::new('-W', 'W ', [CompletionResultType]::ParameterName, 'Filter out messages of a particular word count (ex: -W 312 -W 91,82)')
             [CompletionResult]::new('--filter-words', 'filter-words', [CompletionResultType]::ParameterName, 'Filter out messages of a particular word count (ex: -W 312 -W 91,82)')
-            [CompletionResult]::new('-N', 'N', [CompletionResultType]::ParameterName, 'Filter out messages of a particular line count (ex: -N 20 -N 31,30)')
+            [CompletionResult]::new('-N', 'N ', [CompletionResultType]::ParameterName, 'Filter out messages of a particular line count (ex: -N 20 -N 31,30)')
             [CompletionResult]::new('--filter-lines', 'filter-lines', [CompletionResultType]::ParameterName, 'Filter out messages of a particular line count (ex: -N 20 -N 31,30)')
-            [CompletionResult]::new('-C', 'C', [CompletionResultType]::ParameterName, 'Filter out status codes (deny list) (ex: -C 200 -C 401)')
+            [CompletionResult]::new('-C', 'C ', [CompletionResultType]::ParameterName, 'Filter out status codes (deny list) (ex: -C 200 -C 401)')
             [CompletionResult]::new('--filter-status', 'filter-status', [CompletionResultType]::ParameterName, 'Filter out status codes (deny list) (ex: -C 200 -C 401)')
             [CompletionResult]::new('--filter-similar-to', 'filter-similar-to', [CompletionResultType]::ParameterName, 'Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)')
             [CompletionResult]::new('-s', 's', [CompletionResultType]::ParameterName, 'Status Codes to include (allow list) (default: All Status Codes)')
             [CompletionResult]::new('--status-codes', 'status-codes', [CompletionResultType]::ParameterName, 'Status Codes to include (allow list) (default: All Status Codes)')
-            [CompletionResult]::new('-T', 'T', [CompletionResultType]::ParameterName, 'Number of seconds before a client''s request times out (default: 7)')
+            [CompletionResult]::new('-T', 'T ', [CompletionResultType]::ParameterName, 'Number of seconds before a client''s request times out (default: 7)')
             [CompletionResult]::new('--timeout', 'timeout', [CompletionResultType]::ParameterName, 'Number of seconds before a client''s request times out (default: 7)')
             [CompletionResult]::new('--server-certs', 'server-certs', [CompletionResultType]::ParameterName, 'Add custom root certificate(s) for servers with unknown certificates')
             [CompletionResult]::new('--client-cert', 'client-cert', [CompletionResultType]::ParameterName, 'Add a PEM encoded certificate for mutual authentication (mTLS)')
@@ -66,14 +66,16 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
             [CompletionResult]::new('--threads', 'threads', [CompletionResultType]::ParameterName, 'Number of concurrent threads (default: 50)')
             [CompletionResult]::new('-d', 'd', [CompletionResultType]::ParameterName, 'Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)')
             [CompletionResult]::new('--depth', 'depth', [CompletionResultType]::ParameterName, 'Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)')
-            [CompletionResult]::new('-L', 'L', [CompletionResultType]::ParameterName, 'Limit total number of concurrent scans (default: 0, i.e. no limit)')
+            [CompletionResult]::new('-L', 'L ', [CompletionResultType]::ParameterName, 'Limit total number of concurrent scans (default: 0, i.e. no limit)')
             [CompletionResult]::new('--scan-limit', 'scan-limit', [CompletionResultType]::ParameterName, 'Limit total number of concurrent scans (default: 0, i.e. no limit)')
             [CompletionResult]::new('--parallel', 'parallel', [CompletionResultType]::ParameterName, 'Run parallel feroxbuster instances (one child process per url passed via stdin)')
             [CompletionResult]::new('--rate-limit', 'rate-limit', [CompletionResultType]::ParameterName, 'Limit number of requests per second (per directory) (default: 0, i.e. no limit)')
             [CompletionResult]::new('--time-limit', 'time-limit', [CompletionResultType]::ParameterName, 'Limit total run time of all scans (ex: --time-limit 10m)')
             [CompletionResult]::new('-w', 'w', [CompletionResultType]::ParameterName, 'Path or URL of the wordlist')
             [CompletionResult]::new('--wordlist', 'wordlist', [CompletionResultType]::ParameterName, 'Path or URL of the wordlist')
-            [CompletionResult]::new('-I', 'I', [CompletionResultType]::ParameterName, 'File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)')
+            [CompletionResult]::new('-B', 'B ', [CompletionResultType]::ParameterName, 'Automatically request likely backup extensions for "found" urls (default: ~, .bak, .bak2, .old, .1)')
+            [CompletionResult]::new('--collect-backups', 'collect-backups', [CompletionResultType]::ParameterName, 'Automatically request likely backup extensions for "found" urls (default: ~, .bak, .bak2, .old, .1)')
+            [CompletionResult]::new('-I', 'I ', [CompletionResultType]::ParameterName, 'File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)')
             [CompletionResult]::new('--dont-collect', 'dont-collect', [CompletionResultType]::ParameterName, 'File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)')
             [CompletionResult]::new('-o', 'o', [CompletionResultType]::ParameterName, 'Output file to write results to (use w/ --json for JSON entries)')
             [CompletionResult]::new('--output', 'output', [CompletionResultType]::ParameterName, 'Output file to write results to (use w/ --json for JSON entries)')
@@ -83,7 +85,7 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
             [CompletionResult]::new('--burp-replay', 'burp-replay', [CompletionResultType]::ParameterName, 'Set --replay-proxy to http://127.0.0.1:8080 and set --insecure to true')
             [CompletionResult]::new('--smart', 'smart', [CompletionResultType]::ParameterName, 'Set --auto-tune, --collect-words, and --collect-backups to true')
             [CompletionResult]::new('--thorough', 'thorough', [CompletionResultType]::ParameterName, 'Use the same settings as --smart and set --collect-extensions to true')
-            [CompletionResult]::new('-A', 'A', [CompletionResultType]::ParameterName, 'Use a random User-Agent')
+            [CompletionResult]::new('-A', 'A ', [CompletionResultType]::ParameterName, 'Use a random User-Agent')
             [CompletionResult]::new('--random-agent', 'random-agent', [CompletionResultType]::ParameterName, 'Use a random User-Agent')
             [CompletionResult]::new('-f', 'f', [CompletionResultType]::ParameterName, 'Append / to each request''s URL')
             [CompletionResult]::new('--add-slash', 'add-slash', [CompletionResultType]::ParameterName, 'Append / to each request''s URL')
@@ -99,26 +101,24 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
             [CompletionResult]::new('--dont-extract-links', 'dont-extract-links', [CompletionResultType]::ParameterName, 'Don''t extract links from response body (html, javascript, etc...)')
             [CompletionResult]::new('--auto-tune', 'auto-tune', [CompletionResultType]::ParameterName, 'Automatically lower scan rate when an excessive amount of errors are encountered')
             [CompletionResult]::new('--auto-bail', 'auto-bail', [CompletionResultType]::ParameterName, 'Automatically stop scanning when an excessive amount of errors are encountered')
-            [CompletionResult]::new('-D', 'D', [CompletionResultType]::ParameterName, 'Don''t auto-filter wildcard responses')
+            [CompletionResult]::new('-D', 'D ', [CompletionResultType]::ParameterName, 'Don''t auto-filter wildcard responses')
             [CompletionResult]::new('--dont-filter', 'dont-filter', [CompletionResultType]::ParameterName, 'Don''t auto-filter wildcard responses')
-            [CompletionResult]::new('-E', 'E', [CompletionResultType]::ParameterName, 'Automatically discover extensions and add them to --extensions (unless they''re in --dont-collect)')
+            [CompletionResult]::new('-E', 'E ', [CompletionResultType]::ParameterName, 'Automatically discover extensions and add them to --extensions (unless they''re in --dont-collect)')
             [CompletionResult]::new('--collect-extensions', 'collect-extensions', [CompletionResultType]::ParameterName, 'Automatically discover extensions and add them to --extensions (unless they''re in --dont-collect)')
-            [CompletionResult]::new('-B', 'B', [CompletionResultType]::ParameterName, 'Automatically request likely backup extensions for "found" urls')
-            [CompletionResult]::new('--collect-backups', 'collect-backups', [CompletionResultType]::ParameterName, 'Automatically request likely backup extensions for "found" urls')
             [CompletionResult]::new('-g', 'g', [CompletionResultType]::ParameterName, 'Automatically discover important words from within responses and add them to the wordlist')
             [CompletionResult]::new('--collect-words', 'collect-words', [CompletionResultType]::ParameterName, 'Automatically discover important words from within responses and add them to the wordlist')
             [CompletionResult]::new('-v', 'v', [CompletionResultType]::ParameterName, 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v''s is probably too much)')
             [CompletionResult]::new('--verbosity', 'verbosity', [CompletionResultType]::ParameterName, 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v''s is probably too much)')
-            [CompletionResult]::new('--silent', 'silent', [CompletionResultType]::ParameterName, 'Only print URLs + turn off logging (good for piping a list of urls to other commands)')
+            [CompletionResult]::new('--silent', 'silent', [CompletionResultType]::ParameterName, 'Only print URLs (or JSON w/ --json) + turn off logging (good for piping a list of urls to other commands)')
             [CompletionResult]::new('-q', 'q', [CompletionResultType]::ParameterName, 'Hide progress bars and banner (good for tmux windows w/ notifications)')
             [CompletionResult]::new('--quiet', 'quiet', [CompletionResultType]::ParameterName, 'Hide progress bars and banner (good for tmux windows w/ notifications)')
             [CompletionResult]::new('--json', 'json', [CompletionResultType]::ParameterName, 'Emit JSON logs to --output and --debug-log instead of normal text')
             [CompletionResult]::new('--no-state', 'no-state', [CompletionResultType]::ParameterName, 'Disable state output file (*.state)')
-            [CompletionResult]::new('-U', 'U', [CompletionResultType]::ParameterName, 'Update feroxbuster to the latest version')
+            [CompletionResult]::new('-U', 'U ', [CompletionResultType]::ParameterName, 'Update feroxbuster to the latest version')
             [CompletionResult]::new('--update', 'update', [CompletionResultType]::ParameterName, 'Update feroxbuster to the latest version')
             [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
             [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
-            [CompletionResult]::new('-V', 'V', [CompletionResultType]::ParameterName, 'Print version')
+            [CompletionResult]::new('-V', 'V ', [CompletionResultType]::ParameterName, 'Print version')
             [CompletionResult]::new('--version', 'version', [CompletionResultType]::ParameterName, 'Print version')
             break
         }

shell_completions/feroxbuster.bash

@@ -233,6 +233,14 @@ _feroxbuster() {
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
                     ;;
+                --collect-backups)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -B)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
                 --dont-collect)
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
@@ -263,4 +271,8 @@ _feroxbuster() {
     esac
 }
 
-complete -F _feroxbuster -o bashdefault -o default -o plusdirs feroxbuster
+if [[ "${BASH_VERSINFO[0]}" -eq 4 && "${BASH_VERSINFO[1]}" -ge 4 || "${BASH_VERSINFO[0]}" -gt 4 ]]; then
+    complete -F _feroxbuster -o nosort -o bashdefault -o default -o plusdirs feroxbuster
+else
+    complete -F _feroxbuster -o bashdefault -o default -o plusdirs feroxbuster
+fi

shell_completions/feroxbuster.elv

@@ -27,10 +27,10 @@ set edit:completion:arg-completer[feroxbuster] = {|@words|
             cand --replay-proxy 'Send only unfiltered requests through a Replay Proxy, instead of all requests'
             cand -R 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)'
             cand --replay-codes 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)'
-            cand -a 'Sets the User-Agent (default: feroxbuster/2.10.0)'
-            cand --user-agent 'Sets the User-Agent (default: feroxbuster/2.10.0)'
-            cand -x 'File extension(s) to search for (ex: -x php -x pdf js)'
-            cand --extensions 'File extension(s) to search for (ex: -x php -x pdf js)'
+            cand -a 'Sets the User-Agent (default: feroxbuster/2.10.2)'
+            cand --user-agent 'Sets the User-Agent (default: feroxbuster/2.10.2)'
+            cand -x 'File extension(s) to search for (ex: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex: @ext.txt)'
+            cand --extensions 'File extension(s) to search for (ex: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex: @ext.txt)'
             cand -m 'Which HTTP request method(s) should be sent (default: GET)'
             cand --methods 'Which HTTP request method(s) should be sent (default: GET)'
             cand --data 'Request''s Body; can read data from a file if input starts with an @ (ex: @post.bin)'
@@ -70,6 +70,8 @@ set edit:completion:arg-completer[feroxbuster] = {|@words|
             cand --time-limit 'Limit total run time of all scans (ex: --time-limit 10m)'
             cand -w 'Path or URL of the wordlist'
             cand --wordlist 'Path or URL of the wordlist'
+            cand -B 'Automatically request likely backup extensions for "found" urls (default: ~, .bak, .bak2, .old, .1)'
+            cand --collect-backups 'Automatically request likely backup extensions for "found" urls (default: ~, .bak, .bak2, .old, .1)'
             cand -I 'File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)'
             cand --dont-collect 'File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)'
             cand -o 'Output file to write results to (use w/ --json for JSON entries)'
@@ -100,13 +102,11 @@ set edit:completion:arg-completer[feroxbuster] = {|@words|
             cand --dont-filter 'Don''t auto-filter wildcard responses'
             cand -E 'Automatically discover extensions and add them to --extensions (unless they''re in --dont-collect)'
             cand --collect-extensions 'Automatically discover extensions and add them to --extensions (unless they''re in --dont-collect)'
-            cand -B 'Automatically request likely backup extensions for "found" urls'
-            cand --collect-backups 'Automatically request likely backup extensions for "found" urls'
             cand -g 'Automatically discover important words from within responses and add them to the wordlist'
             cand --collect-words 'Automatically discover important words from within responses and add them to the wordlist'
             cand -v 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v''s is probably too much)'
             cand --verbosity 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v''s is probably too much)'
-            cand --silent 'Only print URLs + turn off logging (good for piping a list of urls to other commands)'
+            cand --silent 'Only print URLs (or JSON w/ --json) + turn off logging (good for piping a list of urls to other commands)'
             cand -q 'Hide progress bars and banner (good for tmux windows w/ notifications)'
             cand --quiet 'Hide progress bars and banner (good for tmux windows w/ notifications)'
             cand --json 'Emit JSON logs to --output and --debug-log instead of normal text'

src/banner/container.rs

@@ -1,13 +1,15 @@
 use super::entry::BannerEntry;
 use crate::{
+    client,
     config::Configuration,
     event_handlers::Handles,
-    utils::{logged_request, parse_url_with_raw_path, status_colorizer},
+    utils::{make_request, parse_url_with_raw_path, status_colorizer},
     DEFAULT_IGNORED_EXTENSIONS, DEFAULT_METHOD, DEFAULT_STATUS_CODES, VERSION,
 };
 use anyhow::{bail, Result};
 use console::{style, Emoji};
 use serde_json::Value;
+use std::collections::HashMap;
 use std::{io::Write, sync::Arc};
 
 /// Url used to query github's api; specifically used to look for the latest tagged release name
@@ -498,7 +500,34 @@ by Ben "epi" Risher {}                 ver: {}"#,
 
         let api_url = parse_url_with_raw_path(url)?;
 
-        let result = logged_request(&api_url, DEFAULT_METHOD, None, handles.clone()).await?;
+        // we don't want to leak sensitive header info / include auth headers
+        // with the github api request, so we'll build a client specifically
+        // for this task. thanks to @stuhlmann for the suggestion!
+        let client = client::initialize(
+            handles.config.timeout,
+            "feroxbuster-update-check",
+            handles.config.redirects,
+            handles.config.insecure,
+            &HashMap::new(),
+            Some(&handles.config.proxy),
+            &handles.config.server_certs,
+            Some(&handles.config.client_cert),
+            Some(&handles.config.client_key),
+        )?;
+        let level = handles.config.output_level;
+        let tx_stats = handles.stats.tx.clone();
+
+        let result = make_request(
+            &client,
+            &api_url,
+            DEFAULT_METHOD,
+            None,
+            level,
+            &handles.config,
+            tx_stats,
+        )
+        .await?;
+
         let body = result.text().await?;
 
         let json_response: Value = serde_json::from_str(&body)?;

src/client.rs

@@ -67,17 +67,19 @@ where
     }
 
     if let (Some(cert_path), Some(key_path)) = (client_cert, client_key) {
-        let cert = std::fs::read(cert_path)?;
-        let key = std::fs::read(key_path)?;
+        if !cert_path.is_empty() && !key_path.is_empty() {
+            let cert = std::fs::read(cert_path)?;
+            let key = std::fs::read(key_path)?;
 
-        let identity = reqwest::Identity::from_pkcs8_pem(&cert, &key).with_context(|| {
-            format!(
-                "either {} or {} are invalid; expecting PEM encoded certificate and key",
-                cert_path, key_path
-            )
-        })?;
+            let identity = reqwest::Identity::from_pkcs8_pem(&cert, &key).with_context(|| {
+                format!(
+                    "either {} or {} are invalid; expecting PEM encoded certificate and key",
+                    cert_path, key_path
+                )
+            })?;
 
-        client = client.identity(identity);
+            client = client.identity(identity);
+        }
     }
 
     Ok(client.build()?)

src/config/container.rs

@@ -1,6 +1,6 @@
 use super::utils::{
-    depth, extract_links, ignored_extensions, methods, report_and_exit, save_state,
-    serialized_type, status_codes, threads, timeout, user_agent, wordlist, OutputLevel,
+    backup_extensions, depth, extract_links, ignored_extensions, methods, report_and_exit,
+    save_state, serialized_type, status_codes, threads, timeout, user_agent, wordlist, OutputLevel,
     RequesterPolicy,
 };
 use crate::config::determine_output_level;
@@ -318,6 +318,9 @@ pub struct Configuration {
     #[serde(default)]
     pub collect_backups: bool,
 
+    #[serde(default = "backup_extensions")]
+    pub backup_extensions: Vec<String>,
+
     /// Automatically discover important words from within responses and add them to the wordlist
     #[serde(default)]
     pub collect_words: bool,
@@ -418,6 +421,7 @@ impl Default for Configuration {
             threads: threads(),
             wordlist: wordlist(),
             dont_collect: ignored_extensions(),
+            backup_extensions: backup_extensions(),
         }
     }
 }
@@ -450,6 +454,7 @@ impl Configuration {
     /// - **extensions**: `None`
     /// - **collect_extensions**: `false`
     /// - **collect_backups**: `false`
+    /// - **backup_extensions**: [`DEFAULT_BACKUP_EXTENSIONS`](constant.DEFAULT_BACKUP_EXTENSIONS.html)
     /// - **collect_words**: `false`
     /// - **dont_collect**: [`DEFAULT_IGNORED_EXTENSIONS`](constant.DEFAULT_RESPONSE_CODES.html)
     /// - **methods**: [`DEFAULT_METHOD`](constant.DEFAULT_METHOD.html)
@@ -655,9 +660,27 @@ impl Configuration {
         }
 
         if let Some(arg) = args.get_many::<String>("extensions") {
-            config.extensions = arg
-                .map(|val| val.trim_start_matches('.').to_string())
-                .collect();
+            let mut extensions = Vec::<String>::new();
+            for ext in arg {
+                if let Some(stripped) = ext.strip_prefix('@') {
+                    let contents = read_to_string(stripped)
+                        .unwrap_or_else(|e| report_and_exit(&e.to_string()));
+                    let exts_from_file = contents.split('\n').filter_map(|s| {
+                        let trimmed = s.trim().trim_start_matches('.');
+
+                        if trimmed.is_empty() || trimmed.starts_with('#') {
+                            None
+                        } else {
+                            Some(trimmed.to_string())
+                        }
+                    });
+
+                    extensions.extend(exts_from_file);
+                } else {
+                    extensions.push(ext.trim().trim_start_matches('.').to_string());
+                }
+            }
+            config.extensions = extensions;
         }
 
         if let Some(arg) = args.get_many::<String>("dont_collect") {
@@ -776,7 +799,11 @@ impl Configuration {
             // if the line below is outside of the if, we'd overwrite true with
             // false if no --silent is used on the command line
             config.silent = true;
-            config.output_level = OutputLevel::Silent;
+            config.output_level = if config.json {
+                OutputLevel::SilentJSON
+            } else {
+                OutputLevel::Silent
+            };
         }
 
         if came_from_cli!(args, "quiet") {
@@ -814,6 +841,20 @@ impl Configuration {
             || came_from_cli!(args, "thorough")
         {
             config.collect_backups = true;
+            config.backup_extensions = backup_extensions();
+
+            if came_from_cli!(args, "collect_backups") {
+                if let Some(arg) = args.get_many::<String>("collect_backups") {
+                    let backup_exts = arg
+                        .map(|ext| ext.trim().to_string())
+                        .collect::<Vec<String>>();
+
+                    if !backup_exts.is_empty() {
+                        // have at least one cli backup, override the defaults
+                        config.backup_extensions = backup_exts;
+                    }
+                }
+            }
         }
 
         if came_from_cli!(args, "collect_words")
@@ -1012,7 +1053,7 @@ impl Configuration {
 
     /// Given a configuration file's location and an instance of `Configuration`, read in
     /// the config file if found and update the current settings with the settings found therein
-    fn parse_and_merge_config(config_file: PathBuf, mut config: &mut Self) -> Result<()> {
+    fn parse_and_merge_config(config_file: PathBuf, config: &mut Self) -> Result<()> {
         if config_file.exists() {
             // save off a string version of the path before it goes out of scope
             let conf_str = config_file.to_str().unwrap_or("").to_string();
@@ -1043,6 +1084,7 @@ impl Configuration {
             new.server_certs,
             Vec::<String>::new()
         );
+        update_if_not_default!(&mut conf.json, new.json, false);
         update_if_not_default!(&mut conf.client_cert, new.client_cert, "");
         update_if_not_default!(&mut conf.client_key, new.client_key, "");
         update_if_not_default!(&mut conf.verbosity, new.verbosity, 0);
@@ -1054,7 +1096,7 @@ impl Configuration {
         update_if_not_default!(&mut conf.collect_backups, new.collect_backups, false);
         update_if_not_default!(&mut conf.collect_words, new.collect_words, false);
         // use updated quiet/silent values to determine output level; same for requester policy
-        conf.output_level = determine_output_level(conf.quiet, conf.silent);
+        conf.output_level = determine_output_level(conf.quiet, conf.silent, conf.json);
         conf.requester_policy = determine_requester_policy(conf.auto_tune, conf.auto_bail);
         update_if_not_default!(&mut conf.output, new.output, "");
         update_if_not_default!(&mut conf.redirects, new.redirects, false);
@@ -1112,10 +1154,14 @@ impl Configuration {
         update_if_not_default!(&mut conf.replay_proxy, new.replay_proxy, "");
         update_if_not_default!(&mut conf.debug_log, new.debug_log, "");
         update_if_not_default!(&mut conf.resume_from, new.resume_from, "");
-        update_if_not_default!(&mut conf.json, new.json, false);
 
         update_if_not_default!(&mut conf.timeout, new.timeout, timeout());
         update_if_not_default!(&mut conf.user_agent, new.user_agent, user_agent());
+        update_if_not_default!(
+            &mut conf.backup_extensions,
+            new.backup_extensions,
+            backup_extensions()
+        );
         update_if_not_default!(&mut conf.random_agent, new.random_agent, false);
         update_if_not_default!(&mut conf.threads, new.threads, threads());
         update_if_not_default!(&mut conf.depth, new.depth, depth());

src/config/tests.rs

@@ -59,6 +59,7 @@ fn setup_config_test() -> Configuration {
             server_certs = ["/some/cert.pem", "/some/other/cert.pem"]
             client_cert = "/some/client/cert.pem"
             client_key = "/some/client/key.pem"
+            backup_extensions = [".save"]
         "#;
     let tmp_dir = TempDir::new().unwrap();
     let file = tmp_dir.path().join(DEFAULT_CONFIG_NAME);
@@ -123,6 +124,7 @@ fn default_configuration() {
     assert_eq!(config.server_certs, Vec::<String>::new());
     assert_eq!(config.client_cert, String::new());
     assert_eq!(config.client_key, String::new());
+    assert_eq!(config.backup_extensions, backup_extensions());
 }
 
 #[test]
@@ -459,6 +461,13 @@ fn config_reads_server_certs() {
     );
 }
 
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_backup_extensions() {
+    let config = setup_config_test();
+    assert_eq!(config.backup_extensions, [".save"]);
+}
+
 #[test]
 /// parse the test config and see that the value parsed is correct
 fn config_reads_client_cert() {

src/config/utils.rs

@@ -1,6 +1,7 @@
 use crate::{
     utils::{module_colorizer, status_colorizer},
-    DEFAULT_IGNORED_EXTENSIONS, DEFAULT_METHOD, DEFAULT_STATUS_CODES, DEFAULT_WORDLIST, VERSION,
+    DEFAULT_BACKUP_EXTENSIONS, DEFAULT_IGNORED_EXTENSIONS, DEFAULT_METHOD, DEFAULT_STATUS_CODES,
+    DEFAULT_WORDLIST, VERSION,
 };
 #[cfg(not(test))]
 use std::process::exit;
@@ -69,6 +70,14 @@ pub(super) fn ignored_extensions() -> Vec<String> {
         .collect()
 }
 
+/// default backup extensions to collect
+pub(super) fn backup_extensions() -> Vec<String> {
+    DEFAULT_BACKUP_EXTENSIONS
+        .iter()
+        .map(|s| s.to_string())
+        .collect()
+}
+
 /// default wordlist
 pub(super) fn wordlist() -> String {
     String::from(DEFAULT_WORDLIST)
@@ -100,6 +109,9 @@ pub enum OutputLevel {
 
     /// silent scan, only print urls (used to be --quiet in versions 1.x.x)
     Silent,
+
+    /// silent scan, but with JSON output
+    SilentJSON,
 }
 
 /// implement a default for OutputLevel
@@ -111,14 +123,22 @@ impl Default for OutputLevel {
 }
 
 /// given the current settings for quiet and silent, determine output_level (DRY helper)
-pub fn determine_output_level(quiet: bool, silent: bool) -> OutputLevel {
+pub fn determine_output_level(quiet: bool, silent: bool, json: bool) -> OutputLevel {
     if quiet && silent {
         // user COULD have both as true in config file, take the more quiet of the two
-        OutputLevel::Silent
+        if json {
+            OutputLevel::SilentJSON
+        } else {
+            OutputLevel::Silent
+        }
     } else if quiet {
         OutputLevel::Quiet
     } else if silent {
-        OutputLevel::Silent
+        if json {
+            OutputLevel::SilentJSON
+        } else {
+            OutputLevel::Silent
+        }
     } else {
         OutputLevel::Default
     }
@@ -166,16 +186,28 @@ mod tests {
     #[test]
     /// test determine_output_level returns higher of the two levels if both given values are true
     fn determine_output_level_returns_correct_results() {
-        let mut level = determine_output_level(true, true);
+        let mut level = determine_output_level(true, true, false);
         assert_eq!(level, OutputLevel::Silent);
 
-        level = determine_output_level(false, true);
+        level = determine_output_level(false, true, false);
         assert_eq!(level, OutputLevel::Silent);
 
-        level = determine_output_level(false, false);
+        let mut level = determine_output_level(true, true, true);
+        assert_eq!(level, OutputLevel::SilentJSON);
+
+        level = determine_output_level(false, true, true);
+        assert_eq!(level, OutputLevel::SilentJSON);
+
+        level = determine_output_level(false, false, false);
         assert_eq!(level, OutputLevel::Default);
 
-        level = determine_output_level(true, false);
+        level = determine_output_level(true, false, false);
+        assert_eq!(level, OutputLevel::Quiet);
+
+        level = determine_output_level(false, false, true);
+        assert_eq!(level, OutputLevel::Default);
+
+        level = determine_output_level(true, false, true);
         assert_eq!(level, OutputLevel::Quiet);
     }
 

src/event_handlers/inputs.rs

@@ -12,6 +12,7 @@ use anyhow::Result;
 use console::style;
 use crossterm::event::{self, Event, KeyCode};
 use std::{
+    env::temp_dir,
     sync::{
         atomic::{AtomicBool, Ordering},
         Arc,
@@ -103,10 +104,36 @@ impl TermInputHandler {
 
         // User didn't set the --no-state flag (so saved_state is still the default true)
         if handles.config.save_state {
-            let state_file = open_file(&filename);
+            let Ok(mut state_file) = open_file(&filename) else {
+                // couldn't open the file, let the user know we're going to try again
+                let error = format!(
+                    "❌ Could not save {}, falling back to {}",
+                    filename,
+                    temp_dir().to_string_lossy()
+                );
+                PROGRESS_PRINTER.println(error);
 
-            let mut buffered_file = state_file?;
-            write_to(&state, &mut buffered_file, true)?;
+                let temp_filename = temp_dir().join(&filename);
+
+                let Ok(mut state_file) = open_file(&temp_filename.to_string_lossy()) else {
+                    // couldn't open the fallback file, let the user know
+                    let error = format!("❌❌ Could not save {:?}, giving up...", temp_filename);
+                    PROGRESS_PRINTER.println(error);
+
+                    log::trace!("exit: sigint_handler (failed to write)");
+                    std::process::exit(1);
+                };
+
+                write_to(&state, &mut state_file, true)?;
+
+                let msg = format!("✅ Saved scan state to {:?}", temp_filename);
+                PROGRESS_PRINTER.println(msg);
+
+                log::trace!("exit: sigint_handler (saved to temp folder)");
+                std::process::exit(1);
+            };
+
+            write_to(&state, &mut state_file, true)?;
         }
 
         log::trace!("exit: sigint_handler (end of program)");

src/event_handlers/outputs.rs

@@ -209,8 +209,12 @@ impl TermOutHandler {
         while let Some(command) = self.receiver.recv().await {
             match command {
                 Command::Report(resp) => {
-                    self.process_response(tx_stats.clone(), resp, ProcessResponseCall::Recursive)
-                        .await?;
+                    if let Err(err) = self
+                        .process_response(tx_stats.clone(), resp, ProcessResponseCall::Recursive)
+                        .await
+                    {
+                        log::warn!("{}", err);
+                    }
                 }
                 Command::Sync(sender) => {
                     sender.send(true).unwrap_or_default();
@@ -328,6 +332,21 @@ impl TermOutHandler {
                     )
                     .await;
 
+                    let Some(handles) = self.handles.as_ref() else {
+                        // shouldn't ever happen, but we'll log and return early if it does
+                        log::error!("handles were unexpectedly None, this shouldn't happen");
+                        return Ok(());
+                    };
+
+                    if handles
+                        .filters
+                        .data
+                        .should_filter_response(&ferox_response, tx_stats.clone())
+                    {
+                        // response was filtered for one reason or another, don't process it
+                        continue;
+                    }
+
                     self.process_response(
                         tx_stats.clone(),
                         Box::new(ferox_response),
@@ -385,7 +404,7 @@ impl TermOutHandler {
 
         if !filename.is_empty() {
             // append rules
-            for suffix in ["~", ".bak", ".bak2", ".old", ".1"] {
+            for suffix in &self.config.backup_extensions {
                 self.add_new_url_to_vec(url, &format!("{filename}{suffix}"), &mut urls);
             }
 

src/heuristics.rs

@@ -332,10 +332,10 @@ impl HeuristicTests {
                     // - http://localhost/adminf1d2541e73c44dcb9d1fb7d93334b280
                     // - http://localhost/admin92969beae6bf4beb855d1622406d87e395c87387a9ad432e8a11245002b709b03cf609d471004154b83bcc1c6ec49f6f
                     let Ok(response) =
-                        logged_request(&nonexistent_url, method, data, self.handles.clone())
-                            .await else {
-                                return None;
-                            };
+                        logged_request(&nonexistent_url, method, data, self.handles.clone()).await
+                    else {
+                        return None;
+                    };
 
                     if !self
                         .handles
@@ -372,7 +372,9 @@ impl HeuristicTests {
                 }
 
                 // check the responses for similarities on which we can filter, multiple may be returned
-                let Some((wildcard_filters, wildcard_responses)) = self.examine_404_like_responses(&responses) else {
+                let Some((wildcard_filters, wildcard_responses)) =
+                    self.examine_404_like_responses(&responses)
+                else {
                     // no match was found during analysis of responses
                     log::warn!("no match found for 404 responses");
                     continue;

src/lib.rs

@@ -59,6 +59,9 @@ pub(crate) const DEFAULT_IGNORED_EXTENSIONS: [&str; 38] = [
     "webm", "ogv", "oga", "flac", "aac", "3gp", "css", "zip", "xls", "xml", "gz", "tgz",
 ];
 
+/// Default set of extensions to search for when auto-collecting backups during scans
+pub(crate) const DEFAULT_BACKUP_EXTENSIONS: [&str; 5] = ["~", ".bak", ".bak2", ".old", ".1"];
+
 /// Default wordlist to use when `-w|--wordlist` isn't specified and not `wordlist` isn't set
 /// in a [ferox-config.toml](constant.DEFAULT_CONFIG_NAME.html) config file.
 ///

src/main.rs

@@ -238,7 +238,15 @@ async fn wrapped_main(config: Arc<Configuration>) -> Result<()> {
 
     let words = if config.wordlist.starts_with("http") {
         // found a url scheme, attempt to download the wordlist
-        let response = config.client.get(&config.wordlist).send().await?;
+        let response = config
+            .client
+            .get(&config.wordlist)
+            .send()
+            .await
+            .context(format!(
+                "Unable to download wordlist from remote url: {}",
+                config.wordlist
+            ))?;
 
         if !response.status().is_success() {
             // status code isn't a 200, bail
@@ -250,14 +258,15 @@ async fn wrapped_main(config: Arc<Configuration>) -> Result<()> {
         }
 
         // attempt to get the filename from the url's path
-        let Some(path_segments) = response
-            .url()
-            .path_segments() else {
-                bail!("Unable to parse path from url: {}", response.url());
-            };
+        let Some(path_segments) = response.url().path_segments() else {
+            bail!("Unable to parse path from url: {}", response.url());
+        };
 
         let Some(filename) = path_segments.last() else {
-            bail!("Unable to parse filename from url's path: {}", response.url().path());
+            bail!(
+                "Unable to parse filename from url's path: {}",
+                response.url().path()
+            );
         };
 
         let filename = filename.to_string();

src/nlp/document.rs

@@ -35,20 +35,21 @@ impl Document {
     fn add_term(&mut self, word: &str) {
         let term = Term::new(word);
 
-        let metadata = self.terms.entry(term).or_insert_with(TermMetaData::new);
+        let metadata = self.terms.entry(term).or_default();
         *metadata.count_mut() += 1;
     }
 
     /// create a new `Document` from the given HTML string
-    pub(crate) fn from_html(raw_html: &str) -> Self {
+    pub(crate) fn from_html(raw_html: &str) -> Option<Self> {
         let selector = Selector::parse("body").unwrap();
 
         let html = Html::parse_document(raw_html);
 
-        let text = html
-            .select(&selector)
-            .next()
-            .unwrap()
+        let Some(element) = html.select(&selector).next() else {
+            return None;
+        };
+
+        let text = element
             .descendants()
             .filter_map(|node| {
                 if !node.value().is_text() && !node.value().is_comment() {
@@ -95,7 +96,7 @@ impl Document {
 
         // call `new` to push the parsed html through the pre-processing pipeline and process all
         // the words
-        Self::new(&text)
+        Some(Self::new(&text))
     }
 
     /// Log normalized weighting scheme for term frequency
@@ -146,19 +147,20 @@ mod tests {
     #[test]
     /// `Document::new` should preprocess html and generate a hashmap of `Term, TermMetadata`
     fn nlp_document_creation_from_html() {
-        let empty = Document::from_html("<html></html>");
+        let empty = Document::from_html("<html></html>").unwrap();
         assert_eq!(empty.number_of_terms, 0);
 
-        let other_empty = Document::from_html("<html><body><p></p></body></html>");
+        let other_empty = Document::from_html("<html><body><p></p></body></html>").unwrap();
         assert_eq!(other_empty.number_of_terms, 0);
 
-        let third_empty = Document::from_html("<!DOCTYPE html><html><!DOCTYPE html><p></p></html>");
+        let third_empty =
+            Document::from_html("<!DOCTYPE html><html><!DOCTYPE html><p></p></html>").unwrap();
         assert_eq!(third_empty.number_of_terms, 0);
 
         // p tag for is_text check and comment for is_comment
         let doc = Document::from_html(
             "<html><body><p>The air quality in Singapore.</p><!--got worse on Wednesday--></body></html>",
-        );
+        ).unwrap();
 
         let expected_terms = ["air", "quality", "singapore", "worse", "wednesday"];
 
@@ -209,7 +211,7 @@ mod tests {
     /// ensure words in script/style tags aren't processed
     fn document_creation_skips_script_and_style_tags() {
         let html = "<body><script>The air quality</script><style>in Singapore</style><p>got worse on Wednesday.</p></body>";
-        let doc = Document::from_html(html);
+        let doc = Document::from_html(html).unwrap();
         let keys = doc.terms().keys().map(|key| key.raw()).collect::<Vec<_>>();
 
         let expected = ["worse", "wednesday"];

src/nlp/term.rs

@@ -35,11 +35,6 @@ pub(super) struct TermMetaData {
 }
 
 impl TermMetaData {
-    /// create a new metadata container
-    pub(super) fn new() -> Self {
-        Self::default()
-    }
-
     /// number of times a `Term` has appeared in any `Document` within the corpus
     pub(super) fn document_frequency(&self) -> usize {
         self.term_frequencies().len()
@@ -90,7 +85,7 @@ mod tests {
     #[test]
     /// test accessors for correctness
     fn nlp_term_metadata_accessor_test() {
-        let mut metadata = TermMetaData::new();
+        let mut metadata = TermMetaData::default();
 
         *metadata.count_mut() += 1;
         assert_eq!(metadata.count(), 1);

src/parser.rs

@@ -177,7 +177,7 @@ pub fn initialize() -> Command {
                 .use_value_delimiter(true)
                 .help_heading("Request settings")
                 .help(
-                    "File extension(s) to search for (ex: -x php -x pdf js)",
+                    "File extension(s) to search for (ex: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex: @ext.txt)",
                 ),
         )
         .arg(
@@ -550,9 +550,9 @@ pub fn initialize() -> Command {
             Arg::new("collect_backups")
                 .short('B')
                 .long("collect-backups")
-                .num_args(0)
+                .num_args(0..)
                 .help_heading("Dynamic collection settings")
-                .help("Automatically request likely backup extensions for \"found\" urls")
+                .help("Automatically request likely backup extensions for \"found\" urls (default: ~, .bak, .bak2, .old, .1)")
         )
         .arg(
             Arg::new("collect_words")
@@ -594,7 +594,7 @@ pub fn initialize() -> Command {
                 .num_args(0)
                 .conflicts_with("quiet")
                 .help_heading("Output settings")
-                .help("Only print URLs + turn off logging (good for piping a list of urls to other commands)")
+                .help("Only print URLs (or JSON w/ --json) + turn off logging (good for piping a list of urls to other commands)")
         )
         .arg(
             Arg::new("quiet")
@@ -645,7 +645,7 @@ pub fn initialize() -> Command {
     let mut app = app
         .group(
             ArgGroup::new("output_files")
-                .args(["debug_log", "output"])
+                .args(["debug_log", "output", "silent"])
                 .multiple(true),
         )
         .arg(

src/response.rs

@@ -485,15 +485,19 @@ impl FeroxSerialize for FeroxResponse {
             message
         } else {
             // not a wildcard, just create a normal entry
-            utils::create_report_string(
-                self.status.as_str(),
-                method,
-                &lines,
-                &words,
-                &chars,
-                &url_with_redirect,
-                self.output_level,
-            )
+            if matches!(self.output_level, OutputLevel::SilentJSON) {
+                self.as_json().unwrap_or_default()
+            } else {
+                utils::create_report_string(
+                    self.status.as_str(),
+                    method,
+                    &lines,
+                    &words,
+                    &chars,
+                    &url_with_redirect,
+                    self.output_level,
+                )
+            }
         }
     }
 

src/scan_manager/mod.rs

@@ -8,7 +8,7 @@ mod state;
 #[cfg(test)]
 mod tests;
 
-pub(self) use menu::Menu;
+use menu::Menu;
 pub use menu::{MenuCmd, MenuCmdResult};
 pub use order::ScanOrder;
 pub use response_container::FeroxResponses;

src/scan_manager/scan.rs

@@ -110,7 +110,7 @@ impl FeroxScan {
 
         match self.task.try_lock() {
             Ok(mut guard) => {
-                if let Some(task) = std::mem::replace(&mut *guard, None) {
+                if let Some(task) = guard.take() {
                     log::trace!("aborting {:?}", self);
                     task.abort();
                     self.set_status(ScanStatus::Cancelled)?;
@@ -175,7 +175,7 @@ impl FeroxScan {
                     let bar_type = match self.output_level {
                         OutputLevel::Default => BarType::Default,
                         OutputLevel::Quiet => BarType::Quiet,
-                        OutputLevel::Silent => BarType::Hidden,
+                        OutputLevel::Silent | OutputLevel::SilentJSON => BarType::Hidden,
                     };
 
                     let pb = add_bar(&self.url, self.num_requests, bar_type);
@@ -194,7 +194,7 @@ impl FeroxScan {
                 let bar_type = match self.output_level {
                     OutputLevel::Default => BarType::Default,
                     OutputLevel::Quiet => BarType::Quiet,
-                    OutputLevel::Silent => BarType::Hidden,
+                    OutputLevel::Silent | OutputLevel::SilentJSON => BarType::Hidden,
                 };
 
                 let pb = add_bar(&self.url, self.num_requests, bar_type);
@@ -268,7 +268,7 @@ impl FeroxScan {
         let mut guard = self.task.lock().await;
 
         if guard.is_some() {
-            if let Some(task) = std::mem::replace(&mut *guard, None) {
+            if let Some(task) = guard.take() {
                 task.await.unwrap();
                 self.set_status(ScanStatus::Complete)
                     .unwrap_or_else(|e| log::warn!("Could not mark scan complete: {}", e))

src/scan_manager/scan_container.rs

@@ -330,6 +330,13 @@ impl FeroxScans {
                     self.menu
                         .println(&format!("{}:", style("Scans").bright().blue()));
                 }
+
+                if let Ok(guard) = scan.status.lock() {
+                    if matches!(*guard, ScanStatus::Cancelled) {
+                        continue;
+                    }
+                }
+
                 // we're only interested in displaying directory scans, as those are
                 // the only ones that make sense to be stopped
                 let scan_msg = format!("{i:3}: {scan}");
@@ -360,7 +367,14 @@ impl FeroxScans {
                         sleep(menu_pause_duration);
                         continue;
                     }
-                    u_scans.index(num).clone()
+
+                    let selected = u_scans.index(num);
+
+                    if matches!(selected.scan_type, ScanType::File) {
+                        continue;
+                    }
+
+                    selected.clone()
                 }
                 Err(..) => continue,
             };
@@ -502,7 +516,7 @@ impl FeroxScans {
         let bar_type = match self.output_level {
             OutputLevel::Default => BarType::Message,
             OutputLevel::Quiet => BarType::Quiet,
-            OutputLevel::Silent => return Ok(()), // fast exit when --silent was used
+            OutputLevel::Silent | OutputLevel::SilentJSON => return Ok(()), // fast exit when --silent was used
         };
 
         if let Ok(scans) = self.scans.read() {
@@ -595,7 +609,7 @@ impl FeroxScans {
                 let bar_type = match self.output_level {
                     OutputLevel::Default => BarType::Default,
                     OutputLevel::Quiet => BarType::Quiet,
-                    OutputLevel::Silent => BarType::Hidden,
+                    OutputLevel::Silent | OutputLevel::SilentJSON => BarType::Hidden,
                 };
 
                 let progress_bar = add_bar(url, bar_length, bar_type);

src/scan_manager/tests.rs

@@ -202,6 +202,7 @@ fn partial_eq_compares_the_id_field() {
 
     assert!(!scan.eq(&scan_two));
 
+    #[allow(clippy::redundant_clone)]
     let scan_two = scan.clone();
 
     assert!(scan.eq(&scan_two));

src/scanner/ferox_scanner.rs

@@ -251,60 +251,57 @@ impl FeroxScanner {
             // heuristics test block:
             let test = heuristics::HeuristicTests::new(self.handles.clone());
 
-            if let Ok(dirlist_result) = test.directory_listing(&self.target_url).await {
-                if dirlist_result.is_some() {
-                    let dirlist_result = dirlist_result.unwrap();
-                    // at this point, we have a DirListingType, and it's not the None variant
-                    // which means we found directory listing based on the heuristic; now we need
-                    // to process the links that are available if --extract-links was used
+            if let Ok(Some(dirlist_result)) = test.directory_listing(&self.target_url).await {
+                // at this point, we have a DirListingType, and it's not the None variant
+                // which means we found directory listing based on the heuristic; now we need
+                // to process the links that are available if --extract-links was used
 
-                    if self.handles.config.extract_links {
-                        let mut extractor = ExtractorBuilder::default()
-                            .response(&dirlist_result.response)
-                            .target(ExtractionTarget::DirectoryListing)
-                            .url(&self.target_url)
-                            .handles(self.handles.clone())
-                            .build()?;
+                if self.handles.config.extract_links {
+                    let mut extractor = ExtractorBuilder::default()
+                        .response(&dirlist_result.response)
+                        .target(ExtractionTarget::DirectoryListing)
+                        .url(&self.target_url)
+                        .handles(self.handles.clone())
+                        .build()?;
 
-                        let result = extractor.extract_from_dir_listing().await?;
+                    let result = extractor.extract_from_dir_listing().await?;
 
-                        extraction_tasks.push(extractor.request_links(result).await?);
+                    extraction_tasks.push(extractor.request_links(result).await?);
 
-                        log::trace!("exit: scan_url -> Directory listing heuristic");
+                    log::trace!("exit: scan_url -> Directory listing heuristic");
 
-                        self.handles.stats.send(AddToF64Field(
-                            DirScanTimes,
-                            scan_timer.elapsed().as_secs_f64(),
-                        ))?;
+                    self.handles.stats.send(AddToF64Field(
+                        DirScanTimes,
+                        scan_timer.elapsed().as_secs_f64(),
+                    ))?;
 
-                        self.handles.stats.send(SubtractFromUsizeField(
-                            TotalExpected,
-                            progress_bar.length().unwrap_or(0) as usize,
-                        ))?;
+                    self.handles.stats.send(SubtractFromUsizeField(
+                        TotalExpected,
+                        progress_bar.length().unwrap_or(0) as usize,
+                    ))?;
+                }
+
+                let mut message = format!("=> {}", style("Directory listing").blue().bright());
+
+                if !self.handles.config.extract_links {
+                    write!(
+                        message,
+                        " (remove {} to scan)",
+                        style("--dont-extract-links").bright().yellow()
+                    )?;
+                }
+
+                if !self.handles.config.force_recursion {
+                    for handle in extraction_tasks.into_iter().flatten() {
+                        _ = handle.await;
                     }
 
-                    let mut message = format!("=> {}", style("Directory listing").blue().bright());
+                    progress_bar.reset_eta();
+                    progress_bar.finish_with_message(message);
 
-                    if !self.handles.config.extract_links {
-                        write!(
-                            message,
-                            " (remove {} to scan)",
-                            style("--dont-extract-links").bright().yellow()
-                        )?;
-                    }
+                    ferox_scan.finish()?;
 
-                    if !self.handles.config.force_recursion {
-                        for handle in extraction_tasks.into_iter().flatten() {
-                            _ = handle.await;
-                        }
-
-                        progress_bar.reset_eta();
-                        progress_bar.finish_with_message(message);
-
-                        ferox_scan.finish()?;
-
-                        return Ok(()); // nothing left to do if we found a dir listing
-                    }
+                    return Ok(()); // nothing left to do if we found a dir listing
                 }
             }
 

src/scanner/requester.rs

@@ -475,12 +475,13 @@ impl Requester {
 
                 if self.handles.config.collect_words {
                     if let Ok(mut guard) = TF_IDF.write() {
-                        let doc = Document::from_html(ferox_response.text());
-                        guard.add_document(doc);
-                        if guard.num_documents() % 12 == 0
-                            || (guard.num_documents() < 5 && guard.num_documents() % 2 == 0)
-                        {
-                            guard.calculate_tf_idf_scores();
+                        if let Some(doc) = Document::from_html(ferox_response.text()) {
+                            guard.add_document(doc);
+                            if guard.num_documents() % 12 == 0
+                                || (guard.num_documents() < 5 && guard.num_documents() % 2 == 0)
+                            {
+                                guard.calculate_tf_idf_scores();
+                            }
                         }
                     }
                 }

src/statistics/tests.rs

@@ -55,7 +55,7 @@ fn save_writes_stats_object_to_disk() {
     stats.add_status_code(StatusCode::OK);
     stats.add_status_code(StatusCode::OK);
     let outfile = NamedTempFile::new().unwrap();
-    if stats.save(174.33, outfile.path().to_str().unwrap()).is_ok() {}
+    assert!(stats.save(174.33, outfile.path().to_str().unwrap()).is_ok());
 
     assert!(stats.as_json().unwrap().contains("statistics"));
     assert!(stats.as_json().unwrap().contains("11")); // requests made

src/utils.rs

@@ -563,6 +563,13 @@ pub fn parse_url_with_raw_path(url: &str) -> Result<Url> {
         bail!("url to parse has no authority and is therefore invalid");
     }
 
+    // thanks to @devx00: the possibility exists for Url to return true for
+    // has_authority, but not have a host/port, so we'll check for that
+    // and bail if it's the case
+    if parsed.host().is_none() {
+        bail!("url to parse doesn't have a host");
+    }
+
     // we have a valid url, the next step is to check the path and see if it's
     // something that url::Url::parse would silently transform
     //
@@ -648,7 +655,7 @@ pub fn parse_url_with_raw_path(url: &str) -> Result<Url> {
     // each of the following is a string that we can expect url::Url::parse to
     // transform. The variety is to ensure we cover most common path traversal
     // encodings
-    let transformation_detectors = vec![
+    let transformation_detectors = [
         // ascii
         "..",
         // single url encoded
@@ -729,6 +736,18 @@ mod tests {
     use crate::config::Configuration;
     use crate::scan_manager::{FeroxScans, ScanOrder};
 
+    #[test]
+    /// parse_url_with_raw_path with javascript:// should not throw an unimplemented! error
+    fn utils_parse_url_with_raw_path_javascript() {
+        let url = "javascript://";
+        let parsed = parse_url_with_raw_path(url);
+        assert!(parsed.is_err());
+        assert!(parsed
+            .unwrap_err()
+            .to_string()
+            .contains("url to parse doesn't have a host"));
+    }
+
     #[test]
     /// multiple tests for parse_url_with_raw_path
     fn utils_parse_url_with_raw_path() {

tests/test_extractor.rs

@@ -640,3 +640,73 @@ fn extractor_recurses_into_403_directories() -> Result<(), Box<dyn std::error::E
     teardown_tmp_directory(tmp_dir);
     Ok(())
 }
+
+#[test]
+/// robots.txt requests shouldn't fire when --dont-extract-links is used
+fn robots_text_extraction_doesnt_run_with_dont_extract_links() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("im a little teapot"); // 18
+    });
+
+    let mock_two = srv.mock(|when, then| {
+        when.method(GET).path("/robots.txt");
+        then.status(200).body(
+            r#"
+            User-agent: *
+            Crawl-delay: 10
+            # CSS, JS, Images
+            Allow: /misc/*.css$
+            Disallow: /misc/stupidfile.php
+               Disallow: /disallowed-subdir/
+            "#,
+        );
+    });
+
+    let mock_file = srv.mock(|when, then| {
+        when.method(GET).path("/misc/stupidfile.php");
+        then.status(200).body("im a little teapot too"); // 22
+    });
+
+    let mock_scanned_file = srv.mock(|when, then| {
+        when.method(GET).path("/misc/LICENSE");
+        then.status(200).body("i too, am a container for tea"); // 29
+    });
+
+    let mock_dir = srv.mock(|when, _| {
+        when.method(GET).path("/misc/");
+    });
+
+    let mock_disallowed = srv.mock(|when, then| {
+        when.method(GET).path("/disallowed-subdir");
+        then.status(404);
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--dont-extract-links")
+        .arg("--no-recursion")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .and(predicate::str::contains("18c"))
+            .and(predicate::str::contains("/misc/stupidfile.php"))
+            .not(),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_dir.hits(), 0);
+    assert_eq!(mock_two.hits(), 0);
+    assert_eq!(mock_file.hits(), 0);
+    assert_eq!(mock_disallowed.hits(), 0);
+    assert_eq!(mock_scanned_file.hits(), 0);
+    teardown_tmp_directory(tmp_dir);
+}

tests/test_filters.rs

@@ -247,3 +247,46 @@ fn filters_similar_should_filter_response() {
     assert_eq!(not_similar.hits(), 1);
     teardown_tmp_directory(tmp_dir);
 }
+
+#[test]
+/// when using --collect-backups, should only see results in output
+/// when the response shouldn't be otherwise filtered
+fn collect_backups_should_be_filtered() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
+
+    let mock = srv.mock(|when: httpmock::When, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("this is a test");
+    });
+
+    let mock_two = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE.bak");
+        then.status(201)
+            .body("im a backup file, but filtered out because im not 200");
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--status-codes")
+        .arg("200")
+        .arg("--collect-backups")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .and(predicate::str::contains("200"))
+            .and(predicate::str::contains("/LICENSE.bak"))
+            .not()
+            .and(predicate::str::contains("201"))
+            .not(),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_two.hits(), 1);
+    teardown_tmp_directory(tmp_dir);
+}

tests/test_scanner.rs

@@ -693,7 +693,7 @@ fn collect_backups_makes_appropriate_requests() {
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE.txt".to_string()], "wordlist").unwrap();
 
-    let valid_paths = vec![
+    let valid_paths = [
         "/LICENSE.txt",
         "/LICENSE.txt~",
         "/LICENSE.txt.bak",