docs: add soutzis as a contributor for bug (#1133)

* docs: update README.md [skip ci]

* docs: update .all-contributorsrc [skip ci]

---------

Co-authored-by: allcontributors[bot] <46447321+allcontributors[bot]@users.noreply.github.com>

.all-contributorsrc

@@ -300,7 +300,8 @@
       "avatar_url": "https://avatars.githubusercontent.com/u/16690056?v=4",
       "profile": "https://github.com/n0kovo",
       "contributions": [
-        "ideas"
+        "ideas",
+        "bug"
       ]
     },
     {
@@ -673,6 +674,132 @@
       "contributions": [
         "ideas"
       ]
+    },
+    {
+      "login": "stuhlmann",
+      "name": "Florian Stuhlmann",
+      "avatar_url": "https://avatars.githubusercontent.com/u/11061864?v=4",
+      "profile": "https://github.com/stuhlmann",
+      "contributions": [
+        "bug"
+      ]
+    },
+    {
+      "login": "Mister7F",
+      "name": "Mister7F",
+      "avatar_url": "https://avatars.githubusercontent.com/u/35213773?v=4",
+      "profile": "https://github.com/Mister7F",
+      "contributions": [
+        "ideas"
+      ]
+    },
+    {
+      "login": "manugramm",
+      "name": "manugramm",
+      "avatar_url": "https://avatars.githubusercontent.com/u/145961515?v=4",
+      "profile": "https://github.com/manugramm",
+      "contributions": [
+        "bug"
+      ]
+    },
+    {
+      "login": "ArthurMuraro",
+      "name": "ArthurMuraro",
+      "avatar_url": "https://avatars.githubusercontent.com/u/73059809?v=4",
+      "profile": "https://github.com/ArthurMuraro",
+      "contributions": [
+        "bug"
+      ]
+    },
+    {
+      "login": "amiremami",
+      "name": "Shadow",
+      "avatar_url": "https://avatars.githubusercontent.com/u/15929497?v=4",
+      "profile": "https://github.com/amiremami",
+      "contributions": [
+        "bug"
+      ]
+    },
+    {
+      "login": "dirhamgithub",
+      "name": "dirhamgithub",
+      "avatar_url": "https://avatars.githubusercontent.com/u/115349974?v=4",
+      "profile": "https://github.com/dirhamgithub",
+      "contributions": [
+        "bug"
+      ]
+    },
+    {
+      "login": "FieldOfRice",
+      "name": "FieldOfRice",
+      "avatar_url": "https://avatars.githubusercontent.com/u/85353?v=4",
+      "profile": "https://github.com/FieldOfRice",
+      "contributions": [
+        "infra"
+      ]
+    },
+    {
+      "login": "NotoriousRebel",
+      "name": "Matt",
+      "avatar_url": "https://avatars.githubusercontent.com/u/36310667?v=4",
+      "profile": "https://github.com/NotoriousRebel",
+      "contributions": [
+        "ideas"
+      ]
+    },
+    {
+      "login": "tritoke",
+      "name": "Sam Leonard",
+      "avatar_url": "https://avatars.githubusercontent.com/u/34941249?v=4",
+      "profile": "https://github.com/tritoke",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "rew1nter",
+      "name": "Rewinter",
+      "avatar_url": "https://avatars.githubusercontent.com/u/64508791?v=4",
+      "profile": "https://github.com/rew1nter",
+      "contributions": [
+        "ideas"
+      ]
+    },
+    {
+      "login": "deadloot",
+      "name": "deadloot",
+      "avatar_url": "https://avatars.githubusercontent.com/u/92878901?v=4",
+      "profile": "https://github.com/deadloot",
+      "contributions": [
+        "ideas"
+      ]
+    },
+    {
+      "login": "Spidle",
+      "name": "Spidle",
+      "avatar_url": "https://avatars.githubusercontent.com/u/90011249?v=4",
+      "profile": "https://github.com/Spidle",
+      "contributions": [
+        "ideas"
+      ]
+    },
+    {
+      "login": "JulianGR",
+      "name": "Julián Gómez",
+      "avatar_url": "https://avatars.githubusercontent.com/u/53094530?v=4",
+      "profile": "https://github.com/JulianGR",
+      "contributions": [
+        "ideas"
+      ]
+    },
+    {
+      "login": "soutzis",
+      "name": "Petros",
+      "avatar_url": "https://avatars.githubusercontent.com/u/25797286?v=4",
+      "profile": "https://github.com/soutzis",
+      "contributions": [
+        "bug"
+      ]
     }
   ],
   "contributorsPerLine": 7,

.github/workflows/build.yml

@@ -73,22 +73,57 @@ jobs:
           name: ${{ matrix.name }}.tar.gz
           path: ${{ matrix.name }}.tar.gz
 
-  # build-deb:
+  build-debug:
-  #   needs: [build-nix]
+    env:
-  #   runs-on: ubuntu-latest
+      IN_PIPELINE: true
-  #   steps:
+    runs-on: ubuntu-latest
-  #     - uses: actions/checkout@master
+    steps:
-  #     - name: Install cargo-deb
+      - uses: actions/checkout@v2
-  #       run: cargo install -f cargo-deb
+      - name: Install System Dependencies
-  #     - name: Install musl toolchain
+        run: |
-  #       run: rustup target add x86_64-unknown-linux-musl
+          env
-  #     - name: Deb Build
+          sudo apt-get update
-  #       run: cargo deb --target=x86_64-unknown-linux-musl
+          sudo apt-get install -y --no-install-recommends libssl-dev pkg-config
-  #     - name: Upload Deb Artifact
+      - uses: actions-rs/toolchain@v1
-  #       uses: actions/upload-artifact@v2
+        with:
-  #       with:
+          toolchain: stable
-  #         name: feroxbuster_amd64.deb
+          target: x86_64-unknown-linux-musl
-  #         path: ./target/x86_64-unknown-linux-musl/debian/*
+          override: true
+      - uses: actions-rs/cargo@v1
+        env:
+          PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig
+          OPENSSL_DIR: /usr/lib/ssl
+        with:
+          use-cross: true
+          command: build
+          args: --target=x86_64-unknown-linux-musl
+      - uses: actions/upload-artifact@v2
+        with:
+          name: x86_64-linux-debug-feroxbuster
+          path: target/x86_64-unknown-linux-musl/debug/feroxbuster
+
+  build-deb:
+    needs: [build-nix]
+    runs-on: ubuntu-latest
+    env:
+      IN_PIPELINE: true
+    steps:
+      - uses: actions/checkout@master
+      - name: Install cargo-deb
+        run: cargo install -f cargo-deb
+      - uses: awalsh128/cache-apt-pkgs-action@v1
+        with:
+          packages: musl-tools # provides musl-gcc
+          version: 1.0
+      - name: Install musl toolchain
+        run: rustup target add x86_64-unknown-linux-musl
+      - name: Deb Build
+        run: cargo deb --target=x86_64-unknown-linux-musl
+      - name: Upload Deb Artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: feroxbuster_amd64.deb
+          path: ./target/x86_64-unknown-linux-musl/debian/*
 
   build-macos:
     env:

.github/workflows/check.yml

@@ -23,7 +23,7 @@ jobs:
         uses: actions-rs/cargo@v1
         with:
           command: nextest
-          args: run --all-features --all-targets --retries 10
+          args: run --all-features --all-targets
 
   fmt:
     name: Rust fmt

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "feroxbuster"
-version = "2.10.1"
+version = "2.10.3"
 authors = ["Ben 'epi' Risher (@epi052)"]
 license = "MIT"
 edition = "2021"
@@ -22,36 +22,36 @@ build = "build.rs"
 maintenance = { status = "actively-developed" }
 
 [build-dependencies]
-clap = { version = "4.3", features = ["wrap_help", "cargo"] }
+clap = { version = "4.5", features = ["wrap_help", "cargo"] }
-clap_complete = "4.3"
+clap_complete = "4.5"
-regex = "1.9"
+regex = "1.10"
 lazy_static = "1.4"
 dirs = "5.0"
 
 [dependencies]
 scraper = "0.18"
 futures = "0.3"
-tokio = { version = "1.29", features = ["full"] }
+tokio = { version = "1.37", features = ["full"] }
 tokio-util = { version = "0.7", features = ["codec"] }
 log = "0.4"
 env_logger = "0.10"
 reqwest = { version = "0.11", features = ["socks", "native-tls-alpn"] }
 # uses feature unification to add 'serde' to reqwest::Url
-url = { version = "2.4", features = ["serde"] }
+url = { version = "2.5", features = ["serde"] }
 serde_regex = "1.1"
-clap = { version = "4.3", features = ["wrap_help", "cargo"] }
+clap = { version = "4.5", features = ["wrap_help", "cargo"] }
 lazy_static = "1.4"
 toml = "0.8"
 serde = { version = "1.0", features = ["derive", "rc"] }
 serde_json = "1.0"
-uuid = { version = "1.4", features = ["v4"] }
+uuid = { version = "1.8", features = ["v4"] }
 # last known working version of indicatif; 0.17.5 has a bug that causes the 
 # scan menu to fail spectacularly 
-indicatif = { version = "0.17.3" }
+indicatif = { version = "0.17.8" }
 console = "0.15"
 openssl = { version = "0.10", features = ["vendored"] }
 dirs = "5.0"
-regex = "1.9"
+regex = "1.10"
 crossterm = "0.27"
 rlimit = "0.10"
 ctrlc = "3.4"
@@ -69,10 +69,10 @@ self_update = { version = "0.36", features = [
 ] }
 
 [dev-dependencies]
-tempfile = "3.6"
+tempfile = "3.10"
 httpmock = "0.6"
 assert_cmd = "2.0"
-predicates = "3.0"
+predicates = "3.1"
 
 [profile.release]
 lto = true

Makefile.toml

@@ -2,6 +2,9 @@
 [tasks.upgrade]
 dependencies = ["upgrade-deps", "update"]
 
+[tasks.check]
+dependencies = ["fmt", "clippy", "test"]
+
 # cleaning
 [tasks.clean-state]
 script = """
@@ -11,7 +14,7 @@ rm ferox-*.state
 # dependency management
 [tasks.upgrade-deps]
 command = "cargo"
-args = ["upgrade", "--to-lockfile", "--exclude", "indicatif", "self_update"]
+args = ["upgrade", "--exclude", "indicatif, self_update"]
 
 [tasks.update]
 command = "cargo"
@@ -24,9 +27,15 @@ script = """
 cargo clippy --all-targets --all-features -- -D warnings
 """
 
+[tasks.fmt]
+clear = true
+script = """
+cargo fmt --all
+"""
+
 # tests
 [tasks.test]
 clear = true
 script = """
-cargo nextest run --all-features --all-targets --retries 10
+cargo nextest run --all-features --all-targets
 """

README.md

@@ -247,7 +247,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/hunter0x8"><img src="https://avatars.githubusercontent.com/u/46222314?v=4?s=100" width="100px;" alt="Muhammad Ahsan"/><br /><sub><b>Muhammad Ahsan</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3Ahunter0x8" title="Bug reports">🐛</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/cortantief"><img src="https://avatars.githubusercontent.com/u/34527333?v=4?s=100" width="100px;" alt="cortantief"/><br /><sub><b>cortantief</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3Acortantief" title="Bug reports">🐛</a> <a href="https://github.com/epi052/feroxbuster/commits?author=cortantief" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/dsaxton"><img src="https://avatars.githubusercontent.com/u/2658661?v=4?s=100" width="100px;" alt="Daniel Saxton"/><br /><sub><b>Daniel Saxton</b></sub></a><br /><a href="#ideas-dsaxton" title="Ideas, Planning, & Feedback">🤔</a> <a href="https://github.com/epi052/feroxbuster/commits?author=dsaxton" title="Code">💻</a></td>
-      <td align="center" valign="top" width="14.28%"><a href="https://github.com/n0kovo"><img src="https://avatars.githubusercontent.com/u/16690056?v=4?s=100" width="100px;" alt="n0kovo"/><br /><sub><b>n0kovo</b></sub></a><br /><a href="#ideas-n0kovo" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/n0kovo"><img src="https://avatars.githubusercontent.com/u/16690056?v=4?s=100" width="100px;" alt="n0kovo"/><br /><sub><b>n0kovo</b></sub></a><br /><a href="#ideas-n0kovo" title="Ideas, Planning, & Feedback">🤔</a> <a href="https://github.com/epi052/feroxbuster/issues?q=author%3An0kovo" title="Bug reports">🐛</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://ring0.lol"><img src="https://avatars.githubusercontent.com/u/1893909?v=4?s=100" width="100px;" alt="Justin Steven"/><br /><sub><b>Justin Steven</b></sub></a><br /><a href="#ideas-justinsteven" title="Ideas, Planning, & Feedback">🤔</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/7047payloads"><img src="https://avatars.githubusercontent.com/u/95562424?v=4?s=100" width="100px;" alt="7047payloads"/><br /><sub><b>7047payloads</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/commits?author=7047payloads" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/unkn0wnsyst3m"><img src="https://avatars.githubusercontent.com/u/21272239?v=4?s=100" width="100px;" alt="unkn0wnsyst3m"/><br /><sub><b>unkn0wnsyst3m</b></sub></a><br /><a href="#ideas-unkn0wnsyst3m" title="Ideas, Planning, & Feedback">🤔</a></td>
@@ -300,6 +300,24 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
     <tr>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/ocervell"><img src="https://avatars.githubusercontent.com/u/9629314?v=4?s=100" width="100px;" alt="Olivier Cervello"/><br /><sub><b>Olivier Cervello</b></sub></a><br /><a href="#ideas-ocervell" title="Ideas, Planning, & Feedback">🤔</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/RavySena"><img src="https://avatars.githubusercontent.com/u/67729597?v=4?s=100" width="100px;" alt="RavySena"/><br /><sub><b>RavySena</b></sub></a><br /><a href="#ideas-RavySena" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/stuhlmann"><img src="https://avatars.githubusercontent.com/u/11061864?v=4?s=100" width="100px;" alt="Florian Stuhlmann"/><br /><sub><b>Florian Stuhlmann</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3Astuhlmann" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/Mister7F"><img src="https://avatars.githubusercontent.com/u/35213773?v=4?s=100" width="100px;" alt="Mister7F"/><br /><sub><b>Mister7F</b></sub></a><br /><a href="#ideas-Mister7F" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/manugramm"><img src="https://avatars.githubusercontent.com/u/145961515?v=4?s=100" width="100px;" alt="manugramm"/><br /><sub><b>manugramm</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3Amanugramm" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/ArthurMuraro"><img src="https://avatars.githubusercontent.com/u/73059809?v=4?s=100" width="100px;" alt="ArthurMuraro"/><br /><sub><b>ArthurMuraro</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3AArthurMuraro" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/amiremami"><img src="https://avatars.githubusercontent.com/u/15929497?v=4?s=100" width="100px;" alt="Shadow"/><br /><sub><b>Shadow</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3Aamiremami" title="Bug reports">🐛</a></td>
+    </tr>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/dirhamgithub"><img src="https://avatars.githubusercontent.com/u/115349974?v=4?s=100" width="100px;" alt="dirhamgithub"/><br /><sub><b>dirhamgithub</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3Adirhamgithub" title="Bug reports">🐛</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/FieldOfRice"><img src="https://avatars.githubusercontent.com/u/85353?v=4?s=100" width="100px;" alt="FieldOfRice"/><br /><sub><b>FieldOfRice</b></sub></a><br /><a href="#infra-FieldOfRice" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/NotoriousRebel"><img src="https://avatars.githubusercontent.com/u/36310667?v=4?s=100" width="100px;" alt="Matt"/><br /><sub><b>Matt</b></sub></a><br /><a href="#ideas-NotoriousRebel" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/tritoke"><img src="https://avatars.githubusercontent.com/u/34941249?v=4?s=100" width="100px;" alt="Sam Leonard"/><br /><sub><b>Sam Leonard</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/commits?author=tritoke" title="Code">💻</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/rew1nter"><img src="https://avatars.githubusercontent.com/u/64508791?v=4?s=100" width="100px;" alt="Rewinter"/><br /><sub><b>Rewinter</b></sub></a><br /><a href="#ideas-rew1nter" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/deadloot"><img src="https://avatars.githubusercontent.com/u/92878901?v=4?s=100" width="100px;" alt="deadloot"/><br /><sub><b>deadloot</b></sub></a><br /><a href="#ideas-deadloot" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/Spidle"><img src="https://avatars.githubusercontent.com/u/90011249?v=4?s=100" width="100px;" alt="Spidle"/><br /><sub><b>Spidle</b></sub></a><br /><a href="#ideas-Spidle" title="Ideas, Planning, & Feedback">🤔</a></td>
+    </tr>
+    <tr>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/JulianGR"><img src="https://avatars.githubusercontent.com/u/53094530?v=4?s=100" width="100px;" alt="Julián Gómez"/><br /><sub><b>Julián Gómez</b></sub></a><br /><a href="#ideas-JulianGR" title="Ideas, Planning, & Feedback">🤔</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/soutzis"><img src="https://avatars.githubusercontent.com/u/25797286?v=4?s=100" width="100px;" alt="Petros"/><br /><sub><b>Petros</b></sub></a><br /><a href="https://github.com/epi052/feroxbuster/issues?q=author%3Asoutzis" title="Bug reports">🐛</a></td>
     </tr>
   </tbody>
 </table>

install-nix.sh

@@ -44,7 +44,7 @@ elif [[ "$(expr substr $(uname -s) 1 5)" == "Linux" ]]; then
     rm "$LIN64_ZIP"
   fi
 
-  if [[ -e ~/.fonts/NotoColorEmoji.ttf ]]; then
+  if [[ "$(fc-list NotoColorEmoji | wc -l)" -gt 0 ]]; then
     echo "[=] Found Noto Emoji Font, skipping install"
   else
     echo "[=] Installing Noto Emoji Font"

shell_completions/_feroxbuster

@@ -24,8 +24,8 @@ _feroxbuster() {
 '--replay-proxy=[Send only unfiltered requests through a Replay Proxy, instead of all requests]:REPLAY_PROXY:_urls' \
 '*-R+[Status Codes to send through a Replay Proxy when found (default\: --status-codes value)]:REPLAY_CODE: ' \
 '*--replay-codes=[Status Codes to send through a Replay Proxy when found (default\: --status-codes value)]:REPLAY_CODE: ' \
-'-a+[Sets the User-Agent (default\: feroxbuster/2.10.1)]:USER_AGENT: ' \
+'-a+[Sets the User-Agent (default\: feroxbuster/2.10.3)]:USER_AGENT: ' \
-'--user-agent=[Sets the User-Agent (default\: feroxbuster/2.10.1)]:USER_AGENT: ' \
+'--user-agent=[Sets the User-Agent (default\: feroxbuster/2.10.3)]:USER_AGENT: ' \
 '*-x+[File extension(s) to search for (ex\: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex\: @ext.txt)]:FILE_EXTENSION: ' \
 '*--extensions=[File extension(s) to search for (ex\: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex\: @ext.txt)]:FILE_EXTENSION: ' \
 '*-m+[Which HTTP request method(s) should be sent (default\: GET)]:HTTP_METHODS: ' \
@@ -62,11 +62,13 @@ _feroxbuster() {
 '--depth=[Maximum recursion depth, a depth of 0 is infinite recursion (default\: 4)]:RECURSION_DEPTH: ' \
 '-L+[Limit total number of concurrent scans (default\: 0, i.e. no limit)]:SCAN_LIMIT: ' \
 '--scan-limit=[Limit total number of concurrent scans (default\: 0, i.e. no limit)]:SCAN_LIMIT: ' \
-'--parallel=[Run parallel feroxbuster instances (one child process per url passed via stdin)]:PARALLEL_SCANS: ' \
+'(-v --verbosity)--parallel=[Run parallel feroxbuster instances (one child process per url passed via stdin)]:PARALLEL_SCANS: ' \
 '(--auto-tune)--rate-limit=[Limit number of requests per second (per directory) (default\: 0, i.e. no limit)]:RATE_LIMIT: ' \
 '--time-limit=[Limit total run time of all scans (ex\: --time-limit 10m)]:TIME_SPEC: ' \
 '-w+[Path or URL of the wordlist]:FILE:_files' \
 '--wordlist=[Path or URL of the wordlist]:FILE:_files' \
+'-B+[Automatically request likely backup extensions for "found" urls (default\: ~, .bak, .bak2, .old, .1)]' \
+'--collect-backups=[Automatically request likely backup extensions for "found" urls (default\: ~, .bak, .bak2, .old, .1)]' \
 '*-I+[File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)]:FILE_EXTENSION: ' \
 '*--dont-collect=[File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)]:FILE_EXTENSION: ' \
 '-o+[Output file to write results to (use w/ --json for JSON entries)]:FILE:_files' \
@@ -97,8 +99,6 @@ _feroxbuster() {
 '--dont-filter[Don'\''t auto-filter wildcard responses]' \
 '-E[Automatically discover extensions and add them to --extensions (unless they'\''re in --dont-collect)]' \
 '--collect-extensions[Automatically discover extensions and add them to --extensions (unless they'\''re in --dont-collect)]' \
-'-B[Automatically request likely backup extensions for "found" urls]' \
-'--collect-backups[Automatically request likely backup extensions for "found" urls]' \
 '-g[Automatically discover important words from within responses and add them to the wordlist]' \
 '--collect-words[Automatically discover important words from within responses and add them to the wordlist]' \
 '(--silent)*-v[Increase verbosity level (use -vv or more for greater effect. \[CAUTION\] 4 -v'\''s is probably too much)]' \

shell_completions/_feroxbuster.ps1

@@ -30,8 +30,8 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
             [CompletionResult]::new('--replay-proxy', 'replay-proxy', [CompletionResultType]::ParameterName, 'Send only unfiltered requests through a Replay Proxy, instead of all requests')
             [CompletionResult]::new('-R', 'R ', [CompletionResultType]::ParameterName, 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)')
             [CompletionResult]::new('--replay-codes', 'replay-codes', [CompletionResultType]::ParameterName, 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)')
-            [CompletionResult]::new('-a', 'a', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.10.1)')
+            [CompletionResult]::new('-a', 'a', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.10.3)')
-            [CompletionResult]::new('--user-agent', 'user-agent', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.10.1)')
+            [CompletionResult]::new('--user-agent', 'user-agent', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/2.10.3)')
             [CompletionResult]::new('-x', 'x', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex: @ext.txt)')
             [CompletionResult]::new('--extensions', 'extensions', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex: @ext.txt)')
             [CompletionResult]::new('-m', 'm', [CompletionResultType]::ParameterName, 'Which HTTP request method(s) should be sent (default: GET)')
@@ -73,6 +73,8 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
             [CompletionResult]::new('--time-limit', 'time-limit', [CompletionResultType]::ParameterName, 'Limit total run time of all scans (ex: --time-limit 10m)')
             [CompletionResult]::new('-w', 'w', [CompletionResultType]::ParameterName, 'Path or URL of the wordlist')
             [CompletionResult]::new('--wordlist', 'wordlist', [CompletionResultType]::ParameterName, 'Path or URL of the wordlist')
+            [CompletionResult]::new('-B', 'B ', [CompletionResultType]::ParameterName, 'Automatically request likely backup extensions for "found" urls (default: ~, .bak, .bak2, .old, .1)')
+            [CompletionResult]::new('--collect-backups', 'collect-backups', [CompletionResultType]::ParameterName, 'Automatically request likely backup extensions for "found" urls (default: ~, .bak, .bak2, .old, .1)')
             [CompletionResult]::new('-I', 'I ', [CompletionResultType]::ParameterName, 'File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)')
             [CompletionResult]::new('--dont-collect', 'dont-collect', [CompletionResultType]::ParameterName, 'File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)')
             [CompletionResult]::new('-o', 'o', [CompletionResultType]::ParameterName, 'Output file to write results to (use w/ --json for JSON entries)')
@@ -103,8 +105,6 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
             [CompletionResult]::new('--dont-filter', 'dont-filter', [CompletionResultType]::ParameterName, 'Don''t auto-filter wildcard responses')
             [CompletionResult]::new('-E', 'E ', [CompletionResultType]::ParameterName, 'Automatically discover extensions and add them to --extensions (unless they''re in --dont-collect)')
             [CompletionResult]::new('--collect-extensions', 'collect-extensions', [CompletionResultType]::ParameterName, 'Automatically discover extensions and add them to --extensions (unless they''re in --dont-collect)')
-            [CompletionResult]::new('-B', 'B ', [CompletionResultType]::ParameterName, 'Automatically request likely backup extensions for "found" urls')
-            [CompletionResult]::new('--collect-backups', 'collect-backups', [CompletionResultType]::ParameterName, 'Automatically request likely backup extensions for "found" urls')
             [CompletionResult]::new('-g', 'g', [CompletionResultType]::ParameterName, 'Automatically discover important words from within responses and add them to the wordlist')
             [CompletionResult]::new('--collect-words', 'collect-words', [CompletionResultType]::ParameterName, 'Automatically discover important words from within responses and add them to the wordlist')
             [CompletionResult]::new('-v', 'v', [CompletionResultType]::ParameterName, 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v''s is probably too much)')

shell_completions/feroxbuster.bash

@@ -34,7 +34,18 @@ _feroxbuster() {
                     return 0
                     ;;
                 --resume-from)
+                    local oldifs
+                    if [ -n "${IFS+x}" ]; then
+                        oldifs="$IFS"
+                    fi
+                    IFS=$'\n'
                     COMPREPLY=($(compgen -f "${cur}"))
+                    if [ -n "${oldifs+x}" ]; then
+                        IFS="$oldifs"
+                    fi
+                    if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
+                        compopt -o filenames
+                    fi
                     return 0
                     ;;
                 --proxy)
@@ -178,15 +189,48 @@ _feroxbuster() {
                     return 0
                     ;;
                 --server-certs)
+                    local oldifs
+                    if [ -n "${IFS+x}" ]; then
+                        oldifs="$IFS"
+                    fi
+                    IFS=$'\n'
                     COMPREPLY=($(compgen -f "${cur}"))
+                    if [ -n "${oldifs+x}" ]; then
+                        IFS="$oldifs"
+                    fi
+                    if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
+                        compopt -o filenames
+                    fi
                     return 0
                     ;;
                 --client-cert)
+                    local oldifs
+                    if [ -n "${IFS+x}" ]; then
+                        oldifs="$IFS"
+                    fi
+                    IFS=$'\n'
                     COMPREPLY=($(compgen -f "${cur}"))
+                    if [ -n "${oldifs+x}" ]; then
+                        IFS="$oldifs"
+                    fi
+                    if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
+                        compopt -o filenames
+                    fi
                     return 0
                     ;;
                 --client-key)
+                    local oldifs
+                    if [ -n "${IFS+x}" ]; then
+                        oldifs="$IFS"
+                    fi
+                    IFS=$'\n'
                     COMPREPLY=($(compgen -f "${cur}"))
+                    if [ -n "${oldifs+x}" ]; then
+                        IFS="$oldifs"
+                    fi
+                    if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
+                        compopt -o filenames
+                    fi
                     return 0
                     ;;
                 --threads)
@@ -226,10 +270,40 @@ _feroxbuster() {
                     return 0
                     ;;
                 --wordlist)
+                    local oldifs
+                    if [ -n "${IFS+x}" ]; then
+                        oldifs="$IFS"
+                    fi
+                    IFS=$'\n'
                     COMPREPLY=($(compgen -f "${cur}"))
+                    if [ -n "${oldifs+x}" ]; then
+                        IFS="$oldifs"
+                    fi
+                    if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
+                        compopt -o filenames
+                    fi
                     return 0
                     ;;
                 -w)
+                    local oldifs
+                    if [ -n "${IFS+x}" ]; then
+                        oldifs="$IFS"
+                    fi
+                    IFS=$'\n'
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    if [ -n "${oldifs+x}" ]; then
+                        IFS="$oldifs"
+                    fi
+                    if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
+                        compopt -o filenames
+                    fi
+                    return 0
+                    ;;
+                --collect-backups)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -B)
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
                     ;;
@@ -242,15 +316,48 @@ _feroxbuster() {
                     return 0
                     ;;
                 --output)
+                    local oldifs
+                    if [ -n "${IFS+x}" ]; then
+                        oldifs="$IFS"
+                    fi
+                    IFS=$'\n'
                     COMPREPLY=($(compgen -f "${cur}"))
+                    if [ -n "${oldifs+x}" ]; then
+                        IFS="$oldifs"
+                    fi
+                    if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
+                        compopt -o filenames
+                    fi
                     return 0
                     ;;
                 -o)
+                    local oldifs
+                    if [ -n "${IFS+x}" ]; then
+                        oldifs="$IFS"
+                    fi
+                    IFS=$'\n'
                     COMPREPLY=($(compgen -f "${cur}"))
+                    if [ -n "${oldifs+x}" ]; then
+                        IFS="$oldifs"
+                    fi
+                    if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
+                        compopt -o filenames
+                    fi
                     return 0
                     ;;
                 --debug-log)
+                    local oldifs
+                    if [ -n "${IFS+x}" ]; then
+                        oldifs="$IFS"
+                    fi
+                    IFS=$'\n'
                     COMPREPLY=($(compgen -f "${cur}"))
+                    if [ -n "${oldifs+x}" ]; then
+                        IFS="$oldifs"
+                    fi
+                    if [[ "${BASH_VERSINFO[0]}" -ge 4 ]]; then
+                        compopt -o filenames
+                    fi
                     return 0
                     ;;
                 *)
@@ -263,4 +370,8 @@ _feroxbuster() {
     esac
 }
 
-complete -F _feroxbuster -o bashdefault -o default -o plusdirs feroxbuster
+if [[ "${BASH_VERSINFO[0]}" -eq 4 && "${BASH_VERSINFO[1]}" -ge 4 || "${BASH_VERSINFO[0]}" -gt 4 ]]; then
+    complete -F _feroxbuster -o nosort -o bashdefault -o default -o plusdirs feroxbuster
+else
+    complete -F _feroxbuster -o bashdefault -o default -o plusdirs feroxbuster
+fi

shell_completions/feroxbuster.elv

@@ -27,8 +27,8 @@ set edit:completion:arg-completer[feroxbuster] = {|@words|
             cand --replay-proxy 'Send only unfiltered requests through a Replay Proxy, instead of all requests'
             cand -R 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)'
             cand --replay-codes 'Status Codes to send through a Replay Proxy when found (default: --status-codes value)'
-            cand -a 'Sets the User-Agent (default: feroxbuster/2.10.1)'
+            cand -a 'Sets the User-Agent (default: feroxbuster/2.10.3)'
-            cand --user-agent 'Sets the User-Agent (default: feroxbuster/2.10.1)'
+            cand --user-agent 'Sets the User-Agent (default: feroxbuster/2.10.3)'
             cand -x 'File extension(s) to search for (ex: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex: @ext.txt)'
             cand --extensions 'File extension(s) to search for (ex: -x php -x pdf js); reads values (newline-separated) from file if input starts with an @ (ex: @ext.txt)'
             cand -m 'Which HTTP request method(s) should be sent (default: GET)'
@@ -70,6 +70,8 @@ set edit:completion:arg-completer[feroxbuster] = {|@words|
             cand --time-limit 'Limit total run time of all scans (ex: --time-limit 10m)'
             cand -w 'Path or URL of the wordlist'
             cand --wordlist 'Path or URL of the wordlist'
+            cand -B 'Automatically request likely backup extensions for "found" urls (default: ~, .bak, .bak2, .old, .1)'
+            cand --collect-backups 'Automatically request likely backup extensions for "found" urls (default: ~, .bak, .bak2, .old, .1)'
             cand -I 'File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)'
             cand --dont-collect 'File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)'
             cand -o 'Output file to write results to (use w/ --json for JSON entries)'
@@ -100,8 +102,6 @@ set edit:completion:arg-completer[feroxbuster] = {|@words|
             cand --dont-filter 'Don''t auto-filter wildcard responses'
             cand -E 'Automatically discover extensions and add them to --extensions (unless they''re in --dont-collect)'
             cand --collect-extensions 'Automatically discover extensions and add them to --extensions (unless they''re in --dont-collect)'
-            cand -B 'Automatically request likely backup extensions for "found" urls'
-            cand --collect-backups 'Automatically request likely backup extensions for "found" urls'
             cand -g 'Automatically discover important words from within responses and add them to the wordlist'
             cand --collect-words 'Automatically discover important words from within responses and add them to the wordlist'
             cand -v 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v''s is probably too much)'

src/banner/container.rs

@@ -1,13 +1,15 @@
 use super::entry::BannerEntry;
 use crate::{
+    client,
     config::Configuration,
     event_handlers::Handles,
-    utils::{logged_request, parse_url_with_raw_path, status_colorizer},
+    utils::{make_request, parse_url_with_raw_path, status_colorizer},
     DEFAULT_IGNORED_EXTENSIONS, DEFAULT_METHOD, DEFAULT_STATUS_CODES, VERSION,
 };
 use anyhow::{bail, Result};
 use console::{style, Emoji};
 use serde_json::Value;
+use std::collections::HashMap;
 use std::{io::Write, sync::Arc};
 
 /// Url used to query github's api; specifically used to look for the latest tagged release name
@@ -498,7 +500,34 @@ by Ben "epi" Risher {}                 ver: {}"#,
 
         let api_url = parse_url_with_raw_path(url)?;
 
-        let result = logged_request(&api_url, DEFAULT_METHOD, None, handles.clone()).await?;
+        // we don't want to leak sensitive header info / include auth headers
+        // with the github api request, so we'll build a client specifically
+        // for this task. thanks to @stuhlmann for the suggestion!
+        let client = client::initialize(
+            handles.config.timeout,
+            "feroxbuster-update-check",
+            handles.config.redirects,
+            handles.config.insecure,
+            &HashMap::new(),
+            Some(&handles.config.proxy),
+            &handles.config.server_certs,
+            Some(&handles.config.client_cert),
+            Some(&handles.config.client_key),
+        )?;
+        let level = handles.config.output_level;
+        let tx_stats = handles.stats.tx.clone();
+
+        let result = make_request(
+            &client,
+            &api_url,
+            DEFAULT_METHOD,
+            None,
+            level,
+            &handles.config,
+            tx_stats,
+        )
+        .await?;
+
         let body = result.text().await?;
 
         let json_response: Value = serde_json::from_str(&body)?;

src/client.rs

@@ -67,17 +67,19 @@ where
     }
 
     if let (Some(cert_path), Some(key_path)) = (client_cert, client_key) {
-        let cert = std::fs::read(cert_path)?;
+        if !cert_path.is_empty() && !key_path.is_empty() {
-        let key = std::fs::read(key_path)?;
+            let cert = std::fs::read(cert_path)?;
+            let key = std::fs::read(key_path)?;
 
-        let identity = reqwest::Identity::from_pkcs8_pem(&cert, &key).with_context(|| {
+            let identity = reqwest::Identity::from_pkcs8_pem(&cert, &key).with_context(|| {
-            format!(
+                format!(
-                "either {} or {} are invalid; expecting PEM encoded certificate and key",
+                    "either {} or {} are invalid; expecting PEM encoded certificate and key",
-                cert_path, key_path
+                    cert_path, key_path
-            )
+                )
-        })?;
+            })?;
 
-        client = client.identity(identity);
+            client = client.identity(identity);
+        }
     }
 
     Ok(client.build()?)

src/config/container.rs

@@ -1,6 +1,6 @@
 use super::utils::{
-    depth, extract_links, ignored_extensions, methods, report_and_exit, save_state,
+    backup_extensions, depth, extract_links, ignored_extensions, methods, report_and_exit,
-    serialized_type, status_codes, threads, timeout, user_agent, wordlist, OutputLevel,
+    save_state, serialized_type, status_codes, threads, timeout, user_agent, wordlist, OutputLevel,
     RequesterPolicy,
 };
 use crate::config::determine_output_level;
@@ -21,7 +21,7 @@ use std::{
     collections::HashMap,
     env::{current_dir, current_exe},
     fs::read_to_string,
-    path::PathBuf,
+    path::{Path, PathBuf},
 };
 
 /// macro helper to abstract away repetitive configuration updates
@@ -318,6 +318,9 @@ pub struct Configuration {
     #[serde(default)]
     pub collect_backups: bool,
 
+    #[serde(default = "backup_extensions")]
+    pub backup_extensions: Vec<String>,
+
     /// Automatically discover important words from within responses and add them to the wordlist
     #[serde(default)]
     pub collect_words: bool,
@@ -418,6 +421,7 @@ impl Default for Configuration {
             threads: threads(),
             wordlist: wordlist(),
             dont_collect: ignored_extensions(),
+            backup_extensions: backup_extensions(),
         }
     }
 }
@@ -450,6 +454,7 @@ impl Configuration {
     /// - **extensions**: `None`
     /// - **collect_extensions**: `false`
     /// - **collect_backups**: `false`
+    /// - **backup_extensions**: [`DEFAULT_BACKUP_EXTENSIONS`](constant.DEFAULT_BACKUP_EXTENSIONS.html)
     /// - **collect_words**: `false`
     /// - **dont_collect**: [`DEFAULT_IGNORED_EXTENSIONS`](constant.DEFAULT_RESPONSE_CODES.html)
     /// - **methods**: [`DEFAULT_METHOD`](constant.DEFAULT_METHOD.html)
@@ -571,9 +576,7 @@ impl Configuration {
         //   - current directory
 
         // merge a config found at /etc/feroxbuster/ferox-config.toml
-        let config_file = PathBuf::new()
+        let config_file = Path::new("/etc/feroxbuster").join(DEFAULT_CONFIG_NAME);
-            .join("/etc/feroxbuster")
-            .join(DEFAULT_CONFIG_NAME);
         Self::parse_and_merge_config(config_file, config)?;
 
         // merge a config found at ~/.config/feroxbuster/ferox-config.toml
@@ -701,6 +704,11 @@ impl Configuration {
             } else {
                 config.data = arg.as_bytes().to_vec();
             }
+
+            if config.methods == methods() {
+                // if the user didn't specify a method, we're going to assume they meant to use POST
+                config.methods = vec![Method::POST.as_str().to_string()];
+            }
         }
 
         if came_from_cli!(args, "stdin") {
@@ -788,7 +796,12 @@ impl Configuration {
                 .collect();
         }
 
-        if came_from_cli!(args, "silent") {
+        if came_from_cli!(args, "quiet") {
+            config.quiet = true;
+            config.output_level = OutputLevel::Quiet;
+        }
+
+        if came_from_cli!(args, "silent") || (config.parallel > 0 && !config.quiet) {
             // the reason this is protected by an if statement:
             // consider a user specifying silent = true in ferox-config.toml
             // if the line below is outside of the if, we'd overwrite true with
@@ -801,11 +814,6 @@ impl Configuration {
             };
         }
 
-        if came_from_cli!(args, "quiet") {
-            config.quiet = true;
-            config.output_level = OutputLevel::Quiet;
-        }
-
         if came_from_cli!(args, "auto_tune")
             || came_from_cli!(args, "smart")
             || came_from_cli!(args, "thorough")
@@ -836,6 +844,20 @@ impl Configuration {
             || came_from_cli!(args, "thorough")
         {
             config.collect_backups = true;
+            config.backup_extensions = backup_extensions();
+
+            if came_from_cli!(args, "collect_backups") {
+                if let Some(arg) = args.get_many::<String>("collect_backups") {
+                    let backup_exts = arg
+                        .map(|ext| ext.trim().to_string())
+                        .collect::<Vec<String>>();
+
+                    if !backup_exts.is_empty() {
+                        // have at least one cli backup, override the defaults
+                        config.backup_extensions = backup_exts;
+                    }
+                }
+            }
         }
 
         if came_from_cli!(args, "collect_words")
@@ -918,7 +940,15 @@ impl Configuration {
                 // all other items in the iterator returned by split, when combined with the
                 // original split deliminator (:), make up the header's final value
                 let value = split_val.collect::<Vec<&str>>().join(":");
-                config.headers.insert(name.to_string(), value.to_string());
+
+                if value.starts_with(' ') && !value.starts_with("  ") {
+                    // first character is a space and the second character isn't
+                    // we can trim the leading space
+                    let trimmed = value.trim_start();
+                    config.headers.insert(name.to_string(), trimmed.to_string());
+                } else {
+                    config.headers.insert(name.to_string(), value.to_string());
+                }
             }
         }
 
@@ -1138,6 +1168,11 @@ impl Configuration {
 
         update_if_not_default!(&mut conf.timeout, new.timeout, timeout());
         update_if_not_default!(&mut conf.user_agent, new.user_agent, user_agent());
+        update_if_not_default!(
+            &mut conf.backup_extensions,
+            new.backup_extensions,
+            backup_extensions()
+        );
         update_if_not_default!(&mut conf.random_agent, new.random_agent, false);
         update_if_not_default!(&mut conf.threads, new.threads, threads());
         update_if_not_default!(&mut conf.depth, new.depth, depth());

src/config/tests.rs

@@ -59,6 +59,7 @@ fn setup_config_test() -> Configuration {
             server_certs = ["/some/cert.pem", "/some/other/cert.pem"]
             client_cert = "/some/client/cert.pem"
             client_key = "/some/client/key.pem"
+            backup_extensions = [".save"]
         "#;
     let tmp_dir = TempDir::new().unwrap();
     let file = tmp_dir.path().join(DEFAULT_CONFIG_NAME);
@@ -123,6 +124,7 @@ fn default_configuration() {
     assert_eq!(config.server_certs, Vec::<String>::new());
     assert_eq!(config.client_cert, String::new());
     assert_eq!(config.client_key, String::new());
+    assert_eq!(config.backup_extensions, backup_extensions());
 }
 
 #[test]
@@ -459,6 +461,13 @@ fn config_reads_server_certs() {
     );
 }
 
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_backup_extensions() {
+    let config = setup_config_test();
+    assert_eq!(config.backup_extensions, [".save"]);
+}
+
 #[test]
 /// parse the test config and see that the value parsed is correct
 fn config_reads_client_cert() {

src/config/utils.rs

@@ -1,6 +1,7 @@
 use crate::{
     utils::{module_colorizer, status_colorizer},
-    DEFAULT_IGNORED_EXTENSIONS, DEFAULT_METHOD, DEFAULT_STATUS_CODES, DEFAULT_WORDLIST, VERSION,
+    DEFAULT_BACKUP_EXTENSIONS, DEFAULT_IGNORED_EXTENSIONS, DEFAULT_METHOD, DEFAULT_STATUS_CODES,
+    DEFAULT_WORDLIST, VERSION,
 };
 #[cfg(not(test))]
 use std::process::exit;
@@ -69,6 +70,14 @@ pub(super) fn ignored_extensions() -> Vec<String> {
         .collect()
 }
 
+/// default backup extensions to collect
+pub(super) fn backup_extensions() -> Vec<String> {
+    DEFAULT_BACKUP_EXTENSIONS
+        .iter()
+        .map(|s| s.to_string())
+        .collect()
+}
+
 /// default wordlist
 pub(super) fn wordlist() -> String {
     String::from(DEFAULT_WORDLIST)

src/event_handlers/command.rs

@@ -85,4 +85,7 @@ pub enum Command {
     /// Give a handler access to an Arc<Handles> instance after the handler has
     /// already been initialized
     AddHandles(Arc<Handles>),
+
+    /// inform the Stats object about which targets are being scanned
+    UpdateTargets(Vec<String>),
 }

src/event_handlers/outputs.rs

@@ -98,6 +98,8 @@ impl FileOutHandler {
 
         log::info!("Writing scan results to {}", self.config.output);
 
+        write_to(&*self.config, &mut file, self.config.json)?;
+
         while let Some(command) = self.receiver.recv().await {
             match command {
                 Command::Report(response) => {
@@ -209,8 +211,12 @@ impl TermOutHandler {
         while let Some(command) = self.receiver.recv().await {
             match command {
                 Command::Report(resp) => {
-                    self.process_response(tx_stats.clone(), resp, ProcessResponseCall::Recursive)
+                    if let Err(err) = self
-                        .await?;
+                        .process_response(tx_stats.clone(), resp, ProcessResponseCall::Recursive)
+                        .await
+                    {
+                        log::warn!("{}", err);
+                    }
                 }
                 Command::Sync(sender) => {
                     sender.send(true).unwrap_or_default();
@@ -400,7 +406,7 @@ impl TermOutHandler {
 
         if !filename.is_empty() {
             // append rules
-            for suffix in ["~", ".bak", ".bak2", ".old", ".1"] {
+            for suffix in &self.config.backup_extensions {
                 self.add_new_url_to_vec(url, &format!("{filename}{suffix}"), &mut urls);
             }
 

src/event_handlers/statistics.rs

@@ -125,6 +125,9 @@ impl StatsHandler {
                 Command::Sync(sender) => {
                     sender.send(true).unwrap_or_default();
                 }
+                Command::UpdateTargets(targets) => {
+                    self.stats.update_targets(targets);
+                }
                 Command::Exit => break,
                 _ => {} // no more commands needed
             }
@@ -132,7 +135,7 @@ impl StatsHandler {
 
         self.bar.finish();
 
-        log::debug!("{:#?}", *self.stats);
+        log::info!("{:#?}", *self.stats);
         log::trace!("exit: start");
         Ok(())
     }

src/filters/tests.rs

@@ -271,7 +271,7 @@ fn remove_function_works_as_expected() {
 
     assert_eq!(data.filters.read().unwrap().len(), 5);
 
-    let expected = vec![
+    let expected = [
         WordsFilter { word_count: 1 },
         WordsFilter { word_count: 3 },
         WordsFilter { word_count: 5 },

src/lib.rs

@@ -59,6 +59,9 @@ pub(crate) const DEFAULT_IGNORED_EXTENSIONS: [&str; 38] = [
     "webm", "ogv", "oga", "flac", "aac", "3gp", "css", "zip", "xls", "xml", "gz", "tgz",
 ];
 
+/// Default set of extensions to search for when auto-collecting backups during scans
+pub(crate) const DEFAULT_BACKUP_EXTENSIONS: [&str; 5] = ["~", ".bak", ".bak2", ".old", ".1"];
+
 /// Default wordlist to use when `-w|--wordlist` isn't specified and not `wordlist` isn't set
 /// in a [ferox-config.toml](constant.DEFAULT_CONFIG_NAME.html) config file.
 ///

src/main.rs

@@ -8,7 +8,7 @@ use std::{
     io::{stderr, BufRead, BufReader},
     ops::Index,
     path::Path,
-    process::{exit, Command},
+    process::{exit, Command, Stdio},
     sync::{atomic::Ordering, Arc},
 };
 
@@ -25,7 +25,8 @@ use feroxbuster::{
     config::{Configuration, OutputLevel},
     event_handlers::{
         Command::{
-            AddHandles, CreateBar, Exit, JoinTasks, LoadStats, ScanInitialUrls, UpdateWordlist,
+            AddHandles, CreateBar, Exit, JoinTasks, LoadStats, ScanInitialUrls, UpdateTargets,
+            UpdateWordlist,
         },
         FiltersHandler, Handles, ScanHandler, StatsHandler, Tasks, TermInputHandler,
         TermOutHandler, SCAN_COMPLETE,
@@ -325,9 +326,15 @@ async fn wrapped_main(config: Arc<Configuration>) -> Result<()> {
     // create new Tasks object, each of these handles is one that will be joined on later
     let tasks = Tasks::new(out_task, stats_task, filters_task, scan_task);
 
-    if !config.time_limit.is_empty() {
+    if !config.time_limit.is_empty() && config.parallel == 0 {
         // --time-limit value not an empty string, need to kick off the thread that enforces
         // the limit
+        //
+        // if --parallel is used, this branch won't execute in the main process, but will in the
+        // children. This is because --parallel is stripped from the children's command line
+        // arguments, so, when spawned, they won't have --parallel, the parallel value will be set
+        // to the default of 0, and will hit this branch. This makes it so that the time limit
+        // is enforced on each individual child process, instead of the main process
         let time_handles = handles.clone();
         tokio::spawn(async move { scan_manager::start_max_time_thread(time_handles).await });
     }
@@ -370,8 +377,7 @@ async fn wrapped_main(config: Arc<Configuration>) -> Result<()> {
 
         let invocation = args();
 
-        let para_regex =
+        let para_regex = Regex::new("--stdin").unwrap();
-            Regex::new("--stdin|-q|--quiet|--silent|--verbosity|-v|-vv|-vvv|-vvvv").unwrap();
 
         // remove stdin since only the original process will process targets
         // remove quiet and silent so we can force silent later to normalize output
@@ -379,8 +385,6 @@ async fn wrapped_main(config: Arc<Configuration>) -> Result<()> {
             .filter(|s| !para_regex.is_match(s))
             .collect::<Vec<String>>();
 
-        original.push("--silent".to_string()); // only output modifier allowed
-
         // we need remove --parallel from command line so we don't hit this branch over and over
         // but we must remove --parallel N manually; the filter above never sees --parallel and the
         // value passed to it at the same time, so can't filter them out in one pass
@@ -455,16 +459,32 @@ async fn wrapped_main(config: Arc<Configuration>) -> Result<()> {
 
             log::debug!("parallel exec: {} {}", bin, args.join(" "));
 
-            tokio::task::spawn_blocking(move || {
+            tokio::task::spawn(async move {
-                let result = Command::new(bin)
+                let mut output = Command::new(bin)
                     .args(&args)
+                    .stdout(Stdio::piped())
                     .spawn()
-                    .expect("failed to spawn a child process")
+                    .expect("failed to spawn a child process");
-                    .wait()
-                    .expect("child process errored during execution");
 
+                let stdout = output.stdout.take().unwrap();
+
+                let mut bufread = BufReader::new(stdout);
+                // output for a single line is a minimum of 51 bytes, so we'll start with that
+                // + a little wiggle room, and grow as needed
+                let mut buf: String = String::with_capacity(128);
+
+                while let Ok(n) = bufread.read_line(&mut buf) {
+                    if n > 0 {
+                        let trimmed = buf.trim();
+                        if !trimmed.is_empty() {
+                            println!("{}", trimmed);
+                        }
+                        buf.clear();
+                    } else {
+                        break;
+                    }
+                }
                 drop(permit);
-                result
             });
         }
 
@@ -488,6 +508,14 @@ async fn wrapped_main(config: Arc<Configuration>) -> Result<()> {
         return Ok(());
     }
 
+    // in order for the Stats object to know about which targets are being scanned, we need to
+    // wait until the parallel branch has been handled before sending the UpdateTargets command
+    // this ensures that only the targets being scanned are sent to the Stats object
+    //
+    // if sent before the parallel branch is handled, the Stats object will have duplicate
+    // targets
+    handles.stats.send(UpdateTargets(targets.clone()))?;
+
     if matches!(config.output_level, OutputLevel::Default) {
         // only print banner if output level is default (no banner on --quiet|--silent)
         let std_stderr = stderr(); // std::io::stderr
@@ -646,7 +674,7 @@ fn main() -> Result<()> {
                 // print the banner to stderr
                 let std_stderr = stderr(); // std::io::stderr
                 let banner = Banner::new(&targets, &config);
-                if !config.quiet && !config.silent {
+                if (!config.quiet && !config.silent) || config.parallel != 0 {
                     banner.print_to(std_stderr, config).unwrap();
                 }
             }

src/nlp/document.rs

@@ -45,9 +45,7 @@ impl Document {
 
         let html = Html::parse_document(raw_html);
 
-        let Some(element) = html.select(&selector).next() else {
+        let element = html.select(&selector).next()?;
-            return None;
-        };
 
         let text = element
             .descendants()

src/parser.rs

@@ -211,7 +211,6 @@ pub fn initialize() -> Command {
                 .num_args(1..)
                 .action(ArgAction::Append)
                 .help_heading("Request settings")
-                .use_value_delimiter(true)
                 .help(
                     "Specify HTTP headers to be used in each request (ex: -H Header:val -H 'stuff: things')",
                 ),
@@ -486,6 +485,7 @@ pub fn initialize() -> Command {
             Arg::new("parallel")
                 .long("parallel")
                 .value_name("PARALLEL_SCANS")
+                .conflicts_with("verbosity")
                 .num_args(1)
                 .requires("stdin")
                 .help_heading("Scan settings")
@@ -550,9 +550,9 @@ pub fn initialize() -> Command {
             Arg::new("collect_backups")
                 .short('B')
                 .long("collect-backups")
-                .num_args(0)
+                .num_args(0..)
                 .help_heading("Dynamic collection settings")
-                .help("Automatically request likely backup extensions for \"found\" urls")
+                .help("Automatically request likely backup extensions for \"found\" urls (default: ~, .bak, .bak2, .old, .1)")
         )
         .arg(
             Arg::new("collect_words")
@@ -648,6 +648,11 @@ pub fn initialize() -> Command {
                 .args(["debug_log", "output", "silent"])
                 .multiple(true),
         )
+        .group(
+            ArgGroup::new("output_limiters")
+                .args(["quiet", "silent"])
+                .multiple(false),
+        )
         .arg(
             Arg::new("update_app")
                 .short('U')

src/response.rs

@@ -21,7 +21,7 @@ use crate::{
     event_handlers::{Command, Handles},
     traits::FeroxSerialize,
     url::FeroxUrl,
-    utils::{self, fmt_err, parse_url_with_raw_path, status_colorizer},
+    utils::{self, fmt_err, parse_url_with_raw_path, status_colorizer, timestamp},
     CommandSender,
 };
 
@@ -63,6 +63,9 @@ pub struct FeroxResponse {
 
     /// Url's file extension, if one exists
     pub(crate) extension: Option<String>,
+
+    /// Timestamp of when this response was received
+    timestamp: f64,
 }
 
 /// implement Default trait for FeroxResponse
@@ -82,6 +85,7 @@ impl Default for FeroxResponse {
             wildcard: false,
             output_level: Default::default(),
             extension: None,
+            timestamp: timestamp(),
         }
     }
 }
@@ -138,6 +142,11 @@ impl FeroxResponse {
         self.content_length
     }
 
+    /// Get the timestamp of this response
+    pub fn timestamp(&self) -> f64 {
+        self.timestamp
+    }
+
     /// Set `FeroxResponse`'s `url` attribute, has no affect if an error occurs
     pub fn set_url(&mut self, url: &str) {
         match parse_url_with_raw_path(url) {
@@ -216,6 +225,7 @@ impl FeroxResponse {
         let status = response.status();
         let headers = response.headers().clone();
         let content_length = response.content_length().unwrap_or(0);
+        let timestamp = timestamp();
 
         // .text() consumes the response, must be called last
         let text = response
@@ -248,6 +258,7 @@ impl FeroxResponse {
             output_level,
             wildcard: false,
             extension: None,
+            timestamp,
         }
     }
 
@@ -423,8 +434,12 @@ impl FeroxSerialize for FeroxResponse {
         let mut url_with_redirect = match (
             self.status().is_redirection(),
             self.headers().get("Location").is_some(),
+            matches!(
+                self.output_level,
+                OutputLevel::Silent | OutputLevel::SilentJSON
+            ),
         ) {
-            (true, true) => {
+            (true, true, false) => {
                 // redirect with Location header, show where it goes if possible
                 let loc = self
                     .headers()
@@ -570,6 +585,7 @@ impl Serialize for FeroxResponse {
             "extension",
             self.extension.as_ref().unwrap_or(&String::new()),
         )?;
+        state.serialize_field("timestamp", &self.timestamp)?;
 
         state.end()
     }
@@ -595,6 +611,7 @@ impl<'de> Deserialize<'de> for FeroxResponse {
             line_count: 0,
             word_count: 0,
             extension: None,
+            timestamp: timestamp(),
         };
 
         let map: HashMap<String, Value> = HashMap::deserialize(deserializer)?;
@@ -668,6 +685,11 @@ impl<'de> Deserialize<'de> for FeroxResponse {
                         response.extension = Some(result.to_string());
                     }
                 }
+                "timestamp" => {
+                    if let Some(result) = value.as_f64() {
+                        response.timestamp = result;
+                    }
+                }
                 _ => {}
             }
         }

src/scan_manager/tests.rs

@@ -314,7 +314,7 @@ fn ferox_scans_serialize() {
 #[test]
 /// given a FeroxResponses, test that it serializes into the proper JSON entry
 fn ferox_responses_serialize() {
-    let json_response = r#"{"type":"response","url":"https://nerdcore.com/css","original_url":"https://nerdcore.com","path":"/css","wildcard":true,"status":301,"method":"GET","content_length":173,"line_count":10,"word_count":16,"headers":{"server":"nginx/1.16.1"},"extension":""}"#;
+    let json_response = r#"{"type":"response","url":"https://nerdcore.com/css","original_url":"https://nerdcore.com","path":"/css","wildcard":true,"status":301,"method":"GET","content_length":173,"line_count":10,"word_count":16,"headers":{"server":"nginx/1.16.1"},"extension":"","timestamp":1711796681.3455093}"#;
     let response: FeroxResponse = serde_json::from_str(json_response).unwrap();
 
     let responses = FeroxResponses::default();
@@ -332,7 +332,7 @@ fn ferox_responses_serialize() {
 /// given a FeroxResponse, test that it serializes into the proper JSON entry
 fn ferox_response_serialize_and_deserialize() {
     // deserialize
-    let json_response = r#"{"type":"response","url":"https://nerdcore.com/css","original_url":"https://nerdcore.com","path":"/css","wildcard":true,"status":301,"method":"GET","content_length":173,"line_count":10,"word_count":16,"headers":{"server":"nginx/1.16.1"},"extension":""}"#;
+    let json_response = r#"{"type":"response","url":"https://nerdcore.com/css","original_url":"https://nerdcore.com","path":"/css","wildcard":true,"status":301,"method":"GET","content_length":173,"line_count":10,"word_count":16,"headers":{"server":"nginx/1.16.1"},"extension":"","timestamp":1711796681.3455093}"#;
     let response: FeroxResponse = serde_json::from_str(json_response).unwrap();
 
     assert_eq!(response.url().as_str(), "https://nerdcore.com/css");
@@ -343,6 +343,7 @@ fn ferox_response_serialize_and_deserialize() {
     assert_eq!(response.line_count(), 10);
     assert_eq!(response.word_count(), 16);
     assert_eq!(response.headers().get("server").unwrap(), "nginx/1.16.1");
+    assert_eq!(response.timestamp(), 1711796681.3455093);
 
     // serialize, however, this can fail when headers are out of order
     let new_json = serde_json::to_string(&response).unwrap();

src/scanner/ferox_scanner.rs

@@ -214,9 +214,9 @@ impl FeroxScanner {
                 .url(&self.target_url)
                 .handles(self.handles.clone())
                 .build()?;
-
+            if let Ok(result) = extractor.extract().await {
-            let result = extractor.extract().await?;
+                extraction_tasks.push(extractor.request_links(result).await?)
-            extraction_tasks.push(extractor.request_links(result).await?)
+            }
         }
 
         let scanned_urls = self.handles.ferox_scans()?;

src/statistics/container.rs

@@ -132,6 +132,9 @@ pub struct Stats {
 
     /// tracker for whether to use json during serialization or not
     json: bool,
+
+    /// tracker for the initial targets that were passed in to the scan
+    targets: Mutex<Vec<String>>,
 }
 
 /// FeroxSerialize implementation for Stats
@@ -196,6 +199,7 @@ impl Serialize for Stats {
         state.serialize_field("request_errors", &atomic_load!(self.request_errors))?;
         state.serialize_field("directory_scan_times", &self.directory_scan_times)?;
         state.serialize_field("total_runtime", &self.total_runtime)?;
+        state.serialize_field("targets", &self.targets)?;
 
         state.end()
     }
@@ -446,6 +450,17 @@ impl<'a> Deserialize<'a> for Stats {
                         }
                     }
                 }
+                "targets" => {
+                    if let Some(arr) = value.as_array() {
+                        for val in arr {
+                            if let Some(parsed) = val.as_str() {
+                                if let Ok(mut guard) = stats.targets.lock() {
+                                    guard.push(parsed.to_string())
+                                }
+                            }
+                        }
+                    }
+                }
                 _ => {}
             }
         }
@@ -514,6 +529,13 @@ impl Stats {
         }
     }
 
+    /// update targets with the given vector of strings
+    pub fn update_targets(&self, targets: Vec<String>) {
+        if let Ok(mut locked_targets) = self.targets.lock() {
+            *locked_targets = targets;
+        }
+    }
+
     /// save an instance of `Stats` to disk after updating the total runtime for the scan
     pub fn save(&self, seconds: f64, location: &str) -> Result<()> {
         let mut file = open_file(location)?;

src/url.rs

@@ -270,7 +270,7 @@ mod tests {
         let pdf = Url::parse("http://localhost/turbo.pdf").unwrap();
         let tar = Url::parse("http://localhost/turbo.tar.gz").unwrap();
 
-        let expected = vec![
+        let expected = [
             vec![base.clone(), js.clone()],
             vec![base.clone(), js.clone(), php.clone()],
             vec![base.clone(), js.clone(), php.clone(), pdf.clone()],

src/utils.rs

@@ -68,6 +68,20 @@ pub fn fmt_err(msg: &str) -> String {
     format!("{}: {}", status_colorizer("ERROR"), msg)
 }
 
+/// simple wrapper to get the current system time as
+/// time elapsed from unix epoch
+pub fn timestamp() -> f64 {
+    let since_the_epoch = SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .unwrap_or_else(|_| Duration::from_secs(0));
+
+    let secs = since_the_epoch.as_secs() as f64;
+    let nanos = since_the_epoch.subsec_nanos() as f64;
+
+    // Convert nanoseconds to fractional seconds and add to secs
+    secs + (nanos / 1_000_000_000.0)
+}
+
 /// given a FeroxResponse, send a TryRecursion command
 ///
 /// moved to utils to allow for calls from extractor and scanner
@@ -563,6 +577,13 @@ pub fn parse_url_with_raw_path(url: &str) -> Result<Url> {
         bail!("url to parse has no authority and is therefore invalid");
     }
 
+    // thanks to @devx00: the possibility exists for Url to return true for
+    // has_authority, but not have a host/port, so we'll check for that
+    // and bail if it's the case
+    if parsed.host().is_none() {
+        bail!("url to parse doesn't have a host");
+    }
+
     // we have a valid url, the next step is to check the path and see if it's
     // something that url::Url::parse would silently transform
     //
@@ -729,6 +750,18 @@ mod tests {
     use crate::config::Configuration;
     use crate::scan_manager::{FeroxScans, ScanOrder};
 
+    #[test]
+    /// parse_url_with_raw_path with javascript:// should not throw an unimplemented! error
+    fn utils_parse_url_with_raw_path_javascript() {
+        let url = "javascript://";
+        let parsed = parse_url_with_raw_path(url);
+        assert!(parsed.is_err());
+        assert!(parsed
+            .unwrap_err()
+            .to_string()
+            .contains("url to parse doesn't have a host"));
+    }
+
     #[test]
     /// multiple tests for parse_url_with_raw_path
     fn utils_parse_url_with_raw_path() {

tests/test_banner.rs

@@ -1146,6 +1146,7 @@ fn banner_prints_parallel() {
     Command::cargo_bin("feroxbuster")
         .unwrap()
         .arg("--stdin")
+        .arg("--quiet")
         .arg("--parallel")
         .arg("4316")
         .arg("--wordlist")

tests/test_main.rs

@@ -108,10 +108,11 @@ fn main_parallel_spawns_children() -> Result<(), Box<dyn std::error::Error>> {
 
     Command::cargo_bin("feroxbuster")
         .unwrap()
+        .env("RUST_LOG", "trace")
         .arg("--stdin")
         .arg("--parallel")
         .arg("2")
-        .arg("-vvvv")
+        .arg("--quiet")
         .arg("--debug-log")
         .arg(outfile.as_os_str())
         .arg("--wordlist")
@@ -172,6 +173,7 @@ fn main_parallel_creates_output_directory() -> Result<(), Box<dyn std::error::Er
     Command::cargo_bin("feroxbuster")
         .unwrap()
         .arg("--stdin")
+        .arg("--quiet")
         .arg("--parallel")
         .arg("2")
         .arg("--output")

tests/test_policies.rs

@@ -22,6 +22,7 @@ use utils::{setup_tmp_directory, teardown_tmp_directory};
 // these words will be used along with pattern matching to trigger different policies
 
 #[test]
+#[ignore]
 /// --auto-bail should cancel a scan with spurious errors  
 fn auto_bail_cancels_scan_with_timeouts() {
     let srv = MockServer::start();
@@ -37,7 +38,7 @@ fn auto_bail_cancels_scan_with_timeouts() {
     let error_mock = srv.mock(|when, then| {
         when.method(GET)
             .path_matches(Regex::new("/[a-zA-Z]{6}error[a-zA-Z]{6}").unwrap());
-        then.delay(Duration::new(3, 0))
+        then.delay(Duration::new(2, 5000))
             .status(200)
             .body("verboten, nerd");
     });
@@ -57,7 +58,7 @@ fn auto_bail_cancels_scan_with_timeouts() {
     Command::cargo_bin("feroxbuster")
         .unwrap()
         .arg("--url")
-        .arg(srv.url("/"))
+        .arg(&srv.url("/"))
         .arg("--wordlist")
         .arg(file.as_os_str())
         .arg("--auto-bail")
@@ -65,10 +66,10 @@ fn auto_bail_cancels_scan_with_timeouts() {
         .arg("--timeout")
         .arg("2")
         .arg("--threads")
-        .arg("4")
+        .arg("8")
         .arg("--debug-log")
         .arg(logfile.as_os_str())
-        .arg("-vvvv")
+        .arg("-vv")
         .arg("--json")
         .assert()
         .success();
@@ -146,7 +147,7 @@ fn auto_bail_cancels_scan_with_403s() {
         .arg("4")
         .arg("--debug-log")
         .arg(logfile.as_os_str())
-        .arg("-vvvv")
+        .arg("-vv")
         .arg("--json")
         .assert()
         .success();
@@ -228,7 +229,7 @@ fn auto_bail_cancels_scan_with_429s() {
         .arg("4")
         .arg("--debug-log")
         .arg(logfile.as_os_str())
-        .arg("-vvvv")
+        .arg("-vvv")
         .arg("--json")
         .assert()
         .success();