diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
index bfe6329..47b50cf 100644
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -11,7 +11,7 @@ Long form explanations of most of the items below can be found in the [CONTRIBUT
## Static analysis checks
- [ ] All rust files are formatted using `cargo fmt`
-- [ ] All `clippy` checks pass when running `cargo clippy --all-targets --all-features -- -D warnings -A clippy::deref_addrof`
+- [ ] All `clippy` checks pass when running `cargo clippy --all-targets --all-features -- -D warnings -A clippy::deref_addrof -A clippy::mutex-atomic`
- [ ] All existing tests pass
## Documentation
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 8d1bbee..bad1d11 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -5,7 +5,7 @@ on: [push]
jobs:
build-nix:
runs-on: ${{ matrix.os }}
- if: github.ref == 'refs/heads/master'
+ if: github.ref == 'refs/heads/main'
strategy:
matrix:
type: [ubuntu-x64, ubuntu-x86]
@@ -73,7 +73,7 @@ jobs:
build-macos:
runs-on: macos-latest
- if: github.ref == 'refs/heads/master'
+ if: github.ref == 'refs/heads/main'
steps:
- uses: actions/checkout@v2
- uses: actions-rs/toolchain@v1
@@ -103,7 +103,7 @@ jobs:
build-windows:
runs-on: ${{ matrix.os }}
- if: github.ref == 'refs/heads/master'
+ if: github.ref == 'refs/heads/main'
strategy:
matrix:
type: [windows-x64, windows-x86]
diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml
index 2b9f46e..5cd64bd 100644
--- a/.github/workflows/check.yml
+++ b/.github/workflows/check.yml
@@ -61,4 +61,4 @@ jobs:
- uses: actions-rs/cargo@v1
with:
command: clippy
- args: --all-targets --all-features -- -D warnings -A clippy::deref_addrof
+ args: --all-targets --all-features -- -D warnings -A clippy::deref_addrof -A clippy::mutex-atomic
diff --git a/.gitignore b/.gitignore
index 538178a..a6b9e1e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -24,3 +24,6 @@ lcov_cobertura.py
# state file created during tests
ferox-http*
+
+# python stuff cuz reasons
+Pipfile*
diff --git a/Cargo.lock b/Cargo.lock
index 196335f..eefc507 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -160,17 +160,17 @@ dependencies = [
[[package]]
name = "async-process"
-version = "1.0.1"
+version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c8cea09c1fb10a317d1b5af8024eeba256d6554763e85ecd90ff8df31c7bbda"
+checksum = "ef37b86e2fa961bae5a4d212708ea0154f904ce31d1a4a7f47e1bbc33a0c040b"
dependencies = [
"async-io",
"blocking",
- "cfg-if 0.1.10",
+ "cfg-if 1.0.0",
"event-listener",
"futures-lite",
"once_cell",
- "signal-hook",
+ "signal-hook 0.3.4",
"winapi",
]
@@ -307,9 +307,9 @@ dependencies = [
[[package]]
name = "bstr"
-version = "0.2.14"
+version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "473fc6b38233f9af7baa94fb5852dca389e3d95b8e21c8e3719301462c5d9faf"
+checksum = "a40b47ad93e1a5404e6c18dec46b628214fee441c70f4ab5d6942142cc268a3d"
dependencies = [
"lazy_static",
"memchr",
@@ -318,9 +318,9 @@ dependencies = [
[[package]]
name = "bumpalo"
-version = "3.5.0"
+version = "3.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f07aa6688c702439a1be0307b6a94dffe1168569e45b9500c1372bc580740d59"
+checksum = "099e596ef14349721d9016f6b80dd3419ea1bf289ab9b44df8e4dfd3a005d5d9"
[[package]]
name = "byteorder"
@@ -442,7 +442,7 @@ dependencies = [
"libc",
"mio",
"parking_lot",
- "signal-hook",
+ "signal-hook 0.1.17",
"winapi",
]
@@ -463,9 +463,9 @@ checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7"
[[package]]
name = "ctor"
-version = "0.1.18"
+version = "0.1.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "10bcb9d7dcbf7002aaffbb53eac22906b64cdcc127971dcc387d8eb7c95d5560"
+checksum = "e8f45d9ad417bcef4817d614a501ab55cdd96a6fdb24f49aab89a54acfd66b19"
dependencies = [
"quote",
"syn",
@@ -498,9 +498,9 @@ dependencies = [
[[package]]
name = "curl-sys"
-version = "0.4.39+curl-7.74.0"
+version = "0.4.40+curl-7.75.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07a8ce861e7b68a0b394e814d7ee9f1b2750ff8bd10372c6ad3bacc10e86f874"
+checksum = "2ffafc1c35958318bd7fdd0582995ce4c72f4f461a8e70499ccee83a619fd562"
dependencies = [
"cc",
"libc",
@@ -584,18 +584,18 @@ checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
[[package]]
name = "encoding_rs"
-version = "0.8.26"
+version = "0.8.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "801bbab217d7f79c0062f4f7205b5d4427c6d1a7bd7aafdd1475f7c59d62b283"
+checksum = "80df024fbc5ac80f87dfef0d9f5209a252f2a497f7f42944cff24d8253cac065"
dependencies = [
"cfg-if 1.0.0",
]
[[package]]
name = "env_logger"
-version = "0.8.2"
+version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f26ecb66b4bdca6c1409b40fb255eefc2bd4f6d135dab3c3124f80ffa2a9661e"
+checksum = "17392a012ea30ef05a610aa97dfb49496e71c9f676b27879922ea5bdf60d9d3f"
dependencies = [
"atty",
"humantime",
@@ -681,13 +681,13 @@ dependencies = [
[[package]]
name = "flume"
-version = "0.10.1"
+version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0362ef9c4c1fa854ff95b4cb78045a86e810d804dc04937961988b45427104a9"
+checksum = "531a685ab99b8f60a271b44d5dd1a76e55124a8c9fa0407b7a8e9cd172d5b588"
dependencies = [
"futures-core",
"futures-sink",
- "pin-project 1.0.4",
+ "pin-project",
"spinning_top",
]
@@ -931,9 +931,9 @@ dependencies = [
[[package]]
name = "httparse"
-version = "1.3.4"
+version = "1.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9"
+checksum = "615caabe2c3160b313d52ccc905335f4ed5f10881dd63dc5699d47e90be85691"
[[package]]
name = "httpdate"
@@ -976,9 +976,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
[[package]]
name = "hyper"
-version = "0.14.2"
+version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12219dc884514cb4a6a03737f4413c0e01c23a1b059b0156004b23f1e19dccbe"
+checksum = "e8e946c2b1349055e0b72ae281b238baf1a3ea7307c7e9f9d64673bdd9c26ac7"
dependencies = [
"bytes",
"futures-channel",
@@ -990,7 +990,7 @@ dependencies = [
"httparse",
"httpdate",
"itoa",
- "pin-project 1.0.4",
+ "pin-project",
"socket2",
"tokio",
"tower-service",
@@ -1013,9 +1013,9 @@ dependencies = [
[[package]]
name = "idna"
-version = "0.2.0"
+version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "02e2673c30ee86b5b96a9cb52ad15718aa1f966f5ab9ad54a8b95d5ca33120a9"
+checksum = "de910d521f7cc3135c4de8db1cb910e0b5ed1dc6f57c381cd07e8e661ce10094"
dependencies = [
"matches",
"unicode-bidi",
@@ -1061,9 +1061,9 @@ checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135"
[[package]]
name = "isahc"
-version = "1.0.3"
+version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff5419136b615bb64a2d0f8ccc91ed2e74c3bcf77e71c1820dbd6663898d1b34"
+checksum = "af3d0a62435883f745c825ec06a03a38d24bf5fa65c43e2c083b6a60ce0058ae"
dependencies = [
"crossbeam-utils",
"curl",
@@ -1176,15 +1176,15 @@ checksum = "66189c12161c65c0023ceb53e2fccc0013311bcb36a7cbd0f9c5e938b408ac96"
[[package]]
name = "libc"
-version = "0.2.84"
+version = "0.2.86"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1cca32fa0182e8c0989459524dc356b8f2b5c10f1b9eb521b7d182c03cf8c5ff"
+checksum = "b7282d924be3275cec7f6756ff4121987bc6481325397dde6ba3e7802b1a8b1c"
[[package]]
name = "libnghttp2-sys"
-version = "0.1.5+1.42.0"
+version = "0.1.6+1.43.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9657455ff47889b70ffd37c3e118e8cdd23fd1f9f3293a285f141070621c4c79"
+checksum = "0af55541a8827e138d59ec9e5877fb6095ece63fb6f4da45e7491b4fbd262855"
dependencies = [
"cc",
"libc",
@@ -1282,12 +1282,12 @@ dependencies = [
[[package]]
name = "nb-connect"
-version = "1.0.2"
+version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8123a81538e457d44b933a02faf885d3fe8408806b23fa700e8f01c6c3a98998"
+checksum = "670361df1bc2399ee1ff50406a0d422587dd3bb0da596e1978fe8e05dabddf4f"
dependencies = [
"libc",
- "winapi",
+ "socket2",
]
[[package]]
@@ -1416,14 +1416,14 @@ dependencies = [
[[package]]
name = "parking_lot_core"
-version = "0.8.2"
+version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ccb628cad4f84851442432c60ad8e1f607e29752d0bf072cbd0baf28aa34272"
+checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018"
dependencies = [
"cfg-if 1.0.0",
"instant",
"libc",
- "redox_syscall 0.1.57",
+ "redox_syscall 0.2.5",
"smallvec",
"winapi",
]
@@ -1461,38 +1461,18 @@ checksum = "28b9b4df73455c861d7cbf8be42f01d3b373ed7f02e378d55fa84eafc6f638b1"
[[package]]
name = "pin-project"
-version = "0.4.27"
+version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2ffbc8e94b38ea3d2d8ba92aea2983b503cd75d0888d75b86bb37970b5698e15"
+checksum = "96fa8ebb90271c4477f144354485b8068bd8f6b78b428b01ba892ca26caf0b63"
dependencies = [
- "pin-project-internal 0.4.27",
-]
-
-[[package]]
-name = "pin-project"
-version = "1.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "95b70b68509f17aa2857863b6fa00bf21fc93674c7a8893de2f469f6aa7ca2f2"
-dependencies = [
- "pin-project-internal 1.0.4",
+ "pin-project-internal",
]
[[package]]
name = "pin-project-internal"
-version = "0.4.27"
+version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "65ad2ae56b6abe3a1ee25f15ee605bacadb9a764edaba9c2bf4103800d4a1895"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
-name = "pin-project-internal"
-version = "1.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "caa25a6393f22ce819b0f50e0be89287292fda8d425be38ee0ca14c4931d9e71"
+checksum = "758669ae3558c6f74bd2a18b41f7ac0b5a195aea6639d6a9b5e5d1ad5ba24c0b"
dependencies = [
"proc-macro2",
"quote",
@@ -1557,15 +1537,15 @@ dependencies = [
[[package]]
name = "predicates-core"
-version = "1.0.1"
+version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fb3dbeaaf793584e29c58c7e3a82bbb3c7c06b63cea68d13b0e3cddc124104dc"
+checksum = "57e35a3326b75e49aa85f5dc6ec15b41108cf5aee58eabb1f274dd18b73c2451"
[[package]]
name = "predicates-tree"
-version = "1.0.1"
+version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aee95d988ee893cb35c06b148c80ed2cd52c8eea927f50ba7a0be1a786aeab73"
+checksum = "15f553275e5721409451eb85e15fd9a860a6e5ab4496eb215987502b5f5391f2"
dependencies = [
"predicates-core",
"treeline",
@@ -1627,9 +1607,9 @@ dependencies = [
[[package]]
name = "quote"
-version = "1.0.8"
+version = "1.0.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "991431c3519a3f36861882da93630ce66b52918dcf1b8e2fd66b397fc96f28df"
+checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7"
dependencies = [
"proc-macro2",
]
@@ -1658,9 +1638,9 @@ dependencies = [
[[package]]
name = "rand_core"
-version = "0.6.1"
+version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c026d7df8b298d90ccbbc5190bd04d85e159eaf5576caeacf8741da93ccbd2e5"
+checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7"
dependencies = [
"getrandom 0.2.2",
]
@@ -1682,9 +1662,9 @@ checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce"
[[package]]
name = "redox_syscall"
-version = "0.2.4"
+version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "05ec8ca9416c5ea37062b502703cd7fcb207736bc294f6e0cf367ac6fc234570"
+checksum = "94341e4e44e24f6b591b59e47a8a027df12e008d73fd5672dbea9cc22f4507d9"
dependencies = [
"bitflags",
]
@@ -1901,6 +1881,16 @@ dependencies = [
"signal-hook-registry",
]
+[[package]]
+name = "signal-hook"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "780f5e3fe0c66f67197236097d89de1e86216f1f6fdeaf47c442f854ab46c240"
+dependencies = [
+ "libc",
+ "signal-hook-registry",
+]
+
[[package]]
name = "signal-hook-registry"
version = "1.3.0"
@@ -1924,9 +1914,9 @@ checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8"
[[package]]
name = "sluice"
-version = "0.5.3"
+version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e24ed1edc8e774f2ec098b0650eec82bfc7c59ddd16cd0e17797bdc92ce2bf1"
+checksum = "8fa0333a60ff2e3474a6775cc611840c2a55610c831dd366503474c02f1a28f5"
dependencies = [
"futures-channel",
"futures-core",
@@ -1997,7 +1987,7 @@ dependencies = [
"cfg-if 1.0.0",
"libc",
"rand",
- "redox_syscall 0.2.4",
+ "redox_syscall 0.2.5",
"remove_dir_all",
"winapi",
]
@@ -2063,9 +2053,9 @@ dependencies = [
[[package]]
name = "thread_local"
-version = "1.1.2"
+version = "1.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d8208a331e1cb318dd5bd76951d2b8fc48ca38a69f5f4e4af1b6a9f8c6236915"
+checksum = "8018d24e04c95ac8790716a5987d0fec4f8b27249ffa0f7d33f1369bdfb88cbd"
dependencies = [
"once_cell",
]
@@ -2149,9 +2139,9 @@ dependencies = [
[[package]]
name = "tokio-stream"
-version = "0.1.2"
+version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76066865172052eb8796c686f0b441a93df8b08d40a950b062ffb9a426f00edd"
+checksum = "1981ad97df782ab506a1f43bf82c967326960d278acf3bf8279809648c3ff3ea"
dependencies = [
"futures-core",
"pin-project-lite",
@@ -2189,9 +2179,9 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6"
[[package]]
name = "tracing"
-version = "0.1.22"
+version = "0.1.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9f47026cdc4080c07e49b37087de021820269d996f581aac150ef9e5583eefe3"
+checksum = "f7d40a22fd029e33300d8d89a5cc8ffce18bb7c587662f54629e94c9de5487f3"
dependencies = [
"cfg-if 1.0.0",
"log",
@@ -2202,9 +2192,9 @@ dependencies = [
[[package]]
name = "tracing-attributes"
-version = "0.1.11"
+version = "0.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "80e0ccfc3378da0cce270c946b676a376943f5cd16aeba64568e7939806f4ada"
+checksum = "43f080ea7e4107844ef4766459426fa2d5c1ada2e47edba05dc7fa99d9629f47"
dependencies = [
"proc-macro2",
"quote",
@@ -2222,11 +2212,11 @@ dependencies = [
[[package]]
name = "tracing-futures"
-version = "0.2.4"
+version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ab7bb6f14721aa00656086e9335d363c5c8747bae02ebe32ea2c7dece5689b4c"
+checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2"
dependencies = [
- "pin-project 0.4.27",
+ "pin-project",
"tracing",
]
@@ -2253,9 +2243,9 @@ dependencies = [
[[package]]
name = "unicode-normalization"
-version = "0.1.16"
+version = "0.1.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a13e63ab62dbe32aeee58d1c5408d35c36c392bba5d9d3142287219721afe606"
+checksum = "07fbfce1c8a97d547e8b5334978438d9d6ec8c20e38f56d4a4374d181493eaef"
dependencies = [
"tinyvec",
]
diff --git a/Cargo.toml b/Cargo.toml
index fce232e..3006ccc 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -25,7 +25,7 @@ futures = { version = "0.3"}
tokio = { version = "1.2.0", features = ["full"] }
tokio-util = {version = "0.6.3", features = ["codec"]}
log = "0.4"
-env_logger = "0.8"
+env_logger = "0.8.3"
reqwest = { version = "0.11", features = ["socks"] }
clap = "2.33"
lazy_static = "1.4"
diff --git a/README.md b/README.md
index 01aee15..740ab68 100644
--- a/README.md
+++ b/README.md
@@ -8,7 +8,7 @@
-
+
@@ -104,6 +104,7 @@ Enumeration.
- [Cancel a Recursive Scan Interactively (new in `v1.12.0`)](#cancel-a-recursive-scan-interactively-new-in-v1120)
- [Limit Number of Requests per Second (Rate Limiting) (new in `v2.0.0`)](#limit-number-of-requests-per-second-rate-limiting-new-in-v200)
- [Silence all Output or Be Kinda Quiet (new in `v2.0.0`)](#silence-all-output-or-be-kinda-quiet-new-in-v200)
+ - [Auto-tune or Auto-bail from Scans (new in `v2.1.0`)](#auto-tune-or-auto-bail-from-scans-new-in-v210)
- [Comparison w/ Similar Tools](#-comparison-w-similar-tools)
- [Common Problems/Issues (FAQ)](#-common-problemsissues-faq)
- [No file descriptors available](#no-file-descriptors-available)
@@ -198,9 +199,9 @@ Download `feroxbuster_amd64.deb` from the [Releases](https://github.com/epi052/f
that, use your favorite package manager to install the `.deb`.
```
-wget -sLO https://github.com/epi052/feroxbuster/releases/latest/download/feroxbuster_amd64.deb.zip
+curl -sLO https://github.com/epi052/feroxbuster/releases/latest/download/feroxbuster_amd64.deb.zip
unzip feroxbuster_amd64.deb.zip
-sudo apt install ./feroxbuster_amd64.deb
+sudo apt install ./feroxbuster_*_amd64.deb
```
### AUR Install
@@ -370,6 +371,8 @@ A pre-made configuration file with examples of all available settings can be fou
# filter_status = [301]
# threads = 1
# timeout = 5
+# auto_tune = true
+# auto_bail = true
# proxy = "http://127.0.0.1:8080"
# replay_proxy = "http://127.0.0.1:8081"
# replay_codes = [200, 302]
@@ -425,6 +428,8 @@ USAGE:
FLAGS:
-f, --add-slash Append / to each request
+ --auto-bail Automatically stop scanning when an excessive amount of errors are encountered
+ --auto-tune Automatically lower scan rate when an excessive amount of errors are encountered
-D, --dont-filter Don't auto-filter wildcard responses
-e, --extract-links Extract links from response body (html, javascript, etc...); make new requests based on
findings (default: false)
@@ -484,7 +489,6 @@ OPTIONS:
-u, --url ... The target URL(s) (required, unless --stdin used)
-a, --user-agent Sets the User-Agent (default: feroxbuster/VERSION)
-w, --wordlist Path to the wordlist
-
```
## 📊 Scan's Display Explained
@@ -871,6 +875,29 @@ Scanning: https://localhost.com/homepage
Scanning: https://localhost.com/api
```
+### Auto-tune or Auto-bail from scans (new in `v2.1.0`)
+
+Version 2.1.0 introduces the `--auto-tune` and `--auto-bail` flags. You can think of these flags as Policies. Both actions (tuning and bailing) are triggered by the same criteria (below). Policies are only enforced after at least 50 requests have been made (or # of threads, if that's > 50).
+
+Policy Enforcement Criteria:
+ - number of general errors (timeouts, etc) is higher than half the number of threads (or at least 25 if threads are lower) (per directory scanned)
+ - 90% of responses are `403|Forbidden` (per directory scanned)
+ - 30% of requests are `429|Too Many Requests` (per directory scanned)
+
+> both demo gifs below use --timeout to overload a single-threaded python web server and elicit timeouts
+
+#### --auto-tune
+
+The AutoTune policy enforces a rate limit on individual directory scans when one of the criteria above is met. The rate limit self-adjusts every (`timeout / 2`) seconds. If the number of errors have increased during that time, the allowed rate of requests is lowered. On the other hand, if the number of errors hasn't moved, the allowed rate of requests is increased. If no additional errors are found after a certain number of checks, the rate limit will be removed completely.
+
+
+
+#### --auto-bail
+
+The AutoBail policy aborts individual directory scans when one of the criteria above is met. They just stop getting scanned, no muss, no fuss.
+
+
+
## 🧐 Comparison w/ Similar Tools
There are quite a few similar tools for forced browsing/content discovery. Burp Suite Pro, Dirb, Dirbuster, etc...
@@ -918,6 +945,8 @@ few of the use-cases in which feroxbuster may be a better fit:
| cancel a recursive scan interactively (`v1.12.0`) | ✔ | | |
| limit number of requests per second (`v2.0.0`) | ✔ | ✔ | ✔ |
| hide progress bars or be silent (or some variation) (`v2.0.0`) | ✔ | ✔ | ✔ |
+| automatically tune scans based on errors/403s/429s (`v2.1.0`) | ✔ | | |
+| automatically stop scans based on errors/403s/429s (`v2.1.0`) | ✔ | | ✔ |
| **huge** number of other options | | | ✔ |
Of note, there's another written-in-rust content discovery tool, [rustbuster](https://github.com/phra/rustbuster). I
diff --git a/ferox-config.toml.example b/ferox-config.toml.example
index ae01134..756b161 100644
--- a/ferox-config.toml.example
+++ b/ferox-config.toml.example
@@ -21,6 +21,8 @@
# rate_limit = 250
# quiet = true
# silent = true
+# auto_tune = true
+# auto_bail = true
# json = true
# output = "/targets/ellingson_mineral_company/gibson.txt"
# debug_log = "/var/log/find-the-derp.log"
diff --git a/img/auto-bail-demo.gif b/img/auto-bail-demo.gif
new file mode 100644
index 0000000..a30d3f5
Binary files /dev/null and b/img/auto-bail-demo.gif differ
diff --git a/img/auto-tune-demo.gif b/img/auto-tune-demo.gif
new file mode 100644
index 0000000..4cc87c7
Binary files /dev/null and b/img/auto-tune-demo.gif differ
diff --git a/shell_completions/_feroxbuster b/shell_completions/_feroxbuster
index c2b896e..a04d295 100644
--- a/shell_completions/_feroxbuster
+++ b/shell_completions/_feroxbuster
@@ -58,7 +58,7 @@ _feroxbuster() {
'*--filter-similar-to=[Filter out pages that are similar to the given page (ex. --filter-similar-to http://site.xyz/soft404)]' \
'-L+[Limit total number of concurrent scans (default: 0, i.e. no limit)]' \
'--scan-limit=[Limit total number of concurrent scans (default: 0, i.e. no limit)]' \
-'--parallel=[Run parallel feroxbuster instances (one child process per url passed via stdin)]' \
+'(--auto-tune)--parallel=[Run parallel feroxbuster instances (one child process per url passed via stdin)]' \
'--rate-limit=[Limit number of requests per second (per directory) (default: 0, i.e. no limit)]' \
'--time-limit=[Limit total run time of all scans (ex: --time-limit 10m)]' \
'(--silent)*-v[Increase verbosity level (use -vv or more for greater effect. \[CAUTION\] 4 -v'\''s is probably too much)]' \
@@ -66,6 +66,8 @@ _feroxbuster() {
'(-q --quiet)--silent[Only print URLs + turn off logging (good for piping a list of urls to other commands)]' \
'-q[Hide progress bars and banner (good for tmux windows w/ notifications)]' \
'--quiet[Hide progress bars and banner (good for tmux windows w/ notifications)]' \
+'(--auto-bail)--auto-tune[Automatically lower scan rate when an excessive amount of errors are encountered]' \
+'--auto-bail[Automatically stop scanning when an excessive amount of errors are encountered]' \
'--json[Emit JSON logs to --output and --debug-log instead of normal text]' \
'-D[Don'\''t auto-filter wildcard responses]' \
'--dont-filter[Don'\''t auto-filter wildcard responses]' \
diff --git a/shell_completions/_feroxbuster.ps1 b/shell_completions/_feroxbuster.ps1
index 3f6417c..ed29564 100644
--- a/shell_completions/_feroxbuster.ps1
+++ b/shell_completions/_feroxbuster.ps1
@@ -71,6 +71,8 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
[CompletionResult]::new('--silent', 'silent', [CompletionResultType]::ParameterName, 'Only print URLs + turn off logging (good for piping a list of urls to other commands)')
[CompletionResult]::new('-q', 'q', [CompletionResultType]::ParameterName, 'Hide progress bars and banner (good for tmux windows w/ notifications)')
[CompletionResult]::new('--quiet', 'quiet', [CompletionResultType]::ParameterName, 'Hide progress bars and banner (good for tmux windows w/ notifications)')
+ [CompletionResult]::new('--auto-tune', 'auto-tune', [CompletionResultType]::ParameterName, 'Automatically lower scan rate when an excessive amount of errors are encountered')
+ [CompletionResult]::new('--auto-bail', 'auto-bail', [CompletionResultType]::ParameterName, 'Automatically stop scanning when an excessive amount of errors are encountered')
[CompletionResult]::new('--json', 'json', [CompletionResultType]::ParameterName, 'Emit JSON logs to --output and --debug-log instead of normal text')
[CompletionResult]::new('-D', 'D', [CompletionResultType]::ParameterName, 'Don''t auto-filter wildcard responses')
[CompletionResult]::new('--dont-filter', 'dont-filter', [CompletionResultType]::ParameterName, 'Don''t auto-filter wildcard responses')
diff --git a/shell_completions/feroxbuster.bash b/shell_completions/feroxbuster.bash
index 777bf2a..a9d5b29 100644
--- a/shell_completions/feroxbuster.bash
+++ b/shell_completions/feroxbuster.bash
@@ -20,7 +20,7 @@ _feroxbuster() {
case "${cmd}" in
feroxbuster)
- opts=" -v -q -D -r -k -n -f -e -h -V -w -u -t -d -T -p -P -R -s -o -a -x -H -Q -S -X -W -N -C -L --verbosity --silent --quiet --json --dont-filter --redirects --insecure --no-recursion --add-slash --stdin --extract-links --help --version --wordlist --url --threads --depth --timeout --proxy --replay-proxy --replay-codes --status-codes --output --resume-from --debug-log --user-agent --extensions --headers --query --filter-size --filter-regex --filter-words --filter-lines --filter-status --filter-similar-to --scan-limit --parallel --rate-limit --time-limit "
+ opts=" -v -q -D -r -k -n -f -e -h -V -w -u -t -d -T -p -P -R -s -o -a -x -H -Q -S -X -W -N -C -L --verbosity --silent --quiet --auto-tune --auto-bail --json --dont-filter --redirects --insecure --no-recursion --add-slash --stdin --extract-links --help --version --wordlist --url --threads --depth --timeout --proxy --replay-proxy --replay-codes --status-codes --output --resume-from --debug-log --user-agent --extensions --headers --query --filter-size --filter-regex --filter-words --filter-lines --filter-status --filter-similar-to --scan-limit --parallel --rate-limit --time-limit "
if [[ ${cur} == -* || ${COMP_CWORD} -eq 1 ]] ; then
COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
return 0
diff --git a/shell_completions/feroxbuster.fish b/shell_completions/feroxbuster.fish
index 59c9b94..7f315ce 100644
--- a/shell_completions/feroxbuster.fish
+++ b/shell_completions/feroxbuster.fish
@@ -27,6 +27,8 @@ complete -c feroxbuster -n "__fish_use_subcommand" -l time-limit -d 'Limit total
complete -c feroxbuster -n "__fish_use_subcommand" -s v -l verbosity -d 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v\'s is probably too much)'
complete -c feroxbuster -n "__fish_use_subcommand" -l silent -d 'Only print URLs + turn off logging (good for piping a list of urls to other commands)'
complete -c feroxbuster -n "__fish_use_subcommand" -s q -l quiet -d 'Hide progress bars and banner (good for tmux windows w/ notifications)'
+complete -c feroxbuster -n "__fish_use_subcommand" -l auto-tune -d 'Automatically lower scan rate when an excessive amount of errors are encountered'
+complete -c feroxbuster -n "__fish_use_subcommand" -l auto-bail -d 'Automatically stop scanning when an excessive amount of errors are encountered'
complete -c feroxbuster -n "__fish_use_subcommand" -l json -d 'Emit JSON logs to --output and --debug-log instead of normal text'
complete -c feroxbuster -n "__fish_use_subcommand" -s D -l dont-filter -d 'Don\'t auto-filter wildcard responses'
complete -c feroxbuster -n "__fish_use_subcommand" -s r -l redirects -d 'Follow redirects'
diff --git a/src/banner/container.rs b/src/banner/container.rs
index 65d3cd3..806081b 100644
--- a/src/banner/container.rs
+++ b/src/banner/container.rs
@@ -1,8 +1,8 @@
use super::entry::BannerEntry;
-use crate::event_handlers::Handles;
use crate::{
config::Configuration,
- utils::{make_request, status_colorizer},
+ event_handlers::Handles,
+ utils::{logged_request, status_colorizer},
VERSION,
};
use anyhow::{bail, Result};
@@ -128,6 +128,12 @@ pub struct Banner {
/// represents Configuration.parallel
parallel: BannerEntry,
+ /// represents Configuration.auto_tune
+ auto_tune: BannerEntry,
+
+ /// represents Configuration.auto_bail
+ auto_bail: BannerEntry,
+
/// current version of feroxbuster
pub(super) version: String,
@@ -254,6 +260,8 @@ impl Banner {
);
let replay_proxy = BannerEntry::new("🎥", "Replay Proxy", &config.replay_proxy);
+ let auto_tune = BannerEntry::new("🎶", "Auto Tune", &config.auto_tune.to_string());
+ let auto_bail = BannerEntry::new("🪣", "Auto Bail", &config.auto_bail.to_string());
let cfg = BannerEntry::new("💉", "Config File", &config.config);
let proxy = BannerEntry::new("💎", "Proxy", &config.proxy);
let threads = BannerEntry::new("🚀", "Threads", &config.threads.to_string());
@@ -288,6 +296,8 @@ impl Banner {
filter_status,
timeout,
user_agent,
+ auto_bail,
+ auto_tune,
proxy,
replay_codes,
replay_proxy,
@@ -359,15 +369,8 @@ by Ben "epi" Risher {} ver: {}"#,
let api_url = Url::parse(url)?;
- let response = make_request(
- &handles.config.client,
- &api_url,
- handles.config.output_level,
- handles.stats.tx.clone(),
- )
- .await?;
-
- let body = response.text().await?;
+ let result = logged_request(&api_url, handles.clone()).await?;
+ let body = result.text().await?;
let json_response: Value = serde_json::from_str(&body)?;
@@ -491,6 +494,13 @@ by Ben "epi" Risher {} ver: {}"#,
writeln!(&mut writer, "{}", self.insecure)?;
}
+ if config.auto_bail {
+ writeln!(&mut writer, "{}", self.auto_bail)?;
+ }
+ if config.auto_tune {
+ writeln!(&mut writer, "{}", self.auto_tune)?;
+ }
+
if config.redirects {
writeln!(&mut writer, "{}", self.redirects)?;
}
diff --git a/src/banner/tests.rs b/src/banner/tests.rs
index 0f997df..4106ba4 100644
--- a/src/banner/tests.rs
+++ b/src/banner/tests.rs
@@ -1,6 +1,6 @@
use super::container::UpdateStatus;
use super::*;
-use crate::{config::Configuration, event_handlers::Handles};
+use crate::{config::Configuration, event_handlers::Handles, scan_manager::FeroxScans};
use httpmock::Method::GET;
use httpmock::MockServer;
use std::{io::stderr, sync::Arc, time::Duration};
@@ -73,8 +73,9 @@ async fn banner_needs_update_returns_up_to_date() {
when.method(GET).path("/latest");
then.status(200).body("{\"tag_name\":\"v1.1.0\"}");
});
+ let scans = Arc::new(FeroxScans::default());
- let handles = Arc::new(Handles::for_testing(None, None).0);
+ let handles = Arc::new(Handles::for_testing(Some(scans), None).0);
let mut banner = Banner::new(&[srv.url("")], &Configuration::new().unwrap());
banner.version = String::from("1.1.0");
@@ -95,7 +96,9 @@ async fn banner_needs_update_returns_out_of_date() {
then.status(200).body("{\"tag_name\":\"v1.1.0\"}");
});
- let handles = Arc::new(Handles::for_testing(None, None).0);
+ let scans = Arc::new(FeroxScans::default());
+
+ let handles = Arc::new(Handles::for_testing(Some(scans), None).0);
let mut banner = Banner::new(&[srv.url("")], &Configuration::new().unwrap());
banner.version = String::from("1.0.1");
diff --git a/src/config/container.rs b/src/config/container.rs
index f1af853..bdae9ef 100644
--- a/src/config/container.rs
+++ b/src/config/container.rs
@@ -1,8 +1,9 @@
use super::utils::{
depth, report_and_exit, save_state, serialized_type, status_codes, threads, timeout,
- user_agent, wordlist, OutputLevel,
+ user_agent, wordlist, OutputLevel, RequesterPolicy,
};
use crate::config::determine_output_level;
+use crate::config::utils::determine_requester_policy;
use crate::{
client, parser, scan_manager::resume_scan, traits::FeroxSerialize, utils::fmt_err,
DEFAULT_CONFIG_NAME,
@@ -124,6 +125,18 @@ pub struct Configuration {
#[serde(skip)]
pub output_level: OutputLevel,
+ /// automatically bail at certain error thresholds
+ #[serde(default)]
+ pub auto_bail: bool,
+
+ /// automatically try to lower request rate in order to reduce errors
+ #[serde(default)]
+ pub auto_tune: bool,
+
+ /// more easily differentiate between the three requester policies
+ #[serde(skip)]
+ pub requester_policy: RequesterPolicy,
+
/// Store log output as NDJSON
#[serde(default)]
pub json: bool,
@@ -249,6 +262,7 @@ impl Default for Configuration {
let replay_codes = status_codes.clone();
let kind = serialized_type();
let output_level = OutputLevel::Default;
+ let requester_policy = RequesterPolicy::Default;
Configuration {
kind,
@@ -258,7 +272,10 @@ impl Default for Configuration {
replay_codes,
status_codes,
replay_client,
+ requester_policy,
dont_filter: false,
+ auto_bail: false,
+ auto_tune: false,
silent: false,
quiet: false,
output_level,
@@ -318,6 +335,8 @@ impl Configuration {
/// - **debug_log**: `None`
/// - **quiet**: `false`
/// - **silent**: `false`
+ /// - **auto_tune**: `false`
+ /// - **auto_bail**: `false`
/// - **save_state**: `true`
/// - **user_agent**: `feroxbuster/VERSION`
/// - **insecure**: `false` (don't be insecure, i.e. don't allow invalid certs)
@@ -380,7 +399,7 @@ impl Configuration {
// read in the user provided options, this produces a separate instance of Configuration
// in order to allow for potentially merging into a --resume-from Configuration
- let cli_config = Self::parse_cli_args(&args)?;
+ let cli_config = Self::parse_cli_args(&args);
// --resume-from used, need to first read the Configuration from disk, and then
// merge the cli_config into the resumed config
@@ -467,7 +486,7 @@ impl Configuration {
/// Given a set of ArgMatches read from the CLI, update and return the default Configuration
/// settings
- fn parse_cli_args(args: &ArgMatches) -> Result {
+ fn parse_cli_args(args: &ArgMatches) -> Self {
let mut config = Configuration::default();
update_config_if_present!(&mut config.threads, args, "threads", usize);
@@ -568,6 +587,16 @@ impl Configuration {
config.output_level = OutputLevel::Quiet;
}
+ if args.is_present("auto_tune") {
+ config.auto_tune = true;
+ config.requester_policy = RequesterPolicy::AutoTune;
+ }
+
+ if args.is_present("auto_bail") {
+ config.auto_bail = true;
+ config.requester_policy = RequesterPolicy::AutoBail;
+ }
+
if args.is_present("dont_filter") {
config.dont_filter = true;
}
@@ -643,7 +672,7 @@ impl Configuration {
}
}
- Ok(config)
+ config
}
/// this function determines if we've gotten a Client configuration change from
@@ -728,8 +757,11 @@ impl Configuration {
update_if_not_default!(&mut conf.verbosity, new.verbosity, 0);
update_if_not_default!(&mut conf.silent, new.silent, false);
update_if_not_default!(&mut conf.quiet, new.quiet, false);
- // use updated quiet/silent values to determin output level
+ update_if_not_default!(&mut conf.auto_bail, new.auto_bail, false);
+ update_if_not_default!(&mut conf.auto_tune, new.auto_tune, false);
+ // use updated quiet/silent values to determine output level; same for requester policy
conf.output_level = determine_output_level(conf.quiet, conf.silent);
+ conf.requester_policy = determine_requester_policy(conf.auto_tune, conf.auto_bail);
update_if_not_default!(&mut conf.output, new.output, "");
update_if_not_default!(&mut conf.redirects, new.redirects, false);
update_if_not_default!(&mut conf.insecure, new.insecure, false);
diff --git a/src/config/mod.rs b/src/config/mod.rs
index 58d9505..275473c 100644
--- a/src/config/mod.rs
+++ b/src/config/mod.rs
@@ -6,4 +6,4 @@ mod utils;
mod tests;
pub use self::container::Configuration;
-pub use self::utils::{determine_output_level, OutputLevel};
+pub use self::utils::{determine_output_level, OutputLevel, RequesterPolicy};
diff --git a/src/config/tests.rs b/src/config/tests.rs
index 2f310d8..88e1b20 100644
--- a/src/config/tests.rs
+++ b/src/config/tests.rs
@@ -16,6 +16,8 @@ fn setup_config_test() -> Configuration {
replay_proxy = "http://127.0.0.1:8081"
quiet = true
silent = true
+ auto_tune = true
+ auto_bail = true
verbosity = 1
scan_limit = 6
parallel = 14
@@ -72,7 +74,11 @@ fn default_configuration() {
assert_eq!(config.scan_limit, 0);
assert_eq!(config.silent, false);
assert_eq!(config.quiet, false);
+ assert_eq!(config.output_level, OutputLevel::Default);
assert_eq!(config.dont_filter, false);
+ assert_eq!(config.auto_tune, false);
+ assert_eq!(config.auto_bail, false);
+ assert_eq!(config.requester_policy, RequesterPolicy::Default);
assert_eq!(config.no_recursion, false);
assert_eq!(config.json, false);
assert_eq!(config.save_state, true);
@@ -197,6 +203,20 @@ fn config_reads_json() {
assert_eq!(config.json, true);
}
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_auto_bail() {
+ let config = setup_config_test();
+ assert_eq!(config.auto_bail, true);
+}
+
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_auto_tune() {
+ let config = setup_config_test();
+ assert_eq!(config.auto_tune, true);
+}
+
#[test]
/// parse the test config and see that the value parsed is correct
fn config_reads_verbosity() {
diff --git a/src/config/utils.rs b/src/config/utils.rs
index 6b630fe..64fe1f7 100644
--- a/src/config/utils.rs
+++ b/src/config/utils.rs
@@ -102,6 +102,41 @@ pub fn determine_output_level(quiet: bool, silent: bool) -> OutputLevel {
}
}
+/// represents actions the Requester should take in certain situations
+#[derive(Debug, PartialEq, Copy, Clone)]
+pub enum RequesterPolicy {
+ /// automatically try to lower request rate in order to reduce errors
+ AutoTune,
+
+ /// automatically bail at certain error thresholds
+ AutoBail,
+
+ /// just let that junk run super natural
+ Default,
+}
+
+/// default implementation for RequesterPolicy
+impl Default for RequesterPolicy {
+ /// Default as default
+ fn default() -> Self {
+ Self::Default
+ }
+}
+
+/// given the current settings for quiet and silent, determine output_level (DRY helper)
+pub fn determine_requester_policy(auto_tune: bool, auto_bail: bool) -> RequesterPolicy {
+ if auto_tune && auto_bail {
+ // user COULD have both as true in config file, take the more aggressive of the two
+ RequesterPolicy::AutoBail
+ } else if auto_tune {
+ RequesterPolicy::AutoTune
+ } else if auto_bail {
+ RequesterPolicy::AutoBail
+ } else {
+ RequesterPolicy::Default
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
@@ -122,6 +157,22 @@ mod tests {
assert_eq!(level, OutputLevel::Quiet);
}
+ #[test]
+ /// test determine_requester_policy returns higher of the two levels if both given values are true
+ fn determine_requester_policy_returns_correct_results() {
+ let mut level = determine_requester_policy(true, true);
+ assert_eq!(level, RequesterPolicy::AutoBail);
+
+ level = determine_requester_policy(false, true);
+ assert_eq!(level, RequesterPolicy::AutoBail);
+
+ level = determine_requester_policy(false, false);
+ assert_eq!(level, RequesterPolicy::Default);
+
+ level = determine_requester_policy(true, false);
+ assert_eq!(level, RequesterPolicy::AutoTune);
+ }
+
#[test]
#[should_panic]
/// report_and_exit should panic/exit when called
diff --git a/src/event_handlers/command.rs b/src/event_handlers/command.rs
index e343c12..34fb72e 100644
--- a/src/event_handlers/command.rs
+++ b/src/event_handlers/command.rs
@@ -25,11 +25,14 @@ pub enum Command {
/// Create the progress bar (`BarType::Total`) that is updated from the stats thread
CreateBar,
- /// Update a `Stats` field that corresponds to the given `StatField` by the given `usize` value
- UpdateUsizeField(StatField, usize),
+ /// Add to a `Stats` field that corresponds to the given `StatField` by the given `usize` value
+ AddToUsizeField(StatField, usize),
+
+ /// Subtract from a `Stats` field that corresponds to the given `StatField` by the given `usize` value
+ SubtractFromUsizeField(StatField, usize),
/// Update a `Stats` field that corresponds to the given `StatField` by the given `f64` value
- UpdateF64Field(StatField, f64),
+ AddToF64Field(StatField, f64),
/// Save a `Stats` object to disk using `reporter::get_cached_file_handle`
Save,
diff --git a/src/event_handlers/outputs.rs b/src/event_handlers/outputs.rs
index ea19efc..bfb8410 100644
--- a/src/event_handlers/outputs.rs
+++ b/src/event_handlers/outputs.rs
@@ -1,4 +1,4 @@
-use super::Command::UpdateUsizeField;
+use super::Command::AddToUsizeField;
use super::*;
use anyhow::{Context, Result};
@@ -195,7 +195,7 @@ impl TermOutHandler {
// print to stdout
ferox_print(&resp.as_str(), &PROGRESS_PRINTER);
- send_command!(tx_stats, UpdateUsizeField(ResourcesDiscovered, 1));
+ send_command!(tx_stats, AddToUsizeField(ResourcesDiscovered, 1));
if self.file_task.is_some() {
// -o used, need to send the report to be written out to disk
@@ -210,7 +210,7 @@ impl TermOutHandler {
if self.config.replay_client.is_some() && should_process_response {
// replay proxy specified/client created and this response's status code is one that
- // should be replayed
+ // should be replayed; not using logged_request due to replay proxy client
make_request(
self.config.replay_client.as_ref().unwrap(),
&resp.url(),
diff --git a/src/event_handlers/scans.rs b/src/event_handlers/scans.rs
index 601095e..1b1cc0d 100644
--- a/src/event_handlers/scans.rs
+++ b/src/event_handlers/scans.rs
@@ -10,11 +10,12 @@ use crate::{
scan_manager::{FeroxScan, FeroxScans, ScanOrder},
scanner::FeroxScanner,
statistics::StatField::TotalScans,
- CommandReceiver, CommandSender, FeroxChannel, Joiner,
+ CommandReceiver, CommandSender, FeroxChannel, Joiner, SLEEP_DURATION,
};
-use super::command::Command::UpdateUsizeField;
+use super::command::Command::AddToUsizeField;
use super::*;
+use tokio::time::Duration;
#[derive(Debug)]
/// Container for recursion transmitter and FeroxScans object
@@ -153,9 +154,7 @@ impl ScanHandler {
tokio::spawn(async move {
while ferox_scans.has_active_scans() {
- for scan in ferox_scans.get_active_scans() {
- scan.join().await;
- }
+ tokio::time::sleep(Duration::from_millis(SLEEP_DURATION + 250)).await;
}
limiter_clone.close();
sender.send(true).expect("oneshot channel failed");
@@ -231,7 +230,7 @@ impl ScanHandler {
}
});
- self.handles.stats.send(UpdateUsizeField(TotalScans, 1))?;
+ self.handles.stats.send(AddToUsizeField(TotalScans, 1))?;
scan.set_task(task).await?;
diff --git a/src/event_handlers/statistics.rs b/src/event_handlers/statistics.rs
index 98f7b16..8a47b1f 100644
--- a/src/event_handlers/statistics.rs
+++ b/src/event_handlers/statistics.rs
@@ -89,6 +89,7 @@ impl StatsHandler {
}
Command::AddStatus(status) => {
self.stats.add_status_code(status);
+
self.increment_bar();
}
Command::AddRequest => {
@@ -99,14 +100,21 @@ impl StatsHandler {
self.stats
.save(start.elapsed().as_secs_f64(), output_file)?;
}
- Command::UpdateUsizeField(field, value) => {
+ Command::AddToUsizeField(field, value) => {
self.stats.update_usize_field(field, value);
if matches!(field, StatField::TotalScans) {
self.bar.set_length(self.stats.total_expected() as u64);
}
}
- Command::UpdateF64Field(field, value) => self.stats.update_f64_field(field, value),
+ Command::SubtractFromUsizeField(field, value) => {
+ self.stats.subtract_from_usize_field(field, value);
+
+ if matches!(field, StatField::TotalExpected) {
+ self.bar.set_length(self.stats.total_expected() as u64);
+ }
+ }
+ Command::AddToF64Field(field, value) => self.stats.update_f64_field(field, value),
Command::CreateBar => {
self.bar = add_bar("", self.stats.total_expected() as u64, BarType::Total);
}
diff --git a/src/extractor/container.rs b/src/extractor/container.rs
index a5263a8..d7c6dcf 100644
--- a/src/extractor/container.rs
+++ b/src/extractor/container.rs
@@ -3,7 +3,7 @@ use crate::{
client,
event_handlers::{
Command,
- Command::{AddError, UpdateUsizeField},
+ Command::{AddError, AddToUsizeField},
Handles,
},
scan_manager::ScanOrder,
@@ -12,7 +12,7 @@ use crate::{
StatField::{LinksExtracted, TotalExpected},
},
url::FeroxUrl,
- utils::make_request,
+ utils::{logged_request, make_request},
};
use anyhow::{bail, Context, Result};
use reqwest::{StatusCode, Url};
@@ -303,13 +303,7 @@ impl<'a> Extractor<'a> {
}
// make the request and store the response
- let new_response = make_request(
- &self.handles.config.client,
- &new_url,
- self.handles.config.output_level,
- self.handles.stats.tx.clone(),
- )
- .await?;
+ let new_response = logged_request(&new_url, self.handles.clone()).await?;
let new_ferox_response =
FeroxResponse::from(new_response, true, self.handles.config.output_level).await;
@@ -384,6 +378,7 @@ impl<'a> Extractor<'a> {
let mut url = Url::parse(&self.url)?;
url.set_path("/robots.txt"); // overwrite existing path with /robots.txt
+ // purposefully not using logged_request here due to using the special client
let response = make_request(
&client,
&url,
@@ -391,6 +386,7 @@ impl<'a> Extractor<'a> {
self.handles.stats.tx.clone(),
)
.await?;
+
let ferox_response =
FeroxResponse::from(response, true, self.handles.config.output_level).await;
@@ -404,10 +400,10 @@ impl<'a> Extractor<'a> {
self.handles
.stats
- .send(UpdateUsizeField(LinksExtracted, num_links))?;
+ .send(AddToUsizeField(LinksExtracted, num_links))?;
self.handles
.stats
- .send(UpdateUsizeField(TotalExpected, num_links * multiplier))?;
+ .send(AddToUsizeField(TotalExpected, num_links * multiplier))?;
Ok(())
}
diff --git a/src/filters/container.rs b/src/filters/container.rs
index 31d3e18..474522f 100644
--- a/src/filters/container.rs
+++ b/src/filters/container.rs
@@ -4,7 +4,7 @@ use anyhow::Result;
use crate::response::FeroxResponse;
use crate::{
- event_handlers::Command::UpdateUsizeField, statistics::StatField::WildcardsFiltered,
+ event_handlers::Command::AddToUsizeField, statistics::StatField::WildcardsFiltered,
CommandSender,
};
@@ -44,7 +44,7 @@ impl FeroxFilters {
if filter.should_filter_response(&response) {
if filter.as_any().downcast_ref::().is_some() {
tx_stats
- .send(UpdateUsizeField(WildcardsFiltered, 1))
+ .send(AddToUsizeField(WildcardsFiltered, 1))
.unwrap_or_default();
}
return true;
diff --git a/src/filters/init.rs b/src/filters/init.rs
index 9b0cf0c..35440b0 100644
--- a/src/filters/init.rs
+++ b/src/filters/init.rs
@@ -5,7 +5,7 @@ use crate::{
event_handlers::Handles,
response::FeroxResponse,
skip_fail,
- utils::{fmt_err, make_request},
+ utils::{fmt_err, logged_request},
Command::AddFilter,
SIMILARITY_THRESHOLD,
};
@@ -72,15 +72,7 @@ pub async fn initialize(handles: Arc) -> Result<()> {
let url = skip_fail!(Url::parse(&similarity_filter));
// attempt to request the given url
- let resp = skip_fail!(
- make_request(
- &handles.config.client,
- &url,
- handles.config.output_level,
- handles.stats.tx.clone()
- )
- .await
- );
+ let resp = skip_fail!(logged_request(&url, handles.clone()).await);
// if successful, create a filter based on the response's body
let fr = FeroxResponse::from(resp, true, handles.config.output_level).await;
diff --git a/src/heuristics.rs b/src/heuristics.rs
index 6f1e4da..3329d91 100644
--- a/src/heuristics.rs
+++ b/src/heuristics.rs
@@ -12,7 +12,7 @@ use crate::{
response::FeroxResponse,
skip_fail,
url::FeroxUrl,
- utils::{ferox_print, fmt_err, make_request, status_colorizer},
+ utils::{ferox_print, fmt_err, logged_request, status_colorizer},
};
/// length of a standard UUID, used when determining wildcard responses
@@ -158,13 +158,7 @@ impl HeuristicTests {
let unique_str = self.unique_string(length);
let nonexistent_url = target.format(&unique_str, None)?;
- let response = make_request(
- &self.handles.config.client,
- &nonexistent_url.to_owned(),
- self.handles.config.output_level,
- self.handles.stats.tx.clone(),
- )
- .await?;
+ let response = logged_request(&nonexistent_url.to_owned(), self.handles.clone()).await?;
if self
.handles
@@ -215,13 +209,8 @@ impl HeuristicTests {
for target_url in target_urls {
let url = FeroxUrl::from_string(&target_url, self.handles.clone());
let request = skip_fail!(url.format("", None));
- let result = make_request(
- &self.handles.config.client,
- &request,
- self.handles.config.output_level,
- self.handles.stats.tx.clone(),
- )
- .await;
+
+ let result = logged_request(&request, self.handles.clone()).await;
match result {
Ok(_) => {
@@ -232,10 +221,17 @@ impl HeuristicTests {
self.handles.config.output_level,
OutputLevel::Default | OutputLevel::Quiet
) {
- ferox_print(
- &format!("Could not connect to {}, skipping...", target_url),
- &PROGRESS_PRINTER,
- );
+ if e.to_string().contains(":SSL") {
+ ferox_print(
+ &format!("Could not connect to {} due to SSL errors (run with -k to ignore), skipping...", target_url),
+ &PROGRESS_PRINTER,
+ );
+ } else {
+ ferox_print(
+ &format!("Could not connect to {}, skipping...", target_url),
+ &PROGRESS_PRINTER,
+ );
+ }
}
log::warn!("{}", e);
}
diff --git a/src/lib.rs b/src/lib.rs
index 2424ae4..b8ee387 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -57,7 +57,10 @@ pub const DEFAULT_WORDLIST: &str =
"/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt";
/// Number of milliseconds to wait between polls of `PAUSE_SCAN` when user pauses a scan
-pub(crate) static SLEEP_DURATION: u64 = 500;
+pub(crate) const SLEEP_DURATION: u64 = 500;
+
+/// The percentage of requests as errors it takes to be deemed too high
+pub const HIGH_ERROR_RATIO: f64 = 0.90;
/// Default list of status codes to report
///
diff --git a/src/main.rs b/src/main.rs
index ecf88b8..7b6971a 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -320,7 +320,7 @@ async fn wrapped_main(config: Arc) -> Result<()> {
// The TermOutHandler spawns a FileOutHandler, so errors in the FileOutHandler never bubble
// up due to the TermOutHandler never awaiting the result of FileOutHandler::start (that's
- // done later here in main). Ping checks that the tx/rx connection to the file handler works
+ // done later here in main). sync checks that the tx/rx connection to the file handler works
if send_to_file && handles.output.sync(send_to_file).await.is_err() {
// output file specified and file handler could not initialize
clean_up(handles, tasks).await?;
diff --git a/src/message.rs b/src/message.rs
index c8b4142..a5186c4 100644
--- a/src/message.rs
+++ b/src/message.rs
@@ -1,9 +1,9 @@
use anyhow::Context;
use console::{style, Color};
+use serde::{Deserialize, Serialize};
use crate::traits::FeroxSerialize;
use crate::utils::fmt_err;
-use serde::{Deserialize, Serialize};
#[derive(Serialize, Deserialize, Default)]
/// Representation of a log entry, can be represented as a human readable string or JSON
@@ -118,4 +118,31 @@ mod tests {
assert_eq!(json.level, message.level);
assert_eq!(json.kind, message.kind);
}
+
+ #[test]
+ /// test defaults for coverage
+ fn message_defaults() {
+ let msg = FeroxMessage::default();
+ assert_eq!(msg.level, String::new());
+ assert_eq!(msg.kind, String::new());
+ assert_eq!(msg.message, String::new());
+ assert_eq!(msg.module, String::new());
+ assert!(msg.time_offset < 0.1);
+ }
+
+ #[test]
+ /// ensure WILDCARD messages serialize to WLD and anything not known to UNK
+ fn message_as_str_edges() {
+ let mut msg = FeroxMessage {
+ message: "message".to_string(),
+ module: "utils".to_string(),
+ time_offset: 1.0,
+ level: "WILDCARD".to_string(),
+ kind: "log".to_string(),
+ };
+ assert!(console::strip_ansi_codes(&msg.as_str()).starts_with("WLD"));
+
+ msg.level = "UNKNOWN".to_string();
+ assert!(console::strip_ansi_codes(&msg.as_str()).starts_with("UNK"));
+ }
}
diff --git a/src/parser.rs b/src/parser.rs
index c8ba04d..0b2a12e 100644
--- a/src/parser.rs
+++ b/src/parser.rs
@@ -130,6 +130,19 @@ pub fn initialize() -> App<'static, 'static> {
.takes_value(false)
.help("Hide progress bars and banner (good for tmux windows w/ notifications)")
)
+ .arg(
+ Arg::with_name("auto_tune")
+ .long("auto-tune")
+ .takes_value(false)
+ .conflicts_with("auto_bail")
+ .help("Automatically lower scan rate when an excessive amount of errors are encountered")
+ )
+ .arg(
+ Arg::with_name("auto_bail")
+ .long("auto-bail")
+ .takes_value(false)
+ .help("Automatically stop scanning when an excessive amount of errors are encountered")
+ )
.arg(
Arg::with_name("json")
.long("json")
@@ -348,6 +361,7 @@ pub fn initialize() -> App<'static, 'static> {
.long("rate-limit")
.value_name("RATE_LIMIT")
.takes_value(true)
+ .conflicts_with("auto_tune")
.help("Limit number of requests per second (per directory) (default: 0, i.e. no limit)")
)
.arg(
diff --git a/src/scan_manager/scan.rs b/src/scan_manager/scan.rs
index ca624c1..22808ba 100644
--- a/src/scan_manager/scan.rs
+++ b/src/scan_manager/scan.rs
@@ -2,6 +2,7 @@ use super::*;
use crate::{
config::OutputLevel,
progress::{add_bar, BarType},
+ scanner::PolicyTrigger,
};
use anyhow::Result;
use console::style;
@@ -12,8 +13,10 @@ use std::{
collections::HashMap,
fmt,
sync::{Arc, Mutex},
+ time::Instant,
};
+use std::sync::atomic::{AtomicUsize, Ordering};
use tokio::{sync, task::JoinHandle};
use uuid::Uuid;
@@ -49,6 +52,18 @@ pub struct FeroxScan {
/// whether or not the user passed --silent|--quiet on the command line
pub(super) output_level: OutputLevel,
+
+ /// tracker for overall number of 403s seen by the FeroxScan instance
+ pub(super) status_403s: AtomicUsize,
+
+ /// tracker for overall number of 429s seen by the FeroxScan instance
+ pub(super) status_429s: AtomicUsize,
+
+ /// tracker for total number of errors encountered by the FeroxScan instance
+ pub(super) errors: AtomicUsize,
+
+ /// tracker for the time at which this scan was started
+ pub(super) start_time: Instant,
}
/// Default implementation for FeroxScan
@@ -67,6 +82,10 @@ impl Default for FeroxScan {
progress_bar: Mutex::new(None),
scan_type: ScanType::File,
output_level: Default::default(),
+ errors: Default::default(),
+ status_429s: Default::default(),
+ status_403s: Default::default(),
+ start_time: Instant::now(),
}
}
}
@@ -75,16 +94,22 @@ impl Default for FeroxScan {
impl FeroxScan {
/// Stop a currently running scan
pub async fn abort(&self) -> Result<()> {
- let mut guard = self.task.lock().await;
+ log::trace!("enter: abort");
- if guard.is_some() {
- if let Some(task) = std::mem::replace(&mut *guard, None) {
- task.abort();
- self.set_status(ScanStatus::Cancelled)?;
- self.stop_progress_bar();
+ match self.task.try_lock() {
+ Ok(mut guard) => {
+ if let Some(task) = std::mem::replace(&mut *guard, None) {
+ log::trace!("aborting {:?}", self);
+ task.abort();
+ self.set_status(ScanStatus::Cancelled)?;
+ self.stop_progress_bar();
+ }
+ }
+ Err(e) => {
+ log::warn!("Could not acquire lock to abort scan (we're already waiting for its results): {:?} {}", self, e);
}
}
-
+ log::trace!("exit: abort");
Ok(())
}
@@ -134,6 +159,7 @@ impl FeroxScan {
pb.reset_elapsed();
let _ = std::mem::replace(&mut *guard, Some(pb.clone()));
+
pb
}
}
@@ -217,6 +243,61 @@ impl FeroxScan {
log::trace!("exit join({:?})", self);
}
+ /// increment the value in question by 1
+ pub(crate) fn add_403(&self) {
+ self.status_403s.fetch_add(1, Ordering::Relaxed);
+ }
+
+ /// increment the value in question by 1
+ pub(crate) fn add_429(&self) {
+ self.status_429s.fetch_add(1, Ordering::Relaxed);
+ }
+
+ /// increment the value in question by 1
+ pub(crate) fn add_error(&self) {
+ self.errors.fetch_add(1, Ordering::Relaxed);
+ }
+
+ /// simple wrapper to call the appropriate getter based on the given PolicyTrigger
+ pub fn num_errors(&self, trigger: PolicyTrigger) -> usize {
+ match trigger {
+ PolicyTrigger::Status403 => self.status_403s(),
+ PolicyTrigger::Status429 => self.status_429s(),
+ PolicyTrigger::Errors => self.errors(),
+ }
+ }
+
+ /// return the number of errors seen by this scan
+ fn errors(&self) -> usize {
+ self.errors.load(Ordering::Relaxed)
+ }
+
+ /// return the number of 403s seen by this scan
+ fn status_403s(&self) -> usize {
+ self.status_403s.load(Ordering::Relaxed)
+ }
+
+ /// return the number of 429s seen by this scan
+ fn status_429s(&self) -> usize {
+ self.status_429s.load(Ordering::Relaxed)
+ }
+
+ /// return the number of requests per second performed by this scan's scanner
+ pub fn requests_per_second(&self) -> u64 {
+ if !self.is_active() {
+ return 0;
+ }
+
+ let reqs = self.requests();
+ let seconds = self.start_time.elapsed().as_secs();
+
+ reqs.checked_div(seconds).unwrap_or(0)
+ }
+
+ /// return the number of requests performed by this scan's scanner
+ pub fn requests(&self) -> u64 {
+ self.progress_bar().position()
+ }
}
/// Display implementation
@@ -360,3 +441,68 @@ impl Default for ScanStatus {
Self::NotStarted
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use std::thread::sleep;
+ use tokio::time::Duration;
+
+ #[test]
+ /// ensure that num_errors returns the correct values for the given PolicyTrigger
+ ///
+ /// covers tests for add_[403,429,error] and the related getters in addition to num_errors
+ fn num_errors_returns_correct_values() {
+ let scan = FeroxScan::new(
+ "http://localhost",
+ ScanType::Directory,
+ ScanOrder::Latest,
+ 1000,
+ OutputLevel::Default,
+ None,
+ );
+
+ scan.add_error();
+ scan.add_403();
+ scan.add_403();
+ scan.add_429();
+ scan.add_429();
+ scan.add_429();
+
+ assert_eq!(scan.num_errors(PolicyTrigger::Errors), 1);
+ assert_eq!(scan.num_errors(PolicyTrigger::Status403), 2);
+ assert_eq!(scan.num_errors(PolicyTrigger::Status429), 3);
+ }
+
+ #[test]
+ /// ensure that requests_per_second returns the correct values
+ fn requests_per_second_returns_correct_values() {
+ let scan = FeroxScan {
+ id: "".to_string(),
+ url: "".to_string(),
+ scan_type: ScanType::Directory,
+ scan_order: ScanOrder::Initial,
+ num_requests: 0,
+ status: Mutex::new(ScanStatus::Running),
+ task: Default::default(),
+ progress_bar: Mutex::new(None),
+ output_level: Default::default(),
+ status_403s: Default::default(),
+ status_429s: Default::default(),
+ errors: Default::default(),
+ start_time: Instant::now(),
+ };
+
+ let pb = scan.progress_bar();
+ pb.set_position(100);
+
+ sleep(Duration::new(1, 0));
+
+ let req_sec = scan.requests_per_second();
+
+ assert_eq!(req_sec, 100);
+
+ scan.finish().unwrap();
+ assert_eq!(scan.requests_per_second(), 0);
+ }
+}
diff --git a/src/scan_manager/scan_container.rs b/src/scan_manager/scan_container.rs
index 6a50cf3..6a298af 100644
--- a/src/scan_manager/scan_container.rs
+++ b/src/scan_manager/scan_container.rs
@@ -9,6 +9,7 @@ use crate::{
SLEEP_DURATION,
};
use anyhow::Result;
+use reqwest::StatusCode;
use serde::{ser::SerializeSeq, Serialize, Serializer};
use std::{
convert::TryInto,
@@ -161,6 +162,63 @@ impl FeroxScans {
None
}
+ pub(super) fn get_base_scan_by_url(&self, url: &str) -> Option> {
+ log::trace!("enter: get_sub_paths_from_path({})", url);
+
+ // rmatch_indices returns tuples in index, match form, i.e. (10, "/")
+ // with the furthest-right match in the first position in the vector
+ let matches: Vec<_> = url.rmatch_indices('/').collect();
+
+ // iterate from the furthest right matching index and check the given url from the
+ // start to the furthest-right '/' character. compare that slice to the urls associated
+ // with directory scans and return the first match, since it should be the 'deepest'
+ // match.
+ // Example:
+ // url: http://shmocalhost/src/release/examples/stuff.php
+ // scans:
+ // http://shmocalhost/src/statistics
+ // http://shmocalhost/src/banner
+ // http://shmocalhost/src/release
+ // http://shmocalhost/src/release/examples
+ //
+ // returns: http://shmocalhost/src/release/examples
+ if let Ok(guard) = self.scans.read() {
+ for (idx, _) in &matches {
+ for scan in guard.iter() {
+ let slice = url.index(0..*idx);
+ if slice == scan.url || format!("{}/", slice).as_str() == scan.url {
+ log::trace!("enter: get_sub_paths_from_path -> {}", scan);
+ return Some(scan.clone());
+ }
+ }
+ }
+ }
+
+ log::trace!("enter: get_sub_paths_from_path -> None");
+ None
+ }
+ /// add one to either 403 or 429 tracker in the scan related to the given url
+ pub fn increment_status_code(&self, url: &str, code: StatusCode) {
+ if let Some(scan) = self.get_base_scan_by_url(url) {
+ match code {
+ StatusCode::TOO_MANY_REQUESTS => {
+ scan.add_429();
+ }
+ StatusCode::FORBIDDEN => {
+ scan.add_403();
+ }
+ _ => {}
+ }
+ }
+ }
+
+ /// add one to either 403 or 429 tracker in the scan related to the given url
+ pub fn increment_error(&self, url: &str) {
+ if let Some(scan) = self.get_base_scan_by_url(url) {
+ scan.add_error();
+ }
+ }
+
/// Print all FeroxScans of type Directory
///
/// Example:
@@ -194,9 +252,11 @@ impl FeroxScans {
}
/// Given a list of indexes, cancel their associated FeroxScans
- async fn cancel_scans(&self, indexes: Vec) {
+ async fn cancel_scans(&self, indexes: Vec) -> usize {
let menu_pause_duration = Duration::from_millis(SLEEP_DURATION);
+ let mut num_cancelled = 0_usize;
+
for num in indexes {
let selected = match self.scans.read() {
Ok(u_scans) => {
@@ -217,32 +277,42 @@ impl FeroxScans {
if input == 'y' || input == '\n' {
self.menu.println(&format!("Stopping {}...", selected.url));
+
selected
.abort()
.await
.unwrap_or_else(|e| log::warn!("Could not cancel task: {}", e));
+
+ let pb = selected.progress_bar();
+ num_cancelled += pb.length() as usize - pb.position() as usize
} else {
self.menu.println("Ok, doing nothing...");
}
sleep(menu_pause_duration);
}
+
+ num_cancelled
}
/// CLI menu that allows for interactive cancellation of recursed-into directories
- async fn interactive_menu(&self) {
+ async fn interactive_menu(&self) -> usize {
self.menu.hide_progress_bars();
self.menu.clear_screen();
self.menu.print_header();
self.display_scans().await;
self.menu.print_footer();
+ let mut num_cancelled = 0_usize;
+
if let Some(input) = self.menu.get_scans_from_user() {
- self.cancel_scans(input).await
+ num_cancelled += self.cancel_scans(input).await;
};
self.menu.clear_screen();
self.menu.show_progress_bars();
+
+ num_cancelled
}
/// prints all known responses that the scanner has already seen
@@ -290,18 +360,19 @@ impl FeroxScans {
///
/// When the value stored in `PAUSE_SCAN` becomes `false`, the function returns, exiting the busy
/// loop
- pub async fn pause(&self, get_user_input: bool) {
+ pub async fn pause(&self, get_user_input: bool) -> usize {
// function uses tokio::time, not std
// local testing showed a pretty slow increase (less than linear) in CPU usage as # of
// concurrent scans rose when SLEEP_DURATION was set to 500, using that as the default for now
let mut interval = time::interval(time::Duration::from_millis(SLEEP_DURATION));
+ let mut num_cancelled = 0_usize;
if INTERACTIVE_BARRIER.load(Ordering::Relaxed) == 0 {
INTERACTIVE_BARRIER.fetch_add(1, Ordering::Relaxed);
if get_user_input {
- self.interactive_menu().await;
+ num_cancelled += self.interactive_menu().await;
PAUSE_SCAN.store(false, Ordering::Relaxed);
self.print_known_responses();
}
@@ -318,8 +389,8 @@ impl FeroxScans {
INTERACTIVE_BARRIER.fetch_sub(1, Ordering::Relaxed);
}
- log::trace!("exit: pause_scan");
- return;
+ log::trace!("exit: pause_scan -> {}", num_cancelled);
+ return num_cancelled;
}
}
}
@@ -351,7 +422,7 @@ impl FeroxScans {
let bar = match scan_type {
ScanType::Directory => {
let bar_type = match self.output_level {
- OutputLevel::Default => BarType::Message,
+ OutputLevel::Default => BarType::Default,
OutputLevel::Quiet => BarType::Quiet,
OutputLevel::Silent => BarType::Hidden,
};
diff --git a/src/scan_manager/tests.rs b/src/scan_manager/tests.rs
index 2dffbba..113f380 100644
--- a/src/scan_manager/tests.rs
+++ b/src/scan_manager/tests.rs
@@ -12,6 +12,7 @@ use indicatif::ProgressBar;
use predicates::prelude::*;
use std::sync::{atomic::Ordering, Arc};
use std::thread::sleep;
+use std::time::Instant;
use tokio::time::{self, Duration};
#[test]
@@ -382,7 +383,7 @@ fn feroxstates_feroxserialize_implementation() {
let json_state = ferox_state.as_json().unwrap();
let expected = format!(
- r#"{{"scans":[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","status":"NotStarted","num_requests":0}}],"config":{{"type":"configuration","wordlist":"/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt","config":"","proxy":"","replay_proxy":"","target_url":"","status_codes":[200,204,301,302,307,308,401,403,405],"replay_codes":[200,204,301,302,307,308,401,403,405],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"silent":false,"quiet":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/{}","redirects":false,"insecure":false,"extensions":[],"headers":{{}},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":false,"depth":4,"scan_limit":0,"parallel":0,"rate_limit":0,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false,"resumed":false,"resume_from":"","save_state":false,"time_limit":"","filter_similar":[]}},"responses":[{{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{{"server":"nginx/1.16.1"}}}}]"#,
+ r#"{{"scans":[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","status":"NotStarted","num_requests":0}}],"config":{{"type":"configuration","wordlist":"/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt","config":"","proxy":"","replay_proxy":"","target_url":"","status_codes":[200,204,301,302,307,308,401,403,405],"replay_codes":[200,204,301,302,307,308,401,403,405],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"silent":false,"quiet":false,"auto_bail":false,"auto_tune":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/{}","redirects":false,"insecure":false,"extensions":[],"headers":{{}},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":false,"depth":4,"scan_limit":0,"parallel":0,"rate_limit":0,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false,"resumed":false,"resume_from":"","save_state":false,"time_limit":"","filter_similar":[]}},"responses":[{{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{{"server":"nginx/1.16.1"}}}}]"#,
saved_id, VERSION
);
println!("{}\n{}", expected, json_state);
@@ -437,10 +438,14 @@ fn feroxscan_display() {
scan_order: ScanOrder::Latest,
scan_type: Default::default(),
num_requests: 0,
+ start_time: Instant::now(),
output_level: OutputLevel::Default,
+ status_403s: Default::default(),
+ status_429s: Default::default(),
status: Default::default(),
task: tokio::sync::Mutex::new(None),
progress_bar: std::sync::Mutex::new(None),
+ errors: Default::default(),
};
let not_started = format!("{}", scan);
@@ -477,12 +482,16 @@ async fn ferox_scan_abort() {
scan_order: ScanOrder::Latest,
scan_type: Default::default(),
num_requests: 0,
+ start_time: Instant::now(),
output_level: OutputLevel::Default,
+ status_403s: Default::default(),
+ status_429s: Default::default(),
status: std::sync::Mutex::new(ScanStatus::Running),
task: tokio::sync::Mutex::new(Some(tokio::spawn(async move {
sleep(Duration::from_millis(SLEEP_DURATION * 2));
}))),
progress_bar: std::sync::Mutex::new(None),
+ errors: Default::default(),
};
scan.abort().await.unwrap();
@@ -516,3 +525,70 @@ fn split_to_nums_is_correct() {
assert_eq!(nums, vec![1, 3, 4]);
}
+
+#[test]
+/// given a deep url, find the correct scan
+fn get_base_scan_by_url_finds_correct_scan() {
+ let urls = FeroxScans::default();
+ let url = "http://localhost";
+ let url1 = "http://localhost/stuff";
+ let url2 = "http://shlocalhost/stuff/things";
+ let url3 = "http://shlocalhost/stuff/things/mostuff";
+ let (_, scan) = urls.add_scan(url, ScanType::Directory, ScanOrder::Latest);
+ let (_, scan1) = urls.add_scan(url1, ScanType::Directory, ScanOrder::Latest);
+ let (_, scan2) = urls.add_scan(url2, ScanType::Directory, ScanOrder::Latest);
+ let (_, scan3) = urls.add_scan(url3, ScanType::Directory, ScanOrder::Latest);
+
+ assert_eq!(
+ urls.get_base_scan_by_url("http://localhost/things.php")
+ .unwrap()
+ .id,
+ scan.id
+ );
+ assert_eq!(
+ urls.get_base_scan_by_url("http://localhost/stuff/things.php")
+ .unwrap()
+ .id,
+ scan1.id
+ );
+ assert_eq!(
+ urls.get_base_scan_by_url("http://shlocalhost/stuff/things/mostuff.php")
+ .unwrap()
+ .id,
+ scan2.id
+ );
+ assert_eq!(
+ urls.get_base_scan_by_url("http://shlocalhost/stuff/things/mostuff/mothings.php")
+ .unwrap()
+ .id,
+ scan3.id
+ );
+}
+
+#[test]
+/// given a shallow url without a trailing slash, find the correct scan
+fn get_base_scan_by_url_finds_correct_scan_without_trailing_slash() {
+ let urls = FeroxScans::default();
+ let url = "http://localhost";
+ let (_, scan) = urls.add_scan(url, ScanType::Directory, ScanOrder::Latest);
+ assert_eq!(
+ urls.get_base_scan_by_url("http://localhost/BKPMiherrortBPKcw")
+ .unwrap()
+ .id,
+ scan.id
+ );
+}
+
+#[test]
+/// given a shallow url with a trailing slash, find the correct scan
+fn get_base_scan_by_url_finds_correct_scan_with_trailing_slash() {
+ let urls = FeroxScans::default();
+ let url = "http://127.0.0.1:41971/";
+ let (_, scan) = urls.add_scan(url, ScanType::Directory, ScanOrder::Latest);
+ assert_eq!(
+ urls.get_base_scan_by_url("http://127.0.0.1:41971/BKPMiherrortBPKcw")
+ .unwrap()
+ .id,
+ scan.id
+ );
+}
diff --git a/src/scanner.rs b/src/scanner.rs
deleted file mode 100644
index 8412683..0000000
--- a/src/scanner.rs
+++ /dev/null
@@ -1,352 +0,0 @@
-use std::{
- cmp::max, collections::HashSet, convert::TryInto, ops::Deref, sync::atomic::Ordering,
- sync::Arc, time::Instant,
-};
-
-use anyhow::{bail, Result};
-use futures::{stream, StreamExt};
-use lazy_static::lazy_static;
-use leaky_bucket::LeakyBucket;
-use tokio::sync::{oneshot, Semaphore};
-
-use crate::{
- event_handlers::{
- Command::{self, AddError, UpdateF64Field, UpdateUsizeField},
- Handles,
- },
- extractor::{
- ExtractionTarget::{ResponseBody, RobotsTxt},
- ExtractorBuilder,
- },
- heuristics,
- response::FeroxResponse,
- scan_manager::{FeroxResponses, ScanOrder, ScanStatus, PAUSE_SCAN},
- statistics::{
- StatError::Other,
- StatField::{DirScanTimes, ExpectedPerScan},
- },
- url::FeroxUrl,
- utils::{fmt_err, make_request},
-};
-use tokio::time::Duration;
-
-lazy_static! {
- /// Vector of FeroxResponse objects
- pub static ref RESPONSES: FeroxResponses = FeroxResponses::default();
- // todo consider removing this
-}
-
-/// Makes multiple requests based on the presence of extensions
-struct Requester {
- /// handles to handlers and config
- handles: Arc,
-
- /// url that will be scanned
- target_url: String,
-
- /// limits requests per second if present
- rate_limiter: Option,
-}
-
-/// Requester implementation
-impl Requester {
- /// given a FeroxScanner, create a Requester
- pub fn from(scanner: &FeroxScanner) -> Result {
- let limit = scanner.handles.config.rate_limit;
- let refill = max(limit / 10, 1); // minimum of 1 per second
- let tokens = max(limit / 2, 1);
- let interval = if refill == 1 { 1000 } else { 100 }; // 1 second if refill is 1
-
- let rate_limiter = if limit > 0 {
- let bucket = LeakyBucket::builder()
- .refill_interval(Duration::from_millis(interval)) // add tokens every 0.1s
- .refill_amount(refill) // ex: 100 req/s -> 10 tokens per 0.1s
- .tokens(tokens) // reduce initial burst, 2 is arbitrary, but felt good
- .max(limit)
- .build()?;
- Some(bucket)
- } else {
- None
- };
-
- Ok(Self {
- rate_limiter,
- handles: scanner.handles.clone(),
- target_url: scanner.target_url.to_owned(),
- })
- }
-
- /// limit the number of requests per second
- pub async fn limit(&self) -> Result<()> {
- self.rate_limiter.as_ref().unwrap().acquire_one().await?;
- Ok(())
- }
-
- /// Wrapper for [make_request](fn.make_request.html)
- ///
- /// Attempts recursion when appropriate and sends Responses to the output handler for processing
- async fn request(&self, word: &str) -> Result<()> {
- log::trace!("enter: request({})", word);
-
- let urls =
- FeroxUrl::from_string(&self.target_url, self.handles.clone()).formatted_urls(word)?;
-
- for url in urls {
- if self.rate_limiter.is_some() {
- // found a rate limiter, limit that junk!
- if let Err(e) = self.limit().await {
- log::warn!("Could not rate limit scan: {}", e);
- self.handles.stats.send(AddError(Other)).unwrap_or_default();
- }
- }
-
- let response = make_request(
- &self.handles.config.client,
- &url,
- self.handles.config.output_level,
- self.handles.stats.tx.clone(),
- )
- .await?;
-
- // response came back without error, convert it to FeroxResponse
- let ferox_response =
- FeroxResponse::from(response, true, self.handles.config.output_level).await;
-
- // do recursion if appropriate
- if !self.handles.config.no_recursion {
- self.handles
- .send_scan_command(Command::TryRecursion(Box::new(ferox_response.clone())))?;
- let (tx, rx) = oneshot::channel::();
- self.handles.send_scan_command(Command::Sync(tx))?;
- rx.await?;
- }
-
- // purposefully doing recursion before filtering. the thought process is that
- // even though this particular url is filtered, subsequent urls may not
- if self
- .handles
- .filters
- .data
- .should_filter_response(&ferox_response, self.handles.stats.tx.clone())
- {
- continue;
- }
-
- if self.handles.config.extract_links && !ferox_response.status().is_redirection() {
- let extractor = ExtractorBuilder::default()
- .target(ResponseBody)
- .response(&ferox_response)
- .handles(self.handles.clone())
- .build()?;
-
- extractor.extract().await?;
- }
-
- // everything else should be reported
- if let Err(e) = ferox_response.send_report(self.handles.output.tx.clone()) {
- log::warn!("Could not send FeroxResponse to output handler: {}", e);
- }
- }
-
- log::trace!("exit: request");
- Ok(())
- }
-}
-
-/// handles the main muscle movement of scanning a url
-pub struct FeroxScanner {
- /// handles to handlers and config
- handles: Arc,
-
- /// url that will be scanned
- target_url: String,
-
- /// whether or not this scanner is targeting an initial target specified by the user or one
- /// found via recursion
- order: ScanOrder,
-
- /// wordlist that's already been read from disk
- wordlist: Arc>,
-
- /// limiter that restricts the number of active FeroxScanners
- scan_limiter: Arc,
-}
-
-/// FeroxScanner implementation
-impl FeroxScanner {
- /// create a new FeroxScanner
- pub fn new(
- target_url: &str,
- order: ScanOrder,
- wordlist: Arc>,
- scan_limiter: Arc,
- handles: Arc,
- ) -> Self {
- Self {
- order,
- handles,
- wordlist,
- scan_limiter,
- target_url: target_url.to_string(),
- }
- }
-
- /// Scan a given url using a given wordlist
- ///
- /// This is the primary entrypoint for the scanner
- pub async fn scan_url(&self) -> Result<()> {
- log::trace!("enter: scan_url");
- log::info!("Starting scan against: {}", self.target_url);
-
- let scan_timer = Instant::now();
-
- if matches!(self.order, ScanOrder::Initial) && self.handles.config.extract_links {
- // only grab robots.txt on the initial scan_url calls. all fresh dirs will be passed
- // to try_recursion
- let extractor = ExtractorBuilder::default()
- .url(&self.target_url)
- .handles(self.handles.clone())
- .target(RobotsTxt)
- .build()?;
-
- let _ = extractor.extract().await;
- }
-
- let scanned_urls = self.handles.ferox_scans()?;
-
- let ferox_scan = match scanned_urls.get_scan_by_url(&self.target_url) {
- Some(scan) => {
- scan.set_status(ScanStatus::Running)?;
- scan
- }
- None => {
- let msg = format!(
- "Could not find FeroxScan associated with {}; this shouldn't happen... exiting",
- self.target_url
- );
- bail!(fmt_err(&msg))
- }
- };
-
- let progress_bar = ferox_scan.progress_bar();
-
- // When acquire is called and the semaphore has remaining permits, the function immediately
- // returns a permit. However, if no remaining permits are available, acquire (asynchronously)
- // waits until an outstanding permit is dropped, at which point, the freed permit is assigned
- // to the caller.
- let _permit = self.scan_limiter.acquire().await;
-
- // Arc clones to be passed around to the various scans
- let looping_words = self.wordlist.clone();
-
- {
- let test = heuristics::HeuristicTests::new(self.handles.clone());
- if let Ok(num_reqs) = test.wildcard(&self.target_url).await {
- progress_bar.inc(num_reqs);
- }
- }
-
- let requester = Arc::new(Requester::from(self)?);
- let increment_len = (self.handles.config.extensions.len() + 1) as u64;
-
- // producer tasks (mp of mpsc); responsible for making requests
- let producers = stream::iter(looping_words.deref().to_owned())
- .map(|word| {
- let pb = progress_bar.clone(); // progress bar is an Arc around internal state
- let scanned_urls_clone = scanned_urls.clone();
- let requester_clone = requester.clone();
- (
- tokio::spawn(async move {
- if PAUSE_SCAN.load(Ordering::Acquire) {
- // for every word in the wordlist, check to see if PAUSE_SCAN is set to true
- // when true; enter a busy loop that only exits by setting PAUSE_SCAN back
- // to false
- scanned_urls_clone.pause(true).await;
- }
- requester_clone.request(&word).await
- }),
- pb,
- )
- })
- .for_each_concurrent(self.handles.config.threads, |(resp, bar)| async move {
- match resp.await {
- Ok(_) => {
- bar.inc(increment_len);
- }
- Err(e) => {
- log::warn!("error awaiting a response: {}", e);
- self.handles.stats.send(AddError(Other)).unwrap_or_default();
- }
- }
- });
-
- // await tx tasks
- log::trace!("awaiting scan producers");
- producers.await;
- log::trace!("done awaiting scan producers");
-
- self.handles.stats.send(UpdateF64Field(
- DirScanTimes,
- scan_timer.elapsed().as_secs_f64(),
- ))?;
-
- ferox_scan.finish()?;
-
- log::trace!("exit: scan_url");
-
- Ok(())
- }
-}
-
-/// Perform steps necessary to run scans that only need to be performed once (warming up the
-/// engine, as it were)
-pub async fn initialize(num_words: usize, handles: Arc) -> Result<()> {
- log::trace!("enter: initialize({}, {:?})", num_words, handles);
-
- // number of requests only needs to be calculated once, and then can be reused
- let num_reqs_expected: u64 = if handles.config.extensions.is_empty() {
- num_words.try_into()?
- } else {
- let total = num_words * (handles.config.extensions.len() + 1);
- total.try_into()?
- };
-
- {
- // no real reason to keep the arc around beyond this call
- let scans = handles.ferox_scans()?;
- scans.set_bar_length(num_reqs_expected);
- }
-
- // tell Stats object about the number of expected requests
- handles.stats.send(UpdateUsizeField(
- ExpectedPerScan,
- num_reqs_expected as usize,
- ))?;
-
- log::trace!("exit: initialize");
- Ok(())
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use crate::config::OutputLevel;
- use crate::scan_manager::FeroxScans;
-
- #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
- #[should_panic]
- /// try to hit struct field coverage of FileOutHandler
- async fn get_scan_by_url_bails_on_unfound_url() {
- let sem = Semaphore::new(10);
- let urls = FeroxScans::new(OutputLevel::Default);
-
- let scanner = FeroxScanner::new(
- "http://localhost",
- ScanOrder::Initial,
- Arc::new(Default::default()),
- Arc::new(sem),
- Arc::new(Handles::for_testing(Some(Arc::new(urls)), None).0),
- );
- scanner.scan_url().await.unwrap();
- }
-}
diff --git a/src/scanner/ferox_scanner.rs b/src/scanner/ferox_scanner.rs
new file mode 100644
index 0000000..2b4a88b
--- /dev/null
+++ b/src/scanner/ferox_scanner.rs
@@ -0,0 +1,185 @@
+use std::{collections::HashSet, ops::Deref, sync::atomic::Ordering, sync::Arc, time::Instant};
+
+use anyhow::{bail, Result};
+use futures::{stream, StreamExt};
+use lazy_static::lazy_static;
+use tokio::sync::Semaphore;
+
+use crate::{
+ event_handlers::{
+ Command::{AddError, AddToF64Field, SubtractFromUsizeField},
+ Handles,
+ },
+ extractor::{ExtractionTarget::RobotsTxt, ExtractorBuilder},
+ heuristics,
+ scan_manager::{FeroxResponses, ScanOrder, ScanStatus, PAUSE_SCAN},
+ statistics::{
+ StatError::Other,
+ StatField::{DirScanTimes, TotalExpected},
+ },
+ utils::fmt_err,
+};
+
+use super::requester::Requester;
+
+lazy_static! {
+ /// Vector of FeroxResponse objects
+ pub static ref RESPONSES: FeroxResponses = FeroxResponses::default();
+ // todo consider removing this
+}
+/// handles the main muscle movement of scanning a url
+pub struct FeroxScanner {
+ /// handles to handlers and config
+ pub(super) handles: Arc,
+
+ /// url that will be scanned
+ pub(super) target_url: String,
+
+ /// whether or not this scanner is targeting an initial target specified by the user or one
+ /// found via recursion
+ order: ScanOrder,
+
+ /// wordlist that's already been read from disk
+ wordlist: Arc>,
+
+ /// limiter that restricts the number of active FeroxScanners
+ scan_limiter: Arc,
+}
+
+/// FeroxScanner implementation
+impl FeroxScanner {
+ /// create a new FeroxScanner
+ pub fn new(
+ target_url: &str,
+ order: ScanOrder,
+ wordlist: Arc>,
+ scan_limiter: Arc,
+ handles: Arc,
+ ) -> Self {
+ Self {
+ order,
+ handles,
+ wordlist,
+ scan_limiter,
+ target_url: target_url.to_string(),
+ }
+ }
+
+ /// Scan a given url using a given wordlist
+ ///
+ /// This is the primary entrypoint for the scanner
+ pub async fn scan_url(&self) -> Result<()> {
+ log::trace!("enter: scan_url");
+ log::info!("Starting scan against: {}", self.target_url);
+
+ let scan_timer = Instant::now();
+
+ if matches!(self.order, ScanOrder::Initial) && self.handles.config.extract_links {
+ // only grab robots.txt on the initial scan_url calls. all fresh dirs will be passed
+ // to try_recursion
+ let extractor = ExtractorBuilder::default()
+ .url(&self.target_url)
+ .handles(self.handles.clone())
+ .target(RobotsTxt)
+ .build()?;
+
+ let _ = extractor.extract().await;
+ }
+
+ let scanned_urls = self.handles.ferox_scans()?;
+
+ let ferox_scan = match scanned_urls.get_scan_by_url(&self.target_url) {
+ Some(scan) => {
+ scan.set_status(ScanStatus::Running)?;
+ scan
+ }
+ None => {
+ let msg = format!(
+ "Could not find FeroxScan associated with {}; this shouldn't happen... exiting",
+ self.target_url
+ );
+ bail!(fmt_err(&msg))
+ }
+ };
+
+ let progress_bar = ferox_scan.progress_bar();
+
+ // When acquire is called and the semaphore has remaining permits, the function immediately
+ // returns a permit. However, if no remaining permits are available, acquire (asynchronously)
+ // waits until an outstanding permit is dropped, at which point, the freed permit is assigned
+ // to the caller.
+ let _permit = self.scan_limiter.acquire().await;
+
+ // Arc clones to be passed around to the various scans
+ let looping_words = self.wordlist.clone();
+
+ {
+ let test = heuristics::HeuristicTests::new(self.handles.clone());
+ if let Ok(num_reqs) = test.wildcard(&self.target_url).await {
+ progress_bar.inc(num_reqs);
+ }
+ }
+
+ let requester = Arc::new(Requester::from(self, ferox_scan.clone())?);
+ let increment_len = (self.handles.config.extensions.len() + 1) as u64;
+
+ // producer tasks (mp of mpsc); responsible for making requests
+ let producers = stream::iter(looping_words.deref().to_owned())
+ .map(|word| {
+ let pb = progress_bar.clone(); // progress bar is an Arc around internal state
+ let scanned_urls_clone = scanned_urls.clone();
+ let requester_clone = requester.clone();
+ let handles_clone = self.handles.clone();
+ (
+ tokio::spawn(async move {
+ if PAUSE_SCAN.load(Ordering::Acquire) {
+ // for every word in the wordlist, check to see if PAUSE_SCAN is set to true
+ // when true; enter a busy loop that only exits by setting PAUSE_SCAN back
+ // to false
+ let num_cancelled = scanned_urls_clone.pause(true).await;
+ if num_cancelled > 0 {
+ handles_clone
+ .stats
+ .send(SubtractFromUsizeField(TotalExpected, num_cancelled))
+ .unwrap_or_else(|e| {
+ log::warn!("Could not update overall scan bar: {}", e)
+ });
+ }
+ }
+ requester_clone
+ .request(&word)
+ .await
+ .unwrap_or_else(|e| log::warn!("Requester encountered an error: {}", e))
+ }),
+ pb,
+ )
+ })
+ .for_each_concurrent(self.handles.config.threads, |(resp, bar)| async move {
+ match resp.await {
+ Ok(_) => {
+ bar.inc(increment_len);
+ }
+ Err(e) => {
+ log::warn!("error awaiting a response: {}", e);
+ self.handles.stats.send(AddError(Other)).unwrap_or_default();
+ }
+ }
+ });
+
+ // await tx tasks
+ log::trace!("awaiting scan producers");
+ producers.await;
+ log::trace!("done awaiting scan producers");
+
+ self.handles.stats.send(AddToF64Field(
+ DirScanTimes,
+ scan_timer.elapsed().as_secs_f64(),
+ ))?;
+
+ ferox_scan.finish()?;
+
+ log::trace!("exit: scan_url");
+
+ Ok(())
+ }
+}
diff --git a/src/scanner/init.rs b/src/scanner/init.rs
new file mode 100644
index 0000000..82ecd80
--- /dev/null
+++ b/src/scanner/init.rs
@@ -0,0 +1,34 @@
+use crate::{
+ event_handlers::{Command::AddToUsizeField, Handles},
+ statistics::StatField::ExpectedPerScan,
+};
+use anyhow::Result;
+use std::{convert::TryInto, sync::Arc};
+
+/// Perform steps necessary to run scans that only need to be performed once (warming up the
+/// engine, as it were)
+pub async fn initialize(num_words: usize, handles: Arc) -> Result<()> {
+ log::trace!("enter: initialize({}, {:?})", num_words, handles);
+
+ // number of requests only needs to be calculated once, and then can be reused
+ let num_reqs_expected: u64 = if handles.config.extensions.is_empty() {
+ num_words.try_into()?
+ } else {
+ let total = num_words * (handles.config.extensions.len() + 1);
+ total.try_into()?
+ };
+
+ {
+ // no real reason to keep the arc around beyond this call
+ let scans = handles.ferox_scans()?;
+ scans.set_bar_length(num_reqs_expected);
+ }
+
+ // tell Stats object about the number of expected requests
+ handles
+ .stats
+ .send(AddToUsizeField(ExpectedPerScan, num_reqs_expected as usize))?;
+
+ log::trace!("exit: initialize");
+ Ok(())
+}
diff --git a/src/scanner/limit_heap.rs b/src/scanner/limit_heap.rs
new file mode 100644
index 0000000..64b8b17
--- /dev/null
+++ b/src/scanner/limit_heap.rs
@@ -0,0 +1,158 @@
+/// bespoke variation on an array-backed max-heap
+///
+/// 255 possible values generated from the initial requests/second
+///
+/// when no additional errors are encountered, the left child is taken (increasing req/sec)
+/// if errors have increased since the last interval, the right child is taken (decreasing req/sec)
+///
+/// formula for each child:
+/// - left: (|parent - current|) / 2 + current
+/// - right: current - ((|parent - current|) / 2)
+#[derive(Debug)]
+pub(super) struct LimitHeap {
+ /// backing array, 255 nodes == height of 7 ( 2^(h+1) -1 nodes )
+ pub(super) inner: [i32; 255],
+
+ /// original # of requests / second
+ pub(super) original: i32,
+
+ /// current position w/in the backing array
+ pub(super) current: usize,
+}
+
+/// default implementation of a LimitHeap
+impl Default for LimitHeap {
+ /// zero-initialize the backing array
+ fn default() -> Self {
+ Self {
+ inner: [0; 255],
+ original: 0,
+ current: 0,
+ }
+ }
+}
+
+/// implementation of a LimitHeap
+impl LimitHeap {
+ /// move to right child, return node's index from which the move was requested
+ pub(super) fn move_right(&mut self) -> usize {
+ if self.has_children() {
+ let tmp = self.current;
+ self.current = self.current * 2 + 2;
+ return tmp;
+ }
+ self.current
+ }
+
+ /// move to left child, return node's index from which the move was requested
+ pub(super) fn move_left(&mut self) -> usize {
+ if self.has_children() {
+ let tmp = self.current;
+ self.current = self.current * 2 + 1;
+ return tmp;
+ }
+ self.current
+ }
+
+ /// move to parent, return node's index from which the move was requested
+ pub(super) fn move_up(&mut self) -> usize {
+ if self.has_parent() {
+ let tmp = self.current;
+ self.current = (self.current - 1) / 2;
+ return tmp;
+ }
+ self.current
+ }
+
+ /// move directly to the given index
+ pub(super) fn move_to(&mut self, index: usize) {
+ self.current = index;
+ }
+
+ /// get the current node's value
+ pub(super) fn value(&self) -> i32 {
+ self.inner[self.current]
+ }
+
+ /// set the current node's value
+ pub(super) fn set_value(&mut self, value: i32) {
+ self.inner[self.current] = value;
+ }
+
+ /// check that this node has a parent (true for all except root)
+ pub(super) fn has_parent(&self) -> bool {
+ self.current > 0
+ }
+
+ /// get node's parent's value or self.original if at the root
+ pub(super) fn parent_value(&mut self) -> i32 {
+ if self.has_parent() {
+ let current = self.move_up();
+ let val = self.value();
+ self.move_to(current);
+ return val;
+ }
+ self.original
+ }
+
+ /// check if the current node has children
+ pub(super) fn has_children(&self) -> bool {
+ // inner structure is a complete tree, just check for the right child
+ self.current * 2 + 2 <= self.inner.len()
+ }
+
+ /// get current node's right child's value
+ fn right_child_value(&mut self) -> i32 {
+ let tmp = self.move_right();
+ let val = self.value();
+ self.move_to(tmp);
+ val
+ }
+
+ /// set current node's left child's value
+ fn set_left_child(&mut self) {
+ let parent = self.parent_value();
+ let current = self.value();
+ let value = ((parent - current).abs() / 2) + current;
+
+ self.move_left();
+ self.set_value(value);
+ self.move_up();
+ }
+
+ /// set current node's right child's value
+ fn set_right_child(&mut self) {
+ let parent = self.parent_value();
+ let current = self.value();
+ let value = current - ((parent - current).abs() / 2);
+
+ self.move_right();
+ self.set_value(value);
+ self.move_up();
+ }
+
+ /// iterate over the backing array, filling in each child's value based on the original value
+ pub(super) fn build(&mut self) {
+ // ex: original is 400
+ // arr[0] == 200
+ // arr[1] (left child) == 300
+ // arr[2] (right child) == 100
+ let root = self.original / 2;
+
+ self.inner[0] = root; // set root node to half of the original value
+ self.inner[1] = ((self.original - root).abs() / 2) + root;
+ self.inner[2] = root - ((self.original - root).abs() / 2);
+
+ // start with index 1 and fill in each child below that node
+ for i in 1..self.inner.len() {
+ self.move_to(i);
+
+ if self.has_children() && self.right_child_value() == 0 {
+ // this node has an unset child since the rchild is 0
+ self.set_left_child();
+ self.set_right_child();
+ }
+ }
+ self.move_to(0); // reset current index to the root of the tree
+ }
+}
diff --git a/src/scanner/mod.rs b/src/scanner/mod.rs
new file mode 100644
index 0000000..7754fa9
--- /dev/null
+++ b/src/scanner/mod.rs
@@ -0,0 +1,12 @@
+mod ferox_scanner;
+mod utils;
+mod init;
+#[cfg(test)]
+mod tests;
+mod limit_heap;
+mod policy_data;
+mod requester;
+
+pub use self::ferox_scanner::{FeroxScanner, RESPONSES};
+pub use self::init::initialize;
+pub use self::utils::PolicyTrigger;
diff --git a/src/scanner/policy_data.rs b/src/scanner/policy_data.rs
new file mode 100644
index 0000000..f6e18bc
--- /dev/null
+++ b/src/scanner/policy_data.rs
@@ -0,0 +1,309 @@
+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+
+use crate::{atomic_load, atomic_store, config::RequesterPolicy};
+
+use super::limit_heap::LimitHeap;
+
+/// data regarding policy and metadata about last enforced trigger etc...
+#[derive(Default, Debug)]
+pub struct PolicyData {
+ /// how to handle exceptional cases such as too many errors / 403s / 429s etc
+ pub(super) policy: RequesterPolicy,
+
+ /// whether or not we're in the middle of a cooldown period
+ pub(super) cooling_down: AtomicBool,
+
+ /// length of time to pause tuning after making an adjustment
+ pub(super) wait_time: u64,
+
+ /// rate limit (at last interval)
+ limit: AtomicUsize,
+
+ /// number of errors (at last interval)
+ pub(super) errors: AtomicUsize,
+
+ /// whether or not the owning Requester should remove the rate_limiter, happens when a scan
+ /// has been limited and moves back up to the point of its original scan speed
+ pub(super) remove_limit: AtomicBool,
+
+ /// heap of values used for adjusting # of requests/second
+ pub(super) heap: std::sync::RwLock,
+}
+
+/// implementation of PolicyData
+impl PolicyData {
+ /// given a RequesterPolicy, create a new PolicyData
+ pub fn new(policy: RequesterPolicy, timeout: u64) -> Self {
+ // can use this as a tweak for how aggressively adjustments should be made when tuning
+ let wait_time = ((timeout as f64 / 2.0) * 1000.0) as u64;
+
+ Self {
+ policy,
+ wait_time,
+ ..Default::default()
+ }
+ }
+
+ /// setter for requests / second; populates the underlying heap with values from req/sec seed
+ pub(super) fn set_reqs_sec(&self, reqs_sec: usize) {
+ if let Ok(mut guard) = self.heap.write() {
+ guard.original = reqs_sec as i32;
+ guard.build();
+ self.set_limit(guard.inner[0] as usize); // set limit to 1/2 of current request rate
+ }
+ }
+
+ /// setter for errors
+ pub(super) fn set_errors(&self, errors: usize) {
+ atomic_store!(self.errors, errors);
+ }
+
+ /// setter for limit
+ fn set_limit(&self, limit: usize) {
+ atomic_store!(self.limit, limit);
+ }
+
+ /// getter for limit
+ pub(super) fn get_limit(&self) -> usize {
+ atomic_load!(self.limit)
+ }
+
+ /// adjust the rate of requests per second up (increase rate)
+ pub(super) fn adjust_up(&self, streak_counter: &usize) {
+ if let Ok(mut heap) = self.heap.try_write() {
+ if *streak_counter > 2 {
+ // streak of 3 upward moves in a row, traverse the tree upward instead of to a
+ // higher-valued branch lower in the tree
+ let current = heap.value();
+ heap.move_up();
+ heap.move_up();
+ if current > heap.value() {
+ // the tree's structure makes it so that sometimes 2 moves up results in a
+ // value greater than the current node's and other times we need to move 3 up
+ // to arrive at a greater value
+ if heap.has_parent() && heap.parent_value() > current {
+ // all nodes except 0th node (root)
+ heap.move_up();
+ } else if !heap.has_parent() {
+ // been here enough that we can try resuming the scan to its original
+ // speed (no limiting at all)
+ atomic_store!(self.remove_limit, true);
+ }
+ }
+ self.set_limit(heap.value() as usize);
+ } else if heap.has_children() {
+ // streak not at 3, just check that we can move down, and do so
+ heap.move_left();
+ self.set_limit(heap.value() as usize);
+ } else {
+ // tree bottomed out, need to move back up the tree a bit
+ let current = heap.value();
+ heap.move_up();
+ heap.move_up();
+
+ if current > heap.value() {
+ heap.move_up();
+ }
+
+ self.set_limit(heap.value() as usize);
+ }
+ }
+ }
+
+ /// adjust the rate of requests per second down (decrease rate)
+ pub(super) fn adjust_down(&self) {
+ if let Ok(mut heap) = self.heap.try_write() {
+ if heap.has_children() {
+ heap.move_right();
+ self.set_limit(heap.value() as usize);
+ }
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ /// PolicyData builds and sets correct values for the inner heap when set_reqs_sec is called
+ fn set_reqs_sec_builds_heap_and_sets_initial_value() {
+ let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+ assert_eq!(pd.wait_time, 3500);
+ pd.set_reqs_sec(400);
+ assert_eq!(pd.get_limit(), 200);
+ assert_eq!(pd.heap.read().unwrap().original, 400);
+ assert_eq!(pd.heap.read().unwrap().current, 0);
+ assert_eq!(pd.heap.read().unwrap().inner[0], 200);
+ assert_eq!(pd.heap.read().unwrap().inner[1], 300);
+ assert_eq!(pd.heap.read().unwrap().inner[2], 100);
+ }
+
+ #[test]
+ /// PolicyData setters/getters tests for code coverage / sanity
+ fn policy_data_getters_and_setters() {
+ let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+ pd.set_errors(20);
+ assert_eq!(pd.errors.load(Ordering::Relaxed), 20);
+ pd.set_limit(200);
+ assert_eq!(pd.get_limit(), 200);
+ }
+
+ #[test]
+ /// PolicyData adjust_down sets the limit to the correct value
+ fn policy_data_adjust_down_simple() {
+ let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+ pd.set_reqs_sec(400);
+ assert_eq!(pd.get_limit(), 200);
+ pd.adjust_down();
+ assert_eq!(pd.get_limit(), 100);
+ }
+
+ #[test]
+ /// PolicyData adjust_down sets the limit to the correct value when no child nodes are present
+ fn policy_data_adjust_down_no_children() {
+ let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+ pd.set_reqs_sec(400);
+ assert_eq!(pd.get_limit(), 200);
+ let mut guard = pd.heap.write().unwrap();
+ guard.move_to(250);
+ guard.set_value(27);
+ pd.set_limit(guard.value() as usize);
+ drop(guard);
+
+ pd.adjust_down();
+ assert_eq!(pd.get_limit(), 27);
+ }
+
+ #[test]
+ /// PolicyData adjust_up sets the limit to the correct value
+ fn policy_data_adjust_up_simple() {
+ let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+ pd.set_reqs_sec(400);
+ assert_eq!(pd.get_limit(), 200);
+ pd.adjust_up(&0);
+ assert_eq!(pd.get_limit(), 300);
+ }
+
+ #[test]
+ /// PolicyData adjust_up sets the limit to the correct value
+ fn policy_data_adjust_up_with_streak_and_2_moves() {
+ // original: 400
+ // [200, 300, 100, 350, 250, 150, 50, 375, 325, 275, 225, 175, 125, 75, 25, ...]
+ let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+ pd.set_reqs_sec(400);
+ assert_eq!(pd.get_limit(), 200);
+
+ // 2 moves
+ pd.heap.write().unwrap().move_to(9);
+ assert_eq!(pd.heap.read().unwrap().value(), 275);
+ pd.adjust_up(&3);
+ assert_eq!(pd.heap.read().unwrap().value(), 300);
+ assert_eq!(pd.limit.load(Ordering::Relaxed), 300);
+ assert_eq!(pd.remove_limit.load(Ordering::Relaxed), false);
+ }
+
+ #[test]
+ /// PolicyData adjust_up sets the limit to the correct value
+ fn policy_data_adjust_up_with_streak_and_2_moves_to_arrive_at_root() {
+ // original: 400
+ // [200, 300, 100, 350, 250, 150, 50, 375, 325, 275, 225, 175, 125, 75, 25, ...]
+ let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+ pd.set_reqs_sec(400);
+ assert_eq!(pd.get_limit(), 200);
+
+ pd.heap.write().unwrap().move_to(4);
+ assert_eq!(pd.heap.read().unwrap().value(), 250);
+ pd.adjust_up(&3);
+ assert_eq!(pd.heap.read().unwrap().value(), 200);
+ assert_eq!(pd.limit.load(Ordering::Relaxed), 200);
+ assert_eq!(pd.remove_limit.load(Ordering::Relaxed), true);
+ }
+
+ #[test]
+ /// PolicyData adjust_up sets the limit to the correct value
+ fn policy_data_adjust_up_with_streak_and_2_moves_to_find_less_than_current() {
+ // original: 400
+ // [200, 300, 100, 350, 250, 150, 50, 375, 325, 275, 225, 175, 125, 75, 25, ...]
+ let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+ pd.set_reqs_sec(400);
+ assert_eq!(pd.get_limit(), 200);
+
+ pd.heap.write().unwrap().move_to(15);
+ assert_eq!(pd.heap.read().unwrap().value(), 387);
+ pd.adjust_up(&3);
+ assert_eq!(pd.heap.read().unwrap().value(), 350);
+ assert_eq!(pd.limit.load(Ordering::Relaxed), 350);
+ assert_eq!(pd.remove_limit.load(Ordering::Relaxed), false);
+ }
+
+ #[test]
+ /// PolicyData adjust_up sets the limit to the correct value
+ fn policy_data_adjust_up_with_streak_and_3_moves() {
+ // original: 400
+ // [200, 300, 100, 350, 250, 150, 50, 375, 325, 275, 225, 175, 125, 75, 25, ...]
+ let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+ pd.set_reqs_sec(400);
+ assert_eq!(pd.get_limit(), 200);
+
+ pd.heap.write().unwrap().move_to(19);
+ assert_eq!(pd.heap.read().unwrap().value(), 287);
+ pd.adjust_up(&3);
+ assert_eq!(pd.heap.read().unwrap().value(), 300);
+ assert_eq!(pd.limit.load(Ordering::Relaxed), 300);
+ assert_eq!(pd.remove_limit.load(Ordering::Relaxed), false);
+ }
+
+ #[test]
+ /// PolicyData adjust_up sets the limit to the correct value
+ fn policy_data_adjust_up_with_no_children_2_moves() {
+ // original: 400
+ // [200, 300, 100, 350, 250, 150, 50, 375, 325, 275, 225, 175, 125, 75, 25, ...]
+ let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+ pd.set_reqs_sec(400);
+ assert_eq!(pd.get_limit(), 200);
+
+ pd.heap.write().unwrap().move_to(241);
+
+ assert_eq!(pd.heap.read().unwrap().value(), 41);
+ pd.adjust_up(&0);
+ assert_eq!(pd.heap.read().unwrap().value(), 43);
+ assert_eq!(pd.limit.load(Ordering::Relaxed), 43);
+ assert_eq!(pd.remove_limit.load(Ordering::Relaxed), false);
+ }
+
+ #[test]
+ /// PolicyData adjust_up sets the limit to the correct value
+ fn policy_data_adjust_up_with_no_children_3_moves() {
+ // original: 400
+ // [200, 300, 100, 350, 250, 150, 50, 375, 325, 275, 225, 175, 125, 75, 25, ...]
+ let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+ pd.set_reqs_sec(400);
+ assert_eq!(pd.get_limit(), 200);
+
+ pd.heap.write().unwrap().move_to(240);
+
+ assert_eq!(pd.heap.read().unwrap().value(), 45);
+ pd.adjust_up(&0);
+ assert_eq!(pd.heap.read().unwrap().value(), 37);
+ assert_eq!(pd.limit.load(Ordering::Relaxed), 37);
+ assert_eq!(pd.remove_limit.load(Ordering::Relaxed), false);
+ }
+
+ #[test]
+ /// hit some of the out of the way corners of limitheap for coverage
+ fn increase_limit_heap_coverage_by_hitting_edge_cases() {
+ let pd = PolicyData::new(RequesterPolicy::AutoBail, 7);
+ pd.set_reqs_sec(400);
+
+ println!("{:?}", pd.heap.read().unwrap()); // debug derivation
+
+ pd.heap.write().unwrap().move_to(240);
+ assert_eq!(pd.heap.write().unwrap().move_right(), 240);
+ assert_eq!(pd.heap.write().unwrap().move_left(), 240);
+
+ pd.heap.write().unwrap().move_to(0);
+ assert_eq!(pd.heap.write().unwrap().move_up(), 0);
+ assert_eq!(pd.heap.write().unwrap().parent_value(), 400);
+ }
+}
diff --git a/src/scanner/requester.rs b/src/scanner/requester.rs
new file mode 100644
index 0000000..41dbbb9
--- /dev/null
+++ b/src/scanner/requester.rs
@@ -0,0 +1,1014 @@
+use std::{
+ cmp::max,
+ sync::{atomic::Ordering, Arc, Mutex},
+};
+
+use anyhow::Result;
+use leaky_bucket::LeakyBucket;
+use tokio::{
+ sync::{oneshot, RwLock},
+ time::{sleep, Duration},
+};
+
+use crate::{
+ atomic_load, atomic_store,
+ config::RequesterPolicy,
+ event_handlers::{
+ Command::{self, AddError, SubtractFromUsizeField},
+ Handles,
+ },
+ extractor::{ExtractionTarget::ResponseBody, ExtractorBuilder},
+ response::FeroxResponse,
+ scan_manager::{FeroxScan, ScanStatus},
+ statistics::{StatError::Other, StatField::TotalExpected},
+ url::FeroxUrl,
+ utils::logged_request,
+ HIGH_ERROR_RATIO,
+};
+
+use super::{policy_data::PolicyData, FeroxScanner, PolicyTrigger};
+
+/// Makes multiple requests based on the presence of extensions
+pub(super) struct Requester {
+ /// handles to handlers and config
+ handles: Arc,
+
+ /// url that will be scanned
+ target_url: String,
+
+ /// limits requests per second if present
+ rate_limiter: RwLock