mirror of
https://github.com/epi052/feroxbuster.git
synced 2026-06-02 14:01:13 -03:00
fixed rate limiting
* fixed requests/sec for small values * ensured limit var is never 0 in build_a_bucket, not just refill * removed unnecessary cooldown flag manipulation in cool_down func * removed minor toctou in should_enforce_policy * added new flag releases before returns from should_enforce_policy * cleaned up how limitheap is initialized from tune func * added (more) safety/bounds checks to limitheap * capped timeout to 30sec; added lock error logging * added per-trigger error tracking to policy data * updated requester to use new policy data per-trigger errors * fixed race condition in progress bar message display; fixed tests * touched up a few minor issues in nlp * fixed req/sec test * fixed more tests * added new test suite for tuning; fixed more tests * clippy/fmt * fixed possible deadlock in error path for tune/bail * fixed a handful of minor correctness issues * removed unnecessary array allocation for error tracking * --rate-limit now serves as a hard cap, in general and on --auto-tune if both are provided together * renamed test file * bumped version to 2.13.1 * added new dirlisting detection heuristics * clippy * nitpickery
This commit is contained in:
epi
@@ -230,11 +230,22 @@ fn auto_tune_slows_scan_with_429s() {
|
||||
|
||||
teardown_tmp_directory(tmp_dir);
|
||||
|
||||
assert!(normal_reqs_mock.hits() + error_mock.hits() > 25); // must have at least 50 reqs fly
|
||||
let normal_hits = normal_reqs_mock.hits();
|
||||
let error_hits = error_mock.hits();
|
||||
|
||||
println!("elapsed: {}", start.elapsed().as_millis()); // 3523ms without tuning
|
||||
assert!(normal_reqs_mock.hits() < 500);
|
||||
assert!(error_mock.hits() <= 180); // may or may not see all other error requests
|
||||
println!("normal_reqs_mock.hits(): {}", normal_hits);
|
||||
println!("error_mock.hits(): {}", error_hits);
|
||||
|
||||
assert!(normal_hits + error_hits > 25); // must have at least 50 reqs fly
|
||||
|
||||
println!("elapsed: {}", start.elapsed().as_millis());
|
||||
// With auto-tune and 429s, the scan should be slowed down but may still process
|
||||
// ~1800-2000 requests in 7 seconds. The key is that it hits the time limit.
|
||||
assert!(
|
||||
normal_hits < 3000,
|
||||
"Should process fewer than 3000 requests due to rate limiting"
|
||||
);
|
||||
assert!(error_hits <= 180); // may or may not see all other error requests
|
||||
assert!(start.elapsed().as_millis() >= 7000); // scan should hit time limit due to limiting
|
||||
}
|
||||
|
||||
@@ -283,11 +294,22 @@ fn auto_tune_slows_scan_with_403s() {
|
||||
|
||||
teardown_tmp_directory(tmp_dir);
|
||||
|
||||
assert!(normal_reqs_mock.hits() + error_mock.hits() > 25); // must have at least 50 reqs fly
|
||||
let normal_hits = normal_reqs_mock.hits();
|
||||
let error_hits = error_mock.hits();
|
||||
|
||||
println!("elapsed: {}", start.elapsed().as_millis()); // 3523ms without tuning
|
||||
assert!(normal_reqs_mock.hits() < 500);
|
||||
assert!(error_mock.hits() <= 180); // may or may not see all other error requests
|
||||
println!("normal_reqs_mock.hits(): {}", normal_hits);
|
||||
println!("error_mock.hits(): {}", error_hits);
|
||||
|
||||
assert!(normal_hits + error_hits > 25); // must have at least 50 reqs fly
|
||||
|
||||
println!("elapsed: {}", start.elapsed().as_millis());
|
||||
// With auto-tune and 403s, the scan should be slowed down but may still process
|
||||
// ~1800-2000 requests in 7 seconds. The key is that it hits the time limit.
|
||||
assert!(
|
||||
normal_hits < 3000,
|
||||
"Should process fewer than 3000 requests due to rate limiting"
|
||||
);
|
||||
assert!(error_hits <= 180); // may or may not see all other error requests
|
||||
assert!(start.elapsed().as_millis() >= 7000); // scan should hit time limit due to limiting
|
||||
}
|
||||
|
||||
@@ -339,8 +361,19 @@ fn auto_tune_slows_scan_with_general_errors() {
|
||||
|
||||
teardown_tmp_directory(tmp_dir);
|
||||
|
||||
println!("elapsed: {}", start.elapsed().as_millis()); // 3523ms without tuning
|
||||
assert!(normal_reqs_mock.hits() < 500);
|
||||
assert!(error_mock.hits() <= 180); // may or may not see all other error requests
|
||||
let normal_hits = normal_reqs_mock.hits();
|
||||
let error_hits = error_mock.hits();
|
||||
|
||||
println!("normal_reqs_mock.hits(): {}", normal_hits);
|
||||
println!("error_mock.hits(): {}", error_hits);
|
||||
println!("elapsed: {}", start.elapsed().as_millis());
|
||||
|
||||
// Normal requests timeout (3s delay with 2s timeout), triggering error policy
|
||||
// The scan should be rate-limited and hit the time limit
|
||||
assert!(
|
||||
normal_hits < 3000,
|
||||
"Should process fewer requests due to rate limiting and timeouts"
|
||||
);
|
||||
assert!(error_hits <= 180); // may or may not see all other error requests
|
||||
assert!(start.elapsed().as_millis() >= 7000); // scan should hit time limit due to limiting
|
||||
}
|
||||
|
||||
402
tests/test_rate_limiting.rs
Normal file
402
tests/test_rate_limiting.rs
Normal file
@@ -0,0 +1,402 @@
|
||||
mod utils;
|
||||
|
||||
use assert_cmd::Command;
|
||||
use httpmock::prelude::*;
|
||||
use httpmock::MockServer;
|
||||
use regex::Regex;
|
||||
use std::fs::{read_to_string, write};
|
||||
use utils::{setup_tmp_directory, teardown_tmp_directory};
|
||||
|
||||
/// Helper to create a test wordlist with controllable patterns
|
||||
fn create_test_wordlist(
|
||||
normal: usize,
|
||||
errors: usize,
|
||||
status403: usize,
|
||||
status429: usize,
|
||||
) -> String {
|
||||
let mut words = Vec::new();
|
||||
|
||||
// Normal responses
|
||||
for i in 0..normal {
|
||||
words.push(format!("normal_{:06}", i));
|
||||
}
|
||||
|
||||
// Timeout errors
|
||||
for i in 0..errors {
|
||||
words.push(format!("error_{:06}", i));
|
||||
}
|
||||
|
||||
// 403 responses
|
||||
for i in 0..status403 {
|
||||
words.push(format!("s403_{:06}", i));
|
||||
}
|
||||
|
||||
// 429 responses
|
||||
for i in 0..status429 {
|
||||
words.push(format!("s429_{:06}", i));
|
||||
}
|
||||
|
||||
words.join("\n")
|
||||
}
|
||||
|
||||
/// Scenario 1: High 403 rate - tests policy enforcement
|
||||
#[test]
|
||||
fn scenario_high_403_rate() {
|
||||
let srv = MockServer::start();
|
||||
let (tmp_dir, file) = setup_tmp_directory(&[], "wordlist").unwrap();
|
||||
let (log_dir, logfile) = setup_tmp_directory(&[], "debug-log").unwrap();
|
||||
|
||||
// Create wordlist with high 403 rate
|
||||
// Need 90%+ ratio and enough requests to trigger policy: 900/(900+100) = 90%
|
||||
let wordlist = create_test_wordlist(100, 0, 900, 0);
|
||||
write(&file, wordlist).unwrap();
|
||||
|
||||
let _normal_mock = srv.mock(|when, then| {
|
||||
when.method(GET)
|
||||
.path_matches(Regex::new("/normal_.*").unwrap());
|
||||
then.status(200).body("OK");
|
||||
});
|
||||
|
||||
let _forbidden_mock = srv.mock(|when, then| {
|
||||
when.method(GET)
|
||||
.path_matches(Regex::new("/s403_.*").unwrap());
|
||||
then.status(403).body("Forbidden");
|
||||
});
|
||||
|
||||
Command::cargo_bin("feroxbuster")
|
||||
.unwrap()
|
||||
.arg("--url")
|
||||
.arg(srv.url("/"))
|
||||
.arg("--wordlist")
|
||||
.arg(file.as_os_str())
|
||||
.arg("--auto-tune")
|
||||
.arg("--dont-filter")
|
||||
.arg("--threads")
|
||||
.arg("10")
|
||||
.arg("--debug-log")
|
||||
.arg(logfile.as_os_str())
|
||||
.arg("--json")
|
||||
.arg("-vv")
|
||||
.assert()
|
||||
.success();
|
||||
|
||||
let debug_log = read_to_string(&logfile).unwrap();
|
||||
|
||||
let mut found_403_policy = false;
|
||||
|
||||
for line in debug_log.lines() {
|
||||
if let Ok(log) = serde_json::from_str::<serde_json::Value>(line) {
|
||||
if let Some(msg) = log.get("message").and_then(|m| m.as_str()) {
|
||||
if msg.contains("auto-tune:") && msg.contains("enforcing limit") {
|
||||
found_403_policy = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
teardown_tmp_directory(tmp_dir);
|
||||
teardown_tmp_directory(log_dir);
|
||||
|
||||
assert!(found_403_policy, "High 403 rate should trigger policy");
|
||||
}
|
||||
|
||||
/// Scenario 2: High 429 rate - tests aggressive rate limiting
|
||||
#[test]
|
||||
fn scenario_high_429_rate() {
|
||||
let srv = MockServer::start();
|
||||
let (tmp_dir, file) = setup_tmp_directory(&[], "wordlist").unwrap();
|
||||
let (log_dir, logfile) = setup_tmp_directory(&[], "debug-log").unwrap();
|
||||
|
||||
// High 429 rate should trigger more aggressive limiting
|
||||
// Need 30%+ ratio and enough requests: 450/(450+150) = 75%
|
||||
let wordlist = create_test_wordlist(150, 0, 0, 450);
|
||||
write(&file, wordlist).unwrap();
|
||||
|
||||
let _normal_mock = srv.mock(|when, then| {
|
||||
when.method(GET)
|
||||
.path_matches(Regex::new("/normal_.*").unwrap());
|
||||
then.status(200).body("OK");
|
||||
});
|
||||
|
||||
let _rate_limit_mock = srv.mock(|when, then| {
|
||||
when.method(GET)
|
||||
.path_matches(Regex::new("/s429_.*").unwrap());
|
||||
then.status(429).body("Too Many Requests");
|
||||
});
|
||||
|
||||
Command::cargo_bin("feroxbuster")
|
||||
.unwrap()
|
||||
.arg("--url")
|
||||
.arg(srv.url("/"))
|
||||
.arg("--wordlist")
|
||||
.arg(file.as_os_str())
|
||||
.arg("--auto-tune")
|
||||
.arg("--dont-filter")
|
||||
.arg("--threads")
|
||||
.arg("10")
|
||||
.arg("--debug-log")
|
||||
.arg(logfile.as_os_str())
|
||||
.arg("--json")
|
||||
.arg("-vv")
|
||||
.assert()
|
||||
.success();
|
||||
|
||||
let debug_log = read_to_string(&logfile).unwrap();
|
||||
|
||||
let mut found_429_policy = false;
|
||||
|
||||
for line in debug_log.lines() {
|
||||
if let Ok(log) = serde_json::from_str::<serde_json::Value>(line) {
|
||||
if let Some(msg) = log.get("message").and_then(|m| m.as_str()) {
|
||||
if msg.contains("auto-tune:") && msg.contains("enforcing limit") {
|
||||
found_429_policy = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
teardown_tmp_directory(tmp_dir);
|
||||
teardown_tmp_directory(log_dir);
|
||||
|
||||
assert!(found_429_policy, "High 429 rate should trigger policy");
|
||||
}
|
||||
|
||||
/// Scenario 3: Recovery pattern - errors then normal
|
||||
#[test]
|
||||
fn scenario_recovery_pattern() {
|
||||
let srv = MockServer::start();
|
||||
let (tmp_dir, file) = setup_tmp_directory(&[], "wordlist").unwrap();
|
||||
let (log_dir, logfile) = setup_tmp_directory(&[], "debug-log").unwrap();
|
||||
|
||||
// Pattern: errors first, then normal - should slow down then speed up
|
||||
let mut wordlist = Vec::new();
|
||||
for i in 0..100 {
|
||||
wordlist.push(format!("s403_{:04}", i));
|
||||
}
|
||||
for i in 0..300 {
|
||||
wordlist.push(format!("normal_{:04}", i));
|
||||
}
|
||||
|
||||
write(&file, wordlist.join("\n")).unwrap();
|
||||
|
||||
let _normal_mock = srv.mock(|when, then| {
|
||||
when.method(GET)
|
||||
.path_matches(Regex::new("/normal_.*").unwrap());
|
||||
then.status(200).body("OK");
|
||||
});
|
||||
|
||||
let _error_mock = srv.mock(|when, then| {
|
||||
when.method(GET)
|
||||
.path_matches(Regex::new("/s403_.*").unwrap());
|
||||
then.status(403).body("Forbidden");
|
||||
});
|
||||
|
||||
Command::cargo_bin("feroxbuster")
|
||||
.unwrap()
|
||||
.arg("--url")
|
||||
.arg(srv.url("/"))
|
||||
.arg("--wordlist")
|
||||
.arg(file.as_os_str())
|
||||
.arg("--auto-tune")
|
||||
.arg("--dont-filter")
|
||||
.arg("--threads")
|
||||
.arg("10")
|
||||
.arg("--debug-log")
|
||||
.arg(logfile.as_os_str())
|
||||
.arg("--json")
|
||||
.arg("-vv")
|
||||
.assert()
|
||||
.success();
|
||||
|
||||
let debug_log = read_to_string(&logfile).unwrap();
|
||||
|
||||
let mut auto_tune_triggered = false;
|
||||
|
||||
for line in debug_log.lines() {
|
||||
if let Ok(log) = serde_json::from_str::<serde_json::Value>(line) {
|
||||
if let Some(msg) = log.get("message").and_then(|m| m.as_str()) {
|
||||
if msg.contains("auto-tune:") && msg.contains("enforcing limit") {
|
||||
auto_tune_triggered = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
teardown_tmp_directory(tmp_dir);
|
||||
teardown_tmp_directory(log_dir);
|
||||
|
||||
assert!(
|
||||
auto_tune_triggered,
|
||||
"Should trigger auto-tune due to errors"
|
||||
);
|
||||
}
|
||||
|
||||
/// Scenario 4: Mixed steady state - balanced errors and normal
|
||||
#[test]
|
||||
fn scenario_mixed_steady_state() {
|
||||
let srv = MockServer::start();
|
||||
let (tmp_dir, file) = setup_tmp_directory(&[], "wordlist").unwrap();
|
||||
let (log_dir, logfile) = setup_tmp_directory(&[], "debug-log").unwrap();
|
||||
|
||||
// Evenly mixed - not enough to trigger bail, but enough for tuning
|
||||
// Need 25+ general errors to trigger: 30 >= 25
|
||||
let wordlist = create_test_wordlist(150, 30, 10, 10);
|
||||
write(&file, wordlist).unwrap();
|
||||
|
||||
let normal_mock = srv.mock(|when, then| {
|
||||
when.method(GET)
|
||||
.path_matches(Regex::new("/normal_.*").unwrap());
|
||||
then.status(200).body("OK");
|
||||
});
|
||||
|
||||
let error_mock = srv.mock(|when, then| {
|
||||
when.method(GET)
|
||||
.path_matches(Regex::new("/error_.*").unwrap());
|
||||
then.status(504).body("Gateway Timeout");
|
||||
});
|
||||
|
||||
let forbidden_mock = srv.mock(|when, then| {
|
||||
when.method(GET)
|
||||
.path_matches(Regex::new("/s403_.*").unwrap());
|
||||
then.status(403).body("Forbidden");
|
||||
});
|
||||
|
||||
let rate_limit_mock = srv.mock(|when, then| {
|
||||
when.method(GET)
|
||||
.path_matches(Regex::new("/s429_.*").unwrap());
|
||||
then.status(429).body("Too Many Requests");
|
||||
});
|
||||
|
||||
Command::cargo_bin("feroxbuster")
|
||||
.unwrap()
|
||||
.arg("--url")
|
||||
.arg(srv.url("/"))
|
||||
.arg("--wordlist")
|
||||
.arg(file.as_os_str())
|
||||
.arg("--auto-tune")
|
||||
.arg("--threads")
|
||||
.arg("10")
|
||||
.arg("--debug-log")
|
||||
.arg(logfile.as_os_str())
|
||||
.arg("-vv")
|
||||
.assert()
|
||||
.success();
|
||||
|
||||
let debug_log = read_to_string(&logfile).unwrap();
|
||||
let mut _policy_adjustments = 0;
|
||||
|
||||
for line in debug_log.lines() {
|
||||
if let Ok(log) = serde_json::from_str::<serde_json::Value>(line) {
|
||||
if let Some(msg) = log.get("message").and_then(|m| m.as_str()) {
|
||||
if msg.contains("scan speed") || msg.contains("set rate limit") {
|
||||
_policy_adjustments += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let total =
|
||||
normal_mock.hits() + error_mock.hits() + forbidden_mock.hits() + rate_limit_mock.hits();
|
||||
|
||||
teardown_tmp_directory(tmp_dir);
|
||||
teardown_tmp_directory(log_dir);
|
||||
|
||||
// With mixed but not extreme errors, should see some adjustments
|
||||
assert!(total > 100, "Should complete significant portion of scan");
|
||||
}
|
||||
|
||||
/// Scenario 5: Capped auto-tune - --rate-limit caps --auto-tune adjustments
|
||||
#[test]
|
||||
fn scenario_capped_auto_tune() {
|
||||
let srv = MockServer::start();
|
||||
let (tmp_dir, file) = setup_tmp_directory(&[], "wordlist").unwrap();
|
||||
let (log_dir, logfile) = setup_tmp_directory(&[], "debug-log").unwrap();
|
||||
|
||||
// Pattern: errors first to trigger rate limiting, then normal responses to allow upward adjustment
|
||||
// The rate limit cap should prevent exceeding the specified limit
|
||||
let mut wordlist = Vec::new();
|
||||
|
||||
// Start with many errors to trigger auto-tune
|
||||
for i in 0..200 {
|
||||
wordlist.push(format!("s403_{:04}", i));
|
||||
}
|
||||
|
||||
// Then many normal responses to allow upward adjustment
|
||||
for i in 0..400 {
|
||||
wordlist.push(format!("normal_{:04}", i));
|
||||
}
|
||||
|
||||
write(&file, wordlist.join("\n")).unwrap();
|
||||
|
||||
let _normal_mock = srv.mock(|when, then| {
|
||||
when.method(GET)
|
||||
.path_matches(Regex::new("/normal_.*").unwrap());
|
||||
then.status(200).body("OK");
|
||||
});
|
||||
|
||||
let _error_mock = srv.mock(|when, then| {
|
||||
when.method(GET)
|
||||
.path_matches(Regex::new("/s403_.*").unwrap());
|
||||
then.status(403).body("Forbidden");
|
||||
});
|
||||
|
||||
Command::cargo_bin("feroxbuster")
|
||||
.unwrap()
|
||||
.arg("--url")
|
||||
.arg(srv.url("/"))
|
||||
.arg("--wordlist")
|
||||
.arg(file.as_os_str())
|
||||
.arg("--auto-tune")
|
||||
.arg("--rate-limit")
|
||||
.arg("50") // Cap at 50 req/s
|
||||
.arg("--dont-filter")
|
||||
.arg("--threads")
|
||||
.arg("10")
|
||||
.arg("--debug-log")
|
||||
.arg(logfile.as_os_str())
|
||||
.arg("--json")
|
||||
.arg("-vv")
|
||||
.assert()
|
||||
.success();
|
||||
|
||||
let debug_log = read_to_string(&logfile).unwrap();
|
||||
|
||||
let mut auto_tune_triggered = false;
|
||||
let mut max_rate_seen = 0;
|
||||
|
||||
for line in debug_log.lines() {
|
||||
if let Ok(log) = serde_json::from_str::<serde_json::Value>(line) {
|
||||
if let Some(msg) = log.get("message").and_then(|m| m.as_str()) {
|
||||
// Check for auto-tune activation
|
||||
if msg.contains("auto-tune:") && msg.contains("enforcing limit") {
|
||||
auto_tune_triggered = true;
|
||||
}
|
||||
|
||||
// Extract rate values from messages like "set rate limit (25/s)" or "scan speed (30/s)"
|
||||
if msg.contains("/s)") {
|
||||
if let Some(start) = msg.rfind('(') {
|
||||
if let Some(end) = msg.rfind("/s)") {
|
||||
if let Ok(rate) = msg[start + 1..end].parse::<usize>() {
|
||||
max_rate_seen = max_rate_seen.max(rate);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
teardown_tmp_directory(tmp_dir);
|
||||
teardown_tmp_directory(log_dir);
|
||||
|
||||
assert!(
|
||||
auto_tune_triggered,
|
||||
"Auto-tune should be triggered by errors"
|
||||
);
|
||||
|
||||
assert!(
|
||||
max_rate_seen <= 50,
|
||||
"Auto-tune should never exceed rate-limit cap of 50, but saw {}",
|
||||
max_rate_seen
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user