h/feroxbuster
1
Fork 0
mirror of https://github.com/epi052/feroxbuster.git synced 2026-06-07 10:01:12 -03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #187 from epi052/2.0-statistics-restructure

Browse Source 
2.0 statistics restructure
This commit is contained in:
epi
2021-01-14 12:21:37 -06:00
committed by GitHub
parent 03282ed4af e7b3c9f7c0
commit df28827c5d
23 changed files with 1213 additions and 989 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Cargo.toml
View File

@@ -42,6 +42,7 @@ crossterm = "0.19"
rlimit = "0.5"
ctrlc = "3.1"
fuzzyhash = "0.2"
anyhow = "1.0"
[dev-dependencies]
tempfile = "3.1"

1089
src/banner.rs
View File

File diff suppressed because it is too large Load Diff

17
src/config.rs
View File

@@ -2,9 +2,10 @@ use crate::{
client, parser,
progress::{add_bar, BarType},
scan_manager::resume_scan,
utils::{module_colorizer, status_colorizer},
utils::{fmt_err, module_colorizer, status_colorizer},
FeroxSerialize, DEFAULT_CONFIG_NAME, DEFAULT_STATUS_CODES, DEFAULT_WORDLIST, VERSION,
};
use anyhow::{Context, Result};
use clap::{value_t, ArgMatches};
use indicatif::{MultiProgress, ProgressBar, ProgressDrawTarget};
use lazy_static::lazy_static;
@@ -875,13 +876,11 @@ impl FeroxSerialize for Configuration {
/// ],
/// ...
/// }\n
fn as_json(&self) -> String {
if let Ok(mut json) = serde_json::to_string(&self) {
json.push('\n');
json
} else {
String::from("{\"error\":\"could not Configuration convert to json\"}")
}
fn as_json(&self) -> Result<String> {
let mut json = serde_json::to_string(&self)
.with_context(|| fmt_err("Could not convert Configuration to JSON"))?;
json.push('\n');
Ok(json)
}
}
@@ -1238,7 +1237,7 @@ mod tests {
let mut config = Configuration::new();
config.timeout = 12;
config.depth = 2;
let config_str = config.as_json();
let config_str = config.as_json().unwrap();
let json: Configuration = serde_json::from_str(&config_str).unwrap();
assert_eq!(json.config, config.config);
assert_eq!(json.wordlist, config.wordlist);

2
src/event_handlers/mod.rs Normal file
View File

@@ -0,0 +1,2 @@
mod statistics;
pub use statistics::StatsHandler;

106
src/event_handlers/statistics.rs Normal file
View File

@@ -0,0 +1,106 @@
use crate::{
config::CONFIGURATION,
progress::{add_bar, BarType},
statistics::{StatCommand, StatField, Stats},
};
use anyhow::Result;
use console::style;
use indicatif::ProgressBar;
use std::{sync::Arc, time::Instant};
use tokio::sync::mpsc::UnboundedReceiver;
/// event handler struct for updating statistics
#[derive(Debug)]
pub struct StatsHandler {
/// overall scan's progress bar
bar: ProgressBar,
/// Receiver half of mpsc from which `StatCommand`s are processed
receiver: UnboundedReceiver<StatCommand>,
/// data class that stores all statistics updates
stats: Arc<Stats>,
}
/// implementation of event handler for statistics
impl StatsHandler {
/// create new event handler builder
pub fn new(stats: Arc<Stats>, rx_stats: UnboundedReceiver<StatCommand>) -> Self {
// will be updated later via StatCommand; delay is for banner to print first
let bar = ProgressBar::hidden();
Self {
bar,
stats,
receiver: rx_stats,
}
}
/// Start a single consumer task (sc side of mpsc)
///
/// The consumer simply receives `StatCommands` and updates the given `Stats` object as appropriate
pub async fn start(&mut self) -> Result<()> {
log::trace!("enter: start({:?})", self);
let start = Instant::now();
while let Some(command) = self.receiver.recv().await {
match command as StatCommand {
StatCommand::AddError(err) => {
self.stats.add_error(err);
self.increment_bar();
}
StatCommand::AddStatus(status) => {
self.stats.add_status_code(status);
self.increment_bar();
}
StatCommand::AddRequest => {
self.stats.add_request();
self.increment_bar();
}
StatCommand::Save => {
self.stats
.save(start.elapsed().as_secs_f64(), &CONFIGURATION.output)?;
}
StatCommand::UpdateUsizeField(field, value) => {
let update_len = matches!(field, StatField::TotalScans);
self.stats.update_usize_field(field, value);
if update_len {
self.bar.set_length(self.stats.total_expected() as u64)
}
}
StatCommand::UpdateF64Field(field, value) => {
self.stats.update_f64_field(field, value)
}
StatCommand::CreateBar => {
self.bar = add_bar("", self.stats.total_expected() as u64, BarType::Total);
}
StatCommand::LoadStats(filename) => {
self.stats.merge_from(&filename)?;
}
StatCommand::Exit => break,
}
}
self.bar.finish();
log::debug!("{:#?}", *self.stats);
log::trace!("exit: start");
Ok(())
}
/// Wrapper around incrementing the overall scan's progress bar
fn increment_bar(&self) {
let msg = format!(
"{}:{:<7} {}:{:<7}",
style("found").green(),
self.stats.resources_discovered(),
style("errors").red(),
self.stats.errors(),
);
self.bar.set_message(&msg);
self.bar.inc(1);
}
}

3
src/filters/mod.rs
View File

@@ -2,7 +2,6 @@
mod traits;
mod wildcard;
mod status_code;
mod shared;
mod words;
mod lines;
mod size;
@@ -23,5 +22,3 @@ pub use self::words::WordsFilter;
use crate::{config::CONFIGURATION, utils::get_url_path_length, FeroxResponse, FeroxSerialize};
use std::any::Any;
use std::fmt::Debug;
// try using pub(in self) or w/e

62
src/lib.rs
View File

@@ -12,8 +12,10 @@ pub mod reporter;
pub mod scan_manager;
pub mod scanner;
pub mod statistics;
mod event_handlers;
use crate::utils::{get_url_path_length, status_colorizer};
use crate::utils::{fmt_err, get_url_path_length, status_colorizer};
use anyhow::{Context, Result};
use console::{style, Color};
use reqwest::header::{HeaderName, HeaderValue};
use reqwest::{header::HeaderMap, Response, StatusCode, Url};
@@ -102,7 +104,7 @@ pub trait FeroxSerialize: Serialize {
fn as_str(&self) -> String;
/// Return an NDJSON representation of the object
fn as_json(&self) -> String;
fn as_json(&self) -> Result<String>;
}
/// A `FeroxResponse`, derived from a `Response` to a submitted `Request`
@@ -342,13 +344,11 @@ impl FeroxSerialize for FeroxResponse {
/// "access-control-allow-origin":"https://localhost.com"
/// }
/// }\n
fn as_json(&self) -> String {
if let Ok(mut json) = serde_json::to_string(&self) {
json.push('\n');
json
} else {
format!("{{\"error\":\"could not convert {} to json\"}}", self.url())
}
fn as_json(&self) -> Result<String> {
let mut json = serde_json::to_string(&self)
.with_context(|| fmt_err(&format!("Could not convert {} to JSON", self.url())))?;
json.push('\n');
Ok(json)
}
}
@@ -487,26 +487,6 @@ pub struct FeroxMessage {
/// Implementation of FeroxMessage
impl FeroxSerialize for FeroxMessage {
/// Create an NDJSON representation of the log message
///
/// (expanded for clarity)
/// ex:
/// {
/// "type": "log",
/// "message": "Sent https://localhost/api to file handler",
/// "level": "DEBUG",
/// "time_offset": 0.86333454,
/// "module": "feroxbuster::reporter"
/// }\n
fn as_json(&self) -> String {
if let Ok(mut json) = serde_json::to_string(&self) {
json.push('\n');
json
} else {
String::from("{\"error\":\"could not convert to json\"}")
}
}
/// Create a string representation of the log message
///
/// ex: 301 10l 16w 173c https://localhost/api
@@ -529,6 +509,28 @@ impl FeroxSerialize for FeroxMessage {
style(&self.message).dim(),
)
}
/// Create an NDJSON representation of the log message
///
/// (expanded for clarity)
/// ex:
/// {
/// "type": "log",
/// "message": "Sent https://localhost/api to file handler",
/// "level": "DEBUG",
/// "time_offset": 0.86333454,
/// "module": "feroxbuster::reporter"
/// }\n
fn as_json(&self) -> Result<String> {
let mut json = serde_json::to_string(&self).with_context(|| {
fmt_err(&format!(
"Could not convert {}:{} to JSON",
self.level, self.message
))
})?;
json.push('\n');
Ok(json)
}
}
#[cfg(test)]
@@ -586,7 +588,7 @@ mod tests {
kind: "log".to_string(),
};
let message_str = message.as_json();
let message_str = message.as_json().unwrap();
let error_margin = f32::EPSILON;

77
src/main.rs
View File

@@ -1,6 +1,7 @@
use anyhow::{bail, Context, Result};
use crossterm::event::{self, Event, KeyCode};
use feroxbuster::{
banner,
banner::{Banner, UPDATE_URL},
config::{CONFIGURATION, PROGRESS_BAR, PROGRESS_PRINTER},
heuristics, logger,
progress::{add_bar, BarType},
@@ -14,8 +15,8 @@ use feroxbuster::{
Stats,
},
update_stat,
utils::{ferox_print, get_current_depth, module_colorizer, status_colorizer},
FeroxError, FeroxResponse, FeroxResult, SLEEP_DURATION, VERSION,
utils::{ferox_print, fmt_err, get_current_depth, status_colorizer},
FeroxError, FeroxResponse, FeroxResult, SLEEP_DURATION,
};
#[cfg(not(target_os = "windows"))]
use feroxbuster::{utils::set_open_file_limit, DEFAULT_OPEN_FILE_LIMIT};
@@ -245,7 +246,7 @@ async fn get_targets() -> FeroxResult<Vec<String>> {
/// async main called from real main, broken out in this way to allow for some synchronous code
/// to be executed before bringing the tokio runtime online
async fn wrapped_main() {
async fn wrapped_main() -> Result<()> {
// join can only be called once, otherwise it causes the thread to panic
tokio::task::spawn_blocking(move || {
// ok, lazy_static! uses (unsurprisingly in retrospect) a lazy loading model where the
@@ -295,7 +296,6 @@ async fn wrapped_main() {
Ok(t) => t,
Err(e) => {
// should only happen in the event that there was an error reading from stdin
log::error!("{} {}", module_colorizer("main::get_targets"), e);
clean_up(
tx_term,
term_handle,
@@ -305,8 +305,8 @@ async fn wrapped_main() {
stats_handle,
save_output,
)
.await;
return;
.await?;
bail!("Could not get determine initial targets: {}", e);
}
};
@@ -315,14 +315,15 @@ async fn wrapped_main() {
if !CONFIGURATION.quiet {
// only print banner if -q isn't used
let std_stderr = stderr(); // std::io::stderr
banner::initialize(
&targets,
&CONFIGURATION,
&VERSION,
std_stderr,
tx_stats.clone(),
)
.await;
let mut banner = Banner::new(&targets, &CONFIGURATION);
// only interested in the side-effect that sets banner.update_status
let _ = banner
.check_for_updates(&CONFIGURATION.client, UPDATE_URL, tx_stats.clone())
.await;
banner.print_to(std_stderr, &CONFIGURATION)?;
}
// discard non-responsive targets
@@ -338,8 +339,8 @@ async fn wrapped_main() {
stats_handle,
save_output,
)
.await;
return;
.await?;
bail!(fmt_err("Could not find any live targets to scan"));
}
// kick off a scan against any targets determined to be responsive
@@ -356,6 +357,7 @@ async fn wrapped_main() {
log::info!("All scans complete!");
}
Err(e) => {
// todo status colorizer here and print is likely not needed
ferox_print(
&format!("{} while scanning: {}", status_colorizer("Error"), e),
&PROGRESS_PRINTER,
@@ -369,7 +371,8 @@ async fn wrapped_main() {
stats_handle,
save_output,
)
.await;
.await?;
// todo bail?
process::exit(1);
}
};
@@ -383,9 +386,10 @@ async fn wrapped_main() {
stats_handle,
save_output,
)
.await;
.await?;
log::trace!("exit: wrapped_main");
Ok(())
}
/// Single cleanup function that handles all the necessary drops/finishes etc required to gracefully
@@ -396,9 +400,9 @@ async fn clean_up(
tx_file: UnboundedSender<FeroxResponse>,
file_handle: Option<JoinHandle<()>>,
tx_stats: UnboundedSender<StatCommand>,
stats_handle: JoinHandle<()>,
stats_handle: JoinHandle<Result<()>>,
save_output: bool,
) {
) -> Result<()> {
log::trace!(
"enter: clean_up({:?}, {:?}, {:?}, {:?}, {:?}, {:?}, {})",
tx_term,
@@ -409,19 +413,14 @@ async fn clean_up(
stats_handle,
save_output
);
update_stat!(tx_stats, StatCommand::Exit); // send exit command and await the end of the future
drop(tx_term);
log::trace!("dropped terminal output handler's transmitter");
log::trace!("awaiting terminal output handler's receiver");
// after dropping tx, we can await the future where rx lived
match term_handle.await {
Ok(_) => {}
Err(e) => {
log::error!("error awaiting terminal output handler's receiver: {}", e);
}
}
term_handle
.await
.with_context(|| fmt_err("Could not await terminal output handler's receiver"))?;
log::trace!("done awaiting terminal output handler's receiver");
log::trace!("tx_file: {:?}", tx_file);
@@ -433,16 +432,17 @@ async fn clean_up(
if save_output {
// but we only await if -o was specified
log::trace!("awaiting file output handler's receiver");
match file_handle.unwrap().await {
Ok(_) => {}
Err(e) => {
log::error!("error awaiting file output handler's receiver: {}", e);
}
}
file_handle
.unwrap()
.await
.with_context(|| fmt_err("Could not await file output handler's receiver"))?;
log::trace!("done awaiting file output handler's receiver");
}
stats_handle.await.unwrap_or_default();
update_stat!(tx_stats, StatCommand::Exit); // send exit command and await the end of the future
stats_handle
.await?
.with_context(|| fmt_err("Could not await statistics handler's receiver"))?;
// mark all scans complete so the terminal input handler will exit cleanly
SCAN_COMPLETE.store(true, Ordering::Relaxed);
@@ -454,6 +454,7 @@ async fn clean_up(
drop(tx_stats);
log::trace!("exit: clean_up");
Ok(())
}
fn main() {
@@ -469,7 +470,9 @@ fn main() {
.build()
{
let future = wrapped_main();
runtime.block_on(future);
if let Err(e) = runtime.block_on(future) {
eprintln!("{}", e);
};
}
log::trace!("exit: main");

2
src/reporter.rs
View File

@@ -220,7 +220,7 @@ pub fn safe_file_write<T>(
// the second log entry being injected into the first.
let contents = if convert_to_json {
value.as_json()
value.as_json().unwrap_or_default() // todo this fn should return result
} else {
value.as_str()
};

11
src/scan_manager.rs
View File

@@ -1,3 +1,4 @@
use crate::utils::fmt_err;
use crate::{
config::{Configuration, CONFIGURATION, PROGRESS_BAR, PROGRESS_PRINTER},
parser::TIMESPEC_REGEX,
@@ -8,6 +9,7 @@ use crate::{
utils::open_file,
FeroxResponse, FeroxSerialize, SLEEP_DURATION,
};
use anyhow::{Context, Result};
use console::{measure_text_width, pad_str, style, Alignment, Term};
use indicatif::{ProgressBar, ProgressDrawTarget};
use serde::{
@@ -652,7 +654,7 @@ impl FeroxScans {
scan_type: ScanType,
stats: Arc<Stats>,
) -> (bool, Arc<Mutex<FeroxScan>>) {
let num_requests = stats.expected_per_scan.load(Ordering::Relaxed) as u64;
let num_requests = stats.expected_per_scan() as u64;
let bar = match scan_type {
ScanType::Directory => {
@@ -782,8 +784,9 @@ impl FeroxSerialize for FeroxState {
}
/// Simple call to produce a JSON string using the given FeroxState
fn as_json(&self) -> String {
serde_json::to_string(&self).unwrap_or_default()
fn as_json(&self) -> Result<String> {
Ok(serde_json::to_string(&self)
.with_context(|| fmt_err("Could not convert scan's running state to JSON"))?)
}
}
@@ -1245,7 +1248,7 @@ mod tests {
assert!(expected_strs.eval(&ferox_state.as_str()));
let json_state = ferox_state.as_json();
let json_state = ferox_state.as_json().unwrap();
let expected = format!(
r#"{{"scans":[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","status":"NotStarted","num_requests":0}}],"config":{{"type":"configuration","wordlist":"/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt","config":"","proxy":"","replay_proxy":"","target_url":"","status_codes":[200,204,301,302,307,308,401,403,405],"replay_codes":[200,204,301,302,307,308,401,403,405],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"quiet":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/{}","redirects":false,"insecure":false,"extensions":[],"headers":{{}},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":false,"depth":4,"scan_limit":0,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false,"resumed":false,"resume_from":"","save_state":false,"time_limit":"","filter_similar":[]}},"responses":[{{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{{"server":"nginx/1.16.1"}}}}]"#,
saved_id, VERSION

25
src/scanner.rs
View File

@@ -22,7 +22,7 @@ use futures::{
use fuzzyhash::FuzzyHash;
use lazy_static::lazy_static;
use regex::Regex;
use reqwest::Url;
use reqwest::{StatusCode, Url};
#[cfg(not(test))]
use std::process::exit;
use std::{
@@ -260,8 +260,9 @@ fn response_is_directory(response: &FeroxResponse) -> bool {
return false;
}
}
} else if response.status().is_success() {
// status code is 2xx, need to check if it ends in /
} else if response.status().is_success() || matches!(response.status(), &StatusCode::FORBIDDEN)
{
// status code is 2xx or 403, need to check if it ends in /
if response.url().as_str().ends_with('/') {
log::debug!("{} is directory suitable for recursion", response.url());
@@ -464,10 +465,14 @@ async fn make_requests(
if !CONFIGURATION.no_recursion {
log::debug!("Recursive extraction: {}", new_ferox_response);
if new_ferox_response.status().is_success()
&& !new_ferox_response.url().as_str().ends_with('/')
if !new_ferox_response.url().as_str().ends_with('/')
&& (new_ferox_response.status().is_success()
|| matches!(new_ferox_response.status(), &StatusCode::FORBIDDEN))
{
// since all of these are 2xx, recursion is only attempted if the
// if the url doesn't end with a /
// and the response code is either a 2xx or 403
// since all of these are 2xx or 403, recursion is only attempted if the
// url ends in a /. I am actually ok with adding the slash and not
// adding it, as both have merit. Leaving it in for now to see how
// things turn out (current as of: v1.1.0)
@@ -540,8 +545,12 @@ async fn scan_robots_txt(
} else if !CONFIGURATION.no_recursion {
log::debug!("Directory extracted from robots.txt: {}", ferox_response);
// todo this code is essentially the same as another piece around ~467 of this file
if ferox_response.status().is_success() && !ferox_response.url().as_str().ends_with('/')
if !ferox_response.url().as_str().ends_with('/')
&& (ferox_response.status().is_success()
|| matches!(ferox_response.status(), &StatusCode::FORBIDDEN))
{
// if the url doesn't end with a /
// and the response code is either a 2xx or 403
ferox_response.set_url(&format!("{}/", ferox_response.url()));
}
@@ -580,7 +589,7 @@ pub async fn scan_url(
let (tx_dir, rx_dir): FeroxChannel<String> = mpsc::unbounded_channel();
if CALL_COUNT.load(Ordering::Relaxed) < stats.initial_targets.load(Ordering::Relaxed) {
if CALL_COUNT.load(Ordering::Relaxed) < stats.initial_targets() {
CALL_COUNT.fetch_add(1, Ordering::Relaxed);
if CONFIGURATION.extract_links {

33
src/statistics/command.rs Normal file
View File

@@ -0,0 +1,33 @@
use super::{error::StatError, field::StatField};
use reqwest::StatusCode;
/// Protocol definition for updating a Stats object via mpsc
#[derive(Debug)]
pub enum StatCommand {
/// Add one to the total number of requests
AddRequest,
/// Add one to the proper field(s) based on the given `StatError`
AddError(StatError),
/// Add one to the proper field(s) based on the given `StatusCode`
AddStatus(StatusCode),
/// Create the progress bar (`BarType::Total`) that is updated from the stats thread
CreateBar,
/// Update a `Stats` field that corresponds to the given `StatField` by the given `usize` value
UpdateUsizeField(StatField, usize),
/// Update a `Stats` field that corresponds to the given `StatField` by the given `f64` value
UpdateF64Field(StatField, f64),
/// Save a `Stats` object to disk using `reporter::get_cached_file_handle`
Save,
/// Load a `Stats` object from disk
LoadStats(String),
/// Break out of the (infinite) mpsc receive loop
Exit,
}

441
src/statistics.rs → src/statistics/container.rs
View File

@@ -1,11 +1,12 @@
use super::{error::StatError, field::StatField};
use crate::utils::fmt_err;
use crate::{
config::CONFIGURATION,
progress::{add_bar, BarType},
reporter::{get_cached_file_handle, safe_file_write},
FeroxChannel, FeroxSerialize,
utils::status_colorizer,
FeroxSerialize,
};
use console::style;
use indicatif::ProgressBar;
use anyhow::{Context, Result};
use reqwest::StatusCode;
use serde::{Deserialize, Serialize};
use std::{
@@ -13,34 +14,9 @@ use std::{
io::BufReader,
sync::{
atomic::{AtomicUsize, Ordering},
Arc, Mutex,
Mutex,
},
time::Instant,
};
use tokio::{
sync::mpsc::{self, UnboundedReceiver, UnboundedSender},
task::JoinHandle,
};
/// Wrapper `Atomic*.fetch_add` to save me from writing Ordering::Relaxed a bajillion times
///
/// default is to increment by 1, second arg can be used to increment by a different value
macro_rules! atomic_increment {
($metric:expr) => {
$metric.fetch_add(1, Ordering::Relaxed);
};
($metric:expr, $value:expr) => {
$metric.fetch_add($value, Ordering::Relaxed);
};
}
/// Wrapper around `Atomic*.load` to save me from writing Ordering::Relaxed a bajillion times
macro_rules! atomic_load {
($metric:expr) => {
$metric.load(Ordering::Relaxed);
};
}
/// Data collection of statistics related to a scan
#[derive(Default, Deserialize, Debug, Serialize)]
@@ -59,7 +35,7 @@ pub struct Stats {
///
/// Note: this is a per-scan expectation; `expected_requests * current # of scans` would be
/// indicative of the current expectation at any given time, but is a moving target.
pub expected_per_scan: AtomicUsize,
expected_per_scan: AtomicUsize,
/// tracker for accumulating total number of requests expected (i.e. as a new scan is started
/// this value should increase by `expected_requests`
@@ -85,7 +61,7 @@ pub struct Stats {
total_scans: AtomicUsize,
/// tracker for initial number of requested targets
pub initial_targets: AtomicUsize,
initial_targets: AtomicUsize,
/// tracker for number of links extracted when `--extract-links` is used; sources are
/// response bodies and robots.txt as of v1.11.0
@@ -158,8 +134,8 @@ impl FeroxSerialize for Stats {
}
/// Simple call to produce a JSON string using the given Stats object
fn as_json(&self) -> String {
serde_json::to_string(&self).unwrap_or_default()
fn as_json(&self) -> Result<String> {
Ok(serde_json::to_string(&self)?)
}
}
@@ -175,8 +151,33 @@ impl Stats {
}
}
/// public getter for expected_per_scan
pub fn expected_per_scan(&self) -> usize {
atomic_load!(self.expected_per_scan)
}
/// public getter for resources_discovered
pub fn resources_discovered(&self) -> usize {
atomic_load!(self.resources_discovered)
}
/// public getter for errors
pub fn errors(&self) -> usize {
atomic_load!(self.errors)
}
/// public getter for total_expected
pub fn total_expected(&self) -> usize {
atomic_load!(self.total_expected)
}
/// public getter for initial_targets
pub fn initial_targets(&self) -> usize {
atomic_load!(self.initial_targets)
}
/// increment `requests` field by one
fn add_request(&self) {
pub fn add_request(&self) {
atomic_increment!(self.requests);
}
@@ -188,17 +189,16 @@ impl Stats {
}
/// save an instance of `Stats` to disk after updating the total runtime for the scan
fn save(&self, seconds: f64, location: &str) {
let buffered_file = match get_cached_file_handle(location) {
Some(file) => file,
None => {
return;
}
};
pub fn save(&self, seconds: f64, location: &str) -> Result<()> {
let buffered_file = get_cached_file_handle(location).with_context(|| {
format!("{}: Could not open {}", status_colorizer("ERROR"), location)
})?;
self.update_runtime(seconds);
safe_file_write(self, buffered_file, CONFIGURATION.json);
Ok(())
}
/// Inspect the given `StatError` and increment the appropriate fields
@@ -242,7 +242,7 @@ impl Stats {
/// - requests
/// - status_403s (when code is 403)
/// - errors (when code is [45]xx)
fn add_status_code(&self, status: StatusCode) {
pub fn add_status_code(&self, status: StatusCode) {
self.add_request();
if status.is_success() {
@@ -291,7 +291,7 @@ impl Stats {
}
/// Update a `Stats` field of type f64
fn update_f64_field(&self, field: StatField, value: f64) {
pub fn update_f64_field(&self, field: StatField, value: f64) {
if let StatField::DirScanTimes = field {
if let Ok(mut locked_times) = self.directory_scan_times.lock() {
locked_times.push(value);
@@ -300,7 +300,7 @@ impl Stats {
}
/// Update a `Stats` field of type usize
fn update_usize_field(&self, field: StatField, value: usize) {
pub fn update_usize_field(&self, field: StatField, value: usize) {
match field {
StatField::ExpectedPerScan => {
atomic_increment!(self.expected_per_scan, value);
@@ -340,290 +340,75 @@ impl Stats {
/// Merge a given `Stats` object from a json entry written to disk when handling a Ctrl+c
///
/// This is only ever called when resuming a scan from disk
pub fn merge_from(&self, filename: &str) {
if let Ok(file) = File::open(filename) {
let reader = BufReader::new(file);
let state: serde_json::Value = serde_json::from_reader(reader).unwrap();
pub fn merge_from(&self, filename: &str) -> Result<()> {
let file = File::open(filename)
.with_context(|| fmt_err(&format!("Could not open {}", filename)))?;
let reader = BufReader::new(file);
let state: serde_json::Value = serde_json::from_reader(reader)?;
if let Some(state_stats) = state.get("statistics") {
if let Ok(d_stats) = serde_json::from_value::<Stats>(state_stats.clone()) {
atomic_increment!(self.successes, atomic_load!(d_stats.successes));
atomic_increment!(self.timeouts, atomic_load!(d_stats.timeouts));
atomic_increment!(self.requests, atomic_load!(d_stats.requests));
atomic_increment!(self.errors, atomic_load!(d_stats.errors));
atomic_increment!(self.redirects, atomic_load!(d_stats.redirects));
atomic_increment!(self.client_errors, atomic_load!(d_stats.client_errors));
atomic_increment!(self.server_errors, atomic_load!(d_stats.server_errors));
atomic_increment!(self.links_extracted, atomic_load!(d_stats.links_extracted));
atomic_increment!(self.status_200s, atomic_load!(d_stats.status_200s));
atomic_increment!(self.status_301s, atomic_load!(d_stats.status_301s));
atomic_increment!(self.status_302s, atomic_load!(d_stats.status_302s));
atomic_increment!(self.status_401s, atomic_load!(d_stats.status_401s));
atomic_increment!(self.status_403s, atomic_load!(d_stats.status_403s));
atomic_increment!(self.status_429s, atomic_load!(d_stats.status_429s));
atomic_increment!(self.status_500s, atomic_load!(d_stats.status_500s));
atomic_increment!(self.status_503s, atomic_load!(d_stats.status_503s));
atomic_increment!(self.status_504s, atomic_load!(d_stats.status_504s));
atomic_increment!(self.status_508s, atomic_load!(d_stats.status_508s));
atomic_increment!(
self.wildcards_filtered,
atomic_load!(d_stats.wildcards_filtered)
);
atomic_increment!(
self.responses_filtered,
atomic_load!(d_stats.responses_filtered)
);
atomic_increment!(
self.resources_discovered,
atomic_load!(d_stats.resources_discovered)
);
atomic_increment!(
self.url_format_errors,
atomic_load!(d_stats.url_format_errors)
);
atomic_increment!(
self.connection_errors,
atomic_load!(d_stats.connection_errors)
);
atomic_increment!(
self.redirection_errors,
atomic_load!(d_stats.redirection_errors)
);
atomic_increment!(self.request_errors, atomic_load!(d_stats.request_errors));
if let Some(state_stats) = state.get("statistics") {
let d_stats = serde_json::from_value::<Stats>(state_stats.clone())?;
atomic_increment!(self.successes, atomic_load!(d_stats.successes));
atomic_increment!(self.timeouts, atomic_load!(d_stats.timeouts));
atomic_increment!(self.requests, atomic_load!(d_stats.requests));
atomic_increment!(self.errors, atomic_load!(d_stats.errors));
atomic_increment!(self.redirects, atomic_load!(d_stats.redirects));
atomic_increment!(self.client_errors, atomic_load!(d_stats.client_errors));
atomic_increment!(self.server_errors, atomic_load!(d_stats.server_errors));
atomic_increment!(self.links_extracted, atomic_load!(d_stats.links_extracted));
atomic_increment!(self.status_200s, atomic_load!(d_stats.status_200s));
atomic_increment!(self.status_301s, atomic_load!(d_stats.status_301s));
atomic_increment!(self.status_302s, atomic_load!(d_stats.status_302s));
atomic_increment!(self.status_401s, atomic_load!(d_stats.status_401s));
atomic_increment!(self.status_403s, atomic_load!(d_stats.status_403s));
atomic_increment!(self.status_429s, atomic_load!(d_stats.status_429s));
atomic_increment!(self.status_500s, atomic_load!(d_stats.status_500s));
atomic_increment!(self.status_503s, atomic_load!(d_stats.status_503s));
atomic_increment!(self.status_504s, atomic_load!(d_stats.status_504s));
atomic_increment!(self.status_508s, atomic_load!(d_stats.status_508s));
atomic_increment!(
self.wildcards_filtered,
atomic_load!(d_stats.wildcards_filtered)
);
atomic_increment!(
self.responses_filtered,
atomic_load!(d_stats.responses_filtered)
);
atomic_increment!(
self.resources_discovered,
atomic_load!(d_stats.resources_discovered)
);
atomic_increment!(
self.url_format_errors,
atomic_load!(d_stats.url_format_errors)
);
atomic_increment!(
self.connection_errors,
atomic_load!(d_stats.connection_errors)
);
atomic_increment!(
self.redirection_errors,
atomic_load!(d_stats.redirection_errors)
);
atomic_increment!(self.request_errors, atomic_load!(d_stats.request_errors));
if let Ok(scan_times) = d_stats.directory_scan_times.lock() {
for scan_time in scan_times.iter() {
self.update_f64_field(StatField::DirScanTimes, *scan_time);
}
}
if let Ok(scan_times) = d_stats.directory_scan_times.lock() {
for scan_time in scan_times.iter() {
self.update_f64_field(StatField::DirScanTimes, *scan_time);
}
}
};
}
Ok(())
}
}
#[derive(Debug)]
/// Enum variants used to inform the `StatCommand` protocol what `Stats` fields should be updated
pub enum StatError {
/// Represents a 403 response code
Status403,
/// Represents a timeout error
Timeout,
/// Represents a URL formatting error
UrlFormat,
/// Represents an error encountered during redirection
Redirection,
/// Represents an error encountered during connection
Connection,
/// Represents an error resulting from the client's request
Request,
/// Represents any other error not explicitly defined above
Other,
}
/// Protocol definition for updating a Stats object via mpsc
#[derive(Debug)]
pub enum StatCommand {
/// Add one to the total number of requests
AddRequest,
/// Add one to the proper field(s) based on the given `StatError`
AddError(StatError),
/// Add one to the proper field(s) based on the given `StatusCode`
AddStatus(StatusCode),
/// Create the progress bar (`BarType::Total`) that is updated from the stats thread
CreateBar,
/// Update a `Stats` field that corresponds to the given `StatField` by the given `usize` value
UpdateUsizeField(StatField, usize),
/// Update a `Stats` field that corresponds to the given `StatField` by the given `f64` value
UpdateF64Field(StatField, f64),
/// Save a `Stats` object to disk using `reporter::get_cached_file_handle`
Save,
/// Load a `Stats` object from disk
LoadStats(String),
/// Break out of the (infinite) mpsc receive loop
Exit,
}
/// Enum representing fields whose updates need to be performed in batches instead of one at
/// a time
#[derive(Debug)]
pub enum StatField {
/// Due to the necessary order of events, the number of requests expected to be sent isn't
/// known until after `statistics::initialize` is called. This command allows for updating
/// the `expected_per_scan` field after initialization
ExpectedPerScan,
/// Translates to `total_scans`
TotalScans,
/// Translates to `links_extracted`
LinksExtracted,
/// Translates to `total_expected`
TotalExpected,
/// Translates to `wildcards_filtered`
WildcardsFiltered,
/// Translates to `responses_filtered`
ResponsesFiltered,
/// Translates to `resources_discovered`
ResourcesDiscovered,
/// Translates to `initial_targets`
InitialTargets,
/// Translates to `directory_scan_times`; assumes a single append to the vector
DirScanTimes,
}
/// Spawn a single consumer task (sc side of mpsc)
///
/// The consumer simply receives `StatCommands` and updates the given `Stats` object as appropriate
pub async fn spawn_statistics_handler(
mut rx_stats: UnboundedReceiver<StatCommand>,
stats: Arc<Stats>,
tx_stats: UnboundedSender<StatCommand>,
) {
log::trace!(
"enter: spawn_statistics_handler({:?}, {:?}, {:?})",
rx_stats,
stats,
tx_stats
);
// will be updated later via StatCommand; delay is for banner to print first
let mut bar = ProgressBar::hidden();
let start = Instant::now();
while let Some(command) = rx_stats.recv().await {
match command as StatCommand {
StatCommand::AddError(err) => {
stats.add_error(err);
increment_bar(&bar, stats.clone());
}
StatCommand::AddStatus(status) => {
stats.add_status_code(status);
increment_bar(&bar, stats.clone());
}
StatCommand::AddRequest => {
stats.add_request();
increment_bar(&bar, stats.clone());
}
StatCommand::Save => stats.save(start.elapsed().as_secs_f64(), &CONFIGURATION.output),
StatCommand::UpdateUsizeField(field, value) => {
let update_len = matches!(field, StatField::TotalScans);
stats.update_usize_field(field, value);
if update_len {
bar.set_length(atomic_load!(stats.total_expected) as u64)
}
}
StatCommand::UpdateF64Field(field, value) => stats.update_f64_field(field, value),
StatCommand::CreateBar => {
bar = add_bar(
"",
atomic_load!(stats.total_expected) as u64,
BarType::Total,
);
}
StatCommand::LoadStats(filename) => {
stats.merge_from(&filename);
}
StatCommand::Exit => break,
}
}
bar.finish();
log::debug!("{:#?}", *stats);
log::trace!("exit: spawn_statistics_handler")
}
/// Wrapper around incrementing the overall scan's progress bar
fn increment_bar(bar: &ProgressBar, stats: Arc<Stats>) {
let msg = format!(
"{}:{:<7} {}:{:<7}",
style("found").green(),
atomic_load!(stats.resources_discovered),
style("errors").red(),
atomic_load!(stats.errors),
);
bar.set_message(&msg);
bar.inc(1);
}
/// Initialize new `Stats` object and the sc side of an mpsc channel that is responsible for
/// updates to the aforementioned object.
pub fn initialize() -> (Arc<Stats>, UnboundedSender<StatCommand>, JoinHandle<()>) {
log::trace!("enter: initialize");
let stats_tracker = Arc::new(Stats::new());
let stats_cloned = stats_tracker.clone();
let (tx_stats, rx_stats): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
let tx_stats_cloned = tx_stats.clone();
let stats_thread = tokio::spawn(async move {
spawn_statistics_handler(rx_stats, stats_cloned, tx_stats_cloned).await
});
log::trace!(
"exit: initialize -> ({:?}, {:?}, {:?})",
stats_tracker,
tx_stats,
stats_thread
);
(stats_tracker, tx_stats, stats_thread)
}
#[cfg(test)]
mod tests {
use super::super::*;
use super::*;
use std::fs::write;
use tempfile::NamedTempFile;
/// simple helper to reduce code reuse
fn setup_stats_test() -> (Arc<Stats>, UnboundedSender<StatCommand>, JoinHandle<()>) {
initialize()
}
/// another helper to stay DRY; must be called after any sent commands and before any checks
/// performed against the Stats object
async fn teardown_stats_test(sender: UnboundedSender<StatCommand>, handle: JoinHandle<()>) {
// send exit and await, once the await completes, stats should be updated
sender.send(StatCommand::Exit).unwrap_or_default();
handle.await.unwrap();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
/// when sent StatCommand::Exit, function should exit its while loop (runs forever otherwise)
async fn statistics_handler_exits() {
let (_, sender, handle) = setup_stats_test();
sender.send(StatCommand::Exit).unwrap_or_default();
handle.await.unwrap(); // blocks on the handler's while loop
// if we've made it here, the test has succeeded
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
/// when sent StatCommand::AddRequest, stats object should reflect the change
async fn statistics_handler_increments_requests() {
@@ -758,7 +543,7 @@ mod tests {
let tfile = NamedTempFile::new().unwrap();
write(&tfile, contents).unwrap();
stats.merge_from(tfile.path().to_str().unwrap());
stats.merge_from(tfile.path().to_str().unwrap()).unwrap();
// as of 1.11.1; all Stats fields are accounted for whether they're updated in merge_from
// or not
@@ -808,26 +593,4 @@ mod tests {
stats.update_runtime(20.2);
assert!((stats.total_runtime.lock().unwrap()[0] - 20.2).abs() < f64::EPSILON);
}
#[test]
/// Stats::save should write contents of Stats to disk
fn save_writes_stats_object_to_disk() {
let stats = Stats::new();
stats.add_request();
stats.add_request();
stats.add_request();
stats.add_request();
stats.add_error(StatError::Timeout);
stats.add_error(StatError::Timeout);
stats.add_error(StatError::Timeout);
stats.add_error(StatError::Timeout);
stats.add_status_code(StatusCode::OK);
stats.add_status_code(StatusCode::OK);
stats.add_status_code(StatusCode::OK);
let outfile = "/tmp/stuff";
stats.save(174.33, outfile);
assert!(stats.as_json().contains("statistics"));
assert!(stats.as_json().contains("11")); // requests made
assert!(stats.as_str().is_empty());
}
}

24
src/statistics/error.rs Normal file
View File

@@ -0,0 +1,24 @@
#[derive(Debug)]
/// Enum variants used to inform the `StatCommand` protocol what `Stats` fields should be updated
pub enum StatError {
/// Represents a 403 response code
Status403,
/// Represents a timeout error
Timeout,
/// Represents a URL formatting error
UrlFormat,
/// Represents an error encountered during redirection
Redirection,
/// Represents an error encountered during connection
Connection,
/// Represents an error resulting from the client's request
Request,
/// Represents any other error not explicitly defined above
Other,
}

33
src/statistics/field.rs Normal file
View File

@@ -0,0 +1,33 @@
/// Enum representing fields whose updates need to be performed in batches instead of one at
/// a time
#[derive(Debug)]
pub enum StatField {
/// Due to the necessary order of events, the number of requests expected to be sent isn't
/// known until after `statistics::initialize` is called. This command allows for updating
/// the `expected_per_scan` field after initialization
ExpectedPerScan,
/// Translates to `total_scans`
TotalScans,
/// Translates to `links_extracted`
LinksExtracted,
/// Translates to `total_expected`
TotalExpected,
/// Translates to `wildcards_filtered`
WildcardsFiltered,
/// Translates to `responses_filtered`
ResponsesFiltered,
/// Translates to `resources_discovered`
ResourcesDiscovered,
/// Translates to `initial_targets`
InitialTargets,
/// Translates to `directory_scan_times`; assumes a single append to the vector
DirScanTimes,
}

34
src/statistics/init.rs Normal file
View File

@@ -0,0 +1,34 @@
use super::{command::StatCommand, container::Stats};
use crate::{event_handlers::StatsHandler, FeroxChannel};
use anyhow::Result;
use std::sync::Arc;
use tokio::{
sync::mpsc::{self, UnboundedSender},
task::JoinHandle,
};
/// Initialize new `Stats` object and the sc side of an mpsc channel that is responsible for
/// updates to the aforementioned object.
pub fn initialize() -> (
Arc<Stats>,
UnboundedSender<StatCommand>,
JoinHandle<Result<()>>,
) {
log::trace!("enter: initialize");
let stats_tracker = Arc::new(Stats::new());
let (tx_stats, rx_stats): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
let mut handler = StatsHandler::new(stats_tracker.clone(), rx_stats);
let stats_thread = tokio::spawn(async move { handler.start().await });
log::trace!(
"exit: initialize -> ({:?}, {:?}, {:?})",
stats_tracker,
tx_stats,
stats_thread
);
(stats_tracker, tx_stats, stats_thread)
}

23
src/statistics/macros.rs Normal file
View File

@@ -0,0 +1,23 @@
#![macro_use]
/// Wrapper `Atomic*.fetch_add` to save me from writing Ordering::Relaxed a bajillion times
///
/// default is to increment by 1, second arg can be used to increment by a different value
#[macro_export]
macro_rules! atomic_increment {
($metric:expr) => {
$metric.fetch_add(1, Ordering::Relaxed);
};
($metric:expr, $value:expr) => {
$metric.fetch_add($value, Ordering::Relaxed);
};
}
/// Wrapper around `Atomic*.load` to save me from writing Ordering::Relaxed a bajillion times
#[macro_export]
macro_rules! atomic_load {
($metric:expr) => {
$metric.load(Ordering::Relaxed);
};
}

17
src/statistics/mod.rs Normal file
View File

@@ -0,0 +1,17 @@
mod error;
mod macros;
mod container;
mod command;
mod field;
mod init;
#[cfg(test)]
mod tests;
pub use self::command::StatCommand;
pub use self::container::Stats;
pub use self::error::StatError;
pub use self::field::StatField;
pub use self::init::initialize;
#[cfg(test)]
use self::tests::{setup_stats_test, teardown_stats_test};

65
src/statistics/tests.rs Normal file
View File

@@ -0,0 +1,65 @@
use super::*;
use crate::FeroxSerialize;
use anyhow::Result;
use reqwest::StatusCode;
use std::sync::Arc;
use tempfile::NamedTempFile;
use tokio::{sync::mpsc::UnboundedSender, task::JoinHandle};
/// simple helper to reduce code reuse
pub fn setup_stats_test() -> (
Arc<Stats>,
UnboundedSender<StatCommand>,
JoinHandle<Result<()>>,
) {
initialize()
}
/// another helper to stay DRY; must be called after any sent commands and before any checks
/// performed against the Stats object
pub async fn teardown_stats_test(
sender: UnboundedSender<StatCommand>,
handle: JoinHandle<Result<()>>,
) {
// send exit and await, once the await completes, stats should be updated
sender.send(StatCommand::Exit).unwrap_or_default();
handle.await.unwrap().unwrap();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
/// when sent StatCommand::Exit, function should exit its while loop (runs forever otherwise)
async fn statistics_handler_exits() {
let (_, sender, handle) = setup_stats_test();
sender.send(StatCommand::Exit).unwrap_or_default();
handle.await.unwrap().unwrap(); // blocks on the handler's while loop
// if we've made it here, the test has succeeded
}
#[test]
/// Stats::save should write contents of Stats to disk
fn save_writes_stats_object_to_disk() {
let stats = Stats::new();
stats.add_request();
stats.add_request();
stats.add_request();
stats.add_request();
stats.add_error(StatError::Timeout);
stats.add_error(StatError::Timeout);
stats.add_error(StatError::Timeout);
stats.add_error(StatError::Timeout);
stats.add_status_code(StatusCode::OK);
stats.add_status_code(StatusCode::OK);
stats.add_status_code(StatusCode::OK);
let outfile = NamedTempFile::new().unwrap();
if stats
.save(174.33, &outfile.path().to_str().unwrap())
.is_ok()
{}
assert!(stats.as_json().unwrap().contains("statistics"));
assert!(stats.as_json().unwrap().contains("11")); // requests made
assert!(stats.as_str().is_empty());
}

32
src/utils.rs
View File

@@ -7,6 +7,7 @@ use crate::{
},
FeroxError, FeroxResult,
};
use anyhow::{bail, Context, Result};
use console::{strip_ansi_codes, style, user_attended};
use indicatif::ProgressBar;
use reqwest::{Client, Response, Url};
@@ -22,25 +23,19 @@ use tokio::sync::mpsc::UnboundedSender;
pub fn open_file(filename: &str) -> Option<Arc<RwLock<io::BufWriter<fs::File>>>> {
log::trace!("enter: open_file({})", filename);
match fs::OpenOptions::new() // std fs
let file = fs::OpenOptions::new() // std fs
.create(true)
.append(true)
.open(filename)
{
Ok(file) => {
let writer = io::BufWriter::new(file); // std io
.with_context(|| fmt_err(&format!("Could not open {}", filename)))
.ok()?;
let locked_file = Some(Arc::new(RwLock::new(writer)));
let writer = io::BufWriter::new(file); // std io
log::trace!("exit: open_file -> {:?}", locked_file);
locked_file
}
Err(e) => {
log::error!("{}", e);
log::trace!("exit: open_file -> None");
None
}
}
let locked_file = Arc::new(RwLock::new(writer));
log::trace!("exit: open_file -> {:?}", locked_file);
Some(locked_file)
}
/// Helper function that determines the current depth of a given url
@@ -105,6 +100,11 @@ pub fn status_colorizer(status: &str) -> String {
}
}
/// simple wrapper to stay DRY
pub fn fmt_err(msg: &str) -> String {
format!("{}: {}", status_colorizer("ERROR"), msg)
}
/// Takes in a string and colors it using console::style
///
/// mainly putting this here in case i want to change the color later, making any changes easy
@@ -285,7 +285,7 @@ pub async fn make_request(
client: &Client,
url: &Url,
tx_stats: UnboundedSender<StatCommand>,
) -> FeroxResult<Response> {
) -> Result<Response> {
log::trace!(
"enter: make_request(CONFIGURATION.Client, {}, {:?})",
url,
@@ -331,7 +331,7 @@ pub async fn make_request(
log::warn!("Error while making request: {}", e);
}
Err(Box::new(e))
bail!("{}", e)
}
Ok(resp) => {
log::trace!("exit: make_request -> {:?}", resp);

4
tests/test_banner.rs
View File

@@ -615,7 +615,9 @@ fn banner_doesnt_print() -> Result<(), Box<dyn std::error::Error>> {
.arg("-q")
.assert()
.success()
.stderr(predicate::str::is_empty());
.stderr(predicate::str::contains(
"Could not find any live targets to scan",
));
Ok(())
}

51
tests/test_extractor.rs
View File

@@ -295,3 +295,54 @@ fn extractor_finds_robots_txt_links_and_displays_files_or_scans_directories() {
assert_eq!(mock_scanned_file.hits(), 1);
teardown_tmp_directory(tmp_dir);
}
#[test]
/// send a request to a page that contains a link that contains a directory that returns a 403
/// --extract-links should find the link and make recurse into the 403 directory, finding LICENSE
fn extractor_recurses_into_403_directories() -> Result<(), Box<dyn std::error::Error>> {
let srv = MockServer::start();
let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
let mock = srv.mock(|when, then| {
when.method(GET).path("/LICENSE");
then.status(200)
.body(&srv.url("'/homepage/assets/img/icons/handshake.svg'"));
});
let mock_two = srv.mock(|when, then| {
when.method(GET).path("/homepage/assets/img/icons/LICENSE");
then.status(200).body("that's just like, your opinion man");
});
let forbidden_dir = srv.mock(|when, then| {
when.method(GET).path("/homepage/assets/img/icons/");
then.status(403);
});
let cmd = Command::cargo_bin("feroxbuster")
.unwrap()
.arg("--url")
.arg(srv.url("/"))
.arg("--wordlist")
.arg(file.as_os_str())
.arg("--extract-links")
.arg("--depth") // need to go past default 4 directories
.arg("0")
.unwrap();
cmd.assert().success().stdout(
predicate::str::contains("/LICENSE")
.count(2)
.and(predicate::str::contains("1w")) // link in /LICENSE
.and(predicate::str::contains("34c")) // recursed LICENSE
.and(predicate::str::contains(
"/homepage/assets/img/icons/LICENSE",
)),
);
assert_eq!(mock.hits(), 1);
assert_eq!(mock_two.hits(), 1);
assert_eq!(forbidden_dir.hits(), 1);
teardown_tmp_directory(tmp_dir);
Ok(())
}

50
tests/test_scanner.rs
View File

@@ -546,3 +546,53 @@ fn scanner_single_request_scan_with_regex_filtered_result() {
assert_eq!(filtered_mock.hits(), 1);
teardown_tmp_directory(tmp_dir);
}
#[test]
/// send a request to a 403 directory, expect recursion to work into the 403
fn scanner_recursion_works_with_403_directories() {
let srv = MockServer::start();
let (tmp_dir, file) =
setup_tmp_directory(&["LICENSE".to_string(), "ignored/".to_string()], "wordlist").unwrap();
let mock = srv.mock(|when, then| {
when.method(GET).path("/LICENSE");
then.status(200).body("this is a test");
});
let forbidden_dir = srv.mock(|when, then| {
when.method(GET).path("/ignored/");
then.status(403);
});
let found_anyway = srv.mock(|when, then| {
when.method(GET).path("/ignored/LICENSE");
then.status(200)
.body("this is a test\nThat rug really tied the room together");
});
let cmd = Command::cargo_bin("feroxbuster")
.unwrap()
.arg("--url")
.arg(srv.url("/"))
.arg("--wordlist")
.arg(file.as_os_str())
.unwrap();
cmd.assert().success().stdout(
predicate::str::contains("/LICENSE")
.count(2)
.and(predicate::str::contains("200").count(2))
.and(predicate::str::contains("403"))
.and(predicate::str::contains("53c"))
.and(predicate::str::contains("14c"))
.and(predicate::str::contains("0c"))
.and(predicate::str::contains("ignored").count(2))
.and(predicate::str::contains("/ignored/LICENSE")),
);
assert_eq!(mock.hits(), 1);
assert_eq!(found_anyway.hits(), 1);
assert_eq!(forbidden_dir.hits(), 1);
teardown_tmp_directory(tmp_dir);
}