h/feroxbuster
1
Fork 0
mirror of https://github.com/epi052/feroxbuster.git synced 2026-06-01 13:01:19 -03:00
Code Issues Packages Projects Releases Wiki Activity

rewrote heuristics

Browse Source
This commit is contained in:
epi
2021-01-28 11:26:23 -06:00
parent 41fdc6a95a
commit 9cecf0c0d4
5 changed files with 238 additions and 228 deletions
Download Patch File Download Diff File Expand all files Collapse all files

433
src/heuristics.rs
View File

@@ -1,63 +1,109 @@
use crate::{
config::{CONFIGURATION, PROGRESS_PRINTER},
config::{Configuration, CONFIGURATION, PROGRESS_PRINTER},
event_handlers::{Command, Handles},
filters::WildcardFilter,
skip_fail,
utils::{ferox_print, format_url, get_url_path_length, make_request, status_colorizer},
FeroxResponse,
};
use anyhow::Result;
use anyhow::{bail, Result};
use console::style;
use indicatif::ProgressBar;
use reqwest::Client;
use std::sync::Arc;
use tokio::sync::mpsc::UnboundedSender;
use uuid::Uuid;
/// length of a standard UUID, used when determining wildcard responses
const UUID_LENGTH: u64 = 32;
/// Simple helper to return a uuid, formatted as lowercase without hyphens
///
/// `length` determines the number of uuids to string together. Each uuid
/// is 32 characters long. So, a length of 1 return a 32 character string,
/// a length of 2 returns a 64 character string, and so on...
fn unique_string(length: usize) -> String {
log::trace!("enter: unique_string({})", length);
let mut ids = vec![];
for _ in 0..length {
ids.push(Uuid::new_v4().to_simple().to_string());
}
let unique_id = ids.join("");
log::trace!("exit: unique_string -> {}", unique_id);
unique_id
/// wrapper around ugly string formatting
macro_rules! format_template {
($template:expr, $length:expr) => {
format!(
$template,
status_colorizer("WLD"),
"-",
"-",
"-",
style("auto-filtering").yellow(),
style($length).cyan(),
style("--dont-filter").yellow()
)
};
}
/// Tests the given url to see if it issues a wildcard response
///
/// In the event that url returns a wildcard response, a
/// [WildcardFilter](struct.WildcardFilter.html) is created and returned to the caller.
pub async fn wildcard_test(
target_url: &str,
bar: ProgressBar,
/// container for heuristics related info
pub struct HeuristicTests<'a> {
/// Handles object for event handler interaction
handles: Arc<Handles>,
) -> Result<()> {
log::trace!(
"enter: wildcard_test({:?}, {:?}, {:?})",
target_url,
bar,
handles,
);
if CONFIGURATION.dont_filter {
// early return, dont_filter scans don't need tested
log::trace!("exit: wildcard_test -> None");
return Ok(());
dont_filter: bool,
quiet: bool,
add_slash: bool,
client: &'a Client,
queries: &'a Vec<(String, String)>,
}
/// HeuristicTests implementation
impl<'a> HeuristicTests<'a> {
/// create a new HeuristicTests struct
pub fn new(handles: Arc<Handles>, config: &'a Configuration) -> Self {
Self {
handles,
dont_filter: config.dont_filter,
quiet: config.quiet,
add_slash: config.add_slash,
client: &config.client,
queries: &config.queries,
}
}
if let Some(ferox_response) = make_wildcard_request(&target_url, 1, handles.clone()).await {
bar.inc(1);
/// Simple helper to return a uuid, formatted as lowercase without hyphens
///
/// `length` determines the number of uuids to string together. Each uuid
/// is 32 characters long. So, a length of 1 return a 32 character string,
/// a length of 2 returns a 64 character string, and so on...
fn unique_string(&self, length: usize) -> String {
log::trace!("enter: unique_string({})", length);
let mut ids = vec![];
for _ in 0..length {
ids.push(Uuid::new_v4().to_simple().to_string());
}
let unique_id = ids.join("");
log::trace!("exit: unique_string -> {}", unique_id);
unique_id
}
/// wrapper for sending a filter to the filters event handler
fn send_filter(&self, filter: WildcardFilter) -> Result<()> {
self.handles
.filters
.send(Command::AddFilter(Box::new(filter)))
}
/// Tests the given url to see if it issues a wildcard response
///
/// In the event that url returns a wildcard response, a
/// [WildcardFilter](struct.WildcardFilter.html) is created and sent to the filters event
/// handler.
///
/// Returns the number of times to increment the caller's progress bar
pub async fn wildcard(&self, target_url: &str) -> Result<u64> {
log::trace!("enter: wildcard_test({:?})", target_url);
if self.dont_filter {
// early return, dont_filter scans don't need tested
log::trace!("exit: wildcard_test -> 0");
return Ok(0);
}
let ferox_response = self.make_wildcard_request(&target_url, 1).await?;
// found a wildcard response
let mut wildcard = WildcardFilter::default();
@@ -65,212 +111,149 @@ pub async fn wildcard_test(
let wc_length = ferox_response.content_length();
if wc_length == 0 {
log::trace!("exit: wildcard_test -> Some({:?})", wildcard);
handles
.filters
.send(Command::AddFilter(Box::new(wildcard)))?;
return Ok(());
log::trace!("exit: wildcard_test -> 1");
self.send_filter(wildcard)?;
return Ok(1);
}
// content length of wildcard is non-zero, perform additional tests:
// make a second request, with a known-sized (64) longer request
if let Some(resp_two) = make_wildcard_request(&target_url, 3, handles.clone()).await {
bar.inc(1);
let resp_two = self.make_wildcard_request(&target_url, 3).await?;
let wc2_length = resp_two.content_length();
let wc2_length = resp_two.content_length();
if wc2_length == wc_length + (UUID_LENGTH * 2) {
// second length is what we'd expect to see if the requested url is
// reflected in the response along with some static content; aka custom 404
let url_len = get_url_path_length(&ferox_response.url());
if wc2_length == wc_length + (UUID_LENGTH * 2) {
// second length is what we'd expect to see if the requested url is
// reflected in the response along with some static content; aka custom 404
let url_len = get_url_path_length(&ferox_response.url());
wildcard.dynamic = wc_length - url_len;
wildcard.dynamic = wc_length - url_len;
if !CONFIGURATION.quiet {
let msg = format!(
"{} {:>9} {:>9} {:>9} Wildcard response is dynamic; {} ({} + url length) responses; toggle this behavior by using {}\n",
status_colorizer("WLD"),
"-",
"-",
"-",
style("auto-filtering").yellow(),
style(wc_length - url_len).cyan(),
style("--dont-filter").yellow()
);
ferox_print(&msg, &PROGRESS_PRINTER);
}
} else if wc_length == wc2_length {
wildcard.size = wc_length;
if !CONFIGURATION.quiet {
let msg = format!(
"{} {:>9} {:>9} {:>9} Wildcard response is static; {} {} responses; toggle this behavior by using {}\n",
status_colorizer("WLD"),
"-",
"-",
"-",
style("auto-filtering").yellow(),
style(wc_length).cyan(),
style("--dont-filter").yellow()
);
ferox_print(&msg, &PROGRESS_PRINTER);
}
if !self.quiet {
let msg = format_template!("{} {:>9} {:>9} {:>9} Wildcard response is dynamic; {} ({} + url length) responses; toggle this behavior by using {}\n", wildcard.dynamic);
ferox_print(&msg, &PROGRESS_PRINTER);
}
} else {
bar.inc(2);
}
} else if wc_length == wc2_length {
wildcard.size = wc_length;
log::trace!("exit: wildcard_test -> Some({:?})", wildcard);
handles
.filters
.send(Command::AddFilter(Box::new(wildcard)))?;
return Ok(());
}
log::trace!("exit: wildcard_test -> None");
Ok(())
}
/// Generates a uuid and appends it to the given target url. The reasoning is that the randomly
/// generated unique string should not exist on and be served by the target web server.
///
/// Once the unique url is created, the request is sent to the server. If the server responds
/// back with a valid status code, the response is considered to be a wildcard response. If that
/// wildcard response has a 3xx status code, that redirection location is displayed to the user.
async fn make_wildcard_request(
target_url: &str,
length: usize,
handles: Arc<Handles>,
) -> Option<FeroxResponse> {
log::trace!(
"enter: make_wildcard_request({}, {}, {:?})",
target_url,
length,
handles
);
let unique_str = unique_string(length);
let nonexistent = match format_url(
target_url,
&unique_str,
CONFIGURATION.add_slash,
&CONFIGURATION.queries,
None,
handles.stats.tx.clone(),
) {
Ok(url) => url,
Err(e) => {
log::error!("{}", e);
log::trace!("exit: make_wildcard_request -> None");
return None;
}
};
match make_request(
&CONFIGURATION.client,
&nonexistent.to_owned(),
handles.stats.tx.clone(),
)
.await
{
Ok(response) => {
if CONFIGURATION
.status_codes
.contains(&response.status().as_u16())
{
// found a wildcard response
let mut ferox_response = FeroxResponse::from(response, true).await;
ferox_response.wildcard = true;
if !CONFIGURATION.quiet
&& !handles
.filters
.data
.should_filter_response(&ferox_response, handles.stats.tx.clone())
&& handles
.output
.send(Command::Report(Box::new(ferox_response.clone())))
.is_err()
{
// abusing short-circuiting to protect the terminal send behind
// not quiet and shouldn't filter out the response
return None;
}
log::trace!("exit: make_wildcard_request -> {}", ferox_response);
return Some(ferox_response);
if !self.quiet {
let msg = format_template!("{} {:>9} {:>9} {:>9} Wildcard response is static; {} {} responses; toggle this behavior by using {}\n", wildcard.size);
ferox_print(&msg, &PROGRESS_PRINTER);
}
}
Err(e) => {
log::warn!("{}", e);
log::trace!("exit: make_wildcard_request -> None");
return None;
}
self.send_filter(wildcard)?;
log::trace!("exit: wildcard_test");
Ok(2)
}
log::trace!("exit: make_wildcard_request -> None");
None
}
/// Generates a uuid and appends it to the given target url. The reasoning is that the randomly
/// generated unique string should not exist on and be served by the target web server.
///
/// Once the unique url is created, the request is sent to the server. If the server responds
/// back with a valid status code, the response is considered to be a wildcard response. If that
/// wildcard response has a 3xx status code, that redirection location is displayed to the user.
async fn make_wildcard_request(
&self,
target_url: &str,
length: usize,
) -> Result<FeroxResponse> {
log::trace!("enter: make_wildcard_request({}, {})", target_url, length);
/// Simply tries to connect to all given sites before starting to scan
///
/// In the event that no sites can be reached, the program will exit.
///
/// Any urls that are found to be alive are returned to the caller.
pub async fn connectivity_test(
target_urls: &[String],
tx_stats: UnboundedSender<Command>,
) -> Vec<String> {
log::trace!(
"enter: connectivity_test({:?}, {:?})",
target_urls,
tx_stats
);
let unique_str = self.unique_string(length);
let mut good_urls = vec![];
for target_url in target_urls {
let request = match format_url(
let nonexistent_url = format_url(
target_url,
"",
CONFIGURATION.add_slash,
&CONFIGURATION.queries,
&unique_str,
self.add_slash,
&self.queries,
None,
tx_stats.clone(),
) {
Ok(url) => url,
Err(e) => {
log::error!("{}", e);
continue;
}
};
self.handles.stats.tx.clone(),
)?;
match make_request(&CONFIGURATION.client, &request, tx_stats.clone()).await {
Ok(_) => {
good_urls.push(target_url.to_owned());
let response = make_request(
&self.client,
&nonexistent_url.to_owned(),
self.handles.stats.tx.clone(),
)
.await?;
if CONFIGURATION
.status_codes
.contains(&response.status().as_u16())
{
// found a wildcard response
let mut ferox_response = FeroxResponse::from(response, true).await;
ferox_response.wildcard = true;
if self
.handles
.filters
.data
.should_filter_response(&ferox_response, self.handles.stats.tx.clone())
{
bail!("filtered response")
}
Err(e) => {
if !CONFIGURATION.quiet {
ferox_print(
&format!("Could not connect to {}, skipping...", target_url),
&PROGRESS_PRINTER,
);
if !self.quiet {
let boxed = Box::new(ferox_response.clone());
self.handles.output.send(Command::Report(boxed))?;
}
log::trace!("exit: make_wildcard_request -> {}", ferox_response);
return Ok(ferox_response);
}
log::trace!("exit: make_wildcard_request -> Err");
bail!("uninteresting status code")
}
/// Simply tries to connect to all given sites before starting to scan
///
/// In the event that no sites can be reached, the program will exit.
///
/// Any urls that are found to be alive are returned to the caller.
pub async fn connectivity(&self, target_urls: &[String]) -> Result<Vec<String>> {
log::trace!("enter: connectivity_test({:?})", target_urls);
let mut good_urls = vec![];
for target_url in target_urls {
let request = skip_fail!(format_url(
target_url,
"",
self.add_slash,
&self.queries,
None,
self.handles.stats.tx.clone(),
));
let result = make_request(&self.client, &request, self.handles.stats.tx.clone()).await;
match result {
Ok(_) => {
good_urls.push(target_url.to_owned());
}
Err(e) => {
if !CONFIGURATION.quiet {
ferox_print(
&format!("Could not connect to {}, skipping...", target_url),
&PROGRESS_PRINTER,
);
}
log::error!("{}", e);
}
log::error!("{}", e);
}
}
if good_urls.is_empty() {
bail!("Could not connect to any target provided");
}
log::trace!("exit: connectivity_test -> {:?}", good_urls);
Ok(good_urls)
}
if good_urls.is_empty() {
log::error!("Could not connect to any target provided, exiting.");
}
log::trace!("exit: connectivity_test -> {:?}", good_urls);
good_urls
}
#[cfg(test)]
@@ -280,8 +263,10 @@ mod tests {
#[test]
/// request a unique string of 32bytes * a value returns correct result
fn heuristics_unique_string_returns_correct_length() {
let (handles, _) = Handles::for_testing(None);
let tester = HeuristicTests::new(Arc::new(handles), &CONFIGURATION);
for i in 0..10 {
assert_eq!(unique_string(i).len(), i * 32);
assert_eq!(tester.unique_string(i).len(), i * 32);
}
}
}

10
src/main.rs
View File

@@ -286,7 +286,15 @@ async fn wrapped_main() -> Result<()> {
}
// discard non-responsive targets
let live_targets = heuristics::connectivity_test(&targets, handles.stats.tx.clone()).await;
let live_targets = {
let test = heuristics::HeuristicTests::new(handles.clone(), &CONFIGURATION);
let result = test.connectivity(&targets).await;
if result.is_err() {
clean_up(handles, tasks).await?;
bail!(fmt_err(&result.unwrap_err().to_string()));
}
result?
};
if live_targets.is_empty() {
clean_up(handles, tasks).await?;

8
src/scanner.rs
View File

@@ -220,10 +220,14 @@ pub async fn scan_url(
let permit = SCAN_LIMITER.acquire().await;
// Arc clones to be passed around to the various scans
let wildcard_bar = progress_bar.clone();
let looping_words = wordlist.clone();
heuristics::wildcard_test(&target_url, wildcard_bar, handles.clone()).await?;
{
let test = heuristics::HeuristicTests::new(handles.clone(), &CONFIGURATION);
if let Ok(num_reqs) = test.wildcard(&target_url).await {
progress_bar.inc(num_reqs);
}
}
// producer tasks (mp of mpsc); responsible for making requests
let producers = stream::iter(looping_words.deref().to_owned())

13
src/utils.rs
View File

@@ -173,6 +173,19 @@ macro_rules! send_command {
};
}
#[macro_export]
macro_rules! skip_fail {
($res:expr) => {
match $res {
Ok(val) => val,
Err(e) => {
log::warn!("An error: {}; skipped.", e);
continue;
}
}
};
}
/// Simple helper to generate a `Url`
///
/// Errors during parsing `url` or joining `word` are propagated up the call stack

2
tests/test_banner.rs
View File

@@ -619,7 +619,7 @@ fn banner_doesnt_print() -> Result<(), Box<dyn std::error::Error>> {
.assert()
.success()
.stderr(predicate::str::contains(
"Could not find any live targets to scan",
"Could not connect to any target provided",
));
Ok(())
}