h/feroxbuster
1
Fork 0
mirror of https://github.com/epi052/feroxbuster.git synced 2026-06-01 21:21:12 -03:00
Code Issues Packages Projects Releases Wiki Activity

filters now sent to the filter handler; still not acted upon

Browse Source
This commit is contained in:
epi
2021-01-24 15:03:43 -06:00
parent 8353978b5a
commit e33816e9da
9 changed files with 138 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/event_handlers/command.rs
View File

@@ -56,6 +56,9 @@ pub enum Command {
/// Command used to test that a spawned task succeeded in initialization
Ping,
/// Just receive a sender and reply, used for slowing down the main thread
Sync(Sender<bool>),
/// Break out of the (infinite) mpsc receive loop
Exit,
}

13
src/event_handlers/container.rs
View File

@@ -1,7 +1,7 @@
use super::*;
use crate::event_handlers::scans::ScanHandle;
use crate::scan_manager::FeroxScans;
use crate::Joiner;
use crate::{CommandSender, Joiner};
use anyhow::{bail, Result};
use std::sync::{Arc, RwLock};
@@ -93,4 +93,15 @@ impl Handles {
bail!("Could not get underlying FeroxScans")
}
/// Helper to easily get the (locked) underlying transmitter
pub fn sender(&self) -> Result<CommandSender> {
if let Ok(guard) = self.scans.read().as_ref() {
if let Some(handle) = guard.as_ref() {
return Ok(handle.tx.clone());
}
}
bail!("Could not get underlying transmitter")
}
}

17
src/event_handlers/filters.rs
View File

@@ -2,7 +2,10 @@ use super::*;
use crate::{filters::FeroxFilters, CommandSender, FeroxChannel, Joiner};
use anyhow::Result;
use std::sync::Arc;
use tokio::sync::mpsc::{self, UnboundedReceiver};
use tokio::sync::{
mpsc::{self, UnboundedReceiver},
oneshot,
};
#[derive(Debug)]
/// Container for filters transmitter and FeroxFilters object
@@ -26,6 +29,14 @@ impl FiltersHandle {
self.tx.send(command)?;
Ok(())
}
/// Sync the handle with the handler
pub async fn sync(&self) -> Result<()> {
let (tx, rx) = oneshot::channel::<bool>();
self.tx.send(Command::Sync(tx))?;
rx.await?;
Ok(())
}
}
/// event handler for updating a single data structure of all active filters
@@ -75,6 +86,10 @@ impl FiltersHandler {
Command::AddFilter(filter) => {
self.data.push(filter)?;
}
Command::Sync(sender) => {
log::debug!("filters: {:?}", self);
sender.send(true).unwrap_or_default();
}
Command::Exit => break,
_ => {} // no other commands needed for FilterHandler
}

4
src/event_handlers/scans.rs
View File

@@ -1,8 +1,8 @@
use super::command::Command::UpdateUsizeField;
use super::*;
use crate::{
scan_manager::{FeroxScan, FeroxScans},
scanner::{scan_url, ScanOrder},
scan_manager::{FeroxScan, FeroxScans, ScanOrder},
scanner::scan_url,
statistics::StatField::TotalScans,
CommandReceiver, CommandSender, FeroxChannel, Joiner,
};

7
src/extractor/builder.rs
View File

@@ -102,8 +102,7 @@ impl<'a> ExtractorBuilder<'a> {
/// builder call to set `tx_recursion`
pub fn recursion_transmitter(&mut self, tx_recursion: CommandSender) -> &mut Self {
// todo change to scans_transmitter or w/e same on struct; don't bother, going to make
// extractor take a Handles object later anyway
// todo change to scans_transmitter or w/e same on struct; don't bother, going to make extractor take a Handles object later anyway
self.tx_recursion = Some(tx_recursion);
self
}
@@ -116,9 +115,7 @@ impl<'a> ExtractorBuilder<'a> {
/// builder call to set `tx_reporter`
pub fn reporter_transmitter(&mut self, tx_reporter: CommandSender) -> &mut Self {
// todo change to outputs or w/e same on struct; don't bother, going to make
// extractor take a Handles object later anyway
// todo change to outputs or w/e same on struct; don't bother, going to make extractor take a Handles object later anyway
self.tx_reporter = Some(tx_reporter);
self
}

8
src/filters/container.rs
View File

@@ -13,8 +13,12 @@ pub struct FeroxFilters {
impl FeroxFilters {
/// add a single FeroxFilter to the collection
pub fn push(&self, filter: Box<dyn FeroxFilter>) -> Result<()> {
if let Ok(mut unlocked) = self.filters.lock() {
unlocked.push(filter)
if let Ok(mut guard) = self.filters.lock() {
if guard.contains(&filter) {
return Ok(());
}
guard.push(filter)
}
Ok(())
}

35
src/main.rs
View File

@@ -26,8 +26,7 @@ use feroxbuster::{
},
heuristics, logger,
scan_manager::{self, PAUSE_SCAN},
scanner::{self, SCANNED_URLS},
send_command,
scanner,
utils::fmt_err,
SLEEP_DURATION,
};
@@ -118,7 +117,7 @@ async fn scan(targets: Vec<String>, handles: Arc<Handles>) -> Result<()> {
handles.send_scan_command(UpdateWordlist(words.clone()))?;
scanner::initialize(words.len(), &CONFIGURATION, handles.stats.tx.clone()).await;
scanner::initialize(words.len(), &CONFIGURATION, handles.clone()).await?;
// at this point, the stat thread's progress bar can be created; things that needed to happen
// first:
@@ -146,7 +145,7 @@ async fn scan(targets: Vec<String>, handles: Arc<Handles>) -> Result<()> {
}
/// Get targets from either commandline or stdin, pass them back to the caller as a Result<Vec>
async fn get_targets() -> Result<Vec<String>> {
async fn get_targets(handles: Arc<Handles>) -> Result<Vec<String>> {
log::trace!("enter: get_targets");
let mut targets = vec![];
@@ -163,8 +162,9 @@ async fn get_targets() -> Result<Vec<String>> {
} else if CONFIGURATION.resumed {
// resume-from can't be used with --url, and --stdin is marked false for every resumed
// scan, making it mutually exclusive from either of the other two options
if let Ok(scans) = SCANNED_URLS.scans.read() {
// todo this block shouldn't be SCANNED_URLS
let ferox_scans = handles.ferox_scans()?;
if let Ok(scans) = ferox_scans.scans.read() {
for scan in scans.iter() {
// SCANNED_URLS gets deserialized scans added to it at program start if --resume-from
// is used, so scans that aren't marked complete still need to be scanned
@@ -175,7 +175,7 @@ async fn get_targets() -> Result<Vec<String>> {
targets.push(scan.url.to_owned());
}
}
};
} else {
targets.push(CONFIGURATION.target_url.clone());
}
@@ -242,9 +242,8 @@ async fn wrapped_main() -> Result<()> {
}
// get targets from command line or stdin
// todo get_targets needs SCANNED_URLS replaced
// todo a bunch of fucking functions needs SCANNED_URLS replaced
let targets = match get_targets().await {
let targets = match get_targets(handles.clone()).await {
Ok(t) => t,
Err(e) => {
// should only happen in the event that there was an error reading from stdin
@@ -305,7 +304,7 @@ async fn wrapped_main() -> Result<()> {
// todo known things not working: confirm same # of requests seen in burp as reported
// todo known things not working: enable build on this branch, compare memory usage with new recursion paradigm
// todo known things not working: banner_print_output_file or w/e is hanging, probably means
// --output is wrong or that no-recursion is wrong
// todo known things not working: scan cancel menu is hard fkn broke
clean_up(handles, tasks).await?;
@@ -316,32 +315,24 @@ async fn wrapped_main() -> Result<()> {
/// Single cleanup function that handles all the necessary drops/finishes etc required to gracefully
/// shutdown the program
async fn clean_up(handles: Arc<Handles>, tasks: Tasks) -> Result<()> {
log::trace!(
"enter: clean_up({:?}, {:?})", // todo missing tasks
handles,
tasks
);
log::trace!("enter: clean_up({:?}, {:?})", handles, tasks);
let (tx, rx) = oneshot::channel::<bool>();
log::error!("oneshot created, sending join command"); // todo
handles.send_scan_command(JoinTasks(tx))?;
log::error!("sent join command"); // todo
rx.await?;
log::info!("All scans complete!");
// terminal handler closes file handler if one is in use
log::error!("Sending Exit"); // todo remove
handles.output.send(Exit)?;
// send_command!(handles.output.tx, Exit); // todo use handles.thing.send in this function
tasks.terminal.await??;
log::trace!("terminal handler closed");
send_command!(handles.filters.tx, Exit);
handles.filters.send(Exit)?;
tasks.filters.await??;
log::trace!("filters handler closed");
send_command!(handles.stats.tx, Exit);
handles.stats.send(Exit)?;
tasks.stats.await??;
log::trace!("stats handler closed");
@@ -352,8 +343,6 @@ async fn clean_up(handles: Arc<Handles>, tasks: Tasks) -> Result<()> {
// the final trace messages above
PROGRESS_PRINTER.finish();
// drop(tx_stats);
log::trace!("exit: clean_up");
Ok(())
}

101
src/scan_manager.rs
View File

@@ -49,10 +49,24 @@ pub static PAUSE_SCAN: AtomicBool = AtomicBool::new(false);
/// Simple enum used to flag a `FeroxScan` as likely a directory or file
#[derive(Debug, Serialize, Deserialize, Copy, Clone)]
pub enum ScanType {
/// Just a file being requested
File,
/// A an entire directory that might be scanned
Directory,
}
#[derive(Debug, Copy, Clone)]
/// Simple enum to designate whether a URL was passed in by the user (Initial) or found during
/// scanning (Latest)
pub enum ScanOrder {
/// Url was passed in by the user
Initial,
/// Url was found during scanning
Latest,
}
/// Default implementation for ScanType
impl Default for ScanType {
/// Return ScanType::File as default
@@ -110,15 +124,18 @@ impl Default for FeroxScan {
/// Implementation of FeroxScan
impl FeroxScan {
/// Stop a currently running scan
pub async fn abort(&self) {
pub async fn abort(&self) -> Result<()> {
let mut guard = self.task.lock().await;
if guard.is_some() {
let task = std::mem::replace(&mut *guard, None).unwrap();
task.abort();
self.set_status(ScanStatus::Cancelled).unwrap(); // todo
self.stop_progress_bar();
if let Some(task) = std::mem::replace(&mut *guard, None) {
task.abort();
self.set_status(ScanStatus::Cancelled)?;
self.stop_progress_bar();
}
}
Ok(())
}
/// small wrapper to set the JoinHandle
@@ -138,29 +155,34 @@ impl FeroxScan {
/// Simple helper to call .finish on the scan's progress bar
fn stop_progress_bar(&self) {
// todo do something with the unwrap see set_status todo note
let guard = self.progress_bar.lock().unwrap();
if guard.is_some() {
(*guard).as_ref().unwrap().finish_at_current_pos()
if let Ok(guard) = self.progress_bar.lock() {
if guard.is_some() {
(*guard).as_ref().unwrap().finish_at_current_pos()
}
}
}
/// Simple helper get a progress bar
pub fn progress_bar(&self) -> ProgressBar {
// todo do something with the unwrap see set_status todo note
let mut guard = self.progress_bar.lock().unwrap();
match self.progress_bar.lock() {
Ok(mut guard) => {
if guard.is_some() {
(*guard).as_ref().unwrap().clone()
} else {
let pb = add_bar(&self.url, self.num_requests, BarType::Default);
pb.reset_elapsed();
if guard.is_some() {
(*guard).as_ref().unwrap().clone()
} else {
let pb = add_bar(&self.url, self.num_requests, BarType::Default);
let _ = std::mem::replace(&mut *guard, Some(pb.clone()));
pb
}
}
Err(_) => {
log::warn!("Could not unlock progress bar on {:?}", self);
let pb = add_bar(&self.url, self.num_requests, BarType::Default);
pb.reset_elapsed();
pb.reset_elapsed();
let _ = std::mem::replace(&mut *guard, Some(pb.clone()));
pb
pb
}
}
}
@@ -208,15 +230,17 @@ impl FeroxScan {
false
}
/// todo doc
/// await a task's completion, similar to a thread's join; perform necessary bookkeeping
pub async fn join(&self) {
log::trace!("enter join({:?})", self);
let mut guard = self.task.lock().await;
if guard.is_some() {
let task = std::mem::replace(&mut *guard, None).unwrap();
task.await.unwrap();
self.set_status(ScanStatus::Complete).unwrap(); // todo
if let Some(task) = std::mem::replace(&mut *guard, None) {
task.await.unwrap();
self.set_status(ScanStatus::Complete)
.unwrap_or_else(|e| log::warn!("Could not mark scan complete: {}", e))
}
}
log::trace!("exit join({:?})", self);
@@ -595,6 +619,7 @@ impl FeroxScans {
// todo check this assumption, as we swap out the task with None once joined
continue;
}
self.menu.println(&format!("fdaf {}", scan));
if matches!(scan.scan_type, ScanType::Directory) {
// we're only interested in displaying directory scans, as those are
@@ -629,7 +654,10 @@ impl FeroxScans {
if input == 'y' || input == '\n' {
self.menu.println(&format!("Stopping {}...", selected.url));
selected.abort().await;
selected
.abort()
.await
.unwrap_or_else(|e| log::warn!("Could not cancel task: {}", e));
} else {
self.menu.println("Ok, doing nothing...");
}
@@ -798,25 +826,6 @@ impl FeroxScans {
}
scans
}
// todo remove probably
// pub async fn join_all(&self) -> usize {
// let mut joined = 0;
// if let Ok(u_scans) = self.scans.read() {
// for scan in u_scans.iter() {
// let mut guard = scan.lock().await;
// if guard.task.is_none() {
// continue;
// }
// guard.join().await;
// joined += 1;
//
// if let Ok(mut u_scan) = scan.lock() {
// }
// }
// }
// joined
// }
}
/// Container around a locked vector of `FeroxResponse`s, adds wrappers for insertion and search
@@ -1460,7 +1469,7 @@ mod tests {
progress_bar: std::sync::Mutex::new(None),
};
scan.abort().await;
scan.abort().await.unwrap();
assert!(matches!(
*scan.status.lock().unwrap(),

129
src/scanner.rs
View File

@@ -1,7 +1,7 @@
use crate::{
config::{Configuration, CONFIGURATION},
event_handlers::{
Command::{self, UpdateF64Field, UpdateUsizeField},
Command::{self, AddFilter, UpdateF64Field, UpdateUsizeField},
Handles,
},
extractor::ExtractorBuilder,
@@ -10,7 +10,7 @@ use crate::{
WordsFilter,
},
heuristics,
scan_manager::{FeroxResponses, FeroxScans, ScanStatus, PAUSE_SCAN},
scan_manager::{FeroxResponses, FeroxScans, ScanOrder, ScanStatus, PAUSE_SCAN},
send_command,
statistics::StatField::{DirScanTimes, ExpectedPerScan, WildcardsFiltered},
traits::FeroxFilter,
@@ -51,43 +51,6 @@ lazy_static! {
}
/// Adds the given FeroxFilter to the given list of FeroxFilter implementors
///
/// If the given list did not already contain the filter, return true; otherwise return false
fn add_filter_to_list_of_ferox_filters(
filter: Box<dyn FeroxFilter>,
ferox_filters: Arc<RwLock<Vec<Box<dyn FeroxFilter>>>>,
) -> bool {
log::trace!(
"enter: add_filter_to_list_of_ferox_filters({:?}, {:?})",
filter,
ferox_filters
);
// todo move to filters handler
match ferox_filters.write() {
Ok(mut filters) => {
// If the set did not contain the assigned filter, true is returned.
// If the set did contain the assigned filter, false is returned.
if filters.contains(&filter) {
log::trace!("exit: add_filter_to_list_of_ferox_filters -> false");
return false;
}
filters.push(filter);
log::trace!("exit: add_filter_to_list_of_ferox_filters -> true");
true
}
Err(e) => {
// poisoned lock
log::error!("Set of wildcard filters poisoned: {}", e);
log::trace!("exit: add_filter_to_list_of_ferox_filters -> false");
false
}
}
}
/// Creates a vector of formatted Urls
///
/// At least one value will be returned (base_url + word)
@@ -325,8 +288,10 @@ async fn make_requests(target_url: &str, word: &str, base_depth: usize, handles:
handles.stats.tx.clone(),
);
let scanned_urls = handles.ferox_scans().unwrap(); // unwrap todo
let tx_scans = handles.scans.read().unwrap().as_ref().unwrap().tx.clone(); // todo abstract away
let scanned_urls = handles.ferox_scans().expect("Could not get FeroxScans");
// todo abstract away, and by that i mean that extractor and try_recursion should either take
// Handles or be put into a struct somewhere
let tx_scans = handles.scans.read().unwrap().as_ref().unwrap().tx.clone();
for url in urls {
if let Ok(response) =
@@ -377,26 +342,13 @@ pub fn send_report(report_sender: CommandSender, response: FeroxResponse) {
match report_sender.send(Command::Report(Box::new(response))) {
Ok(_) => {}
Err(e) => {
log::warn!("{}", e);
// todo back to error
log::error!("{}", e);
}
}
log::trace!("exit: send_report");
}
#[derive(Debug, Copy, Clone)]
/// Simple enum to designate whether a URL was passed in by the user (Initial) or found during
/// scanning (Latest)
pub enum ScanOrder {
// todo is this the right location?
/// Url was passed in by the user
Initial,
/// Url was found during scanning
Latest,
}
/// Scan a given url using a given wordlist
///
/// This is the primary entrypoint for the scanner
@@ -485,18 +437,14 @@ pub async fn scan_url(
None => Box::new(WildcardFilter::default()),
};
// todo move to filters handler
add_filter_to_list_of_ferox_filters(filter, FILTERS.clone());
handles.filters.send(AddFilter(filter))?;
let scanned_urls = handles.ferox_scans()?;
// producer tasks (mp of mpsc); responsible for making requests
// todo .deref().to_owned() seems like they cancel eachother out
let producers = stream::iter(looping_words.deref().to_owned())
.map(|word| {
// todo abstract away more scans shit
let handles_clone = handles.clone();
let _txs = handles.stats.tx.clone();
let pb = progress_bar.clone(); // progress bar is an Arc around internal state
let tgt = target_url.to_string(); // done to satisfy 'static lifetime below
let scanned_urls_clone = scanned_urls.clone();
@@ -539,28 +487,6 @@ pub async fn scan_url(
ferox_scan.finish()?;
// todo remove
// // manually drop tx in order for the rx task's while loops to eval to false
// log::trace!("dropped recursion handler's transmitter");
// drop(tx_dir);
// note: in v1.11.2 i removed the join_all call that used to handle the recurser handles.
// nothing appears to change by having them removed, however, if ever a revert is needed
// this is the place and anything prior to 1.11.2 will have the code to do so
// {
// if let Ok(urls) = SCANNED_URLS.get_scan_by_url(target_url).unwrap().lock() {
// urls.task.as_ref().unwrap().into_inner().unwrap().await;
// }
// }
// todo remove
log::error!("SCAN URL EXIT: {}", target_url);
// for mut fut in futures {
// let x = Arc::try_unwrap(fut).unwrap();
// x.await;
// }
log::trace!("exit: scan_url");
Ok(())
@@ -571,28 +497,28 @@ pub async fn scan_url(
pub async fn initialize(
num_words: usize,
config: &Configuration,
tx_stats: UnboundedSender<Command>,
) {
handles: Arc<Handles>,
) -> Result<()> {
log::trace!(
"enter: initialize({}, {:?}, {:?})",
num_words,
config,
tx_stats
handles
);
// number of requests only needs to be calculated once, and then can be reused
let num_reqs_expected: u64 = if config.extensions.is_empty() {
num_words.try_into().unwrap()
num_words.try_into()?
} else {
let total = num_words * (config.extensions.len() + 1);
total.try_into().unwrap()
total.try_into()?
};
// tell Stats object about the number of expected requests
send_command!(
tx_stats,
UpdateUsizeField(ExpectedPerScan, num_reqs_expected as usize)
);
handles.stats.send(UpdateUsizeField(
ExpectedPerScan,
num_reqs_expected as usize,
))?;
// add any status code filters to `FILTERS` (-C|--filter-status)
for code_filter in &config.filter_status {
@@ -600,7 +526,7 @@ pub async fn initialize(
filter_code: *code_filter,
};
let boxed_filter = Box::new(filter);
add_filter_to_list_of_ferox_filters(boxed_filter, FILTERS.clone());
handles.filters.send(AddFilter(boxed_filter))?;
}
// add any line count filters to `FILTERS` (-N|--filter-lines)
@@ -609,7 +535,7 @@ pub async fn initialize(
line_count: *lines_filter,
};
let boxed_filter = Box::new(filter);
add_filter_to_list_of_ferox_filters(boxed_filter, FILTERS.clone());
handles.filters.send(AddFilter(boxed_filter))?;
}
// add any line count filters to `FILTERS` (-W|--filter-words)
@@ -618,7 +544,7 @@ pub async fn initialize(
word_count: *words_filter,
};
let boxed_filter = Box::new(filter);
add_filter_to_list_of_ferox_filters(boxed_filter, FILTERS.clone());
handles.filters.send(AddFilter(boxed_filter))?;
}
// add any line count filters to `FILTERS` (-S|--filter-size)
@@ -627,7 +553,7 @@ pub async fn initialize(
content_length: *size_filter,
};
let boxed_filter = Box::new(filter);
add_filter_to_list_of_ferox_filters(boxed_filter, FILTERS.clone());
handles.filters.send(AddFilter(boxed_filter))?;
}
// add any regex filters to `FILTERS` (-X|--filter-regex)
@@ -649,7 +575,7 @@ pub async fn initialize(
compiled,
};
let boxed_filter = Box::new(filter);
add_filter_to_list_of_ferox_filters(boxed_filter, FILTERS.clone());
handles.filters.send(AddFilter(boxed_filter))?;
}
// add any similarity filters to `FILTERS` (--filter-similar-to)
@@ -661,10 +587,12 @@ pub async fn initialize(
false,
&Vec::new(),
None,
tx_stats.clone(),
handles.stats.tx.clone(),
) {
// attempt to request the given url
if let Ok(resp) = make_request(&CONFIGURATION.client, &url, tx_stats.clone()).await {
if let Ok(resp) =
make_request(&CONFIGURATION.client, &url, handles.stats.tx.clone()).await
{
// if successful, create a filter based on the response's body
let fr = FeroxResponse::from(resp, true).await;
@@ -677,7 +605,7 @@ pub async fn initialize(
};
let boxed_filter = Box::new(filter);
add_filter_to_list_of_ferox_filters(boxed_filter, FILTERS.clone());
handles.filters.send(AddFilter(boxed_filter))?;
}
}
}
@@ -689,7 +617,10 @@ pub async fn initialize(
SCAN_LIMITER.add_permits(usize::MAX >> 4);
}
handles.filters.sync().await?;
log::trace!("exit: initialize");
Ok(())
}
#[cfg(test)]