From e33816e9da977697fb05e2dcc69e14afad082b0a Mon Sep 17 00:00:00 2001 From: epi Date: Sun, 24 Jan 2021 15:03:43 -0600 Subject: [PATCH] filters now sent to the filter handler; still not acted upon --- src/event_handlers/command.rs | 3 + src/event_handlers/container.rs | 13 +++- src/event_handlers/filters.rs | 17 ++++- src/event_handlers/scans.rs | 4 +- src/extractor/builder.rs | 7 +- src/filters/container.rs | 8 +- src/main.rs | 35 +++------ src/scan_manager.rs | 101 +++++++++++++------------ src/scanner.rs | 129 ++++++++------------------------ 9 files changed, 138 insertions(+), 179 deletions(-) diff --git a/src/event_handlers/command.rs b/src/event_handlers/command.rs index be65dd7..e9bf940 100644 --- a/src/event_handlers/command.rs +++ b/src/event_handlers/command.rs @@ -56,6 +56,9 @@ pub enum Command { /// Command used to test that a spawned task succeeded in initialization Ping, + /// Just receive a sender and reply, used for slowing down the main thread + Sync(Sender), + /// Break out of the (infinite) mpsc receive loop Exit, } diff --git a/src/event_handlers/container.rs b/src/event_handlers/container.rs index 35d67bf..e48bfc9 100644 --- a/src/event_handlers/container.rs +++ b/src/event_handlers/container.rs @@ -1,7 +1,7 @@ use super::*; use crate::event_handlers::scans::ScanHandle; use crate::scan_manager::FeroxScans; -use crate::Joiner; +use crate::{CommandSender, Joiner}; use anyhow::{bail, Result}; use std::sync::{Arc, RwLock}; @@ -93,4 +93,15 @@ impl Handles { bail!("Could not get underlying FeroxScans") } + + /// Helper to easily get the (locked) underlying transmitter + pub fn sender(&self) -> Result { + if let Ok(guard) = self.scans.read().as_ref() { + if let Some(handle) = guard.as_ref() { + return Ok(handle.tx.clone()); + } + } + + bail!("Could not get underlying transmitter") + } } diff --git a/src/event_handlers/filters.rs b/src/event_handlers/filters.rs index b82d0d9..755ed47 100644 --- a/src/event_handlers/filters.rs +++ b/src/event_handlers/filters.rs @@ -2,7 +2,10 @@ use super::*; use crate::{filters::FeroxFilters, CommandSender, FeroxChannel, Joiner}; use anyhow::Result; use std::sync::Arc; -use tokio::sync::mpsc::{self, UnboundedReceiver}; +use tokio::sync::{ + mpsc::{self, UnboundedReceiver}, + oneshot, +}; #[derive(Debug)] /// Container for filters transmitter and FeroxFilters object @@ -26,6 +29,14 @@ impl FiltersHandle { self.tx.send(command)?; Ok(()) } + + /// Sync the handle with the handler + pub async fn sync(&self) -> Result<()> { + let (tx, rx) = oneshot::channel::(); + self.tx.send(Command::Sync(tx))?; + rx.await?; + Ok(()) + } } /// event handler for updating a single data structure of all active filters @@ -75,6 +86,10 @@ impl FiltersHandler { Command::AddFilter(filter) => { self.data.push(filter)?; } + Command::Sync(sender) => { + log::debug!("filters: {:?}", self); + sender.send(true).unwrap_or_default(); + } Command::Exit => break, _ => {} // no other commands needed for FilterHandler } diff --git a/src/event_handlers/scans.rs b/src/event_handlers/scans.rs index a12d0bd..d720440 100644 --- a/src/event_handlers/scans.rs +++ b/src/event_handlers/scans.rs @@ -1,8 +1,8 @@ use super::command::Command::UpdateUsizeField; use super::*; use crate::{ - scan_manager::{FeroxScan, FeroxScans}, - scanner::{scan_url, ScanOrder}, + scan_manager::{FeroxScan, FeroxScans, ScanOrder}, + scanner::scan_url, statistics::StatField::TotalScans, CommandReceiver, CommandSender, FeroxChannel, Joiner, }; diff --git a/src/extractor/builder.rs b/src/extractor/builder.rs index 85fd631..5b78f53 100644 --- a/src/extractor/builder.rs +++ b/src/extractor/builder.rs @@ -102,8 +102,7 @@ impl<'a> ExtractorBuilder<'a> { /// builder call to set `tx_recursion` pub fn recursion_transmitter(&mut self, tx_recursion: CommandSender) -> &mut Self { - // todo change to scans_transmitter or w/e same on struct; don't bother, going to make - // extractor take a Handles object later anyway + // todo change to scans_transmitter or w/e same on struct; don't bother, going to make extractor take a Handles object later anyway self.tx_recursion = Some(tx_recursion); self } @@ -116,9 +115,7 @@ impl<'a> ExtractorBuilder<'a> { /// builder call to set `tx_reporter` pub fn reporter_transmitter(&mut self, tx_reporter: CommandSender) -> &mut Self { - // todo change to outputs or w/e same on struct; don't bother, going to make - // extractor take a Handles object later anyway - + // todo change to outputs or w/e same on struct; don't bother, going to make extractor take a Handles object later anyway self.tx_reporter = Some(tx_reporter); self } diff --git a/src/filters/container.rs b/src/filters/container.rs index ab38f1b..c2ea03c 100644 --- a/src/filters/container.rs +++ b/src/filters/container.rs @@ -13,8 +13,12 @@ pub struct FeroxFilters { impl FeroxFilters { /// add a single FeroxFilter to the collection pub fn push(&self, filter: Box) -> Result<()> { - if let Ok(mut unlocked) = self.filters.lock() { - unlocked.push(filter) + if let Ok(mut guard) = self.filters.lock() { + if guard.contains(&filter) { + return Ok(()); + } + + guard.push(filter) } Ok(()) } diff --git a/src/main.rs b/src/main.rs index 61692aa..3c7447c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -26,8 +26,7 @@ use feroxbuster::{ }, heuristics, logger, scan_manager::{self, PAUSE_SCAN}, - scanner::{self, SCANNED_URLS}, - send_command, + scanner, utils::fmt_err, SLEEP_DURATION, }; @@ -118,7 +117,7 @@ async fn scan(targets: Vec, handles: Arc) -> Result<()> { handles.send_scan_command(UpdateWordlist(words.clone()))?; - scanner::initialize(words.len(), &CONFIGURATION, handles.stats.tx.clone()).await; + scanner::initialize(words.len(), &CONFIGURATION, handles.clone()).await?; // at this point, the stat thread's progress bar can be created; things that needed to happen // first: @@ -146,7 +145,7 @@ async fn scan(targets: Vec, handles: Arc) -> Result<()> { } /// Get targets from either commandline or stdin, pass them back to the caller as a Result -async fn get_targets() -> Result> { +async fn get_targets(handles: Arc) -> Result> { log::trace!("enter: get_targets"); let mut targets = vec![]; @@ -163,8 +162,9 @@ async fn get_targets() -> Result> { } else if CONFIGURATION.resumed { // resume-from can't be used with --url, and --stdin is marked false for every resumed // scan, making it mutually exclusive from either of the other two options - if let Ok(scans) = SCANNED_URLS.scans.read() { - // todo this block shouldn't be SCANNED_URLS + let ferox_scans = handles.ferox_scans()?; + + if let Ok(scans) = ferox_scans.scans.read() { for scan in scans.iter() { // SCANNED_URLS gets deserialized scans added to it at program start if --resume-from // is used, so scans that aren't marked complete still need to be scanned @@ -175,7 +175,7 @@ async fn get_targets() -> Result> { targets.push(scan.url.to_owned()); } - } + }; } else { targets.push(CONFIGURATION.target_url.clone()); } @@ -242,9 +242,8 @@ async fn wrapped_main() -> Result<()> { } // get targets from command line or stdin - // todo get_targets needs SCANNED_URLS replaced // todo a bunch of fucking functions needs SCANNED_URLS replaced - let targets = match get_targets().await { + let targets = match get_targets(handles.clone()).await { Ok(t) => t, Err(e) => { // should only happen in the event that there was an error reading from stdin @@ -305,7 +304,7 @@ async fn wrapped_main() -> Result<()> { // todo known things not working: confirm same # of requests seen in burp as reported // todo known things not working: enable build on this branch, compare memory usage with new recursion paradigm // todo known things not working: banner_print_output_file or w/e is hanging, probably means - // --output is wrong or that no-recursion is wrong + // todo known things not working: scan cancel menu is hard fkn broke clean_up(handles, tasks).await?; @@ -316,32 +315,24 @@ async fn wrapped_main() -> Result<()> { /// Single cleanup function that handles all the necessary drops/finishes etc required to gracefully /// shutdown the program async fn clean_up(handles: Arc, tasks: Tasks) -> Result<()> { - log::trace!( - "enter: clean_up({:?}, {:?})", // todo missing tasks - handles, - tasks - ); + log::trace!("enter: clean_up({:?}, {:?})", handles, tasks); let (tx, rx) = oneshot::channel::(); - log::error!("oneshot created, sending join command"); // todo handles.send_scan_command(JoinTasks(tx))?; - log::error!("sent join command"); // todo rx.await?; log::info!("All scans complete!"); // terminal handler closes file handler if one is in use - log::error!("Sending Exit"); // todo remove handles.output.send(Exit)?; - // send_command!(handles.output.tx, Exit); // todo use handles.thing.send in this function tasks.terminal.await??; log::trace!("terminal handler closed"); - send_command!(handles.filters.tx, Exit); + handles.filters.send(Exit)?; tasks.filters.await??; log::trace!("filters handler closed"); - send_command!(handles.stats.tx, Exit); + handles.stats.send(Exit)?; tasks.stats.await??; log::trace!("stats handler closed"); @@ -352,8 +343,6 @@ async fn clean_up(handles: Arc, tasks: Tasks) -> Result<()> { // the final trace messages above PROGRESS_PRINTER.finish(); - // drop(tx_stats); - log::trace!("exit: clean_up"); Ok(()) } diff --git a/src/scan_manager.rs b/src/scan_manager.rs index bd2735c..3ac570b 100644 --- a/src/scan_manager.rs +++ b/src/scan_manager.rs @@ -49,10 +49,24 @@ pub static PAUSE_SCAN: AtomicBool = AtomicBool::new(false); /// Simple enum used to flag a `FeroxScan` as likely a directory or file #[derive(Debug, Serialize, Deserialize, Copy, Clone)] pub enum ScanType { + /// Just a file being requested File, + + /// A an entire directory that might be scanned Directory, } +#[derive(Debug, Copy, Clone)] +/// Simple enum to designate whether a URL was passed in by the user (Initial) or found during +/// scanning (Latest) +pub enum ScanOrder { + /// Url was passed in by the user + Initial, + + /// Url was found during scanning + Latest, +} + /// Default implementation for ScanType impl Default for ScanType { /// Return ScanType::File as default @@ -110,15 +124,18 @@ impl Default for FeroxScan { /// Implementation of FeroxScan impl FeroxScan { /// Stop a currently running scan - pub async fn abort(&self) { + pub async fn abort(&self) -> Result<()> { let mut guard = self.task.lock().await; if guard.is_some() { - let task = std::mem::replace(&mut *guard, None).unwrap(); - task.abort(); - self.set_status(ScanStatus::Cancelled).unwrap(); // todo - self.stop_progress_bar(); + if let Some(task) = std::mem::replace(&mut *guard, None) { + task.abort(); + self.set_status(ScanStatus::Cancelled)?; + self.stop_progress_bar(); + } } + + Ok(()) } /// small wrapper to set the JoinHandle @@ -138,29 +155,34 @@ impl FeroxScan { /// Simple helper to call .finish on the scan's progress bar fn stop_progress_bar(&self) { - // todo do something with the unwrap see set_status todo note - let guard = self.progress_bar.lock().unwrap(); - - if guard.is_some() { - (*guard).as_ref().unwrap().finish_at_current_pos() + if let Ok(guard) = self.progress_bar.lock() { + if guard.is_some() { + (*guard).as_ref().unwrap().finish_at_current_pos() + } } } /// Simple helper get a progress bar pub fn progress_bar(&self) -> ProgressBar { - // todo do something with the unwrap see set_status todo note - let mut guard = self.progress_bar.lock().unwrap(); + match self.progress_bar.lock() { + Ok(mut guard) => { + if guard.is_some() { + (*guard).as_ref().unwrap().clone() + } else { + let pb = add_bar(&self.url, self.num_requests, BarType::Default); + pb.reset_elapsed(); - if guard.is_some() { - (*guard).as_ref().unwrap().clone() - } else { - let pb = add_bar(&self.url, self.num_requests, BarType::Default); + let _ = std::mem::replace(&mut *guard, Some(pb.clone())); + pb + } + } + Err(_) => { + log::warn!("Could not unlock progress bar on {:?}", self); + let pb = add_bar(&self.url, self.num_requests, BarType::Default); + pb.reset_elapsed(); - pb.reset_elapsed(); - - let _ = std::mem::replace(&mut *guard, Some(pb.clone())); - - pb + pb + } } } @@ -208,15 +230,17 @@ impl FeroxScan { false } - /// todo doc + /// await a task's completion, similar to a thread's join; perform necessary bookkeeping pub async fn join(&self) { log::trace!("enter join({:?})", self); let mut guard = self.task.lock().await; if guard.is_some() { - let task = std::mem::replace(&mut *guard, None).unwrap(); - task.await.unwrap(); - self.set_status(ScanStatus::Complete).unwrap(); // todo + if let Some(task) = std::mem::replace(&mut *guard, None) { + task.await.unwrap(); + self.set_status(ScanStatus::Complete) + .unwrap_or_else(|e| log::warn!("Could not mark scan complete: {}", e)) + } } log::trace!("exit join({:?})", self); @@ -595,6 +619,7 @@ impl FeroxScans { // todo check this assumption, as we swap out the task with None once joined continue; } + self.menu.println(&format!("fdaf {}", scan)); if matches!(scan.scan_type, ScanType::Directory) { // we're only interested in displaying directory scans, as those are @@ -629,7 +654,10 @@ impl FeroxScans { if input == 'y' || input == '\n' { self.menu.println(&format!("Stopping {}...", selected.url)); - selected.abort().await; + selected + .abort() + .await + .unwrap_or_else(|e| log::warn!("Could not cancel task: {}", e)); } else { self.menu.println("Ok, doing nothing..."); } @@ -798,25 +826,6 @@ impl FeroxScans { } scans } - - // todo remove probably - // pub async fn join_all(&self) -> usize { - // let mut joined = 0; - // if let Ok(u_scans) = self.scans.read() { - // for scan in u_scans.iter() { - // let mut guard = scan.lock().await; - // if guard.task.is_none() { - // continue; - // } - // guard.join().await; - // joined += 1; - // - // if let Ok(mut u_scan) = scan.lock() { - // } - // } - // } - // joined - // } } /// Container around a locked vector of `FeroxResponse`s, adds wrappers for insertion and search @@ -1460,7 +1469,7 @@ mod tests { progress_bar: std::sync::Mutex::new(None), }; - scan.abort().await; + scan.abort().await.unwrap(); assert!(matches!( *scan.status.lock().unwrap(), diff --git a/src/scanner.rs b/src/scanner.rs index 2c52666..34aefae 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -1,7 +1,7 @@ use crate::{ config::{Configuration, CONFIGURATION}, event_handlers::{ - Command::{self, UpdateF64Field, UpdateUsizeField}, + Command::{self, AddFilter, UpdateF64Field, UpdateUsizeField}, Handles, }, extractor::ExtractorBuilder, @@ -10,7 +10,7 @@ use crate::{ WordsFilter, }, heuristics, - scan_manager::{FeroxResponses, FeroxScans, ScanStatus, PAUSE_SCAN}, + scan_manager::{FeroxResponses, FeroxScans, ScanOrder, ScanStatus, PAUSE_SCAN}, send_command, statistics::StatField::{DirScanTimes, ExpectedPerScan, WildcardsFiltered}, traits::FeroxFilter, @@ -51,43 +51,6 @@ lazy_static! { } -/// Adds the given FeroxFilter to the given list of FeroxFilter implementors -/// -/// If the given list did not already contain the filter, return true; otherwise return false -fn add_filter_to_list_of_ferox_filters( - filter: Box, - ferox_filters: Arc>>>, -) -> bool { - log::trace!( - "enter: add_filter_to_list_of_ferox_filters({:?}, {:?})", - filter, - ferox_filters - ); - // todo move to filters handler - - match ferox_filters.write() { - Ok(mut filters) => { - // If the set did not contain the assigned filter, true is returned. - // If the set did contain the assigned filter, false is returned. - if filters.contains(&filter) { - log::trace!("exit: add_filter_to_list_of_ferox_filters -> false"); - return false; - } - - filters.push(filter); - - log::trace!("exit: add_filter_to_list_of_ferox_filters -> true"); - true - } - Err(e) => { - // poisoned lock - log::error!("Set of wildcard filters poisoned: {}", e); - log::trace!("exit: add_filter_to_list_of_ferox_filters -> false"); - false - } - } -} - /// Creates a vector of formatted Urls /// /// At least one value will be returned (base_url + word) @@ -325,8 +288,10 @@ async fn make_requests(target_url: &str, word: &str, base_depth: usize, handles: handles.stats.tx.clone(), ); - let scanned_urls = handles.ferox_scans().unwrap(); // unwrap todo - let tx_scans = handles.scans.read().unwrap().as_ref().unwrap().tx.clone(); // todo abstract away + let scanned_urls = handles.ferox_scans().expect("Could not get FeroxScans"); + // todo abstract away, and by that i mean that extractor and try_recursion should either take + // Handles or be put into a struct somewhere + let tx_scans = handles.scans.read().unwrap().as_ref().unwrap().tx.clone(); for url in urls { if let Ok(response) = @@ -377,26 +342,13 @@ pub fn send_report(report_sender: CommandSender, response: FeroxResponse) { match report_sender.send(Command::Report(Box::new(response))) { Ok(_) => {} Err(e) => { - log::warn!("{}", e); - // todo back to error + log::error!("{}", e); } } log::trace!("exit: send_report"); } -#[derive(Debug, Copy, Clone)] -/// Simple enum to designate whether a URL was passed in by the user (Initial) or found during -/// scanning (Latest) -pub enum ScanOrder { - // todo is this the right location? - /// Url was passed in by the user - Initial, - - /// Url was found during scanning - Latest, -} - /// Scan a given url using a given wordlist /// /// This is the primary entrypoint for the scanner @@ -485,18 +437,14 @@ pub async fn scan_url( None => Box::new(WildcardFilter::default()), }; - // todo move to filters handler - add_filter_to_list_of_ferox_filters(filter, FILTERS.clone()); + handles.filters.send(AddFilter(filter))?; let scanned_urls = handles.ferox_scans()?; // producer tasks (mp of mpsc); responsible for making requests - // todo .deref().to_owned() seems like they cancel eachother out let producers = stream::iter(looping_words.deref().to_owned()) .map(|word| { - // todo abstract away more scans shit let handles_clone = handles.clone(); - let _txs = handles.stats.tx.clone(); let pb = progress_bar.clone(); // progress bar is an Arc around internal state let tgt = target_url.to_string(); // done to satisfy 'static lifetime below let scanned_urls_clone = scanned_urls.clone(); @@ -539,28 +487,6 @@ pub async fn scan_url( ferox_scan.finish()?; - // todo remove - // // manually drop tx in order for the rx task's while loops to eval to false - // log::trace!("dropped recursion handler's transmitter"); - // drop(tx_dir); - - // note: in v1.11.2 i removed the join_all call that used to handle the recurser handles. - // nothing appears to change by having them removed, however, if ever a revert is needed - // this is the place and anything prior to 1.11.2 will have the code to do so - // { - // if let Ok(urls) = SCANNED_URLS.get_scan_by_url(target_url).unwrap().lock() { - // urls.task.as_ref().unwrap().into_inner().unwrap().await; - // } - // } - - // todo remove - log::error!("SCAN URL EXIT: {}", target_url); - - // for mut fut in futures { - // let x = Arc::try_unwrap(fut).unwrap(); - // x.await; - // } - log::trace!("exit: scan_url"); Ok(()) @@ -571,28 +497,28 @@ pub async fn scan_url( pub async fn initialize( num_words: usize, config: &Configuration, - tx_stats: UnboundedSender, -) { + handles: Arc, +) -> Result<()> { log::trace!( "enter: initialize({}, {:?}, {:?})", num_words, config, - tx_stats + handles ); // number of requests only needs to be calculated once, and then can be reused let num_reqs_expected: u64 = if config.extensions.is_empty() { - num_words.try_into().unwrap() + num_words.try_into()? } else { let total = num_words * (config.extensions.len() + 1); - total.try_into().unwrap() + total.try_into()? }; // tell Stats object about the number of expected requests - send_command!( - tx_stats, - UpdateUsizeField(ExpectedPerScan, num_reqs_expected as usize) - ); + handles.stats.send(UpdateUsizeField( + ExpectedPerScan, + num_reqs_expected as usize, + ))?; // add any status code filters to `FILTERS` (-C|--filter-status) for code_filter in &config.filter_status { @@ -600,7 +526,7 @@ pub async fn initialize( filter_code: *code_filter, }; let boxed_filter = Box::new(filter); - add_filter_to_list_of_ferox_filters(boxed_filter, FILTERS.clone()); + handles.filters.send(AddFilter(boxed_filter))?; } // add any line count filters to `FILTERS` (-N|--filter-lines) @@ -609,7 +535,7 @@ pub async fn initialize( line_count: *lines_filter, }; let boxed_filter = Box::new(filter); - add_filter_to_list_of_ferox_filters(boxed_filter, FILTERS.clone()); + handles.filters.send(AddFilter(boxed_filter))?; } // add any line count filters to `FILTERS` (-W|--filter-words) @@ -618,7 +544,7 @@ pub async fn initialize( word_count: *words_filter, }; let boxed_filter = Box::new(filter); - add_filter_to_list_of_ferox_filters(boxed_filter, FILTERS.clone()); + handles.filters.send(AddFilter(boxed_filter))?; } // add any line count filters to `FILTERS` (-S|--filter-size) @@ -627,7 +553,7 @@ pub async fn initialize( content_length: *size_filter, }; let boxed_filter = Box::new(filter); - add_filter_to_list_of_ferox_filters(boxed_filter, FILTERS.clone()); + handles.filters.send(AddFilter(boxed_filter))?; } // add any regex filters to `FILTERS` (-X|--filter-regex) @@ -649,7 +575,7 @@ pub async fn initialize( compiled, }; let boxed_filter = Box::new(filter); - add_filter_to_list_of_ferox_filters(boxed_filter, FILTERS.clone()); + handles.filters.send(AddFilter(boxed_filter))?; } // add any similarity filters to `FILTERS` (--filter-similar-to) @@ -661,10 +587,12 @@ pub async fn initialize( false, &Vec::new(), None, - tx_stats.clone(), + handles.stats.tx.clone(), ) { // attempt to request the given url - if let Ok(resp) = make_request(&CONFIGURATION.client, &url, tx_stats.clone()).await { + if let Ok(resp) = + make_request(&CONFIGURATION.client, &url, handles.stats.tx.clone()).await + { // if successful, create a filter based on the response's body let fr = FeroxResponse::from(resp, true).await; @@ -677,7 +605,7 @@ pub async fn initialize( }; let boxed_filter = Box::new(filter); - add_filter_to_list_of_ferox_filters(boxed_filter, FILTERS.clone()); + handles.filters.send(AddFilter(boxed_filter))?; } } } @@ -689,7 +617,10 @@ pub async fn initialize( SCAN_LIMITER.add_permits(usize::MAX >> 4); } + handles.filters.sync().await?; + log::trace!("exit: initialize"); + Ok(()) } #[cfg(test)]