From 923c59faac16c7344dd9a57dc1d600fb40d85d13 Mon Sep 17 00:00:00 2001 From: epi Date: Wed, 3 Feb 2021 07:18:19 -0600 Subject: [PATCH] silent and quiet are gtg --- shell_completions/_feroxbuster | 6 +++--- shell_completions/_feroxbuster.ps1 | 2 +- shell_completions/feroxbuster.fish | 2 +- src/config/container.rs | 15 ++++----------- src/config/mod.rs | 2 +- src/config/utils.rs | 16 +++++++++++++++- src/event_handlers/inputs.rs | 10 +++++++--- src/event_handlers/outputs.rs | 1 - src/extractor/container.rs | 14 +++++++++++--- src/heuristics.rs | 4 ++-- src/main.rs | 8 +++++++- src/parser.rs | 4 +++- src/progress.rs | 2 +- src/response.rs | 4 ++-- src/scan_manager/scan_container.rs | 8 +++++--- src/scan_manager/tests.rs | 4 ++-- src/scan_manager/utils.rs | 6 +++++- src/scanner.rs | 29 +++++++++++++++-------------- src/utils.rs | 12 ++---------- tests/test_heuristics.rs | 4 ++-- 20 files changed, 89 insertions(+), 64 deletions(-) diff --git a/shell_completions/_feroxbuster b/shell_completions/_feroxbuster index dec9eec..5d9a40e 100644 --- a/shell_completions/_feroxbuster +++ b/shell_completions/_feroxbuster @@ -60,9 +60,9 @@ _feroxbuster() { '--scan-limit=[Limit total number of concurrent scans (default: 0, i.e. no limit)]' \ '--rate-limit=[Limit number of requests per second (per directory) (default: 0, i.e. no limit)]' \ '--time-limit=[Limit total run time of all scans (ex: --time-limit 10m)]' \ -'*-v[Increase verbosity level (use -vv or more for greater effect. \[CAUTION\] 4 -v'\''s is probably too much)]' \ -'*--verbosity[Increase verbosity level (use -vv or more for greater effect. \[CAUTION\] 4 -v'\''s is probably too much)]' \ -'--silent[Only print URLs (good for piping a list of urls to other commands)]' \ +'(--silent)*-v[Increase verbosity level (use -vv or more for greater effect. \[CAUTION\] 4 -v'\''s is probably too much)]' \ +'(--silent)*--verbosity[Increase verbosity level (use -vv or more for greater effect. \[CAUTION\] 4 -v'\''s is probably too much)]' \ +'(-q --quiet)--silent[Only print URLs + turn off logging (good for piping a list of urls to other commands)]' \ '-q[Hide progress bars and banner (good for tmux windows w/ notifications)]' \ '--quiet[Hide progress bars and banner (good for tmux windows w/ notifications)]' \ '--json[Emit JSON logs to --output and --debug-log instead of normal text]' \ diff --git a/shell_completions/_feroxbuster.ps1 b/shell_completions/_feroxbuster.ps1 index 1b9aa44..d98a5b0 100644 --- a/shell_completions/_feroxbuster.ps1 +++ b/shell_completions/_feroxbuster.ps1 @@ -67,7 +67,7 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock { [CompletionResult]::new('--time-limit', 'time-limit', [CompletionResultType]::ParameterName, 'Limit total run time of all scans (ex: --time-limit 10m)') [CompletionResult]::new('-v', 'v', [CompletionResultType]::ParameterName, 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v''s is probably too much)') [CompletionResult]::new('--verbosity', 'verbosity', [CompletionResultType]::ParameterName, 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v''s is probably too much)') - [CompletionResult]::new('--silent', 'silent', [CompletionResultType]::ParameterName, 'Only print URLs (good for piping a list of urls to other commands)') + [CompletionResult]::new('--silent', 'silent', [CompletionResultType]::ParameterName, 'Only print URLs + turn off logging (good for piping a list of urls to other commands)') [CompletionResult]::new('-q', 'q', [CompletionResultType]::ParameterName, 'Hide progress bars and banner (good for tmux windows w/ notifications)') [CompletionResult]::new('--quiet', 'quiet', [CompletionResultType]::ParameterName, 'Hide progress bars and banner (good for tmux windows w/ notifications)') [CompletionResult]::new('--json', 'json', [CompletionResultType]::ParameterName, 'Emit JSON logs to --output and --debug-log instead of normal text') diff --git a/shell_completions/feroxbuster.fish b/shell_completions/feroxbuster.fish index 2727ec5..e2354d7 100644 --- a/shell_completions/feroxbuster.fish +++ b/shell_completions/feroxbuster.fish @@ -24,7 +24,7 @@ complete -c feroxbuster -n "__fish_use_subcommand" -s L -l scan-limit -d 'Limit complete -c feroxbuster -n "__fish_use_subcommand" -l rate-limit -d 'Limit number of requests per second (per directory) (default: 0, i.e. no limit)' complete -c feroxbuster -n "__fish_use_subcommand" -l time-limit -d 'Limit total run time of all scans (ex: --time-limit 10m)' complete -c feroxbuster -n "__fish_use_subcommand" -s v -l verbosity -d 'Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v\'s is probably too much)' -complete -c feroxbuster -n "__fish_use_subcommand" -l silent -d 'Only print URLs (good for piping a list of urls to other commands)' +complete -c feroxbuster -n "__fish_use_subcommand" -l silent -d 'Only print URLs + turn off logging (good for piping a list of urls to other commands)' complete -c feroxbuster -n "__fish_use_subcommand" -s q -l quiet -d 'Hide progress bars and banner (good for tmux windows w/ notifications)' complete -c feroxbuster -n "__fish_use_subcommand" -l json -d 'Emit JSON logs to --output and --debug-log instead of normal text' complete -c feroxbuster -n "__fish_use_subcommand" -s D -l dont-filter -d 'Don\'t auto-filter wildcard responses' diff --git a/src/config/container.rs b/src/config/container.rs index ca54fda..02bc002 100644 --- a/src/config/container.rs +++ b/src/config/container.rs @@ -2,6 +2,7 @@ use super::utils::{ depth, report_and_exit, save_state, serialized_type, status_codes, threads, timeout, user_agent, wordlist, OutputLevel, }; +use crate::config::determine_output_level; use crate::{ client, parser, scan_manager::resume_scan, utils::fmt_err, FeroxSerialize, DEFAULT_CONFIG_NAME, }; @@ -368,7 +369,7 @@ impl Configuration { let mut config = Configuration::default(); // read in all config files - let _ = Self::parse_config_files(&mut config); + Self::parse_config_files(&mut config)?; // read in the user provided options, this produces a separate instance of Configuration // in order to allow for potentially merging into a --resume-from Configuration @@ -719,16 +720,8 @@ impl Configuration { update_if_not_default!(&mut conf.verbosity, new.verbosity, 0); update_if_not_default!(&mut conf.silent, new.silent, false); update_if_not_default!(&mut conf.quiet, new.quiet, false); - if new.quiet && new.silent { - // user COULD have both as true in config file, take the more quiet of the two - conf.output_level = OutputLevel::Silent; - } else if new.quiet { - conf.output_level = OutputLevel::Quiet; - } else if new.silent { - conf.output_level = OutputLevel::Silent; - } else { - conf.output_level = OutputLevel::Default; - } + // use updated quiet/silent values to determin output level + conf.output_level = determine_output_level(conf.quiet, conf.silent); update_if_not_default!(&mut conf.output, new.output, ""); update_if_not_default!(&mut conf.redirects, new.redirects, false); update_if_not_default!(&mut conf.insecure, new.insecure, false); diff --git a/src/config/mod.rs b/src/config/mod.rs index 6fa763b..58d9505 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -6,4 +6,4 @@ mod utils; mod tests; pub use self::container::Configuration; -pub use self::utils::OutputLevel; +pub use self::utils::{determine_output_level, OutputLevel}; diff --git a/src/config/utils.rs b/src/config/utils.rs index c6f8189..3a83a77 100644 --- a/src/config/utils.rs +++ b/src/config/utils.rs @@ -68,7 +68,7 @@ pub(super) fn depth() -> usize { } /// enum representing the three possible states for informational output (not logging verbosity) -#[derive(Debug, Copy, Clone)] +#[derive(Debug, Copy, Clone, PartialEq)] pub enum OutputLevel { /// normal scan, no --quiet|--silent Default, @@ -87,3 +87,17 @@ impl Default for OutputLevel { Self::Default } } + +/// given the current settings for quiet and silent, determine output_level (DRY helper) +pub fn determine_output_level(quiet: bool, silent: bool) -> OutputLevel { + if quiet && silent { + // user COULD have both as true in config file, take the more quiet of the two + OutputLevel::Silent + } else if quiet { + OutputLevel::Quiet + } else if silent { + OutputLevel::Silent + } else { + OutputLevel::Default + } +} diff --git a/src/event_handlers/inputs.rs b/src/event_handlers/inputs.rs index 15bf3a0..441a34f 100644 --- a/src/event_handlers/inputs.rs +++ b/src/event_handlers/inputs.rs @@ -1,9 +1,9 @@ use super::*; use crate::{ progress::PROGRESS_PRINTER, - scan_manager::FeroxState, - scan_manager::PAUSE_SCAN, + scan_manager::{FeroxState, PAUSE_SCAN}, scanner::RESPONSES, + statistics::StatError, utils::{open_file, write_to}, SLEEP_DURATION, }; @@ -64,7 +64,11 @@ impl TermInputHandler { }); if result.is_err() { - log::error!("Could not set Ctrl+c handler; scan state will not be saved"); + log::warn!("Could not set Ctrl+c handler; scan state will not be saved"); + self.handles + .stats + .send(Command::AddError(StatError::Other)) + .unwrap_or_default(); } } } diff --git a/src/event_handlers/outputs.rs b/src/event_handlers/outputs.rs index bf7aa9c..8544dd4 100644 --- a/src/event_handlers/outputs.rs +++ b/src/event_handlers/outputs.rs @@ -90,7 +90,6 @@ impl FileOutHandler { skip_fail!(write_to(&*response, &mut file, self.config.json)); } Command::Exit => { - log::error!("file handler got Exit"); break; } Command::Sync(sender) => { diff --git a/src/extractor/container.rs b/src/extractor/container.rs index f74c521..a5263a8 100644 --- a/src/extractor/container.rs +++ b/src/extractor/container.rs @@ -1,9 +1,16 @@ use super::*; use crate::{ client, - event_handlers::{Command, Command::UpdateUsizeField, Handles}, + event_handlers::{ + Command, + Command::{AddError, UpdateUsizeField}, + Handles, + }, scan_manager::ScanOrder, - statistics::StatField::{LinksExtracted, TotalExpected}, + statistics::{ + StatError::Other, + StatField::{LinksExtracted, TotalExpected}, + }, url::FeroxUrl, utils::make_request, }; @@ -163,7 +170,8 @@ impl<'a> Extractor<'a> { } } else { // unexpected error has occurred - log::error!("Could not parse given url: {}", e); + log::warn!("Could not parse given url: {}", e); + self.handles.stats.send(AddError(Other)).unwrap_or_default(); } } } diff --git a/src/heuristics.rs b/src/heuristics.rs index cb20890..49918ef 100644 --- a/src/heuristics.rs +++ b/src/heuristics.rs @@ -233,11 +233,11 @@ impl HeuristicTests { OutputLevel::Default | OutputLevel::Quiet ) { ferox_print( - &format!("Could not connect to {}, skipping...", target_url), + &format!("Could not connect to {}, skipping... (try turning on warnings with -v)", target_url), &PROGRESS_PRINTER, ); } - log::error!("{}", e); + log::warn!("{}", e); } } } diff --git a/src/main.rs b/src/main.rs index e8ea449..ffaf730 100644 --- a/src/main.rs +++ b/src/main.rs @@ -327,7 +327,13 @@ fn main() -> Result<()> { let config = Arc::new(Configuration::new().with_context(|| "Could not create Configuration")?); // setup logging based on the number of -v's used - logger::initialize(config.clone())?; + if matches!( + config.output_level, + OutputLevel::Default | OutputLevel::Quiet + ) { + // don't log on --silent + logger::initialize(config.clone())?; + } // this function uses rlimit, which is not supported on windows #[cfg(not(target_os = "windows"))] diff --git a/src/parser.rs b/src/parser.rs index 0469ece..c186820 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -68,6 +68,7 @@ pub fn initialize() -> App<'static, 'static> { .long("verbosity") .takes_value(false) .multiple(true) + .conflicts_with("silent") .help("Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v's is probably too much)"), ) .arg( @@ -119,7 +120,8 @@ pub fn initialize() -> App<'static, 'static> { Arg::with_name("silent") .long("silent") .takes_value(false) - .help("Only print URLs (good for piping a list of urls to other commands)") + .conflicts_with("quiet") + .help("Only print URLs + turn off logging (good for piping a list of urls to other commands)") ) .arg( Arg::with_name("quiet") diff --git a/src/progress.rs b/src/progress.rs index cdc749f..fe505ab 100644 --- a/src/progress.rs +++ b/src/progress.rs @@ -45,7 +45,7 @@ pub fn add_bar(prefix: &str, length: u64, bar_type: BarType) -> ProgressBar { style.template("[{bar:.yellow/blue}] - {elapsed:<4} {pos:>7}/{len:7} {eta:7} {msg}") } BarType::Quiet => style.template("Scanning: {prefix}"), - }; + }; // todo update README with silent/quiet stuff let progress_bar = PROGRESS_BAR.add(ProgressBar::new(length)); diff --git a/src/response.rs b/src/response.rs index d8ea544..0f4f436 100644 --- a/src/response.rs +++ b/src/response.rs @@ -126,7 +126,7 @@ impl FeroxResponse { self.url = url; } Err(e) => { - log::error!("Could not parse {} into a Url: {}", url, e); + log::warn!("Could not parse {} into a Url: {}", url, e); } }; } @@ -200,7 +200,7 @@ impl FeroxResponse { // await the response's body Ok(text) => text, Err(e) => { - log::error!("Could not parse body from response: {}", e); + log::warn!("Could not parse body from response: {}", e); String::new() } } diff --git a/src/scan_manager/scan_container.rs b/src/scan_manager/scan_container.rs index edf59bc..85ee83e 100644 --- a/src/scan_manager/scan_container.rs +++ b/src/scan_manager/scan_container.rs @@ -97,7 +97,7 @@ impl FeroxScans { scans.push(scan); } Err(e) => { - log::error!("FeroxScans' container's mutex is poisoned: {}", e); + log::warn!("FeroxScans' container's mutex is poisoned: {}", e); return false; } } @@ -248,8 +248,10 @@ impl FeroxScans { pub fn print_known_responses(&self) { if let Ok(mut responses) = RESPONSES.responses.write() { for response in responses.iter_mut() { - let _level = response.output_level; - if !matches!(self.output_level, _level) { + if self.output_level != response.output_level { + // set the output_level prior to printing the response to ensure that the + // response's setting aligns with the overall configuration (since we're + // calling this from a resumed state) response.output_level = self.output_level; } PROGRESS_PRINTER.println(response.as_str()); diff --git a/src/scan_manager/tests.rs b/src/scan_manager/tests.rs index f047289..e011912 100644 --- a/src/scan_manager/tests.rs +++ b/src/scan_manager/tests.rs @@ -436,7 +436,7 @@ fn feroxscan_display() { scan_order: ScanOrder::Latest, scan_type: Default::default(), num_requests: 0, - quiet: false, + output_level: OutputLevel::Default, status: Default::default(), task: tokio::sync::Mutex::new(None), progress_bar: std::sync::Mutex::new(None), @@ -476,7 +476,7 @@ async fn ferox_scan_abort() { scan_order: ScanOrder::Latest, scan_type: Default::default(), num_requests: 0, - quiet: false, + output_level: OutputLevel::Default, status: std::sync::Mutex::new(ScanStatus::Running), task: tokio::sync::Mutex::new(Some(tokio::spawn(async move { sleep(Duration::from_millis(SLEEP_DURATION * 2)); diff --git a/src/scan_manager/utils.rs b/src/scan_manager/utils.rs index eddd4e5..f37a160 100644 --- a/src/scan_manager/utils.rs +++ b/src/scan_manager/utils.rs @@ -46,7 +46,7 @@ pub async fn start_max_time_thread(handles: Arc) { let _ = TermInputHandler::sigint_handler(handles.clone()); } - log::error!( + log::warn!( "Could not parse the value provided ({}), can't enforce time limit", handles.config.time_limit ); @@ -87,6 +87,10 @@ pub fn resume_scan(filename: &str) -> Configuration { } } + log::error!("{:#?}", config); + + log::error!("{:#?}", config); + log::trace!("exit: resume_scan -> {:?}", config); config } diff --git a/src/scanner.rs b/src/scanner.rs index 0f1d3ea..7222b2f 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -11,7 +11,7 @@ use tokio::sync::{oneshot, Semaphore}; use crate::{ event_handlers::{ - Command::{self, UpdateF64Field, UpdateUsizeField}, + Command::{self, AddError, UpdateF64Field, UpdateUsizeField}, Handles, }, extractor::{ @@ -21,7 +21,10 @@ use crate::{ heuristics, response::FeroxResponse, scan_manager::{FeroxResponses, ScanOrder, ScanStatus, PAUSE_SCAN}, - statistics::StatField::{DirScanTimes, ExpectedPerScan}, + statistics::{ + StatError::Other, + StatField::{DirScanTimes, ExpectedPerScan}, + }, url::FeroxUrl, utils::{fmt_err, make_request}, }; @@ -74,13 +77,9 @@ impl Requester { } /// limit the number of requests per second - pub async fn limit(&self) { - self.rate_limiter - .as_ref() - .unwrap() - .acquire_one() - .await - .unwrap(); // todo unwrap + pub async fn limit(&self) -> Result<()> { + self.rate_limiter.as_ref().unwrap().acquire_one().await?; + Ok(()) } /// Wrapper for [make_request](fn.make_request.html) @@ -94,7 +93,11 @@ impl Requester { for url in urls { if self.rate_limiter.is_some() { - self.limit().await; + // found a rate limiter, limit that junk! + if let Err(e) = self.limit().await { + log::warn!("Could not rate limit scan: {}", e); + self.handles.stats.send(AddError(Other)).unwrap_or_default(); + } } let response = make_request( @@ -260,9 +263,6 @@ impl FeroxScanner { // to false scanned_urls_clone.pause(true).await; } - // if requester_clone.rate_limiter.is_some() { - // requester_clone.limit().await; - // } requester_clone.request(&word).await }), pb, @@ -274,7 +274,8 @@ impl FeroxScanner { bar.inc(increment_len); } Err(e) => { - log::error!("error awaiting a response: {}", e); + log::warn!("error awaiting a response: {}", e); + self.handles.stats.send(AddError(Other)).unwrap_or_default(); } } }); diff --git a/src/utils.rs b/src/utils.rs index 15a8fe7..caed60c 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -97,12 +97,9 @@ pub async fn make_request( match client.get(url.to_owned()).send().await { Err(e) => { - let mut log_level = log::Level::Error; - log::trace!("exit: make_request -> {}", e); + if e.is_timeout() { - // only warn for timeouts, while actual errors are still left as errors - log_level = log::Level::Warn; send_command!(tx_stats, AddError(Timeout)); } else if e.is_redirect() { if let Some(last_redirect) = e.url() { @@ -135,12 +132,7 @@ pub async fn make_request( send_command!(tx_stats, AddError(Other)); } - if matches!(log_level, log::Level::Error) { - log::error!("Error while making request: {}", e); - } else { - log::warn!("Error while making request: {}", e); - } - + log::warn!("Error while making request: {}", e); bail!("{}", e) } Ok(resp) => { diff --git a/tests/test_heuristics.rs b/tests/test_heuristics.rs index 504fa9b..443b749 100644 --- a/tests/test_heuristics.rs +++ b/tests/test_heuristics.rs @@ -295,7 +295,7 @@ fn heuristics_wildcard_test_with_two_static_wildcards() { #[test] /// test finds a static wildcard and reports nothing to stdout -fn heuristics_wildcard_test_with_two_static_wildcards_with_quiet_enabled( +fn heuristics_wildcard_test_with_two_static_wildcards_with_silent_enabled( ) -> Result<(), Box> { let srv = MockServer::start(); let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?; @@ -321,7 +321,7 @@ fn heuristics_wildcard_test_with_two_static_wildcards_with_quiet_enabled( .arg("--wordlist") .arg(file.as_os_str()) .arg("--add-slash") - .arg("-q") + .arg("--silent") .unwrap(); teardown_tmp_directory(tmp_dir);