h/feroxbuster
1
Fork 0
mirror of https://github.com/epi052/feroxbuster.git synced 2026-04-22 01:51:13 -03:00
Code Issues Packages Projects Releases Wiki Activity

907 dont skip dir listings (#1192)

Browse Source 
* bumped version to 2.11.0

* updated deps

* new cli options

* added --request-file, --protocol, --scan-dir-listings

* added tests / clippy

* removed errant module definition

* implemented visible bar limiter

* many fixes; feature implemented i believe

* added banner test for limit-bars

* beginning troubleshooting of recursion panic

* put a bandaid on trace-level logging bug

* clippy
This commit is contained in:
epi
2024-09-14 14:00:14 -04:00
committed by GitHub
parent b44c52f0ea
commit 762bfc4e78
29 changed files with 2551 additions and 824 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
src/banner/container.rs
View File

@@ -176,6 +176,15 @@ pub struct Banner {
/// represents Configuration.collect_words
force_recursion: BannerEntry,
/// represents Configuration.protocol
protocol: BannerEntry,
/// represents Configuration.scan_dir_listings
scan_dir_listings: BannerEntry,
/// represents Configuration.limit_bars
limit_bars: BannerEntry,
}
/// implementation of Banner
@@ -320,6 +329,12 @@ impl Banner {
BannerEntry::new("🚫", "Do Not Recurse", &config.no_recursion.to_string())
};
let protocol = if config.protocol.to_lowercase() == "http" {
BannerEntry::new("🔓", "Default Protocol", &config.protocol)
} else {
BannerEntry::new("🔒", "Default Protocol", &config.protocol)
};
let scan_limit = BannerEntry::new(
"🦥",
"Concurrent Scan Limit",
@@ -331,6 +346,11 @@ impl Banner {
let replay_proxy = BannerEntry::new("🎥", "Replay Proxy", &config.replay_proxy);
let auto_tune = BannerEntry::new("🎶", "Auto Tune", &config.auto_tune.to_string());
let auto_bail = BannerEntry::new("🙅", "Auto Bail", &config.auto_bail.to_string());
let scan_dir_listings = BannerEntry::new(
"📂",
"Scan Dir Listings",
&config.scan_dir_listings.to_string(),
);
let cfg = BannerEntry::new("💉", "Config File", &config.config);
let proxy = BannerEntry::new("💎", "Proxy", &config.proxy);
let server_certs = BannerEntry::new(
@@ -341,6 +361,8 @@ impl Banner {
let client_cert = BannerEntry::new("🏅", "Client Certificate", &config.client_cert);
let client_key = BannerEntry::new("🔑", "Client Key", &config.client_key);
let threads = BannerEntry::new("🚀", "Threads", &config.threads.to_string());
let limit_bars =
BannerEntry::new("📊", "Limit Dir Scan Bars", &config.limit_bars.to_string());
let wordlist = BannerEntry::new("📖", "Wordlist", &config.wordlist);
let timeout = BannerEntry::new("💥", "Timeout (secs)", &config.timeout.to_string());
let user_agent = BannerEntry::new("🦡", "User-Agent", &config.user_agent);
@@ -455,6 +477,9 @@ impl Banner {
collect_words,
dont_collect,
config: cfg,
scan_dir_listings,
protocol,
limit_bars,
version: VERSION.to_string(),
update_status: UpdateStatus::Unknown,
}
@@ -595,6 +620,14 @@ by Ben "epi" Risher {} ver: {}"#,
}
// followed by the maybe printed or variably displayed values
if !config.request_file.is_empty() || !config.target_url.starts_with("http") {
writeln!(&mut writer, "{}", self.protocol)?;
}
if config.limit_bars > 0 {
writeln!(&mut writer, "{}", self.limit_bars)?;
}
if !config.config.is_empty() {
writeln!(&mut writer, "{}", self.config)?;
}
@@ -662,6 +695,10 @@ by Ben "epi" Risher {} ver: {}"#,
writeln!(&mut writer, "{}", self.output)?;
}
if config.scan_dir_listings {
writeln!(&mut writer, "{}", self.scan_dir_listings)?;
}
if !config.debug_log.is_empty() {
writeln!(&mut writer, "{}", self.debug_log)?;
}

110
src/config/container.rs
View File

@@ -1,15 +1,16 @@
use super::utils::{
backup_extensions, depth, extract_links, ignored_extensions, methods, report_and_exit,
save_state, serialized_type, status_codes, threads, timeout, user_agent, wordlist, OutputLevel,
backup_extensions, depth, determine_requester_policy, extract_links, ignored_extensions,
methods, parse_request_file, report_and_exit, request_protocol, save_state, serialized_type,
split_header, split_query, status_codes, threads, timeout, user_agent, wordlist, OutputLevel,
RequesterPolicy,
};
use crate::config::determine_output_level;
use crate::config::utils::determine_requester_policy;
use crate::{
client, parser,
scan_manager::resume_scan,
traits::FeroxSerialize,
utils::{fmt_err, parse_url_with_raw_path},
utils::{fmt_err, module_colorizer, parse_url_with_raw_path, status_colorizer},
DEFAULT_CONFIG_NAME,
};
use anyhow::{anyhow, Context, Result};
@@ -332,6 +333,22 @@ pub struct Configuration {
/// Auto update app feature
#[serde(skip)]
pub update_app: bool,
/// whether to recurse into directory listings or not
#[serde(default)]
pub scan_dir_listings: bool,
/// path to a raw request file generated by burp or similar
#[serde(skip)]
pub request_file: String,
/// default request protocol
#[serde(default = "request_protocol")]
pub protocol: String,
/// number of directory scan bars to show at any given time, 0 is no limit
#[serde(default)]
pub limit_bars: usize,
}
impl Default for Configuration {
@@ -378,10 +395,12 @@ impl Default for Configuration {
resumed: false,
stdin: false,
json: false,
scan_dir_listings: false,
verbosity: 0,
scan_limit: 0,
parallel: 0,
rate_limit: 0,
limit_bars: 0,
add_slash: false,
insecure: false,
redirects: false,
@@ -403,6 +422,8 @@ impl Default for Configuration {
time_limit: String::new(),
resume_from: String::new(),
replay_proxy: String::new(),
request_file: String::new(),
protocol: request_protocol(),
server_certs: Vec::new(),
queries: Vec::new(),
extensions: Vec::new(),
@@ -476,12 +497,16 @@ impl Configuration {
/// - **depth**: `4` (maximum recursion depth)
/// - **force_recursion**: `false` (still respects recursion depth)
/// - **scan_limit**: `0` (no limit on concurrent scans imposed)
/// - **limit_bars**: `0` (no limit on number of directory scan bars shown)
/// - **parallel**: `0` (no limit on parallel scans imposed)
/// - **rate_limit**: `0` (no limit on requests per second imposed)
/// - **time_limit**: `None` (no limit on length of scan imposed)
/// - **replay_proxy**: `None` (no limit on concurrent scans imposed)
/// - **replay_codes**: [`DEFAULT_RESPONSE_CODES`](constant.DEFAULT_RESPONSE_CODES.html)
/// - **update_app**: `false`
/// - **scan_dir_listings**: `false`
/// - **request_file**: `None`
/// - **protocol**: `https`
///
/// After which, any values defined in a
/// [ferox-config.toml](constant.DEFAULT_CONFIG_NAME.html) config file will override the
@@ -555,6 +580,18 @@ impl Configuration {
// merge the cli options into the config file options and return the result
Self::merge_config(&mut config, cli_config);
// if the user provided a raw request file as the target, we'll need to parse out
// the provided info and update the config with those values. This call needs to
// come after the cli/config merge so we can allow the cli options to override
// the raw request values (i.e. --headers "stuff: things" should override a "stuff"
// header from the raw request).
//
// Additionally, this call needs to come before client rebuild so that the things
// like user-agent can be set at the client level instead of the header level.
if !config.request_file.is_empty() {
parse_request_file(&mut config)?;
}
// rebuild clients is the last step in either code branch
Self::try_rebuild_clients(&mut config);
@@ -614,10 +651,13 @@ impl Configuration {
update_config_with_num_type_if_present!(&mut config.depth, args, "depth", usize);
update_config_with_num_type_if_present!(&mut config.scan_limit, args, "scan_limit", usize);
update_config_with_num_type_if_present!(&mut config.rate_limit, args, "rate_limit", usize);
update_config_with_num_type_if_present!(&mut config.limit_bars, args, "limit_bars", usize);
update_config_if_present!(&mut config.wordlist, args, "wordlist", String);
update_config_if_present!(&mut config.output, args, "output", String);
update_config_if_present!(&mut config.debug_log, args, "debug_log", String);
update_config_if_present!(&mut config.resume_from, args, "resume_from", String);
update_config_if_present!(&mut config.request_file, args, "request_file", String);
update_config_if_present!(&mut config.protocol, args, "protocol", String);
if let Ok(Some(inner)) = args.try_get_one::<String>("time_limit") {
inner.clone_into(&mut config.time_limit);
@@ -831,6 +871,10 @@ impl Configuration {
config.save_state = false;
}
if came_from_cli!(args, "scan_dir_listings") || came_from_cli!(args, "thorough") {
config.scan_dir_listings = true;
}
if came_from_cli!(args, "dont_filter") {
config.dont_filter = true;
}
@@ -871,6 +915,25 @@ impl Configuration {
// occurrences_of returns 0 if none are found; this is protected in
// an if block for the same reason as the quiet option
config.verbosity = args.get_count("verbosity");
// todo: starting on 2.11.0 (907-dont-skip-dir-listings), trace-level
// logging started causing the following error:
//
// thread 'tokio-runtime-worker' has overflowed its stack
// fatal runtime error: stack overflow
// Aborted (core dumped)
//
// as a temporary fix, we'll disable trace logging to prevent the stack
// overflow until I get time to investigate the root cause
if config.verbosity > 3 {
eprintln!(
"{} {}: Trace level logging is disabled; setting log level to debug",
status_colorizer("WRN"),
module_colorizer("Configuration::parse_cli_args"),
);
config.verbosity = 3;
}
}
if came_from_cli!(args, "no_recursion") {
@@ -932,23 +995,11 @@ impl Configuration {
if let Some(headers) = args.get_many::<String>("headers") {
for val in headers {
let mut split_val = val.split(':');
// explicitly take first split value as header's name
let name = split_val.next().unwrap().trim();
// all other items in the iterator returned by split, when combined with the
// original split deliminator (:), make up the header's final value
let value = split_val.collect::<Vec<&str>>().join(":");
if value.starts_with(' ') && !value.starts_with(" ") {
// first character is a space and the second character isn't
// we can trim the leading space
let trimmed = value.trim_start();
config.headers.insert(name.to_string(), trimmed.to_string());
} else {
config.headers.insert(name.to_string(), value.to_string());
}
let Ok((name, value)) = split_header(val) else {
log::warn!("Invalid header: {}", val);
continue;
};
config.headers.insert(name, value);
}
}
@@ -982,14 +1033,11 @@ impl Configuration {
if let Some(queries) = args.get_many::<String>("queries") {
for val in queries {
// same basic logic used as reading in the headers HashMap above
let mut split_val = val.split('=');
let name = split_val.next().unwrap().trim();
let value = split_val.collect::<Vec<&str>>().join("=");
config.queries.push((name.to_string(), value.to_string()));
let Ok((name, value)) = split_query(val) else {
log::warn!("Invalid query string: {}", val);
continue;
};
config.queries.push((name, value));
}
}
@@ -1111,6 +1159,7 @@ impl Configuration {
update_if_not_default!(&mut conf.client_cert, new.client_cert, "");
update_if_not_default!(&mut conf.client_key, new.client_key, "");
update_if_not_default!(&mut conf.verbosity, new.verbosity, 0);
update_if_not_default!(&mut conf.limit_bars, new.limit_bars, 0);
update_if_not_default!(&mut conf.silent, new.silent, false);
update_if_not_default!(&mut conf.quiet, new.quiet, false);
update_if_not_default!(&mut conf.auto_bail, new.auto_bail, false);
@@ -1171,12 +1220,15 @@ impl Configuration {
Vec::<u16>::new()
);
update_if_not_default!(&mut conf.dont_filter, new.dont_filter, false);
update_if_not_default!(&mut conf.scan_dir_listings, new.scan_dir_listings, false);
update_if_not_default!(&mut conf.scan_limit, new.scan_limit, 0);
update_if_not_default!(&mut conf.parallel, new.parallel, 0);
update_if_not_default!(&mut conf.rate_limit, new.rate_limit, 0);
update_if_not_default!(&mut conf.replay_proxy, new.replay_proxy, "");
update_if_not_default!(&mut conf.debug_log, new.debug_log, "");
update_if_not_default!(&mut conf.resume_from, new.resume_from, "");
update_if_not_default!(&mut conf.request_file, new.request_file, "");
update_if_not_default!(&mut conf.protocol, new.protocol, request_protocol());
update_if_not_default!(&mut conf.timeout, new.timeout, timeout());
update_if_not_default!(&mut conf.user_agent, new.user_agent, user_agent());

36
src/config/tests.rs
View File

@@ -49,6 +49,10 @@ fn setup_config_test() -> Configuration {
json = true
save_state = false
depth = 1
limit_bars = 3
protocol = "http"
request_file = "/some/request/file"
scan_dir_listings = true
force_recursion = true
filter_size = [4120]
filter_regex = ["^ignore me$"]
@@ -87,6 +91,7 @@ fn default_configuration() {
assert_eq!(config.timeout, timeout());
assert_eq!(config.verbosity, 0);
assert_eq!(config.scan_limit, 0);
assert_eq!(config.limit_bars, 0);
assert!(!config.silent);
assert!(!config.quiet);
assert_eq!(config.output_level, OutputLevel::Default);
@@ -107,6 +112,7 @@ fn default_configuration() {
assert!(!config.collect_extensions);
assert!(!config.collect_backups);
assert!(!config.collect_words);
assert!(!config.scan_dir_listings);
assert!(config.regex_denylist.is_empty());
assert_eq!(config.queries, Vec::new());
assert_eq!(config.filter_size, Vec::<u64>::new());
@@ -125,6 +131,8 @@ fn default_configuration() {
assert_eq!(config.client_cert, String::new());
assert_eq!(config.client_key, String::new());
assert_eq!(config.backup_extensions, backup_extensions());
assert_eq!(config.protocol, request_protocol());
assert_eq!(config.request_file, String::new());
}
#[test]
@@ -260,6 +268,13 @@ fn config_reads_verbosity() {
assert_eq!(config.verbosity, 1);
}
#[test]
/// parse the test config and see that the value parsed is correct
fn config_reads_limit_bars() {
let config = setup_config_test();
assert_eq!(config.limit_bars, 3);
}
#[test]
/// parse the test config and see that the value parsed is correct
fn config_reads_output() {
@@ -444,6 +459,27 @@ fn config_reads_time_limit() {
assert_eq!(config.time_limit, "10m");
}
#[test]
/// parse the test config and see that the value parsed is correct
fn config_reads_scan_dir_listings() {
let config = setup_config_test();
assert!(config.scan_dir_listings);
}
#[test]
/// parse the test config and see that the value parsed is correct
fn config_reads_protocol() {
let config = setup_config_test();
assert_eq!(config.protocol, "http");
}
#[test]
/// parse the test config and see that the value parsed is correct
fn config_reads_request_file() {
let config = setup_config_test();
assert_eq!(config.request_file, String::new());
}
#[test]
/// parse the test config and see that the value parsed is correct
fn config_reads_resume_from() {

1056
src/config/utils.rs
View File

File diff suppressed because it is too large Load Diff

9
src/event_handlers/scans.rs
View File

@@ -120,7 +120,10 @@ impl ScanHandler {
pub fn initialize(handles: Arc<Handles>) -> (Joiner, ScanHandle) {
log::trace!("enter: initialize");
let data = Arc::new(FeroxScans::new(handles.config.output_level));
let data = Arc::new(FeroxScans::new(
handles.config.output_level,
handles.config.limit_bars,
));
let (tx, rx): FeroxChannel<Command> = mpsc::unbounded_channel();
let max_depth = handles.config.depth;
@@ -322,7 +325,9 @@ impl ScanHandler {
let scan = if let Some(ferox_scan) = self.data.get_scan_by_url(&target) {
ferox_scan // scan already known
} else {
self.data.add_directory_scan(&target, order).1 // add the new target; return FeroxScan
self.data
.add_directory_scan(&target, order, self.handles.clone())
.1 // add the new target; return FeroxScan
};
if should_test_deny

7
src/extractor/container.rs
View File

@@ -228,8 +228,11 @@ impl<'a> Extractor<'a> {
if resp.is_file() || !resp.is_directory() {
log::debug!("Extracted File: {}", resp);
c_scanned_urls
.add_file_scan(resp.url().as_str(), ScanOrder::Latest);
c_scanned_urls.add_file_scan(
resp.url().as_str(),
ScanOrder::Latest,
c_handles.clone(),
);
if c_handles.config.collect_extensions {
// no real reason this should fail

6
src/extractor/tests.rs
View File

@@ -386,7 +386,11 @@ async fn request_link_bails_on_seen_url() -> Result<()> {
});
let scans = Arc::new(FeroxScans::default());
scans.add_file_scan(&served, ScanOrder::Latest);
scans.add_file_scan(
&served,
ScanOrder::Latest,
Arc::new(Handles::for_testing(None, None).0),
);
let robots = setup_extractor(ExtractionTarget::RobotsTxt, scans.clone());
let body = setup_extractor(ExtractionTarget::ResponseBody, scans);

4
src/main.rs
View File

@@ -197,9 +197,9 @@ async fn get_targets(handles: Arc<Handles>) -> Result<Vec<String>> {
}
}
if !target.starts_with("http") && !target.starts_with("https") {
if !target.starts_with("http") {
// --url hackerone.com
*target = format!("https://{target}");
*target = format!("{}://{target}", handles.config.protocol);
}
}

35
src/parser.rs
View File

@@ -40,12 +40,12 @@ pub fn initialize() -> Command {
Arg::new("url")
.short('u')
.long("url")
.required_unless_present_any(["stdin", "resume_from", "update_app"])
.required_unless_present_any(["stdin", "resume_from", "update_app", "request_file"])
.help_heading("Target selection")
.value_name("URL")
.use_value_delimiter(true)
.value_hint(ValueHint::Url)
.help("The target URL (required, unless [--stdin || --resume-from] used)"),
.help("The target URL (required, unless [--stdin || --resume-from || --request-file] used)"),
)
.arg(
Arg::new("stdin")
@@ -64,6 +64,15 @@ pub fn initialize() -> Command {
.help("State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)")
.conflicts_with("url")
.num_args(1),
).arg(
Arg::new("request_file")
.long("request-file")
.help_heading("Target selection")
.value_hint(ValueHint::FilePath)
.conflicts_with("url")
.num_args(1)
.value_name("REQUEST_FILE")
.help("Raw HTTP request file to use as a template for all requests"),
);
/////////////////////////////////////////////////////////////////////
@@ -100,7 +109,7 @@ pub fn initialize() -> Command {
.num_args(0)
.help_heading("Composite settings")
.conflicts_with_all(["rate_limit", "auto_bail"])
.help("Use the same settings as --smart and set --collect-extensions to true"),
.help("Use the same settings as --smart and set --collect-extensions and --scan-dir-listings to true"),
);
/////////////////////////////////////////////////////////////////////
@@ -248,6 +257,13 @@ pub fn initialize() -> Command {
.help_heading("Request settings")
.num_args(0)
.help("Append / to each request's URL")
).arg(
Arg::new("protocol")
.long("protocol")
.value_name("PROTOCOL")
.num_args(1)
.help_heading("Request settings")
.help("Specify the protocol to use when targeting via --request-file or --url with domain only (default: https)"),
);
/////////////////////////////////////////////////////////////////////
@@ -574,6 +590,12 @@ pub fn initialize() -> Command {
.help(
"File extension(s) to Ignore while collecting extensions (only used with --collect-extensions)",
),
).arg(
Arg::new("scan_dir_listings")
.long("scan-dir-listings")
.num_args(0)
.help_heading("Scan settings")
.help("Force scans to recurse into directory listings")
);
/////////////////////////////////////////////////////////////////////
@@ -638,6 +660,13 @@ pub fn initialize() -> Command {
.num_args(0)
.help_heading("Output settings")
.help("Disable state output file (*.state)")
).arg(
Arg::new("limit_bars")
.long("limit-bars")
.value_name("NUM_BARS_TO_SHOW")
.num_args(1)
.help_heading("Output settings")
.help("Number of directory scan bars to show at any given time (default: no limit)"),
);
/////////////////////////////////////////////////////////////////////

15
src/progress.rs
View File

@@ -31,6 +31,15 @@ pub enum BarType {
/// Add an [indicatif::ProgressBar](https://docs.rs/indicatif/latest/indicatif/struct.ProgressBar.html)
/// to the global [PROGRESS_BAR](../config/struct.PROGRESS_BAR.html)
pub fn add_bar(prefix: &str, length: u64, bar_type: BarType) -> ProgressBar {
let pb = ProgressBar::new(length).with_prefix(prefix.to_string());
update_style(&pb, bar_type);
PROGRESS_BAR.add(pb)
}
/// Update the style of a progress bar based on the `BarType`
pub fn update_style(bar: &ProgressBar, bar_type: BarType) {
let mut style = ProgressStyle::default_bar().progress_chars("#>-").with_key(
"smoothed_per_sec",
|state: &indicatif::ProgressState, w: &mut dyn std::fmt::Write| match (
@@ -66,11 +75,7 @@ pub fn add_bar(prefix: &str, length: u64, bar_type: BarType) -> ProgressBar {
BarType::Quiet => style.template("Scanning: {prefix}").unwrap(),
};
PROGRESS_BAR.add(
ProgressBar::new(length)
.with_style(style)
.with_prefix(prefix.to_string()),
)
bar.set_style(style);
}
#[cfg(test)]

201
src/scan_manager/scan.rs
View File

@@ -1,7 +1,10 @@
use super::*;
use crate::{
config::OutputLevel,
event_handlers::Handles,
progress::update_style,
progress::{add_bar, BarType},
scan_manager::utils::determine_bar_type,
scanner::PolicyTrigger,
};
use anyhow::Result;
@@ -16,10 +19,20 @@ use std::{
time::Instant,
};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use tokio::{sync, task::JoinHandle};
use uuid::Uuid;
#[derive(Debug, Default, Copy, Clone)]
pub enum Visibility {
/// whether a FeroxScan's progress bar is currently shown
#[default]
Visible,
/// whether a FeroxScan's progress bar is currently hidden
Hidden,
}
/// Struct to hold scan-related state
///
/// The purpose of this container is to open up the pathway to aborting currently running tasks and
@@ -58,7 +71,7 @@ pub struct FeroxScan {
pub(super) task: sync::Mutex<Option<JoinHandle<()>>>,
/// The progress bar associated with this scan
pub(super) progress_bar: Mutex<Option<ProgressBar>>,
pub progress_bar: Mutex<Option<ProgressBar>>,
/// whether or not the user passed --silent|--quiet on the command line
pub(super) output_level: OutputLevel,
@@ -74,6 +87,12 @@ pub struct FeroxScan {
/// tracker for the time at which this scan was started
pub(super) start_time: Instant,
/// whether the progress bar is currently visible or hidden
pub(super) visible: AtomicBool,
/// handles object pointer
pub(super) handles: Option<Arc<Handles>>,
}
/// Default implementation for FeroxScan
@@ -86,6 +105,7 @@ impl Default for FeroxScan {
id: new_id,
task: sync::Mutex::new(None), // tokio mutex
status: Mutex::new(ScanStatus::default()),
handles: None,
num_requests: 0,
requests_made_so_far: 0,
scan_order: ScanOrder::Latest,
@@ -98,14 +118,54 @@ impl Default for FeroxScan {
status_429s: Default::default(),
status_403s: Default::default(),
start_time: Instant::now(),
visible: AtomicBool::new(true),
}
}
}
/// Implementation of FeroxScan
impl FeroxScan {
/// return the visibility of the scan as a boolean
pub fn visible(&self) -> bool {
self.visible.load(Ordering::Relaxed)
}
pub fn swap_visibility(&self) {
// fetch_xor toggles the boolean to its opposite and returns the previous value
let visible = self.visible.fetch_xor(true, Ordering::Relaxed);
let Ok(bar) = self.progress_bar.lock() else {
log::warn!("couldn't unlock progress bar for {}", self.url);
return;
};
if bar.is_none() {
log::warn!("there is no progress bar for {}", self.url);
return;
}
let Some(handles) = self.handles.as_ref() else {
log::warn!("couldn't access handles pointer for {}", self.url);
return;
};
let bar_type = if !visible {
// visibility was false before we xor'd the value
match handles.config.output_level {
OutputLevel::Default => BarType::Default,
OutputLevel::Quiet => BarType::Quiet,
OutputLevel::Silent | OutputLevel::SilentJSON => BarType::Hidden,
}
} else {
// visibility was true before we xor'd the value
BarType::Hidden
};
update_style(bar.as_ref().unwrap(), bar_type);
}
/// Stop a currently running scan
pub async fn abort(&self) -> Result<()> {
pub async fn abort(&self, active_bars: usize) -> Result<()> {
log::trace!("enter: abort");
match self.task.try_lock() {
@@ -114,7 +174,7 @@ impl FeroxScan {
log::trace!("aborting {:?}", self);
task.abort();
self.set_status(ScanStatus::Cancelled)?;
self.stop_progress_bar();
self.stop_progress_bar(active_bars);
}
}
Err(e) => {
@@ -151,15 +211,26 @@ impl FeroxScan {
}
/// Simple helper to call .finish on the scan's progress bar
pub(super) fn stop_progress_bar(&self) {
pub(super) fn stop_progress_bar(&self, active_bars: usize) {
if let Ok(guard) = self.progress_bar.lock() {
if guard.is_some() {
let pb = (*guard).as_ref().unwrap();
if pb.position() > self.num_requests {
pb.finish()
let bar_limit = if let Some(handles) = self.handles.as_ref() {
handles.config.limit_bars
} else {
pb.abandon()
0
};
if bar_limit > 0 && bar_limit < active_bars {
pb.finish_and_clear();
return;
}
if pb.position() > self.num_requests {
pb.finish();
} else {
pb.abandon();
}
}
}
@@ -172,12 +243,18 @@ impl FeroxScan {
if guard.is_some() {
(*guard).as_ref().unwrap().clone()
} else {
let bar_type = match self.output_level {
OutputLevel::Default => BarType::Default,
OutputLevel::Quiet => BarType::Quiet,
OutputLevel::Silent | OutputLevel::SilentJSON => BarType::Hidden,
let (active_bars, bar_limit) = if let Some(handles) = self.handles.as_ref() {
if let Ok(scans) = handles.ferox_scans() {
(scans.number_of_bars(), handles.config.limit_bars)
} else {
(0, handles.config.limit_bars)
}
} else {
(0, 0)
};
let bar_type = determine_bar_type(bar_limit, active_bars, self.output_level);
let pb = add_bar(&self.url, self.num_requests, bar_type);
pb.reset_elapsed();
@@ -191,12 +268,18 @@ impl FeroxScan {
Err(_) => {
log::warn!("Could not unlock progress bar on {:?}", self);
let bar_type = match self.output_level {
OutputLevel::Default => BarType::Default,
OutputLevel::Quiet => BarType::Quiet,
OutputLevel::Silent | OutputLevel::SilentJSON => BarType::Hidden,
let (active_bars, bar_limit) = if let Some(handles) = self.handles.as_ref() {
if let Ok(scans) = handles.ferox_scans() {
(scans.number_of_bars(), handles.config.limit_bars)
} else {
(0, handles.config.limit_bars)
}
} else {
(0, 0)
};
let bar_type = determine_bar_type(bar_limit, active_bars, self.output_level);
let pb = add_bar(&self.url, self.num_requests, bar_type);
pb.reset_elapsed();
@@ -206,6 +289,7 @@ impl FeroxScan {
}
/// Given a URL and ProgressBar, create a new FeroxScan, wrap it in an Arc and return it
#[allow(clippy::too_many_arguments)]
pub fn new(
url: &str,
scan_type: ScanType,
@@ -213,6 +297,8 @@ impl FeroxScan {
num_requests: u64,
output_level: OutputLevel,
pb: Option<ProgressBar>,
visibility: bool,
handles: Arc<Handles>,
) -> Arc<Self> {
Arc::new(Self {
url: url.to_string(),
@@ -222,14 +308,16 @@ impl FeroxScan {
num_requests,
output_level,
progress_bar: Mutex::new(pb),
visible: AtomicBool::new(visibility),
handles: Some(handles),
..Default::default()
})
}
/// Mark the scan as complete and stop the scan's progress bar
pub fn finish(&self) -> Result<()> {
pub fn finish(&self, active_bars: usize) -> Result<()> {
self.set_status(ScanStatus::Complete)?;
self.stop_progress_bar();
self.stop_progress_bar(active_bars);
Ok(())
}
@@ -262,6 +350,22 @@ impl FeroxScan {
false
}
/// small wrapper to inspect ScanStatus and see if it's Running
pub fn is_running(&self) -> bool {
if let Ok(guard) = self.status.lock() {
return matches!(*guard, ScanStatus::Running);
}
false
}
/// small wrapper to inspect ScanStatus and see if it's NotStarted
pub fn is_not_started(&self) -> bool {
if let Ok(guard) = self.status.lock() {
return matches!(*guard, ScanStatus::NotStarted);
}
false
}
/// await a task's completion, similar to a thread's join; perform necessary bookkeeping
pub async fn join(&self) {
log::trace!("enter join({:?})", self);
@@ -507,6 +611,8 @@ mod tests {
1000,
OutputLevel::Default,
None,
true,
Arc::new(Handles::for_testing(None, None).0),
);
scan.add_error();
@@ -532,6 +638,7 @@ mod tests {
scan_order: ScanOrder::Initial,
num_requests: 0,
requests_made_so_far: 0,
visible: AtomicBool::new(true),
status: Mutex::new(ScanStatus::Running),
task: Default::default(),
progress_bar: Mutex::new(None),
@@ -540,6 +647,7 @@ mod tests {
status_429s: Default::default(),
errors: Default::default(),
start_time: Instant::now(),
handles: None,
};
let pb = scan.progress_bar();
@@ -551,7 +659,62 @@ mod tests {
assert_eq!(req_sec, 100);
scan.finish().unwrap();
scan.finish(0).unwrap();
assert_eq!(scan.requests_per_second(), 0);
}
#[test]
fn test_swap_visibility() {
let scan = FeroxScan::new(
"http://localhost",
ScanType::Directory,
ScanOrder::Latest,
1000,
OutputLevel::Default,
None,
true,
Arc::new(Handles::for_testing(None, None).0),
);
assert!(scan.visible());
scan.swap_visibility();
assert!(!scan.visible());
scan.swap_visibility();
assert!(scan.visible());
scan.swap_visibility();
assert!(!scan.visible());
scan.swap_visibility();
assert!(scan.visible());
}
#[test]
/// test for is_running method
fn test_is_running() {
let scan = FeroxScan::new(
"http://localhost",
ScanType::Directory,
ScanOrder::Latest,
1000,
OutputLevel::Default,
None,
true,
Arc::new(Handles::for_testing(None, None).0),
);
assert!(scan.is_not_started());
assert!(!scan.is_running());
assert!(!scan.is_complete());
assert!(!scan.is_cancelled());
*scan.status.lock().unwrap() = ScanStatus::Running;
assert!(!scan.is_not_started());
assert!(scan.is_running());
assert!(!scan.is_complete());
assert!(!scan.is_cancelled());
}
}

116
src/scan_manager/scan_container.rs
View File

@@ -12,6 +12,7 @@ use crate::{
config::OutputLevel,
progress::PROGRESS_PRINTER,
progress::{add_bar, BarType},
scan_manager::utils::determine_bar_type,
scan_manager::{MenuCmd, MenuCmdResult},
scanner::RESPONSES,
traits::FeroxSerialize,
@@ -61,6 +62,9 @@ pub struct FeroxScans {
/// vector of extensions discovered and collected during scans
pub(crate) collected_extensions: RwLock<HashSet<String>>,
/// stored value for Configuration.limit_bars
bar_limit: usize,
}
/// Serialize implementation for FeroxScans
@@ -93,9 +97,10 @@ impl Serialize for FeroxScans {
/// Implementation of `FeroxScans`
impl FeroxScans {
/// given an OutputLevel, create a new FeroxScans object
pub fn new(output_level: OutputLevel) -> Self {
pub fn new(output_level: OutputLevel, bar_limit: usize) -> Self {
Self {
output_level,
bar_limit,
..Default::default()
}
}
@@ -388,8 +393,9 @@ impl FeroxScans {
if input == 'y' || input == '\n' {
self.menu.println(&format!("Stopping {}...", selected.url));
let active_bars = self.number_of_bars();
selected
.abort()
.abort(active_bars)
.await
.unwrap_or_else(|e| log::warn!("Could not cancel task: {}", e));
@@ -521,14 +527,22 @@ impl FeroxScans {
/// if a resumed scan is already complete, display a completed progress bar to the user
pub fn print_completed_bars(&self, bar_length: usize) -> Result<()> {
let bar_type = match self.output_level {
OutputLevel::Default => BarType::Message,
OutputLevel::Quiet => BarType::Quiet,
OutputLevel::Silent | OutputLevel::SilentJSON => return Ok(()), // fast exit when --silent was used
};
if self.output_level == OutputLevel::SilentJSON || self.output_level == OutputLevel::Silent
{
// fast exit when --silent was used
return Ok(());
}
let bar_type: BarType =
determine_bar_type(self.bar_limit, self.number_of_bars(), self.output_level);
if let Ok(scans) = self.scans.read() {
for scan in scans.iter() {
if matches!(bar_type, BarType::Hidden) {
// no need to show hidden bars
continue;
}
if scan.is_complete() {
// these scans are complete, and just need to be shown to the user
let pb = add_bar(
@@ -605,6 +619,7 @@ impl FeroxScans {
url: &str,
scan_type: ScanType,
scan_order: ScanOrder,
handles: Arc<Handles>,
) -> (bool, Arc<FeroxScan>) {
let bar_length = if let Ok(guard) = self.bar_length.lock() {
*guard
@@ -612,14 +627,11 @@ impl FeroxScans {
0
};
let active_bars = self.number_of_bars();
let bar_type = determine_bar_type(self.bar_limit, active_bars, self.output_level);
let bar = match scan_type {
ScanType::Directory => {
let bar_type = match self.output_level {
OutputLevel::Default => BarType::Default,
OutputLevel::Quiet => BarType::Quiet,
OutputLevel::Silent | OutputLevel::SilentJSON => BarType::Hidden,
};
let progress_bar = add_bar(url, bar_length, bar_type);
progress_bar.reset_elapsed();
@@ -629,6 +641,8 @@ impl FeroxScans {
ScanType::File => None,
};
let is_visible = !matches!(bar_type, BarType::Hidden);
let ferox_scan = FeroxScan::new(
url,
scan_type,
@@ -636,6 +650,8 @@ impl FeroxScans {
bar_length,
self.output_level,
bar,
is_visible,
handles,
);
// If the set did not contain the scan, true is returned.
@@ -650,9 +666,14 @@ impl FeroxScans {
/// If `FeroxScans` did not already contain the scan, return true; otherwise return false
///
/// Also return a reference to the new `FeroxScan`
pub fn add_directory_scan(&self, url: &str, scan_order: ScanOrder) -> (bool, Arc<FeroxScan>) {
pub fn add_directory_scan(
&self,
url: &str,
scan_order: ScanOrder,
handles: Arc<Handles>,
) -> (bool, Arc<FeroxScan>) {
let normalized = format!("{}/", url.trim_end_matches('/'));
self.add_scan(&normalized, ScanType::Directory, scan_order)
self.add_scan(&normalized, ScanType::Directory, scan_order, handles)
}
/// Given a url, create a new `FeroxScan` and add it to `FeroxScans` as a File Scan
@@ -660,8 +681,65 @@ impl FeroxScans {
/// If `FeroxScans` did not already contain the scan, return true; otherwise return false
///
/// Also return a reference to the new `FeroxScan`
pub fn add_file_scan(&self, url: &str, scan_order: ScanOrder) -> (bool, Arc<FeroxScan>) {
self.add_scan(url, ScanType::File, scan_order)
pub fn add_file_scan(
&self,
url: &str,
scan_order: ScanOrder,
handles: Arc<Handles>,
) -> (bool, Arc<FeroxScan>) {
self.add_scan(url, ScanType::File, scan_order, handles)
}
/// returns the number of active AND visible scans; supports --limit-bars functionality
pub fn number_of_bars(&self) -> usize {
let Ok(scans) = self.scans.read() else {
return 0;
};
// starting at one ensures we don't have an extra bar
// due to counting up from 0 when there's actually 1 bar
let mut count = 1;
for scan in &*scans {
if scan.is_active() && scan.visible() {
count += 1;
}
}
count
}
/// make one hidden bar visible; supports --limit-bars functionality
pub fn make_visible(&self) {
if let Ok(guard) = self.scans.read() {
// when swapping visibility, we'll prefer an actively running scan
// if none are found, we'll
let mut queued = None;
for scan in &*guard {
if !matches!(scan.scan_type, ScanType::Directory) {
// visibility only makes sense for directory scans
continue;
}
if scan.visible() {
continue;
}
if scan.is_running() {
scan.swap_visibility();
return;
}
if queued.is_none() && scan.is_not_started() {
queued = Some(scan.clone());
}
}
if let Some(scan) = queued {
scan.swap_visibility();
}
}
}
/// small helper to determine whether any scans are active or not
@@ -726,7 +804,7 @@ mod tests {
#[test]
/// unknown extension should be added to collected_extensions
fn unknown_extension_is_added_to_collected_extensions() {
let scans = FeroxScans::new(OutputLevel::Default);
let scans = FeroxScans::new(OutputLevel::Default, 0);
assert_eq!(0, scans.collected_extensions.read().unwrap().len());
@@ -739,7 +817,7 @@ mod tests {
#[test]
/// known extension should not be added to collected_extensions
fn known_extension_is_added_to_collected_extensions() {
let scans = FeroxScans::new(OutputLevel::Default);
let scans = FeroxScans::new(OutputLevel::Default, 0);
scans
.collected_extensions
.write()

90
src/scan_manager/tests.rs
View File

@@ -15,6 +15,7 @@ use crate::{
use indicatif::ProgressBar;
use predicates::prelude::*;
use regex::Regex;
use std::sync::atomic::AtomicBool;
use std::sync::{atomic::Ordering, Arc};
use std::thread::sleep;
use std::time::Instant;
@@ -57,7 +58,12 @@ async fn scanner_pause_scan_with_finished_spinner() {
fn add_url_to_list_of_scanned_urls_with_unknown_url() {
let urls = FeroxScans::default();
let url = "http://unknown_url";
let (result, _scan) = urls.add_scan(url, ScanType::Directory, ScanOrder::Latest);
let (result, _scan) = urls.add_scan(
url,
ScanType::Directory,
ScanOrder::Latest,
Arc::new(Handles::for_testing(None, None).0),
);
assert!(result);
}
@@ -75,11 +81,18 @@ fn add_url_to_list_of_scanned_urls_with_known_url() {
pb.length().unwrap(),
OutputLevel::Default,
Some(pb),
true,
Arc::new(Handles::for_testing(None, None).0),
);
assert!(urls.insert(scan));
let (result, _scan) = urls.add_scan(url, ScanType::Directory, ScanOrder::Latest);
let (result, _scan) = urls.add_scan(
url,
ScanType::Directory,
ScanOrder::Latest,
Arc::new(Handles::for_testing(None, None).0),
);
assert!(!result);
}
@@ -97,6 +110,8 @@ fn stop_progress_bar_stops_bar() {
pb.length().unwrap(),
OutputLevel::Default,
Some(pb),
true,
Arc::new(Handles::for_testing(None, None).0),
);
assert!(!scan
@@ -107,7 +122,7 @@ fn stop_progress_bar_stops_bar() {
.unwrap()
.is_finished());
scan.stop_progress_bar();
scan.stop_progress_bar(0);
assert!(scan
.progress_bar
@@ -124,18 +139,25 @@ fn add_url_to_list_of_scanned_urls_with_known_url_without_slash() {
let urls = FeroxScans::default();
let url = "http://unknown_url";
let scan = FeroxScan::new(
let scan: Arc<FeroxScan> = FeroxScan::new(
url,
ScanType::File,
ScanOrder::Latest,
0,
OutputLevel::Default,
None,
true,
Arc::new(Handles::for_testing(None, None).0),
);
assert!(urls.insert(scan));
let (result, _scan) = urls.add_scan(url, ScanType::File, ScanOrder::Latest);
let (result, _scan) = urls.add_scan(
url,
ScanType::File,
ScanOrder::Latest,
Arc::new(Handles::for_testing(None, None).0),
);
assert!(!result);
}
@@ -155,6 +177,8 @@ async fn call_display_scans() {
pb.length().unwrap(),
OutputLevel::Default,
Some(pb),
true,
Arc::new(Handles::for_testing(None, None).0),
);
let scan_two = FeroxScan::new(
url_two,
@@ -163,9 +187,11 @@ async fn call_display_scans() {
pb_two.length().unwrap(),
OutputLevel::Default,
Some(pb_two),
true,
Arc::new(Handles::for_testing(None, None).0),
);
scan_two.finish().unwrap(); // one complete, one incomplete
scan_two.finish(0).unwrap(); // one complete, one incomplete
scan_two
.set_task(tokio::spawn(async move {
sleep(Duration::from_millis(SLEEP_DURATION));
@@ -190,6 +216,8 @@ fn partial_eq_compares_the_id_field() {
0,
OutputLevel::Default,
None,
true,
Arc::new(Handles::for_testing(None, None).0),
);
let scan_two = FeroxScan::new(
url,
@@ -198,6 +226,8 @@ fn partial_eq_compares_the_id_field() {
0,
OutputLevel::Default,
None,
true,
Arc::new(Handles::for_testing(None, None).0),
);
assert!(!scan.eq(&scan_two));
@@ -280,6 +310,8 @@ fn ferox_scan_serialize() {
0,
OutputLevel::Default,
None,
true,
Arc::new(Handles::for_testing(None, None).0),
);
let fs_json = format!(
r#"{{"id":"{}","url":"https://spiritanimal.com","normalized_url":"https://spiritanimal.com/","scan_type":"Directory","status":"NotStarted","num_requests":0,"requests_made_so_far":0}}"#,
@@ -298,6 +330,8 @@ fn ferox_scans_serialize() {
0,
OutputLevel::Default,
None,
true,
Arc::new(Handles::for_testing(None, None).0),
);
let ferox_scans = FeroxScans::default();
let ferox_scans_json = format!(
@@ -360,6 +394,8 @@ fn feroxstates_feroxserialize_implementation() {
0,
OutputLevel::Default,
None,
true,
Arc::new(Handles::for_testing(None, None).0),
);
let ferox_scans = FeroxScans::default();
let saved_id = ferox_scan.id.clone();
@@ -501,12 +537,15 @@ fn feroxstates_feroxserialize_implementation() {
r#""method":"GET""#,
r#""content_length":173"#,
r#""line_count":10"#,
r#""limit_bars":0"#,
r#""word_count":16"#,
r#""headers""#,
r#""server":"nginx/1.16.1"#,
r#""collect_extensions":true"#,
r#""collect_backups":false"#,
r#""collect_words":false"#,
r#""scan_dir_listings":false"#,
r#""protocol":"https""#,
r#""filters":[{"filter_code":100},{"word_count":200},{"content_length":300},{"line_count":400},{"compiled":".*","raw_string":".*"},{"hash":1,"original_url":"http://localhost:12345/"}]"#,
r#""collected_extensions":["php"]"#,
r#""dont_collect":["tif","tiff","ico","cur","bmp","webp","svg","png","jpg","jpeg","jfif","gif","avif","apng","pjpeg","pjp","mov","wav","mpg","mpeg","mp3","mp4","m4a","m4p","m4v","ogg","webm","ogv","oga","flac","aac","3gp","css","zip","xls","xml","gz","tgz"]"#,
@@ -567,8 +606,10 @@ fn feroxscan_display() {
normalized_url: String::from("http://localhost/"),
scan_order: ScanOrder::Latest,
scan_type: Default::default(),
handles: Some(Arc::new(Handles::for_testing(None, None).0)),
num_requests: 0,
requests_made_so_far: 0,
visible: AtomicBool::new(true),
start_time: Instant::now(),
output_level: OutputLevel::Default,
status_403s: Default::default(),
@@ -617,6 +658,7 @@ async fn ferox_scan_abort() {
requests_made_so_far: 0,
start_time: Instant::now(),
output_level: OutputLevel::Default,
visible: AtomicBool::new(true),
status_403s: Default::default(),
status_429s: Default::default(),
status: std::sync::Mutex::new(ScanStatus::Running),
@@ -625,9 +667,10 @@ async fn ferox_scan_abort() {
}))),
progress_bar: std::sync::Mutex::new(None),
errors: Default::default(),
handles: Some(Arc::new(Handles::for_testing(None, None).0)),
};
scan.abort().await.unwrap();
scan.abort(0).await.unwrap();
assert!(matches!(
*scan.status.lock().unwrap(),
@@ -718,15 +761,26 @@ fn split_to_nums_is_correct() {
#[test]
/// given a deep url, find the correct scan
fn get_base_scan_by_url_finds_correct_scan() {
let handles = Arc::new(Handles::for_testing(None, None).0);
let urls = FeroxScans::default();
let url = "http://localhost";
let url1 = "http://localhost/stuff";
let url2 = "http://shlocalhost/stuff/things";
let url3 = "http://shlocalhost/stuff/things/mostuff";
let (_, scan) = urls.add_scan(url, ScanType::Directory, ScanOrder::Latest);
let (_, scan1) = urls.add_scan(url1, ScanType::Directory, ScanOrder::Latest);
let (_, scan2) = urls.add_scan(url2, ScanType::Directory, ScanOrder::Latest);
let (_, scan3) = urls.add_scan(url3, ScanType::Directory, ScanOrder::Latest);
let (_, scan) = urls.add_scan(url, ScanType::Directory, ScanOrder::Latest, handles.clone());
let (_, scan1) = urls.add_scan(
url1,
ScanType::Directory,
ScanOrder::Latest,
handles.clone(),
);
let (_, scan2) = urls.add_scan(
url2,
ScanType::Directory,
ScanOrder::Latest,
handles.clone(),
);
let (_, scan3) = urls.add_scan(url3, ScanType::Directory, ScanOrder::Latest, handles);
assert_eq!(
urls.get_base_scan_by_url("http://localhost/things.php")
@@ -759,7 +813,12 @@ fn get_base_scan_by_url_finds_correct_scan() {
fn get_base_scan_by_url_finds_correct_scan_without_trailing_slash() {
let urls = FeroxScans::default();
let url = "http://localhost";
let (_, scan) = urls.add_scan(url, ScanType::Directory, ScanOrder::Latest);
let (_, scan) = urls.add_scan(
url,
ScanType::Directory,
ScanOrder::Latest,
Arc::new(Handles::for_testing(None, None).0),
);
assert_eq!(
urls.get_base_scan_by_url("http://localhost/BKPMiherrortBPKcw")
.unwrap()
@@ -773,7 +832,12 @@ fn get_base_scan_by_url_finds_correct_scan_without_trailing_slash() {
fn get_base_scan_by_url_finds_correct_scan_with_trailing_slash() {
let urls = FeroxScans::default();
let url = "http://127.0.0.1:41971/";
let (_, scan) = urls.add_scan(url, ScanType::Directory, ScanOrder::Latest);
let (_, scan) = urls.add_scan(
url,
ScanType::Directory,
ScanOrder::Latest,
Arc::new(Handles::for_testing(None, None).0),
);
assert_eq!(
urls.get_base_scan_by_url("http://127.0.0.1:41971/BKPMiherrortBPKcw")
.unwrap()

83
src/scan_manager/utils.rs
View File

@@ -1,7 +1,12 @@
#[cfg(not(test))]
use crate::event_handlers::TermInputHandler;
use crate::{
config::Configuration, event_handlers::Handles, parser::TIMESPEC_REGEX, scanner::RESPONSES,
config::{Configuration, OutputLevel},
event_handlers::Handles,
parser::TIMESPEC_REGEX,
progress::BarType,
scan_manager::scan::Visibility,
scanner::RESPONSES,
};
use std::{fs::File, io::BufReader, sync::Arc};
@@ -90,3 +95,79 @@ pub fn resume_scan(filename: &str) -> Configuration {
log::trace!("exit: resume_scan -> {:?}", config);
config
}
/// determine the type of progress bar to display
/// takes both --limit-bars and output-level (--quiet|--silent|etc)
/// into account to arrive at a `BarType`
pub fn determine_bar_type(
bar_limit: usize,
number_of_bars: usize,
output_level: OutputLevel,
) -> BarType {
let visibility = if bar_limit == 0 {
// no limit from cli, just set the value to visible
// this protects us from a mutex unlock in number_of_bars
// in the normal case
Visibility::Visible
} else if bar_limit < number_of_bars {
// active bars exceed limit; hidden
Visibility::Hidden
} else {
Visibility::Visible
};
match (output_level, visibility) {
(OutputLevel::Default, Visibility::Visible) => BarType::Default,
(OutputLevel::Quiet, Visibility::Visible) => BarType::Quiet,
(OutputLevel::Default, Visibility::Hidden) => BarType::Hidden,
(OutputLevel::Quiet, Visibility::Hidden) => BarType::Hidden,
(OutputLevel::Silent | OutputLevel::SilentJSON, _) => BarType::Hidden,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_no_limit_visible() {
let bar_type = determine_bar_type(0, 1, OutputLevel::Default);
assert!(matches!(bar_type, BarType::Default));
}
#[test]
fn test_limit_exceeded_hidden() {
let bar_type = determine_bar_type(1, 2, OutputLevel::Default);
assert!(matches!(bar_type, BarType::Hidden));
}
#[test]
fn test_limit_not_exceeded_visible() {
let bar_type = determine_bar_type(2, 1, OutputLevel::Default);
assert!(matches!(bar_type, BarType::Default));
}
#[test]
fn test_quiet_visible() {
let bar_type = determine_bar_type(0, 1, OutputLevel::Quiet);
assert!(matches!(bar_type, BarType::Quiet));
}
#[test]
fn test_quiet_hidden() {
let bar_type = determine_bar_type(1, 2, OutputLevel::Quiet);
assert!(matches!(bar_type, BarType::Hidden));
}
#[test]
fn test_silent_hidden() {
let bar_type = determine_bar_type(0, 1, OutputLevel::Silent);
assert!(matches!(bar_type, BarType::Hidden));
}
#[test]
fn test_silent_json_hidden() {
let bar_type = determine_bar_type(0, 1, OutputLevel::SilentJSON);
assert!(matches!(bar_type, BarType::Hidden));
}
}

28
src/scanner/ferox_scanner.rs
View File

@@ -283,6 +283,14 @@ impl FeroxScanner {
let mut message = format!("=> {}", style("Directory listing").blue().bright());
if !self.handles.config.scan_dir_listings {
write!(
message,
" (add {} to scan)",
style("--scan-dir-listings").bright().yellow()
)?;
}
if !self.handles.config.extract_links {
write!(
message,
@@ -291,7 +299,7 @@ impl FeroxScanner {
)?;
}
if !self.handles.config.force_recursion {
if !self.handles.config.force_recursion && !self.handles.config.scan_dir_listings {
for handle in extraction_tasks.into_iter().flatten() {
_ = handle.await;
}
@@ -299,7 +307,14 @@ impl FeroxScanner {
progress_bar.reset_eta();
progress_bar.finish_with_message(message);
ferox_scan.finish()?;
if self.handles.config.limit_bars > 0 {
let scans = self.handles.ferox_scans()?;
let num_bars = scans.number_of_bars();
ferox_scan.finish(num_bars)?;
scans.make_visible();
} else {
ferox_scan.finish(0)?;
}
return Ok(()); // nothing left to do if we found a dir listing
}
@@ -382,7 +397,14 @@ impl FeroxScanner {
_ = handle.await;
}
ferox_scan.finish()?;
if self.handles.config.limit_bars > 0 {
let scans = self.handles.ferox_scans()?;
let num_bars = scans.number_of_bars();
ferox_scan.finish(num_bars)?;
scans.make_visible();
} else {
ferox_scan.finish(0)?;
}
log::trace!("exit: scan_url");

11
src/scanner/requester.rs
View File

@@ -313,9 +313,12 @@ impl Requester {
.set_status(ScanStatus::Cancelled)
.unwrap_or_else(|e| log::warn!("Could not set scan status: {}", e));
let scans = self.handles.ferox_scans()?;
let active_bars = scans.number_of_bars();
// kill the scan
self.ferox_scan
.abort()
.abort(active_bars)
.await
.unwrap_or_else(|e| log::warn!("Could not bail on scan: {}", e));
@@ -646,6 +649,8 @@ mod tests {
1000,
OutputLevel::Default,
None,
true,
handles.clone(),
);
scan.set_status(ScanStatus::Running).unwrap();
@@ -1144,6 +1149,8 @@ mod tests {
1000,
OutputLevel::Default,
None,
true,
Arc::new(Handles::for_testing(None, None).0),
);
scan.set_status(ScanStatus::Running).unwrap();
scan.add_429();
@@ -1177,7 +1184,7 @@ mod tests {
200
);
scan.finish().unwrap();
scan.finish(0).unwrap();
assert!(start.elapsed().as_millis() >= 2000);
}
}

2
src/scanner/tests.rs
View File

@@ -15,7 +15,7 @@ use super::*;
/// try to hit struct field coverage of FileOutHandler
async fn get_scan_by_url_bails_on_unfound_url() {
let sem = Semaphore::new(10);
let urls = FeroxScans::new(OutputLevel::Default);
let urls = FeroxScans::new(OutputLevel::Default, 0);
let scanner = FeroxScanner::new(
"http://localhost",

66
src/utils.rs
View File

@@ -975,7 +975,11 @@ mod tests {
let tested_url = Url::parse("https://testdomain.com/denied/").unwrap();
let scans = Arc::new(FeroxScans::default());
scans.add_directory_scan(scan_url, ScanOrder::Initial);
scans.add_directory_scan(
scan_url,
ScanOrder::Initial,
Arc::new(Handles::for_testing(None, None).0),
);
let mut config = Configuration::new().unwrap();
config.url_denylist = vec![Url::parse(deny_url).unwrap()];
@@ -994,7 +998,11 @@ mod tests {
let tested_url = Url::parse("https://testdomain.com/denied/").unwrap();
let scans = Arc::new(FeroxScans::default());
scans.add_directory_scan(scan_url, ScanOrder::Initial);
scans.add_directory_scan(
scan_url,
ScanOrder::Initial,
Arc::new(Handles::for_testing(None, None).0),
);
let mut config = Configuration::new().unwrap();
config.url_denylist = vec![Url::parse(deny_url).unwrap()];
@@ -1013,7 +1021,11 @@ mod tests {
let tested_url = Url::parse("https://testdomain.com/denied/").unwrap();
let scans = Arc::new(FeroxScans::default());
scans.add_directory_scan(scan_url, ScanOrder::Initial);
scans.add_directory_scan(
scan_url,
ScanOrder::Initial,
Arc::new(Handles::for_testing(None, None).0),
);
let mut config = Configuration::new().unwrap();
config.url_denylist = vec![Url::parse(deny_url).unwrap()];
@@ -1034,7 +1046,11 @@ mod tests {
let tested_url = Url::parse("https://testdomain.com/denied/").unwrap();
let scans = Arc::new(FeroxScans::default());
scans.add_directory_scan(scan_url, ScanOrder::Initial);
scans.add_directory_scan(
scan_url,
ScanOrder::Initial,
Arc::new(Handles::for_testing(None, None).0),
);
let mut config = Configuration::new().unwrap();
config.url_denylist = vec![Url::parse(deny_url).unwrap()];
@@ -1062,7 +1078,11 @@ mod tests {
let tested_url = Url::parse("https://testdomain.com/denied/").unwrap();
let scans = Arc::new(FeroxScans::default());
scans.add_directory_scan(scan_url, ScanOrder::Initial);
scans.add_directory_scan(
scan_url,
ScanOrder::Initial,
Arc::new(Handles::for_testing(None, None).0),
);
let mut config = Configuration::new().unwrap();
config.url_denylist = vec![Url::parse(deny_url).unwrap()];
@@ -1080,7 +1100,11 @@ mod tests {
let tested_url = Url::parse("https://testdomain.com/api/denied/").unwrap();
let scans = Arc::new(FeroxScans::default());
scans.add_directory_scan(scan_url, ScanOrder::Initial);
scans.add_directory_scan(
scan_url,
ScanOrder::Initial,
Arc::new(Handles::for_testing(None, None).0),
);
let mut config = Configuration::new().unwrap();
config.url_denylist = vec![Url::parse(deny_url).unwrap()];
@@ -1099,7 +1123,11 @@ mod tests {
let tested_url = Url::parse("https://testdomain.com/not-denied/").unwrap();
let scans = Arc::new(FeroxScans::default());
scans.add_directory_scan(scan_url, ScanOrder::Initial);
scans.add_directory_scan(
scan_url,
ScanOrder::Initial,
Arc::new(Handles::for_testing(None, None).0),
);
let mut config = Configuration::new().unwrap();
config.url_denylist = vec![Url::parse(deny_url).unwrap()];
@@ -1118,7 +1146,11 @@ mod tests {
let tested_url = Url::parse("https://testdomain.com/stuff/").unwrap();
let scans = Arc::new(FeroxScans::default());
scans.add_directory_scan(scan_url, ScanOrder::Initial);
scans.add_directory_scan(
scan_url,
ScanOrder::Initial,
Arc::new(Handles::for_testing(None, None).0),
);
let mut config = Configuration::new().unwrap();
config.url_denylist = vec![Url::parse(deny_url).unwrap()];
@@ -1137,7 +1169,11 @@ mod tests {
let tested_url = Url::parse("https://testdomain.com/api/not-denied/").unwrap();
let scans = Arc::new(FeroxScans::default());
scans.add_directory_scan(scan_url, ScanOrder::Initial);
scans.add_directory_scan(
scan_url,
ScanOrder::Initial,
Arc::new(Handles::for_testing(None, None).0),
);
let mut config = Configuration::new().unwrap();
config.url_denylist = vec![Url::parse(deny_url).unwrap()];
@@ -1157,7 +1193,11 @@ mod tests {
let tested_url = Url::parse("https://testdomain.com/denied/").unwrap();
let scans = Arc::new(FeroxScans::default());
scans.add_directory_scan(scan_url, ScanOrder::Initial);
scans.add_directory_scan(
scan_url,
ScanOrder::Initial,
Arc::new(Handles::for_testing(None, None).0),
);
let mut config = Configuration::new().unwrap();
config.regex_denylist = vec![Regex::new(deny_pattern).unwrap()];
@@ -1178,7 +1218,11 @@ mod tests {
let tested_https_url = Url::parse("https://testdomain.com/denied/").unwrap();
let scans = Arc::new(FeroxScans::default());
scans.add_directory_scan(scan_url, ScanOrder::Initial);
scans.add_directory_scan(
scan_url,
ScanOrder::Initial,
Arc::new(Handles::for_testing(None, None).0),
);
let mut config = Configuration::new().unwrap();
config.regex_denylist = vec![Regex::new(deny_pattern).unwrap()];