h/feroxbuster
1
Fork 0
mirror of https://github.com/epi052/feroxbuster.git synced 2026-05-21 03:21:13 -03:00
Code Issues Packages Projects Releases Wiki Activity

removed global num_requests tracker; logic in statistics now

Browse Source
This commit is contained in:
epi
2021-01-03 09:08:03 -06:00
parent 892352914a
commit ab3177ff7f
11 changed files with 184 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
img/demo.gif
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 4.6 MiB

After

Width:  |  Height:  |  Size: 716 KiB

3
src/banner.rs
View File

@@ -545,9 +545,6 @@ by Ben "epi" Risher {} ver: {}"#,
.unwrap_or_default();
writeln!(&mut writer, "{}", addl_section).unwrap_or_default();
// todo: this isn't printing properly anymore, feels like the totals bar is overwriting it
// writeln!(&mut writer, "{}", addl_section).unwrap_or_default();
// writeln!(&mut writer, "{}", addl_section).unwrap_or_default();
}
#[cfg(test)]

1
src/extractor.rs
View File

@@ -204,6 +204,7 @@ pub async fn request_feroxresponse_from_new_link(
CONFIGURATION.add_slash,
&CONFIGURATION.queries,
None,
tx_stats.clone(),
) {
Ok(url) => url,
Err(_) => {

9
src/heuristics.rs
View File

@@ -2,10 +2,7 @@ use crate::{
config::{CONFIGURATION, PROGRESS_PRINTER},
filters::WildcardFilter,
scanner::should_filter_response,
statistics::{
StatCommand::{self, AddError},
StatError::UrlFormat,
},
statistics::StatCommand,
utils::{ferox_print, format_url, get_url_path_length, make_request, status_colorizer},
FeroxResponse,
};
@@ -168,6 +165,7 @@ async fn make_wildcard_request(
CONFIGURATION.add_slash,
&CONFIGURATION.queries,
None,
tx_stats.clone(),
) {
Ok(url) => url,
Err(e) => {
@@ -239,11 +237,10 @@ pub async fn connectivity_test(
CONFIGURATION.add_slash,
&CONFIGURATION.queries,
None,
tx_stats.clone(),
) {
Ok(url) => url,
Err(e) => {
// todo this probably makes more sense inside format_url, similar to make_request
update_stat!(tx_stats, AddError(UrlFormat));
log::error!("{}", e);
continue;
}

29
src/main.rs
View File

@@ -1,15 +1,19 @@
use crossterm::event::{self, Event, KeyCode};
use feroxbuster::progress::BarType;
use feroxbuster::{
banner,
config::{CONFIGURATION, PROGRESS_BAR, PROGRESS_PRINTER},
extractor::{extract_robots_txt, request_feroxresponse_from_new_link},
heuristics, logger,
progress::add_bar,
progress::{add_bar, BarType},
reporter,
scan_manager::{self, PAUSE_SCAN},
scanner::{self, scan_url, send_report, RESPONSES, SCANNED_URLS},
statistics::{self, StatCommand},
statistics::{
self,
StatCommand::{self, UpdateUsizeField},
StatField::InitialTargets,
Stats,
},
update_stat,
utils::{ferox_print, get_current_depth, module_colorizer, status_colorizer},
FeroxError, FeroxResponse, FeroxResult, FeroxSerialize, SLEEP_DURATION, VERSION,
@@ -103,13 +107,15 @@ fn get_unique_words_from_wordlist(path: &str) -> FeroxResult<Arc<HashSet<String>
/// Determine whether it's a single url scan or urls are coming from stdin, then scan as needed
async fn scan(
mut targets: Vec<String>,
stats: Arc<Stats>,
tx_term: UnboundedSender<FeroxResponse>,
tx_file: UnboundedSender<FeroxResponse>,
tx_stats: UnboundedSender<StatCommand>,
) -> FeroxResult<()> {
log::trace!(
"enter: scan({:?}, {:?}, {:?}, {:?})",
"enter: scan({:?}, {:?}, {:?}, {:?}, {:?})",
targets,
stats,
tx_term,
tx_file,
tx_stats
@@ -182,10 +188,10 @@ async fn scan(
};
if ferox_response.is_file() {
SCANNED_URLS.add_file_scan(&robot_link);
SCANNED_URLS.add_file_scan(&robot_link, stats.clone());
send_report(tx_term.clone(), ferox_response);
} else {
let (unknown, _) = SCANNED_URLS.add_directory_scan(&robot_link);
let (unknown, _) = SCANNED_URLS.add_directory_scan(&robot_link, stats.clone());
if !unknown {
// known directory; can skip (unlikely)
@@ -200,13 +206,13 @@ async fn scan(
}
let mut tasks = vec![];
let num_targets = targets.len();
for target in targets {
let word_clone = words.clone();
let term_clone = tx_term.clone();
let file_clone = tx_file.clone();
let stats_clone = tx_stats.clone();
let tx_stats_clone = tx_stats.clone();
let stats_clone = stats.clone();
let task = tokio::spawn(async move {
let base_depth = get_current_depth(&target);
@@ -214,10 +220,10 @@ async fn scan(
&target,
word_clone,
base_depth,
num_targets,
stats_clone,
term_clone,
file_clone,
stats_clone,
tx_stats_clone,
)
.await;
});
@@ -339,6 +345,8 @@ async fn wrapped_main() {
}
};
update_stat!(tx_stats, UpdateUsizeField(InitialTargets, targets.len()));
if !CONFIGURATION.quiet {
// only print banner if -q isn't used
let std_stderr = stderr(); // std::io::stderr
@@ -372,6 +380,7 @@ async fn wrapped_main() {
// kick off a scan against any targets determined to be responsive
match scan(
live_targets,
stats,
tx_term.clone(),
tx_file.clone(),
tx_stats.clone(),

4
src/progress.rs
View File

@@ -58,14 +58,16 @@ mod tests {
let p1 = add_bar("prefix", 2, BarType::Hidden); // hidden
let p2 = add_bar("prefix", 2, BarType::Message); // no per second field
let p3 = add_bar("prefix", 2, BarType::Default); // normal bar
// todo add Total bartype to test
let p4 = add_bar("prefix", 2, BarType::Total); // totals bar
p1.finish();
p2.finish();
p3.finish();
p4.finish();
assert!(p1.is_finished());
assert!(p2.is_finished());
assert!(p3.is_finished());
assert!(p4.is_finished());
}
}

1
src/reporter.rs
View File

@@ -198,7 +198,6 @@ async fn spawn_file_reporter(
safe_file_write(&response, buffered_file.clone(), CONFIGURATION.json);
}
// todo if --summary was used, do this, else pass
update_stat!(tx_stats, StatCommand::Save);
log::trace!("exit: spawn_file_reporter");

74
src/scan_manager.rs
View File

@@ -3,7 +3,7 @@ use crate::{
parser::TIMESPEC_REGEX,
progress::{add_bar, BarType},
reporter::safe_file_write,
scanner::{NUMBER_OF_REQUESTS, RESPONSES, SCANNED_URLS},
scanner::{RESPONSES, SCANNED_URLS},
statistics::Stats,
utils::open_file,
FeroxResponse, FeroxSerialize, SLEEP_DURATION,
@@ -73,6 +73,9 @@ pub struct FeroxScan {
/// The type of scan
pub scan_type: ScanType,
/// Number of requests to populate the progress bar with
num_requests: u64,
/// Whether or not this scan has completed
pub complete: bool,
@@ -93,6 +96,7 @@ impl Default for FeroxScan {
id: new_id,
task: None,
complete: false,
num_requests: 0,
url: String::new(),
progress_bar: None,
scan_type: ScanType::File,
@@ -123,8 +127,7 @@ impl FeroxScan {
if let Some(pb) = &self.progress_bar {
pb.clone()
} else {
let num_requests = NUMBER_OF_REQUESTS.load(Ordering::Relaxed);
let pb = add_bar(&self.url, num_requests, BarType::Default);
let pb = add_bar(&self.url, self.num_requests, BarType::Default);
pb.reset_elapsed();
@@ -135,10 +138,16 @@ impl FeroxScan {
}
/// Given a URL and ProgressBar, create a new FeroxScan, wrap it in an Arc and return it
pub fn new(url: &str, scan_type: ScanType, pb: Option<ProgressBar>) -> Arc<Mutex<Self>> {
pub fn new(
url: &str,
scan_type: ScanType,
num_requests: u64,
pb: Option<ProgressBar>,
) -> Arc<Mutex<Self>> {
Arc::new(Mutex::new(Self {
url: url.to_string(),
scan_type,
num_requests,
progress_bar: pb,
..Default::default()
}))
@@ -435,12 +444,17 @@ impl FeroxScans {
/// If `FeroxScans` did not already contain the scan, return true; otherwise return false
///
/// Also return a reference to the new `FeroxScan`
fn add_scan(&self, url: &str, scan_type: ScanType) -> (bool, Arc<Mutex<FeroxScan>>) {
fn add_scan(
&self,
url: &str,
scan_type: ScanType,
stats: Arc<Stats>,
) -> (bool, Arc<Mutex<FeroxScan>>) {
let bar = match scan_type {
ScanType::Directory => {
let progress_bar = add_bar(
&url,
NUMBER_OF_REQUESTS.load(Ordering::Relaxed),
stats.expected_per_scan.load(Ordering::Relaxed) as u64,
BarType::Default,
);
@@ -451,7 +465,9 @@ impl FeroxScans {
ScanType::File => None,
};
let ferox_scan = FeroxScan::new(&url, scan_type, bar);
let num_requests = stats.expected_per_scan.load(Ordering::Relaxed) as u64;
let ferox_scan = FeroxScan::new(&url, scan_type, num_requests, bar);
// If the set did not contain the scan, true is returned.
// If the set did contain the scan, false is returned.
@@ -465,8 +481,12 @@ impl FeroxScans {
/// If `FeroxScans` did not already contain the scan, return true; otherwise return false
///
/// Also return a reference to the new `FeroxScan`
pub fn add_directory_scan(&self, url: &str) -> (bool, Arc<Mutex<FeroxScan>>) {
self.add_scan(&url, ScanType::Directory)
pub fn add_directory_scan(
&self,
url: &str,
stats: Arc<Stats>,
) -> (bool, Arc<Mutex<FeroxScan>>) {
self.add_scan(&url, ScanType::Directory, stats)
}
/// Given a url, create a new `FeroxScan` and add it to `FeroxScans` as a File Scan
@@ -474,8 +494,8 @@ impl FeroxScans {
/// If `FeroxScans` did not already contain the scan, return true; otherwise return false
///
/// Also return a reference to the new `FeroxScan`
pub fn add_file_scan(&self, url: &str) -> (bool, Arc<Mutex<FeroxScan>>) {
self.add_scan(&url, ScanType::File)
pub fn add_file_scan(&self, url: &str, stats: Arc<Stats>) -> (bool, Arc<Mutex<FeroxScan>>) {
self.add_scan(&url, ScanType::File, stats)
}
}
@@ -796,8 +816,9 @@ mod tests {
/// add an unknown url to the hashset, expect true
fn add_url_to_list_of_scanned_urls_with_unknown_url() {
let urls = FeroxScans::default();
let stats = Arc::new(Stats::new());
let url = "http://unknown_url";
let (result, _scan) = urls.add_scan(url, ScanType::Directory);
let (result, _scan) = urls.add_scan(url, ScanType::Directory, stats);
assert_eq!(result, true);
}
@@ -807,11 +828,13 @@ mod tests {
let urls = FeroxScans::default();
let pb = ProgressBar::new(1);
let url = "http://unknown_url/";
let scan = FeroxScan::new(url, ScanType::Directory, Some(pb));
let stats = Arc::new(Stats::new());
let scan = FeroxScan::new(url, ScanType::Directory, pb.length(), Some(pb));
assert_eq!(urls.insert(scan), true);
let (result, _scan) = urls.add_scan(url, ScanType::Directory);
let (result, _scan) = urls.add_scan(url, ScanType::Directory, stats);
assert_eq!(result, false);
}
@@ -821,7 +844,8 @@ mod tests {
fn abort_stops_progress_bar() {
let pb = ProgressBar::new(1);
let url = "http://unknown_url/";
let scan = FeroxScan::new(url, ScanType::Directory, Some(pb));
let scan = FeroxScan::new(url, ScanType::Directory, pb.length(), Some(pb));
assert_eq!(
scan.lock()
@@ -851,11 +875,13 @@ mod tests {
fn add_url_to_list_of_scanned_urls_with_known_url_without_slash() {
let urls = FeroxScans::default();
let url = "http://unknown_url";
let scan = FeroxScan::new(url, ScanType::File, None);
let stats = Arc::new(Stats::new());
let scan = FeroxScan::new(url, ScanType::File, 0, None);
assert_eq!(urls.insert(scan), true);
let (result, _scan) = urls.add_scan(url, ScanType::File);
let (result, _scan) = urls.add_scan(url, ScanType::File, stats);
assert_eq!(result, false);
}
@@ -868,8 +894,8 @@ mod tests {
let pb_two = ProgressBar::new(2);
let url = "http://unknown_url/";
let url_two = "http://unknown_url/fa";
let scan = FeroxScan::new(url, ScanType::Directory, Some(pb));
let scan_two = FeroxScan::new(url_two, ScanType::Directory, Some(pb_two));
let scan = FeroxScan::new(url, ScanType::Directory, pb.length(), Some(pb));
let scan_two = FeroxScan::new(url_two, ScanType::Directory, pb_two.length(), Some(pb_two));
scan_two.lock().unwrap().finish(); // one complete, one incomplete
@@ -882,8 +908,8 @@ mod tests {
/// ensure that PartialEq compares FeroxScan.id fields
fn partial_eq_compares_the_id_field() {
let url = "http://unknown_url/";
let scan = FeroxScan::new(url, ScanType::Directory, None);
let scan_two = FeroxScan::new(url, ScanType::Directory, None);
let scan = FeroxScan::new(url, ScanType::Directory, 0, None);
let scan_two = FeroxScan::new(url, ScanType::Directory, 0, None);
assert!(!scan.lock().unwrap().eq(&scan_two.lock().unwrap()));
@@ -942,7 +968,7 @@ mod tests {
#[test]
/// given a FeroxScan, test that it serializes into the proper JSON entry
fn ferox_scan_serialize() {
let fs = FeroxScan::new("https://spiritanimal.com", ScanType::Directory, None);
let fs = FeroxScan::new("https://spiritanimal.com", ScanType::Directory, 0, None);
let fs_json = format!(
r#"{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","complete":false}}"#,
fs.lock().unwrap().id
@@ -956,7 +982,7 @@ mod tests {
#[test]
/// given a FeroxScans, test that it serializes into the proper JSON entry
fn ferox_scans_serialize() {
let ferox_scan = FeroxScan::new("https://spiritanimal.com", ScanType::Directory, None);
let ferox_scan = FeroxScan::new("https://spiritanimal.com", ScanType::Directory, 0, None);
let ferox_scans = FeroxScans::default();
let ferox_scans_json = format!(
r#"[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","complete":false}}]"#,
@@ -1010,7 +1036,7 @@ mod tests {
#[test]
/// test FeroxSerialize implementation of FeroxState
fn feroxstates_feroxserialize_implementation() {
let ferox_scan = FeroxScan::new("https://spiritanimal.com", ScanType::Directory, None);
let ferox_scan = FeroxScan::new("https://spiritanimal.com", ScanType::Directory, 0, None);
let saved_id = ferox_scan.lock().unwrap().id.clone();
SCANNED_URLS.insert(ferox_scan);

93
src/scanner.rs
View File

@@ -1,4 +1,4 @@
use crate::statistics::StatCommand::UpdateF64Field;
use crate::statistics::Stats;
use crate::{
config::{Configuration, CONFIGURATION},
extractor::{get_links, request_feroxresponse_from_new_link},
@@ -9,7 +9,7 @@ use crate::{
heuristics,
scan_manager::{FeroxResponses, FeroxScans, PAUSE_SCAN},
statistics::{
StatCommand::{self, UpdateUsizeField},
StatCommand::{self, UpdateF64Field, UpdateUsizeField},
StatField::{DirScanTimes, ExpectedPerScan, TotalScans, WildcardsFiltered},
},
utils::{format_url, get_current_depth, make_request},
@@ -29,8 +29,9 @@ use std::{
collections::HashSet,
convert::TryInto,
ops::Deref,
sync::atomic::{AtomicU64, AtomicUsize, Ordering},
sync::atomic::{AtomicUsize, Ordering},
sync::{Arc, RwLock},
time::Instant,
};
use tokio::{
sync::{
@@ -47,9 +48,6 @@ use tokio::{
/// --stdin means this will be incremented by the number of targets passed via STDIN
static CALL_COUNT: AtomicUsize = AtomicUsize::new(0);
/// Single atomic number that gets holds the number of requests to be sent per directory scanned
pub static NUMBER_OF_REQUESTS: AtomicU64 = AtomicU64::new(0); // todo move to stats
lazy_static! {
/// Set of urls that have been sent to [scan_url](fn.scan_url.html), used for deduplication
pub static ref SCANNED_URLS: FeroxScans = FeroxScans::default();
@@ -107,17 +105,17 @@ fn spawn_recursion_handler(
mut recursion_channel: UnboundedReceiver<String>,
wordlist: Arc<HashSet<String>>,
base_depth: usize,
num_targets: usize,
stats: Arc<Stats>,
tx_term: UnboundedSender<FeroxResponse>,
tx_file: UnboundedSender<FeroxResponse>,
tx_stats: UnboundedSender<StatCommand>,
) -> BoxFuture<'static, Vec<JoinHandle<()>>> {
log::trace!(
"enter: spawn_recursion_handler({:?}, wordlist[{} words...], {}, {}, {:?}, {:?}, {:?})",
"enter: spawn_recursion_handler({:?}, wordlist[{} words...], {}, {:?}, {:?}, {:?}, {:?})",
recursion_channel,
wordlist.len(),
base_depth,
num_targets,
stats,
tx_term,
tx_file,
tx_stats
@@ -127,7 +125,7 @@ fn spawn_recursion_handler(
let mut scans = vec![];
while let Some(resp) = recursion_channel.recv().await {
let (unknown, _) = SCANNED_URLS.add_directory_scan(&resp);
let (unknown, _) = SCANNED_URLS.add_directory_scan(&resp, stats.clone());
if !unknown {
// not unknown, i.e. we've seen the url before and don't need to scan again
@@ -140,7 +138,8 @@ fn spawn_recursion_handler(
let term_clone = tx_term.clone();
let file_clone = tx_file.clone();
let stats_clone = tx_stats.clone();
let tx_stats_clone = tx_stats.clone();
let stats_clone = stats.clone();
let resp_clone = resp.clone();
let list_clone = wordlist.clone();
@@ -149,10 +148,10 @@ fn spawn_recursion_handler(
resp_clone.to_owned().as_str(),
list_clone,
base_depth,
num_targets,
stats_clone,
term_clone,
file_clone,
stats_clone,
tx_stats_clone,
)
.await
});
@@ -173,12 +172,18 @@ fn spawn_recursion_handler(
///
/// If any extensions were passed to the program, each extension will add a
/// (base_url + word + ext) Url to the vector
fn create_urls(target_url: &str, word: &str, extensions: &[String]) -> Vec<Url> {
fn create_urls(
target_url: &str,
word: &str,
extensions: &[String],
tx_stats: UnboundedSender<StatCommand>,
) -> Vec<Url> {
log::trace!(
"enter: create_urls({}, {}, {:?})",
"enter: create_urls({}, {}, {:?}, {:?})",
target_url,
word,
extensions
extensions,
tx_stats
);
let mut urls = vec![];
@@ -189,6 +194,7 @@ fn create_urls(target_url: &str, word: &str, extensions: &[String]) -> Vec<Url>
CONFIGURATION.add_slash,
&CONFIGURATION.queries,
None,
tx_stats.clone(),
) {
urls.push(url); // default request, i.e. no extension
}
@@ -200,6 +206,7 @@ fn create_urls(target_url: &str, word: &str, extensions: &[String]) -> Vec<Url>
CONFIGURATION.add_slash,
&CONFIGURATION.queries,
Some(ext),
tx_stats.clone(),
) {
urls.push(url); // any extensions passed in
}
@@ -375,21 +382,28 @@ async fn make_requests(
target_url: &str,
word: &str,
base_depth: usize,
stats: Arc<Stats>,
dir_chan: UnboundedSender<String>,
report_chan: UnboundedSender<FeroxResponse>,
tx_stats: UnboundedSender<StatCommand>,
) {
log::trace!(
"enter: make_requests({}, {}, {}, {:?}, {:?}, {:?})",
"enter: make_requests({}, {}, {}, {:?}, {:?}, {:?}, {:?})",
target_url,
word,
base_depth,
stats,
dir_chan,
report_chan,
tx_stats
);
let urls = create_urls(&target_url, &word, &CONFIGURATION.extensions);
let urls = create_urls(
&target_url,
&word,
&CONFIGURATION.extensions,
tx_stats.clone(),
);
for url in urls {
if let Ok(response) = make_request(&CONFIGURATION.client, &url, tx_stats.clone()).await {
@@ -431,7 +445,8 @@ async fn make_requests(
// very likely a file, simply request and report
log::debug!("Singular extraction: {}", new_ferox_response);
SCANNED_URLS.add_file_scan(&new_ferox_response.url().to_string());
SCANNED_URLS
.add_file_scan(&new_ferox_response.url().to_string(), stats.clone());
send_report(report_chan.clone(), new_ferox_response);
@@ -484,17 +499,17 @@ pub async fn scan_url(
target_url: &str,
wordlist: Arc<HashSet<String>>,
base_depth: usize,
num_targets: usize,
stats: Arc<Stats>,
tx_term: UnboundedSender<FeroxResponse>,
tx_file: UnboundedSender<FeroxResponse>,
tx_stats: UnboundedSender<StatCommand>,
) {
log::trace!(
"enter: scan_url({:?}, wordlist[{} words...], {}, {}, {:?}, {:?}, {:?})",
"enter: scan_url({:?}, wordlist[{} words...], {}, {:?}, {:?}, {:?}, {:?})",
target_url,
wordlist.len(),
base_depth,
num_targets,
stats,
tx_term,
tx_file,
tx_stats
@@ -502,19 +517,18 @@ pub async fn scan_url(
log::info!("Starting scan against: {}", target_url);
// todo import
let scan_timer = std::time::Instant::now();
let scan_timer = Instant::now();
let (tx_dir, rx_dir): FeroxChannel<String> = mpsc::unbounded_channel();
if CALL_COUNT.load(Ordering::Relaxed) < num_targets {
if CALL_COUNT.load(Ordering::Relaxed) < stats.initial_targets.load(Ordering::Relaxed) {
CALL_COUNT.fetch_add(1, Ordering::Relaxed);
update_stat!(tx_stats, UpdateUsizeField(TotalScans, 1));
// this protection allows us to add the first scanned url to SCANNED_URLS
// from within the scan_url function instead of the recursion handler
SCANNED_URLS.add_directory_scan(&target_url);
SCANNED_URLS.add_directory_scan(&target_url, stats.clone());
}
let ferox_scan = match SCANNED_URLS.get_scan_by_url(&target_url) {
@@ -551,13 +565,14 @@ pub async fn scan_url(
let recurser_stats_clone = tx_stats.clone();
let recurser_words = wordlist.clone();
let looping_words = wordlist.clone();
let looping_stats = stats.clone();
let recurser = tokio::spawn(async move {
spawn_recursion_handler(
rx_dir,
recurser_words,
base_depth,
num_targets,
stats.clone(),
recurser_term_clone,
recurser_file_clone,
recurser_stats_clone,
@@ -588,6 +603,7 @@ pub async fn scan_url(
let txs = tx_stats.clone();
let pb = progress_bar.clone(); // progress bar is an Arc around internal state
let tgt = target_url.to_string(); // done to satisfy 'static lifetime below
let lst = looping_stats.clone();
(
tokio::spawn(async move {
if PAUSE_SCAN.load(Ordering::Acquire) {
@@ -598,7 +614,7 @@ pub async fn scan_url(
// todo change to true when issue #107 is resolved
SCANNED_URLS.pause(false).await;
}
make_requests(&tgt, &word, base_depth, txd, txr, txs).await
make_requests(&tgt, &word, base_depth, lst, txd, txr, txs).await
}),
pb,
)
@@ -665,8 +681,6 @@ pub async fn initialize(
total.try_into().unwrap()
};
NUMBER_OF_REQUESTS.store(num_reqs_expected, Ordering::Relaxed);
// tell Stats object about the number of expected requests
update_stat!(
tx_stats,
@@ -734,7 +748,14 @@ pub async fn initialize(
// add any similarity filters to `FILTERS` (--filter-similar-to)
for similarity_filter in &config.filter_similar {
// url as-is based on input, ignores user-specified url manipulation options (add-slash etc)
if let Ok(url) = format_url(&similarity_filter, &"", false, &Vec::new(), None) {
if let Ok(url) = format_url(
&similarity_filter,
&"",
false,
&Vec::new(),
None,
tx_stats.clone(),
) {
// attempt to request the given url
if let Ok(resp) = make_request(&CONFIGURATION.client, &url, tx_stats.clone()).await {
// if successful, create a filter based on the response's body
@@ -771,14 +792,16 @@ mod tests {
#[test]
/// sending url + word without any extensions should get back one url with the joined word
fn create_urls_no_extension_returns_base_url_with_word() {
let urls = create_urls("http://localhost", "turbo", &[]);
let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
let urls = create_urls("http://localhost", "turbo", &[], tx);
assert_eq!(urls, [Url::parse("http://localhost/turbo").unwrap()])
}
#[test]
/// sending url + word + 1 extension should get back two urls, one base and one with extension
fn create_urls_one_extension_returns_two_urls() {
let urls = create_urls("http://localhost", "turbo", &[String::from("js")]);
let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
let urls = create_urls("http://localhost", "turbo", &[String::from("js")], tx);
assert_eq!(
urls,
[
@@ -816,8 +839,10 @@ mod tests {
vec![base, js, php, pdf, tar],
];
let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
for (i, ext_set) in ext_vec.into_iter().enumerate() {
let urls = create_urls("http://localhost", "turbo", &ext_set);
let urls = create_urls("http://localhost", "turbo", &ext_set, tx.clone());
assert_eq!(urls, expected[i]);
}
}

11
src/statistics.rs
View File

@@ -70,7 +70,7 @@ pub struct Stats {
///
/// Note: this is a per-scan expectation; `expected_requests * current # of scans` would be
/// indicative of the current expectation at any given time, but is a moving target.
expected_per_scan: AtomicUsize,
pub expected_per_scan: AtomicUsize,
/// tracker for accumulating total number of requests expected (i.e. as a new scan is started
/// this value should increase by `expected_requests`
@@ -95,6 +95,9 @@ pub struct Stats {
/// recursed into and affects the total number of expected requests
total_scans: AtomicUsize,
/// tracker for initial number of requested targets
pub initial_targets: AtomicUsize,
/// tracker for number of links extracted when `--extract-links` is used; sources are
/// response bodies and robots.txt as of v1.11.0
links_extracted: AtomicUsize,
@@ -299,6 +302,9 @@ impl Stats {
StatField::ResourcesDiscovered => {
atomic_increment!(self.resources_discovered, value);
}
StatField::InitialTargets => {
atomic_increment!(self.initial_targets, value);
}
_ => {} // f64 fields
}
}
@@ -463,6 +469,9 @@ pub enum StatField {
/// Translates to `resources_discovered`
ResourcesDiscovered,
/// Translates to `initial_targets`
InitialTargets,
/// Translates to `directory_scan_times`; assumes a single append to the vector
DirScanTimes,
}

60
src/utils.rs
View File

@@ -1,10 +1,10 @@
#![macro_use]
use crate::statistics::{
StatCommand::{self, AddError, AddStatus},
StatError::{Connection, Other, Redirection, Request, Timeout},
};
use crate::{
config::{CONFIGURATION, PROGRESS_PRINTER},
statistics::{
StatCommand::{self, AddError, AddStatus},
StatError::{Connection, Other, Redirection, Request, Timeout, UrlFormat},
},
FeroxError, FeroxResult,
};
use console::{strip_ansi_codes, style, user_attended};
@@ -166,6 +166,14 @@ pub fn ferox_print(msg: &str, bar: &ProgressBar) {
}
}
#[macro_export]
/// wrapper to improve code readability
macro_rules! update_stat {
($tx:expr, $value:expr) => {
$tx.send($value).unwrap_or_default();
};
}
/// Simple helper to generate a `Url`
///
/// Errors during parsing `url` or joining `word` are propagated up the call stack
@@ -175,14 +183,16 @@ pub fn format_url(
add_slash: bool,
queries: &[(String, String)],
extension: Option<&str>,
tx_stats: UnboundedSender<StatCommand>,
) -> FeroxResult<Url> {
log::trace!(
"enter: format_url({}, {}, {}, {:?} {:?})",
"enter: format_url({}, {}, {}, {:?} {:?}, {:?})",
url,
word,
add_slash,
queries,
extension
extension,
tx_stats
);
if Url::parse(&word).is_ok() {
@@ -201,6 +211,8 @@ pub fn format_url(
let err = FeroxError { message };
update_stat!(tx_stats, AddError(UrlFormat));
log::trace!("exit: format_url -> {}", err);
return Err(Box::new(err));
}
@@ -260,6 +272,7 @@ pub fn format_url(
}
}
Err(e) => {
update_stat!(tx_stats, AddError(UrlFormat));
log::trace!("exit: format_url -> {}", e);
log::error!("Could not join {} with {}", word, base_url);
Err(Box::new(e))
@@ -267,14 +280,6 @@ pub fn format_url(
}
}
#[macro_export]
/// wrapper to improve code readability
macro_rules! update_stat {
($tx:expr, $value:expr) => {
$tx.send($value).unwrap_or_default();
};
}
/// Initiate request to the given `Url` using `Client`
pub async fn make_request(
client: &Client,
@@ -430,6 +435,8 @@ pub fn normalize_url(url: &str) -> String {
#[cfg(test)]
mod tests {
use super::*;
use crate::FeroxChannel;
use tokio::sync::mpsc;
#[test]
/// set_open_file_limit with a low requested limit succeeds
@@ -497,8 +504,9 @@ mod tests {
#[test]
/// base url + 1 word + no slash + no extension
fn format_url_normal() {
let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
assert_eq!(
format_url("http://localhost", "stuff", false, &Vec::new(), None).unwrap(),
format_url("http://localhost", "stuff", false, &Vec::new(), None, tx).unwrap(),
reqwest::Url::parse("http://localhost/stuff").unwrap()
);
}
@@ -506,8 +514,9 @@ mod tests {
#[test]
/// base url + no word + no slash + no extension
fn format_url_no_word() {
let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
assert_eq!(
format_url("http://localhost", "", false, &Vec::new(), None).unwrap(),
format_url("http://localhost", "", false, &Vec::new(), None, tx).unwrap(),
reqwest::Url::parse("http://localhost").unwrap()
);
}
@@ -515,13 +524,15 @@ mod tests {
#[test]
/// base url + word + no slash + no extension + queries
fn format_url_joins_queries() {
let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
assert_eq!(
format_url(
"http://localhost",
"lazer",
false,
&[(String::from("stuff"), String::from("things"))],
None
None,
tx
)
.unwrap(),
reqwest::Url::parse("http://localhost/lazer?stuff=things").unwrap()
@@ -531,13 +542,15 @@ mod tests {
#[test]
/// base url + no word + no slash + no extension + queries
fn format_url_without_word_joins_queries() {
let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
assert_eq!(
format_url(
"http://localhost",
"",
false,
&[(String::from("stuff"), String::from("things"))],
None
None,
tx
)
.unwrap(),
reqwest::Url::parse("http://localhost/?stuff=things").unwrap()
@@ -548,14 +561,16 @@ mod tests {
#[should_panic]
/// no base url is an error
fn format_url_no_url() {
format_url("", "stuff", false, &Vec::new(), None).unwrap();
let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
format_url("", "stuff", false, &Vec::new(), None, tx).unwrap();
}
#[test]
/// word prepended with slash is adjusted for correctness
fn format_url_word_with_preslash() {
let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
assert_eq!(
format_url("http://localhost", "/stuff", false, &Vec::new(), None).unwrap(),
format_url("http://localhost", "/stuff", false, &Vec::new(), None, tx).unwrap(),
reqwest::Url::parse("http://localhost/stuff").unwrap()
);
}
@@ -563,8 +578,9 @@ mod tests {
#[test]
/// word with appended slash allows the slash to persist
fn format_url_word_with_postslash() {
let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
assert_eq!(
format_url("http://localhost", "stuff/", false, &Vec::new(), None).unwrap(),
format_url("http://localhost", "stuff/", false, &Vec::new(), None, tx).unwrap(),
reqwest::Url::parse("http://localhost/stuff/").unwrap()
);
}
@@ -572,12 +588,14 @@ mod tests {
#[test]
/// word that is a fully formed url, should return an error
fn format_url_word_that_is_a_url() {
let (tx, _): FeroxChannel<StatCommand> = mpsc::unbounded_channel();
let url = format_url(
"http://localhost",
"http://schmocalhost",
false,
&Vec::new(),
None,
tx,
);
assert!(url.is_err());
}