From b84c8cbdf4617d1a31d0e4dc878a1f0f2b60d9eb Mon Sep 17 00:00:00 2001 From: epi Date: Wed, 23 Sep 2020 06:27:21 -0500 Subject: [PATCH] added progress bars to output --- Cargo.toml | 1 + src/config.rs | 2 ++ src/heuristics.rs | 84 +++++++++++++++++++++++++++++++---------------- src/lib.rs | 1 + src/main.rs | 6 ++-- src/progress.rs | 21 ++++++++++++ src/scanner.rs | 57 ++++++++++++++++++++++++-------- 7 files changed, 128 insertions(+), 44 deletions(-) create mode 100644 src/progress.rs diff --git a/Cargo.toml b/Cargo.toml index e8a486a..a09e552 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ toml = "0.5" serde = { version = "1.0", features = ["derive"] } uuid = { version = "0.8", features = ["v4"] } ansi_term = "0.12" +indicatif = "0.15" [dev-dependencies] tempfile = "3.1" diff --git a/src/config.rs b/src/config.rs index f5a236f..fe630a1 100644 --- a/src/config.rs +++ b/src/config.rs @@ -2,6 +2,7 @@ use crate::utils::status_colorizer; use crate::{client, parser}; use crate::{DEFAULT_CONFIG_NAME, DEFAULT_STATUS_CODES, DEFAULT_WORDLIST, VERSION}; use clap::value_t; +use indicatif::MultiProgress; use lazy_static::lazy_static; use reqwest::{Client, StatusCode}; use serde::Deserialize; @@ -14,6 +15,7 @@ use std::process::exit; lazy_static! { /// Global configuration variable. pub static ref CONFIGURATION: Configuration = Configuration::new(); + pub static ref PROGRESS_BAR: MultiProgress = MultiProgress::new(); } /// Represents the final, global configuration of the program. diff --git a/src/heuristics.rs b/src/heuristics.rs index e906c11..43812e0 100644 --- a/src/heuristics.rs +++ b/src/heuristics.rs @@ -1,9 +1,11 @@ use crate::config::CONFIGURATION; +use crate::progress; use crate::scanner::{format_url, make_request}; use crate::utils::{get_url_path_length, status_colorizer}; use reqwest::Response; use std::process; use uuid::Uuid; +use indicatif::ProgressBar; const UUID_LENGTH: u64 = 32; @@ -44,7 +46,7 @@ fn unique_string(length: usize) -> String { /// /// In the event that url returns a wildcard response, a /// [WildcardFilter](struct.WildcardFilter.html) is created and returned to the caller. -pub async fn wildcard_test(target_url: &str) -> Option { +pub async fn wildcard_test(target_url: &str, bar: ProgressBar) -> Option { log::trace!("enter: wildcard_test({:?})", target_url); if CONFIGURATION.dontfilter { @@ -53,7 +55,9 @@ pub async fn wildcard_test(target_url: &str) -> Option { return None; } - if let Some(resp_one) = make_wildcard_request(&target_url, 1).await { + if let Some(resp_one) = make_wildcard_request(&target_url, 1, bar.clone()).await { + bar.inc(1); + // found a wildcard response let mut wildcard = WildcardFilter::default(); @@ -66,7 +70,9 @@ pub async fn wildcard_test(target_url: &str) -> Option { // content length of wildcard is non-zero, perform additional tests: // make a second request, with a known-sized (64) longer request - if let Some(resp_two) = make_wildcard_request(&target_url, 3).await { + if let Some(resp_two) = make_wildcard_request(&target_url, 3, bar.clone()).await { + bar.inc(1); + let wc2_length = resp_two.content_length().unwrap_or(0); if wc2_length == wc_length + (UUID_LENGTH * 2) { @@ -74,22 +80,30 @@ pub async fn wildcard_test(target_url: &str) -> Option { // reflected in the response along with some static content; aka custom 404 let url_len = get_url_path_length(&resp_one.url()); - println!( + bar.println(format!( "[{}] - Url is being reflected in wildcard response, i.e. a dynamic wildcard", status_colorizer("WILDCARD") - ); - println!( - "[{}] - Auto-filtering out responses that are [({} + url length) bytes] long; this behavior can be turned off by using --dontfilter", - status_colorizer("WILDCARD"), - wc_length - url_len, + )); + bar.println( + format!( + "[{}] - Auto-filtering out responses that are [({} + url length) bytes] long; this behavior can be turned off by using --dontfilter", + status_colorizer("WILDCARD"), + wc_length - url_len, + ) ); wildcard.dynamic = wc_length - url_len; } else if wc_length == wc2_length { - println!("[{}] - Wildcard response is a static size; auto-filtering out responses of size [{} bytes]; this behavior can be turned off by using --dontfilter", status_colorizer("WILDCARD"), wc_length); + bar.println(format!( + "[{}] - Wildcard response is a static size; auto-filtering out responses of size [{} bytes]; this behavior can be turned off by using --dontfilter", + status_colorizer("WILDCARD"), + wc_length + )); wildcard.size = wc_length; } + } else { + bar.inc(2); } log::trace!("exit: wildcard_test -> Some({:?})", wildcard); @@ -106,7 +120,7 @@ pub async fn wildcard_test(target_url: &str) -> Option { /// Once the unique url is created, the request is sent to the server. If the server responds /// back with a valid status code, the response is considered to be a wildcard response. If that /// wildcard response has a 3xx status code, that redirection location is displayed to the user. -async fn make_wildcard_request(target_url: &str, length: usize) -> Option { +async fn make_wildcard_request(target_url: &str, length: usize, bar: ProgressBar) -> Option { log::trace!("enter: make_wildcard_request({}, {})", target_url, length); let unique_str = unique_string(length); @@ -137,31 +151,37 @@ async fn make_wildcard_request(target_url: &str, length: usize) -> Option {}", - wildcard, - response.url(), - next_loc_str + bar.println( + format!( + "[{}] {} redirects to => {}", + wildcard, + response.url(), + next_loc_str + ) ); } else { - println!( - "[{}] {} redirects to => {:?}", - wildcard, - response.url(), - next_loc + bar.println( + format!( + "[{}] {} redirects to => {:?}", + wildcard, + response.url(), + next_loc + ) ); } } @@ -190,6 +210,9 @@ pub async fn connectivity_test(target_urls: &[String]) -> Vec { let mut good_urls = vec![]; + // hidden bar just to get ProgressBar::println functionality + let bar = progress::add_bar("", 1, true); + for target_url in target_urls { let request = match format_url( target_url, @@ -201,6 +224,7 @@ pub async fn connectivity_test(target_urls: &[String]) -> Vec { Ok(url) => url, Err(e) => { log::error!("{}", e); + bar.inc(1); continue; } }; @@ -210,12 +234,14 @@ pub async fn connectivity_test(target_urls: &[String]) -> Vec { good_urls.push(target_url.to_owned()); } Err(e) => { - println!("Could not connect to {}, skipping...", target_url); + bar.println(format!("Could not connect to {}, skipping...", target_url)); log::error!("{}", e); } } } + bar.finish(); + if good_urls.is_empty() { log::error!("Could not connect to any target provided, exiting."); log::trace!("exit: connectivity_test"); diff --git a/src/lib.rs b/src/lib.rs index 2c3baf2..257c348 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,6 +4,7 @@ pub mod config; pub mod heuristics; pub mod logger; pub mod parser; +pub mod progress; pub mod scanner; pub mod utils; diff --git a/src/main.rs b/src/main.rs index 05ac3bc..36573f2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -71,7 +71,6 @@ async fn scan(targets: Vec) -> FeroxResult<()> { // drive execution of all accumulated futures futures::future::join_all(tasks).await; - log::trace!("exit: scan"); Ok(()) @@ -120,13 +119,16 @@ async fn main() { if !CONFIGURATION.quiet { // only print banner if -q isn't used banner::initialize(&targets); + // progress::initialize(); } // discard non-responsive targets let live_targets = heuristics::connectivity_test(&targets).await; match scan(live_targets).await { - Ok(_) => log::info!("Done"), + Ok(_) => { + log::info!("Done"); + } Err(e) => log::error!("An error occurred: {}", e), }; diff --git a/src/progress.rs b/src/progress.rs new file mode 100644 index 0000000..175d05f --- /dev/null +++ b/src/progress.rs @@ -0,0 +1,21 @@ +use crate::config::PROGRESS_BAR; +use indicatif::{ProgressBar, ProgressStyle}; + +pub fn add_bar(prefix: &str, length: u64, hidden: bool) -> ProgressBar { + let style = ProgressStyle::default_bar() + .template("[{bar:.cyan/blue}] - {elapsed:<4} {pos:>7}/{len:7} {per_sec:7} {prefix}") + .progress_chars("#>-"); + + let progress_bar = if hidden { + // PROGRESS_BAR.add(ProgressBar::hidden()) + ProgressBar::hidden() + } else { + PROGRESS_BAR.add(ProgressBar::new(length)) + }; + + progress_bar.set_style(style); + + progress_bar.set_prefix(&prefix); + + progress_bar +} diff --git a/src/scanner.rs b/src/scanner.rs index 9915ec5..38df99c 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -1,9 +1,10 @@ -use crate::config::CONFIGURATION; +use crate::config::{CONFIGURATION, PROGRESS_BAR}; use crate::heuristics::WildcardFilter; use crate::utils::{get_current_depth, get_url_path_length, status_colorizer}; -use crate::{heuristics, FeroxResult}; +use crate::{heuristics, progress, FeroxResult}; use futures::future::{BoxFuture, FutureExt}; use futures::{stream, StreamExt}; +use indicatif::ProgressBar; use reqwest::{Client, Response, Url}; use std::collections::HashSet; use std::fs::OpenOptions; @@ -12,6 +13,7 @@ use std::ops::Deref; use std::sync::Arc; use tokio::sync::mpsc::{self, UnboundedReceiver, UnboundedSender}; use tokio::task::JoinHandle; +use std::convert::TryInto; /// Simple helper to generate a `Url` /// @@ -164,23 +166,27 @@ async fn spawn_file_reporter(mut report_channel: UnboundedReceiver) { /// /// The consumer simply receives responses and prints them if they meet the given /// reporting criteria -async fn spawn_terminal_reporter(mut report_channel: UnboundedReceiver) { +async fn spawn_terminal_reporter( + mut report_channel: UnboundedReceiver, + bar: ProgressBar, +) { log::trace!("enter: spawn_terminal_reporter({:?})", report_channel); + //todo trace while let Some(resp) = report_channel.recv().await { log::debug!("received {} on reporting channel", resp.url()); if CONFIGURATION.statuscodes.contains(&resp.status().as_u16()) { if CONFIGURATION.quiet { - println!("{}", resp.url()); + bar.println(format!("{}", resp.url())); } else { let status = status_colorizer(&resp.status().to_string()); - println!( + bar.println(format!( "[{}] - {} - [{} bytes]", status, resp.url(), resp.content_length().unwrap_or(0) - ); + )); } } log::debug!("report complete: {}", resp.url()); @@ -492,11 +498,26 @@ pub async fn scan_url(target_url: &str, wordlist: Arc>, base_dep let (tx_dir, rx_dir): (UnboundedSender, UnboundedReceiver) = mpsc::unbounded_channel(); + let num_reqs_expected: u64 = if CONFIGURATION.extensions.is_empty() { + wordlist.len().try_into().unwrap() + } else { + let total = wordlist.len() * (CONFIGURATION.extensions.len() + 1); + total.try_into().unwrap() + }; + + let progress_bar = progress::add_bar(&target_url, num_reqs_expected, false); + progress_bar.reset_elapsed(); + + let bar_future = tokio::task::spawn_blocking(move || PROGRESS_BAR.join().unwrap()); + + let reporter_bar = progress_bar.clone(); + let wildcard_bar = progress_bar.clone(); + let reporter = if !CONFIGURATION.output.is_empty() { // output file defined tokio::spawn(async move { spawn_file_reporter(rx_rpt).await }) } else { - tokio::spawn(async move { spawn_terminal_reporter(rx_rpt).await }) + tokio::spawn(async move { spawn_terminal_reporter(rx_rpt, reporter_bar).await }) }; // lifetime satisfiers, as it's an Arc, clones are cheap anyway @@ -508,7 +529,7 @@ pub async fn scan_url(target_url: &str, wordlist: Arc>, base_dep async move { spawn_recursion_handler(rx_dir, recurser_words, base_depth).await }, ); - let filter = match heuristics::wildcard_test(&target_url).await { + let filter = match heuristics::wildcard_test(&target_url, wildcard_bar).await { Some(f) => { if CONFIGURATION.dontfilter { // don't auto filter, i.e. use the defaults @@ -526,14 +547,20 @@ pub async fn scan_url(target_url: &str, wordlist: Arc>, base_dep let wc_filter = filter.clone(); let txd = tx_dir.clone(); let txr = tx_rpt.clone(); + let pb = progress_bar.clone(); // progress bar is an Arc around internal state let tgt = target_url.to_string(); // done to satisfy 'static lifetime below - tokio::spawn(async move { - make_requests(&tgt, &word, base_depth, wc_filter, txd, txr).await - }) + ( + tokio::spawn(async move { + make_requests(&tgt, &word, base_depth, wc_filter, txd, txr).await + }), + pb, + ) }) - .for_each_concurrent(CONFIGURATION.threads, |resp| async move { + .for_each_concurrent(CONFIGURATION.threads, |(resp, bar)| async move { match resp.await { - Ok(_) => {} + Ok(_) => { + bar.inc(1); + } Err(e) => { log::error!("error awaiting a response: {}", e); } @@ -545,6 +572,8 @@ pub async fn scan_url(target_url: &str, wordlist: Arc>, base_dep producers.await; log::trace!("done awaiting scan producers"); + progress_bar.finish(); + // manually drop tx in order for the rx task's while loops to eval to false log::trace!("dropped recursion handler's transmitter"); drop(tx_dir); @@ -567,6 +596,8 @@ pub async fn scan_url(target_url: &str, wordlist: Arc>, base_dep } log::trace!("done awaiting report receiver"); + bar_future.await.unwrap(); + log::trace!("exit: scan_url"); }