Merge pull request #110 from epi052/FEATURE-105-add-replay-proxy

added replay-proxy option

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "feroxbuster"
-version = "1.4.0"
+version = "1.5.0"
 authors = ["Ben 'epi' Risher <epibar052@gmail.com>"]
 license = "MIT"
 edition = "2018"
@@ -19,7 +19,7 @@ futures = { version = "0.3"}
 tokio = { version = "0.2", features = ["full"] }
 tokio-util = {version = "0.3", features = ["codec"]}
 log = "0.4"
-env_logger = "0.7"
+env_logger = "0.8"
 reqwest = { version = "0.10", features = ["socks"] }
 clap = "2"
 lazy_static = "1.4"

README.md

@@ -82,6 +82,7 @@ This attack is also known as Predictable Resource Location, File Enumeration, Di
     - [Pass auth token via query parameter](#pass-auth-token-via-query-parameter)
     - [Limit Total Number of Concurrent Scans (new in `v1.2.0`)](#limit-total-number-of-concurrent-scans-new-in-v120)
     - [Filter Response by Status Code  (new in `v1.3.0`)](#filter-response-by-status-code--new-in-v130)
+    - [Replay Responses to a Proxy based on Status Code (new in `v1.5.0`)](#replay-responses-to-a-proxy-based-on-status-code-new-in-v150)
 - [Comparison w/ Similar Tools](#-comparison-w-similar-tools)
 - [Common Problems/Issues (FAQ)](#-common-problemsissues-faq)
     - [No file descriptors available](#no-file-descriptors-available)
@@ -276,9 +277,11 @@ A pre-made configuration file with examples of all available settings can be fou
 # wordlist = "/wordlists/jhaddix/all.txt"
 # status_codes = [200, 500]
 # filter_status = [301]
+# replay_codes = [301]
 # threads = 1
 # timeout = 5
 # proxy = "http://127.0.0.1:8080"
+# replay_proxy = "http://127.0.0.1:8081"
 # verbosity = 1
 # scan_limit = 6
 # quiet = true
@@ -335,11 +338,15 @@ OPTIONS:
     -d, --depth <RECURSION_DEPTH>           Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)
     -x, --extensions <FILE_EXTENSION>...    File extension(s) to search for (ex: -x php -x pdf js)
     -S, --filter-size <SIZE>...             Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)
-    -C, --filter-status <STATUS_CODE>...    Filter out status codes (deny list) (ex: -C 200 -S 401)
+    -C, --filter-status <STATUS_CODE>...    Filter out status codes (deny list) (ex: -C 200 -C 401)
     -H, --headers <HEADER>...               Specify HTTP headers (ex: -H Header:val 'stuff: things')
     -o, --output <FILE>                     Output file to write results to (default: stdout)
     -p, --proxy <PROXY>                     Proxy to use for requests (ex: http(s)://host:port, socks5://host:port)
     -Q, --query <QUERY>...                  Specify URL query parameters (ex: -Q token=stuff -Q secret=key)
+    -R, --replay-codes <REPLAY_CODE>...     Status Codes to send through a Replay Proxy when found (default: --status
+                                            -codes value)
+    -P, --replay-proxy <REPLAY_PROXY>       Send only unfiltered requests through a Replay Proxy, instead of all
+                                            requests
     -L, --scan-limit <SCAN_LIMIT>           Limit total number of concurrent scans (default: 0, i.e. no limit)
     -s, --status-codes <STATUS_CODE>...     Status Codes to include (allow list) (default: 200 204 301 302 307 308 401
                                             403 405)
@@ -459,6 +466,20 @@ each one is checked against a list of known filters and either displayed or not
 ./feroxbuster -u http://127.1 --filter-status 301
 ```
 
+### Replay Responses to a Proxy based on Status Code (new in `v1.5.0`)
+
+The `--replay-proxy` and `--replay-codes` options were added as a way to only send a select few responses to a proxy.  This is in stark contrast to `--proxy` which proxies EVERY request.  
+
+Imagine you only care about proxying responses that have either the status code `200` or `302` (or you just don't want to clutter up your Burp history).  These two options will allow you to fine-tune what gets proxied and what doesn't.  
+
+```
+./feroxbuster -u http://127.1 --replay-proxy http://localhost:8080 --replay-codes 200 302 --insecure
+```
+
+Of note: this means that for every response that matches your replay criteria, you'll end up sending the request that generated that response a second time.  Depending on the target and your engagement terms (if any), it may not make sense from a traffic generated perspective.
+
+![replay-proxy-demo](img/replay-proxy-demo.gif)
+
 ## 🧐 Comparison w/ Similar Tools
 
 There are quite a few similar tools for forced browsing/content discovery.  Burp Suite Pro, Dirb, Dirbuster, etc... 

ferox-config.toml.example

@@ -13,6 +13,8 @@
 # threads = 1
 # timeout = 5
 # proxy = "http://127.0.0.1:8080"
+# replay_proxy = "http://127.0.0.1:8081"
+# replay_codes = [200, 302]
 # verbosity = 1
 # scan_limit = 6
 # quiet = true

src/banner.rs

@@ -246,6 +246,35 @@ by Ben "epi" Risher {}                  ver: {}"#,
         .unwrap_or_default(); // 💎
     }
 
+    if !config.replay_proxy.is_empty() {
+        // i include replay codes logic here because in config.rs, replay codes are set to the
+        // value in status codes, meaning it's never empty
+
+        let mut replay_codes = vec![];
+
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!("\u{1f3a5}", "Replay Proxy", config.replay_proxy)
+        )
+        .unwrap_or_default(); // 🎥
+
+        for code in &config.replay_codes {
+            replay_codes.push(status_colorizer(&code.to_string()))
+        }
+
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!(
+                "\u{1f4fc}",
+                "Replay Proxy Codes",
+                format!("[{}]", replay_codes.join(", "))
+            )
+        )
+        .unwrap_or_default(); // 📼
+    }
+
     if !config.headers.is_empty() {
         for (name, value) in &config.headers {
             writeln!(
@@ -438,7 +467,7 @@ by Ben "epi" Risher {}                  ver: {}"#,
     // ⏯
     writeln!(
         &mut writer,
-        " \u{23ef}  Press [{}] to {}|{} your scan",
+        " \u{23ef}   Press [{}] to {}|{} your scan",
         style("ENTER").yellow(),
         style("pause").red(),
         style("resume").green()

src/client.rs

@@ -62,11 +62,6 @@ pub fn initialize(
     match client.build() {
         Ok(client) => client,
         Err(e) => {
-            eprintln!(
-                "{} {} Could not create a Client with the given configuration, exiting.",
-                status_colorizer("ERROR"),
-                module_colorizer("Client::build")
-            );
             eprintln!(
                 "{} {} {}",
                 status_colorizer("ERROR"),

src/config.rs

@@ -10,6 +10,7 @@ use std::collections::HashMap;
 use std::env::{current_dir, current_exe};
 use std::fs::read_to_string;
 use std::path::PathBuf;
+#[cfg(not(test))]
 use std::process::exit;
 
 lazy_static! {
@@ -23,6 +24,21 @@ lazy_static! {
     pub static ref PROGRESS_PRINTER: ProgressBar = progress::add_bar("", 0, true);
 }
 
+/// simple helper to clean up some code reuse below; panics under test / exits in prod
+fn report_and_exit(err: &str) -> ! {
+    eprintln!(
+        "{} {}: {}",
+        status_colorizer("ERROR"),
+        module_colorizer("Configuration::new"),
+        err
+    );
+
+    #[cfg(test)]
+    panic!();
+    #[cfg(not(test))]
+    exit(1);
+}
+
 /// Represents the final, global configuration of the program.
 ///
 /// This struct is the combination of the following:
@@ -47,6 +63,10 @@ pub struct Configuration {
     #[serde(default)]
     pub proxy: String,
 
+    /// Replay Proxy to use for requests (ex: http(s)://host:port, socks5://host:port)
+    #[serde(default)]
+    pub replay_proxy: String,
+
     /// The target URL
     #[serde(default)]
     pub target_url: String,
@@ -55,6 +75,10 @@ pub struct Configuration {
     #[serde(default = "status_codes")]
     pub status_codes: Vec<u16>,
 
+    /// Status Codes to replay to the Replay Proxy (default: whatever is passed to --status-code)
+    #[serde(default)]
+    pub replay_codes: Vec<u16>,
+
     /// Status Codes to filter out (deny list)
     #[serde(default)]
     pub filter_status: Vec<u16>,
@@ -63,6 +87,10 @@ pub struct Configuration {
     #[serde(skip)]
     pub client: Client,
 
+    /// Instance of [reqwest::Client](https://docs.rs/reqwest/latest/reqwest/struct.Client.html)
+    #[serde(skip)]
+    pub replay_client: Option<Client>,
+
     /// Number of concurrent threads (default: 50)
     #[serde(default = "threads")]
     pub threads: usize,
@@ -183,11 +211,17 @@ impl Default for Configuration {
         let timeout = timeout();
         let user_agent = user_agent();
         let client = client::initialize(timeout, &user_agent, false, false, &HashMap::new(), None);
+        let replay_client = None;
+        let status_codes = status_codes();
+        let replay_codes = status_codes.clone();
 
         Configuration {
             client,
             timeout,
             user_agent,
+            replay_codes,
+            status_codes,
+            replay_client,
             dont_filter: false,
             quiet: false,
             stdin: false,
@@ -202,15 +236,15 @@ impl Default for Configuration {
             config: String::new(),
             output: String::new(),
             target_url: String::new(),
+            replay_proxy: String::new(),
             queries: Vec::new(),
             extensions: Vec::new(),
             filter_size: Vec::new(),
             filter_status: Vec::new(),
             headers: HashMap::new(),
-            threads: threads(),
             depth: depth(),
+            threads: threads(),
             wordlist: wordlist(),
-            status_codes: status_codes(),
         }
     }
 }
@@ -244,6 +278,8 @@ impl Configuration {
     /// - **dont_filter**: `false` (auto filter wildcard responses)
     /// - **depth**: `4` (maximum recursion depth)
     /// - **scan_limit**: `0` (no limit on concurrent scans imposed)
+    /// - **replay_proxy**: `None` (no limit on concurrent scans imposed)
+    /// - **replay_codes**: [`DEFAULT_RESPONSE_CODES`](constant.DEFAULT_RESPONSE_CODES.html)
     ///
     /// After which, any values defined in a
     /// [ferox-config.toml](constant.DEFAULT_CONFIG_NAME.html) config file will override the
@@ -348,35 +384,35 @@ impl Configuration {
                 .unwrap() // already known good
                 .map(|code| {
                     StatusCode::from_bytes(code.as_bytes())
-                        .unwrap_or_else(|e| {
-                            eprintln!(
-                                "{} {}: {}",
-                                status_colorizer("ERROR"),
-                                module_colorizer("Configuration::new"),
-                                e
-                            );
-                            exit(1)
-                        })
+                        .unwrap_or_else(|e| report_and_exit(&e.to_string()))
                         .as_u16()
                 })
                 .collect();
         }
 
+        if args.values_of("replay_codes").is_some() {
+            // replay codes passed in by the user
+            config.replay_codes = args
+                .values_of("replay_codes")
+                .unwrap() // already known good
+                .map(|code| {
+                    StatusCode::from_bytes(code.as_bytes())
+                        .unwrap_or_else(|e| report_and_exit(&e.to_string()))
+                        .as_u16()
+                })
+                .collect();
+        } else {
+            // not passed in by the user, use whatever value is held in status_codes
+            config.replay_codes = config.status_codes.clone();
+        }
+
         if args.values_of("filter_status").is_some() {
             config.filter_status = args
                 .values_of("filter_status")
                 .unwrap() // already known good
                 .map(|code| {
                     StatusCode::from_bytes(code.as_bytes())
-                        .unwrap_or_else(|e| {
-                            eprintln!(
-                                "{} {}: {}",
-                                status_colorizer("ERROR"),
-                                module_colorizer("Configuration::new"),
-                                e
-                            );
-                            exit(1)
-                        })
+                        .unwrap_or_else(|e| report_and_exit(&e.to_string()))
                         .as_u16()
                 })
                 .collect();
@@ -395,15 +431,8 @@ impl Configuration {
                 .values_of("filter_size")
                 .unwrap() // already known good
                 .map(|size| {
-                    size.parse::<u64>().unwrap_or_else(|e| {
-                        eprintln!(
-                            "{} {}: {}",
-                            status_colorizer("ERROR"),
-                            module_colorizer("Configuration::new"),
-                            e
-                        );
-                        exit(1)
-                    })
+                    size.parse::<u64>()
+                        .unwrap_or_else(|e| report_and_exit(&e.to_string()))
                 })
                 .collect();
         }
@@ -451,6 +480,10 @@ impl Configuration {
             config.proxy = String::from(args.value_of("proxy").unwrap());
         }
 
+        if args.value_of("replay_proxy").is_some() {
+            config.replay_proxy = String::from(args.value_of("replay_proxy").unwrap());
+        }
+
         if args.value_of("user_agent").is_some() {
             config.user_agent = String::from(args.value_of("user_agent").unwrap());
         }
@@ -526,6 +559,18 @@ impl Configuration {
             }
         }
 
+        if !config.replay_proxy.is_empty() {
+            // only set replay_client when replay_proxy is set
+            config.replay_client = Some(client::initialize(
+                config.timeout,
+                &config.user_agent,
+                config.redirects,
+                config.insecure,
+                &config.headers,
+                Some(&config.replay_proxy),
+            ));
+        }
+
         config
     }
 
@@ -574,6 +619,8 @@ impl Configuration {
         settings.filter_status = settings_to_merge.filter_status;
         settings.dont_filter = settings_to_merge.dont_filter;
         settings.scan_limit = settings_to_merge.scan_limit;
+        settings.replay_proxy = settings_to_merge.replay_proxy;
+        settings.replay_codes = settings_to_merge.replay_codes;
     }
 
     /// If present, read in `DEFAULT_CONFIG_NAME` and deserialize the specified values
@@ -610,9 +657,11 @@ mod tests {
         let data = r#"
             wordlist = "/some/path"
             status_codes = [201, 301, 401]
+            replay_codes = [201, 301]
             threads = 40
             timeout = 5
             proxy = "http://127.0.0.1:8080"
+            replay_proxy = "http://127.0.0.1:8081"
             quiet = true
             verbosity = 1
             scan_limit = 6
@@ -645,7 +694,10 @@ mod tests {
         assert_eq!(config.proxy, String::new());
         assert_eq!(config.target_url, String::new());
         assert_eq!(config.config, String::new());
+        assert_eq!(config.replay_proxy, String::new());
         assert_eq!(config.status_codes, status_codes());
+        assert_eq!(config.replay_codes, config.status_codes);
+        assert!(config.replay_client.is_none());
         assert_eq!(config.threads, threads());
         assert_eq!(config.depth, depth());
         assert_eq!(config.timeout, timeout());
@@ -680,6 +732,13 @@ mod tests {
         assert_eq!(config.status_codes, vec![201, 301, 401]);
     }
 
+    #[test]
+    /// parse the test config and see that the value parsed is correct
+    fn config_reads_replay_codes() {
+        let config = setup_config_test();
+        assert_eq!(config.replay_codes, vec![201, 301]);
+    }
+
     #[test]
     /// parse the test config and see that the value parsed is correct
     fn config_reads_threads() {
@@ -715,6 +774,13 @@ mod tests {
         assert_eq!(config.proxy, "http://127.0.0.1:8080");
     }
 
+    #[test]
+    /// parse the test config and see that the value parsed is correct
+    fn config_reads_replay_proxy() {
+        let config = setup_config_test();
+        assert_eq!(config.replay_proxy, "http://127.0.0.1:8081");
+    }
+
     #[test]
     /// parse the test config and see that the value parsed is correct
     fn config_reads_quiet() {
@@ -825,4 +891,11 @@ mod tests {
         queries.push(("rick".to_string(), "astley".to_string()));
         assert_eq!(config.queries, queries);
     }
+
+    #[test]
+    #[should_panic]
+    /// test that an error message is printed and panic is called when report_and_exit is called
+    fn config_report_and_exit_works() {
+        report_and_exit("some message");
+    }
 }

src/heuristics.rs

@@ -10,7 +10,6 @@ use crate::{
 };
 use console::style;
 use indicatif::ProgressBar;
-use std::process;
 use tokio::sync::mpsc::UnboundedSender;
 use uuid::Uuid;
 
@@ -287,14 +286,6 @@ pub async fn connectivity_test(target_urls: &[String]) -> Vec<String> {
 
     if good_urls.is_empty() {
         log::error!("Could not connect to any target provided, exiting.");
-        log::trace!("exit: connectivity_test");
-        eprintln!(
-            "{} {} Could not connect to any target provided",
-            status_colorizer("ERROR"),
-            module_colorizer("heuristics::connectivity_test"),
-        );
-
-        process::exit(1);
     }
 
     log::trace!("exit: connectivity_test -> {:?}", good_urls);
@@ -316,8 +307,7 @@ fn try_send_message_to_file(msg: &str, tx_file: UnboundedSender<String>, save_ou
             }
             Err(e) => {
                 log::error!(
-                    "{} {} {}",
-                    status_colorizer("ERROR"),
+                    "{} {}",
                     module_colorizer("heuristics::try_send_message_to_file"),
                     e
                 );

src/lib.rs

@@ -15,11 +15,26 @@ use reqwest::{
     header::HeaderMap,
     {Response, StatusCode, Url},
 };
+use std::{error, fmt};
 use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender};
 
 /// Generic Result type to ease error handling in async contexts
-pub type FeroxResult<T> =
-    std::result::Result<T, Box<dyn std::error::Error + Send + Sync + 'static>>;
+pub type FeroxResult<T> = std::result::Result<T, Box<dyn error::Error + Send + Sync + 'static>>;
+
+/// Simple Error implementation to allow for custom error returns
+#[derive(Debug, Default)]
+pub struct FeroxError {
+    /// fancy string that can be printed via Display
+    pub message: String,
+}
+
+impl error::Error for FeroxError {}
+
+impl fmt::Display for FeroxError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}", &self.message)
+    }
+}
 
 /// Generic mpsc::unbounded_channel type to tidy up some code
 pub type FeroxChannel<T> = (UnboundedSender<T>, UnboundedReceiver<T>);

src/main.rs

@@ -1,11 +1,11 @@
 use crossterm::event::{self, Event, KeyCode};
 use feroxbuster::{
     banner,
-    config::{CONFIGURATION, PROGRESS_PRINTER},
+    config::{CONFIGURATION, PROGRESS_BAR, PROGRESS_PRINTER},
     heuristics, logger, reporter,
     scanner::{scan_url, PAUSE_SCAN},
     utils::{ferox_print, get_current_depth, module_colorizer, status_colorizer},
-    FeroxResponse, FeroxResult, SLEEP_DURATION, VERSION,
+    FeroxError, FeroxResponse, FeroxResult, SLEEP_DURATION, VERSION,
 };
 use futures::StreamExt;
 use std::{
@@ -19,7 +19,7 @@ use std::{
     },
     time::Duration,
 };
-use tokio::{io, sync::mpsc::UnboundedSender};
+use tokio::{io, sync::mpsc::UnboundedSender, task::JoinHandle};
 use tokio_util::codec::{FramedRead, LinesCodec};
 
 /// Atomic boolean flag, used to determine whether or not the terminal input handler should exit
@@ -61,12 +61,6 @@ fn get_unique_words_from_wordlist(path: &str) -> FeroxResult<Arc<HashSet<String>
     let file = match File::open(&path) {
         Ok(f) => f,
         Err(e) => {
-            eprintln!(
-                "{} {} {}",
-                status_colorizer("ERROR"),
-                module_colorizer("main::get_unique_words_from_wordlist"),
-                e
-            );
             log::error!("Could not open wordlist: {}", e);
             log::trace!("exit: get_unique_words_from_wordlist -> {}", e);
 
@@ -111,13 +105,9 @@ async fn scan(
             .await??;
 
     if words.len() == 0 {
-        eprintln!(
-            "{} {} Did not find any words in {}",
-            status_colorizer("ERROR"),
-            module_colorizer("main::scan"),
-            CONFIGURATION.wordlist
-        );
-        process::exit(1);
+        let mut err = FeroxError::default();
+        err.message = format!("Did not find any words in {}", CONFIGURATION.wordlist);
+        return Err(Box::new(err));
     }
 
     let mut tasks = vec![];
@@ -142,6 +132,7 @@ async fn scan(
     Ok(())
 }
 
+/// Get targets from either commandline or stdin, pass them back to the caller as a Result<Vec>
 async fn get_targets() -> FeroxResult<Vec<String>> {
     log::trace!("enter: get_targets");
 
@@ -165,12 +156,23 @@ async fn get_targets() -> FeroxResult<Vec<String>> {
     Ok(targets)
 }
 
-#[tokio::main]
-async fn main() {
-    // setup logging based on the number of -v's used
-    logger::initialize(CONFIGURATION.verbosity);
+/// async main called from real main, broken out in this way to allow for some synchronous code
+/// to be executed before bringing the tokio runtime online
+async fn wrapped_main() {
+    // join can only be called once, otherwise it causes the thread to panic
+    tokio::task::spawn_blocking(move || {
+        // ok, lazy_static! uses (unsurprisingly in retrospect) a lazy loading model where the
+        // thing obtained through deref isn't actually created until it's used. This created a
+        // problem when initializing the logger as it relied on PROGRESS_PRINTER which may or may
+        // not have been created by the time it was needed for logging (really only occurred in
+        // heuristics / banner / main). In order to initialize logging properly, we need to ensure
+        // PROGRESS_PRINTER and PROGRESS_BAR have been used at least once.  This call satisfies
+        // that constraint
+        PROGRESS_PRINTER.println("");
+        PROGRESS_BAR.join().unwrap();
+    });
 
-    // can't trace main until after logger is initialized
+    // can't trace main until after logger is initialized and the above task is started
     log::trace!("enter: main");
     log::debug!("{:#?}", *CONFIGURATION);
 
@@ -189,17 +191,9 @@ async fn main() {
         Ok(t) => t,
         Err(e) => {
             // should only happen in the event that there was an error reading from stdin
-            log::error!("{}", e);
-            ferox_print(
-                &format!(
-                    "{} {} {}",
-                    status_colorizer("ERROR"),
-                    module_colorizer("main::get_targets"),
-                    e
-                ),
-                &PROGRESS_PRINTER,
-            );
-            process::exit(1);
+            log::error!("{} {}", module_colorizer("main::get_targets"), e);
+            clean_up(tx_term, term_handle, tx_file, file_handle, save_output).await;
+            return;
         }
     };
 
@@ -212,15 +206,49 @@ async fn main() {
     // discard non-responsive targets
     let live_targets = heuristics::connectivity_test(&targets).await;
 
+    if live_targets.is_empty() {
+        clean_up(tx_term, term_handle, tx_file, file_handle, save_output).await;
+        return;
+    }
+
     // kick off a scan against any targets determined to be responsive
     match scan(live_targets, tx_term.clone(), tx_file.clone()).await {
         Ok(_) => {
             log::info!("All scans complete!");
         }
-        Err(e) => log::error!("An error occurred: {}", e),
+        Err(e) => {
+            ferox_print(
+                &format!("{} while scanning: {}", status_colorizer("Error"), e),
+                &PROGRESS_PRINTER,
+            );
+            clean_up(tx_term, term_handle, tx_file, file_handle, save_output).await;
+            process::exit(1);
+        }
     };
 
-    // manually drop tx in order for the rx task's while loops to eval to false
+    clean_up(tx_term, term_handle, tx_file, file_handle, save_output).await;
+
+    log::trace!("exit: main");
+}
+
+/// Single cleanup function that handles all the necessary drops/finishes etc required to gracefully
+/// shutdown the program
+async fn clean_up(
+    tx_term: UnboundedSender<FeroxResponse>,
+    term_handle: JoinHandle<()>,
+    tx_file: UnboundedSender<String>,
+    file_handle: Option<JoinHandle<()>>,
+    save_output: bool,
+) {
+    log::trace!(
+        "enter: clean_up({:?}, {:?}, {:?}, {:?}, {}",
+        tx_term,
+        term_handle,
+        tx_file,
+        file_handle,
+        save_output
+    );
+
     drop(tx_term);
     log::trace!("dropped terminal output handler's transmitter");
 
@@ -255,9 +283,19 @@ async fn main() {
     // mark all scans complete so the terminal input handler will exit cleanly
     SCAN_COMPLETE.store(true, Ordering::Relaxed);
 
-    log::trace!("exit: main");
-
     // clean-up function for the MultiProgress bar; must be called last in order to still see
-    // the final trace message above
+    // the final trace messages above
     PROGRESS_PRINTER.finish();
+
+    log::trace!("exit: clean_up");
+}
+
+fn main() {
+    // setup logging based on the number of -v's used
+    logger::initialize(CONFIGURATION.verbosity);
+
+    if let Ok(mut runtime) = tokio::runtime::Runtime::new() {
+        let future = wrapped_main();
+        runtime.block_on(future);
+    }
 }

src/parser.rs

@@ -67,6 +67,29 @@ pub fn initialize() -> App<'static, 'static> {
                     "Proxy to use for requests (ex: http(s)://host:port, socks5://host:port)",
                 ),
         )
+        .arg(
+            Arg::with_name("replay_proxy")
+                .short("P")
+                .long("replay-proxy")
+                .takes_value(true)
+                .value_name("REPLAY_PROXY")
+                .help(
+                    "Send only unfiltered requests through a Replay Proxy, instead of all requests",
+                ),
+        )
+        .arg(
+            Arg::with_name("replay_codes")
+                .short("R")
+                .long("replay-codes")
+                .value_name("REPLAY_CODE")
+                .takes_value(true)
+                .multiple(true)
+                .use_delimiter(true)
+                .requires("replay_proxy")
+                .help(
+                    "Status Codes to send through a Replay Proxy when found (default: --status-codes value)",
+                ),
+        )
         .arg(
             Arg::with_name("status_codes")
                 .short("s")
@@ -204,7 +227,7 @@ pub fn initialize() -> App<'static, 'static> {
                 .multiple(true)
                 .use_delimiter(true)
                 .help(
-                    "Filter out status codes (deny list) (ex: -C 200 -S 401)",
+                    "Filter out status codes (deny list) (ex: -C 200 -C 401)",
                 ),
         )
         .arg(

src/reporter.rs

@@ -1,5 +1,5 @@
 use crate::config::{CONFIGURATION, PROGRESS_PRINTER};
-use crate::utils::{ferox_print, status_colorizer};
+use crate::utils::{ferox_print, make_request, status_colorizer};
 use crate::{FeroxChannel, FeroxResponse};
 use console::strip_ansi_codes;
 use std::io::Write;
@@ -92,7 +92,7 @@ async fn spawn_terminal_reporter(
     );
 
     while let Some(resp) = resp_chan.recv().await {
-        log::debug!("received {} on reporting channel", resp.url());
+        log::trace!("received {} on reporting channel", resp.url());
 
         if CONFIGURATION.status_codes.contains(&resp.status().as_u16()) {
             let report = if CONFIGURATION.quiet {
@@ -126,7 +126,20 @@ async fn spawn_terminal_reporter(
                 }
             }
         }
-        log::debug!("report complete: {}", resp.url());
+        log::trace!("report complete: {}", resp.url());
+
+        if CONFIGURATION.replay_client.is_some()
+            && CONFIGURATION.replay_codes.contains(&resp.status().as_u16())
+        {
+            // replay proxy specified/client created and this response's status code is one that
+            // should be replayed
+            match make_request(CONFIGURATION.replay_client.as_ref().unwrap(), &resp.url()).await {
+                Ok(_) => {}
+                Err(e) => {
+                    log::error!("{}", e);
+                }
+            }
+        }
     }
     log::trace!("exit: spawn_terminal_reporter");
 }

src/scanner.rs

@@ -1,5 +1,5 @@
 use crate::{
-    config::{CONFIGURATION, PROGRESS_BAR},
+    config::CONFIGURATION,
     extractor::get_links,
     filters::{FeroxFilter, StatusCodeFilter, WildcardFilter},
     heuristics, progress,
@@ -619,11 +619,9 @@ pub async fn scan_url(
     progress_bar.reset_elapsed();
 
     if CALL_COUNT.load(Ordering::Relaxed) == 0 {
-        // join can only be called once, otherwise it causes the thread to panic
-        tokio::task::spawn_blocking(move || PROGRESS_BAR.join().unwrap());
         CALL_COUNT.fetch_add(1, Ordering::Relaxed);
 
-        // this protection around join also allows us to add the first scanned url to SCANNED_URLS
+        // this protection allows us to add the first scanned url to SCANNED_URLS
         // from within the scan_url function instead of the recursion handler
         add_url_to_list_of_scanned_urls(&target_url, &SCANNED_URLS);
 

tests/test_banner.rs

@@ -23,7 +23,7 @@ fn banner_prints_proxy() -> Result<(), Box<dyn std::error::Error>> {
         .pipe_stdin(file)
         .unwrap()
         .assert()
-        .failure()
+        .success()
         .stderr(
             predicate::str::contains("─┬─")
                 .and(predicate::str::contains("Target Url"))
@@ -43,6 +43,46 @@ fn banner_prints_proxy() -> Result<(), Box<dyn std::error::Error>> {
     Ok(())
 }
 
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + replay proxy
+fn banner_prints_replay_proxy() -> Result<(), Box<dyn std::error::Error>> {
+    let urls = vec![
+        String::from("http://localhost"),
+        String::from("http://schmocalhost"),
+    ];
+    let (tmp_dir, file) = setup_tmp_directory(&urls, "wordlist")?;
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--stdin")
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--replay-proxy")
+        .arg("http://127.0.0.1:8081")
+        .pipe_stdin(file)
+        .unwrap()
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("http://schmocalhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Replay Proxy"))
+                .and(predicate::str::contains("http://127.0.0.1:8081"))
+                .and(predicate::str::contains("─┴─")),
+        );
+
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
 #[test]
 /// test allows non-existent wordlist to trigger the banner printing to stderr
 /// expect to see all mandatory prints + multiple headers
@@ -56,7 +96,7 @@ fn banner_prints_headers() -> Result<(), Box<dyn std::error::Error>> {
         .arg("-H")
         .arg("mostuff:mothings")
         .assert()
-        .failure()
+        .success()
         .stderr(
             predicate::str::contains("─┬─")
                 .and(predicate::str::contains("Target Url"))
@@ -87,7 +127,7 @@ fn banner_prints_filter_sizes() -> Result<(), Box<dyn std::error::Error>> {
         .arg("--filter-size")
         .arg("44444444")
         .assert()
-        .failure()
+        .success()
         .stderr(
             predicate::str::contains("─┬─")
                 .and(predicate::str::contains("Target Url"))
@@ -118,7 +158,7 @@ fn banner_prints_queries() -> Result<(), Box<dyn std::error::Error>> {
         .arg("--query")
         .arg("stuff=things")
         .assert()
-        .failure()
+        .success()
         .stderr(
             predicate::str::contains("─┬─")
                 .and(predicate::str::contains("Target Url"))
@@ -147,7 +187,7 @@ fn banner_prints_status_codes() -> Result<(), Box<dyn std::error::Error>> {
         .arg("-s")
         .arg("201,301,401")
         .assert()
-        .failure()
+        .success()
         .stderr(
             predicate::str::contains("─┬─")
                 .and(predicate::str::contains("Target Url"))
@@ -163,6 +203,37 @@ fn banner_prints_status_codes() -> Result<(), Box<dyn std::error::Error>> {
     Ok(())
 }
 
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + replay codes
+fn banner_prints_replay_codes() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--replay-codes")
+        .arg("200,302")
+        .arg("--replay-proxy")
+        .arg("http://localhost:8081")
+        .assert()
+        .success()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Replay Proxy"))
+                .and(predicate::str::contains("http://localhost:8081"))
+                .and(predicate::str::contains("Replay Proxy Codes"))
+                .and(predicate::str::contains("[200, 302]"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
 #[test]
 /// test allows non-existent wordlist to trigger the banner printing to stderr
 /// expect to see all mandatory prints + output file
@@ -174,7 +245,7 @@ fn banner_prints_output_file() -> Result<(), Box<dyn std::error::Error>> {
         .arg("--output")
         .arg("/super/cool/path")
         .assert()
-        .failure()
+        .success()
         .stderr(
             predicate::str::contains("─┬─")
                 .and(predicate::str::contains("Target Url"))
@@ -201,7 +272,7 @@ fn banner_prints_insecure() -> Result<(), Box<dyn std::error::Error>> {
         .arg("http://localhost")
         .arg("-k")
         .assert()
-        .failure()
+        .success()
         .stderr(
             predicate::str::contains("─┬─")
                 .and(predicate::str::contains("Target Url"))
@@ -228,7 +299,7 @@ fn banner_prints_redirects() -> Result<(), Box<dyn std::error::Error>> {
         .arg("http://localhost")
         .arg("-r")
         .assert()
-        .failure()
+        .success()
         .stderr(
             predicate::str::contains("─┬─")
                 .and(predicate::str::contains("Target Url"))
@@ -258,7 +329,7 @@ fn banner_prints_extensions() -> Result<(), Box<dyn std::error::Error>> {
         .arg("--extensions")
         .arg("pdf")
         .assert()
-        .failure()
+        .success()
         .stderr(
             predicate::str::contains("─┬─")
                 .and(predicate::str::contains("Target Url"))
@@ -285,7 +356,7 @@ fn banner_prints_dont_filter() -> Result<(), Box<dyn std::error::Error>> {
         .arg("http://localhost")
         .arg("--dont-filter")
         .assert()
-        .failure()
+        .success()
         .stderr(
             predicate::str::contains("─┬─")
                 .and(predicate::str::contains("Target Url"))
@@ -312,7 +383,7 @@ fn banner_prints_verbosity_one() -> Result<(), Box<dyn std::error::Error>> {
         .arg("http://localhost")
         .arg("-v")
         .assert()
-        .failure()
+        .success()
         .stderr(
             predicate::str::contains("─┬─")
                 .and(predicate::str::contains("Target Url"))
@@ -339,7 +410,7 @@ fn banner_prints_verbosity_two() -> Result<(), Box<dyn std::error::Error>> {
         .arg("http://localhost")
         .arg("-vv")
         .assert()
-        .failure()
+        .success()
         .stderr(
             predicate::str::contains("─┬─")
                 .and(predicate::str::contains("Target Url"))
@@ -366,7 +437,7 @@ fn banner_prints_verbosity_three() -> Result<(), Box<dyn std::error::Error>> {
         .arg("http://localhost")
         .arg("-vvv")
         .assert()
-        .failure()
+        .success()
         .stderr(
             predicate::str::contains("─┬─")
                 .and(predicate::str::contains("Target Url"))
@@ -393,7 +464,7 @@ fn banner_prints_verbosity_four() -> Result<(), Box<dyn std::error::Error>> {
         .arg("http://localhost")
         .arg("-vvvv")
         .assert()
-        .failure()
+        .success()
         .stderr(
             predicate::str::contains("─┬─")
                 .and(predicate::str::contains("Target Url"))
@@ -420,7 +491,7 @@ fn banner_prints_add_slash() -> Result<(), Box<dyn std::error::Error>> {
         .arg("http://localhost")
         .arg("-f")
         .assert()
-        .failure()
+        .success()
         .stderr(
             predicate::str::contains("─┬─")
                 .and(predicate::str::contains("Target Url"))
@@ -448,7 +519,7 @@ fn banner_prints_infinite_depth() -> Result<(), Box<dyn std::error::Error>> {
         .arg("--depth")
         .arg("0")
         .assert()
-        .failure()
+        .success()
         .stderr(
             predicate::str::contains("─┬─")
                 .and(predicate::str::contains("Target Url"))
@@ -476,7 +547,7 @@ fn banner_prints_recursion_depth() -> Result<(), Box<dyn std::error::Error>> {
         .arg("--depth")
         .arg("343214")
         .assert()
-        .failure()
+        .success()
         .stderr(
             predicate::str::contains("─┬─")
                 .and(predicate::str::contains("Target Url"))
@@ -503,7 +574,7 @@ fn banner_prints_no_recursion() -> Result<(), Box<dyn std::error::Error>> {
         .arg("http://localhost")
         .arg("-n")
         .assert()
-        .failure()
+        .success()
         .stderr(
             predicate::str::contains("─┬─")
                 .and(predicate::str::contains("Target Url"))
@@ -522,7 +593,7 @@ fn banner_prints_no_recursion() -> Result<(), Box<dyn std::error::Error>> {
 
 #[test]
 /// test allows non-existent wordlist to trigger the banner printing to stderr
-/// expect to see only the error of could not connect
+/// expect to see nothing
 fn banner_doesnt_print() -> Result<(), Box<dyn std::error::Error>> {
     Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -530,10 +601,8 @@ fn banner_doesnt_print() -> Result<(), Box<dyn std::error::Error>> {
         .arg("http://localhost")
         .arg("-q")
         .assert()
-        .failure()
-        .stderr(predicate::str::contains(
-            "ERROR heuristics::connectivity_test Could not connect to any target provided",
-        ));
+        .success()
+        .stderr(predicate::str::is_empty());
     Ok(())
 }
 
@@ -547,7 +616,7 @@ fn banner_prints_extract_links() -> Result<(), Box<dyn std::error::Error>> {
         .arg("http://localhost")
         .arg("-e")
         .assert()
-        .failure()
+        .success()
         .stderr(
             predicate::str::contains("─┬─")
                 .and(predicate::str::contains("Target Url"))
@@ -575,7 +644,7 @@ fn banner_prints_scan_limit() -> Result<(), Box<dyn std::error::Error>> {
         .arg("-L")
         .arg("4")
         .assert()
-        .failure()
+        .success()
         .stderr(
             predicate::str::contains("─┬─")
                 .and(predicate::str::contains("Target Url"))
@@ -603,7 +672,7 @@ fn banner_prints_filter_status() -> Result<(), Box<dyn std::error::Error>> {
         .arg("-C")
         .arg("200")
         .assert()
-        .failure()
+        .success()
         .stderr(
             predicate::str::contains("─┬─")
                 .and(predicate::str::contains("Target Url"))

tests/test_config.rs

@@ -18,7 +18,7 @@ fn read_in_config_file_for_settings() -> Result<(), Box<dyn std::error::Error>>
         .arg(file.as_os_str())
         .arg("-vvvv")
         .assert()
-        .failure()
+        .success()
         .stderr(predicate::str::contains("│ 37"));
 
     teardown_tmp_directory(tmp_dir);

tests/test_heuristics.rs

@@ -19,11 +19,9 @@ fn test_single_target_cannot_connect() -> Result<(), Box<dyn std::error::Error>>
         .arg("--wordlist")
         .arg(file.as_os_str())
         .assert()
-        .failure()
-        .stderr(
-            predicate::str::contains("Could not connect to any target provided")
-                .and(predicate::str::contains("ERROR"))
-                .and(predicate::str::contains("heuristics::connectivity_test")),
+        .success()
+        .stdout(
+            predicate::str::contains("Could not connect to http://fjdksafjkdsajfkdsajkfdsajkfsdjkdsfdsafdsafdsajkr3l2ajfdskafdsjk, skipping...", )
         );
 
     teardown_tmp_directory(tmp_dir);
@@ -47,11 +45,9 @@ fn test_two_targets_cannot_connect() -> Result<(), Box<dyn std::error::Error>> {
         .pipe_stdin(file)
         .unwrap()
         .assert()
-        .failure()
-        .stderr(
-            predicate::str::contains("Could not connect to any target provided")
-                .and(predicate::str::contains("ERROR"))
-                .and(predicate::str::contains("heuristics::connectivity_test")),
+        .success()
+        .stdout(
+            predicate::str::contains("Could not connect to http://fjdksafjkdsajfkdsajkfdsajkfsdjkdsfdsafdsafdsajkr3l2ajfdskafdsjk, skipping...", )
         );
 
     teardown_tmp_directory(tmp_dir);

tests/test_main.rs

@@ -25,10 +25,8 @@ fn main_use_root_owned_file_as_wordlist() -> Result<(), Box<dyn std::error::Erro
         .arg("/etc/shadow")
         .arg("-vvvv")
         .assert()
-        .success()
-        .stderr(predicate::str::contains(
-            "ERROR main::get_unique_words_from_wordlist Permission denied (os error 13)",
-        ));
+        .failure()
+        .stdout(predicate::str::contains("Permission denied (os error 13)"));
 
     // connectivity test hits it once
     assert_eq!(mock.times_called(), 1);
@@ -57,9 +55,7 @@ fn main_use_empty_wordlist() -> Result<(), Box<dyn std::error::Error>> {
         .arg("-vvvv")
         .assert()
         .failure()
-        .stderr(predicate::str::contains(
-            "ERROR main::scan Did not find any words in",
-        ));
+        .stdout(predicate::str::contains("Did not find any words in"));
 
     assert_eq!(mock.times_called(), 1);
 
@@ -83,11 +79,9 @@ fn main_use_empty_stdin_targets() -> Result<(), Box<dyn std::error::Error>> {
         .pipe_stdin(file)
         .unwrap()
         .assert()
-        .failure()
+        .success()
         .stderr(
             predicate::str::contains("Could not connect to any target provided")
-                .and(predicate::str::contains("ERROR"))
-                .and(predicate::str::contains("heuristics::connectivity_test"))
                 .and(predicate::str::contains("Target Url"))
                 .not(), // no target url found
         );

tests/test_scanner.rs

@@ -411,3 +411,52 @@ fn scanner_single_request_scan_with_filtered_result() -> Result<(), Box<dyn std:
     teardown_tmp_directory(tmp_dir);
     Ok(())
 }
+
+#[test]
+/// send a single valid request, expect a 200 response that then gets routed to the replay
+/// proxy
+fn scanner_single_request_replayed_to_proxy() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let proxy = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/LICENSE")
+        .return_status(200)
+        .return_body("this is a test")
+        .create_on(&srv);
+
+    let mock_two = Mock::new()
+        .expect_method(GET)
+        .expect_path("/LICENSE")
+        .return_status(200)
+        .return_body("this is a test")
+        .create_on(&proxy);
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--replay-proxy")
+        .arg(format!("http://{}", proxy.address().to_string()))
+        .arg("--replay-codes")
+        .arg("200")
+        .unwrap();
+
+    cmd.assert()
+        .success()
+        .stdout(
+            predicate::str::contains("/LICENSE")
+                .and(predicate::str::contains("200"))
+                .and(predicate::str::contains("14")),
+        )
+        .stderr(predicate::str::contains("Replay Proxy Codes"));
+
+    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock_two.times_called(), 1);
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}