added FeroxResponse, old Response channels replaced with FeroxResponse

src/lib.rs

@@ -10,7 +10,9 @@ pub mod reporter;
 pub mod scanner;
 pub mod utils;
 
-use reqwest::StatusCode;
+use crate::config::CONFIGURATION;
+
+use reqwest::{Url, StatusCode, Response};
 use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender};
 
 /// Generic Result type to ease error handling in async contexts
@@ -59,6 +61,66 @@ pub const DEFAULT_STATUS_CODES: [StatusCode; 9] = [
 /// Expected location is in the same directory as the feroxbuster binary.
 pub const DEFAULT_CONFIG_NAME: &str = "ferox-config.toml";
 
+/// A `FeroxResponse`, derived from a `Response` to a submitted `Request`
+#[derive(Debug)]
+pub struct FeroxResponse {
+    /// todo doc
+    pub url: Url,
+
+    /// todo doc
+    pub status: StatusCode,
+
+    /// todo doc
+    pub text: String,
+
+    /// todo doc
+    pub content_length: u64
+}
+
+/// todo doc
+impl FeroxResponse {
+    /// Get the `StatusCode` of this `FeroxResponse`
+    pub fn status(&self) -> &StatusCode {
+        &self.status
+    }
+
+    /// Get the final `Url` of this `FeroxResponse`.
+    pub fn url(&self) -> &Url {
+        &self.url
+    }
+
+    /// Get the full response text
+    pub fn text(&self) -> &str {
+        &self.text
+    }
+
+    /// Get the content-length of this response, if known
+    pub fn content_length(&self) -> u64 {
+        self.content_length
+    }
+
+    /// todo doc
+    pub async fn new(response: Response) -> Self {
+        let url = response.url().clone();
+        let status = response.status().clone();
+        let content_length = response.content_length().unwrap_or(0);
+
+        let text = if CONFIGURATION.extract_links {
+            // .text() consumes the response, must be called last
+            response.text().await.unwrap()
+        } else {
+            String::new()
+        };
+
+        FeroxResponse {
+            url,
+            status,
+            content_length,
+            text
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

src/main.rs

@@ -1,9 +1,8 @@
 use feroxbuster::config::{CONFIGURATION, PROGRESS_PRINTER};
 use feroxbuster::scanner::scan_url;
 use feroxbuster::utils::{ferox_print, get_current_depth, module_colorizer, status_colorizer};
-use feroxbuster::{banner, heuristics, logger, reporter, FeroxResult};
+use feroxbuster::{banner, heuristics, logger, reporter, FeroxResult, FeroxResponse};
 use futures::StreamExt;
-use reqwest::Response;
 use std::collections::HashSet;
 use std::fs::File;
 use std::io::{BufRead, BufReader};
@@ -58,7 +57,7 @@ fn get_unique_words_from_wordlist(path: &str) -> FeroxResult<Arc<HashSet<String>
 /// Determine whether it's a single url scan or urls are coming from stdin, then scan as needed
 async fn scan(
     targets: Vec<String>,
-    tx_term: UnboundedSender<Response>,
+    tx_term: UnboundedSender<FeroxResponse>,
     tx_file: UnboundedSender<String>,
 ) -> FeroxResult<()> {
     log::trace!("enter: scan({:?}, {:?}, {:?})", targets, tx_term, tx_file);

src/reporter.rs

@@ -1,8 +1,7 @@
 use crate::config::{CONFIGURATION, PROGRESS_PRINTER};
 use crate::utils::{ferox_print, status_colorizer};
-use crate::FeroxChannel;
+use crate::{FeroxResponse, FeroxChannel};
 use console::strip_ansi_codes;
-use reqwest::Response;
 use std::io::Write;
 use std::sync::{Arc, Once, RwLock};
 use std::{fs, io};
@@ -41,14 +40,14 @@ pub fn initialize(
     output_file: &str,
     save_output: bool,
 ) -> (
-    UnboundedSender<Response>,
+    UnboundedSender<FeroxResponse>,
     UnboundedSender<String>,
     JoinHandle<()>,
     Option<JoinHandle<()>>,
 ) {
     log::trace!("enter: initialize({}, {})", output_file, save_output);
 
-    let (tx_rpt, rx_rpt): FeroxChannel<Response> = mpsc::unbounded_channel();
+    let (tx_rpt, rx_rpt): FeroxChannel<FeroxResponse> = mpsc::unbounded_channel();
     let (tx_file, rx_file): FeroxChannel<String> = mpsc::unbounded_channel();
 
     let file_clone = tx_file.clone();
@@ -81,7 +80,7 @@ pub fn initialize(
 /// The consumer simply receives responses and prints them if they meet the given
 /// reporting criteria
 async fn spawn_terminal_reporter(
-    mut resp_chan: UnboundedReceiver<Response>,
+    mut resp_chan: UnboundedReceiver<FeroxResponse>,
     file_chan: UnboundedSender<String>,
     save_output: bool,
 ) {
@@ -107,7 +106,7 @@ async fn spawn_terminal_reporter(
                     // 200       3280 https://localhost.com/FAQ
                     "{} {:>10} {}\n",
                     status,
-                    resp.content_length().unwrap_or(0),
+                    resp.content_length(),
                     resp.url()
                 )
             };

src/scanner.rs

@@ -1,7 +1,7 @@
 use crate::config::{CONFIGURATION, PROGRESS_BAR};
 use crate::heuristics::WildcardFilter;
 use crate::utils::{format_url, get_current_depth, get_url_path_length, make_request};
-use crate::{heuristics, progress, FeroxChannel};
+use crate::{heuristics, progress, FeroxChannel, FeroxResponse};
 use futures::future::{BoxFuture, FutureExt};
 use futures::{stream, StreamExt};
 use lazy_static::lazy_static;
@@ -66,7 +66,7 @@ fn spawn_recursion_handler(
     mut recursion_channel: UnboundedReceiver<String>,
     wordlist: Arc<HashSet<String>>,
     base_depth: usize,
-    tx_term: UnboundedSender<Response>,
+    tx_term: UnboundedSender<FeroxResponse>,
     tx_file: UnboundedSender<String>,
 ) -> BoxFuture<'static, Vec<JoinHandle<()>>> {
     log::trace!(
@@ -301,7 +301,7 @@ async fn make_requests(
     base_depth: usize,
     filter: Arc<WildcardFilter>,
     dir_chan: UnboundedSender<String>,
-    report_chan: UnboundedSender<Response>,
+    report_chan: UnboundedSender<FeroxResponse>,
 ) {
     log::trace!(
         "enter: make_requests({}, {}, {}, {:?}, {:?})",
@@ -357,8 +357,10 @@ async fn make_requests(
                 }
             }
 
+            let ferox_response = FeroxResponse::new(response).await;
+
             // everything else should be reported
-            match report_chan.send(response) {
+            match report_chan.send(ferox_response) {
                 Ok(_) => {
                     log::debug!("sent {}/{} over reporting channel", &target_url, &word);
                 }
@@ -378,7 +380,7 @@ pub async fn scan_url(
     target_url: &str,
     wordlist: Arc<HashSet<String>>,
     base_depth: usize,
-    tx_term: UnboundedSender<Response>,
+    tx_term: UnboundedSender<FeroxResponse>,
     tx_file: UnboundedSender<String>,
 ) {
     log::trace!(