diff --git a/ferox-config.toml.example b/ferox-config.toml.example index b9f53a2..6bac2d2 100644 --- a/ferox-config.toml.example +++ b/ferox-config.toml.example @@ -19,6 +19,7 @@ # scan_limit = 6 # quiet = true # output = "/targets/ellingson_mineral_company/gibson.txt" +# debug_log = "/var/log/find-the-derp.log" # user_agent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0" # redirects = true # insecure = true diff --git a/src/config.rs b/src/config.rs index 9efc219..d800da4 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,11 +1,11 @@ use crate::utils::{module_colorizer, status_colorizer}; use crate::{client, parser, progress}; -use crate::{DEFAULT_CONFIG_NAME, DEFAULT_STATUS_CODES, DEFAULT_WORDLIST, VERSION}; +use crate::{FeroxSerialize, DEFAULT_CONFIG_NAME, DEFAULT_STATUS_CODES, DEFAULT_WORDLIST, VERSION}; use clap::value_t; use indicatif::{MultiProgress, ProgressBar, ProgressDrawTarget}; use lazy_static::lazy_static; use reqwest::{Client, StatusCode}; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::env::{current_dir, current_exe}; use std::fs::read_to_string; @@ -49,8 +49,12 @@ fn report_and_exit(err: &str) -> ! { /// In that order. /// /// Inspired by and derived from https://github.com/PhilipDaniels/rust-config-example -#[derive(Debug, Clone, Deserialize)] +#[derive(Debug, Clone, Deserialize, Serialize)] pub struct Configuration { + #[serde(rename = "type", default = "serialized_type")] + /// Name of this type of struct, used for serialization, i.e. `{"type":"configuration"}` + kind: String, + /// Path to the wordlist #[serde(default = "wordlist")] pub wordlist: String, @@ -107,10 +111,19 @@ pub struct Configuration { #[serde(default)] pub quiet: bool, + /// Store log output as NDJSON + #[serde(default)] + pub json: bool, + /// Output file to write results to (default: stdout) #[serde(default)] pub output: String, + /// File in which to store debug output, used in conjunction with verbosity to dictate which + /// logs are written + #[serde(default)] + pub debug_log: String, + /// Sets the User-Agent (default: feroxbuster/VERSION) #[serde(default = "user_agent")] pub user_agent: String, @@ -180,6 +193,11 @@ pub struct Configuration { // defaults in the event that a ferox-config.toml is found but one or more of the values below // aren't listed in the config. This way, we get the correct defaults upon Deserialization +/// default Configuration type for use in json output +fn serialized_type() -> String { + String::from("configuration") +} + /// default timeout value fn timeout() -> u64 { 7 @@ -222,8 +240,10 @@ impl Default for Configuration { let replay_client = None; let status_codes = status_codes(); let replay_codes = status_codes.clone(); + let kind = serialized_type(); Configuration { + kind, client, timeout, user_agent, @@ -233,6 +253,7 @@ impl Default for Configuration { dont_filter: false, quiet: false, stdin: false, + json: false, verbosity: 0, scan_limit: 0, add_slash: false, @@ -243,6 +264,7 @@ impl Default for Configuration { proxy: String::new(), config: String::new(), output: String::new(), + debug_log: String::new(), target_url: String::new(), replay_proxy: String::new(), queries: Vec::new(), @@ -275,8 +297,9 @@ impl Configuration { /// - **status_codes**: [`DEFAULT_RESPONSE_CODES`](constant.DEFAULT_RESPONSE_CODES.html) /// - **filter_status**: `None` /// - **output**: `None` (print to stdout) + /// - **debug_log**: `None` /// - **quiet**: `false` - /// - **user_agent**: `feroxer/VERSION` + /// - **user_agent**: `feroxbuster/VERSION` /// - **insecure**: `false` (don't be insecure, i.e. don't allow invalid certs) /// - **extensions**: `None` /// - **filter_size**: `None` @@ -287,6 +310,7 @@ impl Configuration { /// - **no_recursion**: `false` (recursively scan enumerated sub-directories) /// - **add_slash**: `false` /// - **stdin**: `false` + /// - **json**: `false` /// - **dont_filter**: `false` (auto filter wildcard responses) /// - **depth**: `4` (maximum recursion depth) /// - **scan_limit**: `0` (no limit on concurrent scans imposed) @@ -385,6 +409,7 @@ impl Configuration { update_config_if_present!(&mut config.scan_limit, args, "scan_limit", usize); update_config_if_present!(&mut config.wordlist, args, "wordlist", String); update_config_if_present!(&mut config.output, args, "output", String); + update_config_if_present!(&mut config.debug_log, args, "debug_log", String); if let Some(arg) = args.values_of("status_codes") { config.status_codes = arg @@ -481,6 +506,10 @@ impl Configuration { config.extract_links = true; } + if args.is_present("json") { + config.json = true; + } + if args.is_present("stdin") { config.stdin = true; } else { @@ -625,6 +654,8 @@ impl Configuration { settings.scan_limit = settings_to_merge.scan_limit; settings.replay_proxy = settings_to_merge.replay_proxy; settings.replay_codes = settings_to_merge.replay_codes; + settings.debug_log = settings_to_merge.debug_log; + settings.json = settings_to_merge.json; } /// If present, read in `DEFAULT_CONFIG_NAME` and deserialize the specified values @@ -650,6 +681,47 @@ impl Configuration { } } +/// Implementation of FeroxMessage +impl FeroxSerialize for Configuration { + /// Simple wrapper around create_report_string + fn as_str(&self) -> String { + format!("{:#?}\n", *self) + } + + /// Create an NDJSON representation of the current scan's Configuration + /// + /// (expanded for clarity) + /// ex: + /// { + /// "type":"configuration", + /// "wordlist":"test", + /// "config":"/home/epi/.config/feroxbuster/ferox-config.toml", + /// "proxy":"", + /// "replay_proxy":"", + /// "target_url":"https://localhost.com", + /// "status_codes":[ + /// 200, + /// 204, + /// 301, + /// 302, + /// 307, + /// 308, + /// 401, + /// 403, + /// 405 + /// ], + /// ... + /// }\n + fn as_json(&self) -> String { + if let Ok(mut json) = serde_json::to_string(&self) { + json.push('\n'); + json + } else { + String::from("{\"error\":\"could not Configuration convert to json\"}") + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -670,6 +742,7 @@ mod tests { verbosity = 1 scan_limit = 6 output = "/some/otherpath" + debug_log = "/yet/anotherpath" redirects = true insecure = true extensions = ["html", "php", "js"] @@ -680,6 +753,7 @@ mod tests { stdin = true dont_filter = true extract_links = true + json = true depth = 1 filter_size = [4120] filter_word_count = [994, 992] @@ -699,6 +773,7 @@ mod tests { assert_eq!(config.wordlist, wordlist()); assert_eq!(config.proxy, String::new()); assert_eq!(config.target_url, String::new()); + assert_eq!(config.debug_log, String::new()); assert_eq!(config.config, String::new()); assert_eq!(config.replay_proxy, String::new()); assert_eq!(config.status_codes, status_codes()); @@ -712,6 +787,7 @@ mod tests { assert_eq!(config.quiet, false); assert_eq!(config.dont_filter, false); assert_eq!(config.no_recursion, false); + assert_eq!(config.json, false); assert_eq!(config.stdin, false); assert_eq!(config.add_slash, false); assert_eq!(config.redirects, false); @@ -733,6 +809,13 @@ mod tests { assert_eq!(config.wordlist, "/some/path"); } + #[test] + /// parse the test config and see that the value parsed is correct + fn config_reads_debug_log() { + let config = setup_config_test(); + assert_eq!(config.debug_log, "/yet/anotherpath"); + } + #[test] /// parse the test config and see that the value parsed is correct fn config_reads_status_codes() { @@ -796,6 +879,13 @@ mod tests { assert_eq!(config.quiet, true); } + #[test] + /// parse the test config and see that the value parsed is correct + fn config_reads_json() { + let config = setup_config_test(); + assert_eq!(config.json, true); + } + #[test] /// parse the test config and see that the value parsed is correct fn config_reads_verbosity() { diff --git a/src/heuristics.rs b/src/heuristics.rs index 7b53ab2..7ffef98 100644 --- a/src/heuristics.rs +++ b/src/heuristics.rs @@ -2,10 +2,7 @@ use crate::{ config::{CONFIGURATION, PROGRESS_PRINTER}, filters::WildcardFilter, scanner::should_filter_response, - utils::{ - ferox_print, format_url, get_url_path_length, make_request, module_colorizer, - status_colorizer, - }, + utils::{ferox_print, format_url, get_url_path_length, make_request, status_colorizer}, FeroxResponse, }; use console::style; @@ -42,7 +39,7 @@ fn unique_string(length: usize) -> String { pub async fn wildcard_test( target_url: &str, bar: ProgressBar, - tx_file: UnboundedSender, + tx_file: UnboundedSender, ) -> Option { log::trace!( "enter: wildcard_test({:?}, {:?}, {:?})", @@ -89,46 +86,45 @@ pub async fn wildcard_test( if !CONFIGURATION.quiet { let msg = format!( - "{} {:>8}l {:>8}w {:>8}c Wildcard response is dynamic; {} ({} + url length) responses; toggle this behavior by using {}\n", + "{:<33} Wildcard response is dynamic; {} ({} + url length) responses; toggle this behavior by using {}\n", status_colorizer("WLD"), - ferox_response.line_count(), - ferox_response.word_count(), - wildcard.dynamic, style("auto-filtering").yellow(), style(wc_length - url_len).cyan(), style("--dont-filter").yellow() - ); + ); ferox_print(&msg, &PROGRESS_PRINTER); - try_send_message_to_file( - &msg, - tx_file.clone(), - !CONFIGURATION.output.is_empty(), - ); + // tx_file.send(resp_two); + + // try_send_message_to_file( + // &msg, + // tx_file.clone(), + // !CONFIGURATION.output.is_empty(), + // ); } } else if wc_length == wc2_length { wildcard.size = wc_length; if !CONFIGURATION.quiet { let msg = format!( - "{} {:>8}l {:>8}w {:>8}c Wildcard response is static; {} {} responses; toggle this behavior by using {}\n", + "{} {:>9} {:>9} {:>9} Wildcard response is static; {} {} responses; toggle this behavior by using {}\n", status_colorizer("WLD"), - ferox_response.line_count(), - ferox_response.word_count(), - wc_length, + "-", + "-", + "-", style("auto-filtering").yellow(), style(wc_length).cyan(), style("--dont-filter").yellow() ); + // tx_file.send(resp_two); ferox_print(&msg, &PROGRESS_PRINTER); - - try_send_message_to_file( - &msg, - tx_file.clone(), - !CONFIGURATION.output.is_empty(), - ); + // try_send_message_to_file( + // &msg, + // tx_file.clone(), + // !CONFIGURATION.output.is_empty(), + // ); } } } else { @@ -152,7 +148,7 @@ pub async fn wildcard_test( async fn make_wildcard_request( target_url: &str, length: usize, - tx_file: UnboundedSender, + tx_file: UnboundedSender, ) -> Option { log::trace!( "enter: make_wildcard_request({}, {}, {:?})", @@ -178,8 +174,6 @@ async fn make_wildcard_request( } }; - let wildcard = status_colorizer("WLD"); - match make_request(&CONFIGURATION.client, &nonexistent.to_owned()).await { Ok(response) => { if CONFIGURATION @@ -187,58 +181,16 @@ async fn make_wildcard_request( .contains(&response.status().as_u16()) { // found a wildcard response - let ferox_response = FeroxResponse::from(response, true).await; - let url_len = get_url_path_length(&ferox_response.url()); - let content_len = ferox_response.content_length(); - let content_words = ferox_response.word_count(); - let content_lines = ferox_response.line_count(); + let mut ferox_response = FeroxResponse::from(response, true).await; + ferox_response.wildcard = true; if !CONFIGURATION.quiet && !should_filter_response(&ferox_response) { - let msg = format!( - "{} {:>8}l {:>8}w {:>8}c Got {} for {} (url length: {})\n", - wildcard, - content_lines, - content_words, - content_len, - status_colorizer(&ferox_response.status().as_str()), - ferox_response.url(), - url_len - ); - - ferox_print(&msg, &PROGRESS_PRINTER); - - try_send_message_to_file( - &msg, - tx_file.clone(), - !CONFIGURATION.output.is_empty(), - ); + // todo unwrap + if tx_file.send(ferox_response.clone()).is_err() { + return None; + }; } - if ferox_response.status().is_redirection() { - // show where it goes, if possible - if let Some(next_loc) = ferox_response.headers().get("Location") { - let next_loc_str = next_loc.to_str().unwrap_or("Unknown"); - if !CONFIGURATION.quiet && !should_filter_response(&ferox_response) { - let msg = format!( - "{} {:>8}l {:>8}w {:>8}c {} redirects to => {}\n", - wildcard, - content_lines, - content_words, - content_len, - ferox_response.url(), - next_loc_str - ); - - ferox_print(&msg, &PROGRESS_PRINTER); - - try_send_message_to_file( - &msg, - tx_file.clone(), - !CONFIGURATION.output.is_empty(), - ); - } - } - } log::trace!("exit: make_wildcard_request -> {}", ferox_response); return Some(ferox_response); } @@ -303,35 +255,9 @@ pub async fn connectivity_test(target_urls: &[String]) -> Vec { good_urls } -/// simple helper to keep DRY; sends a message using the transmitter side of the given mpsc channel -/// the receiver is expected to be the side that saves the message to CONFIGURATION.output. -fn try_send_message_to_file(msg: &str, tx_file: UnboundedSender, save_output: bool) { - log::trace!("enter: try_send_message_to_file({}, {:?})", msg, tx_file); - - if save_output { - match tx_file.send(msg.to_string()) { - Ok(_) => { - log::trace!( - "sent message from heuristics::try_send_message_to_file to file handler" - ); - } - Err(e) => { - log::error!( - "{} {}", - module_colorizer("heuristics::try_send_message_to_file"), - e - ); - } - } - } - log::trace!("exit: try_send_message_to_file"); -} - #[cfg(test)] mod tests { use super::*; - use crate::FeroxChannel; - use tokio::sync::mpsc; #[test] /// request a unique string of 32bytes * a value returns correct result @@ -348,41 +274,4 @@ mod tests { assert_eq!(wcf.size, 0); assert_eq!(wcf.dynamic, 0); } - - #[tokio::test(core_threads = 1)] - /// tests that given a message and transmitter, the function sends the message across the - /// channel - async fn heuristics_try_send_message_to_file_sends_when_true() { - let (tx, mut rx): FeroxChannel = mpsc::unbounded_channel(); - let msg = "It really tied the room together."; - let should_save = true; - try_send_message_to_file(&msg, tx, should_save); - - assert_eq!(rx.recv().await.unwrap(), msg); - } - - #[tokio::test(core_threads = 1)] - #[should_panic] - /// tests that when save_output is false, nothing is sent to the receiver - async fn heuristics_try_send_message_to_file_sends_when_false() { - let (tx, mut rx): FeroxChannel = mpsc::unbounded_channel(); - let msg = "I'm the Dude, so that's what you call me."; - let should_save = false; - try_send_message_to_file(&msg, tx, should_save); - - assert_ne!(rx.recv().await.unwrap(), msg); - } - - #[tokio::test(core_threads = 1)] - /// tests that when save_output is true, but the receiver is closed, nothing is sent to the receiver - /// this test doesn't assert anything, but reaches the error block of the given function and - /// can be verified with --nocapture and RUST_LOG being set - async fn heuristics_try_send_message_to_file_sends_with_closed_receiver() { - env_logger::init(); - let (tx, mut rx): FeroxChannel = mpsc::unbounded_channel(); - let msg = "Hey, nice marmot."; - let should_save = true; - rx.close(); - try_send_message_to_file(&msg, tx, should_save); - } } diff --git a/src/lib.rs b/src/lib.rs index 2bc7404..74f917a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -12,7 +12,11 @@ pub mod scan_manager; pub mod scanner; pub mod utils; +use crate::utils::{get_url_path_length, status_colorizer}; +use console::{style, Color}; use reqwest::{header::HeaderMap, Response, StatusCode, Url}; +use serde::{ser::SerializeStruct, Serialize, Serializer}; +use std::collections::HashMap; use std::{error, fmt}; use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; @@ -82,6 +86,30 @@ pub const DEFAULT_STATUS_CODES: [StatusCode; 9] = [ /// Expected location is in the same directory as the feroxbuster binary. pub const DEFAULT_CONFIG_NAME: &str = "ferox-config.toml"; +/// FeroxSerialize trait; represents different types that are Serialize and also implement +/// as_str / as_json methods +pub trait FeroxSerialize: Serialize { + /// Return a String representation of the object, generally the human readable version of the + /// implementor + fn as_str(&self) -> String; + + /// Return an NDJSON representation of the object + fn as_json(&self) -> String; +} + +// todo remove if unused +/// Simple enum for determining wildcard response type +pub enum WildcardType { + /// Wildcard response that is always the same size + Static, + + /// Wildcard response that changes based on input from the client (i.e. url changes) + Dynamic, + + /// Not a wildcard response + None, +} + /// A `FeroxResponse`, derived from a `Response` to a submitted `Request` #[derive(Debug, Clone)] pub struct FeroxResponse { @@ -99,6 +127,9 @@ pub struct FeroxResponse { /// The `Headers` of this `FeroxResponse` headers: HeaderMap, + + /// Wildcard response status + wildcard: bool, } /// Implement Display for FeroxResponse @@ -216,10 +247,206 @@ impl FeroxResponse { content_length, text, headers, + wildcard: false, } } } +/// Implement FeroxSerialize for FeroxResponse +impl FeroxSerialize for FeroxResponse { + /// Simple wrapper around create_report_string + fn as_str(&self) -> String { + let lines = self.line_count().to_string(); + let words = self.word_count().to_string(); + let chars = self.content_length().to_string(); + let status = self.status().as_str(); + let wild_status = status_colorizer("WLD"); + + if self.wildcard { + // response is a wildcard, special messages abound when this is the case... + + // create the base message + let mut message = format!( + "{} {:>8}l {:>8}w {:>8}c Got {} for {} (url length: {})\n", + wild_status, + lines, + words, + chars, + status_colorizer(&status), + self.url(), + get_url_path_length(&self.url()) + ); + + if self.status().is_redirection() { + // when it's a redirect, show where it goes, if possible + if let Some(next_loc) = self.headers().get("Location") { + let next_loc_str = next_loc.to_str().unwrap_or("Unknown"); + + let redirect_msg = format!( + "{} {:>9} {:>9} {:>9} {} redirects to => {}\n", + wild_status, + "-", + "-", + "-", + self.url(), + next_loc_str + ); + + message.push_str(&redirect_msg); + } + } + + // base message + redirection message (if appropriate) + message + } else { + // not a wildcard, just create a normal entry + utils::create_report_string( + self.status.as_str(), + &lines, + &words, + &chars, + self.url().as_str(), + ) + } + } + + /// Create an NDJSON representation of the FeroxResponse + /// + /// (expanded for clarity) + /// ex: + /// { + /// "type":"response", + /// "url":"https://localhost.com/images", + /// "path":"/images", + /// "status":301, + /// "content_length":179, + /// "line_count":10, + /// "word_count":16, + /// "headers":{ + /// "x-content-type-options":"nosniff", + /// "strict-transport-security":"max-age=31536000; includeSubDomains", + /// "x-frame-options":"SAMEORIGIN", + /// "connection":"keep-alive", + /// "server":"nginx/1.16.1", + /// "content-type":"text/html; charset=UTF-8", + /// "referrer-policy":"origin-when-cross-origin", + /// "content-security-policy":"default-src 'none'", + /// "access-control-allow-headers":"X-Requested-With", + /// "x-xss-protection":"1; mode=block", + /// "content-length":"179", + /// "date":"Mon, 23 Nov 2020 15:33:24 GMT", + /// "location":"/images/", + /// "access-control-allow-origin":"https://localhost.com" + /// } + /// }\n + fn as_json(&self) -> String { + if let Ok(mut json) = serde_json::to_string(&self) { + json.push('\n'); + json + } else { + format!("{{\"error\":\"could not convert {} to json\"}}", self.url()) + } + } +} + +/// Serialize implementation for FeroxResponse +impl Serialize for FeroxResponse { + /// Function that handles serialization of a FeroxResponse to NDJSON + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut headers = HashMap::new(); + let mut state = serializer.serialize_struct("FeroxResponse", 7)?; + + // need to convert the HeaderMap to a HashMap in order to pass it to the serializer + for (key, value) in &self.headers { + let k = key.as_str().to_owned(); + let v = String::from_utf8_lossy(value.as_bytes()); + headers.insert(k, v); + } + + state.serialize_field("type", "response")?; + state.serialize_field("url", self.url.as_str())?; + state.serialize_field("path", self.url.path())?; + state.serialize_field("wildcard", &self.wildcard)?; + state.serialize_field("status", &self.status.as_u16())?; + state.serialize_field("content_length", &self.content_length)?; + state.serialize_field("line_count", &self.line_count())?; + state.serialize_field("word_count", &self.word_count())?; + state.serialize_field("headers", &headers)?; + + state.end() + } +} + +#[derive(Serialize, Default)] +/// Representation of a log entry, can be represented as a human readable string or JSON +pub struct FeroxMessage { + // todo probably move to lib + #[serde(rename = "type")] + /// Name of this type of struct, used for serialization, i.e. `{"type":"log"}` + kind: String, + + /// The log message + pub message: String, + + /// The log level + pub level: String, + + /// The number of seconds elapsed since the scan started + pub time_offset: f32, + + /// The module from which log::* was called + pub module: String, +} + +/// Implementation of FeroxMessage +impl FeroxSerialize for FeroxMessage { + /// Create an NDJSON representation of the log message + /// + /// (expanded for clarity) + /// ex: + /// { + /// "type": "log", + /// "message": "Sent https://localhost/api to file handler", + /// "level": "DEBUG", + /// "time_offset": 0.86333454, + /// "module": "feroxbuster::reporter" + /// }\n + fn as_json(&self) -> String { + if let Ok(mut json) = serde_json::to_string(&self) { + json.push('\n'); + json + } else { + String::from("{\"error\":\"could not convert to json\"}") + } + } + + /// Create a string representation of the log message + /// + /// ex: 301 10l 16w 173c https://localhost/api + fn as_str(&self) -> String { + let (level_name, level_color) = match self.level.as_str() { + "ERROR" => ("ERR", Color::Red), + "WARN" => ("WRN", Color::Red), + "INFO" => ("INF", Color::Cyan), + "DEBUG" => ("DBG", Color::Yellow), + "TRACE" => ("TRC", Color::Magenta), + "WILDCARD" => ("WLD", Color::Cyan), + _ => ("UNK", Color::White), + }; + + format!( + "{} {:10.03} {} {}\n", + style(level_name).bg(level_color).black(), + style(self.time_offset).dim(), + self.module, + style(&self.message).dim(), + ) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/logger.rs b/src/logger.rs index 3c681a3..2356b41 100644 --- a/src/logger.rs +++ b/src/logger.rs @@ -1,6 +1,9 @@ -use crate::config::{CONFIGURATION, PROGRESS_PRINTER}; -use crate::reporter::{get_cached_file_handle, safe_file_write}; -use console::{style, Color}; +use crate::{ + config::{CONFIGURATION, PROGRESS_PRINTER}, + reporter::safe_file_write, + utils::open_file, + FeroxMessage, FeroxSerialize, +}; use env_logger::Builder; use std::env; use std::time::Instant; @@ -28,44 +31,27 @@ pub fn initialize(verbosity: u8) { let start = Instant::now(); let mut builder = Builder::from_default_env(); - // I REALLY wanted the logger to also use the reporting channels found in the `reporter` - // module. However, in order to properly clean up the channels, all references to the - // transmitter side of a channel need to go out of scope, then you can await the future into - // which the receiver was moved. - // - // The problem was that putting a transmitter reference in this closure, which gets initialized - // as part of the global logger, made it so that I couldn't destroy/leak/take/swap the last - // reference to allow the channels to gracefully close. - // - // The workaround was to have a RwLock around the file and allow both the logger and the - // file handler to both write independent of each other. - let locked_file = get_cached_file_handle(&CONFIGURATION.output); + let debug_file = open_file(&CONFIGURATION.debug_log); + + if let Some(buffered_file) = debug_file.clone() { + // write out the configuration to the debug file if it exists + safe_file_write(&*CONFIGURATION, buffered_file, CONFIGURATION.json); + } builder .format(move |_, record| { - let t = start.elapsed().as_secs_f32(); - let level = record.level(); - - let (level_name, level_color) = match level { - log::Level::Error => ("ERR", Color::Red), - log::Level::Warn => ("WRN", Color::Red), - log::Level::Info => ("INF", Color::Cyan), - log::Level::Debug => ("DBG", Color::Yellow), - log::Level::Trace => ("TRC", Color::Magenta), + let log_entry = FeroxMessage { + message: record.args().to_string(), + level: record.level().to_string(), + time_offset: start.elapsed().as_secs_f32(), + module: record.target().to_string(), + kind: "log".to_string(), }; - let msg = format!( - "{} {:10.03} {} {}\n", - style(level_name).bg(level_color).black(), - style(t).dim(), - record.target(), - style(record.args()).dim(), - ); + PROGRESS_PRINTER.println(&log_entry.as_str()); - PROGRESS_PRINTER.println(&msg); - - if let Some(buffered_file) = locked_file.clone() { - safe_file_write(&msg, buffered_file); + if let Some(buffered_file) = debug_file.clone() { + safe_file_write(&log_entry, buffered_file, CONFIGURATION.json); } Ok(()) diff --git a/src/main.rs b/src/main.rs index baf1711..88f80af 100644 --- a/src/main.rs +++ b/src/main.rs @@ -97,7 +97,7 @@ fn get_unique_words_from_wordlist(path: &str) -> FeroxResult async fn scan( targets: Vec, tx_term: UnboundedSender, - tx_file: UnboundedSender, + tx_file: UnboundedSender, ) -> FeroxResult<()> { log::trace!("enter: scan({:?}, {:?}, {:?})", targets, tx_term, tx_file); // cloning an Arc is cheap (it's basically a pointer into the heap) @@ -187,7 +187,6 @@ async fn wrapped_main() { // can't trace main until after logger is initialized and the above task is started log::trace!("enter: main"); - log::debug!("{:#?}", *CONFIGURATION); // spawn a thread that listens for keyboard input on stdin, when a user presses enter // the input handler will toggle PAUSE_SCAN, which in turn is used to pause and resume @@ -249,7 +248,7 @@ async fn wrapped_main() { async fn clean_up( tx_term: UnboundedSender, term_handle: JoinHandle<()>, - tx_file: UnboundedSender, + tx_file: UnboundedSender, file_handle: Option>, save_output: bool, ) { diff --git a/src/parser.rs b/src/parser.rs index ed50e7b..0edf776 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -109,6 +109,12 @@ pub fn initialize() -> App<'static, 'static> { .takes_value(false) .help("Only print URLs; Don't print status codes, response size, running config, etc...") ) + .arg( + Arg::with_name("json") + .long("json") + .takes_value(false) + .help("Emit JSON logs to --output and --debug-log instead of normal text") + ) .arg( Arg::with_name("dont_filter") .short("D") @@ -121,7 +127,14 @@ pub fn initialize() -> App<'static, 'static> { .short("o") .long("output") .value_name("FILE") - .help("Output file to write results to (default: stdout)") + .help("Output file to write results to (default: stdout; use w/ --json for JSON entries)") + .takes_value(true), + ) + .arg( + Arg::with_name("debug_log") + .long("debug-log") + .value_name("FILE") + .help("Output file to write log entries (use w/ --json for JSON entries)") .takes_value(true), ) .arg( diff --git a/src/reporter.rs b/src/reporter.rs index 09d9407..2bf8b6c 100644 --- a/src/reporter.rs +++ b/src/reporter.rs @@ -1,6 +1,8 @@ -use crate::config::{CONFIGURATION, PROGRESS_PRINTER}; -use crate::utils::{create_report_string, ferox_print, make_request}; -use crate::{FeroxChannel, FeroxResponse}; +use crate::{ + config::{CONFIGURATION, PROGRESS_PRINTER}, + utils::{ferox_print, make_request, open_file}, + FeroxChannel, FeroxResponse, FeroxSerialize, +}; use console::strip_ansi_codes; use std::io::Write; use std::sync::{Arc, Once, RwLock}; @@ -41,14 +43,14 @@ pub fn initialize( save_output: bool, ) -> ( UnboundedSender, - UnboundedSender, + UnboundedSender, JoinHandle<()>, Option>, ) { log::trace!("enter: initialize({}, {})", output_file, save_output); let (tx_rpt, rx_rpt): FeroxChannel = mpsc::unbounded_channel(); - let (tx_file, rx_file): FeroxChannel = mpsc::unbounded_channel(); + let (tx_file, rx_file): FeroxChannel = mpsc::unbounded_channel(); let file_clone = tx_file.clone(); @@ -81,7 +83,7 @@ pub fn initialize( /// reporting criteria async fn spawn_terminal_reporter( mut resp_chan: UnboundedReceiver, - file_chan: UnboundedSender, + file_chan: UnboundedSender, save_output: bool, ) { log::trace!( @@ -95,20 +97,12 @@ async fn spawn_terminal_reporter( log::trace!("received {} on reporting channel", resp.url()); if CONFIGURATION.status_codes.contains(&resp.status().as_u16()) { - let report = create_report_string( - resp.status().as_str(), - &resp.line_count().to_string(), - &resp.word_count().to_string(), - &resp.content_length().to_string(), - &resp.url().to_string(), - ); - // print to stdout - ferox_print(&report, &PROGRESS_PRINTER); + ferox_print(&resp.as_str(), &PROGRESS_PRINTER); if save_output { // -o used, need to send the report to be written out to disk - match file_chan.send(report.to_string()) { + match file_chan.send(resp.clone()) { Ok(_) => { log::debug!("Sent {} to file handler", resp.url()); } @@ -140,7 +134,10 @@ async fn spawn_terminal_reporter( /// /// The consumer simply receives responses and writes them to the given output file if they meet /// the given reporting criteria -async fn spawn_file_reporter(mut report_channel: UnboundedReceiver, output_file: &str) { +async fn spawn_file_reporter( + mut report_channel: UnboundedReceiver, + output_file: &str, +) { let buffered_file = match get_cached_file_handle(&CONFIGURATION.output) { Some(file) => file, None => { @@ -157,47 +154,33 @@ async fn spawn_file_reporter(mut report_channel: UnboundedReceiver, outp log::info!("Writing scan results to {}", output_file); - while let Some(report) = report_channel.recv().await { - safe_file_write(&report, buffered_file.clone()); + while let Some(response) = report_channel.recv().await { + safe_file_write(&response, buffered_file.clone(), CONFIGURATION.json); } log::trace!("exit: spawn_file_reporter"); } -/// Given the path to a file, open the file in append mode (create it if it doesn't exist) and -/// return a reference to the file that is buffered and locked -fn open_file(filename: &str) -> Option>>> { - log::trace!("enter: open_file({})", filename); - - match fs::OpenOptions::new() // std fs - .create(true) - .append(true) - .open(filename) - { - Ok(file) => { - let writer = io::BufWriter::new(file); // std io - - let locked_file = Some(Arc::new(RwLock::new(writer))); - - log::trace!("exit: open_file -> {:?}", locked_file); - locked_file - } - Err(e) => { - log::error!("{}", e); - log::trace!("exit: open_file -> None"); - None - } - } -} - /// Given a string and a reference to a locked buffered file, write the contents and flush /// the buffer to disk. -pub fn safe_file_write(contents: &str, locked_file: Arc>>) { +pub fn safe_file_write( + value: &T, + locked_file: Arc>>, + convert_to_json: bool, +) where + T: FeroxSerialize, +{ // note to future self: adding logging of anything other than error to this function // is a bad idea. we call this function while processing records generated by the logger. // If we then call log::... while already processing some logging output, it results in // the second log entry being injected into the first. + let contents = if convert_to_json { + value.as_json() + } else { + value.as_str() + }; + let contents = strip_ansi_codes(&contents); if let Ok(mut handle) = locked_file.write() { diff --git a/src/scan_manager.rs b/src/scan_manager.rs index 7c3c138..4abc54f 100644 --- a/src/scan_manager.rs +++ b/src/scan_manager.rs @@ -66,7 +66,7 @@ impl FeroxScan { self.stop_progress_bar(); if let Some(_task) = &self.task { - // task.abort(); todo uncomment once upgraded to tokio 0.3 + // task.abort(); todo uncomment once upgraded to tokio 0.3 (issue #107) } } @@ -271,8 +271,8 @@ impl FeroxScans { let mut user_input = String::new(); std::io::stdin().read_line(&mut user_input).unwrap(); - // todo actual logic for parsing user input in a way that allows for - // calling .abort on the scan retrieved based on the input (issue #107) + // todo (issue #107) actual logic for parsing user input in a way that allows for + // calling .abort on the scan retrieved based on the input } } diff --git a/src/scanner.rs b/src/scanner.rs index 271c3ab..7bd121c 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -91,7 +91,7 @@ fn spawn_recursion_handler( wordlist: Arc>, base_depth: usize, tx_term: UnboundedSender, - tx_file: UnboundedSender, + tx_file: UnboundedSender, ) -> BoxFuture<'static, Vec>> { log::trace!( "enter: spawn_recursion_handler({:?}, wordlist[{} words...], {}, {:?}, {:?})", @@ -466,7 +466,7 @@ pub async fn scan_url( wordlist: Arc>, base_depth: usize, tx_term: UnboundedSender, - tx_file: UnboundedSender, + tx_file: UnboundedSender, ) { log::trace!( "enter: scan_url({:?}, wordlist[{} words...], {}, {:?}, {:?})", @@ -516,7 +516,7 @@ pub async fn scan_url( // Arc clones to be passed around to the various scans let wildcard_bar = progress_bar.clone(); - let heuristics_file_clone = tx_file.clone(); + let heuristics_term_clone = tx_term.clone(); let recurser_term_clone = tx_term.clone(); let recurser_file_clone = tx_file.clone(); let recurser_words = wordlist.clone(); @@ -535,7 +535,7 @@ pub async fn scan_url( // add any wildcard filters to `FILTERS` let filter = - match heuristics::wildcard_test(&target_url, wildcard_bar, heuristics_file_clone).await { + match heuristics::wildcard_test(&target_url, wildcard_bar, heuristics_term_clone).await { Some(f) => Box::new(f), None => Box::new(WildcardFilter::default()), }; diff --git a/src/utils.rs b/src/utils.rs index 3e3f6af..f8eeb17 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -8,6 +8,34 @@ use reqwest::{Client, Response, Url}; #[cfg(not(target_os = "windows"))] use rlimit::{getrlimit, setrlimit, Resource, Rlim}; use std::convert::TryInto; +use std::sync::{Arc, RwLock}; +use std::{fs, io}; + +/// Given the path to a file, open the file in append mode (create it if it doesn't exist) and +/// return a reference to the file that is buffered and locked +pub fn open_file(filename: &str) -> Option>>> { + log::trace!("enter: open_file({})", filename); + + match fs::OpenOptions::new() // std fs + .create(true) + .append(true) + .open(filename) + { + Ok(file) => { + let writer = io::BufWriter::new(file); // std io + + let locked_file = Some(Arc::new(RwLock::new(writer))); + + log::trace!("exit: open_file -> {:?}", locked_file); + locked_file + } + Err(e) => { + log::error!("{}", e); + log::trace!("exit: open_file -> None"); + None + } + } +} /// Helper function that determines the current depth of a given url /// diff --git a/tests/test_extractor.rs b/tests/test_extractor.rs index fb5f7a3..86cd91d 100644 --- a/tests/test_extractor.rs +++ b/tests/test_extractor.rs @@ -131,10 +131,10 @@ fn extractor_finds_relative_url() -> Result<(), Box> { #[test] /// send a request to a page that contains an relative link, follow it, and find the same link again /// should follow then filter -fn extractor_finds_same_relative_url_twice() -> Result<(), Box> { +fn extractor_finds_same_relative_url_twice() { let srv = MockServer::start(); let (tmp_dir, file) = - setup_tmp_directory(&["LICENSE".to_string(), "README".to_string()], "wordlist")?; + setup_tmp_directory(&["LICENSE".to_string(), "README".to_string()], "wordlist").unwrap(); let mock = Mock::new() .expect_method(GET) @@ -177,7 +177,6 @@ fn extractor_finds_same_relative_url_twice() -> Result<(), Box Result<(), Box #[test] /// test finds a dynamic wildcard and reports as much to stdout and a file -fn test_dynamic_wildcard_request_found() -> Result<(), Box> { +fn test_dynamic_wildcard_request_found() { let srv = MockServer::start(); - let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?; + let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap(); let outfile = tmp_dir.path().join("outfile"); let mock = Mock::new() @@ -166,31 +166,19 @@ fn test_dynamic_wildcard_request_found() -> Result<(), Box Result<(), Box Result<(), Box> { +fn heuristics_wildcard_test_with_two_static_wildcards() { let srv = MockServer::start(); - let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?; + let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap(); let mock = Mock::new() .expect_method(GET) @@ -265,7 +253,6 @@ fn heuristics_wildcard_test_with_two_static_wildcards() -> Result<(), Box