Merge pull request #147 from epi052/144-resume-scan

added 1.9 images to repo

.github/workflows/check.yml

@@ -61,4 +61,4 @@ jobs:
       - uses: actions-rs/cargo@v1
         with:
           command: clippy
-          args: --all-targets --all-features -- -D warnings -A clippy::unnecessary_unwrap -A clippy::deref_addrof
+          args: --all-targets --all-features -- -D warnings -A clippy::deref_addrof

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "feroxbuster"
-version = "1.8.0"
+version = "1.9.0"
 authors = ["Ben 'epi' Risher <epibar052@gmail.com>"]
 license = "MIT"
 edition = "2018"
@@ -34,6 +34,7 @@ dirs = "3.0"
 regex = "1"
 crossterm = "0.18"
 rlimit = "0.5"
+ctrlc = "3.1"
 
 [dev-dependencies]
 tempfile = "3.1"

README.md

@@ -73,19 +73,20 @@ This attack is also known as Predictable Resource Location, File Enumeration, Di
     - [ferox-config.toml](#ferox-configtoml)
     - [Command Line Parsing](#command-line-parsing)
 - [Example Usage](#-example-usage)
-    - [Pause and Resume Scans (new in `v1.4.0`)](#pause-and-resume-scans-new-in-v140)
     - [Multiple Values](#multiple-values)
-    - [Extract Links from Response Body (new in `v1.1.0`)](#extract-links-from-response-body-new-in-v110)
     - [Include Headers](#include-headers)
     - [IPv6, Non-recursive scan with INFO logging enabled](#ipv6-non-recursive-scan-with-info-level-logging-enabled)
     - [Read urls from STDIN; pipe only resulting urls out to another tool](#read-urls-from-stdin-pipe-only-resulting-urls-out-to-another-tool)
     - [Proxy traffic through Burp](#proxy-traffic-through-burp)
     - [Proxy traffic through a SOCKS proxy](#proxy-traffic-through-a-socks-proxy)
     - [Pass auth token via query parameter](#pass-auth-token-via-query-parameter)
+    - [Extract Links from Response Body (new in `v1.1.0`)](#extract-links-from-response-body-new-in-v110)
     - [Limit Total Number of Concurrent Scans (new in `v1.2.0`)](#limit-total-number-of-concurrent-scans-new-in-v120)
     - [Filter Response by Status Code  (new in `v1.3.0`)](#filter-response-by-status-code--new-in-v130)
-    - [Filter Response Using a Regular Expression (new in `v1.8.0`)](#filter-response-using-a-regular-expression-new-in-v180)
+    - [Pause an Active Scan (new in `v1.4.0`)](#pause-an-active-scan-new-in-v140)
     - [Replay Responses to a Proxy based on Status Code (new in `v1.5.0`)](#replay-responses-to-a-proxy-based-on-status-code-new-in-v150)
+    - [Filter Response Using a Regular Expression (new in `v1.8.0`)](#filter-response-using-a-regular-expression-new-in-v180)
+    - [Stop and Resume Scans (save scan's state to disk) (new in `v1.9.0`)](#stop-and-resume-scans---resume-from-file-new-in-v190)
 - [Comparison w/ Similar Tools](#-comparison-w-similar-tools)
 - [Common Problems/Issues (FAQ)](#-common-problemsissues-faq)
     - [No file descriptors available](#no-file-descriptors-available)
@@ -257,6 +258,7 @@ Configuration begins with with the following built-in default values baked into
 - recursion depth: `4`
 - auto-filter wildcards - `true`
 - output: `stdout`
+- save_state: `true` (create a state file in cwd when `Ctrl+C` is received)
 
 ### Threads and Connection Limits At A High-Level
 
@@ -348,6 +350,7 @@ A pre-made configuration file with examples of all available settings can be fou
 # filter_word_count = [993]
 # filter_line_count = [35, 36]
 # queries = [["name","value"], ["rick", "astley"]]
+# save_state = false
 
 # headers can be specified on multiple lines or as an inline table
 #
@@ -404,6 +407,8 @@ OPTIONS:
                                             -codes value)
     -P, --replay-proxy <REPLAY_PROXY>       Send only unfiltered requests through a Replay Proxy, instead of all
                                             requests
+        --resume-from <STATE_FILE>          State file from which to resume a partially complete scan (ex. --resume-from
+                                            ferox-1606586780.state)
     -L, --scan-limit <SCAN_LIMIT>           Limit total number of concurrent scans (default: 0, i.e. no limit)
     -s, --status-codes <STATUS_CODE>...     Status Codes to include (allow list) (default: 200 204 301 302 307 308 401
                                             403 405)
@@ -416,12 +421,6 @@ OPTIONS:
 
 ## 🧰 Example Usage
 
-### Pause and Resume Scans (new in `v1.4.0`)
-
-Scans can be paused and resumed by pressing the ENTER key (shown below)
-
-![pause-resume-demo](img/pause-resume-demo.gif)
-
 ### Multiple Values
 
 Options that take multiple values are very flexible.  Consider the following ways of specifying extensions:
@@ -440,6 +439,36 @@ All of the methods above (multiple flags, space separated, comma separated, etc.
 ./feroxbuster -u http://127.1 -H Accept:application/json "Authorization: Bearer {token}"
 ```
 
+### IPv6, non-recursive scan with INFO-level logging enabled
+
+```
+./feroxbuster -u http://[::1] --no-recursion -vv
+```
+
+### Read urls from STDIN; pipe only resulting urls out to another tool
+
+```
+cat targets | ./feroxbuster --stdin --quiet -s 200 301 302 --redirects -x js | fff -s 200 -o js-files
+```
+
+### Proxy traffic through Burp
+
+```
+./feroxbuster -u http://127.1 --insecure --proxy http://127.0.0.1:8080
+```
+
+### Proxy traffic through a SOCKS proxy
+
+```
+./feroxbuster -u http://127.1 --proxy socks5://127.0.0.1:9050
+```
+
+### Pass auth token via query parameter 
+
+```
+./feroxbuster -u http://127.1 --query token=0123456789ABCDEF
+```
+
 ### Extract Links from Response Body (New in `v1.1.0`) 
 
 Search through the body of valid responses (html, javascript, etc...) for additional endpoints to scan. This turns
@@ -470,37 +499,6 @@ With `--extract-links`
 
 ![extract-scan-cmp-normal](img/extract-scan-cmp-normal.gif)
 
-
-### IPv6, non-recursive scan with INFO-level logging enabled
-
-```
-./feroxbuster -u http://[::1] --no-recursion -vv
-```
-
-### Read urls from STDIN; pipe only resulting urls out to another tool
-
-```
-cat targets | ./feroxbuster --stdin --quiet -s 200 301 302 --redirects -x js | fff -s 200 -o js-files
-```
-
-### Proxy traffic through Burp
-
-```
-./feroxbuster -u http://127.1 --insecure --proxy http://127.0.0.1:8080
-```
-
-### Proxy traffic through a SOCKS proxy
-
-```
-./feroxbuster -u http://127.1 --proxy socks5://127.0.0.1:9050
-```
-
-### Pass auth token via query parameter 
-
-```
-./feroxbuster -u http://127.1 --query token=0123456789ABCDEF
-```
-
 ### Limit Total Number of Concurrent Scans (new in `v1.2.0`)
 
 Limit the number of scans permitted to run at any given time.  Recursion will still identify new directories, but newly
@@ -523,18 +521,11 @@ each one is checked against a list of known filters and either displayed or not
 ./feroxbuster -u http://127.1 --filter-status 301
 ```
 
-### Filter Response Using a Regular Expression (new in `v1.8.0`) 
+### Pause an Active Scan (new in `v1.4.0`)
 
-Version 1.3.0 included an overhaul to the filtering system which will allow for a wide array of filters to be added 
-with minimal effort. The latest addition is a Regular Expression Filter. As responses come back from the scanned server,
-the **body** of the response is checked against the filter's regular expression.  If the expression is found in the 
-body, then that response is filtered out.  
+Scans can be paused and resumed by pressing the ENTER key (shown below)
 
-**NOTE: Using regular expressions to filter large responses or many regular expressions may negatively impact performance.**  
-
-```
-./feroxbuster -u http://127.1 --filter-regex '[aA]ccess [dD]enied.?' --output results.txt --json
-```
+![pause-resume-demo](img/pause-resume-demo.gif)
 
 ### Replay Responses to a Proxy based on Status Code (new in `v1.5.0`)
 
@@ -550,6 +541,80 @@ Of note: this means that for every response that matches your replay criteria, y
 
 ![replay-proxy-demo](img/replay-proxy-demo.gif)
 
+### Filter Response Using a Regular Expression (new in `v1.8.0`) 
+
+Version 1.3.0 included an overhaul to the filtering system which will allow for a wide array of filters to be added 
+with minimal effort. The latest addition is a Regular Expression Filter. As responses come back from the scanned server,
+the **body** of the response is checked against the filter's regular expression.  If the expression is found in the 
+body, then that response is filtered out.  
+
+**NOTE: Using regular expressions to filter large responses or many regular expressions may negatively impact performance.**  
+
+```
+./feroxbuster -u http://127.1 --filter-regex '[aA]ccess [dD]enied.?' --output results.txt --json
+```
+
+### Stop and Resume Scans (`--resume-from FILE`) (new in `v1.9.0`)
+
+Version 1.9.0 adds a few features that allow for completely stopping a scan, and resuming that same scan from a file on disk. 
+
+A simple `Ctrl+C` during a scan will create a file that contains information about the scan that was cancelled.
+
+![save-state](img/save-state.png)
+
+```json
+// example snippet of state file
+
+{
+   "scans":[
+      {
+         "id":"057016a14769414aac9a7a62707598cb",
+         "url":"https://localhost.com",
+         "scan_type":"Directory",
+         "complete":true
+      },
+      {
+         "id":"400b2323a16f43468a04ffcbbeba34c6",
+         "url":"https://localhost.com/css",
+         "scan_type":"Directory",
+         "complete":false
+      }
+   ],
+   "config":{
+      "wordlist":"/wordlists/seclists/Discovery/Web-Content/common.txt",
+      "...":"..."
+   },
+   "responses":[
+      {
+         "type":"response",
+         "url":"https://localhost.com/Login",
+         "path":"/Login",
+         "wildcard":false,
+         "status":302,
+         "content_length":0,
+         "line_count":0,
+         "word_count":0,
+         "headers":{
+            "content-length":"0",
+            "server":"nginx/1.16.1"
+         }
+      }
+   ]
+},
+```
+
+Based on the example image above, the same scan can be resumed by using `feroxbuster --resume-from ferox-http_localhost-1606947491.state`.  Directories that were already complete are not rescanned, however partially complete scans are started from the beginning.  
+
+![resumed-scan](img/resumed-scan.gif)
+
+In order to prevent state file creation when `Ctrl+C` is pressed, you can simply add the entry below to your `ferox-config.toml`.
+
+```toml
+# ferox-config.toml
+
+save_state = false
+```
+
 ## 🧐 Comparison w/ Similar Tools
 
 There are quite a few similar tools for forced browsing/content discovery.  Burp Suite Pro, Dirb, Dirbuster, etc... 

ferox-config.toml.example

@@ -36,6 +36,7 @@
 # filter_word_count = [993]
 # filter_line_count = [35, 36]
 # queries = [["name","value"], ["rick", "astley"]]
+# save_state = false
 
 # headers can be specified on multiple lines or as an inline table
 #

src/config.rs

@@ -1,3 +1,4 @@
+use crate::scan_manager::resume_scan;
 use crate::utils::{module_colorizer, status_colorizer};
 use crate::{client, parser, progress};
 use crate::{FeroxSerialize, DEFAULT_CONFIG_NAME, DEFAULT_STATUS_CODES, DEFAULT_WORDLIST, VERSION};
@@ -21,7 +22,7 @@ lazy_static! {
     pub static ref PROGRESS_BAR: MultiProgress = MultiProgress::with_draw_target(ProgressDrawTarget::stdout());
 
     /// Global progress bar that is only used for printing messages that don't jack up other bars
-    pub static ref PROGRESS_PRINTER: ProgressBar = progress::add_bar("", 0, true);
+    pub static ref PROGRESS_PRINTER: ProgressBar = progress::add_bar("", 0, true, false);
 }
 
 /// simple helper to clean up some code reuse below; panics under test / exits in prod
@@ -191,9 +192,19 @@ pub struct Configuration {
     /// Don't auto-filter wildcard responses
     #[serde(default)]
     pub dont_filter: bool,
+
+    /// Scan started from a state file, not from CLI args
+    #[serde(default)]
+    pub resumed: bool,
+
+    /// Whether or not a scan's current state should be saved when user presses Ctrl+C
+    ///
+    /// Not configurable from CLI; can only be set from a config file
+    #[serde(default = "save_state")]
+    pub save_state: bool,
 }
 
-// functions timeout, threads, status_codes, user_agent, wordlist, and depth are used to provide
+// functions timeout, threads, status_codes, user_agent, wordlist, save_state, and depth are used to provide
 // defaults in the event that a ferox-config.toml is found but one or more of the values below
 // aren't listed in the config.  This way, we get the correct defaults upon Deserialization
 
@@ -207,6 +218,11 @@ fn timeout() -> u64 {
     7
 }
 
+/// default save_state value
+fn save_state() -> bool {
+    true
+}
+
 /// default threads value
 fn threads() -> usize {
     50
@@ -256,6 +272,7 @@ impl Default for Configuration {
             replay_client,
             dont_filter: false,
             quiet: false,
+            resumed: false,
             stdin: false,
             json: false,
             verbosity: 0,
@@ -265,6 +282,7 @@ impl Default for Configuration {
             redirects: false,
             no_recursion: false,
             extract_links: false,
+            save_state: true,
             proxy: String::new(),
             config: String::new(),
             output: String::new(),
@@ -304,6 +322,7 @@ impl Configuration {
     /// - **output**: `None` (print to stdout)
     /// - **debug_log**: `None`
     /// - **quiet**: `false`
+    /// - **save_state**: `true`
     /// - **user_agent**: `feroxbuster/VERSION`
     /// - **insecure**: `false` (don't be insecure, i.e. don't allow invalid certs)
     /// - **extensions**: `None`
@@ -346,6 +365,29 @@ impl Configuration {
             return Configuration::default();
         }
 
+        let args = parser::initialize().get_matches();
+
+        if let Some(filename) = args.value_of("resume_from") {
+            // when resuming a scan, instead of normal configuration loading, we just
+            // load the config from disk by calling resume_scan
+            let mut previous_config = resume_scan(filename);
+
+            // the resumed flag isn't printed in the banner and really has no business being
+            // serialized or included in much of the usual config logic; simply setting it to true
+            // here and being done with it
+            previous_config.resumed = true;
+
+            // if the user used --stdin, we already have all the scans started (or complete), we
+            // need to flip stdin to false so that the 'read from stdin' logic doesn't fire (if
+            // not flipped to false, the program hangs waiting for input from stdin again)
+            previous_config.stdin = false;
+
+            // clients aren't serialized, have to remake them from the previous config
+            Self::try_rebuild_clients(&mut previous_config);
+
+            return previous_config;
+        }
+
         // Get the default configuration, this is what will apply if nothing
         // else is specified.
         let mut config = Configuration::default();
@@ -392,8 +434,6 @@ impl Configuration {
             Self::parse_and_merge_config(config_file, &mut config);
         }
 
-        let args = parser::initialize().get_matches();
-
         macro_rules! update_config_if_present {
             ($c:expr, $m:ident, $v:expr, $t:ty) => {
                 match value_t!($m, $v, $t) {
@@ -569,50 +609,55 @@ impl Configuration {
             }
         }
 
-        // this if statement determines if we've gotten a Client configuration change from
-        // either the config file or command line arguments; if we have, we need to rebuild
-        // the client and store it in the config struct
-        if !config.proxy.is_empty()
-            || config.timeout != timeout()
-            || config.user_agent != user_agent()
-            || config.redirects
-            || config.insecure
-            || !config.headers.is_empty()
+        Self::try_rebuild_clients(&mut config);
+
+        config
+    }
+
+    /// this function determines if we've gotten a Client configuration change from
+    /// either the config file or command line arguments; if we have, we need to rebuild
+    /// the client and store it in the config struct
+    fn try_rebuild_clients(configuration: &mut Configuration) {
+        if !configuration.proxy.is_empty()
+            || configuration.timeout != timeout()
+            || configuration.user_agent != user_agent()
+            || configuration.redirects
+            || configuration.insecure
+            || !configuration.headers.is_empty()
+            || configuration.resumed
         {
-            if config.proxy.is_empty() {
-                config.client = client::initialize(
-                    config.timeout,
-                    &config.user_agent,
-                    config.redirects,
-                    config.insecure,
-                    &config.headers,
+            if configuration.proxy.is_empty() {
+                configuration.client = client::initialize(
+                    configuration.timeout,
+                    &configuration.user_agent,
+                    configuration.redirects,
+                    configuration.insecure,
+                    &configuration.headers,
                     None,
                 )
             } else {
-                config.client = client::initialize(
-                    config.timeout,
-                    &config.user_agent,
-                    config.redirects,
-                    config.insecure,
-                    &config.headers,
-                    Some(&config.proxy),
+                configuration.client = client::initialize(
+                    configuration.timeout,
+                    &configuration.user_agent,
+                    configuration.redirects,
+                    configuration.insecure,
+                    &configuration.headers,
+                    Some(&configuration.proxy),
                 )
             }
         }
 
-        if !config.replay_proxy.is_empty() {
+        if !configuration.replay_proxy.is_empty() {
             // only set replay_client when replay_proxy is set
-            config.replay_client = Some(client::initialize(
-                config.timeout,
-                &config.user_agent,
-                config.redirects,
-                config.insecure,
-                &config.headers,
-                Some(&config.replay_proxy),
+            configuration.replay_client = Some(client::initialize(
+                configuration.timeout,
+                &configuration.user_agent,
+                configuration.redirects,
+                configuration.insecure,
+                &configuration.headers,
+                Some(&configuration.replay_proxy),
             ));
         }
-
-        config
     }
 
     /// Given a configuration file's location and an instance of `Configuration`, read in
@@ -665,6 +710,7 @@ impl Configuration {
         settings.scan_limit = settings_to_merge.scan_limit;
         settings.replay_proxy = settings_to_merge.replay_proxy;
         settings.replay_codes = settings_to_merge.replay_codes;
+        settings.save_state = settings_to_merge.save_state;
         settings.debug_log = settings_to_merge.debug_log;
         settings.json = settings_to_merge.json;
     }
@@ -765,6 +811,7 @@ mod tests {
             dont_filter = true
             extract_links = true
             json = true
+            save_state = false
             depth = 1
             filter_size = [4120]
             filter_regex = ["^ignore me$"]
@@ -800,6 +847,7 @@ mod tests {
         assert_eq!(config.dont_filter, false);
         assert_eq!(config.no_recursion, false);
         assert_eq!(config.json, false);
+        assert_eq!(config.save_state, true);
         assert_eq!(config.stdin, false);
         assert_eq!(config.add_slash, false);
         assert_eq!(config.redirects, false);
@@ -1004,6 +1052,13 @@ mod tests {
         assert_eq!(config.filter_status, vec![201]);
     }
 
+    #[test]
+    /// parse the test config and see that the value parsed is correct
+    fn config_reads_save_state() {
+        let config = setup_config_test();
+        assert_eq!(config.save_state, false);
+    }
+
     #[test]
     /// parse the test config and see that the values parsed are correct
     fn config_reads_headers() {

src/filters.rs

@@ -360,6 +360,8 @@ mod tests {
             wildcard: true,
             url: Url::parse("http://localhost").unwrap(),
             content_length: 100,
+            word_count: 50,
+            line_count: 25,
             headers: reqwest::header::HeaderMap::new(),
             status: reqwest::StatusCode::OK,
         };
@@ -380,6 +382,8 @@ mod tests {
             wildcard: true,
             url: Url::parse("http://localhost/stuff").unwrap(),
             content_length: 100,
+            word_count: 50,
+            line_count: 25,
             headers: reqwest::header::HeaderMap::new(),
             status: reqwest::StatusCode::OK,
         };
@@ -400,6 +404,8 @@ mod tests {
             wildcard: false,
             url: Url::parse("http://localhost/stuff").unwrap(),
             content_length: 100,
+            word_count: 50,
+            line_count: 25,
             headers: reqwest::header::HeaderMap::new(),
             status: reqwest::StatusCode::OK,
         };

src/lib.rs

@@ -14,9 +14,13 @@ pub mod utils;
 
 use crate::utils::{get_url_path_length, status_colorizer};
 use console::{style, Color};
+use reqwest::header::{HeaderName, HeaderValue};
 use reqwest::{header::HeaderMap, Response, StatusCode, Url};
-use serde::{ser::SerializeStruct, Deserialize, Serialize, Serializer};
+use serde::{ser::SerializeStruct, Deserialize, Deserializer, Serialize, Serializer};
+use serde_json::Value;
 use std::collections::HashMap;
+use std::convert::{TryFrom, TryInto};
+use std::str::FromStr;
 use std::{error, fmt};
 use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender};
 
@@ -112,6 +116,12 @@ pub struct FeroxResponse {
     /// The content-length of this response, if known
     content_length: u64,
 
+    /// The number of lines contained in the body of this response, if known
+    line_count: usize,
+
+    /// The number of words contained in the body of this response, if known
+    word_count: usize,
+
     /// The `Headers` of this `FeroxResponse`
     headers: HeaderMap,
 
@@ -194,15 +204,12 @@ impl FeroxResponse {
 
     /// Returns line count of the response text.
     pub fn line_count(&self) -> usize {
-        self.text().lines().count()
+        self.line_count
     }
 
     /// Returns word count of the response text.
     pub fn word_count(&self) -> usize {
-        self.text()
-            .lines()
-            .map(|s| s.split_whitespace().count())
-            .sum()
+        self.word_count
     }
 
     /// Create a new `FeroxResponse` from the given `Response`
@@ -228,18 +235,23 @@ impl FeroxResponse {
             String::new()
         };
 
+        let line_count = text.lines().count();
+        let word_count = text.lines().map(|s| s.split_whitespace().count()).sum();
+
         FeroxResponse {
             url,
             status,
             content_length,
             text,
             headers,
+            line_count,
+            word_count,
             wildcard: false,
         }
     }
 }
 
-/// Implement FeroxSerialize for FeroxResponse
+/// Implement FeroxSerialusize::from(ize for FeroxRespons)e
 impl FeroxSerialize for FeroxResponse {
     /// Simple wrapper around create_report_string
     fn as_str(&self) -> String {
@@ -359,18 +371,99 @@ impl Serialize for FeroxResponse {
         state.serialize_field("wildcard", &self.wildcard)?;
         state.serialize_field("status", &self.status.as_u16())?;
         state.serialize_field("content_length", &self.content_length)?;
-        state.serialize_field("line_count", &self.line_count())?;
-        state.serialize_field("word_count", &self.word_count())?;
+        state.serialize_field("line_count", &self.line_count)?;
+        state.serialize_field("word_count", &self.word_count)?;
         state.serialize_field("headers", &headers)?;
 
         state.end()
     }
 }
 
+/// Deserialize implementation for FeroxResponse
+impl<'de> Deserialize<'de> for FeroxResponse {
+    /// Deserialize a FeroxResponse from a serde_json::Value
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let mut response = Self {
+            url: Url::parse("http://localhost").unwrap(),
+            status: StatusCode::OK,
+            text: String::new(),
+            content_length: 0,
+            headers: HeaderMap::new(),
+            wildcard: false,
+            line_count: 0,
+            word_count: 0,
+        };
+
+        let map: HashMap<String, Value> = HashMap::deserialize(deserializer)?;
+
+        for (key, value) in &map {
+            match key.as_str() {
+                "url" => {
+                    if let Some(url) = value.as_str() {
+                        if let Ok(parsed) = Url::parse(url) {
+                            response.url = parsed;
+                        }
+                    }
+                }
+                "status" => {
+                    if let Some(num) = value.as_u64() {
+                        if let Ok(smaller) = u16::try_from(num) {
+                            if let Ok(status) = StatusCode::from_u16(smaller) {
+                                response.status = status;
+                            }
+                        }
+                    }
+                }
+                "content_length" => {
+                    if let Some(num) = value.as_u64() {
+                        response.content_length = num;
+                    }
+                }
+                "line_count" => {
+                    if let Some(num) = value.as_u64() {
+                        response.line_count = num.try_into().unwrap_or_default();
+                    }
+                }
+                "word_count" => {
+                    if let Some(num) = value.as_u64() {
+                        response.word_count = num.try_into().unwrap_or_default();
+                    }
+                }
+                "headers" => {
+                    let mut headers = HeaderMap::<HeaderValue>::default();
+
+                    if let Some(map_headers) = value.as_object() {
+                        for (h_key, h_value) in map_headers {
+                            let h_value_str = h_value.as_str().unwrap_or("");
+                            let h_name = HeaderName::from_str(h_key)
+                                .unwrap_or_else(|_| HeaderName::from_str("Unknown").unwrap());
+                            let h_value_parsed = HeaderValue::from_str(h_value_str)
+                                .unwrap_or_else(|_| HeaderValue::from_str("Unknown").unwrap());
+                            headers.insert(h_name, h_value_parsed);
+                        }
+                    }
+
+                    response.headers = headers;
+                }
+                "wildcard" => {
+                    if let Some(result) = value.as_bool() {
+                        response.wildcard = result;
+                    }
+                }
+                _ => {}
+            }
+        }
+
+        Ok(response)
+    }
+}
+
 #[derive(Serialize, Deserialize, Default)]
 /// Representation of a log entry, can be represented as a human readable string or JSON
 pub struct FeroxMessage {
-    // todo probably move to lib
     #[serde(rename = "type")]
     /// Name of this type of struct, used for serialization, i.e. `{"type":"log"}`
     kind: String,

src/main.rs

@@ -1,16 +1,18 @@
 use crossterm::event::{self, Event, KeyCode};
+use feroxbuster::progress::add_bar;
 use feroxbuster::{
     banner,
     config::{CONFIGURATION, PROGRESS_BAR, PROGRESS_PRINTER},
     heuristics, logger, reporter,
-    scan_manager::PAUSE_SCAN,
-    scanner::{self, scan_url},
+    scan_manager::{self, PAUSE_SCAN},
+    scanner::{self, scan_url, RESPONSES, SCANNED_URLS},
     utils::{ferox_print, get_current_depth, module_colorizer, status_colorizer},
-    FeroxError, FeroxResponse, FeroxResult, SLEEP_DURATION, VERSION,
+    FeroxError, FeroxResponse, FeroxResult, FeroxSerialize, SLEEP_DURATION, VERSION,
 };
 #[cfg(not(target_os = "windows"))]
 use feroxbuster::{utils::set_open_file_limit, DEFAULT_OPEN_FILE_LIMIT};
 use futures::StreamExt;
+use std::convert::TryInto;
 use std::{
     collections::HashSet,
     fs::File,
@@ -115,6 +117,31 @@ async fn scan(
 
     scanner::initialize(words.len(), &CONFIGURATION);
 
+    if CONFIGURATION.resumed {
+        if let Ok(scans) = SCANNED_URLS.scans.lock() {
+            for scan in scans.iter() {
+                if let Ok(locked_scan) = scan.lock() {
+                    if locked_scan.complete {
+                        // these scans are complete, and just need to be shown to the user
+                        let pb = add_bar(
+                            &locked_scan.url,
+                            words.len().try_into().unwrap_or_default(),
+                            false,
+                            true,
+                        );
+                        pb.finish();
+                    }
+                }
+            }
+        }
+
+        if let Ok(responses) = RESPONSES.responses.read() {
+            for response in responses.iter() {
+                PROGRESS_PRINTER.println(response.as_str());
+            }
+        }
+    }
+
     let mut tasks = vec![];
 
     for target in targets {
@@ -152,6 +179,22 @@ async fn get_targets() -> FeroxResult<Vec<String>> {
         while let Some(line) = reader.next().await {
             targets.push(line?);
         }
+    } else if CONFIGURATION.resumed {
+        // resume-from can't be used with any other flag, making it mutually exclusive from either
+        // of the other two options
+        if let Ok(scans) = SCANNED_URLS.scans.lock() {
+            for scan in scans.iter() {
+                // SCANNED_URLS gets deserialized scans added to it at program start if --resume-from
+                // is used, so scans that aren't marked complete still need to be scanned
+                if let Ok(locked_scan) = scan.lock() {
+                    if locked_scan.complete {
+                        // this one's already done, ignore it
+                        continue;
+                    }
+                    targets.push(locked_scan.url.to_owned());
+                }
+            }
+        }
     } else {
         targets.push(CONFIGURATION.target_url.clone());
     }
@@ -298,6 +341,11 @@ fn main() {
     // setup logging based on the number of -v's used
     logger::initialize(CONFIGURATION.verbosity);
 
+    if CONFIGURATION.save_state {
+        // start the ctrl+c handler
+        scan_manager::initialize();
+    }
+
     // this function uses rlimit, which is not supported on windows
     #[cfg(not(target_os = "windows"))]
     set_open_file_limit(DEFAULT_OPEN_FILE_LIMIT);

src/parser.rs

@@ -1,5 +1,5 @@
 use crate::VERSION;
-use clap::{App, Arg};
+use clap::{App, Arg, ArgGroup};
 
 /// Create and return an instance of [clap::App](https://docs.rs/clap/latest/clap/struct.App.html), i.e. the Command Line Interface's configuration
 pub fn initialize() -> App<'static, 'static> {
@@ -19,7 +19,7 @@ pub fn initialize() -> App<'static, 'static> {
             Arg::with_name("url")
                 .short("u")
                 .long("url")
-                .required_unless("stdin")
+                .required_unless_one(&["stdin", "resume_from"])
                 .value_name("URL")
                 .multiple(true)
                 .use_delimiter(true)
@@ -113,6 +113,7 @@ pub fn initialize() -> App<'static, 'static> {
             Arg::with_name("json")
                 .long("json")
                 .takes_value(false)
+                .requires("output_files")
                 .help("Emit JSON logs to --output and --debug-log instead of normal text")
         )
         .arg(
@@ -130,6 +131,14 @@ pub fn initialize() -> App<'static, 'static> {
                 .help("Output file to write results to (use w/ --json for JSON entries)")
                 .takes_value(true),
         )
+        .arg(
+            Arg::with_name("resume_from")
+                .long("resume-from")
+                .value_name("STATE_FILE")
+                .help("State file from which to resume a partially complete scan (ex. --resume-from ferox-1606586780.state)")
+                .conflicts_with_all(&["wordlist", "url", "threads", "depth", "timeout", "verbosity", "proxy", "replay_proxy", "replay_codes", "status_codes", "quiet", "json", "dont_filter", "output", "debug_log", "user_agent", "redirects", "insecure", "extensions", "headers", "queries", "no_recursion", "add_slash", "stdin", "filter_size", "filter_regex", "filter_words", "filter_lines", "filter_status", "extract_links", "scan_limit"])
+                .takes_value(true),
+        )
         .arg(
             Arg::with_name("debug_log")
                 .long("debug-log")
@@ -294,6 +303,10 @@ pub fn initialize() -> App<'static, 'static> {
                 .takes_value(true)
                 .help("Limit total number of concurrent scans (default: 0, i.e. no limit)")
         )
+        .group(ArgGroup::with_name("output_files")
+            .args(&["debug_log", "output"])
+            .multiple(true)
+        )
         .after_help(r#"NOTE:
     Options that take multiple values are very flexible.  Consider the following ways of specifying
     extensions:

src/progress.rs

@@ -3,9 +3,16 @@ use indicatif::{ProgressBar, ProgressStyle};
 
 /// Add an [indicatif::ProgressBar](https://docs.rs/indicatif/latest/indicatif/struct.ProgressBar.html)
 /// to the global [PROGRESS_BAR](../config/struct.PROGRESS_BAR.html)
-pub fn add_bar(prefix: &str, length: u64, hidden: bool) -> ProgressBar {
+pub fn add_bar(prefix: &str, length: u64, hidden: bool, hide_per_sec: bool) -> ProgressBar {
     let style = if hidden || CONFIGURATION.quiet {
         ProgressStyle::default_bar().template("")
+    } else if hide_per_sec {
+        ProgressStyle::default_bar()
+            .template(&format!(
+                "[{{bar:.cyan/blue}}] - {{elapsed:<4}} {{pos:>7}}/{{len:7}} {:7} {{prefix}}",
+                "-"
+            ))
+            .progress_chars("#>-")
     } else {
         ProgressStyle::default_bar()
             .template("[{bar:.cyan/blue}] - {elapsed:<4} {pos:>7}/{len:7} {per_sec:7} {prefix}")
@@ -20,3 +27,24 @@ pub fn add_bar(prefix: &str, length: u64, hidden: bool) -> ProgressBar {
 
     progress_bar
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    /// hit all code branches for add_bar
+    fn add_bar_with_all_configurations() {
+        let p1 = add_bar("prefix", 2, true, false); // hidden
+        let p2 = add_bar("prefix", 2, false, true); // no per second field
+        let p3 = add_bar("prefix", 2, false, false); // normal bar
+
+        p1.finish();
+        p2.finish();
+        p3.finish();
+
+        assert!(p1.is_finished());
+        assert!(p2.is_finished());
+        assert!(p3.is_finished());
+    }
+}

src/reporter.rs

@@ -1,5 +1,6 @@
 use crate::{
     config::{CONFIGURATION, PROGRESS_PRINTER},
+    scanner::RESPONSES,
     utils::{ferox_print, make_request, open_file},
     FeroxChannel, FeroxResponse, FeroxSerialize,
 };
@@ -96,7 +97,11 @@ async fn spawn_terminal_reporter(
     while let Some(resp) = resp_chan.recv().await {
         log::trace!("received {} on reporting channel", resp.url());
 
-        if CONFIGURATION.status_codes.contains(&resp.status().as_u16()) {
+        let contains_sentry = CONFIGURATION.status_codes.contains(&resp.status().as_u16());
+        let unknown_sentry = !RESPONSES.contains(&resp); // !contains == unknown
+        let should_process_response = contains_sentry && unknown_sentry;
+
+        if should_process_response {
             // print to stdout
             ferox_print(&resp.as_str(), &PROGRESS_PRINTER);
 
@@ -114,9 +119,7 @@ async fn spawn_terminal_reporter(
         }
         log::trace!("report complete: {}", resp.url());
 
-        if CONFIGURATION.replay_client.is_some()
-            && CONFIGURATION.replay_codes.contains(&resp.status().as_u16())
-        {
+        if CONFIGURATION.replay_client.is_some() && should_process_response {
             // replay proxy specified/client created and this response's status code is one that
             // should be replayed
             match make_request(CONFIGURATION.replay_client.as_ref().unwrap(), &resp.url()).await {
@@ -126,6 +129,13 @@ async fn spawn_terminal_reporter(
                 }
             }
         }
+
+        if should_process_response {
+            // add response to RESPONSES for serialization in case of ctrl+c
+            // placed all by its lonesome like this so that RESPONSES can take ownership
+            // of the FeroxResponse
+            RESPONSES.insert(resp);
+        }
     }
     log::trace!("exit: spawn_terminal_reporter");
 }

src/scan_manager.rs

@@ -1,11 +1,28 @@
-use crate::{config::PROGRESS_PRINTER, progress, scanner::NUMBER_OF_REQUESTS, SLEEP_DURATION};
+use crate::config::Configuration;
+use crate::reporter::safe_file_write;
+use crate::utils::open_file;
+use crate::{
+    config::{CONFIGURATION, PROGRESS_PRINTER},
+    progress,
+    scanner::{NUMBER_OF_REQUESTS, RESPONSES, SCANNED_URLS},
+    FeroxResponse, FeroxSerialize, SLEEP_DURATION,
+};
 use console::style;
 use indicatif::{ProgressBar, ProgressStyle};
 use lazy_static::lazy_static;
+use serde::{
+    ser::{SerializeSeq, SerializeStruct},
+    Deserialize, Deserializer, Serialize, Serializer,
+};
+use serde_json::Value;
+use std::collections::HashMap;
 use std::{
     cmp::PartialEq,
     fmt,
+    fs::File,
+    io::BufReader,
     sync::{Arc, Mutex, RwLock},
+    time::{SystemTime, UNIX_EPOCH},
 };
 use std::{
     io::{stderr, Write},
@@ -28,12 +45,20 @@ static INTERACTIVE_BARRIER: AtomicUsize = AtomicUsize::new(0);
 pub static PAUSE_SCAN: AtomicBool = AtomicBool::new(false);
 
 /// Simple enum used to flag a `FeroxScan` as likely a directory or file
-#[derive(Debug)]
+#[derive(Debug, Serialize, Deserialize)]
 pub enum ScanType {
     File,
     Directory,
 }
 
+/// Default implementation for ScanType
+impl Default for ScanType {
+    /// Return ScanType::File as default
+    fn default() -> Self {
+        Self::File
+    }
+}
+
 /// Struct to hold scan-related state
 ///
 /// The purpose of this container is to open up the pathway to aborting currently running tasks and
@@ -59,17 +84,8 @@ pub struct FeroxScan {
     pub progress_bar: Option<ProgressBar>,
 }
 
-/// Implementation of FeroxScan
-impl FeroxScan {
-    /// Stop a currently running scan
-    pub fn abort(&self) {
-        self.stop_progress_bar();
-
-        if let Some(_task) = &self.task {
-            // task.abort();  todo uncomment once upgraded to tokio 0.3 (issue #107)
-        }
-    }
-
+/// Default implementation for FeroxScan
+impl Default for FeroxScan {
     /// Create a default FeroxScan, populates ID with a new UUID
     fn default() -> Self {
         let new_id = Uuid::new_v4().to_simple().to_string();
@@ -83,6 +99,18 @@ impl FeroxScan {
             scan_type: ScanType::File,
         }
     }
+}
+
+/// Implementation of FeroxScan
+impl FeroxScan {
+    /// Stop a currently running scan
+    pub fn abort(&self) {
+        self.stop_progress_bar();
+
+        if let Some(_task) = &self.task {
+            // task.abort();  todo uncomment once upgraded to tokio 0.3 (issue #107)
+        }
+    }
 
     /// Simple helper to call .finish on the scan's progress bar
     fn stop_progress_bar(&self) {
@@ -97,7 +125,7 @@ impl FeroxScan {
             pb.clone()
         } else {
             let num_requests = NUMBER_OF_REQUESTS.load(Ordering::Relaxed);
-            let pb = progress::add_bar(&self.url, num_requests, false);
+            let pb = progress::add_bar(&self.url, num_requests, false, false);
 
             pb.reset_elapsed();
 
@@ -145,6 +173,69 @@ impl PartialEq for FeroxScan {
     }
 }
 
+/// Serialize implementation for FeroxScan
+impl Serialize for FeroxScan {
+    /// Function that handles serialization of a FeroxScan
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let mut state = serializer.serialize_struct("FeroxScan", 4)?;
+
+        state.serialize_field("id", &self.id)?;
+        state.serialize_field("url", &self.url)?;
+        state.serialize_field("scan_type", &self.scan_type)?;
+        state.serialize_field("complete", &self.complete)?;
+
+        state.end()
+    }
+}
+
+/// Deserialize implementation for FeroxScan
+impl<'de> Deserialize<'de> for FeroxScan {
+    /// Deserialize a FeroxScan from a serde_json::Value
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let mut scan = Self::default();
+
+        let map: HashMap<String, Value> = HashMap::deserialize(deserializer)?;
+
+        for (key, value) in &map {
+            match key.as_str() {
+                "id" => {
+                    if let Some(id) = value.as_str() {
+                        scan.id = id.to_string();
+                    }
+                }
+                "scan_type" => {
+                    if let Some(scan_type) = value.as_str() {
+                        scan.scan_type = match scan_type {
+                            "File" => ScanType::File,
+                            "Directory" => ScanType::Directory,
+                            _ => ScanType::File,
+                        }
+                    }
+                }
+                "complete" => {
+                    if let Some(complete) = value.as_bool() {
+                        scan.complete = complete;
+                    }
+                }
+                "url" => {
+                    if let Some(url) = value.as_str() {
+                        scan.url = url.to_string();
+                    }
+                }
+                _ => {}
+            }
+        }
+
+        Ok(scan)
+    }
+}
+
 /// Container around a locked hashset of `FeroxScan`s, adds wrappers for insertion and searching
 #[derive(Debug, Default)]
 pub struct FeroxScans {
@@ -152,6 +243,31 @@ pub struct FeroxScans {
     pub scans: Mutex<Vec<Arc<Mutex<FeroxScan>>>>,
 }
 
+/// Serialize implementation for FeroxScans
+impl Serialize for FeroxScans {
+    /// Function that handles serialization of FeroxScans
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        if let Ok(scans) = self.scans.lock() {
+            let mut seq = serializer.serialize_seq(Some(scans.len()))?;
+
+            for scan in scans.iter() {
+                if let Ok(unlocked) = scan.lock() {
+                    seq.serialize_element(&*unlocked)?;
+                }
+            }
+
+            seq.end()
+        } else {
+            // if for some reason we can't unlock the mutex, just write an empty list
+            let seq = serializer.serialize_seq(Some(0))?;
+            seq.end()
+        }
+    }
+}
+
 /// Implementation of `FeroxScans`
 impl FeroxScans {
     /// Add a `FeroxScan` to the internal container
@@ -324,8 +440,12 @@ impl FeroxScans {
     fn add_scan(&self, url: &str, scan_type: ScanType) -> (bool, Arc<Mutex<FeroxScan>>) {
         let bar = match scan_type {
             ScanType::Directory => {
-                let progress_bar =
-                    progress::add_bar(&url, NUMBER_OF_REQUESTS.load(Ordering::Relaxed), false);
+                let progress_bar = progress::add_bar(
+                    &url,
+                    NUMBER_OF_REQUESTS.load(Ordering::Relaxed),
+                    false,
+                    false,
+                );
 
                 progress_bar.reset_elapsed();
 
@@ -382,9 +502,212 @@ fn get_single_spinner() -> ProgressBar {
     spinner
 }
 
+/// Container around a locked vector of `FeroxResponse`s, adds wrappers for insertion and search
+#[derive(Debug, Default)]
+pub struct FeroxResponses {
+    /// Internal structure: locked hashset of `FeroxScan`s
+    pub responses: Arc<RwLock<Vec<FeroxResponse>>>,
+}
+
+/// Serialize implementation for FeroxResponses
+impl Serialize for FeroxResponses {
+    /// Function that handles serialization of FeroxResponses
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        if let Ok(responses) = self.responses.read() {
+            let mut seq = serializer.serialize_seq(Some(responses.len()))?;
+
+            for response in responses.iter() {
+                seq.serialize_element(response)?;
+            }
+
+            seq.end()
+        } else {
+            // if for some reason we can't unlock the mutex, just write an empty list
+            let seq = serializer.serialize_seq(Some(0))?;
+            seq.end()
+        }
+    }
+}
+
+/// Implementation of `FeroxResponses`
+impl FeroxResponses {
+    /// Add a `FeroxResponse` to the internal container
+    pub fn insert(&self, response: FeroxResponse) {
+        match self.responses.write() {
+            Ok(mut responses) => {
+                responses.push(response);
+            }
+            Err(e) => {
+                log::error!("FeroxResponses' container's mutex is poisoned: {}", e);
+            }
+        }
+    }
+
+    /// Simple check for whether or not a FeroxResponse is contained within the inner container
+    pub fn contains(&self, other: &FeroxResponse) -> bool {
+        match self.responses.read() {
+            Ok(responses) => {
+                for response in responses.iter() {
+                    if response.url == other.url {
+                        return true;
+                    }
+                }
+            }
+            Err(e) => {
+                log::error!("FeroxResponses' container's mutex is poisoned: {}", e);
+            }
+        }
+        false
+    }
+}
+/// Data container for (de)?serialization of multiple items
+#[derive(Serialize, Debug)]
+pub struct FeroxState {
+    /// Known scans
+    scans: &'static FeroxScans,
+
+    /// Current running config
+    config: &'static Configuration,
+
+    /// Known responses
+    responses: &'static FeroxResponses,
+}
+
+/// FeroxSerialize implementation for FeroxState
+impl FeroxSerialize for FeroxState {
+    /// Simply return debug format of FeroxState to satisfy as_str
+    fn as_str(&self) -> String {
+        format!("{:?}", self)
+    }
+
+    /// Simple call to produce a JSON string using the given FeroxState
+    fn as_json(&self) -> String {
+        serde_json::to_string(&self).unwrap_or_default()
+    }
+}
+
+/// Initialize the ctrl+c handler that saves scan state to disk
+pub fn initialize() {
+    log::trace!("enter: initialize");
+
+    let result = ctrlc::set_handler(move || {
+        let ts = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .unwrap()
+            .as_secs();
+
+        let slug = if !CONFIGURATION.target_url.is_empty() {
+            // target url populated
+            CONFIGURATION
+                .target_url
+                .replace("://", "_")
+                .replace("/", "_")
+                .replace(".", "_")
+        } else {
+            // stdin used
+            "stdin".to_string()
+        };
+
+        let filename = format!("ferox-{}-{}.state", slug, ts);
+        let warning = format!(
+            "🚨 Caught {} 🚨 saving scan state to {} ...",
+            style("ctrl+c").yellow(),
+            filename
+        );
+
+        PROGRESS_PRINTER.println(warning);
+
+        let state = FeroxState {
+            config: &CONFIGURATION,
+            scans: &SCANNED_URLS,
+            responses: &RESPONSES,
+        };
+
+        let state_file = open_file(&filename);
+
+        if let Some(buffered_file) = state_file {
+            safe_file_write(&state, buffered_file, true);
+        }
+
+        std::process::exit(1);
+    });
+
+    if result.is_err() {
+        log::error!("Could not set Ctrl+c handler");
+        std::process::exit(1);
+    }
+
+    log::trace!("exit: initialize");
+}
+
+/// Primary logic used to load a Configuration from disk and populate the appropriate data
+/// structures
+pub fn resume_scan(filename: &str) -> Configuration {
+    log::trace!("enter: resume_scan({})", filename);
+
+    let file = File::open(filename).unwrap_or_else(|e| {
+        log::error!("{}", e);
+        log::error!("Could not open state file, exiting");
+        std::process::exit(1);
+    });
+
+    let reader = BufReader::new(file);
+    let state: serde_json::Value = serde_json::from_reader(reader).unwrap();
+
+    let conf = state.get("config").unwrap_or_else(|| {
+        log::error!("Could not load configuration from state file, exiting");
+        std::process::exit(1);
+    });
+
+    let config = serde_json::from_value(conf.clone()).unwrap_or_else(|e| {
+        log::error!("{}", e);
+        log::error!("Could not deserialize configuration found in state file, exiting");
+        std::process::exit(1);
+    });
+
+    // let scans: FeroxScans = serde_json::from_value(state.get("scans").unwrap().clone()).unwrap();
+    if let Some(responses) = state.get("responses") {
+        if let Some(arr_responses) = responses.as_array() {
+            for response in arr_responses {
+                if let Ok(deser_resp) = serde_json::from_value(response.clone()) {
+                    RESPONSES.insert(deser_resp);
+                }
+            }
+        }
+    }
+
+    if let Some(scans) = state.get("scans") {
+        if let Some(arr_scans) = scans.as_array() {
+            for scan in arr_scans {
+                let deser_scan: FeroxScan =
+                    serde_json::from_value(scan.clone()).unwrap_or_default();
+                // need to determine if it's complete and based on that create a progress bar
+                // populate it accordingly based on completion
+                SCANNED_URLS.insert(Arc::new(Mutex::new(deser_scan)));
+            }
+        }
+    }
+
+    log::trace!("exit: resume_scan -> {:?}", config);
+    config
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
+    use predicates::prelude::*;
+
+    #[test]
+    /// test that ScanType's default is File
+    fn default_scantype_is_file() {
+        match ScanType::default() {
+            ScanType::File => {}
+            ScanType::Directory => panic!(),
+        }
+    }
 
     #[test]
     /// test that get_single_spinner returns the correct spinner
@@ -528,4 +851,142 @@ mod tests {
         assert!(scan.progress_bar.is_some()); // new pb created
         assert!(!pb.is_finished()) // not finished
     }
+
+    #[test]
+    /// given a JSON entry representing a FeroxScan, test that it deserializes into the proper type
+    /// with the right attributes
+    fn ferox_scan_deserialize() {
+        let fs_json = r#"{"id":"057016a14769414aac9a7a62707598cb","url":"https://spiritanimal.com","scan_type":"Directory","complete":true}"#;
+        let fs_json_two = r#"{"id":"057016a14769414aac9a7a62707598cb","url":"https://spiritanimal.com","scan_type":"Not Correct","complete":true}"#;
+
+        let fs: FeroxScan = serde_json::from_str(fs_json).unwrap();
+        let fs_two: FeroxScan = serde_json::from_str(fs_json_two).unwrap();
+        assert_eq!(fs.url, "https://spiritanimal.com");
+
+        match fs.scan_type {
+            ScanType::Directory => {}
+            ScanType::File => {
+                panic!();
+            }
+        }
+        match fs_two.scan_type {
+            ScanType::Directory => {
+                panic!();
+            }
+            ScanType::File => {}
+        }
+
+        match fs.progress_bar {
+            None => {}
+            Some(_) => {
+                panic!();
+            }
+        }
+        assert_eq!(fs.complete, true);
+        assert_eq!(fs.id, "057016a14769414aac9a7a62707598cb");
+    }
+
+    #[test]
+    /// given a FeroxScan, test that it serializes into the proper JSON entry
+    fn ferox_scan_serialize() {
+        let fs = FeroxScan::new("https://spiritanimal.com", ScanType::Directory, None);
+        let fs_json = format!(
+            r#"{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","complete":false}}"#,
+            fs.lock().unwrap().id
+        );
+        assert_eq!(
+            fs_json,
+            serde_json::to_string(&*fs.lock().unwrap()).unwrap()
+        );
+    }
+
+    #[test]
+    /// given a FeroxScans, test that it serializes into the proper JSON entry
+    fn ferox_scans_serialize() {
+        let ferox_scan = FeroxScan::new("https://spiritanimal.com", ScanType::Directory, None);
+        let ferox_scans = FeroxScans::default();
+        let ferox_scans_json = format!(
+            r#"[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","complete":false}}]"#,
+            ferox_scan.lock().unwrap().id
+        );
+        ferox_scans.scans.lock().unwrap().push(ferox_scan);
+        assert_eq!(
+            ferox_scans_json,
+            serde_json::to_string(&ferox_scans).unwrap()
+        );
+    }
+
+    #[test]
+    /// given a FeroxResponses, test that it serializes into the proper JSON entry
+    fn ferox_responses_serialize() {
+        let json_response = r#"{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{"server":"nginx/1.16.1"}}"#;
+        let response: FeroxResponse = serde_json::from_str(json_response).unwrap();
+
+        let responses = FeroxResponses::default();
+        responses.insert(response);
+        // responses has a response now
+
+        // serialized should be a list of responses
+        let expected = format!("[{}]", json_response);
+
+        let serialized = serde_json::to_string(&responses).unwrap();
+        assert_eq!(expected, serialized);
+    }
+
+    #[test]
+    /// given a FeroxResponse, test that it serializes into the proper JSON entry
+    fn ferox_response_serialize_and_deserialize() {
+        // deserialize
+        let json_response = r#"{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{"server":"nginx/1.16.1"}}"#;
+        let response: FeroxResponse = serde_json::from_str(json_response).unwrap();
+
+        assert_eq!(response.url.as_str(), "https://nerdcore.com/css");
+        assert_eq!(response.url.path(), "/css");
+        assert_eq!(response.wildcard, true);
+        assert_eq!(response.status.as_u16(), 301);
+        assert_eq!(response.content_length, 173);
+        assert_eq!(response.line_count, 10);
+        assert_eq!(response.word_count, 16);
+        assert_eq!(response.headers.get("server").unwrap(), "nginx/1.16.1");
+
+        // serialize, however, this can fail when headers are out of order
+        let new_json = serde_json::to_string(&response).unwrap();
+        assert_eq!(json_response, new_json);
+    }
+
+    #[test]
+    /// test FeroxSerialize implementation of FeroxState
+    fn feroxstates_feroxserialize_implementation() {
+        let ferox_scan = FeroxScan::new("https://spiritanimal.com", ScanType::Directory, None);
+        let saved_id = ferox_scan.lock().unwrap().id.clone();
+        SCANNED_URLS.insert(ferox_scan);
+
+        let json_response = r#"{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{"server":"nginx/1.16.1"}}"#;
+        let response: FeroxResponse = serde_json::from_str(json_response).unwrap();
+        RESPONSES.insert(response);
+
+        let ferox_state = FeroxState {
+            scans: &SCANNED_URLS,
+            responses: &RESPONSES,
+            config: &CONFIGURATION,
+        };
+
+        let expected_strs = predicates::str::contains("scans: FeroxScans").and(
+            predicate::str::contains("config: Configuration")
+                .and(predicate::str::contains("responses: FeroxResponses"))
+                .and(predicate::str::contains("nerdcore.com"))
+                .and(predicate::str::contains("/css"))
+                .and(predicate::str::contains("https://spiritanimal.com")),
+        );
+
+        assert!(expected_strs.eval(&ferox_state.as_str()));
+
+        let json_state = ferox_state.as_json();
+        let expected = format!(
+            r#"{{"scans":[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","complete":false}}],"config":{{"type":"configuration","wordlist":"/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt","config":"","proxy":"","replay_proxy":"","target_url":"","status_codes":[200,204,301,302,307,308,401,403,405],"replay_codes":[200,204,301,302,307,308,401,403,405],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"quiet":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/1.9.0","redirects":false,"insecure":false,"extensions":[],"headers":{{}},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":false,"depth":4,"scan_limit":0,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false,"resumed":false,"save_state":true}},"responses":[{{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{{"server":"nginx/1.16.1"}}}}]}}"#,
+            saved_id
+        );
+
+        assert!(predicates::str::similar(expected).eval(&json_state));
+    }
 }

src/scanner.rs

@@ -6,7 +6,7 @@ use crate::{
         WordsFilter,
     },
     heuristics,
-    scan_manager::{FeroxScans, PAUSE_SCAN},
+    scan_manager::{FeroxResponses, FeroxScans, PAUSE_SCAN},
     utils::{format_url, get_current_depth, make_request},
     FeroxChannel, FeroxResponse,
 };
@@ -47,6 +47,9 @@ lazy_static! {
     /// Vector of implementors of the FeroxFilter trait
     static ref FILTERS: Arc<RwLock<Vec<Box<dyn FeroxFilter>>>> = Arc::new(RwLock::new(Vec::<Box<dyn FeroxFilter>>::new()));
 
+    /// Vector of FeroxResponse objects
+    pub static ref RESPONSES: FeroxResponses = FeroxResponses::default();
+
     /// Bounded semaphore used as a barrier to limit concurrent scans
     static ref SCAN_LIMITER: Semaphore = Semaphore::new(CONFIGURATION.scan_limit);
 }

tests/test_banner.rs

@@ -711,6 +711,8 @@ fn banner_prints_json() {
         .arg("--url")
         .arg("http://localhost")
         .arg("--json")
+        .arg("--output")
+        .arg("/dev/null")
         .assert()
         .success()
         .stderr(

tests/test_scan_manager.rs

@@ -0,0 +1,93 @@
+mod utils;
+use assert_cmd::Command;
+use httpmock::Method::GET;
+use httpmock::{Mock, MockServer};
+use predicates::prelude::*;
+use utils::{setup_tmp_directory, teardown_tmp_directory};
+
+#[test]
+/// pass a known serialized scan with 1 scan complete and 1 not. expect the incomplete scan to
+/// start and the complete to not start. expect the responses, scans, and configuration structures
+/// to be populated based off the contents of the given state file
+fn resume_scan_works() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) =
+        setup_tmp_directory(&["css".to_string(), "stuff".to_string()], "wordlist").unwrap();
+
+    // localhost:PORT/ <- complete
+    // localhost:PORT/js <- will get scanned with /css and /stuff
+    let complete_scan = format!(
+        r#"{{"id":"057016a14769414aac9a7a62707598cb","url":"{}","scan_type":"Directory","complete":true}}"#,
+        srv.url("/")
+    );
+    let incomplete_scan = format!(
+        r#"{{"id":"400b2323a16f43468a04ffcbbeba34c6","url":"{}","scan_type":"Directory","complete":false}}"#,
+        srv.url("/js")
+    );
+    let scans = format!(r#""scans":[{},{}]"#, complete_scan, incomplete_scan);
+
+    let config = format!(
+        r#""config": {{"type":"configuration","wordlist":"{}","config":"","proxy":"","replay_proxy":"","target_url":"{}","status_codes":[200,204,301,302,307,308,401,403,405],"replay_codes":[200,204,301,302,307,308,401,403,405],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"quiet":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/1.9.0","redirects":false,"insecure":false,"extensions":[],"headers":{{}},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":false,"depth":2,"scan_limit":1,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false}}"#,
+        file.to_string_lossy(),
+        srv.url("/")
+    );
+
+    // // localhost:PORT/js/css has already been seen, expect not to be scanned
+    let response = format!(
+        r#"{{"type":"response","url":"{}","path":"/js/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{{"server":"nginx/1.16.1"}}}}"#,
+        srv.url("/js/css")
+    );
+    let responses = format!(r#""responses":[{}]"#, response);
+
+    // not scanned because /js is not complete, and /js/stuff response is not known
+    let not_scanned_yet = Mock::new()
+        .expect_method(GET)
+        .expect_path("/js/stuff")
+        .return_status(200)
+        .return_body("i expect to be scanned")
+        .create_on(&srv);
+
+    // will get scanned because /js is not complete, but because response of /js/css is known, the
+    // response will not be in stdout
+    let already_scanned = Mock::new()
+        .expect_method(GET)
+        .expect_path("/js/css")
+        .return_status(200)
+        .create_on(&srv);
+
+    // already scanned because scan on / is complete
+    let also_already_scanned = Mock::new()
+        .expect_method(GET)
+        .expect_path("/css")
+        .return_status(200)
+        .return_body("two words")
+        .create_on(&srv);
+
+    let state_file_contents = format!("{{{},{},{}}}", scans, config, responses);
+    let (tmp_dir2, state_file) = setup_tmp_directory(&[state_file_contents], "state-file").unwrap();
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--resume-from")
+        .arg(state_file.as_os_str())
+        .assert()
+        .success()
+        .stdout(
+            predicate::str::contains("/js/stuff")
+                .and(predicate::str::contains("22c"))
+                .and(predicate::str::contains("5w"))
+                .and(predicate::str::contains("/js/css"))
+                .not()
+                .and(predicate::str::contains("2w"))
+                .not()
+                .and(predicate::str::contains("9c"))
+                .not(),
+        );
+
+    teardown_tmp_directory(tmp_dir);
+    teardown_tmp_directory(tmp_dir2);
+
+    assert_eq!(already_scanned.times_called(), 1);
+    assert_eq!(also_already_scanned.times_called(), 0);
+    assert_eq!(not_scanned_yet.times_called(), 1);
+}