Merge pull request #163 from epi052/137-extract-robots-txt

add robots.txt extraction to increase scan coverage

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "feroxbuster"
-version = "1.10.0"
+version = "1.10.2"
 authors = ["Ben 'epi' Risher <epibar052@gmail.com>"]
 license = "MIT"
 edition = "2018"
@@ -34,7 +34,7 @@ serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 uuid = { version = "0.8", features = ["v4"] }
 indicatif = "0.15"
-console = "0.12"
+console = "0.13"
 openssl = { version = "0.10", features = ["vendored"] }
 dirs = "3.0"
 regex = "1"
@@ -44,7 +44,7 @@ ctrlc = "3.1"
 
 [dev-dependencies]
 tempfile = "3.1"
-httpmock = "0.4.5"
+httpmock = "0.5.2"
 assert_cmd = "1.0.1"
 predicates = "1.0.5"
 

README.md

@@ -93,6 +93,9 @@ This attack is also known as Predictable Resource Location, File Enumeration, Di
 - [Common Problems/Issues (FAQ)](#-common-problemsissues-faq)
     - [No file descriptors available](#no-file-descriptors-available)
     - [Progress bars print one line at a time](#progress-bars-print-one-line-at-a-time)
+    - [What do each of the numbers beside the URL mean?](#what-do-each-of-the-numbers-beside-the-url-mean)
+    - [Connection closed before message completed](#connection-closed-before-message-completed)
+    - [SSL Error routines:tls_process_server_certificate:certificate verify failed](#ssl-error-routinestls_process_server_certificatecertificate-verify-failed)
 
 ## 💿 Installation
 
@@ -649,6 +652,12 @@ A valid time_spec can be passed to `--time-limit` in order to force a shutdown a
 
 ![time-limit](img/time-limit.gif)
 
+### Extract Links from robots.txt (New in `v1.10.2`)
+
+In addition to [extracting links from the response body](#extract-links-from-response-body-new-in-v110), using 
+`--extract-links` makes a request to `/robots.txt` and examines all `Allow` and `Disallow` entries.  Directory entries 
+are added to the scan queue, while file entries are requested and then reported if appropriate.  
+
 ## 🧐 Comparison w/ Similar Tools
 
 There are quite a few similar tools for forced browsing/content discovery.  Burp Suite Pro, Dirb, Dirbuster, etc... 
@@ -692,6 +701,7 @@ a few of the use-cases in which feroxbuster may be a better fit:
 | filter out responses by regular expression (`v1.8.0`)                        | ✔ |   | ✔ |
 | save scan's state to disk (can pick up where it left off) (`v1.9.0`)         | ✔ |   |   |
 | maximum run time limit (`v1.10.0`)                                           | ✔ |   | ✔ |
+| use robots.txt to increase scan coverage (`v1.10.2`)                         | ✔ |   |   |
 | **huge** number of other options                                             |   |   | ✔ |
 
 Of note, there's another written-in-rust content discovery tool, [rustbuster](https://github.com/phra/rustbuster). I 
@@ -783,3 +793,27 @@ consider using `-q` to suppress the progress bars.
 ### What do each of the numbers beside the URL mean?
 
 Please refer to [this section](#filter-response-by-word-count--line-count--new-in-v160) where each number's meaning and how to use it to filter responses is discussed.
+
+### Connection closed before message completed
+
+The error in question can be boiled down to 'networking stuff'. `feroxbuster` uses [reqwest](https://docs.rs/reqwest/latest/) which uses [hyper](https://docs.rs/hyper/latest/hyper/) to make requests to the server. [This issue report](https://github.com/hyperium/hyper/issues/2136#issuecomment-589345238) to the hyper project explains what is happening (quoted below to save you a click). This isn't a bug so much as it's a target-specific tuning issue. When lowering the `-t` value, the error doesn't occur (or happens much less frequently).
+
+This isn't a bug. Simply slow down the scan. A `-t` value of 50 was chosen as a sane default that's still quite fast out of the box. However, network related errors may occur when the client and/or server become over-saturated.  The [Threads and Connection Limits At A High-Level](#threads-and-connection-limits-at-a-high-level) section details how to accomplish per-target tuning.
+
+> This is just due to the racy nature of networking.
+> 
+> hyper has a connection pool of idle connections, and it selected one to send your request. Most of the time, hyper will receive the server's FIN and drop the dead connection from its pool. But occasionally, a connection will be selected from the pool and written to at the same time the server is deciding to close the connection. Since hyper already wrote some of the request, it can't really retry it automatically on a new connection, since the server may have acted already.
+
+### SSL Error routines:tls_process_server_certificate:certificate verify failed
+
+In the event you see an error similar to 
+
+![self-signed](img/insecure.png)
+
+```
+error trying to connect: error:1416F086:SSL routines:tls_process_server_certificate:certificate verify failed:ssl/statem/statem_clnt.c:1913: (self signed certificate)
+```
+
+You just need to add the `-k|--insecure` flag to your command.
+
+`feroxbuster` rejects self-signed certs and other "insecure" certificates/site configurations by default. You can choose to scan these services anyway by telling `feroxbuster` to ignore insecure server certs.

build.rs

@@ -5,6 +5,12 @@ use clap::Shell;
 include!("src/parser.rs");
 
 fn main() {
+    println!("cargo:rerun-if-env-changed=src/parser.rs");
+
+    if std::env::var("DOCS_RS").is_ok() {
+        return; // only build when we're not generating docs
+    }
+
     let outdir = "shell_completions";
 
     let mut app = initialize();

src/banner.rs

@@ -1,6 +1,6 @@
 use crate::config::{Configuration, CONFIGURATION};
 use crate::utils::{make_request, status_colorizer};
-use console::style;
+use console::{style, Emoji};
 use reqwest::{Client, Url};
 use serde_json::Value;
 use std::io::Write;
@@ -126,6 +126,14 @@ async fn needs_update(client: &Client, url: &str, bin_version: &str) -> UpdateSt
     unknown
 }
 
+/// Simple wrapper for emoji or fallback when terminal doesn't support emoji
+fn format_emoji(emoji: &str) -> String {
+    let width = console::measure_text_width(emoji);
+    let pad_len = width * width;
+    let pad = format!("{:<pad_len$}", "\u{0020}", pad_len = pad_len);
+    Emoji(emoji, &pad).to_string()
+}
+
 /// Prints the banner to stdout.
 ///
 /// Only prints those settings which are either always present, or passed in by the user.
@@ -138,10 +146,10 @@ where
  ___  ___  __   __     __      __         __   ___
 |__  |__  |__) |__) | /  `    /  \ \_/ | |  \ |__
 |    |___ |  \ |  \ | \__,    \__/ / \ | |__/ |___
-by Ben "epi" Risher {}                  ver: {}"#,
+by Ben "epi" Risher {}                 ver: {}"#,
-        '\u{1F913}', version
+        Emoji("🤓", &format!("{:<2}", "\u{0020}")),
+        version
     );
-
     let status = needs_update(&CONFIGURATION.client, UPDATE_URL, version).await;
 
     let top = "───────────────────────────┬──────────────────────";
@@ -156,7 +164,7 @@ by Ben "epi" Risher {}                  ver: {}"#,
         writeln!(
             &mut writer,
             "{}",
-            format_banner_entry!("\u{1F3af}", "Target Url", target)
+            format_banner_entry!(format_emoji("🎯"), "Target Url", target)
         )
         .unwrap_or_default(); // 🎯
     }
@@ -170,14 +178,14 @@ by Ben "epi" Risher {}                  ver: {}"#,
     writeln!(
         &mut writer,
         "{}",
-        format_banner_entry!("\u{1F680}", "Threads", config.threads)
+        format_banner_entry!(format_emoji("🚀"), "Threads", config.threads)
     )
     .unwrap_or_default(); // 🚀
 
     writeln!(
         &mut writer,
         "{}",
-        format_banner_entry!("\u{1f4d6}", "Wordlist", config.wordlist)
+        format_banner_entry!(format_emoji("📖"), "Wordlist", config.wordlist)
     )
     .unwrap_or_default(); // 📖
 
@@ -185,7 +193,7 @@ by Ben "epi" Risher {}                  ver: {}"#,
         &mut writer,
         "{}",
         format_banner_entry!(
-            "\u{1F197}",
+            format_emoji("🆗"),
             "Status Codes",
             format!("[{}]", codes.join(", "))
         )
@@ -205,7 +213,7 @@ by Ben "epi" Risher {}                  ver: {}"#,
             &mut writer,
             "{}",
             format_banner_entry!(
-                "\u{1f5d1}",
+                format_emoji("🗑"),
                 "Status Code Filters",
                 format!("[{}]", code_filters.join(", "))
             )
@@ -216,14 +224,14 @@ by Ben "epi" Risher {}                  ver: {}"#,
     writeln!(
         &mut writer,
         "{}",
-        format_banner_entry!("\u{1f4a5}", "Timeout (secs)", config.timeout)
+        format_banner_entry!(format_emoji("💥"), "Timeout (secs)", config.timeout)
     )
     .unwrap_or_default(); // 💥
 
     writeln!(
         &mut writer,
         "{}",
-        format_banner_entry!("\u{1F9a1}", "User-Agent", config.user_agent)
+        format_banner_entry!(format_emoji("🦡"), "User-Agent", config.user_agent)
     )
     .unwrap_or_default(); // 🦡
 
@@ -232,7 +240,7 @@ by Ben "epi" Risher {}                  ver: {}"#,
         writeln!(
             &mut writer,
             "{}",
-            format_banner_entry!("\u{1f489}", "Config File", config.config)
+            format_banner_entry!(format_emoji("💉"), "Config File", config.config)
         )
         .unwrap_or_default(); // 💉
     }
@@ -241,7 +249,7 @@ by Ben "epi" Risher {}                  ver: {}"#,
         writeln!(
             &mut writer,
             "{}",
-            format_banner_entry!("\u{1f48e}", "Proxy", config.proxy)
+            format_banner_entry!(format_emoji("💎"), "Proxy", config.proxy)
         )
         .unwrap_or_default(); // 💎
     }
@@ -255,7 +263,7 @@ by Ben "epi" Risher {}                  ver: {}"#,
         writeln!(
             &mut writer,
             "{}",
-            format_banner_entry!("\u{1f3a5}", "Replay Proxy", config.replay_proxy)
+            format_banner_entry!(format_emoji("🎥"), "Replay Proxy", config.replay_proxy)
         )
         .unwrap_or_default(); // 🎥
 
@@ -267,7 +275,7 @@ by Ben "epi" Risher {}                  ver: {}"#,
             &mut writer,
             "{}",
             format_banner_entry!(
-                "\u{1f4fc}",
+                format_emoji("📼"),
                 "Replay Proxy Codes",
                 format!("[{}]", replay_codes.join(", "))
             )
@@ -280,7 +288,7 @@ by Ben "epi" Risher {}                  ver: {}"#,
             writeln!(
                 &mut writer,
                 "{}",
-                format_banner_entry!("\u{1f92f}", "Header", name, value)
+                format_banner_entry!(format_emoji("🤯"), "Header", name, value)
             )
             .unwrap_or_default(); // 🤯
         }
@@ -291,7 +299,7 @@ by Ben "epi" Risher {}                  ver: {}"#,
             writeln!(
                 &mut writer,
                 "{}",
-                format_banner_entry!("\u{1f4a2}", "Size Filter", filter)
+                format_banner_entry!(format_emoji("💢"), "Size Filter", filter)
             )
             .unwrap_or_default(); // 💢
         }
@@ -301,7 +309,7 @@ by Ben "epi" Risher {}                  ver: {}"#,
         writeln!(
             &mut writer,
             "{}",
-            format_banner_entry!("\u{1f4a2}", "Word Count Filter", filter)
+            format_banner_entry!(format_emoji("💢"), "Word Count Filter", filter)
         )
         .unwrap_or_default(); // 💢
     }
@@ -310,7 +318,7 @@ by Ben "epi" Risher {}                  ver: {}"#,
         writeln!(
             &mut writer,
             "{}",
-            format_banner_entry!("\u{1f4a2}", "Line Count Filter", filter)
+            format_banner_entry!(format_emoji("💢"), "Line Count Filter", filter)
         )
         .unwrap_or_default(); // 💢
     }
@@ -319,7 +327,7 @@ by Ben "epi" Risher {}                  ver: {}"#,
         writeln!(
             &mut writer,
             "{}",
-            format_banner_entry!("\u{1f4a2}", "Regex Filter", filter)
+            format_banner_entry!(format_emoji("💢"), "Regex Filter", filter)
         )
         .unwrap_or_default(); // 💢
     }
@@ -328,7 +336,7 @@ by Ben "epi" Risher {}                  ver: {}"#,
         writeln!(
             &mut writer,
             "{}",
-            format_banner_entry!("\u{1F50E}", "Extract Links", config.extract_links)
+            format_banner_entry!(format_emoji("🔎"), "Extract Links", config.extract_links)
         )
         .unwrap_or_default(); // 🔎
     }
@@ -337,7 +345,7 @@ by Ben "epi" Risher {}                  ver: {}"#,
         writeln!(
             &mut writer,
             "{}",
-            format_banner_entry!("\u{1F9d4}", "JSON Output", config.json)
+            format_banner_entry!(format_emoji("🧔"), "JSON Output", config.json)
         )
         .unwrap_or_default(); // 🧔
     }
@@ -348,7 +356,7 @@ by Ben "epi" Risher {}                  ver: {}"#,
                 &mut writer,
                 "{}",
                 format_banner_entry!(
-                    "\u{1f914}",
+                    format_emoji("🤔"),
                     "Query Parameter",
                     format!("{}={}", query.0, query.1)
                 )
@@ -361,7 +369,7 @@ by Ben "epi" Risher {}                  ver: {}"#,
         writeln!(
             &mut writer,
             "{}",
-            format_banner_entry!("\u{1f4be}", "Output File", config.output)
+            format_banner_entry!(format_emoji("💾"), "Output File", config.output)
         )
         .unwrap_or_default(); // 💾
     }
@@ -370,7 +378,7 @@ by Ben "epi" Risher {}                  ver: {}"#,
         writeln!(
             &mut writer,
             "{}",
-            format_banner_entry!("\u{1fab2}", "Debugging Log", config.debug_log)
+            format_banner_entry!(format_emoji("🪲"), "Debugging Log", config.debug_log)
         )
         .unwrap_or_default(); // 🪲
     }
@@ -380,7 +388,7 @@ by Ben "epi" Risher {}                  ver: {}"#,
             &mut writer,
             "{}",
             format_banner_entry!(
-                "\u{1f4b2}",
+                format_emoji("💲"),
                 "Extensions",
                 format!("[{}]", config.extensions.join(", "))
             )
@@ -392,7 +400,7 @@ by Ben "epi" Risher {}                  ver: {}"#,
         writeln!(
             &mut writer,
             "{}",
-            format_banner_entry!("\u{1f513}", "Insecure", config.insecure)
+            format_banner_entry!(format_emoji("🔓"), "Insecure", config.insecure)
         )
         .unwrap_or_default(); // 🔓
     }
@@ -401,7 +409,7 @@ by Ben "epi" Risher {}                  ver: {}"#,
         writeln!(
             &mut writer,
             "{}",
-            format_banner_entry!("\u{1f4cd}", "Follow Redirects", config.redirects)
+            format_banner_entry!(format_emoji("📍"), "Follow Redirects", config.redirects)
         )
         .unwrap_or_default(); // 📍
     }
@@ -410,53 +418,31 @@ by Ben "epi" Risher {}                  ver: {}"#,
         writeln!(
             &mut writer,
             "{}",
-            format_banner_entry!("\u{1f92a}", "Filter Wildcards", !config.dont_filter)
+            format_banner_entry!(format_emoji("🤪"), "Filter Wildcards", !config.dont_filter)
         )
         .unwrap_or_default(); // 🤪
     }
 
-    match config.verbosity {
+    let volume = ["🔈", "🔉", "🔊", "📢"];
+    if let 1..=4 = config.verbosity {
         //speaker medium volume (increasing with verbosity to loudspeaker)
-        1 => {
+        writeln!(
-            writeln!(
+            &mut writer,
-                &mut writer,
+            "{}",
-                "{}",
+            format_banner_entry!(
-                format_banner_entry!("\u{1f508}", "Verbosity", config.verbosity)
+                format_emoji(volume[config.verbosity as usize - 1]),
+                "Verbosity",
+                config.verbosity
             )
-            .unwrap_or_default(); // 🔈
+        )
-        }
+        .unwrap_or_default();
-        2 => {
-            writeln!(
-                &mut writer,
-                "{}",
-                format_banner_entry!("\u{1f509}", "Verbosity", config.verbosity)
-            )
-            .unwrap_or_default(); // 🔉
-        }
-        3 => {
-            writeln!(
-                &mut writer,
-                "{}",
-                format_banner_entry!("\u{1f50a}", "Verbosity", config.verbosity)
-            )
-            .unwrap_or_default(); // 🔊
-        }
-        4 => {
-            writeln!(
-                &mut writer,
-                "{}",
-                format_banner_entry!("\u{1f4e2}", "Verbosity", config.verbosity)
-            )
-            .unwrap_or_default(); // 📢
-        }
-        _ => {}
     }
 
     if config.add_slash {
         writeln!(
             &mut writer,
             "{}",
-            format_banner_entry!("\u{1fa93}", "Add Slash", config.add_slash)
+            format_banner_entry!(format_emoji("🪓"), "Add Slash", config.add_slash)
         )
         .unwrap_or_default(); // 🪓
     }
@@ -466,14 +452,14 @@ by Ben "epi" Risher {}                  ver: {}"#,
             writeln!(
                 &mut writer,
                 "{}",
-                format_banner_entry!("\u{1f503}", "Recursion Depth", "INFINITE")
+                format_banner_entry!(format_emoji("🔃"), "Recursion Depth", "INFINITE")
             )
             .unwrap_or_default(); // 🔃
         } else {
             writeln!(
                 &mut writer,
                 "{}",
-                format_banner_entry!("\u{1f503}", "Recursion Depth", config.depth)
+                format_banner_entry!(format_emoji("🔃"), "Recursion Depth", config.depth)
             )
             .unwrap_or_default(); // 🔃
         }
@@ -481,7 +467,7 @@ by Ben "epi" Risher {}                  ver: {}"#,
         writeln!(
             &mut writer,
             "{}",
-            format_banner_entry!("\u{1f6ab}", "Do Not Recurse", config.no_recursion)
+            format_banner_entry!(format_emoji("🚫"), "Do Not Recurse", config.no_recursion)
         )
         .unwrap_or_default(); // 🚫
     }
@@ -490,7 +476,11 @@ by Ben "epi" Risher {}                  ver: {}"#,
         writeln!(
             &mut writer,
             "{}",
-            format_banner_entry!("\u{1f9a5}", "Concurrent Scan Limit", config.scan_limit)
+            format_banner_entry!(
+                format_emoji("🦥"),
+                "Concurrent Scan Limit",
+                config.scan_limit
+            )
         )
         .unwrap_or_default(); // 🦥
     }
@@ -499,7 +489,7 @@ by Ben "epi" Risher {}                  ver: {}"#,
         writeln!(
             &mut writer,
             "{}",
-            format_banner_entry!("\u{1f556}", "Time Limit", config.time_limit)
+            format_banner_entry!(format_emoji("🕖"), "Time Limit", config.time_limit)
         )
         .unwrap_or_default(); // 🕖
     }
@@ -509,7 +499,7 @@ by Ben "epi" Risher {}                  ver: {}"#,
             &mut writer,
             "{}",
             format_banner_entry!(
-                "\u{1f389}",
+                format_emoji("🎉"),
                 "New Version Available",
                 "https://github.com/epi052/feroxbuster/releases/latest"
             )
@@ -521,12 +511,14 @@ by Ben "epi" Risher {}                  ver: {}"#,
     // ⏯
     writeln!(
         &mut writer,
-        " \u{23ef}   Press [{}] to {}|{} your scan",
+        " {}   Press [{}] to {}|{} your scan",
+        format_emoji("⏯"),
         style("ENTER").yellow(),
         style("pause").red(),
         style("resume").green()
     )
     .unwrap_or_default();
+
     writeln!(&mut writer, "{}", addl_section).unwrap_or_default();
 }
 
@@ -535,7 +527,7 @@ mod tests {
     use super::*;
     use crate::VERSION;
     use httpmock::Method::GET;
-    use httpmock::{Mock, MockServer};
+    use httpmock::MockServer;
     use std::fs::read_to_string;
     use std::io::stderr;
     use std::time::Duration;
@@ -620,16 +612,14 @@ mod tests {
     async fn banner_needs_update_returns_up_to_date() {
         let srv = MockServer::start();
 
-        let mock = Mock::new()
+        let mock = srv.mock(|when, then| {
-            .expect_method(GET)
+            when.method(GET).path("/latest");
-            .expect_path("/latest")
+            then.status(200).body("{\"tag_name\":\"v1.1.0\"}");
-            .return_status(200)
+        });
-            .return_body("{\"tag_name\":\"v1.1.0\"}")
-            .create_on(&srv);
 
         let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.1.0").await;
 
-        assert_eq!(mock.times_called(), 1);
+        assert_eq!(mock.hits(), 1);
         assert!(matches!(result, UpdateStatus::UpToDate));
     }
 
@@ -638,16 +628,14 @@ mod tests {
     async fn banner_needs_update_returns_out_of_date() {
         let srv = MockServer::start();
 
-        let mock = Mock::new()
+        let mock = srv.mock(|when, then| {
-            .expect_method(GET)
+            when.method(GET).path("/latest");
-            .expect_path("/latest")
+            then.status(200).body("{\"tag_name\":\"v1.1.0\"}");
-            .return_status(200)
+        });
-            .return_body("{\"tag_name\":\"v1.1.0\"}")
-            .create_on(&srv);
 
         let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1").await;
 
-        assert_eq!(mock.times_called(), 1);
+        assert_eq!(mock.hits(), 1);
         assert!(matches!(result, UpdateStatus::OutOfDate));
     }
 
@@ -656,17 +644,16 @@ mod tests {
     async fn banner_needs_update_returns_unknown_on_timeout() {
         let srv = MockServer::start();
 
-        let mock = Mock::new()
+        let mock = srv.mock(|when, then| {
-            .expect_method(GET)
+            when.method(GET).path("/latest");
-            .expect_path("/latest")
+            then.status(200)
-            .return_status(200)
+                .body("{\"tag_name\":\"v1.1.0\"}")
-            .return_body("{\"tag_name\":\"v1.1.0\"}")
+                .delay(Duration::from_secs(8));
-            .return_with_delay(Duration::from_secs(8))
+        });
-            .create_on(&srv);
 
         let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1").await;
 
-        assert_eq!(mock.times_called(), 1);
+        assert_eq!(mock.hits(), 1);
         assert!(matches!(result, UpdateStatus::Unknown));
     }
 
@@ -675,16 +662,14 @@ mod tests {
     async fn banner_needs_update_returns_unknown_on_bad_json_response() {
         let srv = MockServer::start();
 
-        let mock = Mock::new()
+        let mock = srv.mock(|when, then| {
-            .expect_method(GET)
+            when.method(GET).path("/latest");
-            .expect_path("/latest")
+            then.status(200).body("not json");
-            .return_status(200)
+        });
-            .return_body("not json")
-            .create_on(&srv);
 
         let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1").await;
 
-        assert_eq!(mock.times_called(), 1);
+        assert_eq!(mock.hits(), 1);
         assert!(matches!(result, UpdateStatus::Unknown));
     }
 
@@ -693,16 +678,15 @@ mod tests {
     async fn banner_needs_update_returns_unknown_on_json_without_correct_tag() {
         let srv = MockServer::start();
 
-        let mock = Mock::new()
+        let mock = srv.mock(|when, then| {
-            .expect_method(GET)
+            when.method(GET).path("/latest");
-            .expect_path("/latest")
+            then.status(200)
-            .return_status(200)
+                .body("{\"no tag_name\": \"doesn't exist\"}");
-            .return_body("{\"no tag_name\": \"doesn't exist\"}")
+        });
-            .create_on(&srv);
 
         let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1").await;
 
-        assert_eq!(mock.times_called(), 1);
+        assert_eq!(mock.hits(), 1);
         assert!(matches!(result, UpdateStatus::Unknown));
     }
 }

src/extractor.rs

@@ -1,4 +1,10 @@
-use crate::FeroxResponse;
+use crate::{
+    client,
+    config::{Configuration, CONFIGURATION},
+    scanner::SCANNED_URLS,
+    utils::{format_url, make_request},
+    FeroxResponse,
+};
 use lazy_static::lazy_static;
 use regex::Regex;
 use reqwest::Url;
@@ -9,9 +15,18 @@ use std::collections::HashSet;
 /// Incorporates change from this [Pull Request](https://github.com/GerbenJavado/LinkFinder/pull/66/files)
 const LINKFINDER_REGEX: &str = r#"(?:"|')(((?:[a-zA-Z]{1,10}://|//)[^"'/]{1,}\.[a-zA-Z]{2,}[^"']{0,})|((?:/|\.\./|\./)[^"'><,;| *()(%%$^/\\\[\]][^"'><,;|()]{1,})|([a-zA-Z0-9_\-/]{1,}/[a-zA-Z0-9_\-/]{1,}\.(?:[a-zA-Z]{1,4}|action)(?:[\?|#][^"|']{0,}|))|([a-zA-Z0-9_\-/]{1,}/[a-zA-Z0-9_\-/]{3,}(?:[\?|#][^"|']{0,}|))|([a-zA-Z0-9_\-.]{1,}\.(?:php|asp|aspx|jsp|json|action|html|js|txt|xml)(?:[\?|#][^"|']{0,}|)))(?:"|')"#;
 
+/// Regular expression to pull url paths from robots.txt
+///
+/// ref: https://developers.google.com/search/reference/robots_txt
+const ROBOTS_TXT_REGEX: &str =
+    r#"(?m)^ *(Allow|Disallow): *(?P<url_path>[a-zA-Z0-9._/?#@!&'()+,;%=-]+?)$"#; // multi-line (?m)
+
 lazy_static! {
     /// `LINKFINDER_REGEX` as a regex::Regex type
-    static ref REGEX: Regex = Regex::new(LINKFINDER_REGEX).unwrap();
+    static ref LINKS_REGEX: Regex = Regex::new(LINKFINDER_REGEX).unwrap();
+
+    /// `ROBOTS_TXT_REGEX` as a regex::Regex type
+    static ref ROBOTS_REGEX: Regex = Regex::new(ROBOTS_TXT_REGEX).unwrap();
 }
 
 /// Iterate over a given path, return a list of every sub-path found
@@ -90,7 +105,7 @@ pub async fn get_links(response: &FeroxResponse) -> HashSet<String> {
 
     let body = response.text();
 
-    for capture in REGEX.captures_iter(&body) {
+    for capture in LINKS_REGEX.captures_iter(&body) {
         // remove single & double quotes from both ends of the capture
         // capture[0] is the entire match, additional capture groups start at [1]
         let link = capture[0].trim_matches(|c| c == '\'' || c == '"');
@@ -105,27 +120,14 @@ pub async fn get_links(response: &FeroxResponse) -> HashSet<String> {
                     continue;
                 }
 
-                for sub_path in get_sub_paths_from_path(absolute.path()) {
+                add_all_sub_paths(absolute.path(), &response, &mut links);
-                    // take a url fragment like homepage/assets/img/icons/handshake.svg and
-                    // incrementally add
-                    //     - homepage/assets/img/icons/
-                    //     - homepage/assets/img/
-                    //     - homepage/assets/
-                    //     - homepage/
-                    log::debug!("Adding {} to {:?}", sub_path, links);
-                    add_link_to_set_of_links(&sub_path, &response.url(), &mut links);
-                }
             }
             Err(e) => {
                 // this is the expected error that happens when we try to parse a url fragment
                 //     ex: Url::parse("/login") -> Err("relative URL without a base")
                 // while this is technically an error, these are good results for us
                 if e.to_string().contains("relative URL without a base") {
-                    for sub_path in get_sub_paths_from_path(link) {
+                    add_all_sub_paths(link, &response, &mut links);
-                        // incrementally save all sub-paths that led to the relative url's resource
-                        log::debug!("Adding {} to {:?}", sub_path, links);
-                        add_link_to_set_of_links(&sub_path, &response.url(), &mut links);
-                    }
                 } else {
                     // unexpected error has occurred
                     log::error!("Could not parse given url: {}", e);
@@ -135,6 +137,152 @@ pub async fn get_links(response: &FeroxResponse) -> HashSet<String> {
     }
 
     log::trace!("exit: get_links -> {:?}", links);
+
+    links
+}
+
+/// take a url fragment like homepage/assets/img/icons/handshake.svg and
+/// incrementally add
+///     - homepage/assets/img/icons/
+///     - homepage/assets/img/
+///     - homepage/assets/
+///     - homepage/
+fn add_all_sub_paths(url_path: &str, response: &FeroxResponse, mut links: &mut HashSet<String>) {
+    log::trace!(
+        "enter: add_all_sub_paths({}, {}, {:?})",
+        url_path,
+        response,
+        links
+    );
+
+    for sub_path in get_sub_paths_from_path(url_path) {
+        log::debug!("Adding {} to {:?}", sub_path, links);
+        add_link_to_set_of_links(&sub_path, &response.url(), &mut links);
+    }
+
+    log::trace!("exit: add_all_sub_paths");
+}
+
+/// Wrapper around link extraction logic
+/// currently used in two places:
+///   - links from response bodys
+///   - links from robots.txt responses
+///
+/// general steps taken:
+///   - create a new Url object based on cli options/args
+///   - check if the new Url has already been seen/scanned -> None
+///   - make a request to the new Url ? -> Some(response) : None
+pub async fn request_feroxresponse_from_new_link(url: &str) -> Option<FeroxResponse> {
+    log::trace!("enter: request_feroxresponse_from_new_link({})", url);
+
+    // create a url based on the given command line options, return None on error
+    let new_url = match format_url(
+        &url,
+        &"",
+        CONFIGURATION.add_slash,
+        &CONFIGURATION.queries,
+        None,
+    ) {
+        Ok(url) => url,
+        Err(_) => {
+            log::trace!("exit: request_feroxresponse_from_new_link -> None");
+            return None;
+        }
+    };
+
+    if SCANNED_URLS.get_scan_by_url(&new_url.to_string()).is_some() {
+        //we've seen the url before and don't need to scan again
+        log::trace!("exit: request_feroxresponse_from_new_link -> None");
+        return None;
+    }
+
+    // make the request and store the response
+    let new_response = match make_request(&CONFIGURATION.client, &new_url).await {
+        Ok(resp) => resp,
+        Err(_) => {
+            log::trace!("exit: request_feroxresponse_from_new_link -> None");
+            return None;
+        }
+    };
+
+    let new_ferox_response = FeroxResponse::from(new_response, true).await;
+
+    log::trace!(
+        "exit: request_feroxresponse_from_new_link -> {:?}",
+        new_ferox_response
+    );
+    Some(new_ferox_response)
+}
+
+/// helper function that simply requests /robots.txt on the given url's base url
+///
+/// example:
+///     http://localhost/api/users -> http://localhost/robots.txt
+///     
+/// The length of the given path has no effect on what's requested; it's always
+/// base url + /robots.txt
+pub async fn request_robots_txt(base_url: &str, config: &Configuration) -> Option<FeroxResponse> {
+    log::trace!("enter: get_robots_file({})", base_url);
+
+    // more often than not, domain/robots.txt will redirect to www.domain/robots.txt or something
+    // similar; to account for that, create a client that will follow redirects, regardless of
+    // what the user specified for the scanning client. Other than redirects, it will respect
+    // all other user specified settings
+    let follow_redirects = true;
+
+    let proxy = if config.proxy.is_empty() {
+        None
+    } else {
+        Some(config.proxy.as_str())
+    };
+
+    let client = client::initialize(
+        config.timeout,
+        &config.user_agent,
+        follow_redirects,
+        config.insecure,
+        &config.headers,
+        proxy,
+    );
+
+    if let Ok(mut url) = Url::parse(base_url) {
+        url.set_path("/robots.txt"); // overwrite existing path with /robots.txt
+
+        if let Ok(response) = make_request(&client, &url).await {
+            let ferox_response = FeroxResponse::from(response, true).await;
+
+            log::trace!("exit: get_robots_file -> {}", ferox_response);
+            return Some(ferox_response);
+        }
+    }
+
+    None
+}
+
+/// Entry point to perform link extraction from robots.txt
+///
+/// `base_url` can have paths and subpaths, however robots.txt will be requested from the
+/// root of the url
+/// given the url:
+///     http://localhost/stuff/things
+/// this function requests:
+///     http://localhost/robots.txt
+pub async fn extract_robots_txt(base_url: &str, config: &Configuration) -> HashSet<String> {
+    log::trace!("enter: extract_robots_txt({}, CONFIGURATION)", base_url);
+    let mut links = HashSet::new();
+
+    if let Some(response) = request_robots_txt(&base_url, &config).await {
+        for capture in ROBOTS_REGEX.captures_iter(response.text.as_str()) {
+            if let Some(new_path) = capture.name("url_path") {
+                if let Ok(mut new_url) = Url::parse(base_url) {
+                    new_url.set_path(new_path.as_str());
+                    add_all_sub_paths(new_url.path(), &response, &mut links);
+                }
+            }
+        }
+    }
+
+    log::trace!("exit: extract_robots_txt -> {:?}", links);
     links
 }
 
@@ -143,7 +291,7 @@ mod tests {
     use super::*;
     use crate::utils::make_request;
     use httpmock::Method::GET;
-    use httpmock::{Mock, MockServer};
+    use httpmock::MockServer;
     use reqwest::Client;
 
     #[test]
@@ -245,12 +393,12 @@ mod tests {
     ) -> Result<(), Box<dyn std::error::Error>> {
         let srv = MockServer::start();
 
-        let mock = Mock::new()
+        let mock = srv.mock(|when, then|{
-            .expect_method(GET)
+            when.method(GET)
-            .expect_path("/some-path")
+                .path("/some-path");
-            .return_status(200)
+            then.status(200)
-            .return_body("\"http://defintely.not.a.thing.probably.com/homepage/assets/img/icons/handshake.svg\"")
+                .body("\"http://defintely.not.a.thing.probably.com/homepage/assets/img/icons/handshake.svg\"");
-            .create_on(&srv);
+        });
 
         let client = Client::new();
         let url = Url::parse(&srv.url("/some-path")).unwrap();
@@ -263,7 +411,30 @@ mod tests {
 
         assert!(links.is_empty());
 
-        assert_eq!(mock.times_called(), 1);
+        assert_eq!(mock.hits(), 1);
         Ok(())
     }
+
+    #[tokio::test(core_threads = 1)]
+    /// test that /robots.txt is correctly requested given a base url (happy path)
+    async fn request_robots_txt_with_and_without_proxy() {
+        let srv = MockServer::start();
+
+        let mock = srv.mock(|when, then| {
+            when.method(GET).path("/robots.txt");
+            then.status(200).body("this is a test");
+        });
+
+        let mut config = Configuration::default();
+
+        request_robots_txt(&srv.url("/api/users/stuff/things"), &config).await;
+
+        // note: the proxy doesn't actually do anything other than hit a different code branch
+        // in this unit test; it would however have an effect on an integration test
+        config.proxy = srv.url("/ima-proxy");
+
+        request_robots_txt(&srv.url("/api/different/path"), &config).await;
+
+        assert_eq!(mock.hits(), 2);
+    }
 }

src/main.rs

@@ -1,11 +1,13 @@
 use crossterm::event::{self, Event, KeyCode};
-use feroxbuster::progress::add_bar;
 use feroxbuster::{
     banner,
     config::{CONFIGURATION, PROGRESS_BAR, PROGRESS_PRINTER},
-    heuristics, logger, reporter,
+    extractor::{extract_robots_txt, request_feroxresponse_from_new_link},
+    heuristics, logger,
+    progress::add_bar,
+    reporter,
     scan_manager::{self, PAUSE_SCAN},
-    scanner::{self, scan_url, RESPONSES, SCANNED_URLS},
+    scanner::{self, scan_url, send_report, RESPONSES, SCANNED_URLS},
     utils::{ferox_print, get_current_depth, module_colorizer, status_colorizer},
     FeroxError, FeroxResponse, FeroxResult, FeroxSerialize, SLEEP_DURATION, VERSION,
 };
@@ -97,7 +99,7 @@ fn get_unique_words_from_wordlist(path: &str) -> FeroxResult<Arc<HashSet<String>
 
 /// Determine whether it's a single url scan or urls are coming from stdin, then scan as needed
 async fn scan(
-    targets: Vec<String>,
+    mut targets: Vec<String>,
     tx_term: UnboundedSender<FeroxResponse>,
     tx_file: UnboundedSender<FeroxResponse>,
 ) -> FeroxResult<()> {
@@ -142,6 +144,37 @@ async fn scan(
         }
     }
 
+    if CONFIGURATION.extract_links {
+        for target in targets.clone() {
+            // modifying the targets vector, so we can't have a reference to it while we borrow
+            // it as mutable; thus the clone
+            let robots_links = extract_robots_txt(&target, &CONFIGURATION).await;
+
+            for robot_link in robots_links {
+                // create a url based on the given command line options, continue on error
+                let ferox_response = match request_feroxresponse_from_new_link(&robot_link).await {
+                    Some(resp) => resp,
+                    None => continue,
+                };
+
+                if ferox_response.is_file() {
+                    SCANNED_URLS.add_file_scan(&robot_link);
+                    send_report(tx_term.clone(), ferox_response);
+                } else {
+                    let (unknown, _) = SCANNED_URLS.add_directory_scan(&robot_link);
+
+                    if !unknown {
+                        // known directory; can skip (unlikely)
+                        continue;
+                    }
+
+                    // unknown directory; add to targets for scanning
+                    targets.push(robot_link);
+                }
+            }
+        }
+    }
+
     let mut tasks = vec![];
 
     for target in targets {

src/scanner.rs

@@ -1,6 +1,6 @@
 use crate::{
     config::{Configuration, CONFIGURATION},
-    extractor::get_links,
+    extractor::{get_links, request_feroxresponse_from_new_link},
     filters::{
         FeroxFilter, LinesFilter, RegexFilter, SizeFilter, StatusCodeFilter, WildcardFilter,
         WordsFilter,
@@ -385,30 +385,11 @@ async fn make_requests(
                 let new_links = get_links(&ferox_response).await;
 
                 for new_link in new_links {
-                    // create a url based on the given command line options, continue on error
+                    let mut new_ferox_response =
-                    let new_url = match format_url(
+                        match request_feroxresponse_from_new_link(&new_link).await {
-                        &new_link,
+                            Some(resp) => resp,
-                        &"",
+                            None => continue,
-                        CONFIGURATION.add_slash,
+                        };
-                        &CONFIGURATION.queries,
-                        None,
-                    ) {
-                        Ok(url) => url,
-                        Err(_) => continue,
-                    };
-
-                    if SCANNED_URLS.get_scan_by_url(&new_url.to_string()).is_some() {
-                        //we've seen the url before and don't need to scan again
-                        continue;
-                    }
-
-                    // make the request and store the response
-                    let new_response = match make_request(&CONFIGURATION.client, &new_url).await {
-                        Ok(resp) => resp,
-                        Err(_) => continue,
-                    };
-
-                    let mut new_ferox_response = FeroxResponse::from(new_response, true).await;
 
                     // filter if necessary
                     if should_filter_response(&new_ferox_response) {
@@ -419,7 +400,7 @@ async fn make_requests(
                         // very likely a file, simply request and report
                         log::debug!("Singular extraction: {}", new_ferox_response);
 
-                        SCANNED_URLS.add_file_scan(&new_url.to_string());
+                        SCANNED_URLS.add_file_scan(&new_ferox_response.url().to_string());
 
                         send_report(report_chan.clone(), new_ferox_response);
 
@@ -452,7 +433,7 @@ async fn make_requests(
 }
 
 /// Simple helper to send a `FeroxResponse` over the tx side of an `mpsc::unbounded_channel`
-fn send_report(report_sender: UnboundedSender<FeroxResponse>, response: FeroxResponse) {
+pub fn send_report(report_sender: UnboundedSender<FeroxResponse>, response: FeroxResponse) {
     log::trace!("enter: send_report({:?}, {}", report_sender, response);
 
     match report_sender.send(response) {

tests/test_extractor.rs

@@ -1,7 +1,7 @@
 mod utils;
 use assert_cmd::prelude::*;
 use httpmock::Method::GET;
-use httpmock::{Mock, MockServer};
+use httpmock::MockServer;
 use predicates::prelude::*;
 use std::process::Command;
 use utils::{setup_tmp_directory, teardown_tmp_directory};
@@ -13,18 +13,17 @@ fn extractor_finds_absolute_url() -> Result<(), Box<dyn std::error::Error>> {
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
+    let mock = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET).path("/LICENSE");
-        .expect_path("/LICENSE")
+        then.status(200)
-        .return_status(200)
+            .body(&srv.url("'/homepage/assets/img/icons/handshake.svg'"));
-        .return_body(&srv.url("'/homepage/assets/img/icons/handshake.svg'"))
+    });
-        .create_on(&srv);
 
-    let mock_two = Mock::new()
+    let mock_two = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET)
-        .expect_path("/homepage/assets/img/icons/handshake.svg")
+            .path("/homepage/assets/img/icons/handshake.svg");
-        .return_status(200)
+        then.status(200);
-        .create_on(&srv);
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -43,8 +42,8 @@ fn extractor_finds_absolute_url() -> Result<(), Box<dyn std::error::Error>> {
             )),
     );
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
-    assert_eq!(mock_two.times_called(), 1);
+    assert_eq!(mock_two.hits(), 1);
     teardown_tmp_directory(tmp_dir);
     Ok(())
 }
@@ -56,12 +55,11 @@ fn extractor_finds_absolute_url_to_different_domain() -> Result<(), Box<dyn std:
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
+    let mock = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET).path("/LICENSE");
-        .expect_path("/LICENSE")
+        then.status(200)
-        .return_status(200)
+            .body("\"http://localhost/homepage/assets/img/icons/handshake.svg\"");
-        .return_body("\"http://localhost/homepage/assets/img/icons/handshake.svg\"")
+    });
-        .create_on(&srv);
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -81,7 +79,7 @@ fn extractor_finds_absolute_url_to_different_domain() -> Result<(), Box<dyn std:
             .not(),
     );
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
     teardown_tmp_directory(tmp_dir);
     Ok(())
 }
@@ -92,18 +90,17 @@ fn extractor_finds_relative_url() -> Result<(), Box<dyn std::error::Error>> {
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
+    let mock = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET).path("/LICENSE");
-        .expect_path("/LICENSE")
+        then.status(200)
-        .return_status(200)
+            .body("\"/homepage/assets/img/icons/handshake.svg\"");
-        .return_body("\"/homepage/assets/img/icons/handshake.svg\"")
+    });
-        .create_on(&srv);
 
-    let mock_two = Mock::new()
+    let mock_two = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET)
-        .expect_path("/homepage/assets/img/icons/handshake.svg")
+            .path("/homepage/assets/img/icons/handshake.svg");
-        .return_status(200)
+        then.status(200);
-        .create_on(&srv);
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -122,8 +119,8 @@ fn extractor_finds_relative_url() -> Result<(), Box<dyn std::error::Error>> {
             )),
     );
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
-    assert_eq!(mock_two.times_called(), 1);
+    assert_eq!(mock_two.hits(), 1);
     teardown_tmp_directory(tmp_dir);
     Ok(())
 }
@@ -136,25 +133,23 @@ fn extractor_finds_same_relative_url_twice() {
     let (tmp_dir, file) =
         setup_tmp_directory(&["LICENSE".to_string(), "README".to_string()], "wordlist").unwrap();
 
-    let mock = Mock::new()
+    let mock = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET).path("/LICENSE");
-        .expect_path("/LICENSE")
+        then.status(200)
-        .return_status(200)
+            .body(&srv.url("\"/homepage/assets/img/icons/handshake.svg\""));
-        .return_body(&srv.url("\"/homepage/assets/img/icons/handshake.svg\""))
+    });
-        .create_on(&srv);
 
-    let mock_two = Mock::new()
+    let mock_two = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET).path("/README");
-        .expect_path("/README")
+        then.status(200)
-        .return_body(&srv.url("\"/homepage/assets/img/icons/handshake.svg\""))
+            .body(&srv.url("\"/homepage/assets/img/icons/handshake.svg\""));
-        .return_status(200)
+    });
-        .create_on(&srv);
 
-    let mock_three = Mock::new()
+    let mock_three = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET)
-        .expect_path("/homepage/assets/img/icons/handshake.svg")
+            .path("/homepage/assets/img/icons/handshake.svg");
-        .return_status(200)
+        then.status(200);
-        .create_on(&srv);
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -168,15 +163,14 @@ fn extractor_finds_same_relative_url_twice() {
     cmd.assert().success().stdout(
         predicate::str::contains("/LICENSE")
             .and(predicate::str::contains("200"))
-            .and(predicate::str::contains(
+            // .count(1) asserts that we only see the endpoint reported once, even though there
-                "/homepage/assets/img/icons/handshake.svg",
+            // is the potential to request the same url twice
-            )),
+            .and(predicate::str::contains("/homepage/assets/img/icons/handshake.svg").count(1)),
     );
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
-    assert_eq!(mock_two.times_called(), 1);
+    assert_eq!(mock_two.hits(), 1);
-    assert!(mock_three.times_called() <= 2); // todo: sometimes this is 2 instead of 1
+    assert!(mock_three.hits() <= 2);
-                                             // the expectation is one, suggesting a race condition... investigate and fix
     teardown_tmp_directory(tmp_dir);
 }
 
@@ -188,19 +182,17 @@ fn extractor_finds_filtered_content() -> Result<(), Box<dyn std::error::Error>>
     let (tmp_dir, file) =
         setup_tmp_directory(&["LICENSE".to_string(), "README".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
+    let mock = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET).path("/LICENSE");
-        .expect_path("/LICENSE")
+        then.status(200)
-        .return_status(200)
+            .body(&srv.url("\"/homepage/assets/img/icons/handshake.svg\""));
-        .return_body(&srv.url("\"/homepage/assets/img/icons/handshake.svg\""))
+    });
-        .create_on(&srv);
 
-    let mock_two = Mock::new()
+    let mock_two = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET)
-        .expect_path("/homepage/assets/img/icons/handshake.svg")
+            .path("/homepage/assets/img/icons/handshake.svg");
-        .return_body("im a little teapot")
+        then.status(200).body("im a little teapot");
-        .return_status(200)
+    });
-        .create_on(&srv);
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -222,8 +214,72 @@ fn extractor_finds_filtered_content() -> Result<(), Box<dyn std::error::Error>>
             .not(),
     );
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
-    assert_eq!(mock_two.times_called(), 1);
+    assert_eq!(mock_two.hits(), 1);
     teardown_tmp_directory(tmp_dir);
     Ok(())
 }
+
+#[test]
+/// serve a robots.txt with a file and and a folder link contained within it. ferox should
+/// find both links and request each one. Additionally, a scan should start with the directory
+/// link found, meaning the wordlist will be thrown at the sub directory
+fn extractor_finds_robots_txt_links_and_displays_files_or_scans_directories() {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
+
+    let mock = srv.mock(|when, then| {
+        when.method(GET).path("/LICENSE");
+        then.status(200).body("im a little teapot"); // 18
+    });
+
+    let mock_two = srv.mock(|when, then| {
+        when.method(GET).path("/robots.txt");
+        then.status(200).body(
+            r#"
+            User-agent: *
+            Crawl-delay: 10
+            # CSS, JS, Images
+            Allow: /misc/*.css$
+            Disallow: /misc/stupidfile.php
+               Disallow: /disallowed-subdir/
+            "#,
+        );
+    });
+
+    let mock_file = srv.mock(|when, then| {
+        when.method(GET).path("/misc/stupidfile.php");
+        then.status(200).body("im a little teapot too"); // 22
+    });
+
+    let mock_dir = srv.mock(|when, then| {
+        when.method(GET).path("/disallowed-subdir/LICENSE");
+        then.status(200).body("i too, am a container for tea"); // 29
+    });
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--extract-links")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE") // 2 directories contain LICENSE
+            .count(2)
+            .and(predicate::str::contains("18c"))
+            .and(predicate::str::contains("/misc/stupidfile.php"))
+            .and(predicate::str::contains("22c"))
+            .and(predicate::str::contains("/disallowed-subdir/LICENSE"))
+            .and(predicate::str::contains("29c"))
+            .and(predicate::str::contains("200").count(3)),
+    );
+
+    assert_eq!(mock.hits(), 1);
+    assert_eq!(mock_dir.hits(), 1);
+    assert_eq!(mock_two.hits(), 1);
+    assert_eq!(mock_file.hits(), 1);
+    teardown_tmp_directory(tmp_dir);
+}

tests/test_filters.rs

@@ -1,7 +1,7 @@
 mod utils;
 use assert_cmd::prelude::*;
 use httpmock::Method::GET;
-use httpmock::{Mock, MockServer};
+use httpmock::MockServer;
 use predicates::prelude::*;
 use std::process::Command;
 use utils::{setup_tmp_directory, teardown_tmp_directory};
@@ -14,19 +14,15 @@ fn filters_status_code_should_filter_response() {
     let (tmp_dir, file) =
         setup_tmp_directory(&["LICENSE".to_string(), "file.js".to_string()], "wordlist").unwrap();
 
-    let mock = Mock::new()
+    let mock = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET).path("/LICENSE");
-        .expect_path("/LICENSE")
+        then.status(302).body("this is a test");
-        .return_status(302)
+    });
-        .return_body("this is a test")
-        .create_on(&srv);
 
-    let mock_two = Mock::new()
+    let mock_two = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET).path("/file.js");
-        .expect_path("/file.js")
+        then.status(200).body("this is also a test of some import");
-        .return_status(200)
+    });
-        .return_body("this is also a test of some import")
-        .create_on(&srv);
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -51,8 +47,8 @@ fn filters_status_code_should_filter_response() {
             .and(predicate::str::contains("34c")),
     );
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
-    assert_eq!(mock_two.times_called(), 1);
+    assert_eq!(mock_two.hits(), 1);
     teardown_tmp_directory(tmp_dir);
 }
 
@@ -64,19 +60,16 @@ fn filters_lines_should_filter_response() {
     let (tmp_dir, file) =
         setup_tmp_directory(&["LICENSE".to_string(), "file.js".to_string()], "wordlist").unwrap();
 
-    let mock = Mock::new()
+    let mock = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET).path("/LICENSE");
-        .expect_path("/LICENSE")
+        then.status(302).body("this is a test");
-        .return_status(302)
+    });
-        .return_body("this is a test")
-        .create_on(&srv);
 
-    let mock_two = Mock::new()
+    let mock_two = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET).path("/file.js");
-        .expect_path("/file.js")
+        then.status(200)
-        .return_status(200)
+            .body("this is also a test of some import\nwith 2 lines, no less");
-        .return_body("this is also a test of some import\nwith 2 lines, no less")
+    });
-        .create_on(&srv);
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -100,8 +93,8 @@ fn filters_lines_should_filter_response() {
             .not(),
     );
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
-    assert_eq!(mock_two.times_called(), 1);
+    assert_eq!(mock_two.hits(), 1);
     teardown_tmp_directory(tmp_dir);
 }
 
@@ -113,19 +106,16 @@ fn filters_words_should_filter_response() {
     let (tmp_dir, file) =
         setup_tmp_directory(&["LICENSE".to_string(), "file.js".to_string()], "wordlist").unwrap();
 
-    let mock = Mock::new()
+    let mock = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET).path("/LICENSE");
-        .expect_path("/LICENSE")
+        then.status(302).body("this is a test");
-        .return_status(302)
+    });
-        .return_body("this is a test")
-        .create_on(&srv);
 
-    let mock_two = Mock::new()
+    let mock_two = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET).path("/file.js");
-        .expect_path("/file.js")
+        then.status(200)
-        .return_status(200)
+            .body("this is also a test of some import\nwith 2 lines, no less");
-        .return_body("this is also a test of some import\nwith 2 lines, no less")
+    });
-        .create_on(&srv);
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -149,8 +139,8 @@ fn filters_words_should_filter_response() {
             .not(),
     );
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
-    assert_eq!(mock_two.times_called(), 1);
+    assert_eq!(mock_two.hits(), 1);
     teardown_tmp_directory(tmp_dir);
 }
 
@@ -162,19 +152,16 @@ fn filters_size_should_filter_response() {
     let (tmp_dir, file) =
         setup_tmp_directory(&["LICENSE".to_string(), "file.js".to_string()], "wordlist").unwrap();
 
-    let mock = Mock::new()
+    let mock = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET).path("/LICENSE");
-        .expect_path("/LICENSE")
+        then.status(302).body("this is a test");
-        .return_status(302)
+    });
-        .return_body("this is a test")
-        .create_on(&srv);
 
-    let mock_two = Mock::new()
+    let mock_two = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET).path("/file.js");
-        .expect_path("/file.js")
+        then.status(200)
-        .return_status(200)
+            .body("this is also a test of some import\nwith 2 lines, no less");
-        .return_body("this is also a test of some import\nwith 2 lines, no less")
+    });
-        .create_on(&srv);
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -198,7 +185,7 @@ fn filters_size_should_filter_response() {
             .not(),
     );
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
-    assert_eq!(mock_two.times_called(), 1);
+    assert_eq!(mock_two.hits(), 1);
     teardown_tmp_directory(tmp_dir);
 }

tests/test_heuristics.rs

@@ -2,7 +2,7 @@ mod utils;
 use assert_cmd::prelude::*;
 use assert_cmd::Command;
 use httpmock::Method::GET;
-use httpmock::{Mock, MockServer, Regex};
+use httpmock::{MockServer, Regex};
 use predicates::prelude::*;
 use utils::{setup_tmp_directory, teardown_tmp_directory};
 
@@ -65,12 +65,10 @@ fn test_one_good_and_one_bad_target_scan_succeeds() -> Result<(), Box<dyn std::e
     let urls = vec![not_real, srv.url("/"), String::from("LICENSE")];
     let (tmp_dir, file) = setup_tmp_directory(&urls, "wordlist")?;
 
-    let mock = Mock::new()
+    let mock = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET).path("/LICENSE");
-        .expect_path("/LICENSE")
+        then.status(200).body("this is a test");
-        .return_status(200)
+    });
-        .return_body("this is a test")
-        .create_on(&srv);
 
     let mut cmd = Command::cargo_bin("feroxbuster").unwrap();
 
@@ -86,7 +84,7 @@ fn test_one_good_and_one_bad_target_scan_succeeds() -> Result<(), Box<dyn std::e
                 .and(predicate::str::contains("200"))
                 .and(predicate::str::contains("14")),
         );
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
 
     teardown_tmp_directory(tmp_dir);
     Ok(())
@@ -98,12 +96,11 @@ fn test_static_wildcard_request_found() -> Result<(), Box<dyn std::error::Error>
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
+    let mock = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET)
-        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap())
+            .path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap());
-        .return_status(200)
+        then.status(200).body("this is a test");
-        .return_body("this is a test")
+    });
-        .create_on(&srv);
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -123,7 +120,7 @@ fn test_static_wildcard_request_found() -> Result<(), Box<dyn std::error::Error>
             .and(predicate::str::contains("(url length: 32)")),
     );
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
     Ok(())
 }
 
@@ -134,19 +131,17 @@ fn test_dynamic_wildcard_request_found() {
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
     let outfile = tmp_dir.path().join("outfile");
 
-    let mock = Mock::new()
+    let mock = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET)
-        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap())
+            .path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap());
-        .return_status(200)
+        then.status(200)
-        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
+            .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
-        .create_on(&srv);
+    });
 
-    let mock2 = Mock::new()
+    let mock2 = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET).path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap());
-        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap())
+        then.status(200).body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
-        .return_status(200)
+    });
-        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
-        .create_on(&srv);
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -177,8 +172,8 @@ fn test_dynamic_wildcard_request_found() {
             .and(predicate::str::contains("(url length: 96)")),
     );
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
-    assert_eq!(mock2.times_called(), 1);
+    assert_eq!(mock2.hits(), 1);
 }
 
 #[test]
@@ -187,12 +182,11 @@ fn heuristics_static_wildcard_request_with_dont_filter() -> Result<(), Box<dyn s
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
+    let mock = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET)
-        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap())
+            .path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap());
-        .return_status(200)
+        then.status(200).body("this is a test");
-        .return_body("this is a test")
+    });
-        .create_on(&srv);
 
     Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -205,7 +199,7 @@ fn heuristics_static_wildcard_request_with_dont_filter() -> Result<(), Box<dyn s
 
     teardown_tmp_directory(tmp_dir);
 
-    assert_eq!(mock.times_called(), 0);
+    assert_eq!(mock.hits(), 0);
     Ok(())
 }
 
@@ -215,19 +209,19 @@ fn heuristics_wildcard_test_with_two_static_wildcards() {
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
 
-    let mock = Mock::new()
+    let mock = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET)
-        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap())
+            .path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap());
-        .return_status(200)
+        then.status(200)
-        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
+            .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
-        .create_on(&srv);
+    });
 
-    let mock2 = Mock::new()
+    let mock2 = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET)
-        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap())
+            .path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap());
-        .return_status(200)
+        then.status(200)
-        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
+            .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
-        .create_on(&srv);
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -251,8 +245,8 @@ fn heuristics_wildcard_test_with_two_static_wildcards() {
             )),
     );
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
-    assert_eq!(mock2.times_called(), 1);
+    assert_eq!(mock2.hits(), 1);
 }
 
 #[test]
@@ -262,19 +256,19 @@ fn heuristics_wildcard_test_with_two_static_wildcards_with_quiet_enabled(
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
+    let mock = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET)
-        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap())
+            .path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap());
-        .return_status(200)
+        then.status(200)
-        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
+            .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
-        .create_on(&srv);
+    });
 
-    let mock2 = Mock::new()
+    let mock2 = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET)
-        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap())
+            .path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap());
-        .return_status(200)
+        then.status(200)
-        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
+            .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
-        .create_on(&srv);
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -290,8 +284,8 @@ fn heuristics_wildcard_test_with_two_static_wildcards_with_quiet_enabled(
 
     cmd.assert().success().stdout(predicate::str::is_empty());
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
-    assert_eq!(mock2.times_called(), 1);
+    assert_eq!(mock2.hits(), 1);
     Ok(())
 }
 
@@ -302,19 +296,19 @@ fn heuristics_wildcard_test_with_two_static_wildcards_and_output_to_file() {
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
     let outfile = tmp_dir.path().join("outfile");
 
-    let mock = Mock::new()
+    let mock = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET)
-        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap())
+            .path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap());
-        .return_status(200)
+        then.status(200)
-        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
+            .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
-        .create_on(&srv);
+    });
 
-    let mock2 = Mock::new()
+    let mock2 = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET)
-        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap())
+            .path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap());
-        .return_status(200)
+        then.status(200)
-        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
+            .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
-        .create_on(&srv);
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -348,8 +342,8 @@ fn heuristics_wildcard_test_with_two_static_wildcards_and_output_to_file() {
             )),
     );
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
-    assert_eq!(mock2.times_called(), 1);
+    assert_eq!(mock2.hits(), 1);
 }
 
 #[test]
@@ -361,20 +355,20 @@ fn heuristics_wildcard_test_with_redirect_as_response_code(
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
     let outfile = tmp_dir.path().join("outfile");
 
-    let mock = Mock::new()
+    let mock = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET)
-        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap())
+            .path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap());
-        .return_status(301)
+        then.status(301)
-        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
+            .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
-        .create_on(&srv);
+    });
 
-    let mock2 = Mock::new()
+    let mock2 = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET)
-        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap())
+            .path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap());
-        .return_status(301)
+        then.status(301)
-        .return_header("Location", &srv.url("/some-redirect"))
+            .header("Location", &srv.url("/some-redirect"))
-        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
+            .body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
-        .create_on(&srv);
+    });
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -407,7 +401,7 @@ fn heuristics_wildcard_test_with_redirect_as_response_code(
             .and(predicate::str::contains("WLD")),
     );
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
-    assert_eq!(mock2.times_called(), 1);
+    assert_eq!(mock2.hits(), 1);
     Ok(())
 }

tests/test_main.rs

@@ -1,7 +1,7 @@
 mod utils;
 use assert_cmd::Command;
 use httpmock::Method::GET;
-use httpmock::{Mock, MockServer};
+use httpmock::MockServer;
 use predicates::prelude::*;
 use utils::{setup_tmp_directory, teardown_tmp_directory};
 
@@ -10,12 +10,10 @@ use utils::{setup_tmp_directory, teardown_tmp_directory};
 fn main_use_root_owned_file_as_wordlist() -> Result<(), Box<dyn std::error::Error>> {
     let srv = MockServer::start();
 
-    let mock = Mock::new()
+    let mock = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET).path("/");
-        .expect_path("/")
+        then.status(200).body("this is a test");
-        .return_status(200)
+    });
-        .return_body("this is a test")
-        .create_on(&srv);
 
     Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -29,7 +27,7 @@ fn main_use_root_owned_file_as_wordlist() -> Result<(), Box<dyn std::error::Erro
         .stdout(predicate::str::contains("Permission denied (os error 13)"));
 
     // connectivity test hits it once
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
     Ok(())
 }
 
@@ -39,12 +37,10 @@ fn main_use_empty_wordlist() -> Result<(), Box<dyn std::error::Error>> {
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&[], "wordlist")?;
 
-    let mock = Mock::new()
+    let mock = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET).path("/");
-        .expect_path("/")
+        then.status(200).body("this is a test");
-        .return_status(200)
+    });
-        .return_body("this is a test")
-        .create_on(&srv);
 
     Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -57,7 +53,7 @@ fn main_use_empty_wordlist() -> Result<(), Box<dyn std::error::Error>> {
         .failure()
         .stdout(predicate::str::contains("Did not find any words in"));
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
 
     teardown_tmp_directory(tmp_dir);
     Ok(())

tests/test_scan_manager.rs

@@ -1,7 +1,7 @@
 mod utils;
 use assert_cmd::Command;
 use httpmock::Method::GET;
-use httpmock::{Mock, MockServer};
+use httpmock::MockServer;
 use predicates::prelude::*;
 use std::fs::{read_to_string, write};
 use std::path::Path;
@@ -43,28 +43,23 @@ fn resume_scan_works() {
     let responses = format!(r#""responses":[{}]"#, response);
 
     // not scanned because /js is not complete, and /js/stuff response is not known
-    let not_scanned_yet = Mock::new()
+    let not_scanned_yet = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET).path("/js/stuff");
-        .expect_path("/js/stuff")
+        then.status(200).body("i expect to be scanned");
-        .return_status(200)
+    });
-        .return_body("i expect to be scanned")
-        .create_on(&srv);
 
     // will get scanned because /js is not complete, but because response of /js/css is known, the
     // response will not be in stdout
-    let already_scanned = Mock::new()
+    let already_scanned = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET).path("/js/css");
-        .expect_path("/js/css")
+        then.status(200);
-        .return_status(200)
+    });
-        .create_on(&srv);
 
     // already scanned because scan on / is complete
-    let also_already_scanned = Mock::new()
+    let also_already_scanned = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET).path("/css");
-        .expect_path("/css")
+        then.status(200).body("two words");
-        .return_status(200)
+    });
-        .return_body("two words")
-        .create_on(&srv);
 
     let state_file_contents = format!("{{{},{},{}}}", scans, config, responses);
     let (tmp_dir2, state_file) = setup_tmp_directory(&[state_file_contents], "state-file").unwrap();
@@ -90,9 +85,9 @@ fn resume_scan_works() {
     teardown_tmp_directory(tmp_dir);
     teardown_tmp_directory(tmp_dir2);
 
-    assert_eq!(already_scanned.times_called(), 1);
+    assert_eq!(already_scanned.hits(), 1);
-    assert_eq!(also_already_scanned.times_called(), 0);
+    assert_eq!(also_already_scanned.hits(), 0);
-    assert_eq!(not_scanned_yet.times_called(), 1);
+    assert_eq!(not_scanned_yet.hits(), 1);
 }
 
 #[test]

tests/test_scanner.rs

@@ -1,7 +1,7 @@
 mod utils;
 use assert_cmd::prelude::*;
 use httpmock::Method::GET;
-use httpmock::{Mock, MockServer};
+use httpmock::MockServer;
 use predicates::prelude::*;
 use std::process::Command;
 use utils::{setup_tmp_directory, teardown_tmp_directory};
@@ -12,12 +12,10 @@ fn scanner_single_request_scan() -> Result<(), Box<dyn std::error::Error>> {
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
+    let mock = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET).path("/LICENSE");
-        .expect_path("/LICENSE")
+        then.status(200).body("this is a test");
-        .return_status(200)
+    });
-        .return_body("this is a test")
-        .create_on(&srv);
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -34,7 +32,7 @@ fn scanner_single_request_scan() -> Result<(), Box<dyn std::error::Error>> {
             .and(predicate::str::contains("14")),
     );
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
     teardown_tmp_directory(tmp_dir);
     Ok(())
 }
@@ -51,33 +49,26 @@ fn scanner_recursive_request_scan() -> Result<(), Box<dyn std::error::Error>> {
     ];
     let (tmp_dir, file) = setup_tmp_directory(&urls, "wordlist")?;
 
-    let js_mock = Mock::new()
+    let js_mock = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET).path("/js");
-        .expect_path("/js")
+        then.status(301).header("Location", &srv.url("/js/"));
-        .return_status(301)
+    });
-        .return_header("Location", &srv.url("/js/"))
-        .create_on(&srv);
 
-    let js_prod_mock = Mock::new()
+    let js_prod_mock = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET).path("/js/prod");
-        .expect_path("/js/prod")
+        then.status(301).header("Location", &srv.url("/js/prod/"));
-        .return_status(301)
+    });
-        .return_header("Location", &srv.url("/js/prod/"))
-        .create_on(&srv);
 
-    let js_dev_mock = Mock::new()
+    let js_dev_mock = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET).path("/js/dev");
-        .expect_path("/js/dev")
+        then.status(301).header("Location", &srv.url("/js/dev/"));
-        .return_status(301)
+    });
-        .return_header("Location", &srv.url("/js/dev/"))
-        .create_on(&srv);
 
-    let js_dev_file_mock = Mock::new()
+    let js_dev_file_mock = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET).path("/js/dev/file.js");
-        .expect_path("/js/dev/file.js")
+        then.status(200)
-        .return_status(200)
+            .body("this is a test and is more bytes than other ones");
-        .return_body("this is a test and is more bytes than other ones")
+    });
-        .create_on(&srv);
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -98,10 +89,10 @@ fn scanner_recursive_request_scan() -> Result<(), Box<dyn std::error::Error>> {
             .and(predicate::str::is_match("200.*js/dev/file.js").unwrap()),
     );
 
-    assert_eq!(js_mock.times_called(), 1);
+    assert_eq!(js_mock.hits(), 1);
-    assert_eq!(js_prod_mock.times_called(), 1);
+    assert_eq!(js_prod_mock.hits(), 1);
-    assert_eq!(js_dev_mock.times_called(), 1);
+    assert_eq!(js_dev_mock.hits(), 1);
-    assert_eq!(js_dev_file_mock.times_called(), 1);
+    assert_eq!(js_dev_file_mock.hits(), 1);
 
     teardown_tmp_directory(tmp_dir);
 
@@ -121,33 +112,26 @@ fn scanner_recursive_request_scan_using_only_success_responses(
     ];
     let (tmp_dir, file) = setup_tmp_directory(&urls, "wordlist")?;
 
-    let js_mock = Mock::new()
+    let js_mock = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET).path("/js/");
-        .expect_path("/js/")
+        then.status(200).header("Location", &srv.url("/js/"));
-        .return_status(200)
+    });
-        .return_header("Location", &srv.url("/js/"))
-        .create_on(&srv);
 
-    let js_prod_mock = Mock::new()
+    let js_prod_mock = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET).path("/js/prod/");
-        .expect_path("/js/prod/")
+        then.status(200).header("Location", &srv.url("/js/prod/"));
-        .return_status(200)
+    });
-        .return_header("Location", &srv.url("/js/prod/"))
-        .create_on(&srv);
 
-    let js_dev_mock = Mock::new()
+    let js_dev_mock = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET).path("/js/dev/");
-        .expect_path("/js/dev/")
+        then.status(200).header("Location", &srv.url("/js/dev/"));
-        .return_status(200)
+    });
-        .return_header("Location", &srv.url("/js/dev/"))
-        .create_on(&srv);
 
-    let js_dev_file_mock = Mock::new()
+    let js_dev_file_mock = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET).path("/js/dev/file.js");
-        .expect_path("/js/dev/file.js")
+        then.status(200)
-        .return_status(200)
+            .body("this is a test and is more bytes than other ones");
-        .return_body("this is a test and is more bytes than other ones")
+    });
-        .create_on(&srv);
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -169,10 +153,10 @@ fn scanner_recursive_request_scan_using_only_success_responses(
             .and(predicate::str::is_match("200.*js/dev/file.js").unwrap()),
     );
 
-    assert_eq!(js_mock.times_called(), 1);
+    assert_eq!(js_mock.hits(), 1);
-    assert_eq!(js_prod_mock.times_called(), 1);
+    assert_eq!(js_prod_mock.hits(), 1);
-    assert_eq!(js_dev_mock.times_called(), 1);
+    assert_eq!(js_dev_mock.hits(), 1);
-    assert_eq!(js_dev_file_mock.times_called(), 1);
+    assert_eq!(js_dev_file_mock.hits(), 1);
 
     teardown_tmp_directory(tmp_dir);
 
@@ -185,12 +169,10 @@ fn scanner_single_request_scan_with_file_output() -> Result<(), Box<dyn std::err
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
+    let mock = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET).path("/LICENSE");
-        .expect_path("/LICENSE")
+        then.status(200).body("this is a test");
-        .return_status(200)
+    });
-        .return_body("this is a test")
-        .create_on(&srv);
 
     let outfile = tmp_dir.path().join("output");
 
@@ -211,7 +193,7 @@ fn scanner_single_request_scan_with_file_output() -> Result<(), Box<dyn std::err
     assert!(contents.contains("200"));
     assert!(contents.contains("14"));
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
     teardown_tmp_directory(tmp_dir);
     Ok(())
 }
@@ -223,12 +205,10 @@ fn scanner_single_request_scan_with_file_output_and_tack_q(
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
+    let mock = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET).path("/LICENSE");
-        .expect_path("/LICENSE")
+        then.status(200).body("this is a test");
-        .return_status(200)
+    });
-        .return_body("this is a test")
-        .create_on(&srv);
 
     let outfile = tmp_dir.path().join("output");
 
@@ -249,7 +229,7 @@ fn scanner_single_request_scan_with_file_output_and_tack_q(
     let url = srv.url("/LICENSE");
     assert!(contents.contains(&url));
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
     teardown_tmp_directory(tmp_dir);
     Ok(())
 }
@@ -261,12 +241,10 @@ fn scanner_single_request_scan_with_invalid_file_output() -> Result<(), Box<dyn
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
+    let mock = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET).path("/LICENSE");
-        .expect_path("/LICENSE")
+        then.status(200).body("this is a test");
-        .return_status(200)
+    });
-        .return_body("this is a test")
-        .create_on(&srv);
 
     let outfile = tmp_dir.path(); // outfile is a directory
 
@@ -285,7 +263,7 @@ fn scanner_single_request_scan_with_invalid_file_output() -> Result<(), Box<dyn
     let contents = std::fs::read_to_string(outfile);
     assert!(contents.is_err());
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
     teardown_tmp_directory(tmp_dir);
     Ok(())
 }
@@ -296,12 +274,10 @@ fn scanner_single_request_quiet_scan() -> Result<(), Box<dyn std::error::Error>>
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
+    let mock = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET).path("/LICENSE");
-        .expect_path("/LICENSE")
+        then.status(200).body("this is a test");
-        .return_status(200)
+    });
-        .return_body("this is a test")
-        .create_on(&srv);
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -321,7 +297,7 @@ fn scanner_single_request_quiet_scan() -> Result<(), Box<dyn std::error::Error>>
             .not(),
     );
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
     teardown_tmp_directory(tmp_dir);
     Ok(())
 }
@@ -334,12 +310,10 @@ fn scanner_single_request_returns_301_without_location_header(
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
+    let mock = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET).path("/LICENSE");
-        .expect_path("/LICENSE")
+        then.status(301).body("this is a test");
-        .return_body("this is a test")
+    });
-        .return_status(301)
-        .create_on(&srv);
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -359,7 +333,7 @@ fn scanner_single_request_returns_301_without_location_header(
             .and(predicate::str::contains("14")),
     );
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
     teardown_tmp_directory(tmp_dir);
     Ok(())
 }
@@ -372,19 +346,15 @@ fn scanner_single_request_replayed_to_proxy() -> Result<(), Box<dyn std::error::
     let proxy = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
+    let mock = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET).path("/LICENSE");
-        .expect_path("/LICENSE")
+        then.status(200).body("this is a test");
-        .return_status(200)
+    });
-        .return_body("this is a test")
-        .create_on(&srv);
 
-    let mock_two = Mock::new()
+    let mock_two = proxy.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET).path("/LICENSE");
-        .expect_path("/LICENSE")
+        then.status(200).body("this is a test");
-        .return_status(200)
+    });
-        .return_body("this is a test")
-        .create_on(&proxy);
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -407,8 +377,8 @@ fn scanner_single_request_replayed_to_proxy() -> Result<(), Box<dyn std::error::
         )
         .stderr(predicate::str::contains("Replay Proxy Codes"));
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
-    assert_eq!(mock_two.times_called(), 1);
+    assert_eq!(mock_two.hits(), 1);
     teardown_tmp_directory(tmp_dir);
     Ok(())
 }
@@ -420,19 +390,15 @@ fn scanner_single_request_scan_with_filtered_result() -> Result<(), Box<dyn std:
     let (tmp_dir, file) =
         setup_tmp_directory(&["LICENSE".to_string(), "ignored".to_string()], "wordlist")?;
 
-    let mock = Mock::new()
+    let mock = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET).path("/LICENSE");
-        .expect_path("/LICENSE")
+        then.status(200).body("this is a not a test");
-        .return_status(200)
+    });
-        .return_body("this is a not a test")
-        .create_on(&srv);
 
-    let filtered_mock = Mock::new()
+    let filtered_mock = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET).path("/ignored");
-        .expect_path("/ignored")
+        then.status(200).body("this is a test");
-        .return_status(200)
+    });
-        .return_body("this is a test")
-        .create_on(&srv);
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -455,8 +421,8 @@ fn scanner_single_request_scan_with_filtered_result() -> Result<(), Box<dyn std:
             .not(),
     );
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
-    assert_eq!(filtered_mock.times_called(), 1);
+    assert_eq!(filtered_mock.hits(), 1);
     teardown_tmp_directory(tmp_dir);
     Ok(())
 }
@@ -467,12 +433,10 @@ fn scanner_single_request_scan_with_debug_logging() {
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
 
-    let mock = Mock::new()
+    let mock = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET).path("/LICENSE");
-        .expect_path("/LICENSE")
+        then.status(200).body("this is a test");
-        .return_status(200)
+    });
-        .return_body("this is a test")
-        .create_on(&srv);
 
     let outfile = tmp_dir.path().join("debug.log");
 
@@ -496,7 +460,7 @@ fn scanner_single_request_scan_with_debug_logging() {
     assert!(contents.contains("feroxbuster All scans complete!"));
     assert!(contents.contains("feroxbuster exit: terminal_input_handler"));
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
     teardown_tmp_directory(tmp_dir);
 }
 
@@ -506,12 +470,10 @@ fn scanner_single_request_scan_with_debug_logging_as_json() {
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist").unwrap();
 
-    let mock = Mock::new()
+    let mock = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET).path("/LICENSE");
-        .expect_path("/LICENSE")
+        then.status(200).body("this is a test");
-        .return_status(200)
+    });
-        .return_body("this is a test")
-        .create_on(&srv);
 
     let outfile = tmp_dir.path().join("debug.log");
 
@@ -538,7 +500,7 @@ fn scanner_single_request_scan_with_debug_logging_as_json() {
     assert!(contents.contains("All scans complete!"));
     assert!(contents.contains("exit: terminal_input_handler"));
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
     teardown_tmp_directory(tmp_dir);
 }
 
@@ -549,19 +511,16 @@ fn scanner_single_request_scan_with_regex_filtered_result() {
     let (tmp_dir, file) =
         setup_tmp_directory(&["LICENSE".to_string(), "ignored".to_string()], "wordlist").unwrap();
 
-    let mock = Mock::new()
+    let mock = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET).path("/LICENSE");
-        .expect_path("/LICENSE")
+        then.status(200).body("this is a test");
-        .return_status(200)
+    });
-        .return_body("this is a not a test")
-        .create_on(&srv);
 
-    let filtered_mock = Mock::new()
+    let filtered_mock = srv.mock(|when, then| {
-        .expect_method(GET)
+        when.method(GET).path("/ignored");
-        .expect_path("/ignored")
+        then.status(200)
-        .return_status(200)
+            .body("this is a test\nThat rug really tied the room together");
-        .return_body("this is a test\nThat rug really tied the room together")
+    });
-        .create_on(&srv);
 
     let cmd = Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -583,7 +542,7 @@ fn scanner_single_request_scan_with_regex_filtered_result() {
             .not(),
     );
 
-    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock.hits(), 1);
-    assert_eq!(filtered_mock.times_called(), 1);
+    assert_eq!(filtered_mock.hits(), 1);
     teardown_tmp_directory(tmp_dir);
 }