Merge pull request #90 from epi052/78-check-for-updates-on-startup

feroxbuster now checks for updates on startup

.github/actions-rs/grcov.yml

@@ -1,7 +1,8 @@
-branch: true
+branch: false
 ignore-not-existing: true
 llvm: true
 output-type: lcov
 output-path: ./lcov.info
+# excl-br-line: "^\\s*((debug_)?assert(_eq|_ne)?!|#\\[derive\\(|log::)"
 ignore:
   - "../*"

.github/workflows/build.yml

@@ -41,10 +41,19 @@ jobs:
           use-cross: true
           command: build
           args: --release --target=${{ matrix.target }}
+      - name: Build tar.gz for homebrew installs
+        if: matrix.type == 'ubuntu-x64'
+        run: |
+          tar czf ${{ matrix.name }}.tar.gz -C target/x86_64-unknown-linux-musl/release feroxbuster
       - uses: actions/upload-artifact@v2
         with:
           name: ${{ matrix.name }}
           path: ${{ matrix.path }}
+      - uses: actions/upload-artifact@v2
+        if: matrix.type == 'ubuntu-x64'
+        with:
+          name: ${{ matrix.name }}.tar.gz
+          path: ${{ matrix.name }}.tar.gz
 
   build-deb:
     needs: [build-nix]

.github/workflows/coverage.yml

@@ -20,8 +20,8 @@ jobs:
           args: --all-features --no-fail-fast
         env:
           CARGO_INCREMENTAL: '0'
-          RUSTFLAGS: '-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Cpanic=abort -Zpanic_abort_tests'
-          RUSTDOCFLAGS: '-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Cpanic=abort -Zpanic_abort_tests'
+          RUSTFLAGS: '-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort'
+          RUSTDOCFLAGS: '-Cpanic=abort'
       - uses: actions-rs/grcov@v0.1
       - name: Convert lcov to xml
         run: |

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "feroxbuster"
-version = "1.0.3"
+version = "1.1.1"
 authors = ["Ben 'epi' Risher <epibar052@gmail.com>"]
 license = "MIT"
 edition = "2018"
@@ -25,11 +25,13 @@ clap = "2"
 lazy_static = "1.4"
 toml = "0.5"
 serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
 uuid = { version = "0.8", features = ["v4"] }
 indicatif = "0.15"
 console = "0.12"
 openssl = { version = "0.10", features = ["vendored"] }
 dirs = "3.0"
+regex = "1"
 
 [dev-dependencies]
 tempfile = "3.1"

README.md

@@ -59,19 +59,20 @@ This attack is also known as Predictable Resource Location, File Enumeration, Di
 
 📖 Table of Contents
 -----------------
-- [Downloads](#-downloads)
 - [Installation](#-installation)
     - [Download a Release](#download-a-release)
     - [Homebrew on MacOS and Linux](#homebrew-on-macos-and-linux)
     - [Cargo Install](#cargo-install)
     - [apt Install](#apt-install)
+    - [AUR Install](#aur-install)
     - [Docker Install](#docker-install)
-- [Configuration](#-configuration)
+- [Configuration](#%EF%B8%8F-configuration)
     - [Default Values](#default-values)
     - [ferox-config.toml](#ferox-configtoml)
     - [Command Line Parsing](#command-line-parsing)
 - [Example Usage](#-example-usage)
     - [Multiple Values](#multiple-values)
+    - [Extract Links from Response Body (new in `v1.1.0`)](#extract-links-from-response-body-new-in-v110)
     - [Include Headers](#include-headers)
     - [IPv6, Non-recursive scan with INFO logging enabled](#ipv6-non-recursive-scan-with-info-level-logging-enabled)
     - [Read urls from STDIN; pipe only resulting urls out to another tool](#read-urls-from-stdin-pipe-only-resulting-urls-out-to-another-tool)
@@ -84,13 +85,47 @@ This attack is also known as Predictable Resource Location, File Enumeration, Di
 
 ### Download a Release
 
-Releases for multiple architectures can be found in the [Releases](https://github.com/epi052/feroxbuster/releases) section.  Builds for the following systems are currently supported:
+Releases for multiple architectures can be found in the [Releases](https://github.com/epi052/feroxbuster/releases) section.  The latest release for each of the following systems can be downloaded and executed as shown below.
 
-- Linux x86
-- Linux x86_64
-- MacOS x86_64
-- Windows x86
-- Windows x86_64
+#### Linux x86
+```
+curl -sLO https://github.com/epi052/feroxbuster/releases/latest/download/x86-linux-feroxbuster.zip
+unzip x86-linux-feroxbuster.zip
+chmod +x ./feroxbuster
+./feroxbuster -V
+```
+#### Linux x86_64
+
+```
+curl -sLO https://github.com/epi052/feroxbuster/releases/latest/download/x86_64-linux-feroxbuster.zip
+unzip x86_64-linux-feroxbuster.zip
+chmod +x ./feroxbuster
+./feroxbuster -V
+```
+
+#### MacOS x86_64
+```
+curl -sLO https://github.com/epi052/feroxbuster/releases/latest/download/x86_64-macos-feroxbuster.zip
+unzip x86_64-macos-feroxbuster.zip
+chmod +x ./feroxbuster
+./feroxbuster -V
+```
+
+#### Windows x86
+
+```
+https://github.com/epi052/feroxbuster/releases/latest/download/x86-windows-feroxbuster.exe.zip
+Expand-Archive .\feroxbuster.zip
+.\feroxbuster\feroxbuster.exe -V
+```
+
+#### Windows x86_64
+
+```
+Invoke-WebRequest https://github.com/epi052/feroxbuster/releases/latest/download/x86_64-windows-feroxbuster.exe.zip -OutFile feroxbuster.zip
+Expand-Archive .\feroxbuster.zip
+.\feroxbuster\feroxbuster.exe -V
+```
 
 ### Homebrew on MacOS and Linux
 
@@ -120,12 +155,22 @@ cargo install feroxbuster
 
 ### apt Install
 
-Head to the [Releases](https://github.com/epi052/feroxbuster/releases) section and download `feroxbuster_amd64.deb`.  After that, use your favorite package manager to install the .deb.
+Download `feroxbuster_amd64.deb` from the [Releases](https://github.com/epi052/feroxbuster/releases) section.  After that, use your favorite package manager to install the `.deb`.
 
 ```
+wget -sLO https://github.com/epi052/feroxbuster/releases/latest/download/feroxbuster_amd64.deb.zip
+unzip feroxbuster_amd64.deb.zip
 sudo apt install ./feroxbuster_amd64.deb
 ```
 
+### AUR Install
+
+Install `feroxbuster-git` on Arch Linux with your AUR helper of choice:
+
+```
+yay -S feroxbuster-git
+```
+
 ### Docker Install
 
 > The following steps assume you have docker installed / setup
@@ -206,6 +251,11 @@ built-in defaults.
 - The same directory as the `feroxbuster` executable (per-user)
 - The user's current working directory (per-target)
 
+> `CONFIG_DIR` is defined as the following:
+> - Linux: `$XDG_CONFIG_HOME` or `$HOME/.config` i.e. `/home/bob/.config`
+> - MacOs: `$HOME/Library/Application Support` i.e. `/Users/bob/Library/Application Support`
+> - Windows: `{FOLDERID_RoamingAppData}` i.e. `C:\Users\Bob\AppData\Roaming`
+
 If more than one valid configuration file is found, each one overwrites the values found previously.  
 
 If no configuration file is found, nothing happens at this stage.
@@ -251,6 +301,7 @@ A pre-made configuration file with examples of all available settings can be fou
 # addslash = true
 # stdin = true
 # dontfilter = true
+# extract_links = true
 # depth = 1
 # sizefilters = [5174]
 # queries = [["name","value"], ["rick", "astley"]]
@@ -277,16 +328,18 @@ USAGE:
     feroxbuster [FLAGS] [OPTIONS] --url <URL>...
 
 FLAGS:
-    -f, --addslash       Append / to each request
-    -D, --dontfilter     Don't auto-filter wildcard responses
-    -h, --help           Prints help information
-    -k, --insecure       Disables TLS certificate validation
-    -n, --norecursion    Do not scan recursively
-    -q, --quiet          Only print URLs; Don't print status codes, response size, running config, etc...
-    -r, --redirects      Follow redirects
-        --stdin          Read url(s) from STDIN
-    -V, --version        Prints version information
-    -v, --verbosity      Increase verbosity level (use -vv or more for greater effect)
+    -f, --addslash         Append / to each request
+    -D, --dontfilter       Don't auto-filter wildcard responses
+    -e, --extract-links    Extract links from response body (html, javascript, etc...); make new requests based on
+                           findings (default: false)
+    -h, --help             Prints help information
+    -k, --insecure         Disables TLS certificate validation
+    -n, --norecursion      Do not scan recursively
+    -q, --quiet            Only print URLs; Don't print status codes, response size, running config, etc...
+    -r, --redirects        Follow redirects
+        --stdin            Read url(s) from STDIN
+    -V, --version          Prints version information
+    -v, --verbosity        Increase verbosity level (use -vv or more for greater effect)
 
 OPTIONS:
     -d, --depth <RECURSION_DEPTH>           Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)
@@ -324,6 +377,26 @@ All of the methods above (multiple flags, space separated, comma separated, etc.
 ./feroxbuster -u http://127.1 -H Accept:application/json "Authorization: Bearer {token}"
 ```
 
+### Extract Links from Response Body (New in `v1.1.0`) 
+
+Search through the body of valid responses (html, javascript, etc...) for additional endpoints to scan. This turns
+`feroxbuster` into a hybrid that looks for both linked and unlinked content. 
+
+Example request/response with `--extract-links` enabled:
+- Make request to `http://example.com/index.html`
+- Receive, and read in, the `body` of the response
+- Search the `body` for absolute and relative links (i.e. `homepage/assets/img/icons/handshake.svg`)
+- Add the following directories for recursive scanning:
+    - `http://example.com/homepage`
+    - `http://example.com/homepage/assets`
+    - `http://example.com/homepage/assets/img`
+    - `http://example.com/homepage/assets/img/icons`
+- Make a single request to `http://example.com/homepage/assets/img/icons/handshake.svg`
+
+```
+./feroxbuster -u http://127.1 --extract-links
+```
+
 ### IPv6, non-recursive scan with INFO-level logging enabled
 
 ```
@@ -369,26 +442,28 @@ a few of the use-cases in which feroxbuster may be a better fit:
 - You want to be able to run your content discovery as part of some crazy 12 command unix **pipeline extravaganza**
 - You want to scan through a **SOCKS** proxy
 - You want **auto-filtering** of Wildcard responses by default
+- You want an integrated **link extractor** to increase discovered endpoints
 - You want **recursion** along with some other thing mentioned above (ffuf also does recursion)
 - You want a **configuration file** option for overriding built-in default values for your scans
 
-|                                                     | feroxbuster | gobuster | ffuf |
-|-----------------------------------------------------|---|---|---|
-| fast                                                | ✔ | ✔ | ✔ |
-| easy to use                                         | ✔ | ✔ |   |
-| blacklist status codes (in addition to whitelist)   |   | ✔ | ✔ |
-| allows recursion                                    | ✔ |   | ✔ |
-| can specify query parameters                        | ✔ |   | ✔ |
-| SOCKS proxy support                                 | ✔ |   |   |
-| multiple target scan (via stdin or multiple -u)     | ✔ |   | ✔ |
-| configuration file for default value override       | ✔ |   | ✔ |
-| can accept urls via STDIN as part of a pipeline     | ✔ |   | ✔ |
-| can accept wordlists via STDIN                      |   | ✔ | ✔ |
-| filter by response size                             | ✔ |   | ✔ |
-| auto-filter wildcard responses                      | ✔ |   | ✔ |
-| performs other scans (vhost, dns, etc)              |   | ✔ | ✔ |
-| time delay / rate limiting                          |   | ✔ | ✔ |
-| **huge** number of other options                    |   |   | ✔ |
+|                                                                  | feroxbuster | gobuster | ffuf |
+|------------------------------------------------------------------|---|---|---|
+| fast                                                             | ✔ | ✔ | ✔ |
+| easy to use                                                      | ✔ | ✔ |   |
+| blacklist status codes (in addition to whitelist)                |   | ✔ | ✔ |
+| allows recursion                                                 | ✔ |   | ✔ |
+| can specify query parameters                                     | ✔ |   | ✔ |
+| SOCKS proxy support                                              | ✔ |   |   |
+| extracts links from response body to increase scan coverage      | ✔ |   |   |
+| multiple target scan (via stdin or multiple -u)                  | ✔ |   | ✔ |
+| configuration file for default value override                    | ✔ |   | ✔ |
+| can accept urls via STDIN as part of a pipeline                  | ✔ |   | ✔ |
+| can accept wordlists via STDIN                                   |   | ✔ | ✔ |
+| filter by response size                                          | ✔ |   | ✔ |
+| auto-filter wildcard responses                                   | ✔ |   | ✔ |
+| performs other scans (vhost, dns, etc)                           |   | ✔ | ✔ |
+| time delay / rate limiting                                       |   | ✔ | ✔ |
+| **huge** number of other options                                 |   |   | ✔ |
 
 Of note, there's another written-in-rust content discovery tool, [rustbuster](https://github.com/phra/rustbuster). I 
 came across rustbuster when I was naming my tool (😢). I don't have any experience using it, but it appears to 

ferox-config.toml.example

@@ -23,6 +23,7 @@
 # addslash = true
 # stdin = true
 # dontfilter = true
+# extract_links = true
 # depth = 1
 # sizefilters = [5174]
 # queries = [["name","value"], ["rick", "astley"]]

src/banner.rs

@@ -1,4 +1,8 @@
-use crate::{config::Configuration, utils::status_colorizer, VERSION};
+use crate::config::{Configuration, CONFIGURATION};
+use crate::utils::{make_request, status_colorizer};
+use reqwest::{Client, Url};
+use serde_json::Value;
+use std::io::Write;
 
 /// macro helper to abstract away repetitive string formatting
 macro_rules! format_banner_entry_helper {
@@ -40,31 +44,119 @@ macro_rules! format_banner_entry {
     };
 }
 
+/// Url used to query github's api; specifically used to look for the latest tagged release name
+const UPDATE_URL: &str = "https://api.github.com/repos/epi052/feroxbuster/releases/latest";
+
+/// Simple enum to hold three different update states
+#[derive(Debug)]
+enum UpdateStatus {
+    /// this version and latest release are the same
+    UpToDate,
+
+    /// this version and latest release are not the same
+    OutOfDate,
+
+    /// some error occurred during version check
+    Unknown,
+}
+
+/// Makes a request to the given url, expecting to receive a JSON response that contains a field
+/// named `tag_name` that holds a value representing the latest tagged release of this tool.
+///
+/// ex: v1.1.0
+///
+/// Returns `UpdateStatus`
+async fn needs_update(client: &Client, url: &str, bin_version: &str) -> UpdateStatus {
+    log::trace!("enter: needs_update({:?}, {})", client, url);
+
+    let unknown = UpdateStatus::Unknown;
+
+    let api_url = match Url::parse(url) {
+        Ok(url) => url,
+        Err(e) => {
+            log::error!("{}", e);
+            log::trace!("exit: needs_update -> {:?}", unknown);
+            return unknown;
+        }
+    };
+
+    if let Ok(response) = make_request(&client, &api_url).await {
+        let body = response.text().await.unwrap_or_default();
+
+        let json_response: Value = serde_json::from_str(&body).unwrap_or_default();
+
+        if json_response.is_null() {
+            // unwrap_or_default above should result in a null value for the json_response variable
+            log::error!("Could not parse JSON from response body");
+            log::trace!("exit: needs_update -> {:?}", unknown);
+            return unknown;
+        }
+
+        let latest_version = match json_response["tag_name"].as_str() {
+            Some(tag) => tag.trim_start_matches('v'),
+            None => {
+                log::error!("Could not get version field from JSON response");
+                log::debug!("{}", json_response);
+                log::trace!("exit: needs_update -> {:?}", unknown);
+                return unknown;
+            }
+        };
+
+        // if we've gotten this far, we have a string in the form of X.X.X where X is a number
+        // all that's left is to compare the current version with the version found above
+
+        return if latest_version == bin_version {
+            // there's really only two possible outcomes if we accept that the tag conforms to
+            // the X.X.X pattern:
+            //   1. the version strings match, meaning we're up to date
+            //   2. the version strings do not match, meaning we're out of date
+            //
+            // except for developers working on this code, nobody should ever be in a situation
+            // where they have a version greater than the latest tagged release
+            log::trace!("exit: needs_update -> UpdateStatus::UpToDate");
+            UpdateStatus::UpToDate
+        } else {
+            log::trace!("exit: needs_update -> UpdateStatus::OutOfDate");
+            UpdateStatus::OutOfDate
+        };
+    }
+
+    log::trace!("exit: needs_update -> {:?}", unknown);
+    unknown
+}
+
 /// Prints the banner to stdout.
 ///
 /// Only prints those settings which are either always present, or passed in by the user.
-pub fn initialize(targets: &[String], config: &Configuration) {
+pub async fn initialize<W>(targets: &[String], config: &Configuration, version: &str, mut writer: W)
+where
+    W: Write,
+{
     let artwork = format!(
         r#"
  ___  ___  __   __     __      __         __   ___
 |__  |__  |__) |__) | /  `    /  \ \_/ | |  \ |__
 |    |___ |  \ |  \ | \__,    \__/ / \ | |__/ |___
 by Ben "epi" Risher {}                  ver: {}"#,
-        '\u{1F913}', VERSION
+        '\u{1F913}', version
     );
 
+    let status = needs_update(&CONFIGURATION.client, UPDATE_URL, version).await;
+
     let top = "───────────────────────────┬──────────────────────";
     let bottom = "───────────────────────────┴──────────────────────";
 
-    eprintln!("{}", artwork);
-    eprintln!("{}", top);
+    writeln!(&mut writer, "{}", artwork).unwrap_or_default();
+    writeln!(&mut writer, "{}", top).unwrap_or_default();
 
     // begin with always printed items
     for target in targets {
-        eprintln!(
+        writeln!(
+            &mut writer,
             "{}",
             format_banner_entry!("\u{1F3af}", "Target Url", target)
-        ); // 🎯
+        )
+        .unwrap_or_default(); // 🎯
     }
 
     let mut codes = vec![];
@@ -73,206 +165,419 @@ by Ben "epi" Risher {}                  ver: {}"#,
         codes.push(status_colorizer(&code.to_string()))
     }
 
-    eprintln!(
+    writeln!(
+        &mut writer,
         "{}",
         format_banner_entry!("\u{1F680}", "Threads", config.threads)
-    ); // 🚀
-    eprintln!(
+    )
+    .unwrap_or_default(); // 🚀
+
+    writeln!(
+        &mut writer,
         "{}",
         format_banner_entry!("\u{1f4d6}", "Wordlist", config.wordlist)
-    ); // 📖
-    eprintln!(
+    )
+    .unwrap_or_default(); // 📖
+
+    writeln!(
+        &mut writer,
         "{}",
         format_banner_entry!(
             "\u{1F197}",
             "Status Codes",
             format!("[{}]", codes.join(", "))
         )
-    ); // 🆗
-    eprintln!(
+    )
+    .unwrap_or_default(); // 🆗
+
+    writeln!(
+        &mut writer,
         "{}",
         format_banner_entry!("\u{1f4a5}", "Timeout (secs)", config.timeout)
-    ); // 💥
-    eprintln!(
+    )
+    .unwrap_or_default(); // 💥
+
+    writeln!(
+        &mut writer,
         "{}",
         format_banner_entry!("\u{1F9a1}", "User-Agent", config.useragent)
-    ); // 🦡
+    )
+    .unwrap_or_default(); // 🦡
 
     // followed by the maybe printed or variably displayed values
     if !config.config.is_empty() {
-        eprintln!(
+        writeln!(
+            &mut writer,
             "{}",
             format_banner_entry!("\u{1f489}", "Config File", config.config)
-        ); // 💉
+        )
+        .unwrap_or_default(); // 💉
     }
 
     if !config.proxy.is_empty() {
-        eprintln!(
+        writeln!(
+            &mut writer,
             "{}",
             format_banner_entry!("\u{1f48e}", "Proxy", config.proxy)
-        ); // 💎
+        )
+        .unwrap_or_default(); // 💎
     }
 
     if !config.headers.is_empty() {
         for (name, value) in &config.headers {
-            eprintln!(
+            writeln!(
+                &mut writer,
                 "{}",
                 format_banner_entry!("\u{1f92f}", "Header", name, value)
-            ); // 🤯
+            )
+            .unwrap_or_default(); // 🤯
         }
     }
 
     if !config.sizefilters.is_empty() {
         for filter in &config.sizefilters {
-            eprintln!(
+            writeln!(
+                &mut writer,
                 "{}",
                 format_banner_entry!("\u{1f4a2}", "Size Filter", filter)
-            ); // 💢
+            )
+            .unwrap_or_default(); // 💢
         }
     }
 
+    if config.extract_links {
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!("\u{1F50E}", "Extract Links", config.extract_links)
+        )
+        .unwrap_or_default(); // 🔎
+    }
+
     if !config.queries.is_empty() {
         for query in &config.queries {
-            eprintln!(
+            writeln!(
+                &mut writer,
                 "{}",
                 format_banner_entry!(
                     "\u{1f914}",
                     "Query Parameter",
                     format!("{}={}", query.0, query.1)
                 )
-            ); // 🤔
+            )
+            .unwrap_or_default(); // 🤔
         }
     }
 
     if !config.output.is_empty() {
-        eprintln!(
+        writeln!(
+            &mut writer,
             "{}",
             format_banner_entry!("\u{1f4be}", "Output File", config.output)
-        ); // 💾
+        )
+        .unwrap_or_default(); // 💾
     }
 
     if !config.extensions.is_empty() {
-        eprintln!(
+        writeln!(
+            &mut writer,
             "{}",
             format_banner_entry!(
                 "\u{1f4b2}",
                 "Extensions",
                 format!("[{}]", config.extensions.join(", "))
             )
-        ); // 💲
+        )
+        .unwrap_or_default(); // 💲
     }
 
     if config.insecure {
-        eprintln!(
+        writeln!(
+            &mut writer,
             "{}",
             format_banner_entry!("\u{1f513}", "Insecure", config.insecure)
-        ); // 🔓
+        )
+        .unwrap_or_default(); // 🔓
     }
 
     if config.redirects {
-        eprintln!(
+        writeln!(
+            &mut writer,
             "{}",
             format_banner_entry!("\u{1f4cd}", "Follow Redirects", config.redirects)
-        ); // 📍
+        )
+        .unwrap_or_default(); // 📍
     }
 
     if config.dontfilter {
-        eprintln!(
+        writeln!(
+            &mut writer,
             "{}",
             format_banner_entry!("\u{1f92a}", "Filter Wildcards", !config.dontfilter)
-        ); // 🤪
+        )
+        .unwrap_or_default(); // 🤪
     }
 
     match config.verbosity {
         //speaker medium volume (increasing with verbosity to loudspeaker)
         1 => {
-            eprintln!(
+            writeln!(
+                &mut writer,
                 "{}",
                 format_banner_entry!("\u{1f508}", "Verbosity", config.verbosity)
-            ); // 🔈
+            )
+            .unwrap_or_default(); // 🔈
         }
         2 => {
-            eprintln!(
+            writeln!(
+                &mut writer,
                 "{}",
                 format_banner_entry!("\u{1f509}", "Verbosity", config.verbosity)
-            ); // 🔉
+            )
+            .unwrap_or_default(); // 🔉
         }
         3 => {
-            eprintln!(
+            writeln!(
+                &mut writer,
                 "{}",
                 format_banner_entry!("\u{1f50a}", "Verbosity", config.verbosity)
-            ); // 🔊
+            )
+            .unwrap_or_default(); // 🔊
         }
         4 => {
-            eprintln!(
+            writeln!(
+                &mut writer,
                 "{}",
                 format_banner_entry!("\u{1f4e2}", "Verbosity", config.verbosity)
-            ); // 📢
+            )
+            .unwrap_or_default(); // 📢
         }
         _ => {}
     }
 
     if config.addslash {
-        eprintln!(
+        writeln!(
+            &mut writer,
             "{}",
             format_banner_entry!("\u{1fa93}", "Add Slash", config.addslash)
-        ); // 🪓
+        )
+        .unwrap_or_default(); // 🪓
     }
 
     if !config.norecursion {
         if config.depth == 0 {
-            eprintln!(
+            writeln!(
+                &mut writer,
                 "{}",
                 format_banner_entry!("\u{1f503}", "Recursion Depth", "INFINITE")
-            ); // 🔃
+            )
+            .unwrap_or_default(); // 🔃
         } else {
-            eprintln!(
+            writeln!(
+                &mut writer,
                 "{}",
                 format_banner_entry!("\u{1f503}", "Recursion Depth", config.depth)
-            ); // 🔃
+            )
+            .unwrap_or_default(); // 🔃
         }
     } else {
-        eprintln!(
+        writeln!(
+            &mut writer,
             "{}",
             format_banner_entry!("\u{1f6ab}", "Do Not Recurse", config.norecursion)
-        ); // 🚫
+        )
+        .unwrap_or_default(); // 🚫
     }
 
-    eprintln!("{}", bottom);
+    if matches!(status, UpdateStatus::OutOfDate) {
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!(
+                "\u{1f389}",
+                "New Version Available",
+                "https://github.com/epi052/feroxbuster/releases/latest"
+            )
+        )
+        .unwrap_or_default(); // 🎉
+    }
+
+    writeln!(&mut writer, "{}", bottom).unwrap_or_default();
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::VERSION;
+    use httpmock::Method::GET;
+    use httpmock::{Mock, MockServer};
+    use std::fs::read_to_string;
+    use std::io::stderr;
+    use std::time::Duration;
+    use tempfile::NamedTempFile;
 
-    #[test]
+    #[tokio::test(core_threads = 1)]
     /// test to hit no execution of targets for loop in banner
-    fn banner_without_targets() {
+    async fn banner_intialize_without_targets() {
         let config = Configuration::default();
-        initialize(&[], &config);
+        initialize(&[], &config, VERSION, stderr()).await;
     }
 
-    #[test]
+    #[tokio::test(core_threads = 1)]
     /// test to hit no execution of statuscode for loop in banner
-    fn banner_without_status_codes() {
+    async fn banner_intialize_without_status_codes() {
         let mut config = Configuration::default();
         config.statuscodes = vec![];
-        initialize(&[String::from("http://localhost")], &config);
+        initialize(
+            &[String::from("http://localhost")],
+            &config,
+            VERSION,
+            stderr(),
+        )
+        .await;
     }
 
-    #[test]
+    #[tokio::test(core_threads = 1)]
     /// test to hit an empty config file
-    fn banner_without_config_file() {
+    async fn banner_intialize_without_config_file() {
         let mut config = Configuration::default();
         config.config = String::new();
-        initialize(&[String::from("http://localhost")], &config);
+        initialize(
+            &[String::from("http://localhost")],
+            &config,
+            VERSION,
+            stderr(),
+        )
+        .await;
     }
 
-    #[test]
+    #[tokio::test(core_threads = 1)]
     /// test to hit an empty config file
-    fn banner_without_queries() {
+    async fn banner_intialize_without_queries() {
         let mut config = Configuration::default();
         config.queries = vec![(String::new(), String::new())];
-        initialize(&[String::from("http://localhost")], &config);
+        initialize(
+            &[String::from("http://localhost")],
+            &config,
+            VERSION,
+            stderr(),
+        )
+        .await;
+    }
+
+    #[tokio::test(core_threads = 1)]
+    /// test to show that a new version is available for download
+    async fn banner_intialize_with_mismatched_version() {
+        let config = Configuration::default();
+        let file = NamedTempFile::new().unwrap();
+        initialize(
+            &[String::from("http://localhost")],
+            &config,
+            "mismatched-version",
+            &file,
+        )
+        .await;
+        let contents = read_to_string(file.path()).unwrap();
+        println!("contents: {}", contents);
+        assert!(contents.contains("New Version Available"));
+        assert!(contents.contains("https://github.com/epi052/feroxbuster/releases/latest"));
+    }
+
+    #[tokio::test(core_threads = 1)]
+    /// test that
+    async fn banner_needs_update_returns_unknown_with_bad_url() {
+        let result = needs_update(&CONFIGURATION.client, &"", VERSION).await;
+        assert!(matches!(result, UpdateStatus::Unknown));
+    }
+
+    #[tokio::test(core_threads = 1)]
+    /// test return value of good url to needs_update
+    async fn banner_needs_update_returns_up_to_date() {
+        let srv = MockServer::start();
+
+        let mock = Mock::new()
+            .expect_method(GET)
+            .expect_path("/latest")
+            .return_status(200)
+            .return_body("{\"tag_name\":\"v1.1.0\"}")
+            .create_on(&srv);
+
+        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.1.0").await;
+
+        assert_eq!(mock.times_called(), 1);
+        assert!(matches!(result, UpdateStatus::UpToDate));
+    }
+
+    #[tokio::test(core_threads = 1)]
+    /// test return value of good url to needs_update that returns a newer version than current
+    async fn banner_needs_update_returns_out_of_date() {
+        let srv = MockServer::start();
+
+        let mock = Mock::new()
+            .expect_method(GET)
+            .expect_path("/latest")
+            .return_status(200)
+            .return_body("{\"tag_name\":\"v1.1.0\"}")
+            .create_on(&srv);
+
+        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1").await;
+
+        assert_eq!(mock.times_called(), 1);
+        assert!(matches!(result, UpdateStatus::OutOfDate));
+    }
+
+    #[tokio::test(core_threads = 1)]
+    /// test return value of good url that times out
+    async fn banner_needs_update_returns_unknown_on_timeout() {
+        let srv = MockServer::start();
+
+        let mock = Mock::new()
+            .expect_method(GET)
+            .expect_path("/latest")
+            .return_status(200)
+            .return_body("{\"tag_name\":\"v1.1.0\"}")
+            .return_with_delay(Duration::from_secs(8))
+            .create_on(&srv);
+
+        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1").await;
+
+        assert_eq!(mock.times_called(), 1);
+        assert!(matches!(result, UpdateStatus::Unknown));
+    }
+
+    #[tokio::test(core_threads = 1)]
+    /// test return value of good url with bad json response
+    async fn banner_needs_update_returns_unknown_on_bad_json_response() {
+        let srv = MockServer::start();
+
+        let mock = Mock::new()
+            .expect_method(GET)
+            .expect_path("/latest")
+            .return_status(200)
+            .return_body("not json")
+            .create_on(&srv);
+
+        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1").await;
+
+        assert_eq!(mock.times_called(), 1);
+        assert!(matches!(result, UpdateStatus::Unknown));
+    }
+
+    #[tokio::test(core_threads = 1)]
+    /// test return value of good url with json response that lacks the tag_name field
+    async fn banner_needs_update_returns_unknown_on_json_without_correct_tag() {
+        let srv = MockServer::start();
+
+        let mock = Mock::new()
+            .expect_method(GET)
+            .expect_path("/latest")
+            .return_status(200)
+            .return_body("{\"no tag_name\": \"doesn't exist\"}")
+            .create_on(&srv);
+
+        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1").await;
+
+        assert_eq!(mock.times_called(), 1);
+        assert!(matches!(result, UpdateStatus::Unknown));
     }
 }

src/client.rs

@@ -1,9 +1,9 @@
 use crate::utils::{module_colorizer, status_colorizer};
-use console::style;
 use reqwest::header::HeaderMap;
 use reqwest::{redirect::Policy, Client, Proxy};
 use std::collections::HashMap;
 use std::convert::TryInto;
+#[cfg(not(test))]
 use std::process::exit;
 use std::time::Duration;
 
@@ -22,18 +22,8 @@ pub fn initialize(
         Policy::none()
     };
 
-    let header_map: HeaderMap = match headers.try_into() {
-        Ok(map) => map,
-        Err(e) => {
-            eprintln!(
-                "{} {} {}",
-                status_colorizer("ERROR"),
-                module_colorizer("Client::initialize"),
-                e
-            );
-            exit(1);
-        }
-    };
+    // try_into returns infallible as its error, unwrap is safe here
+    let header_map: HeaderMap = headers.try_into().unwrap();
 
     let client = Client::builder()
         .timeout(Duration::new(timeout, 0))
@@ -55,9 +45,13 @@ pub fn initialize(
                 eprintln!(
                     "{} {} {}",
                     status_colorizer("ERROR"),
-                    style("Client::initialize").cyan(),
+                    module_colorizer("Client::initialize"),
                     e
                 );
+
+                #[cfg(test)]
+                panic!();
+                #[cfg(not(test))]
                 exit(1);
             }
         }
@@ -79,7 +73,32 @@ pub fn initialize(
                 module_colorizer("Client::build"),
                 e
             );
+
+            #[cfg(test)]
+            panic!();
+            #[cfg(not(test))]
             exit(1);
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    #[should_panic]
+    /// create client with a bad proxy, expect panic
+    fn client_with_bad_proxy() {
+        let headers = HashMap::new();
+        initialize(0, "stuff", true, false, &headers, Some("not a valid proxy"));
+    }
+
+    #[test]
+    /// create client with a proxy, expect no error
+    fn client_with_good_proxy() {
+        let headers = HashMap::new();
+        let proxy = "http://127.0.0.1:8080";
+        initialize(0, "stuff", true, true, &headers, Some(proxy));
+    }
+}

src/config.rs

@@ -107,6 +107,10 @@ pub struct Configuration {
     #[serde(default)]
     pub norecursion: bool,
 
+    /// Extract links from html/javscript
+    #[serde(default)]
+    pub extract_links: bool,
+
     /// Append / to each request
     #[serde(default)]
     pub addslash: bool,
@@ -182,8 +186,9 @@ impl Default for Configuration {
             verbosity: 0,
             addslash: false,
             insecure: false,
-            norecursion: false,
             redirects: false,
+            norecursion: false,
+            extract_links: false,
             proxy: String::new(),
             config: String::new(),
             output: String::new(),
@@ -206,6 +211,7 @@ impl Configuration {
     ///
     /// - **timeout**: `5` seconds
     /// - **redirects**: `false`
+    /// - **extract-links**: `false`
     /// - **wordlist**: [`DEFAULT_WORDLIST`](constant.DEFAULT_WORDLIST.html)
     /// - **config**: `None`
     /// - **threads**: `50`
@@ -325,7 +331,12 @@ impl Configuration {
                 .map(|code| {
                     StatusCode::from_bytes(code.as_bytes())
                         .unwrap_or_else(|e| {
-                            eprintln!("[!] Error encountered: {}", e);
+                            eprintln!(
+                                "{} {}: {}",
+                                status_colorizer("ERROR"),
+                                module_colorizer("Configuration::new"),
+                                e
+                            );
                             exit(1)
                         })
                         .as_u16()
@@ -347,7 +358,12 @@ impl Configuration {
                 .unwrap() // already known good
                 .map(|size| {
                     size.parse::<u64>().unwrap_or_else(|e| {
-                        eprintln!("[!] Error encountered: {}", e);
+                        eprintln!(
+                            "{} {}: {}",
+                            status_colorizer("ERROR"),
+                            module_colorizer("Configuration::new"),
+                            e
+                        );
                         exit(1)
                     })
                 })
@@ -380,6 +396,10 @@ impl Configuration {
             config.addslash = args.is_present("addslash");
         }
 
+        if args.is_present("extract_links") {
+            config.extract_links = args.is_present("extract_links");
+        }
+
         if args.is_present("stdin") {
             config.stdin = args.is_present("stdin");
         } else {
@@ -504,6 +524,7 @@ impl Configuration {
         settings.useragent = settings_to_merge.useragent;
         settings.redirects = settings_to_merge.redirects;
         settings.insecure = settings_to_merge.insecure;
+        settings.extract_links = settings_to_merge.extract_links;
         settings.extensions = settings_to_merge.extensions;
         settings.headers = settings_to_merge.headers;
         settings.queries = settings_to_merge.queries;
@@ -564,6 +585,7 @@ mod tests {
             addslash = true
             stdin = true
             dontfilter = true
+            extract_links = true
             depth = 1
             sizefilters = [4120]
         "#;
@@ -592,6 +614,7 @@ mod tests {
         assert_eq!(config.stdin, false);
         assert_eq!(config.addslash, false);
         assert_eq!(config.redirects, false);
+        assert_eq!(config.extract_links, false);
         assert_eq!(config.insecure, false);
         assert_eq!(config.queries, Vec::new());
         assert_eq!(config.extensions, Vec::<String>::new());
@@ -704,6 +727,13 @@ mod tests {
         assert_eq!(config.addslash, true);
     }
 
+    #[test]
+    /// parse the test config and see that the value parsed is correct
+    fn config_reads_extract_links() {
+        let config = setup_config_test();
+        assert_eq!(config.extract_links, true);
+    }
+
     #[test]
     /// parse the test config and see that the value parsed is correct
     fn config_reads_extensions() {

src/extractor.rs

@@ -0,0 +1,269 @@
+use crate::FeroxResponse;
+use lazy_static::lazy_static;
+use regex::Regex;
+use reqwest::Url;
+use std::collections::HashSet;
+
+/// Regular expression used in [LinkFinder](https://github.com/GerbenJavado/LinkFinder)
+///
+/// Incorporates change from this [Pull Request](https://github.com/GerbenJavado/LinkFinder/pull/66/files)
+const LINKFINDER_REGEX: &str = r#"(?:"|')(((?:[a-zA-Z]{1,10}://|//)[^"'/]{1,}\.[a-zA-Z]{2,}[^"']{0,})|((?:/|\.\./|\./)[^"'><,;| *()(%%$^/\\\[\]][^"'><,;|()]{1,})|([a-zA-Z0-9_\-/]{1,}/[a-zA-Z0-9_\-/]{1,}\.(?:[a-zA-Z]{1,4}|action)(?:[\?|#][^"|']{0,}|))|([a-zA-Z0-9_\-/]{1,}/[a-zA-Z0-9_\-/]{3,}(?:[\?|#][^"|']{0,}|))|([a-zA-Z0-9_\-.]{1,}\.(?:php|asp|aspx|jsp|json|action|html|js|txt|xml)(?:[\?|#][^"|']{0,}|)))(?:"|')"#;
+
+lazy_static! {
+    /// `LINKFINDER_REGEX` as a regex::Regex type
+    static ref REGEX: Regex = Regex::new(LINKFINDER_REGEX).unwrap();
+}
+
+/// Iterate over a given path, return a list of every sub-path found
+///
+/// example: `path` contains a link fragment `homepage/assets/img/icons/handshake.svg`
+/// the following fragments would be returned:
+///   - homepage/assets/img/icons/handshake.svg
+///   - homepage/assets/img/icons/
+///   - homepage/assets/img/
+///   - homepage/assets/
+///   - homepage/
+fn get_sub_paths_from_path(path: &str) -> Vec<String> {
+    log::trace!("enter: get_sub_paths_from_path({})", path);
+    let mut paths = vec![];
+
+    // filter out any empty strings caused by .split
+    let mut parts: Vec<&str> = path.split('/').filter(|s| !s.is_empty()).collect();
+
+    let length = parts.len();
+
+    for _ in 0..length {
+        // iterate over all parts of the path
+        if parts.is_empty() {
+            // pop left us with an empty vector, we're done
+            break;
+        }
+
+        let possible_path = parts.join("/");
+
+        if possible_path.is_empty() {
+            // .join can result in an empty string, which we don't need, ignore
+            continue;
+        }
+
+        paths.push(possible_path); // good sub-path found
+        parts.pop(); // use .pop() to remove the last part of the path and continue iteration
+    }
+
+    log::trace!("exit: get_sub_paths_from_path -> {:?}", paths);
+    paths
+}
+
+/// simple helper to stay DRY, trys to join a url + fragment and add it to the `links` HashSet
+fn add_link_to_set_of_links(link: &str, url: &Url, links: &mut HashSet<String>) {
+    log::trace!(
+        "enter: add_link_to_set_of_links({}, {}, {:?})",
+        link,
+        url.to_string(),
+        links
+    );
+    match url.join(&link) {
+        Ok(new_url) => {
+            links.insert(new_url.to_string());
+        }
+        Err(e) => {
+            log::error!("Could not join given url to the base url: {}", e);
+        }
+    }
+    log::trace!("exit: add_link_to_set_of_links");
+}
+
+/// Given a `reqwest::Response`, perform the following actions
+///   - parse the response's text for links using the linkfinder regex
+///   - for every link found take its url path and parse each sub-path
+///     - example: Response contains a link fragment `homepage/assets/img/icons/handshake.svg`
+///       with a base url of http://localhost, the following urls would be returned:
+///         - homepage/assets/img/icons/handshake.svg
+///         - homepage/assets/img/icons/
+///         - homepage/assets/img/
+///         - homepage/assets/
+///         - homepage/
+pub async fn get_links(response: &FeroxResponse) -> HashSet<String> {
+    log::trace!("enter: get_links({})", response.url().as_str());
+
+    let mut links = HashSet::<String>::new();
+
+    let body = response.text();
+
+    for capture in REGEX.captures_iter(&body) {
+        // remove single & double quotes from both ends of the capture
+        // capture[0] is the entire match, additional capture groups start at [1]
+        let link = capture[0].trim_matches(|c| c == '\'' || c == '"');
+
+        match Url::parse(link) {
+            Ok(absolute) => {
+                if absolute.domain() != response.url().domain()
+                    || absolute.host() != response.url().host()
+                {
+                    // domains/ips are not the same, don't scan things that aren't part of the original
+                    // target url
+                    continue;
+                }
+
+                for sub_path in get_sub_paths_from_path(absolute.path()) {
+                    // take a url fragment like homepage/assets/img/icons/handshake.svg and
+                    // incrementally add
+                    //     - homepage/assets/img/icons/
+                    //     - homepage/assets/img/
+                    //     - homepage/assets/
+                    //     - homepage/
+                    log::debug!("Adding {} to {:?}", sub_path, links);
+                    add_link_to_set_of_links(&sub_path, &response.url(), &mut links);
+                }
+            }
+            Err(e) => {
+                // this is the expected error that happens when we try to parse a url fragment
+                //     ex: Url::parse("/login") -> Err("relative URL without a base")
+                // while this is technically an error, these are good results for us
+                if e.to_string().contains("relative URL without a base") {
+                    for sub_path in get_sub_paths_from_path(link) {
+                        // incrementally save all sub-paths that led to the relative url's resource
+                        log::debug!("Adding {} to {:?}", sub_path, links);
+                        add_link_to_set_of_links(&sub_path, &response.url(), &mut links);
+                    }
+                } else {
+                    // unexpected error has occurred
+                    log::error!("Could not parse given url: {}", e);
+                }
+            }
+        }
+    }
+
+    log::trace!("exit: get_links -> {:?}", links);
+    links
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::utils::make_request;
+    use httpmock::Method::GET;
+    use httpmock::{Mock, MockServer};
+    use reqwest::Client;
+
+    #[test]
+    /// extract sub paths from the given url fragment; expect 4 sub paths and that all are
+    /// in the expected array
+    fn extractor_get_sub_paths_from_path_with_multiple_paths() {
+        let path = "homepage/assets/img/icons/handshake.svg";
+        let paths = get_sub_paths_from_path(&path);
+        let expected = vec![
+            "homepage",
+            "homepage/assets",
+            "homepage/assets/img",
+            "homepage/assets/img/icons",
+            "homepage/assets/img/icons/handshake.svg",
+        ];
+
+        assert_eq!(paths.len(), expected.len());
+        for expected_path in expected {
+            assert_eq!(paths.contains(&expected_path.to_string()), true);
+        }
+    }
+
+    #[test]
+    /// extract sub paths from the given url fragment; expect 2 sub paths and that all are
+    /// in the expected array. the fragment is wrapped in slashes to ensure no empty strings are
+    /// returned
+    fn extractor_get_sub_paths_from_path_with_enclosing_slashes() {
+        let path = "/homepage/assets/";
+        let paths = get_sub_paths_from_path(&path);
+        let expected = vec!["homepage", "homepage/assets"];
+
+        assert_eq!(paths.len(), expected.len());
+        for expected_path in expected {
+            assert_eq!(paths.contains(&expected_path.to_string()), true);
+        }
+    }
+
+    #[test]
+    /// extract sub paths from the given url fragment; expect 1 sub path, no forward slashes are
+    /// included
+    fn extractor_get_sub_paths_from_path_with_only_a_word() {
+        let path = "homepage";
+        let paths = get_sub_paths_from_path(&path);
+        let expected = vec!["homepage"];
+
+        assert_eq!(paths.len(), expected.len());
+        for expected_path in expected {
+            assert_eq!(paths.contains(&expected_path.to_string()), true);
+        }
+    }
+
+    #[test]
+    /// extract sub paths from the given url fragment; expect 1 sub path, forward slash removed
+    fn extractor_get_sub_paths_from_path_with_an_absolute_word() {
+        let path = "/homepage";
+        let paths = get_sub_paths_from_path(&path);
+        let expected = vec!["homepage"];
+
+        assert_eq!(paths.len(), expected.len());
+        for expected_path in expected {
+            assert_eq!(paths.contains(&expected_path.to_string()), true);
+        }
+    }
+
+    #[test]
+    /// test that a full url and fragment are joined correctly, then added to the given list
+    /// i.e. the happy path
+    fn extractor_add_link_to_set_of_links_happy_path() {
+        let url = Url::parse("https://localhost").unwrap();
+        let mut links = HashSet::<String>::new();
+        let link = "admin";
+
+        assert_eq!(links.len(), 0);
+        add_link_to_set_of_links(link, &url, &mut links);
+
+        assert_eq!(links.len(), 1);
+        assert!(links.contains("https://localhost/admin"));
+    }
+
+    #[test]
+    /// test that an invalid path fragment doesn't add anything to the set of links
+    fn extractor_add_link_to_set_of_links_with_non_base_url() {
+        let url = Url::parse("https://localhost").unwrap();
+        let mut links = HashSet::<String>::new();
+        let link = "\\\\\\\\";
+
+        assert_eq!(links.len(), 0);
+        add_link_to_set_of_links(link, &url, &mut links);
+
+        assert_eq!(links.len(), 0);
+        assert!(links.is_empty());
+    }
+
+    #[tokio::test(core_threads = 1)]
+    /// use make_request to generate a Response, and use the Response to test get_links;
+    /// the response will contain an absolute path to a domain that is not part of the scanned
+    /// domain; expect an empty set returned
+    async fn extractor_get_links_with_absolute_url_that_differs_from_target_domain(
+    ) -> Result<(), Box<dyn std::error::Error>> {
+        let srv = MockServer::start();
+
+        let mock = Mock::new()
+            .expect_method(GET)
+            .expect_path("/some-path")
+            .return_status(200)
+            .return_body("\"http://defintely.not.a.thing.probably.com/homepage/assets/img/icons/handshake.svg\"")
+            .create_on(&srv);
+
+        let client = Client::new();
+        let url = Url::parse(&srv.url("/some-path")).unwrap();
+
+        let response = make_request(&client, &url).await.unwrap();
+
+        let ferox_response = FeroxResponse::from(response, true).await;
+
+        let links = get_links(&ferox_response).await;
+
+        assert!(links.is_empty());
+
+        assert_eq!(mock.times_called(), 1);
+        Ok(())
+    }
+}

src/heuristics.rs

@@ -1,4 +1,5 @@
 use crate::config::{CONFIGURATION, PROGRESS_PRINTER};
+use crate::scanner::should_filter_response;
 use crate::utils::{
     ferox_print, format_url, get_url_path_length, make_request, module_colorizer, status_colorizer,
 };
@@ -6,6 +7,7 @@ use console::style;
 use indicatif::ProgressBar;
 use reqwest::Response;
 use std::process;
+use tokio::sync::mpsc::UnboundedSender;
 use uuid::Uuid;
 
 /// length of a standard UUID, used when determining wildcard responses
@@ -19,9 +21,13 @@ const UUID_LENGTH: u64 = 32;
 ///
 /// `size` is size of the response that should be included with filters passed via runtime
 /// configuration and any static wildcard lengths.
-#[derive(Default, Debug)]
+#[derive(Default, Debug, PartialEq, Copy, Clone)]
 pub struct WildcardFilter {
+    /// size of the response that will later be combined with the length of the path of the url
+    /// requested
     pub dynamic: u64,
+
+    /// size of the response that should be included with filters passed via runtime configuration
     pub size: u64,
 }
 
@@ -48,8 +54,17 @@ fn unique_string(length: usize) -> String {
 ///
 /// In the event that url returns a wildcard response, a
 /// [WildcardFilter](struct.WildcardFilter.html) is created and returned to the caller.
-pub async fn wildcard_test(target_url: &str, bar: ProgressBar) -> Option<WildcardFilter> {
-    log::trace!("enter: wildcard_test({:?})", target_url);
+pub async fn wildcard_test(
+    target_url: &str,
+    bar: ProgressBar,
+    tx_file: UnboundedSender<String>,
+) -> Option<WildcardFilter> {
+    log::trace!(
+        "enter: wildcard_test({:?}, {:?}, {:?})",
+        target_url,
+        bar,
+        tx_file
+    );
 
     if CONFIGURATION.dontfilter {
         // early return, dontfilter scans don't need tested
@@ -57,7 +72,10 @@ pub async fn wildcard_test(target_url: &str, bar: ProgressBar) -> Option<Wildcar
         return None;
     }
 
-    if let Some(resp_one) = make_wildcard_request(&target_url, 1).await {
+    let clone_req_one = tx_file.clone();
+    let clone_req_two = tx_file.clone();
+
+    if let Some(resp_one) = make_wildcard_request(&target_url, 1, clone_req_one).await {
         bar.inc(1);
 
         // found a wildcard response
@@ -72,7 +90,7 @@ pub async fn wildcard_test(target_url: &str, bar: ProgressBar) -> Option<Wildcar
 
         // content length of wildcard is non-zero, perform additional tests:
         //   make a second request, with a known-sized (64) longer request
-        if let Some(resp_two) = make_wildcard_request(&target_url, 3).await {
+        if let Some(resp_two) = make_wildcard_request(&target_url, 3, clone_req_two).await {
             bar.inc(1);
 
             let wc2_length = resp_two.content_length().unwrap_or(0);
@@ -82,32 +100,50 @@ pub async fn wildcard_test(target_url: &str, bar: ProgressBar) -> Option<Wildcar
                 // reflected in the response along with some static content; aka custom 404
                 let url_len = get_url_path_length(&resp_one.url());
 
-                if !CONFIGURATION.quiet {
-                    ferox_print(
-                    &format!(
-                            "{} {:>10} Wildcard response is dynamic; {} ({} + url length) responses; toggle this behavior by using {}",
+                wildcard.dynamic = wc_length - url_len;
+
+                if !CONFIGURATION.quiet
+                    && !should_filter_response(&wildcard.dynamic, &resp_one.url())
+                {
+                    let msg = format!(
+                            "{} {:>10} Wildcard response is dynamic; {} ({} + url length) responses; toggle this behavior by using {}\n",
                             status_colorizer("WLD"),
-                            wc_length - url_len,
+                            wildcard.dynamic,
                             style("auto-filtering").yellow(),
                             style(wc_length - url_len).cyan(),
                             style("--dontfilter").yellow()
-                        ), &PROGRESS_PRINTER
+                        );
+
+                    ferox_print(&msg, &PROGRESS_PRINTER);
+
+                    try_send_message_to_file(
+                        &msg,
+                        tx_file.clone(),
+                        !CONFIGURATION.output.is_empty(),
                     );
                 }
-
-                wildcard.dynamic = wc_length - url_len;
             } else if wc_length == wc2_length {
-                if !CONFIGURATION.quiet {
-                    ferox_print(&format!(
-                        "{} {:>10} Wildcard response is static; {} {} responses; toggle this behavior by using {}",
+                wildcard.size = wc_length;
+
+                if !CONFIGURATION.quiet && !should_filter_response(&wildcard.size, &resp_one.url())
+                {
+                    let msg = format!(
+                        "{} {:>10} Wildcard response is static; {} {} responses; toggle this behavior by using {}\n",
                         status_colorizer("WLD"),
                         wc_length,
                         style("auto-filtering").yellow(),
                         style(wc_length).cyan(),
                         style("--dontfilter").yellow()
-                    ), &PROGRESS_PRINTER);
+                    );
+
+                    ferox_print(&msg, &PROGRESS_PRINTER);
+
+                    try_send_message_to_file(
+                        &msg,
+                        tx_file.clone(),
+                        !CONFIGURATION.output.is_empty(),
+                    );
                 }
-                wildcard.size = wc_length;
             }
         } else {
             bar.inc(2);
@@ -127,8 +163,17 @@ pub async fn wildcard_test(target_url: &str, bar: ProgressBar) -> Option<Wildcar
 /// Once the unique url is created, the request is sent to the server. If the server responds
 /// back with a valid status code, the response is considered to be a wildcard response. If that
 /// wildcard response has a 3xx status code, that redirection location is displayed to the user.
-async fn make_wildcard_request(target_url: &str, length: usize) -> Option<Response> {
-    log::trace!("enter: make_wildcard_request({}, {})", target_url, length);
+async fn make_wildcard_request(
+    target_url: &str,
+    length: usize,
+    tx_file: UnboundedSender<String>,
+) -> Option<Response> {
+    log::trace!(
+        "enter: make_wildcard_request({}, {}, {:?})",
+        target_url,
+        length,
+        tx_file
+    );
 
     let unique_str = unique_string(length);
 
@@ -159,45 +204,46 @@ async fn make_wildcard_request(target_url: &str, length: usize) -> Option<Respon
                 let url_len = get_url_path_length(&response.url());
                 let content_len = response.content_length().unwrap_or(0);
 
-                if !CONFIGURATION.quiet {
-                    ferox_print(
-                        &format!(
-                            "{} {:>10} Got {} for {} (url length: {})",
-                            wildcard,
-                            content_len,
-                            status_colorizer(&response.status().as_str()),
-                            response.url(),
-                            url_len
-                        ),
-                        &PROGRESS_PRINTER,
+                if !CONFIGURATION.quiet && !should_filter_response(&content_len, &response.url()) {
+                    let msg = format!(
+                        "{} {:>10} Got {} for {} (url length: {})\n",
+                        wildcard,
+                        content_len,
+                        status_colorizer(&response.status().as_str()),
+                        response.url(),
+                        url_len
+                    );
+
+                    ferox_print(&msg, &PROGRESS_PRINTER);
+
+                    try_send_message_to_file(
+                        &msg,
+                        tx_file.clone(),
+                        !CONFIGURATION.output.is_empty(),
                     );
                 }
+
                 if response.status().is_redirection() {
                     // show where it goes, if possible
                     if let Some(next_loc) = response.headers().get("Location") {
-                        if let Ok(next_loc_str) = next_loc.to_str() {
-                            if !CONFIGURATION.quiet {
-                                ferox_print(
-                                    &format!(
-                                        "{} {:>10} {} redirects to => {}",
-                                        wildcard,
-                                        content_len,
-                                        response.url(),
-                                        next_loc_str
-                                    ),
-                                    &PROGRESS_PRINTER,
-                                );
-                            }
-                        } else if !CONFIGURATION.quiet {
-                            ferox_print(
-                                &format!(
-                                    "{} {:>10} {} redirects to => {:?}",
-                                    wildcard,
-                                    content_len,
-                                    response.url(),
-                                    next_loc
-                                ),
-                                &PROGRESS_PRINTER,
+                        let next_loc_str = next_loc.to_str().unwrap_or("Unknown");
+                        if !CONFIGURATION.quiet
+                            && !should_filter_response(&content_len, &response.url())
+                        {
+                            let msg = format!(
+                                "{} {:>10} {} redirects to => {}\n",
+                                wildcard,
+                                content_len,
+                                response.url(),
+                                next_loc_str
+                            );
+
+                            ferox_print(&msg, &PROGRESS_PRINTER);
+
+                            try_send_message_to_file(
+                                &msg,
+                                tx_file.clone(),
+                                !CONFIGURATION.output.is_empty(),
                             );
                         }
                     }
@@ -274,15 +320,87 @@ pub async fn connectivity_test(target_urls: &[String]) -> Vec<String> {
     good_urls
 }
 
+/// simple helper to keep DRY; sends a message using the transmitter side of the given mpsc channel
+/// the receiver is expected to be the side that saves the message to CONFIGURATION.output.
+fn try_send_message_to_file(msg: &str, tx_file: UnboundedSender<String>, save_output: bool) {
+    log::trace!("enter: try_send_message_to_file({}, {:?})", msg, tx_file);
+
+    if save_output {
+        match tx_file.send(msg.to_string()) {
+            Ok(_) => {
+                log::trace!(
+                    "sent message from heuristics::try_send_message_to_file to file handler"
+                );
+            }
+            Err(e) => {
+                log::error!(
+                    "{} {} {}",
+                    status_colorizer("ERROR"),
+                    module_colorizer("heuristics::try_send_message_to_file"),
+                    e
+                );
+            }
+        }
+    }
+    log::trace!("exit: try_send_message_to_file");
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::FeroxChannel;
+    use tokio::sync::mpsc;
 
     #[test]
     /// request a unique string of 32bytes * a value returns correct result
-    fn unique_string_returns_correct_length() {
+    fn heuristics_unique_string_returns_correct_length() {
         for i in 0..10 {
             assert_eq!(unique_string(i).len(), i * 32);
         }
     }
+
+    #[test]
+    /// simply test the default values for wildcardfilter, expect 0, 0
+    fn heuristics_wildcardfilter_dafaults() {
+        let wcf = WildcardFilter::default();
+        assert_eq!(wcf.size, 0);
+        assert_eq!(wcf.dynamic, 0);
+    }
+
+    #[tokio::test(core_threads = 1)]
+    /// tests that given a message and transmitter, the function sends the message across the
+    /// channel
+    async fn heuristics_try_send_message_to_file_sends_when_true() {
+        let (tx, mut rx): FeroxChannel<String> = mpsc::unbounded_channel();
+        let msg = "It really tied the room together.";
+        let should_save = true;
+        try_send_message_to_file(&msg, tx, should_save);
+
+        assert_eq!(rx.recv().await.unwrap(), msg);
+    }
+
+    #[tokio::test(core_threads = 1)]
+    #[should_panic]
+    /// tests that when save_output is false, nothing is sent to the receiver
+    async fn heuristics_try_send_message_to_file_sends_when_false() {
+        let (tx, mut rx): FeroxChannel<String> = mpsc::unbounded_channel();
+        let msg = "I'm the Dude, so that's what you call me.";
+        let should_save = false;
+        try_send_message_to_file(&msg, tx, should_save);
+
+        assert_ne!(rx.recv().await.unwrap(), msg);
+    }
+
+    #[tokio::test(core_threads = 1)]
+    /// tests that when save_output is true, but the receiver is closed, nothing is sent to the receiver
+    /// this test doesn't assert anything, but reaches the error block of the given function and
+    /// can be verified with --nocapture and RUST_LOG being set
+    async fn heuristics_try_send_message_to_file_sends_with_closed_receiver() {
+        env_logger::init();
+        let (tx, mut rx): FeroxChannel<String> = mpsc::unbounded_channel();
+        let msg = "Hey, nice marmot.";
+        let should_save = true;
+        rx.close();
+        try_send_message_to_file(&msg, tx, should_save);
+    }
 }

src/lib.rs

@@ -1,19 +1,26 @@
 pub mod banner;
 pub mod client;
 pub mod config;
+pub mod extractor;
 pub mod heuristics;
 pub mod logger;
 pub mod parser;
 pub mod progress;
+pub mod reporter;
 pub mod scanner;
 pub mod utils;
 
-use reqwest::StatusCode;
+use reqwest::header::HeaderMap;
+use reqwest::{Response, StatusCode, Url};
+use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender};
 
 /// Generic Result type to ease error handling in async contexts
 pub type FeroxResult<T> =
     std::result::Result<T, Box<dyn std::error::Error + Send + Sync + 'static>>;
 
+/// Generic mpsc::unbounded_channel type to tidy up some code
+pub type FeroxChannel<T> = (UnboundedSender<T>, UnboundedReceiver<T>);
+
 /// Version pulled from Cargo.toml at compile time
 pub const VERSION: &str = env!("CARGO_PKG_VERSION");
 
@@ -53,6 +60,118 @@ pub const DEFAULT_STATUS_CODES: [StatusCode; 9] = [
 /// Expected location is in the same directory as the feroxbuster binary.
 pub const DEFAULT_CONFIG_NAME: &str = "ferox-config.toml";
 
+/// A `FeroxResponse`, derived from a `Response` to a submitted `Request`
+#[derive(Debug)]
+pub struct FeroxResponse {
+    /// The final `Url` of this `FeroxResponse`
+    url: Url,
+
+    /// The `StatusCode` of this `FeroxResponse`
+    status: StatusCode,
+
+    /// The full response text
+    text: String,
+
+    /// The content-length of this response, if known
+    content_length: u64,
+
+    /// The `Headers` of this `FeroxResponse`
+    headers: HeaderMap,
+}
+
+/// `FeroxResponse` implementation
+impl FeroxResponse {
+    /// Get the `StatusCode` of this `FeroxResponse`
+    pub fn status(&self) -> &StatusCode {
+        &self.status
+    }
+
+    /// Get the final `Url` of this `FeroxResponse`.
+    pub fn url(&self) -> &Url {
+        &self.url
+    }
+
+    /// Get the full response text
+    pub fn text(&self) -> &str {
+        &self.text
+    }
+
+    /// Get the `Headers` of this `FeroxResponse`
+    pub fn headers(&self) -> &HeaderMap {
+        &self.headers
+    }
+
+    /// Get the content-length of this response, if known
+    pub fn content_length(&self) -> u64 {
+        self.content_length
+    }
+
+    /// Set `FeroxResponse`'s `url` attribute, has no affect if an error occurs
+    pub fn set_url(&mut self, url: &str) {
+        match Url::parse(&url) {
+            Ok(url) => {
+                self.url = url;
+            }
+            Err(e) => {
+                log::error!("Could not parse {} into a Url: {}", url, e);
+            }
+        };
+    }
+
+    /// Make a reasonable guess at whether the response is a file or not
+    ///
+    /// Examines the last part of a path to determine if it has an obvious extension
+    /// i.e. http://localhost/some/path/stuff.js where stuff.js indicates a file
+    ///
+    /// Additionally, inspects query parameters, as they're also often indicative of a file
+    pub fn is_file(&self) -> bool {
+        let has_extension = match self.url.path_segments() {
+            Some(path) => {
+                if let Some(last) = path.last() {
+                    last.contains('.') // last segment has some sort of extension, probably
+                } else {
+                    false
+                }
+            }
+            None => false,
+        };
+
+        self.url.query_pairs().count() > 0 || has_extension
+    }
+
+    /// Create a new `FeroxResponse` from the given `Response`
+    pub async fn from(response: Response, read_body: bool) -> Self {
+        let url = response.url().clone();
+        let status = response.status();
+        let headers = response.headers().clone();
+        let content_length = response.content_length().unwrap_or(0);
+
+        let text = if read_body {
+            // .text() consumes the response, must be called last
+            // additionally, --extract-links is currently the only place we use the body of the
+            // response, so we forego the processing if not performing extraction
+            match response.text().await {
+                // await the response's body
+                Ok(text) => text,
+                Err(e) => {
+                    log::error!("Could not parse body from response: {}", e);
+                    String::new()
+                }
+            }
+        } else {
+            String::new()
+        };
+
+        FeroxResponse {
+            url,
+            status,
+            content_length,
+            text,
+            headers,
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

src/logger.rs

@@ -1,4 +1,5 @@
-use crate::config::PROGRESS_PRINTER;
+use crate::config::{CONFIGURATION, PROGRESS_PRINTER};
+use crate::reporter::{get_cached_file_handle, safe_file_write};
 use console::{style, Color};
 use env_logger::Builder;
 use std::env;
@@ -27,6 +28,19 @@ pub fn initialize(verbosity: u8) {
     let start = Instant::now();
     let mut builder = Builder::from_default_env();
 
+    // I REALLY wanted the logger to also use the reporting channels found in the `reporter`
+    // module. However, in order to properly clean up the channels, all references to the
+    // transmitter side of a channel need to go out of scope, then you can await the future into
+    // which the receiver was moved.
+    //
+    // The problem was that putting a transmitter reference in this closure, which gets initialized
+    // as part of the global logger, made it so that I couldn't destroy/leak/take/swap the last
+    // reference to allow the channels to gracefully close.
+    //
+    // The workaround was to have a RwLock around the file and allow both the logger and the
+    // file handler to both write independent of each other.
+    let locked_file = get_cached_file_handle(&CONFIGURATION.output);
+
     builder
         .format(move |_, record| {
             let t = start.elapsed().as_secs_f32();
@@ -41,13 +55,18 @@ pub fn initialize(verbosity: u8) {
             };
 
             let msg = format!(
-                "{} {:10.03} {}",
+                "{} {:10.03} {}\n",
                 style(level_name).bg(level_color).black(),
                 style(t).dim(),
                 style(record.args()).dim(),
             );
 
-            PROGRESS_PRINTER.println(msg);
+            PROGRESS_PRINTER.println(&msg);
+
+            if let Some(buffered_file) = locked_file.clone() {
+                safe_file_write(&msg, buffered_file);
+            }
+
             Ok(())
         })
         .init();

src/main.rs

@@ -1,14 +1,15 @@
 use feroxbuster::config::{CONFIGURATION, PROGRESS_PRINTER};
 use feroxbuster::scanner::scan_url;
 use feroxbuster::utils::{ferox_print, get_current_depth, module_colorizer, status_colorizer};
-use feroxbuster::{banner, heuristics, logger, FeroxResult};
+use feroxbuster::{banner, heuristics, logger, reporter, FeroxResponse, FeroxResult, VERSION};
 use futures::StreamExt;
 use std::collections::HashSet;
 use std::fs::File;
-use std::io::{BufRead, BufReader};
+use std::io::{stderr, BufRead, BufReader};
 use std::process;
 use std::sync::Arc;
 use tokio::io;
+use tokio::sync::mpsc::UnboundedSender;
 use tokio_util::codec::{FramedRead, LinesCodec};
 
 /// Create a HashSet of Strings from the given wordlist then stores it inside an Arc
@@ -36,7 +37,13 @@ fn get_unique_words_from_wordlist(path: &str) -> FeroxResult<Arc<HashSet<String>
     let mut words = HashSet::new();
 
     for line in reader.lines() {
-        words.insert(line?);
+        let result = line?;
+
+        if result.starts_with('#') || result.is_empty() {
+            continue;
+        }
+
+        words.insert(result);
     }
 
     log::trace!(
@@ -48,8 +55,12 @@ fn get_unique_words_from_wordlist(path: &str) -> FeroxResult<Arc<HashSet<String>
 }
 
 /// Determine whether it's a single url scan or urls are coming from stdin, then scan as needed
-async fn scan(targets: Vec<String>) -> FeroxResult<()> {
-    log::trace!("enter: scan");
+async fn scan(
+    targets: Vec<String>,
+    tx_term: UnboundedSender<FeroxResponse>,
+    tx_file: UnboundedSender<String>,
+) -> FeroxResult<()> {
+    log::trace!("enter: scan({:?}, {:?}, {:?})", targets, tx_term, tx_file);
     // cloning an Arc is cheap (it's basically a pointer into the heap)
     // so that will allow for cheap/safe sharing of a single wordlist across multi-target scans
     // as well as additional directories found as part of recursion
@@ -70,11 +81,13 @@ async fn scan(targets: Vec<String>) -> FeroxResult<()> {
     let mut tasks = vec![];
 
     for target in targets {
-        let wordclone = words.clone();
+        let word_clone = words.clone();
+        let term_clone = tx_term.clone();
+        let file_clone = tx_file.clone();
 
         let task = tokio::spawn(async move {
             let base_depth = get_current_depth(&target);
-            scan_url(&target, wordclone, base_depth).await;
+            scan_url(&target, word_clone, base_depth, term_clone, file_clone).await;
         });
 
         tasks.push(task);
@@ -112,11 +125,18 @@ async fn get_targets() -> FeroxResult<Vec<String>> {
 
 #[tokio::main]
 async fn main() {
+    // setup logging based on the number of -v's used
     logger::initialize(CONFIGURATION.verbosity);
 
+    // can't trace main until after logger is initialized
     log::trace!("enter: main");
     log::debug!("{:#?}", *CONFIGURATION);
 
+    let save_output = !CONFIGURATION.output.is_empty(); // was -o used?
+
+    let (tx_term, tx_file, term_handle, file_handle) =
+        reporter::initialize(&CONFIGURATION.output, save_output);
+
     // get targets from command line or stdin
     let targets = match get_targets().await {
         Ok(t) => t,
@@ -138,20 +158,56 @@ async fn main() {
 
     if !CONFIGURATION.quiet {
         // only print banner if -q isn't used
-        banner::initialize(&targets, &CONFIGURATION);
+        let std_stderr = stderr(); // std::io::stderr
+        banner::initialize(&targets, &CONFIGURATION, &VERSION, std_stderr).await;
     }
 
     // discard non-responsive targets
     let live_targets = heuristics::connectivity_test(&targets).await;
 
-    match scan(live_targets).await {
+    // kick off a scan against any targets determined to be responsive
+    match scan(live_targets, tx_term.clone(), tx_file.clone()).await {
         Ok(_) => {
-            log::info!("Done");
+            log::info!("All scans complete!");
         }
         Err(e) => log::error!("An error occurred: {}", e),
     };
 
-    PROGRESS_PRINTER.finish();
+    // manually drop tx in order for the rx task's while loops to eval to false
+    drop(tx_term);
+    log::trace!("dropped terminal output handler's transmitter");
+
+    log::trace!("awaiting terminal output handler's receiver");
+    // after dropping tx, we can await the future where rx lived
+    match term_handle.await {
+        Ok(_) => {}
+        Err(e) => {
+            log::error!("error awaiting terminal output handler's receiver: {}", e);
+        }
+    }
+    log::trace!("done awaiting terminal output handler's receiver");
+
+    log::trace!("tx_file: {:?}", tx_file);
+    // the same drop/await process used on the terminal handler is repeated for the file handler
+    // we drop the file transmitter every time, because it's created no matter what
+    drop(tx_file);
+
+    log::trace!("dropped file output handler's transmitter");
+    if save_output {
+        // but we only await if -o was specified
+        log::trace!("awaiting file output handler's receiver");
+        match file_handle.unwrap().await {
+            Ok(_) => {}
+            Err(e) => {
+                log::error!("error awaiting file output handler's receiver: {}", e);
+            }
+        }
+        log::trace!("done awaiting file output handler's receiver");
+    }
 
     log::trace!("exit: main");
+
+    // clean-up function for the MultiProgress bar; must be called last in order to still see
+    // the final trace message above
+    PROGRESS_PRINTER.finish();
 }

src/parser.rs

@@ -195,6 +195,13 @@ pub fn initialize() -> App<'static, 'static> {
                     "Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)",
                 ),
         )
+        .arg(
+            Arg::with_name("extract_links")
+                .short("e")
+                .long("extract-links")
+                .takes_value(false)
+                .help("Extract links from response body (html, javascript, etc...); make new requests based on findings (default: false)")
+        )
 
         .after_help(r#"NOTE:
     Options that take multiple values are very flexible.  Consider the following ways of specifying
@@ -225,7 +232,22 @@ EXAMPLES:
     Pass auth token via query parameter
         ./feroxbuster -u http://127.1 --query token=0123456789ABCDEF
 
+    Find links in javascript/html and make additional requests based on results
+        ./feroxbuster -u http://127.1 --extract-links
+
     Ludicrous speed... go!
         ./feroxbuster -u http://127.1 -t 200
     "#)
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    /// initalize parser, expect a clap::App returned
+    fn parser_initialize_gives_defaults() {
+        let app = initialize();
+        assert_eq!(app.get_name(), "feroxbuster");
+    }
+}

src/reporter.rs

@@ -0,0 +1,229 @@
+use crate::config::{CONFIGURATION, PROGRESS_PRINTER};
+use crate::utils::{ferox_print, status_colorizer};
+use crate::{FeroxChannel, FeroxResponse};
+use console::strip_ansi_codes;
+use std::io::Write;
+use std::sync::{Arc, Once, RwLock};
+use std::{fs, io};
+use tokio::sync::mpsc::{self, UnboundedReceiver, UnboundedSender};
+use tokio::task::JoinHandle;
+
+/// Singleton buffered file behind an Arc/RwLock; used for file writes from two locations:
+///     - [logger::initialize](../logger/fn.initialize.html) (specifically a closure on the global logger instance)
+///     - `reporter::spawn_file_handler`
+pub static mut LOCKED_FILE: Option<Arc<RwLock<io::BufWriter<fs::File>>>> = None;
+
+/// An initializer Once variable used to create `LOCKED_FILE`
+static INIT: Once = Once::new();
+
+// Accessing a `static mut` is unsafe much of the time, but if we do so
+// in a synchronized fashion (e.g., write once or read all) then we're
+// good to go!
+//
+// This function will only call `open_file` once, and will
+// otherwise always return the value returned from the first invocation.
+pub fn get_cached_file_handle(filename: &str) -> Option<Arc<RwLock<io::BufWriter<fs::File>>>> {
+    unsafe {
+        INIT.call_once(|| {
+            LOCKED_FILE = open_file(&filename);
+        });
+        LOCKED_FILE.clone()
+    }
+}
+
+/// Creates all required output handlers (terminal, file) and returns
+/// the transmitter sides of each mpsc along with each receiver's future's JoinHandle to be awaited
+///
+/// Any other module that needs to write a Response to stdout or output results to a file should
+/// be passed a clone of the appropriate returned transmitter
+pub fn initialize(
+    output_file: &str,
+    save_output: bool,
+) -> (
+    UnboundedSender<FeroxResponse>,
+    UnboundedSender<String>,
+    JoinHandle<()>,
+    Option<JoinHandle<()>>,
+) {
+    log::trace!("enter: initialize({}, {})", output_file, save_output);
+
+    let (tx_rpt, rx_rpt): FeroxChannel<FeroxResponse> = mpsc::unbounded_channel();
+    let (tx_file, rx_file): FeroxChannel<String> = mpsc::unbounded_channel();
+
+    let file_clone = tx_file.clone();
+
+    let term_reporter =
+        tokio::spawn(async move { spawn_terminal_reporter(rx_rpt, file_clone, save_output).await });
+
+    let file_reporter = if save_output {
+        // -o used, need to spawn the thread for writing to disk
+        let file_clone = output_file.to_string();
+        Some(tokio::spawn(async move {
+            spawn_file_reporter(rx_file, &file_clone).await
+        }))
+    } else {
+        None
+    };
+
+    log::trace!(
+        "exit: initialize -> ({:?}, {:?}, {:?}, {:?})",
+        tx_rpt,
+        tx_file,
+        term_reporter,
+        file_reporter
+    );
+    (tx_rpt, tx_file, term_reporter, file_reporter)
+}
+
+/// Spawn a single consumer task (sc side of mpsc)
+///
+/// The consumer simply receives responses and prints them if they meet the given
+/// reporting criteria
+async fn spawn_terminal_reporter(
+    mut resp_chan: UnboundedReceiver<FeroxResponse>,
+    file_chan: UnboundedSender<String>,
+    save_output: bool,
+) {
+    log::trace!(
+        "enter: spawn_terminal_reporter({:?}, {:?}, {})",
+        resp_chan,
+        file_chan,
+        save_output
+    );
+
+    while let Some(resp) = resp_chan.recv().await {
+        log::debug!("received {} on reporting channel", resp.url());
+
+        if CONFIGURATION.statuscodes.contains(&resp.status().as_u16()) {
+            let report = if CONFIGURATION.quiet {
+                // -q used, just need the url
+                format!("{}\n", resp.url())
+            } else {
+                // normal printing with status and size
+                let status = status_colorizer(&resp.status().as_str());
+                format!(
+                    // example output
+                    // 200       3280 https://localhost.com/FAQ
+                    "{} {:>10} {}\n",
+                    status,
+                    resp.content_length(),
+                    resp.url()
+                )
+            };
+
+            // print to stdout
+            ferox_print(&report, &PROGRESS_PRINTER);
+
+            if save_output {
+                // -o used, need to send the report to be written out to disk
+                match file_chan.send(report.to_string()) {
+                    Ok(_) => {
+                        log::debug!("Sent {} to file handler", resp.url());
+                    }
+                    Err(e) => {
+                        log::error!("Could not send {} to file handler: {}", resp.url(), e);
+                    }
+                }
+            }
+        }
+        log::debug!("report complete: {}", resp.url());
+    }
+    log::trace!("exit: spawn_terminal_reporter");
+}
+
+/// Spawn a single consumer task (sc side of mpsc)
+///
+/// The consumer simply receives responses and writes them to the given output file if they meet
+/// the given reporting criteria
+async fn spawn_file_reporter(mut report_channel: UnboundedReceiver<String>, output_file: &str) {
+    let buffered_file = match get_cached_file_handle(&CONFIGURATION.output) {
+        Some(file) => file,
+        None => {
+            log::trace!("exit: spawn_file_reporter");
+            return;
+        }
+    };
+
+    log::trace!(
+        "enter: spawn_file_reporter({:?}, {})",
+        report_channel,
+        output_file
+    );
+
+    log::info!("Writing scan results to {}", output_file);
+
+    while let Some(report) = report_channel.recv().await {
+        safe_file_write(&report, buffered_file.clone());
+    }
+
+    log::trace!("exit: spawn_file_reporter");
+}
+
+/// Given the path to a file, open the file in append mode (create it if it doesn't exist) and
+/// return a reference to the file that is buffered and locked
+fn open_file(filename: &str) -> Option<Arc<RwLock<io::BufWriter<fs::File>>>> {
+    log::trace!("enter: open_file({})", filename);
+
+    match fs::OpenOptions::new() // std fs
+        .create(true)
+        .append(true)
+        .open(filename)
+    {
+        Ok(file) => {
+            let writer = io::BufWriter::new(file); // std io
+
+            let locked_file = Some(Arc::new(RwLock::new(writer)));
+
+            log::trace!("exit: open_file -> {:?}", locked_file);
+            locked_file
+        }
+        Err(e) => {
+            log::error!("{}", e);
+            log::trace!("exit: open_file -> None");
+            None
+        }
+    }
+}
+
+/// Given a string and a reference to a locked buffered file, write the contents and flush
+/// the buffer to disk.
+pub fn safe_file_write(contents: &str, locked_file: Arc<RwLock<io::BufWriter<fs::File>>>) {
+    // note to future self: adding logging of anything other than error to this function
+    // is a bad idea. we call this function while processing records generated by the logger.
+    // If we then call log::... while already processing some logging output, it results in
+    // the second log entry being injected into the first.
+
+    let contents = strip_ansi_codes(&contents);
+
+    if let Ok(mut handle) = locked_file.write() {
+        // write lock acquired
+        match handle.write(contents.as_bytes()) {
+            Ok(_) => {}
+            Err(e) => {
+                log::error!("could not write report to disk: {}", e);
+            }
+        }
+
+        match handle.flush() {
+            // this function is used within async functions/loops, so i'm flushing so that in
+            // the event of a ctrl+c or w/e results seen so far are saved instead of left lying
+            // around in the buffer
+            Ok(_) => {}
+            Err(e) => {
+                log::error!("error writing to file: {}", e);
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    #[should_panic]
+    /// asserts that an empty string for a filename returns None
+    fn reporter_get_cached_file_handle_without_filename_returns_none() {
+        let _used = get_cached_file_handle(&"").unwrap();
+    }
+}

src/scanner.rs

@@ -1,128 +1,29 @@
-use crate::config::{CONFIGURATION, PROGRESS_BAR, PROGRESS_PRINTER};
+use crate::config::{CONFIGURATION, PROGRESS_BAR};
+use crate::extractor::get_links;
 use crate::heuristics::WildcardFilter;
-use crate::utils::{
-    ferox_print, format_url, get_current_depth, get_url_path_length, make_request, status_colorizer,
-};
-use crate::{heuristics, progress};
+use crate::utils::{format_url, get_current_depth, get_url_path_length, make_request};
+use crate::{heuristics, progress, FeroxChannel, FeroxResponse};
 use futures::future::{BoxFuture, FutureExt};
 use futures::{stream, StreamExt};
 use lazy_static::lazy_static;
-use reqwest::{Response, Url};
+use reqwest::Url;
 use std::collections::HashSet;
 use std::convert::TryInto;
 use std::ops::Deref;
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::{Arc, RwLock};
-use tokio::fs;
-use tokio::io::{self, AsyncWriteExt};
 use tokio::sync::mpsc::{self, UnboundedReceiver, UnboundedSender};
 use tokio::task::JoinHandle;
 
+/// Single atomic number that gets incremented once, used to track first scan vs. all others
 static CALL_COUNT: AtomicUsize = AtomicUsize::new(0);
 
 lazy_static! {
-    /// Global configuration state
+    /// Set of urls that have been sent to [scan_url](fn.scan_url.html), used for deduplication
     static ref SCANNED_URLS: RwLock<HashSet<String>> = RwLock::new(HashSet::new());
-}
 
-/// Spawn a single consumer task (sc side of mpsc)
-///
-/// The consumer simply receives responses and writes them to the given output file if they meet
-/// the given reporting criteria
-async fn spawn_file_reporter(mut report_channel: UnboundedReceiver<Response>) {
-    log::trace!("enter: spawn_file_reporter({:?}", report_channel);
-
-    log::info!("Writing scan results to {}", CONFIGURATION.output);
-
-    match fs::OpenOptions::new() // tokio fs
-        .create(true)
-        .append(true)
-        .open(&CONFIGURATION.output)
-        .await
-    {
-        Ok(outfile) => {
-            log::debug!("{:?} opened in append mode", outfile);
-
-            let mut writer = io::BufWriter::new(outfile); // tokio BufWriter
-
-            while let Some(resp) = report_channel.recv().await {
-                log::debug!("received {} on reporting channel", resp.url());
-
-                if CONFIGURATION.statuscodes.contains(&resp.status().as_u16()) {
-                    let report = if CONFIGURATION.quiet {
-                        format!("{}\n", resp.url())
-                    } else {
-                        // example output
-                        // 200       3280 https://localhost.com/FAQ
-                        format!(
-                            "{} {:>10} {}\n",
-                            resp.status().as_str(),
-                            resp.content_length().unwrap_or(0),
-                            resp.url()
-                        )
-                    };
-
-                    match writer.write(report.as_bytes()).await {
-                        Ok(written) => {
-                            log::trace!("wrote {} bytes to {}", written, CONFIGURATION.output);
-                        }
-                        Err(e) => {
-                            log::error!("could not write report to disk: {}", e);
-                        }
-                    }
-                }
-
-                match writer.flush().await {
-                    // i'm flushing inside the while loop so in the event of a ctrl+c or w/e
-                    // results seen so far are saved instead of left lying around in the buffer
-                    Ok(_) => {}
-                    Err(e) => {
-                        log::error!("error writing to file: {}", e);
-                    }
-                }
-
-                log::debug!("report complete: {}", resp.url());
-            }
-        }
-        Err(e) => {
-            log::error!("error opening file: {}", e);
-        }
-    }
-
-    log::trace!("exit: spawn_file_reporter");
-}
-
-/// Spawn a single consumer task (sc side of mpsc)
-///
-/// The consumer simply receives responses and prints them if they meet the given
-/// reporting criteria
-async fn spawn_terminal_reporter(mut report_channel: UnboundedReceiver<Response>) {
-    log::trace!("enter: spawn_terminal_reporter({:?})", report_channel);
-
-    while let Some(resp) = report_channel.recv().await {
-        log::debug!("received {} on reporting channel", resp.url());
-
-        if CONFIGURATION.statuscodes.contains(&resp.status().as_u16()) {
-            if CONFIGURATION.quiet {
-                ferox_print(&format!("{}", resp.url()), &PROGRESS_PRINTER);
-            } else {
-                let status = status_colorizer(&resp.status().as_str());
-                ferox_print(
-                    &format!(
-                        // example output
-                        // 200       3280 https://localhost.com/FAQ
-                        "{} {:>10} {}",
-                        status,
-                        resp.content_length().unwrap_or(0),
-                        resp.url()
-                    ),
-                    &PROGRESS_PRINTER,
-                );
-            }
-        }
-        log::debug!("report complete: {}", resp.url());
-    }
-    log::trace!("exit: spawn_terminal_reporter");
+    /// Vector of WildcardFilters that have been ID'd through heuristics
+    static ref WILDCARD_FILTERS: Arc<RwLock<Vec<Arc<WildcardFilter>>>> = Arc::new(RwLock::new(Vec::<Arc<WildcardFilter>>::new()));
 }
 
 /// Adds the given url to `SCANNED_URLS`
@@ -162,6 +63,42 @@ fn add_url_to_list_of_scanned_urls(resp: &str, scanned_urls: &RwLock<HashSet<Str
     }
 }
 
+/// Adds the given WildcardFilter to `WILDCARD_FILTERS`
+///
+/// If `WILDCARD_FILTERS` did not already contain the filter, return true; otherwise return false
+fn add_filter_to_list_of_wildcard_filters(
+    filter: Arc<WildcardFilter>,
+    wildcard_filters: Arc<RwLock<Vec<Arc<WildcardFilter>>>>,
+) -> bool {
+    log::trace!(
+        "enter: add_filter_to_list_of_wildcard_filters({:?}, {:?})",
+        filter,
+        wildcard_filters
+    );
+
+    match wildcard_filters.write() {
+        Ok(mut filters) => {
+            // If the set did not contain the assigned filter, true is returned.
+            // If the set did contain the assigned filter, false is returned.
+            if filters.contains(&filter) {
+                log::trace!("exit: add_filter_to_list_of_wildcard_filters -> false");
+                return false;
+            }
+
+            filters.push(filter);
+
+            log::trace!("exit: add_filter_to_list_of_wildcard_filters -> true");
+            true
+        }
+        Err(e) => {
+            // poisoned lock
+            log::error!("Set of wildcard filters poisoned: {}", e);
+            log::trace!("exit: add_filter_to_list_of_wildcard_filters -> false");
+            false
+        }
+    }
+}
+
 /// Spawn a single consumer task (sc side of mpsc)
 ///
 /// The consumer simply receives Urls and scans them
@@ -169,12 +106,16 @@ fn spawn_recursion_handler(
     mut recursion_channel: UnboundedReceiver<String>,
     wordlist: Arc<HashSet<String>>,
     base_depth: usize,
+    tx_term: UnboundedSender<FeroxResponse>,
+    tx_file: UnboundedSender<String>,
 ) -> BoxFuture<'static, Vec<JoinHandle<()>>> {
     log::trace!(
-        "enter: spawn_recursion_handler({:?}, wordlist[{} words...], {})",
+        "enter: spawn_recursion_handler({:?}, wordlist[{} words...], {}, {:?}, {:?})",
         recursion_channel,
         wordlist.len(),
-        base_depth
+        base_depth,
+        tx_term,
+        tx_file
     );
 
     let boxed_future = async move {
@@ -188,10 +129,21 @@ fn spawn_recursion_handler(
             }
 
             log::info!("received {} on recursion channel", resp);
-            let clonedresp = resp.clone();
-            let clonedlist = wordlist.clone();
+
+            let term_clone = tx_term.clone();
+            let file_clone = tx_file.clone();
+            let resp_clone = resp.clone();
+            let list_clone = wordlist.clone();
+
             scans.push(tokio::spawn(async move {
-                scan_url(clonedresp.to_owned().as_str(), clonedlist, base_depth).await
+                scan_url(
+                    resp_clone.to_owned().as_str(),
+                    list_clone,
+                    base_depth,
+                    term_clone,
+                    file_clone,
+                )
+                .await
             }));
         }
         scans
@@ -248,7 +200,7 @@ fn create_urls(target_url: &str, word: &str, extensions: &[String]) -> Vec<Url>
 ///
 /// handles 2xx and 3xx responses by either checking if the url ends with a / (2xx)
 /// or if the Location header is present and matches the base url + / (3xx)
-fn response_is_directory(response: &Response) -> bool {
+fn response_is_directory(response: &FeroxResponse) -> bool {
     log::trace!("enter: is_directory({:?})", response);
 
     if response.status().is_redirection() {
@@ -300,10 +252,15 @@ fn response_is_directory(response: &Response) -> bool {
 ///
 /// Essentially looks at the Url path and determines how many directories are present in the
 /// given Url
-fn reached_max_depth(url: &Url, base_depth: usize) -> bool {
-    log::trace!("enter: reached_max_depth({}, {})", url, base_depth);
+fn reached_max_depth(url: &Url, base_depth: usize, max_depth: usize) -> bool {
+    log::trace!(
+        "enter: reached_max_depth({}, {}, {})",
+        url,
+        base_depth,
+        max_depth
+    );
 
-    if CONFIGURATION.depth == 0 {
+    if max_depth == 0 {
         // early return, as 0 means recurse forever; no additional processing needed
         log::trace!("exit: reached_max_depth -> false");
         return false;
@@ -311,7 +268,7 @@ fn reached_max_depth(url: &Url, base_depth: usize) -> bool {
 
     let depth = get_current_depth(url.as_str());
 
-    if depth - base_depth >= CONFIGURATION.depth {
+    if depth - base_depth >= max_depth {
         return true;
     }
 
@@ -323,7 +280,7 @@ fn reached_max_depth(url: &Url, base_depth: usize) -> bool {
 ///
 /// When a recursion opportunity is found, the new url is sent across the recursion channel
 async fn try_recursion(
-    response: &Response,
+    response: &FeroxResponse,
     base_depth: usize,
     transmitter: UnboundedSender<String>,
 ) {
@@ -334,7 +291,9 @@ async fn try_recursion(
         transmitter
     );
 
-    if !reached_max_depth(response.url(), base_depth) && response_is_directory(&response) {
+    if !reached_max_depth(response.url(), base_depth, CONFIGURATION.depth)
+        && response_is_directory(&response)
+    {
         if CONFIGURATION.redirects {
             // response is 2xx can simply send it because we're following redirects
             log::info!("Added new directory to recursive scan: {}", response.url());
@@ -345,9 +304,8 @@ async fn try_recursion(
                 }
                 Err(e) => {
                     log::error!(
-                        "could not send {} across {:?}: {}",
+                        "Could not send {} to recursion handler: {}",
                         response.url(),
-                        transmitter,
                         e
                     );
                 }
@@ -361,9 +319,8 @@ async fn try_recursion(
                 Ok(_) => {}
                 Err(e) => {
                     log::error!(
-                        "could not send {}/ across {:?}: {}",
+                        "Could not send {}/ to recursion handler: {}",
                         response.url(),
-                        transmitter,
                         e
                     );
                 }
@@ -373,6 +330,54 @@ async fn try_recursion(
     log::trace!("exit: try_recursion");
 }
 
+/// Simple helper to stay DRY; determines whether or not a given `FeroxResponse` should be reported
+/// to the user or not.
+pub fn should_filter_response(content_len: &u64, url: &Url) -> bool {
+    if CONFIGURATION.sizefilters.contains(content_len) {
+        // filtered value from --sizefilters, move on to the next url
+        log::debug!("size filter: filtered out {}", url);
+        return true;
+    }
+
+    match WILDCARD_FILTERS.read() {
+        Ok(filters) => {
+            for filter in filters.iter() {
+                if CONFIGURATION.dontfilter {
+                    // quick return if dontfilter is set
+                    return false;
+                }
+
+                if filter.size > 0 && filter.size == *content_len {
+                    // static wildcard size found during testing
+                    // size isn't default, size equals response length, and auto-filter is on
+                    log::debug!("static wildcard: filtered out {}", url);
+                    return true;
+                }
+
+                if filter.dynamic > 0 {
+                    // dynamic wildcard offset found during testing
+
+                    // I'm about to manually split this url path instead of using reqwest::Url's
+                    // builtin parsing. The reason is that they call .split() on the url path
+                    // except that I don't want an empty string taking up the last index in the
+                    // event that the url ends with a forward slash.  It's ugly enough to be split
+                    // into its own function for readability.
+                    let url_len = get_url_path_length(&url);
+
+                    if url_len + filter.dynamic == *content_len {
+                        log::debug!("dynamic wildcard: filtered out {}", url);
+                        return true;
+                    }
+                }
+            }
+        }
+        Err(e) => {
+            log::error!("{}", e);
+        }
+    }
+    false
+}
+
 /// Wrapper for [make_request](fn.make_request.html)
 ///
 /// Handles making multiple requests based on the presence of extensions
@@ -382,9 +387,8 @@ async fn make_requests(
     target_url: &str,
     word: &str,
     base_depth: usize,
-    filter: Arc<WildcardFilter>,
     dir_chan: UnboundedSender<String>,
-    report_chan: UnboundedSender<Response>,
+    report_chan: UnboundedSender<FeroxResponse>,
 ) {
     log::trace!(
         "enter: make_requests({}, {}, {}, {:?}, {:?})",
@@ -399,79 +403,139 @@ async fn make_requests(
 
     for url in urls {
         if let Ok(response) = make_request(&CONFIGURATION.client, &url).await {
-            // response came back without error
+            // response came back without error, convert it to FeroxResponse
+            let ferox_response = FeroxResponse::from(response, CONFIGURATION.extract_links).await;
 
             // do recursion if appropriate
-            if !CONFIGURATION.norecursion && response_is_directory(&response) {
-                try_recursion(&response, base_depth, dir_chan.clone()).await;
+            if !CONFIGURATION.norecursion {
+                try_recursion(&ferox_response, base_depth, dir_chan.clone()).await;
             }
 
             // purposefully doing recursion before filtering. the thought process is that
             // even though this particular url is filtered, subsequent urls may not
 
-            let content_len = &response.content_length().unwrap_or(0);
+            let content_len = &ferox_response.content_length();
 
-            if CONFIGURATION.sizefilters.contains(content_len) {
-                // filtered value from --sizefilters, move on to the next url
-                log::debug!("size filter: filtered out {}", response.url());
+            if should_filter_response(content_len, &ferox_response.url()) {
                 continue;
             }
 
-            if filter.size > 0 && filter.size == *content_len && !CONFIGURATION.dontfilter {
-                // static wildcard size found during testing
-                // size isn't default, size equals response length, and auto-filter is on
-                log::debug!("static wildcard: filtered out {}", response.url());
-                continue;
-            }
+            if CONFIGURATION.extract_links && !ferox_response.status().is_redirection() {
+                let new_links = get_links(&ferox_response).await;
 
-            if filter.dynamic > 0 && !CONFIGURATION.dontfilter {
-                // dynamic wildcard offset found during testing
+                for new_link in new_links {
+                    let unknown = add_url_to_list_of_scanned_urls(&new_link, &SCANNED_URLS);
 
-                // I'm about to manually split this url path instead of using reqwest::Url's
-                // builtin parsing. The reason is that they call .split() on the url path
-                // except that I don't want an empty string taking up the last index in the
-                // event that the url ends with a forward slash.  It's ugly enough to be split
-                // into its own function for readability.
-                let url_len = get_url_path_length(&response.url());
+                    if !unknown {
+                        // not unknown, i.e. we've seen the url before and don't need to scan again
+                        continue;
+                    }
 
-                if url_len + filter.dynamic == *content_len {
-                    log::debug!("dynamic wildcard: filtered out {}", response.url());
-                    continue;
+                    // create a url based on the given command line options, continue on error
+                    let new_url = match format_url(
+                        &new_link,
+                        &"",
+                        CONFIGURATION.addslash,
+                        &CONFIGURATION.queries,
+                        None,
+                    ) {
+                        Ok(url) => url,
+                        Err(_) => continue,
+                    };
+
+                    // make the request and store the response
+                    let new_response = match make_request(&CONFIGURATION.client, &new_url).await {
+                        Ok(resp) => resp,
+                        Err(_) => continue,
+                    };
+
+                    let mut new_ferox_response =
+                        FeroxResponse::from(new_response, CONFIGURATION.extract_links).await;
+
+                    // filter if necessary
+                    let new_content_len = &new_ferox_response.content_length();
+                    if should_filter_response(new_content_len, &new_ferox_response.url()) {
+                        continue;
+                    }
+
+                    if new_ferox_response.is_file() {
+                        // very likely a file, simply request and report
+                        log::debug!(
+                            "Singular extraction: {} ({})",
+                            new_ferox_response.url(),
+                            new_ferox_response.status().as_str(),
+                        );
+
+                        send_report(report_chan.clone(), new_ferox_response);
+
+                        continue;
+                    }
+
+                    if !CONFIGURATION.norecursion {
+                        log::debug!(
+                            "Recursive extraction: {} ({})",
+                            new_ferox_response.url(),
+                            new_ferox_response.status().as_str()
+                        );
+
+                        if new_ferox_response.status().is_success()
+                            && !new_ferox_response.url().as_str().ends_with('/')
+                        {
+                            // since all of these are 2xx, recursion is only attempted if the
+                            // url ends in a /. I am actually ok with adding the slash and not
+                            // adding it, as both have merit.  Leaving it in for now to see how
+                            // things turn out (current as of: v1.1.0)
+                            new_ferox_response.set_url(&format!("{}/", new_ferox_response.url()));
+                        }
+
+                        try_recursion(&new_ferox_response, base_depth, dir_chan.clone()).await;
+                    }
                 }
             }
 
             // everything else should be reported
-            match report_chan.send(response) {
-                Ok(_) => {
-                    log::debug!("sent {}/{} over reporting channel", &target_url, &word);
-                }
-                Err(e) => {
-                    log::error!("wtf: {}", e);
-                }
-            }
+            send_report(report_chan.clone(), ferox_response);
         }
     }
     log::trace!("exit: make_requests");
 }
 
+/// Simple helper to send a `FeroxResponse` over the tx side of an `mpsc::unbounded_channel`
+fn send_report(report_sender: UnboundedSender<FeroxResponse>, response: FeroxResponse) {
+    log::trace!("enter: send_report({:?}, {:?}", report_sender, response);
+
+    match report_sender.send(response) {
+        Ok(_) => {}
+        Err(e) => {
+            log::error!("{}", e);
+        }
+    }
+
+    log::trace!("exit: send_report");
+}
+
 /// Scan a given url using a given wordlist
 ///
 /// This is the primary entrypoint for the scanner
-pub async fn scan_url(target_url: &str, wordlist: Arc<HashSet<String>>, base_depth: usize) {
+pub async fn scan_url(
+    target_url: &str,
+    wordlist: Arc<HashSet<String>>,
+    base_depth: usize,
+    tx_term: UnboundedSender<FeroxResponse>,
+    tx_file: UnboundedSender<String>,
+) {
     log::trace!(
-        "enter: scan_url({:?}, wordlist[{} words...], {})",
+        "enter: scan_url({:?}, wordlist[{} words...], {}, {:?}, {:?})",
         target_url,
         wordlist.len(),
-        base_depth
+        base_depth,
+        tx_term,
+        tx_file
     );
 
     log::info!("Starting scan against: {}", target_url);
 
-    let (tx_rpt, rx_rpt): (UnboundedSender<Response>, UnboundedReceiver<Response>) =
-        mpsc::unbounded_channel();
-
-    let (tx_dir, rx_dir): (UnboundedSender<String>, UnboundedReceiver<String>) =
-        mpsc::unbounded_channel();
+    let (tx_dir, rx_dir): FeroxChannel<String> = mpsc::unbounded_channel();
 
     let num_reqs_expected: u64 = if CONFIGURATION.extensions.is_empty() {
         wordlist.len().try_into().unwrap()
@@ -493,48 +557,42 @@ pub async fn scan_url(target_url: &str, wordlist: Arc<HashSet<String>>, base_dep
         add_url_to_list_of_scanned_urls(&target_url, &SCANNED_URLS);
     }
 
+    // Arc clones to be passed around to the various scans
     let wildcard_bar = progress_bar.clone();
-
-    let reporter = if !CONFIGURATION.output.is_empty() {
-        // output file defined
-        tokio::spawn(async move { spawn_file_reporter(rx_rpt).await })
-    } else {
-        tokio::spawn(async move { spawn_terminal_reporter(rx_rpt).await })
-    };
-
-    // lifetime satisfiers, as it's an Arc, clones are cheap anyway
-    let looping_words = wordlist.clone();
+    let heuristics_file_clone = tx_file.clone();
+    let recurser_term_clone = tx_term.clone();
+    let recurser_file_clone = tx_file.clone();
     let recurser_words = wordlist.clone();
+    let looping_words = wordlist.clone();
 
-    let recurser =
-        tokio::spawn(
-            async move { spawn_recursion_handler(rx_dir, recurser_words, base_depth).await },
-        );
+    let recurser = tokio::spawn(async move {
+        spawn_recursion_handler(
+            rx_dir,
+            recurser_words,
+            base_depth,
+            recurser_term_clone,
+            recurser_file_clone,
+        )
+        .await
+    });
 
-    let filter = match heuristics::wildcard_test(&target_url, wildcard_bar).await {
-        Some(f) => {
-            if CONFIGURATION.dontfilter {
-                // don't auto filter, i.e. use the defaults
-                Arc::new(WildcardFilter::default())
-            } else {
-                Arc::new(f)
-            }
-        }
-        None => Arc::new(WildcardFilter::default()),
-    };
+    let filter =
+        match heuristics::wildcard_test(&target_url, wildcard_bar, heuristics_file_clone).await {
+            Some(f) => Arc::new(f),
+            None => Arc::new(WildcardFilter::default()),
+        };
+
+    add_filter_to_list_of_wildcard_filters(filter.clone(), WILDCARD_FILTERS.clone());
 
     // producer tasks (mp of mpsc); responsible for making requests
     let producers = stream::iter(looping_words.deref().to_owned())
         .map(|word| {
-            let wc_filter = filter.clone();
             let txd = tx_dir.clone();
-            let txr = tx_rpt.clone();
+            let txr = tx_term.clone();
             let pb = progress_bar.clone(); // progress bar is an Arc around internal state
             let tgt = target_url.to_string(); // done to satisfy 'static lifetime below
             (
-                tokio::spawn(async move {
-                    make_requests(&tgt, &word, base_depth, wc_filter, txd, txr).await
-                }),
+                tokio::spawn(async move { make_requests(&tgt, &word, base_depth, txd, txr).await }),
                 pb,
             )
         })
@@ -565,18 +623,6 @@ pub async fn scan_url(target_url: &str, wordlist: Arc<HashSet<String>>, base_dep
     futures::future::join_all(recurser.await.unwrap()).await;
     log::trace!("done awaiting recursive scan receiver/scans");
 
-    // same thing here, drop report tx so the rx can finish up
-    log::trace!("dropped report handler's transmitter");
-    drop(tx_rpt);
-
-    log::trace!("awaiting report receiver");
-    match reporter.await {
-        Ok(_) => {}
-        Err(e) => {
-            log::error!("error awaiting report receiver: {}", e);
-        }
-    }
-    log::trace!("done awaiting report receiver");
     log::trace!("exit: scan_url");
 }
 
@@ -638,6 +684,46 @@ mod tests {
         }
     }
 
+    #[test]
+    /// call reached_max_depth with max depth of zero, which is infinite recursion, expect false
+    fn reached_max_depth_returns_early_on_zero() {
+        let url = Url::parse("http://localhost").unwrap();
+        let result = reached_max_depth(&url, 0, 0);
+        assert!(!result);
+    }
+
+    #[test]
+    /// call reached_max_depth with url depth equal to max depth, expect true
+    fn reached_max_depth_current_depth_equals_max() {
+        let url = Url::parse("http://localhost/one/two").unwrap();
+        let result = reached_max_depth(&url, 0, 2);
+        assert!(result);
+    }
+
+    #[test]
+    /// call reached_max_depth with url dpeth less than max depth, expect false
+    fn reached_max_depth_current_depth_less_than_max() {
+        let url = Url::parse("http://localhost").unwrap();
+        let result = reached_max_depth(&url, 0, 2);
+        assert!(!result);
+    }
+
+    #[test]
+    /// call reached_max_depth with url of 2, base depth of 2, and max depth of 2, expect false
+    fn reached_max_depth_base_depth_equals_max_depth() {
+        let url = Url::parse("http://localhost/one/two").unwrap();
+        let result = reached_max_depth(&url, 2, 2);
+        assert!(!result);
+    }
+
+    #[test]
+    /// call reached_max_depth with url depth greater than max depth, expect true
+    fn reached_max_depth_current_greater_than_max() {
+        let url = Url::parse("http://localhost/one/two/three").unwrap();
+        let result = reached_max_depth(&url, 0, 2);
+        assert!(result);
+    }
+
     #[test]
     /// add an unknown url to the hashset, expect true
     fn add_url_to_list_of_scanned_urls_with_unknown_url() {
@@ -672,4 +758,30 @@ mod tests {
 
         assert_eq!(add_url_to_list_of_scanned_urls(url, &urls), false);
     }
+
+    #[test]
+    /// add a wildcard filter with the `size` attribute set to WILDCARD_FILTERS and ensure that
+    /// should_filter_response correctly returns true
+    fn should_filter_response_filters_wildcard_size() {
+        let mut filter = WildcardFilter::default();
+        let url = Url::parse("http://localhost").unwrap();
+        filter.size = 18;
+        let filter = Arc::new(filter);
+        add_filter_to_list_of_wildcard_filters(filter, WILDCARD_FILTERS.clone());
+        let result = should_filter_response(&18, &url);
+        assert!(result);
+    }
+
+    #[test]
+    /// add a wildcard filter with the `dynamic` attribute set to WILDCARD_FILTERS and ensure that
+    /// should_filter_response correctly returns true
+    fn should_filter_response_filters_wildcard_dynamic() {
+        let mut filter = WildcardFilter::default();
+        let url = Url::parse("http://localhost/some-path").unwrap();
+        filter.dynamic = 9;
+        let filter = Arc::new(filter);
+        add_filter_to_list_of_wildcard_filters(filter, WILDCARD_FILTERS.clone());
+        let result = should_filter_response(&18, &url);
+        assert!(result);
+    }
 }

src/utils.rs

@@ -160,7 +160,11 @@ pub fn format_url(
     //
     // the transforms that occur here will need to keep this in mind, i.e. add a slash to preserve
     // the current directory sent as part of the url
-    let url = if !url.ends_with('/') {
+    let url = if word.is_empty() {
+        // v1.0.6: added during --extract-links feature inplementation to support creating urls
+        // that were extracted from response bodies, i.e. http://localhost/some/path/js/main.js
+        url.to_string()
+    } else if !url.ends_with('/') {
         format!("{}/", url)
     } else {
         url.to_string()

tests/test_banner.rs

@@ -11,7 +11,7 @@ fn banner_prints_proxy() -> Result<(), Box<dyn std::error::Error>> {
         String::from("http://localhost"),
         String::from("http://schmocalhost"),
     ];
-    let (tmp_dir, file) = setup_tmp_directory(&urls)?;
+    let (tmp_dir, file) = setup_tmp_directory(&urls, "wordlist")?;
 
     Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -536,3 +536,30 @@ fn banner_doesnt_print() -> Result<(), Box<dyn std::error::Error>> {
         ));
     Ok(())
 }
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + extract-links
+fn banner_prints_extract_links() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-e")
+        .assert()
+        .failure()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Extract Links"))
+                .and(predicate::str::contains("true"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}

tests/test_config.rs

@@ -0,0 +1,27 @@
+mod utils;
+use assert_cmd::prelude::*;
+use predicates::prelude::*;
+use std::process::Command;
+use utils::{setup_tmp_directory, teardown_tmp_directory};
+
+#[test]
+/// send a single valid request, expect a 200 response
+fn read_in_config_file_for_settings() -> Result<(), Box<dyn std::error::Error>> {
+    let (tmp_dir, file) = setup_tmp_directory(&["threads = 37".to_string()], "ferox-config.toml")?;
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .current_dir(&tmp_dir)
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-vvvv")
+        .assert()
+        .failure()
+        .stderr(predicate::str::contains("│ 37"));
+
+    teardown_tmp_directory(tmp_dir);
+
+    Ok(())
+}

tests/test_extractor.rs

@@ -0,0 +1,229 @@
+mod utils;
+use assert_cmd::prelude::*;
+use httpmock::Method::GET;
+use httpmock::{Mock, MockServer};
+use predicates::prelude::*;
+use std::process::Command;
+use utils::{setup_tmp_directory, teardown_tmp_directory};
+
+#[test]
+/// send a request to a page that contains a relative link, --extract-links should find the link
+/// and make a request to the new link
+fn extractor_finds_absolute_url() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/LICENSE")
+        .return_status(200)
+        .return_body(&srv.url("'/homepage/assets/img/icons/handshake.svg'"))
+        .create_on(&srv);
+
+    let mock_two = Mock::new()
+        .expect_method(GET)
+        .expect_path("/homepage/assets/img/icons/handshake.svg")
+        .return_status(200)
+        .create_on(&srv);
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--extract-links")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .and(predicate::str::contains("200"))
+            .and(predicate::str::contains(
+                "/homepage/assets/img/icons/handshake.svg",
+            )),
+    );
+
+    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock_two.times_called(), 1);
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// send a request to a page that contains an absolute link to another domain, scanner should not
+/// follow
+fn extractor_finds_absolute_url_to_different_domain() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/LICENSE")
+        .return_status(200)
+        .return_body("\"http://localhost/homepage/assets/img/icons/handshake.svg\"")
+        .create_on(&srv);
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--extract-links")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .and(predicate::str::contains("200"))
+            .and(predicate::str::contains(
+                "/homepage/assets/img/icons/handshake.svg",
+            ))
+            .not(),
+    );
+
+    assert_eq!(mock.times_called(), 1);
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// send a request to a page that contains a relative link, should follow
+fn extractor_finds_relative_url() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/LICENSE")
+        .return_status(200)
+        .return_body("\"/homepage/assets/img/icons/handshake.svg\"")
+        .create_on(&srv);
+
+    let mock_two = Mock::new()
+        .expect_method(GET)
+        .expect_path("/homepage/assets/img/icons/handshake.svg")
+        .return_status(200)
+        .create_on(&srv);
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--extract-links")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .and(predicate::str::contains("200"))
+            .and(predicate::str::contains(
+                "/homepage/assets/img/icons/handshake.svg",
+            )),
+    );
+
+    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock_two.times_called(), 1);
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// send a request to a page that contains an relative link, follow it, and find the same link again
+/// should follow then filter
+fn extractor_finds_same_relative_url_twice() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) =
+        setup_tmp_directory(&["LICENSE".to_string(), "README".to_string()], "wordlist")?;
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/LICENSE")
+        .return_status(200)
+        .return_body(&srv.url("\"/homepage/assets/img/icons/handshake.svg\""))
+        .create_on(&srv);
+
+    let mock_two = Mock::new()
+        .expect_method(GET)
+        .expect_path("/README")
+        .return_body(&srv.url("\"/homepage/assets/img/icons/handshake.svg\""))
+        .return_status(200)
+        .create_on(&srv);
+
+    let mock_three = Mock::new()
+        .expect_method(GET)
+        .expect_path("/homepage/assets/img/icons/handshake.svg")
+        .return_status(200)
+        .create_on(&srv);
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--extract-links")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .and(predicate::str::contains("200"))
+            .and(predicate::str::contains(
+                "/homepage/assets/img/icons/handshake.svg",
+            )),
+    );
+
+    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock_two.times_called(), 1);
+    assert_eq!(mock_three.times_called(), 1);
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// send a request to a page that contains an absolute link that leads to a page with a sizefilter
+/// that should filter it out, expect not to see the second response reported
+fn extractor_finds_filtered_content() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) =
+        setup_tmp_directory(&["LICENSE".to_string(), "README".to_string()], "wordlist")?;
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/LICENSE")
+        .return_status(200)
+        .return_body(&srv.url("\"/homepage/assets/img/icons/handshake.svg\""))
+        .create_on(&srv);
+
+    let mock_two = Mock::new()
+        .expect_method(GET)
+        .expect_path("/homepage/assets/img/icons/handshake.svg")
+        .return_body("im a little teapot")
+        .return_status(200)
+        .create_on(&srv);
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--extract-links")
+        .arg("--sizefilter")
+        .arg("18")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .and(predicate::str::contains("200"))
+            .and(predicate::str::contains(
+                "/homepage/assets/img/icons/handshake.svg",
+            ))
+            .not(),
+    );
+
+    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock_two.times_called(), 1);
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}

tests/test_heuristics.rs

@@ -10,7 +10,7 @@ use utils::{setup_tmp_directory, teardown_tmp_directory};
 /// test passes one bad target via -u to the scanner, expected result is that the
 /// scanner dies
 fn test_single_target_cannot_connect() -> Result<(), Box<dyn std::error::Error>> {
-    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()])?;
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
     Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -37,7 +37,7 @@ fn test_two_targets_cannot_connect() -> Result<(), Box<dyn std::error::Error>> {
     let not_real =
         String::from("http://fjdksafjkdsajfkdsajkfdsajkfsdjkdsfdsafdsafdsajkr3l2ajfdskafdsjk");
     let urls = vec![not_real.clone(), not_real];
-    let (tmp_dir, file) = setup_tmp_directory(&urls)?;
+    let (tmp_dir, file) = setup_tmp_directory(&urls, "wordlist")?;
 
     Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -67,7 +67,7 @@ fn test_one_good_and_one_bad_target_scan_succeeds() -> Result<(), Box<dyn std::e
     let not_real =
         String::from("http://fjdksafjkdsajfkdsajkfdsajkfsdjkdsfdsafdsafdsajkr3l2ajfdskafdsjk");
     let urls = vec![not_real, srv.url("/"), String::from("LICENSE")];
-    let (tmp_dir, file) = setup_tmp_directory(&urls)?;
+    let (tmp_dir, file) = setup_tmp_directory(&urls, "wordlist")?;
 
     let mock = Mock::new()
         .expect_method(GET)
@@ -100,7 +100,7 @@ fn test_one_good_and_one_bad_target_scan_succeeds() -> Result<(), Box<dyn std::e
 /// test finds a static wildcard and reports as much to stdout
 fn test_static_wildcard_request_found() -> Result<(), Box<dyn std::error::Error>> {
     let srv = MockServer::start();
-    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()])?;
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
     let mock = Mock::new()
         .expect_method(GET)
@@ -132,10 +132,11 @@ fn test_static_wildcard_request_found() -> Result<(), Box<dyn std::error::Error>
 }
 
 #[test]
-/// test finds a dynamic wildcard and reports as much to stdout
+/// test finds a dynamic wildcard and reports as much to stdout and a file
 fn test_dynamic_wildcard_request_found() -> Result<(), Box<dyn std::error::Error>> {
     let srv = MockServer::start();
-    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()])?;
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+    let outfile = tmp_dir.path().join("outfile");
 
     let mock = Mock::new()
         .expect_method(GET)
@@ -158,10 +159,26 @@ fn test_dynamic_wildcard_request_found() -> Result<(), Box<dyn std::error::Error
         .arg("--wordlist")
         .arg(file.as_os_str())
         .arg("--addslash")
+        .arg("--output")
+        .arg(outfile.as_os_str())
         .unwrap();
 
+    let contents = std::fs::read_to_string(outfile).unwrap();
+
     teardown_tmp_directory(tmp_dir);
 
+    assert_eq!(contents.contains("WLD"), true);
+    assert_eq!(contents.contains("Got"), true);
+    assert_eq!(contents.contains("200"), true);
+    assert_eq!(contents.contains("auto-filtering"), true);
+    assert_eq!(contents.contains("(url length: 32)"), true);
+    assert_eq!(contents.contains("(url length: 96)"), true);
+    assert_eq!(contents.contains("Wildcard response is dynamic"), true);
+    assert_eq!(
+        contents.contains("(14 + url length) responses; toggle this behavior by using"),
+        true
+    );
+
     cmd.assert().success().stdout(
         predicate::str::contains("WLD")
             .and(predicate::str::contains("Got"))
@@ -179,3 +196,242 @@ fn test_dynamic_wildcard_request_found() -> Result<(), Box<dyn std::error::Error
     assert_eq!(mock2.times_called(), 1);
     Ok(())
 }
+
+#[test]
+/// uses dontfilter, so the normal wildcard test should never happen
+fn heuristics_static_wildcard_request_with_dontfilter() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap())
+        .return_status(200)
+        .return_body("this is a test")
+        .create_on(&srv);
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--dontfilter")
+        .unwrap();
+
+    teardown_tmp_directory(tmp_dir);
+
+    assert_eq!(mock.times_called(), 0);
+    Ok(())
+}
+
+#[test]
+/// test finds a static wildcard and reports as much to stdout
+fn heuristics_wildcard_test_with_two_static_wildcards() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap())
+        .return_status(200)
+        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
+        .create_on(&srv);
+
+    let mock2 = Mock::new()
+        .expect_method(GET)
+        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap())
+        .return_status(200)
+        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
+        .create_on(&srv);
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--addslash")
+        .unwrap();
+
+    teardown_tmp_directory(tmp_dir);
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("WLD")
+            .and(predicate::str::contains("Got"))
+            .and(predicate::str::contains("200"))
+            .and(predicate::str::contains("(url length: 32)"))
+            .and(predicate::str::contains("(url length: 96)"))
+            .and(predicate::str::contains(
+                "Wildcard response is static; auto-filtering 46",
+            )),
+    );
+
+    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock2.times_called(), 1);
+    Ok(())
+}
+
+#[test]
+/// test finds a static wildcard and reports nothing to stdout
+fn heuristics_wildcard_test_with_two_static_wildcards_with_quiet_enabled(
+) -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap())
+        .return_status(200)
+        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
+        .create_on(&srv);
+
+    let mock2 = Mock::new()
+        .expect_method(GET)
+        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap())
+        .return_status(200)
+        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
+        .create_on(&srv);
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--addslash")
+        .arg("-q")
+        .unwrap();
+
+    teardown_tmp_directory(tmp_dir);
+
+    cmd.assert().success().stdout(predicate::str::is_empty());
+
+    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock2.times_called(), 1);
+    Ok(())
+}
+
+#[test]
+/// test finds a static wildcard and reports as much to stdout and a file
+fn heuristics_wildcard_test_with_two_static_wildcards_and_output_to_file(
+) -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+    let outfile = tmp_dir.path().join("outfile");
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap())
+        .return_status(200)
+        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
+        .create_on(&srv);
+
+    let mock2 = Mock::new()
+        .expect_method(GET)
+        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap())
+        .return_status(200)
+        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
+        .create_on(&srv);
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--addslash")
+        .arg("--output")
+        .arg(outfile.as_os_str())
+        .unwrap();
+
+    let contents = std::fs::read_to_string(outfile).unwrap();
+
+    teardown_tmp_directory(tmp_dir);
+
+    assert_eq!(contents.contains("WLD"), true);
+    assert_eq!(contents.contains("Got"), true);
+    assert_eq!(contents.contains("200"), true);
+    assert_eq!(contents.contains("(url length: 32)"), true);
+    assert_eq!(contents.contains("(url length: 96)"), true);
+    assert_eq!(
+        contents.contains("Wildcard response is static; auto-filtering 46"),
+        true
+    );
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("WLD")
+            .and(predicate::str::contains("Got"))
+            .and(predicate::str::contains("200"))
+            .and(predicate::str::contains("(url length: 32)"))
+            .and(predicate::str::contains("(url length: 96)"))
+            .and(predicate::str::contains(
+                "Wildcard response is static; auto-filtering 46",
+            )),
+    );
+
+    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock2.times_called(), 1);
+
+    Ok(())
+}
+
+#[test]
+/// test finds a static wildcard that returns 3xx, expect redirects to => in response as well as
+/// in the output file
+fn heuristics_wildcard_test_with_redirect_as_response_code(
+) -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+    let outfile = tmp_dir.path().join("outfile");
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{32}/").unwrap())
+        .return_status(301)
+        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
+        .create_on(&srv);
+
+    let mock2 = Mock::new()
+        .expect_method(GET)
+        .expect_path_matches(Regex::new("/[a-zA-Z0-9]{96}/").unwrap())
+        .return_status(301)
+        .return_header("Location", &srv.url("/some-redirect"))
+        .return_body("this is a testAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
+        .create_on(&srv);
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--addslash")
+        .arg("--output")
+        .arg(outfile.as_os_str())
+        .unwrap();
+
+    let contents = std::fs::read_to_string(outfile).unwrap();
+
+    teardown_tmp_directory(tmp_dir);
+
+    assert_eq!(contents.contains("WLD"), true);
+    assert_eq!(contents.contains("301"), true);
+    assert_eq!(contents.contains("/some-redirect"), true);
+    assert_eq!(contents.contains("redirects to => "), true);
+    assert_eq!(contents.contains(&srv.url("/")), true);
+    assert_eq!(contents.contains("(url length: 32)"), true);
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("redirects to => ")
+            .and(predicate::str::contains("/some-redirect"))
+            .and(predicate::str::contains("301"))
+            .and(predicate::str::contains(srv.url("/")))
+            .and(predicate::str::contains("(url length: 32)"))
+            .and(predicate::str::contains("WLD")),
+    );
+
+    assert_eq!(mock.times_called(), 1);
+    assert_eq!(mock2.times_called(), 1);
+    Ok(())
+}

tests/test_main.rs

@@ -39,7 +39,7 @@ fn main_use_root_owned_file_as_wordlist() -> Result<(), Box<dyn std::error::Erro
 /// send the function an empty file
 fn main_use_empty_wordlist() -> Result<(), Box<dyn std::error::Error>> {
     let srv = MockServer::start();
-    let (tmp_dir, file) = setup_tmp_directory(&[])?;
+    let (tmp_dir, file) = setup_tmp_directory(&[], "wordlist")?;
 
     let mock = Mock::new()
         .expect_method(GET)
@@ -70,7 +70,7 @@ fn main_use_empty_wordlist() -> Result<(), Box<dyn std::error::Error>> {
 #[test]
 /// send nothing over stdin, expect heuristics to be upset during connectivity test
 fn main_use_empty_stdin_targets() -> Result<(), Box<dyn std::error::Error>> {
-    let (tmp_dir, file) = setup_tmp_directory(&[])?;
+    let (tmp_dir, file) = setup_tmp_directory(&[], "wordlist")?;
 
     // get_targets is called before scan, so the empty wordlist shouldn't trigger
     // the 'Did not find any words' error

tests/test_scanner.rs

@@ -8,9 +8,9 @@ use utils::{setup_tmp_directory, teardown_tmp_directory};
 
 #[test]
 /// send a single valid request, expect a 200 response
-fn test_single_request_scan() -> Result<(), Box<dyn std::error::Error>> {
+fn scanner_single_request_scan() -> Result<(), Box<dyn std::error::Error>> {
     let srv = MockServer::start();
-    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()])?;
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
 
     let mock = Mock::new()
         .expect_method(GET)
@@ -49,7 +49,7 @@ fn scanner_recursive_request_scan() -> Result<(), Box<dyn std::error::Error>> {
         "dev".to_string(),
         "file.js".to_string(),
     ];
-    let (tmp_dir, file) = setup_tmp_directory(&urls)?;
+    let (tmp_dir, file) = setup_tmp_directory(&urls, "wordlist")?;
 
     let js_mock = Mock::new()
         .expect_method(GET)
@@ -107,3 +107,306 @@ fn scanner_recursive_request_scan() -> Result<(), Box<dyn std::error::Error>> {
 
     Ok(())
 }
+
+#[test]
+/// send a valid request, follow 200s into new directories, expect 200 responses
+fn scanner_recursive_request_scan_using_only_success_responses(
+) -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let urls = [
+        "js/".to_string(),
+        "prod/".to_string(),
+        "dev/".to_string(),
+        "file.js".to_string(),
+    ];
+    let (tmp_dir, file) = setup_tmp_directory(&urls, "wordlist")?;
+
+    let js_mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/js/")
+        .return_status(200)
+        .return_header("Location", &srv.url("/js/"))
+        .create_on(&srv);
+
+    let js_prod_mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/js/prod/")
+        .return_status(200)
+        .return_header("Location", &srv.url("/js/prod/"))
+        .create_on(&srv);
+
+    let js_dev_mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/js/dev/")
+        .return_status(200)
+        .return_header("Location", &srv.url("/js/dev/"))
+        .create_on(&srv);
+
+    let js_dev_file_mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/js/dev/file.js")
+        .return_status(200)
+        .return_body("this is a test and is more bytes than other ones")
+        .create_on(&srv);
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-vvvv")
+        .arg("-t")
+        .arg("1")
+        .arg("--redirects")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::is_match("200.*js")
+            .unwrap()
+            .and(predicate::str::is_match("200.*js/prod").unwrap())
+            .and(predicate::str::is_match("200.*js/dev").unwrap())
+            .and(predicate::str::is_match("200.*js/dev/file.js").unwrap()),
+    );
+
+    assert_eq!(js_mock.times_called(), 1);
+    assert_eq!(js_prod_mock.times_called(), 1);
+    assert_eq!(js_dev_mock.times_called(), 1);
+    assert_eq!(js_dev_file_mock.times_called(), 1);
+
+    teardown_tmp_directory(tmp_dir);
+
+    Ok(())
+}
+
+#[test]
+/// send a single valid request, get a response, and write it to disk
+fn scanner_single_request_scan_with_file_output() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/LICENSE")
+        .return_status(200)
+        .return_body("this is a test")
+        .create_on(&srv);
+
+    let outfile = tmp_dir.path().join("output");
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-vvvv")
+        .arg("-o")
+        .arg(outfile.as_os_str())
+        .unwrap();
+
+    let contents = std::fs::read_to_string(outfile)?;
+
+    assert!(contents.contains("/LICENSE"));
+    assert!(contents.contains("200"));
+    assert!(contents.contains("14"));
+
+    assert_eq!(mock.times_called(), 1);
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// send a single valid request with -q, get a response, and write only the url to disk
+fn scanner_single_request_scan_with_file_output_and_tack_q(
+) -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/LICENSE")
+        .return_status(200)
+        .return_body("this is a test")
+        .create_on(&srv);
+
+    let outfile = tmp_dir.path().join("output");
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-vvvv")
+        .arg("-q")
+        .arg("-o")
+        .arg(outfile.as_os_str())
+        .unwrap();
+
+    let contents = std::fs::read_to_string(outfile)?;
+
+    let url = srv.url("/LICENSE");
+    assert!(contents.contains(&url));
+
+    assert_eq!(mock.times_called(), 1);
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// send an invalid output file, expect nothing to be written to disk
+fn scanner_single_request_scan_with_invalid_file_output() -> Result<(), Box<dyn std::error::Error>>
+{
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/LICENSE")
+        .return_status(200)
+        .return_body("this is a test")
+        .create_on(&srv);
+
+    let outfile = tmp_dir.path(); // outfile is a directory
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-vvvv")
+        .arg("-q")
+        .arg("-o")
+        .arg(outfile.as_os_str())
+        .unwrap();
+
+    let contents = std::fs::read_to_string(outfile);
+    assert!(contents.is_err());
+
+    assert_eq!(mock.times_called(), 1);
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// send a single valid request using -q, expect only the url on stdout
+fn scanner_single_request_quiet_scan() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/LICENSE")
+        .return_status(200)
+        .return_body("this is a test")
+        .create_on(&srv);
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-x")
+        .arg("js,html")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains(srv.url("/LICENSE"))
+            .and(predicate::str::contains("200"))
+            .not()
+            .and(predicate::str::contains("14"))
+            .not(),
+    );
+
+    assert_eq!(mock.times_called(), 1);
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// send single valid request, get back a 301 without a Location header, expect false
+fn scanner_single_request_returns_301_without_location_header(
+) -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()], "wordlist")?;
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/LICENSE")
+        .return_status(301)
+        .create_on(&srv);
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-T")
+        .arg("5")
+        .arg("-a")
+        .arg("some-user-agent-string")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains(srv.url("/LICENSE"))
+            .and(predicate::str::contains("301"))
+            .and(predicate::str::contains("14"))
+            .not(),
+    );
+
+    assert_eq!(mock.times_called(), 1);
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// send a single valid request, filter the size of the response, expect one out of 2 urls
+fn scanner_single_request_scan_with_filtered_result() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) =
+        setup_tmp_directory(&["LICENSE".to_string(), "ignored".to_string()], "wordlist")?;
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/LICENSE")
+        .return_status(200)
+        .return_body("this is a not a test")
+        .create_on(&srv);
+
+    let filtered_mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/ignored")
+        .return_status(200)
+        .return_body("this is a test")
+        .create_on(&srv);
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-n")
+        .arg("-S")
+        .arg("14")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::contains("/LICENSE")
+            .and(predicate::str::contains("200"))
+            .and(predicate::str::contains("20"))
+            .and(predicate::str::contains("ignored"))
+            .not()
+            .and(predicate::str::contains("14"))
+            .not(),
+    );
+
+    assert_eq!(mock.times_called(), 1);
+    assert_eq!(filtered_mock.times_called(), 1);
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}

tests/utils/mod.rs

@@ -3,12 +3,13 @@ use std::path::PathBuf;
 use tempfile::TempDir;
 
 /// integration test helper: creates a temp directory, and writes `words` to
-/// a file named `wordlist` in the temp directory
+/// a file named `filename` in the temp directory
 pub fn setup_tmp_directory(
     words: &[String],
+    filename: &str,
 ) -> Result<(TempDir, PathBuf), Box<dyn std::error::Error>> {
     let tmp_dir = TempDir::new()?;
-    let file = tmp_dir.path().join("wordlist");
+    let file = tmp_dir.path().join(&filename);
     write(&file, words.join("\n"))?;
     Ok((tmp_dir, file))
 }