fixed possible thread panic due to multiple calls to join

[Documentation] README.md matrix fixes for ffuf

.github/actions-rs/grcov.yml

@@ -0,0 +1,7 @@
+branch: true
+ignore-not-existing: true
+llvm: true
+output-type: lcov
+output-path: ./lcov.info
+ignore:
+  - "../*"

.github/workflows/build.yml

@@ -1,69 +1,8 @@
-name: CI Pipeline
+name: CD Pipeline
 
 on: [push]
 
 jobs:
-  check:
-    name: Check
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-      - uses: actions-rs/cargo@v1
-        with:
-          command: check
-
-  test:
-    name: Test Suite
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-      - uses: actions-rs/cargo@v1
-        with:
-          command: test
-
-  fmt:
-    name: Rust fmt
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-      - run: rustup component add rustfmt
-      - uses: actions-rs/cargo@v1
-        with:
-          command: fmt
-          args: --all -- --check
-
-  clippy:
-    name: Clippy
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-      - run: rustup component add clippy
-      - uses: actions-rs/cargo@v1
-        with:
-          command: clippy
-          args: --all-targets --all-features -- -D warnings -A clippy::unnecessary_unwrap
-
-
   build-nix:
     runs-on: ${{ matrix.os }}
     if: github.ref == 'refs/heads/master'

.github/workflows/check.yml

@@ -0,0 +1,64 @@
+name: CI Pipeline
+
+on: [push]
+
+jobs:
+  check:
+    name: Check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+      - uses: actions-rs/cargo@v1
+        with:
+          command: check
+
+  test:
+    name: Test Suite
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+      - uses: actions-rs/cargo@v1
+        with:
+          command: test
+
+  fmt:
+    name: Rust fmt
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+      - run: rustup component add rustfmt
+      - uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --all -- --check
+
+  clippy:
+    name: Clippy
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+      - run: rustup component add clippy
+      - uses: actions-rs/cargo@v1
+        with:
+          command: clippy
+          args: --all-targets --all-features -- -D warnings -A clippy::unnecessary_unwrap

.github/workflows/coverage.yml

@@ -0,0 +1,36 @@
+on: [push]
+
+name: Code Coverage Pipeline
+
+jobs:
+  upload-coverage:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: nightly
+          override: true
+      - uses: actions-rs/cargo@v1
+        with:
+          command: clean
+      - uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --all-features --no-fail-fast
+        env:
+          CARGO_INCREMENTAL: '0'
+          RUSTFLAGS: '-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Cpanic=abort -Zpanic_abort_tests'
+          RUSTDOCFLAGS: '-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Cpanic=abort -Zpanic_abort_tests'
+      - uses: actions-rs/grcov@v0.1
+      - name: Convert lcov to xml
+        run: |
+          curl -O https://raw.githubusercontent.com/eriwen/lcov-to-cobertura-xml/master/lcov_cobertura/lcov_cobertura.py
+          chmod +x lcov_cobertura.py
+          ./lcov_cobertura.py ./lcov.info
+      - uses: codecov/codecov-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          file: ./coverage.xml
+          name: codecov-umbrella
+          fail_ci_if_error: true

.gitignore

@@ -19,5 +19,6 @@ ferox-config.toml
 # images for the README on github
 img/**
 
-# personal script to check code coverage using nightly compiler
+# scripts to check code coverage using nightly compiler
 check-coverage.sh
+lcov_cobertura.py

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "feroxbuster"
-version = "0.2.1"
+version = "1.0.1"
 authors = ["Ben 'epi' Risher <epibar052@gmail.com>"]
 license = "MIT"
 edition = "2018"
@@ -26,7 +26,6 @@ lazy_static = "1.4"
 toml = "0.5"
 serde = { version = "1.0", features = ["derive"] }
 uuid = { version = "0.8", features = ["v4"] }
-ansi_term = "0.12"
 indicatif = "0.15"
 console = "0.12"
 openssl = { version = "0.10", features = ["vendored"] }

README.md

@@ -7,27 +7,56 @@
 <h4 align="center">A simple, fast, recursive content discovery tool written in Rust</h4>
 
 <p align="center">
-  <img src="https://img.shields.io/github/workflow/status/epi052/feroxbuster/CI%20Pipeline/master?logo=github">
+  <a href="https://github.com/epi052/feroxbuster/actions?query=workflow%3A%22CI+Pipeline%22">
-  <img src="https://img.shields.io/github/downloads/epi052/feroxbuster/total?label=downloads&logo=github&color=inactive" alt="github downloads">
+    <img src="https://img.shields.io/github/workflow/status/epi052/feroxbuster/CI%20Pipeline/master?logo=github">
-  <img src="https://img.shields.io/github/issues-closed-raw/s0md3v/Photon.svg">
+  </a>
-  <img src="https://img.shields.io/github/last-commit/epi052/feroxbuster?logo=github">
+
-  <img src="https://img.shields.io/crates/v/feroxbuster?color=blue&label=version&logo=rust">
+  <a href="https://github.com/epi052/feroxbuster/releases">
-  <img src="https://img.shields.io/crates/d/feroxbuster?label=downloads&logo=rust&color=inactive">
+    <img src="https://img.shields.io/github/downloads/epi052/feroxbuster/total?label=downloads&logo=github&color=inactive" alt="github downloads">
+  </a>
+
+  <a href="https://github.com/epi052/feroxbuster/commits/master">
+    <img src="https://img.shields.io/github/last-commit/epi052/feroxbuster?logo=github">
+  </a>
+
+  <a href="https://crates.io/crates/feroxbuster">
+    <img src="https://img.shields.io/crates/v/feroxbuster?color=blue&label=version&logo=rust">
+  </a>
+ 
+  <a href="https://crates.io/crates/feroxbuster">
+    <img src="https://img.shields.io/crates/d/feroxbuster?label=downloads&logo=rust&color=inactive">
+  </a>
+
+  <a href="https://codecov.io/gh/epi052/feroxbuster">
+    <img src="https://codecov.io/gh/epi052/feroxbuster/branch/master/graph/badge.svg" />
+  </a>
 </p>
 
 ![demo](img/demo.gif)
 
 <p align="center">
-  <a href="https://github.com/epi052/feroxbuster/releases">Releases</a> •
+  🦀
-  <a href="#-example-usage">Example Usage</a> •
+  <a href="https://github.com/epi052/feroxbuster/releases">Releases</a> ✨
-  <a href="https://github.com/epi052/feroxbuster/blob/master/CONTRIBUTING.md">Contributing</a> •
+  <a href="#-example-usage">Example Usage</a> ✨
+  <a href="https://github.com/epi052/feroxbuster/blob/master/CONTRIBUTING.md">Contributing</a> ✨
   <a href="https://docs.rs/feroxbuster/latest/feroxbuster/">Documentation</a>
+  🦀
 </p>
 
 ## 😕 What the heck is a ferox anyway?
 
 Ferox is short for Ferric Oxide. Ferric Oxide, simply put, is rust.  The name rustbuster was taken, so I decided on a variation.  🤷	
 
+## 🤔 What's it do tho? 
+
+`feroxbuster` is a tool designed to perform [Forced Browsing](https://owasp.org/www-community/attacks/Forced_browsing).  
+
+Forced browsing is an attack where the aim is to enumerate and access resources that are not referenced by the web application, but are still accessible by an attacker.
+
+`feroxbuster` uses brute force combined with a wordlist to search for unlinked content in target directories. These resources may store sensitive information about web applications and operational systems, such as source code, credentials, internal network addressing, etc...
+
+This attack is also known as Predictable Resource Location, File Enumeration, Directory Enumeration, and Resource Enumeration.
+
 📖 Table of Contents
 -----------------
 - [Downloads](#-downloads)
@@ -97,10 +126,10 @@ After setting built-in default values, any values defined in a `ferox-config.tom
 built-in defaults.  
 
 `feroxbuster` searches for `ferox-config.toml` in the following locations (in the order shown):
-- `/etc/feroxbuster/`
+- `/etc/feroxbuster/` (global)
-- `CONFIG_DIR/ferxobuster/`
+- `CONFIG_DIR/ferxobuster/` (per-user)
-- The same directory as the `feroxbuster` executable
+- The same directory as the `feroxbuster` executable (per-user)
-- The user's current working directory
+- The user's current working directory (per-target)
 
 If more than one valid configuration file is found, each one overwrites the values found previously.  
 
@@ -276,10 +305,10 @@ a few of the use-cases in which feroxbuster may be a better fit:
 | allows recursion                                    | ✔ |   | ✔ |
 | can specify query parameters                        | ✔ |   | ✔ |
 | SOCKS proxy support                                 | ✔ |   |   |
-| multiple target scan (via stdin or multiple -u)     | ✔ |   |   |
+| multiple target scan (via stdin or multiple -u)     | ✔ |   | ✔ |
 | configuration file for default value override       | ✔ |   | ✔ |
-| can accept urls via STDIN as part of a pipeline     | ✔ |   |   |
+| can accept urls via STDIN as part of a pipeline     | ✔ |   | ✔ |
-| can accept wordlists via STDIN                      |   | ✔ |   |
+| can accept wordlists via STDIN                      |   | ✔ | ✔ |
 | filter by response size                             | ✔ |   | ✔ |
 | auto-filter wildcard responses                      | ✔ |   | ✔ |
 | performs other scans (vhost, dns, etc)              |   | ✔ | ✔ |

src/banner.rs

@@ -1,4 +1,4 @@
-use crate::{config::CONFIGURATION, utils::status_colorizer, VERSION};
+use crate::{config::Configuration, utils::status_colorizer, VERSION};
 
 /// macro helper to abstract away repetitive string formatting
 macro_rules! format_banner_entry_helper {
@@ -43,7 +43,7 @@ macro_rules! format_banner_entry {
 /// Prints the banner to stdout.
 ///
 /// Only prints those settings which are either always present, or passed in by the user.
-pub fn initialize(targets: &[String]) {
+pub fn initialize(targets: &[String], config: &Configuration) {
     let artwork = format!(
         r#"
  ___  ___  __   __     __      __         __   ___
@@ -69,17 +69,17 @@ by Ben "epi" Risher {}                  ver: {}"#,
 
     let mut codes = vec![];
 
-    for code in &CONFIGURATION.statuscodes {
+    for code in &config.statuscodes {
         codes.push(status_colorizer(&code.to_string()))
     }
 
     eprintln!(
         "{}",
-        format_banner_entry!("\u{1F680}", "Threads", CONFIGURATION.threads)
+        format_banner_entry!("\u{1F680}", "Threads", config.threads)
     ); // 🚀
     eprintln!(
         "{}",
-        format_banner_entry!("\u{1f4d6}", "Wordlist", CONFIGURATION.wordlist)
+        format_banner_entry!("\u{1f4d6}", "Wordlist", config.wordlist)
     ); // 📖
     eprintln!(
         "{}",
@@ -91,30 +91,30 @@ by Ben "epi" Risher {}                  ver: {}"#,
     ); // 🆗
     eprintln!(
         "{}",
-        format_banner_entry!("\u{1f4a5}", "Timeout (secs)", CONFIGURATION.timeout)
+        format_banner_entry!("\u{1f4a5}", "Timeout (secs)", config.timeout)
     ); // 💥
     eprintln!(
         "{}",
-        format_banner_entry!("\u{1F9a1}", "User-Agent", CONFIGURATION.useragent)
+        format_banner_entry!("\u{1F9a1}", "User-Agent", config.useragent)
     ); // 🦡
 
     // followed by the maybe printed or variably displayed values
-    if !CONFIGURATION.config.is_empty() {
+    if !config.config.is_empty() {
         eprintln!(
             "{}",
-            format_banner_entry!("\u{1f489}", "Config File", CONFIGURATION.config)
+            format_banner_entry!("\u{1f489}", "Config File", config.config)
         ); // 💉
     }
 
-    if !CONFIGURATION.proxy.is_empty() {
+    if !config.proxy.is_empty() {
         eprintln!(
             "{}",
-            format_banner_entry!("\u{1f48e}", "Proxy", CONFIGURATION.proxy)
+            format_banner_entry!("\u{1f48e}", "Proxy", config.proxy)
         ); // 💎
     }
 
-    if !CONFIGURATION.headers.is_empty() {
+    if !config.headers.is_empty() {
-        for (name, value) in &CONFIGURATION.headers {
+        for (name, value) in &config.headers {
             eprintln!(
                 "{}",
                 format_banner_entry!("\u{1f92f}", "Header", name, value)
@@ -122,8 +122,8 @@ by Ben "epi" Risher {}                  ver: {}"#,
         }
     }
 
-    if !CONFIGURATION.sizefilters.is_empty() {
+    if !config.sizefilters.is_empty() {
-        for filter in &CONFIGURATION.sizefilters {
+        for filter in &config.sizefilters {
             eprintln!(
                 "{}",
                 format_banner_entry!("\u{1f4a2}", "Size Filter", filter)
@@ -131,8 +131,8 @@ by Ben "epi" Risher {}                  ver: {}"#,
         }
     }
 
-    if !CONFIGURATION.queries.is_empty() {
+    if !config.queries.is_empty() {
-        for query in &CONFIGURATION.queries {
+        for query in &config.queries {
             eprintln!(
                 "{}",
                 format_banner_entry!(
@@ -144,83 +144,83 @@ by Ben "epi" Risher {}                  ver: {}"#,
         }
     }
 
-    if !CONFIGURATION.output.is_empty() {
+    if !config.output.is_empty() {
         eprintln!(
             "{}",
-            format_banner_entry!("\u{1f4be}", "Output File", CONFIGURATION.output)
+            format_banner_entry!("\u{1f4be}", "Output File", config.output)
         ); // 💾
     }
 
-    if !CONFIGURATION.extensions.is_empty() {
+    if !config.extensions.is_empty() {
         eprintln!(
             "{}",
             format_banner_entry!(
                 "\u{1f4b2}",
                 "Extensions",
-                format!("[{}]", CONFIGURATION.extensions.join(", "))
+                format!("[{}]", config.extensions.join(", "))
             )
         ); // 💲
     }
 
-    if CONFIGURATION.insecure {
+    if config.insecure {
         eprintln!(
             "{}",
-            format_banner_entry!("\u{1f513}", "Insecure", CONFIGURATION.insecure)
+            format_banner_entry!("\u{1f513}", "Insecure", config.insecure)
         ); // 🔓
     }
 
-    if CONFIGURATION.redirects {
+    if config.redirects {
         eprintln!(
             "{}",
-            format_banner_entry!("\u{1f4cd}", "Follow Redirects", CONFIGURATION.redirects)
+            format_banner_entry!("\u{1f4cd}", "Follow Redirects", config.redirects)
         ); // 📍
     }
 
-    if CONFIGURATION.dontfilter {
+    if config.dontfilter {
         eprintln!(
             "{}",
-            format_banner_entry!("\u{1f92a}", "Filter Wildcards", !CONFIGURATION.dontfilter)
+            format_banner_entry!("\u{1f92a}", "Filter Wildcards", !config.dontfilter)
         ); // 🤪
     }
 
-    match CONFIGURATION.verbosity {
+    match config.verbosity {
         //speaker medium volume (increasing with verbosity to loudspeaker)
         1 => {
             eprintln!(
                 "{}",
-                format_banner_entry!("\u{1f508}", "Verbosity", CONFIGURATION.verbosity)
+                format_banner_entry!("\u{1f508}", "Verbosity", config.verbosity)
             ); // 🔈
         }
         2 => {
             eprintln!(
                 "{}",
-                format_banner_entry!("\u{1f509}", "Verbosity", CONFIGURATION.verbosity)
+                format_banner_entry!("\u{1f509}", "Verbosity", config.verbosity)
             ); // 🔉
         }
         3 => {
             eprintln!(
                 "{}",
-                format_banner_entry!("\u{1f50a}", "Verbosity", CONFIGURATION.verbosity)
+                format_banner_entry!("\u{1f50a}", "Verbosity", config.verbosity)
             ); // 🔊
         }
         4 => {
             eprintln!(
                 "{}",
-                format_banner_entry!("\u{1f4e2}", "Verbosity", CONFIGURATION.verbosity)
+                format_banner_entry!("\u{1f4e2}", "Verbosity", config.verbosity)
             ); // 📢
         }
         _ => {}
     }
 
-    if CONFIGURATION.addslash {
+    if config.addslash {
         eprintln!(
             "{}",
-            format_banner_entry!("\u{1fa93}", "Add Slash", CONFIGURATION.addslash)
+            format_banner_entry!("\u{1fa93}", "Add Slash", config.addslash)
         ); // 🪓
     }
 
-    if !CONFIGURATION.norecursion {
+    if !config.norecursion {
-        if CONFIGURATION.depth == 0 {
+        if config.depth == 0 {
             eprintln!(
                 "{}",
                 format_banner_entry!("\u{1f503}", "Recursion Depth", "INFINITE")
@@ -228,15 +228,51 @@ by Ben "epi" Risher {}                  ver: {}"#,
         } else {
             eprintln!(
                 "{}",
-                format_banner_entry!("\u{1f503}", "Recursion Depth", CONFIGURATION.depth)
+                format_banner_entry!("\u{1f503}", "Recursion Depth", config.depth)
             ); // 🔃
         }
     } else {
         eprintln!(
             "{}",
-            format_banner_entry!("\u{1f6ab}", "Do Not Recurse", CONFIGURATION.norecursion)
+            format_banner_entry!("\u{1f6ab}", "Do Not Recurse", config.norecursion)
         ); // 🚫
     }
 
     eprintln!("{}", bottom);
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    /// test to hit no execution of targets for loop in banner
+    fn banner_without_targets() {
+        let config = Configuration::default();
+        initialize(&[], &config);
+    }
+
+    #[test]
+    /// test to hit no execution of statuscode for loop in banner
+    fn banner_without_status_codes() {
+        let mut config = Configuration::default();
+        config.statuscodes = vec![];
+        initialize(&[String::from("http://localhost")], &config);
+    }
+
+    #[test]
+    /// test to hit an empty config file
+    fn banner_without_config_file() {
+        let mut config = Configuration::default();
+        config.config = String::new();
+        initialize(&[String::from("http://localhost")], &config);
+    }
+
+    #[test]
+    /// test to hit an empty config file
+    fn banner_without_queries() {
+        let mut config = Configuration::default();
+        config.queries = vec![(String::new(), String::new())];
+        initialize(&[String::from("http://localhost")], &config);
+    }
+}

src/client.rs

@@ -1,5 +1,5 @@
-use crate::utils::status_colorizer;
+use crate::utils::{module_colorizer, status_colorizer};
-use ansi_term::Color::Cyan;
+use console::style;
 use reqwest::header::HeaderMap;
 use reqwest::{redirect::Policy, Client, Proxy};
 use std::collections::HashMap;
@@ -28,7 +28,7 @@ pub fn initialize(
             eprintln!(
                 "{} {} {}",
                 status_colorizer("ERROR"),
-                Cyan.paint("Client::initialize"),
+                module_colorizer("Client::initialize"),
                 e
             );
             exit(1);
@@ -49,13 +49,13 @@ pub fn initialize(
                 eprintln!(
                     "{} {} Could not add proxy ({:?}) to Client configuration",
                     status_colorizer("ERROR"),
-                    Cyan.paint("Client::initialize"),
+                    module_colorizer("Client::initialize"),
                     proxy
                 );
                 eprintln!(
                     "{} {} {}",
                     status_colorizer("ERROR"),
-                    Cyan.paint("Client::initialize"),
+                    style("Client::initialize").cyan(),
                     e
                 );
                 exit(1);
@@ -71,12 +71,12 @@ pub fn initialize(
             eprintln!(
                 "{} {} Could not create a Client with the given configuration, exiting.",
                 status_colorizer("ERROR"),
-                Cyan.paint("Client::build")
+                module_colorizer("Client::build")
             );
             eprintln!(
                 "{} {} {}",
                 status_colorizer("ERROR"),
-                Cyan.paint("Client::build"),
+                module_colorizer("Client::build"),
                 e
             );
             exit(1);

src/config.rs

@@ -1,7 +1,6 @@
-use crate::utils::status_colorizer;
+use crate::utils::{module_colorizer, status_colorizer};
 use crate::{client, parser, progress};
 use crate::{DEFAULT_CONFIG_NAME, DEFAULT_STATUS_CODES, DEFAULT_WORDLIST, VERSION};
-use ansi_term::Color::Cyan;
 use clap::value_t;
 use indicatif::{MultiProgress, ProgressBar, ProgressDrawTarget};
 use lazy_static::lazy_static;
@@ -246,6 +245,11 @@ impl Configuration {
     /// The resulting [Configuration](struct.Configuration.html) is a singleton with a `static`
     /// lifetime.
     pub fn new() -> Self {
+        // when compiling for test, we want to eliminate the runtime dependency of the parser
+        if cfg!(test) {
+            return Configuration::default();
+        }
+
         // Get the default configuration, this is what will apply if nothing
         // else is specified.
         let mut config = Configuration::default();
@@ -524,7 +528,7 @@ impl Configuration {
                     println!(
                         "{} {} {}",
                         status_colorizer("ERROR"),
-                        Cyan.paint("config::parse_config"),
+                        module_colorizer("config::parse_config"),
                         e
                     );
                 }

src/heuristics.rs

@@ -1,6 +1,8 @@
 use crate::config::{CONFIGURATION, PROGRESS_PRINTER};
-use crate::utils::{ferox_print, format_url, get_url_path_length, make_request, status_colorizer};
+use crate::utils::{
-use ansi_term::Color::{Cyan, Yellow};
+    ferox_print, format_url, get_url_path_length, make_request, module_colorizer, status_colorizer,
+};
+use console::style;
 use indicatif::ProgressBar;
 use reqwest::Response;
 use std::process;
@@ -86,9 +88,9 @@ pub async fn wildcard_test(target_url: &str, bar: ProgressBar) -> Option<Wildcar
                             "{} {:>10} Wildcard response is dynamic; {} ({} + url length) responses; toggle this behavior by using {}",
                             status_colorizer("WLD"),
                             wc_length - url_len,
-                            Yellow.paint("auto-filtering"),
+                            style("auto-filtering").yellow(),
-                            Cyan.paint(format!("{}", wc_length - url_len)),
+                            style(wc_length - url_len).cyan(),
-                            Yellow.paint("--dontfilter")
+                            style("--dontfilter").yellow()
                         ), &PROGRESS_PRINTER
                     );
                 }
@@ -100,9 +102,9 @@ pub async fn wildcard_test(target_url: &str, bar: ProgressBar) -> Option<Wildcar
                         "{} {:>10} Wildcard response is static; {} {} responses; toggle this behavior by using {}",
                         status_colorizer("WLD"),
                         wc_length,
-                        Yellow.paint("auto-filtering"),
+                        style("auto-filtering").yellow(),
-                        Cyan.paint(format!("{}", wc_length)),
+                        style(wc_length).cyan(),
-                        Yellow.paint("--dontfilter")
+                        style("--dontfilter").yellow()
                     ), &PROGRESS_PRINTER);
                 }
                 wildcard.size = wc_length;
@@ -261,7 +263,7 @@ pub async fn connectivity_test(target_urls: &[String]) -> Vec<String> {
         eprintln!(
             "{} {} Could not connect to any target provided",
             status_colorizer("ERROR"),
-            Cyan.paint("heuristics::connectivity_test"),
+            module_colorizer("heuristics::connectivity_test"),
         );
 
         process::exit(1);
@@ -277,6 +279,7 @@ mod tests {
     use super::*;
 
     #[test]
+    /// request a unique string of 32bytes * a value returns correct result
     fn unique_string_returns_correct_length() {
         for i in 0..10 {
             assert_eq!(unique_string(i).len(), i * 32);

src/lib.rs

@@ -52,3 +52,29 @@ pub const DEFAULT_STATUS_CODES: [StatusCode; 9] = [
 ///
 /// Expected location is in the same directory as the feroxbuster binary.
 pub const DEFAULT_CONFIG_NAME: &str = "ferox-config.toml";
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    /// asserts default config name is correct
+    fn default_config_name() {
+        assert_eq!(DEFAULT_CONFIG_NAME, "ferox-config.toml");
+    }
+
+    #[test]
+    /// asserts default wordlist is correct
+    fn default_wordlist() {
+        assert_eq!(
+            DEFAULT_WORDLIST,
+            "/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt"
+        );
+    }
+
+    #[test]
+    /// asserts default version is correct
+    fn default_version() {
+        assert_eq!(VERSION, env!("CARGO_PKG_VERSION"));
+    }
+}

src/main.rs

@@ -1,7 +1,6 @@
-use ansi_term::Color::Cyan;
 use feroxbuster::config::{CONFIGURATION, PROGRESS_PRINTER};
 use feroxbuster::scanner::scan_url;
-use feroxbuster::utils::{get_current_depth, status_colorizer};
+use feroxbuster::utils::{ferox_print, get_current_depth, module_colorizer, status_colorizer};
 use feroxbuster::{banner, heuristics, logger, FeroxResult};
 use futures::StreamExt;
 use std::collections::HashSet;
@@ -22,7 +21,7 @@ fn get_unique_words_from_wordlist(path: &str) -> FeroxResult<Arc<HashSet<String>
             eprintln!(
                 "{} {} {}",
                 status_colorizer("ERROR"),
-                Cyan.paint("main::get_unique_words_from_wordlist"),
+                module_colorizer("main::get_unique_words_from_wordlist"),
                 e
             );
             log::error!("Could not open wordlist: {}", e);
@@ -37,20 +36,14 @@ fn get_unique_words_from_wordlist(path: &str) -> FeroxResult<Arc<HashSet<String>
     let mut words = HashSet::new();
 
     for line in reader.lines() {
-        match line {
+        words.insert(line?);
-            Ok(word) => {
-                words.insert(word);
-            }
-            Err(e) => {
-                log::warn!("Could not parse current line from wordlist : {}", e);
-            }
-        }
     }
 
     log::trace!(
         "exit: get_unique_words_from_wordlist -> Arc<wordlist[{} words...]>",
         words.len()
     );
+
     Ok(Arc::new(words))
 }
 
@@ -68,7 +61,7 @@ async fn scan(targets: Vec<String>) -> FeroxResult<()> {
         eprintln!(
             "{} {} Did not find any words in {}",
             status_colorizer("ERROR"),
-            Cyan.paint("main::scan"),
+            module_colorizer("main::scan"),
             CONFIGURATION.wordlist
         );
         process::exit(1);
@@ -94,7 +87,7 @@ async fn scan(targets: Vec<String>) -> FeroxResult<()> {
     Ok(())
 }
 
-async fn get_targets() -> Vec<String> {
+async fn get_targets() -> FeroxResult<Vec<String>> {
     log::trace!("enter: get_targets");
 
     let mut targets = vec![];
@@ -106,14 +99,7 @@ async fn get_targets() -> Vec<String> {
         let mut reader = FramedRead::new(stdin, LinesCodec::new());
 
         while let Some(line) = reader.next().await {
-            match line {
+            targets.push(line?);
-                Ok(target) => {
-                    targets.push(target);
-                }
-                Err(e) => {
-                    log::error!("{}", e);
-                }
-            }
         }
     } else {
         targets.push(CONFIGURATION.target_url.clone());
@@ -121,7 +107,7 @@ async fn get_targets() -> Vec<String> {
 
     log::trace!("exit: get_targets -> {:?}", targets);
 
-    targets
+    Ok(targets)
 }
 
 #[tokio::main]
@@ -132,11 +118,27 @@ async fn main() {
     log::debug!("{:#?}", *CONFIGURATION);
 
     // get targets from command line or stdin
-    let targets = get_targets().await;
+    let targets = match get_targets().await {
+        Ok(t) => t,
+        Err(e) => {
+            // should only happen in the event that there was an error reading from stdin
+            log::error!("{}", e);
+            ferox_print(
+                &format!(
+                    "{} {} {}",
+                    status_colorizer("ERROR"),
+                    module_colorizer("main::get_targets"),
+                    e
+                ),
+                &PROGRESS_PRINTER,
+            );
+            process::exit(1);
+        }
+    };
 
     if !CONFIGURATION.quiet {
         // only print banner if -q isn't used
-        banner::initialize(&targets);
+        banner::initialize(&targets, &CONFIGURATION);
     }
 
     // discard non-responsive targets

src/scanner.rs

@@ -10,12 +10,15 @@ use reqwest::{Response, Url};
 use std::collections::HashSet;
 use std::convert::TryInto;
 use std::ops::Deref;
+use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::Arc;
 use tokio::fs;
 use tokio::io::{self, AsyncWriteExt};
 use tokio::sync::mpsc::{self, UnboundedReceiver, UnboundedSender};
 use tokio::task::JoinHandle;
 
+static CALL_COUNT: AtomicUsize = AtomicUsize::new(0);
+
 /// Spawn a single consumer task (sc side of mpsc)
 ///
 /// The consumer simply receives responses and writes them to the given output file if they meet
@@ -430,10 +433,10 @@ pub async fn scan_url(target_url: &str, wordlist: Arc<HashSet<String>>, base_dep
     let progress_bar = progress::add_bar(&target_url, num_reqs_expected, false);
     progress_bar.reset_elapsed();
 
-    if get_current_depth(&target_url) - base_depth == 0 {
+    if CALL_COUNT.load(Ordering::Relaxed) == 0 {
         // join can only be called once, otherwise it causes the thread to panic
-        // when current depth - base depth equals zero, we're in the first call to scan_url
         tokio::task::spawn_blocking(move || PROGRESS_BAR.join().unwrap());
+        CALL_COUNT.fetch_add(1, Ordering::Relaxed);
     }
 
     let wildcard_bar = progress_bar.clone();
@@ -522,3 +525,62 @@ pub async fn scan_url(target_url: &str, wordlist: Arc<HashSet<String>>, base_dep
     log::trace!("done awaiting report receiver");
     log::trace!("exit: scan_url");
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    /// sending url + word without any extensions should get back one url with the joined word
+    fn create_urls_no_extension_returns_base_url_with_word() {
+        let urls = create_urls("http://localhost", "turbo", &[]);
+        assert_eq!(urls, [Url::parse("http://localhost/turbo").unwrap()])
+    }
+
+    #[test]
+    /// sending url + word + 1 extension should get back two urls, one base and one with extension
+    fn create_urls_one_extension_returns_two_urls() {
+        let urls = create_urls("http://localhost", "turbo", &[String::from("js")]);
+        assert_eq!(
+            urls,
+            [
+                Url::parse("http://localhost/turbo").unwrap(),
+                Url::parse("http://localhost/turbo.js").unwrap()
+            ]
+        )
+    }
+
+    #[test]
+    /// sending url + word + multiple extensions should get back n+1 urls
+    fn create_urls_multiple_extensions_returns_n_plus_one_urls() {
+        let ext_vec = vec![
+            vec![String::from("js")],
+            vec![String::from("js"), String::from("php")],
+            vec![String::from("js"), String::from("php"), String::from("pdf")],
+            vec![
+                String::from("js"),
+                String::from("php"),
+                String::from("pdf"),
+                String::from("tar.gz"),
+            ],
+        ];
+
+        let base = Url::parse("http://localhost/turbo").unwrap();
+        let js = Url::parse("http://localhost/turbo.js").unwrap();
+        let php = Url::parse("http://localhost/turbo.php").unwrap();
+        let pdf = Url::parse("http://localhost/turbo.pdf").unwrap();
+        let tar = Url::parse("http://localhost/turbo.tar.gz").unwrap();
+
+        let expected = vec![
+            vec![base.clone(), js.clone()],
+            vec![base.clone(), js.clone(), php.clone()],
+            vec![base.clone(), js.clone(), php.clone(), pdf.clone()],
+            vec![base, js, php, pdf, tar],
+        ];
+
+        for (i, ext_set) in ext_vec.into_iter().enumerate() {
+            let urls = create_urls("http://localhost", "turbo", &ext_set);
+            assert_eq!(urls, expected[i]);
+        }
+    }
+}

src/utils.rs

@@ -1,6 +1,5 @@
 use crate::FeroxResult;
-use ansi_term::Color::{Blue, Cyan, Green, Red, Yellow};
+use console::{strip_ansi_codes, style, user_attended};
-use console::{strip_ansi_codes, user_attended};
 use indicatif::ProgressBar;
 use reqwest::Url;
 use reqwest::{Client, Response};
@@ -63,17 +62,24 @@ pub fn get_current_depth(target: &str) -> usize {
 /// Takes in a string and examines the first character to return a color version of the same string
 pub fn status_colorizer(status: &str) -> String {
     match status.chars().next() {
-        Some('1') => Blue.paint(status).to_string(), // informational
+        Some('1') => style(status).blue().to_string(), // informational
-        Some('2') => Green.bold().paint(status).to_string(), // success
+        Some('2') => style(status).green().to_string(), // success
-        Some('3') => Yellow.paint(status).to_string(), // redirects
+        Some('3') => style(status).yellow().to_string(), // redirects
-        Some('4') => Red.paint(status).to_string(),  // client error
+        Some('4') => style(status).red().to_string(),  // client error
-        Some('5') => Red.paint(status).to_string(),  // server error
+        Some('5') => style(status).red().to_string(),  // server error
-        Some('W') => Cyan.paint(status).to_string(), // wildcard
+        Some('W') => style(status).cyan().to_string(), // wildcard
-        Some('E') => Red.paint(status).to_string(),  // error
+        Some('E') => style(status).red().to_string(),  // error
-        _ => status.to_string(),                     // ¯\_(ツ)_/¯
+        _ => status.to_string(),                       // ¯\_(ツ)_/¯
     }
 }
 
+/// Takes in a string and colors it using console::style
+///
+/// mainly putting this here in case i want to change the color later, making any changes easy
+pub fn module_colorizer(modname: &str) -> String {
+    style(modname).cyan().to_string()
+}
+
 /// Gets the length of a url's path
 ///
 /// example: http://localhost/stuff -> 5
@@ -228,30 +234,42 @@ mod tests {
     use super::*;
 
     #[test]
+    /// base url returns 1
     fn get_current_depth_base_url_returns_1() {
         let depth = get_current_depth("http://localhost");
         assert_eq!(depth, 1);
     }
 
     #[test]
+    /// base url with slash returns 1
     fn get_current_depth_base_url_with_slash_returns_1() {
         let depth = get_current_depth("http://localhost/");
         assert_eq!(depth, 1);
     }
 
     #[test]
+    /// base url + 1 dir returns 2
     fn get_current_depth_one_dir_returns_2() {
         let depth = get_current_depth("http://localhost/src");
         assert_eq!(depth, 2);
     }
 
     #[test]
+    /// base url + 1 dir and slash returns 2
     fn get_current_depth_one_dir_with_slash_returns_2() {
         let depth = get_current_depth("http://localhost/src/");
         assert_eq!(depth, 2);
     }
 
     #[test]
+    /// base url + 1 dir and slash returns 2
+    fn get_current_depth_single_forward_slash_is_zero() {
+        let depth = get_current_depth("");
+        assert_eq!(depth, 0);
+    }
+
+    #[test]
+    /// base url + 1 word + no slash + no extension
     fn format_url_normal() {
         assert_eq!(
             format_url("http://localhost", "stuff", false, &Vec::new(), None).unwrap(),
@@ -260,6 +278,7 @@ mod tests {
     }
 
     #[test]
+    /// base url + no word + no slash + no extension
     fn format_url_no_word() {
         assert_eq!(
             format_url("http://localhost", "", false, &Vec::new(), None).unwrap(),
@@ -267,13 +286,47 @@ mod tests {
         );
     }
 
+    #[test]
+    /// base url + word + no slash + no extension + queries
+    fn format_url_joins_queries() {
+        assert_eq!(
+            format_url(
+                "http://localhost",
+                "lazer",
+                false,
+                &[(String::from("stuff"), String::from("things"))],
+                None
+            )
+            .unwrap(),
+            reqwest::Url::parse("http://localhost/lazer?stuff=things").unwrap()
+        );
+    }
+
+    #[test]
+    /// base url + no word + no slash + no extension + queries
+    fn format_url_without_word_joins_queries() {
+        assert_eq!(
+            format_url(
+                "http://localhost",
+                "",
+                false,
+                &[(String::from("stuff"), String::from("things"))],
+                None
+            )
+            .unwrap(),
+            reqwest::Url::parse("http://localhost/?stuff=things").unwrap()
+        );
+    }
+
     #[test]
     #[should_panic]
+    /// no base url is an error
     fn format_url_no_url() {
         format_url("", "stuff", false, &Vec::new(), None).unwrap();
     }
 
     #[test]
+    /// word prepended with slash is adjusted for correctness
     fn format_url_word_with_preslash() {
         assert_eq!(
             format_url("http://localhost", "/stuff", false, &Vec::new(), None).unwrap(),
@@ -282,10 +335,59 @@ mod tests {
     }
 
     #[test]
+    /// word with appended slash allows the slash to persist
     fn format_url_word_with_postslash() {
         assert_eq!(
             format_url("http://localhost", "stuff/", false, &Vec::new(), None).unwrap(),
             reqwest::Url::parse("http://localhost/stuff/").unwrap()
         );
     }
+
+    #[test]
+    /// status colorizer uses red for 500s
+    fn status_colorizer_uses_red_for_500s() {
+        assert_eq!(status_colorizer("500"), style("500").red().to_string());
+    }
+
+    #[test]
+    /// status colorizer uses red for 400s
+    fn status_colorizer_uses_red_for_400s() {
+        assert_eq!(status_colorizer("400"), style("400").red().to_string());
+    }
+
+    #[test]
+    /// status colorizer uses red for errors
+    fn status_colorizer_uses_red_for_errors() {
+        assert_eq!(status_colorizer("ERROR"), style("ERROR").red().to_string());
+    }
+
+    #[test]
+    /// status colorizer uses cyan for wildcards
+    fn status_colorizer_uses_cyan_for_wildcards() {
+        assert_eq!(status_colorizer("WLD"), style("WLD").cyan().to_string());
+    }
+
+    #[test]
+    /// status colorizer uses blue for 100s
+    fn status_colorizer_uses_blue_for_100s() {
+        assert_eq!(status_colorizer("100"), style("100").blue().to_string());
+    }
+
+    #[test]
+    /// status colorizer uses green for 200s
+    fn status_colorizer_uses_green_for_200s() {
+        assert_eq!(status_colorizer("200"), style("200").green().to_string());
+    }
+
+    #[test]
+    /// status colorizer uses yellow for 300s
+    fn status_colorizer_uses_yellow_for_300s() {
+        assert_eq!(status_colorizer("300"), style("300").yellow().to_string());
+    }
+
+    #[test]
+    /// status colorizer doesnt color anything else
+    fn status_colorizer_returns_as_is() {
+        assert_eq!(status_colorizer("farfignewton"), "farfignewton".to_string());
+    }
 }

tests/test_banner.rs

@@ -0,0 +1,538 @@
+mod utils;
+use assert_cmd::Command;
+use predicates::prelude::*;
+use utils::{setup_tmp_directory, teardown_tmp_directory};
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + proxy
+fn banner_prints_proxy() -> Result<(), Box<dyn std::error::Error>> {
+    let urls = vec![
+        String::from("http://localhost"),
+        String::from("http://schmocalhost"),
+    ];
+    let (tmp_dir, file) = setup_tmp_directory(&urls)?;
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--stdin")
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("--proxy")
+        .arg("http://127.0.0.1:8080")
+        .pipe_stdin(file)
+        .unwrap()
+        .assert()
+        .failure()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("http://schmocalhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Proxy"))
+                .and(predicate::str::contains("http://127.0.0.1:8080"))
+                .and(predicate::str::contains("─┴─")),
+        );
+
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + multiple headers
+fn banner_prints_headers() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--headers")
+        .arg("stuff:things")
+        .arg("-H")
+        .arg("mostuff:mothings")
+        .assert()
+        .failure()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Header"))
+                .and(predicate::str::contains("stuff: things"))
+                .and(predicate::str::contains("mostuff: mothings"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + multiple size filters
+fn banner_prints_size_filters() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-S")
+        .arg("789456123")
+        .arg("--sizefilter")
+        .arg("44444444")
+        .assert()
+        .failure()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Size Filter"))
+                .and(predicate::str::contains("789456123"))
+                .and(predicate::str::contains("44444444"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + queries
+fn banner_prints_queries() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-Q")
+        .arg("token=supersecret")
+        .arg("--query")
+        .arg("stuff=things")
+        .assert()
+        .failure()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Query Parameter"))
+                .and(predicate::str::contains("token=supersecret"))
+                .and(predicate::str::contains("stuff=things"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + status codes
+fn banner_prints_status_codes() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-s")
+        .arg("201,301,401")
+        .assert()
+        .failure()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("[201, 301, 401]"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + output file
+fn banner_prints_output_file() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--output")
+        .arg("/super/cool/path")
+        .assert()
+        .failure()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Output File"))
+                .and(predicate::str::contains("/super/cool/path"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + insecure
+fn banner_prints_insecure() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-k")
+        .assert()
+        .failure()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Insecure"))
+                .and(predicate::str::contains("true"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + follow redirects
+fn banner_prints_redirects() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-r")
+        .assert()
+        .failure()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Follow Redirects"))
+                .and(predicate::str::contains("true"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + extensions
+fn banner_prints_extensions() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-x")
+        .arg("js")
+        .arg("--extensions")
+        .arg("pdf")
+        .assert()
+        .failure()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Extensions"))
+                .and(predicate::str::contains("[js, pdf]"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + dontfilter
+fn banner_prints_dontfilter() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--dontfilter")
+        .assert()
+        .failure()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Filter Wildcards"))
+                .and(predicate::str::contains("false"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + verbosity=1
+fn banner_prints_verbosity_one() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-v")
+        .assert()
+        .failure()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Verbosity"))
+                .and(predicate::str::contains("│ 1"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + verbosity=2
+fn banner_prints_verbosity_two() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-vv")
+        .assert()
+        .failure()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Verbosity"))
+                .and(predicate::str::contains("│ 2"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + verbosity=3
+fn banner_prints_verbosity_three() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-vvv")
+        .assert()
+        .failure()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Verbosity"))
+                .and(predicate::str::contains("│ 3"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + verbosity=4
+fn banner_prints_verbosity_four() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-vvvv")
+        .assert()
+        .failure()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Verbosity"))
+                .and(predicate::str::contains("│ 4"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + add slash
+fn banner_prints_add_slash() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-f")
+        .assert()
+        .failure()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Add Slash"))
+                .and(predicate::str::contains("true"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + INFINITE recursion
+fn banner_prints_infinite_depth() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--depth")
+        .arg("0")
+        .assert()
+        .failure()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Recursion Depth"))
+                .and(predicate::str::contains("INFINITE"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + recursion depth
+fn banner_prints_recursion_depth() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("--depth")
+        .arg("343214")
+        .assert()
+        .failure()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Recursion Depth"))
+                .and(predicate::str::contains("343214"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + no recursion
+fn banner_prints_no_recursion() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-n")
+        .assert()
+        .failure()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Do Not Recurse"))
+                .and(predicate::str::contains("true"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see only the error of could not connect
+fn banner_doesnt_print() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-q")
+        .assert()
+        .failure()
+        .stderr(predicate::str::contains(
+            "ERROR heuristics::connectivity_test Could not connect to any target provided",
+        ));
+    Ok(())
+}

tests/test_heuristics.rs

@@ -12,17 +12,19 @@ use utils::{setup_tmp_directory, teardown_tmp_directory};
 fn test_single_target_cannot_connect() -> Result<(), Box<dyn std::error::Error>> {
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()])?;
 
-    let cmd = std::panic::catch_unwind(|| {
+    Command::cargo_bin("feroxbuster")
-        Command::cargo_bin("feroxbuster")
+        .unwrap()
-            .unwrap()
+        .arg("--url")
-            .arg("--url")
+        .arg("http://fjdksafjkdsajfkdsajkfdsajkfsdjkdsfdsafdsafdsajkr3l2ajfdskafdsjk")
-            .arg("http://fjdksafjkdsajfkdsajkfdsajkfsdjkdsfdsafdsafdsajkr3l2ajfdskafdsjk")
+        .arg("--wordlist")
-            .arg("--wordlist")
+        .arg(file.as_os_str())
-            .arg(file.as_os_str())
+        .assert()
-            .unwrap()
+        .failure()
-    });
+        .stderr(
-
+            predicate::str::contains("Could not connect to any target provided")
-    assert!(cmd.is_err());
+                .and(predicate::str::contains("ERROR"))
+                .and(predicate::str::contains("heuristics::connectivity_test")),
+        );
 
     teardown_tmp_directory(tmp_dir);
     Ok(())
@@ -37,18 +39,20 @@ fn test_two_targets_cannot_connect() -> Result<(), Box<dyn std::error::Error>> {
     let urls = vec![not_real.clone(), not_real];
     let (tmp_dir, file) = setup_tmp_directory(&urls)?;
 
-    let cmd = std::panic::catch_unwind(|| {
+    Command::cargo_bin("feroxbuster")
-        Command::cargo_bin("feroxbuster")
+        .unwrap()
-            .unwrap()
+        .arg("--stdin")
-            .arg("--stdin")
+        .arg("--wordlist")
-            .arg("--wordlist")
+        .arg(file.as_os_str())
-            .arg(file.as_os_str())
+        .pipe_stdin(file)
-            .pipe_stdin(file)
+        .unwrap()
-            .unwrap()
+        .assert()
-            .unwrap()
+        .failure()
-    });
+        .stderr(
-
+            predicate::str::contains("Could not connect to any target provided")
-    assert!(cmd.is_err());
+                .and(predicate::str::contains("ERROR"))
+                .and(predicate::str::contains("heuristics::connectivity_test")),
+        );
 
     teardown_tmp_directory(tmp_dir);
     Ok(())

tests/test_main.rs

@@ -0,0 +1,98 @@
+mod utils;
+use assert_cmd::Command;
+use httpmock::Method::GET;
+use httpmock::{Mock, MockServer};
+use predicates::prelude::*;
+use utils::{setup_tmp_directory, teardown_tmp_directory};
+
+#[test]
+/// send the function a file to which we dont have permission in order to execute error branch
+fn main_use_root_owned_file_as_wordlist() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/")
+        .return_status(200)
+        .return_body("this is a test")
+        .create_on(&srv);
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg("/etc/shadow")
+        .arg("-vvvv")
+        .assert()
+        .success()
+        .stderr(predicate::str::contains(
+            "ERROR main::get_unique_words_from_wordlist Permission denied (os error 13)",
+        ));
+
+    // connectivity test hits it once
+    assert_eq!(mock.times_called(), 1);
+    Ok(())
+}
+
+#[test]
+/// send the function an empty file
+fn main_use_empty_wordlist() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let (tmp_dir, file) = setup_tmp_directory(&[])?;
+
+    let mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/")
+        .return_status(200)
+        .return_body("this is a test")
+        .create_on(&srv);
+
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-vvvv")
+        .assert()
+        .failure()
+        .stderr(predicate::str::contains(
+            "ERROR main::scan Did not find any words in",
+        ));
+
+    assert_eq!(mock.times_called(), 1);
+
+    teardown_tmp_directory(tmp_dir);
+    Ok(())
+}
+
+#[test]
+/// send nothing over stdin, expect heuristics to be upset during connectivity test
+fn main_use_empty_stdin_targets() -> Result<(), Box<dyn std::error::Error>> {
+    let (tmp_dir, file) = setup_tmp_directory(&[])?;
+
+    // get_targets is called before scan, so the empty wordlist shouldn't trigger
+    // the 'Did not find any words' error
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--stdin")
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-vvv")
+        .pipe_stdin(file)
+        .unwrap()
+        .assert()
+        .failure()
+        .stderr(
+            predicate::str::contains("Could not connect to any target provided")
+                .and(predicate::str::contains("ERROR"))
+                .and(predicate::str::contains("heuristics::connectivity_test"))
+                .and(predicate::str::contains("Target Url"))
+                .not(), // no target url found
+        );
+
+    teardown_tmp_directory(tmp_dir);
+
+    Ok(())
+}

tests/test_scanner.rs

@@ -7,6 +7,7 @@ use std::process::Command;
 use utils::{setup_tmp_directory, teardown_tmp_directory};
 
 #[test]
+/// send a single valid request, expect a 200 response
 fn test_single_request_scan() -> Result<(), Box<dyn std::error::Error>> {
     let srv = MockServer::start();
     let (tmp_dir, file) = setup_tmp_directory(&["LICENSE".to_string()])?;
@@ -24,6 +25,7 @@ fn test_single_request_scan() -> Result<(), Box<dyn std::error::Error>> {
         .arg(srv.url("/"))
         .arg("--wordlist")
         .arg(file.as_os_str())
+        .arg("-vvvv")
         .unwrap();
 
     cmd.assert().success().stdout(
@@ -36,3 +38,72 @@ fn test_single_request_scan() -> Result<(), Box<dyn std::error::Error>> {
     teardown_tmp_directory(tmp_dir);
     Ok(())
 }
+
+#[test]
+/// send a valid request, follow redirects into new directories, expect 301/200 responses
+fn scanner_recursive_request_scan() -> Result<(), Box<dyn std::error::Error>> {
+    let srv = MockServer::start();
+    let urls = [
+        "js".to_string(),
+        "prod".to_string(),
+        "dev".to_string(),
+        "file.js".to_string(),
+    ];
+    let (tmp_dir, file) = setup_tmp_directory(&urls)?;
+
+    let js_mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/js")
+        .return_status(301)
+        .return_header("Location", &srv.url("/js/"))
+        .create_on(&srv);
+
+    let js_prod_mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/js/prod")
+        .return_status(301)
+        .return_header("Location", &srv.url("/js/prod/"))
+        .create_on(&srv);
+
+    let js_dev_mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/js/dev")
+        .return_status(301)
+        .return_header("Location", &srv.url("/js/dev/"))
+        .create_on(&srv);
+
+    let js_dev_file_mock = Mock::new()
+        .expect_method(GET)
+        .expect_path("/js/dev/file.js")
+        .return_status(200)
+        .return_body("this is a test and is more bytes than other ones")
+        .create_on(&srv);
+
+    let cmd = Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg(srv.url("/"))
+        .arg("--wordlist")
+        .arg(file.as_os_str())
+        .arg("-vvvv")
+        .arg("-t")
+        .arg("1")
+        .unwrap();
+
+    cmd.assert().success().stdout(
+        predicate::str::is_match("301.*js")
+            .unwrap()
+            .and(predicate::str::is_match("301.*js/prod").unwrap())
+            .and(predicate::str::is_match("301.*js/dev").unwrap())
+            .and(predicate::str::is_match("200.*js/dev/file.js").unwrap()),
+    );
+
+    assert_eq!(js_mock.times_called(), 1);
+    assert_eq!(js_prod_mock.times_called(), 1);
+    assert_eq!(js_dev_mock.times_called(), 1);
+    assert_eq!(js_dev_file_mock.times_called(), 1);
+
+    teardown_tmp_directory(tmp_dir);
+
+    Ok(())
+}