Merge pull request #96 from epi052/FEATURE-limit-number-of-scans

Added ability to limit number of scans

.github/workflows/build.yml

@@ -41,6 +41,9 @@ jobs:
           use-cross: true
           command: build
           args: --release --target=${{ matrix.target }}
+      - name: Strip symbols from binary
+        run: |
+          strip -s ${{ matrix.path }}
       - name: Build tar.gz for homebrew installs
         if: matrix.type == 'ubuntu-x64'
         run: |
@@ -83,6 +86,9 @@ jobs:
           use-cross: true
           command: build
           args: --release --target=x86_64-apple-darwin
+      - name: Strip symbols from binary
+        run: |
+          strip -u -r target/x86_64-apple-darwin/release/feroxbuster
       - name: Build tar.gz for homebrew installs
         run: |
           tar czf x86_64-macos-feroxbuster.tar.gz -C target/x86_64-apple-darwin/release feroxbuster

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "feroxbuster"
-version = "1.1.0"
+version = "1.2.0"
 authors = ["Ben 'epi' Risher <epibar052@gmail.com>"]
 license = "MIT"
 edition = "2018"
@@ -25,6 +25,7 @@ clap = "2"
 lazy_static = "1.4"
 toml = "0.5"
 serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
 uuid = { version = "0.8", features = ["v4"] }
 indicatif = "0.15"
 console = "0.12"

README.md

@@ -79,7 +79,10 @@ This attack is also known as Predictable Resource Location, File Enumeration, Di
     - [Proxy traffic through Burp](#proxy-traffic-through-burp)
     - [Proxy traffic through a SOCKS proxy](#proxy-traffic-through-a-socks-proxy)
     - [Pass auth token via query parameter](#pass-auth-token-via-query-parameter)
+    - [Limit Total Number of Concurrent Scans (new in `v1.2.0`)](#limit-total-number-of-concurrent-scans-new-in-v120)
 - [Comparison w/ Similar Tools](#-comparison-w-similar-tools)
+- [Common Problems/Issues (FAQ)](#-common-problemsissues-faq)
+    - [No file descriptors available](#no-file-descriptors-available)
 
 ## 💿 Installation
 
@@ -235,6 +238,7 @@ Configuration begins with with the following built-in default values baked into
 - wordlist: `/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt`
 - threads: `50`
 - verbosity: `0` (no logging enabled)
+- scan_limit: `0` (no limit imposed on concurrent scans)
 - statuscodes: `200 204 301 302 307 308 401 403 405`
 - useragent: `feroxbuster/VERSION`
 - recursion depth: `4`
@@ -291,6 +295,7 @@ A pre-made configuration file with examples of all available settings can be fou
 # timeout = 5
 # proxy = "http://127.0.0.1:8080"
 # verbosity = 1
+# scan_limit = 6
 # quiet = true
 # output = "/targets/ellingson_mineral_company/gibson.txt"
 # useragent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0"
@@ -348,6 +353,7 @@ OPTIONS:
     -o, --output <FILE>                     Output file to write results to (default: stdout)
     -p, --proxy <PROXY>                     Proxy to use for requests (ex: http(s)://host:port, socks5://host:port)
     -Q, --query <QUERY>...                  Specify URL query parameters (ex: -Q token=stuff -Q secret=key)
+    -L, --scan-limit <SCAN_LIMIT>           Limit total number of concurrent scans (default: 7)
     -S, --sizefilter <SIZE>...              Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)
     -s, --statuscodes <STATUS_CODE>...      Status Codes of interest (default: 200 204 301 302 307 308 401 403 405)
     -t, --threads <THREADS>                 Number of concurrent threads (default: 50)
@@ -421,12 +427,23 @@ cat targets | ./feroxbuster --stdin --quiet -s 200 301 302 --redirects -x js | f
 ./feroxbuster -u http://127.1 --proxy socks5://127.0.0.1:9050
 ```
 
-### Pass auth token via query parameter
+### Pass auth token via query parameter 
 
 ```
 ./feroxbuster -u http://127.1 --query token=0123456789ABCDEF
 ```
 
+### Limit Total Number of Concurrent Scans (new in `v1.2.0`)
+
+Limit the number of scans permitted to run at any given time.  Recursion will still identify new directories, but newly
+discovered directories can only begin scanning when the total number of active scans drops below the value passed to 
+`--scan-limit`.
+
+```
+./feroxbuster -u http://127.1 --scan-limit 2
+```
+
+![limit-demo](img/limit-demo.gif)
 
 ## 🧐 Comparison w/ Similar Tools
 
@@ -470,3 +487,74 @@ came across rustbuster when I was naming my tool (😢). I don't have any experi
 be able to do POST requests with an HTTP body, has SOCKS support, and has an 8.3 shortname scanner (in addition to vhost
 dns, directory, etc...).  In short, it definitely looks interesting and may be what you're looking for as it has some 
 capability I haven't seen in similar tools.  
+
+## 🤯 Common Problems/Issues (FAQ)
+
+### No file descriptors available
+
+Why do I get a bunch of `No file descriptors available (os error 24)` errors?
+
+---
+
+There are a few potential causes of this error.  The simplest is that your operating system sets an open file limit that is aggressively low.  Through personal testing, I've found that `4096` is a reasonable open file limit (this will vary based on your exact setup).
+
+There are quite a few options to solve this particular problem, of which a handful are shown below.  
+
+#### Increase the Number of Open Files
+
+We'll start by increasing the number of open files the OS allows. On my Kali install, the default was `1024`, and I know some MacOS installs use `256` 😕.
+
+##### Edit `/etc/security/limits.conf`
+
+One option to up the limit is to edit `/etc/security/limits.conf` so that it includes the two lines below.  
+
+- `*` represents all users
+- `hard` and `soft` indicate the hard and soft limits for the OS 
+- `nofile` is the number of open files option. 
+
+```
+/etc/security/limits.conf
+-------------------------
+...
+*        soft nofile 4096
+*        hard nofile 8192
+...
+```
+
+##### Use `ulimit` directly
+
+A faster option, that is **not** persistent, is to simply use the `ulimit` command to change the setting.
+
+```
+ulimit -n 4096
+```
+
+#### Additional Tweaks (may not be needed)
+
+If you still find yourself hitting the file limit with the above changes, there are a few additional tweaks that may help.  
+
+> This section was shamelessly stolen from this [stackoverflow answer](https://stackoverflow.com/a/3923785).  More information is included in that post and is recommended reading if you end up needing to use this section.
+
+✨ Special thanks to HTB user [@sparkla](https://www.hackthebox.eu/home/users/profile/221599) for their help with identifying these additional tweaks ✨
+
+##### Increase the ephemeral port range, and decrease the tcp_fin_timeout.
+
+The ephermal port range defines the maximum number of outbound sockets a host can create from a particular I.P. address. The fin_timeout defines the minimum time these sockets will stay in TIME_WAIT state (unusable after being used once). Usual system defaults are
+
+- `net.ipv4.ip_local_port_range = 32768   61000`
+- `net.ipv4.tcp_fin_timeout = 60`
+
+This basically means your system cannot consistently guarantee more than `(61000 - 32768) / 60 = 470` sockets per second.
+
+```
+sudo sysctl net.ipv4.ip_local_port_range="15000 61000"
+sudo sysctl net.ipv4.tcp_fin_timeout=30
+```
+
+##### Allow socket reuse while in a `TIME_WAIT` status
+
+This allows fast cycling of sockets in time_wait state and re-using them. Make sure to read post [Coping with the TCP TIME-WAIT](https://vincent.bernat.ch/en/blog/2014-tcp-time-wait-state-linux) from Vincent Bernat to understand the implications.
+
+```
+sudo sysctl net.ipv4.tcp_tw_reuse=1 
+```

ferox-config.toml.example

@@ -13,6 +13,7 @@
 # timeout = 5
 # proxy = "http://127.0.0.1:8080"
 # verbosity = 1
+# scan_limit = 6
 # quiet = true
 # output = "/targets/ellingson_mineral_company/gibson.txt"
 # useragent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0"

src/banner.rs

@@ -1,4 +1,8 @@
-use crate::{config::Configuration, utils::status_colorizer, VERSION};
+use crate::config::{Configuration, CONFIGURATION};
+use crate::utils::{make_request, status_colorizer};
+use reqwest::{Client, Url};
+use serde_json::Value;
+use std::io::Write;
 
 /// macro helper to abstract away repetitive string formatting
 macro_rules! format_banner_entry_helper {
@@ -40,31 +44,119 @@ macro_rules! format_banner_entry {
     };
 }
 
+/// Url used to query github's api; specifically used to look for the latest tagged release name
+const UPDATE_URL: &str = "https://api.github.com/repos/epi052/feroxbuster/releases/latest";
+
+/// Simple enum to hold three different update states
+#[derive(Debug)]
+enum UpdateStatus {
+    /// this version and latest release are the same
+    UpToDate,
+
+    /// this version and latest release are not the same
+    OutOfDate,
+
+    /// some error occurred during version check
+    Unknown,
+}
+
+/// Makes a request to the given url, expecting to receive a JSON response that contains a field
+/// named `tag_name` that holds a value representing the latest tagged release of this tool.
+///
+/// ex: v1.1.0
+///
+/// Returns `UpdateStatus`
+async fn needs_update(client: &Client, url: &str, bin_version: &str) -> UpdateStatus {
+    log::trace!("enter: needs_update({:?}, {})", client, url);
+
+    let unknown = UpdateStatus::Unknown;
+
+    let api_url = match Url::parse(url) {
+        Ok(url) => url,
+        Err(e) => {
+            log::error!("{}", e);
+            log::trace!("exit: needs_update -> {:?}", unknown);
+            return unknown;
+        }
+    };
+
+    if let Ok(response) = make_request(&client, &api_url).await {
+        let body = response.text().await.unwrap_or_default();
+
+        let json_response: Value = serde_json::from_str(&body).unwrap_or_default();
+
+        if json_response.is_null() {
+            // unwrap_or_default above should result in a null value for the json_response variable
+            log::error!("Could not parse JSON from response body");
+            log::trace!("exit: needs_update -> {:?}", unknown);
+            return unknown;
+        }
+
+        let latest_version = match json_response["tag_name"].as_str() {
+            Some(tag) => tag.trim_start_matches('v'),
+            None => {
+                log::error!("Could not get version field from JSON response");
+                log::debug!("{}", json_response);
+                log::trace!("exit: needs_update -> {:?}", unknown);
+                return unknown;
+            }
+        };
+
+        // if we've gotten this far, we have a string in the form of X.X.X where X is a number
+        // all that's left is to compare the current version with the version found above
+
+        return if latest_version == bin_version {
+            // there's really only two possible outcomes if we accept that the tag conforms to
+            // the X.X.X pattern:
+            //   1. the version strings match, meaning we're up to date
+            //   2. the version strings do not match, meaning we're out of date
+            //
+            // except for developers working on this code, nobody should ever be in a situation
+            // where they have a version greater than the latest tagged release
+            log::trace!("exit: needs_update -> UpdateStatus::UpToDate");
+            UpdateStatus::UpToDate
+        } else {
+            log::trace!("exit: needs_update -> UpdateStatus::OutOfDate");
+            UpdateStatus::OutOfDate
+        };
+    }
+
+    log::trace!("exit: needs_update -> {:?}", unknown);
+    unknown
+}
+
 /// Prints the banner to stdout.
 ///
 /// Only prints those settings which are either always present, or passed in by the user.
-pub fn initialize(targets: &[String], config: &Configuration) {
+pub async fn initialize<W>(targets: &[String], config: &Configuration, version: &str, mut writer: W)
+where
+    W: Write,
+{
     let artwork = format!(
         r#"
  ___  ___  __   __     __      __         __   ___
 |__  |__  |__) |__) | /  `    /  \ \_/ | |  \ |__
 |    |___ |  \ |  \ | \__,    \__/ / \ | |__/ |___
 by Ben "epi" Risher {}                  ver: {}"#,
-        '\u{1F913}', VERSION
+        '\u{1F913}', version
     );
 
+    let status = needs_update(&CONFIGURATION.client, UPDATE_URL, version).await;
+
     let top = "───────────────────────────┬──────────────────────";
     let bottom = "───────────────────────────┴──────────────────────";
 
-    eprintln!("{}", artwork);
-    eprintln!("{}", top);
+    writeln!(&mut writer, "{}", artwork).unwrap_or_default();
+    writeln!(&mut writer, "{}", top).unwrap_or_default();
 
     // begin with always printed items
     for target in targets {
-        eprintln!(
+        writeln!(
+            &mut writer,
             "{}",
             format_banner_entry!("\u{1F3af}", "Target Url", target)
-        ); // 🎯
+        )
+        .unwrap_or_default(); // 🎯
     }
 
     let mut codes = vec![];
@@ -73,217 +165,428 @@ by Ben "epi" Risher {}                  ver: {}"#,
         codes.push(status_colorizer(&code.to_string()))
     }
 
-    eprintln!(
+    writeln!(
+        &mut writer,
         "{}",
         format_banner_entry!("\u{1F680}", "Threads", config.threads)
-    ); // 🚀
+    )
+    .unwrap_or_default(); // 🚀
 
-    eprintln!(
+    writeln!(
+        &mut writer,
         "{}",
         format_banner_entry!("\u{1f4d6}", "Wordlist", config.wordlist)
-    ); // 📖
+    )
+    .unwrap_or_default(); // 📖
 
-    eprintln!(
+    writeln!(
+        &mut writer,
         "{}",
         format_banner_entry!(
             "\u{1F197}",
             "Status Codes",
             format!("[{}]", codes.join(", "))
         )
-    ); // 🆗
+    )
+    .unwrap_or_default(); // 🆗
 
-    eprintln!(
+    writeln!(
+        &mut writer,
         "{}",
         format_banner_entry!("\u{1f4a5}", "Timeout (secs)", config.timeout)
-    ); // 💥
+    )
+    .unwrap_or_default(); // 💥
 
-    eprintln!(
+    writeln!(
+        &mut writer,
         "{}",
         format_banner_entry!("\u{1F9a1}", "User-Agent", config.useragent)
-    ); // 🦡
+    )
+    .unwrap_or_default(); // 🦡
 
     // followed by the maybe printed or variably displayed values
     if !config.config.is_empty() {
-        eprintln!(
+        writeln!(
+            &mut writer,
             "{}",
             format_banner_entry!("\u{1f489}", "Config File", config.config)
-        ); // 💉
+        )
+        .unwrap_or_default(); // 💉
     }
 
     if !config.proxy.is_empty() {
-        eprintln!(
+        writeln!(
+            &mut writer,
             "{}",
             format_banner_entry!("\u{1f48e}", "Proxy", config.proxy)
-        ); // 💎
+        )
+        .unwrap_or_default(); // 💎
     }
 
     if !config.headers.is_empty() {
         for (name, value) in &config.headers {
-            eprintln!(
+            writeln!(
+                &mut writer,
                 "{}",
                 format_banner_entry!("\u{1f92f}", "Header", name, value)
-            ); // 🤯
+            )
+            .unwrap_or_default(); // 🤯
         }
     }
 
     if !config.sizefilters.is_empty() {
         for filter in &config.sizefilters {
-            eprintln!(
+            writeln!(
+                &mut writer,
                 "{}",
                 format_banner_entry!("\u{1f4a2}", "Size Filter", filter)
-            ); // 💢
+            )
+            .unwrap_or_default(); // 💢
         }
     }
 
     if config.extract_links {
-        eprintln!(
+        writeln!(
+            &mut writer,
             "{}",
             format_banner_entry!("\u{1F50E}", "Extract Links", config.extract_links)
-        ); // 🔎
+        )
+        .unwrap_or_default(); // 🔎
     }
 
     if !config.queries.is_empty() {
         for query in &config.queries {
-            eprintln!(
+            writeln!(
+                &mut writer,
                 "{}",
                 format_banner_entry!(
                     "\u{1f914}",
                     "Query Parameter",
                     format!("{}={}", query.0, query.1)
                 )
-            ); // 🤔
+            )
+            .unwrap_or_default(); // 🤔
         }
     }
 
     if !config.output.is_empty() {
-        eprintln!(
+        writeln!(
+            &mut writer,
             "{}",
             format_banner_entry!("\u{1f4be}", "Output File", config.output)
-        ); // 💾
+        )
+        .unwrap_or_default(); // 💾
     }
 
     if !config.extensions.is_empty() {
-        eprintln!(
+        writeln!(
+            &mut writer,
             "{}",
             format_banner_entry!(
                 "\u{1f4b2}",
                 "Extensions",
                 format!("[{}]", config.extensions.join(", "))
             )
-        ); // 💲
+        )
+        .unwrap_or_default(); // 💲
     }
 
     if config.insecure {
-        eprintln!(
+        writeln!(
+            &mut writer,
             "{}",
             format_banner_entry!("\u{1f513}", "Insecure", config.insecure)
-        ); // 🔓
+        )
+        .unwrap_or_default(); // 🔓
     }
 
     if config.redirects {
-        eprintln!(
+        writeln!(
+            &mut writer,
             "{}",
             format_banner_entry!("\u{1f4cd}", "Follow Redirects", config.redirects)
-        ); // 📍
+        )
+        .unwrap_or_default(); // 📍
     }
 
     if config.dontfilter {
-        eprintln!(
+        writeln!(
+            &mut writer,
             "{}",
             format_banner_entry!("\u{1f92a}", "Filter Wildcards", !config.dontfilter)
-        ); // 🤪
+        )
+        .unwrap_or_default(); // 🤪
     }
 
     match config.verbosity {
         //speaker medium volume (increasing with verbosity to loudspeaker)
         1 => {
-            eprintln!(
+            writeln!(
+                &mut writer,
                 "{}",
                 format_banner_entry!("\u{1f508}", "Verbosity", config.verbosity)
-            ); // 🔈
+            )
+            .unwrap_or_default(); // 🔈
         }
         2 => {
-            eprintln!(
+            writeln!(
+                &mut writer,
                 "{}",
                 format_banner_entry!("\u{1f509}", "Verbosity", config.verbosity)
-            ); // 🔉
+            )
+            .unwrap_or_default(); // 🔉
         }
         3 => {
-            eprintln!(
+            writeln!(
+                &mut writer,
                 "{}",
                 format_banner_entry!("\u{1f50a}", "Verbosity", config.verbosity)
-            ); // 🔊
+            )
+            .unwrap_or_default(); // 🔊
         }
         4 => {
-            eprintln!(
+            writeln!(
+                &mut writer,
                 "{}",
                 format_banner_entry!("\u{1f4e2}", "Verbosity", config.verbosity)
-            ); // 📢
+            )
+            .unwrap_or_default(); // 📢
         }
         _ => {}
     }
 
     if config.addslash {
-        eprintln!(
+        writeln!(
+            &mut writer,
             "{}",
             format_banner_entry!("\u{1fa93}", "Add Slash", config.addslash)
-        ); // 🪓
+        )
+        .unwrap_or_default(); // 🪓
     }
 
     if !config.norecursion {
         if config.depth == 0 {
-            eprintln!(
+            writeln!(
+                &mut writer,
                 "{}",
                 format_banner_entry!("\u{1f503}", "Recursion Depth", "INFINITE")
-            ); // 🔃
+            )
+            .unwrap_or_default(); // 🔃
         } else {
-            eprintln!(
+            writeln!(
+                &mut writer,
                 "{}",
                 format_banner_entry!("\u{1f503}", "Recursion Depth", config.depth)
-            ); // 🔃
+            )
+            .unwrap_or_default(); // 🔃
         }
     } else {
-        eprintln!(
+        writeln!(
+            &mut writer,
             "{}",
             format_banner_entry!("\u{1f6ab}", "Do Not Recurse", config.norecursion)
-        ); // 🚫
+        )
+        .unwrap_or_default(); // 🚫
     }
 
-    eprintln!("{}", bottom);
+    if CONFIGURATION.scan_limit > 0 {
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!("\u{1f9a5}", "Concurrent Scan Limit", config.scan_limit)
+        )
+        .unwrap_or_default(); // 🦥
+    }
+
+    if matches!(status, UpdateStatus::OutOfDate) {
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!(
+                "\u{1f389}",
+                "New Version Available",
+                "https://github.com/epi052/feroxbuster/releases/latest"
+            )
+        )
+        .unwrap_or_default(); // 🎉
+    }
+
+    writeln!(&mut writer, "{}", bottom).unwrap_or_default();
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::VERSION;
+    use httpmock::Method::GET;
+    use httpmock::{Mock, MockServer};
+    use std::fs::read_to_string;
+    use std::io::stderr;
+    use std::time::Duration;
+    use tempfile::NamedTempFile;
 
-    #[test]
+    #[tokio::test(core_threads = 1)]
     /// test to hit no execution of targets for loop in banner
-    fn banner_without_targets() {
+    async fn banner_intialize_without_targets() {
         let config = Configuration::default();
-        initialize(&[], &config);
+        initialize(&[], &config, VERSION, stderr()).await;
     }
 
-    #[test]
+    #[tokio::test(core_threads = 1)]
     /// test to hit no execution of statuscode for loop in banner
-    fn banner_without_status_codes() {
+    async fn banner_intialize_without_status_codes() {
         let mut config = Configuration::default();
         config.statuscodes = vec![];
-        initialize(&[String::from("http://localhost")], &config);
+        initialize(
+            &[String::from("http://localhost")],
+            &config,
+            VERSION,
+            stderr(),
+        )
+        .await;
     }
 
-    #[test]
+    #[tokio::test(core_threads = 1)]
     /// test to hit an empty config file
-    fn banner_without_config_file() {
+    async fn banner_intialize_without_config_file() {
         let mut config = Configuration::default();
         config.config = String::new();
-        initialize(&[String::from("http://localhost")], &config);
+        initialize(
+            &[String::from("http://localhost")],
+            &config,
+            VERSION,
+            stderr(),
+        )
+        .await;
     }
 
-    #[test]
+    #[tokio::test(core_threads = 1)]
     /// test to hit an empty config file
-    fn banner_without_queries() {
+    async fn banner_intialize_without_queries() {
         let mut config = Configuration::default();
         config.queries = vec![(String::new(), String::new())];
-        initialize(&[String::from("http://localhost")], &config);
+        initialize(
+            &[String::from("http://localhost")],
+            &config,
+            VERSION,
+            stderr(),
+        )
+        .await;
+    }
+
+    #[tokio::test(core_threads = 1)]
+    /// test to show that a new version is available for download
+    async fn banner_intialize_with_mismatched_version() {
+        let config = Configuration::default();
+        let file = NamedTempFile::new().unwrap();
+        initialize(
+            &[String::from("http://localhost")],
+            &config,
+            "mismatched-version",
+            &file,
+        )
+        .await;
+        let contents = read_to_string(file.path()).unwrap();
+        println!("contents: {}", contents);
+        assert!(contents.contains("New Version Available"));
+        assert!(contents.contains("https://github.com/epi052/feroxbuster/releases/latest"));
+    }
+
+    #[tokio::test(core_threads = 1)]
+    /// test that
+    async fn banner_needs_update_returns_unknown_with_bad_url() {
+        let result = needs_update(&CONFIGURATION.client, &"", VERSION).await;
+        assert!(matches!(result, UpdateStatus::Unknown));
+    }
+
+    #[tokio::test(core_threads = 1)]
+    /// test return value of good url to needs_update
+    async fn banner_needs_update_returns_up_to_date() {
+        let srv = MockServer::start();
+
+        let mock = Mock::new()
+            .expect_method(GET)
+            .expect_path("/latest")
+            .return_status(200)
+            .return_body("{\"tag_name\":\"v1.1.0\"}")
+            .create_on(&srv);
+
+        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.1.0").await;
+
+        assert_eq!(mock.times_called(), 1);
+        assert!(matches!(result, UpdateStatus::UpToDate));
+    }
+
+    #[tokio::test(core_threads = 1)]
+    /// test return value of good url to needs_update that returns a newer version than current
+    async fn banner_needs_update_returns_out_of_date() {
+        let srv = MockServer::start();
+
+        let mock = Mock::new()
+            .expect_method(GET)
+            .expect_path("/latest")
+            .return_status(200)
+            .return_body("{\"tag_name\":\"v1.1.0\"}")
+            .create_on(&srv);
+
+        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1").await;
+
+        assert_eq!(mock.times_called(), 1);
+        assert!(matches!(result, UpdateStatus::OutOfDate));
+    }
+
+    #[tokio::test(core_threads = 1)]
+    /// test return value of good url that times out
+    async fn banner_needs_update_returns_unknown_on_timeout() {
+        let srv = MockServer::start();
+
+        let mock = Mock::new()
+            .expect_method(GET)
+            .expect_path("/latest")
+            .return_status(200)
+            .return_body("{\"tag_name\":\"v1.1.0\"}")
+            .return_with_delay(Duration::from_secs(8))
+            .create_on(&srv);
+
+        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1").await;
+
+        assert_eq!(mock.times_called(), 1);
+        assert!(matches!(result, UpdateStatus::Unknown));
+    }
+
+    #[tokio::test(core_threads = 1)]
+    /// test return value of good url with bad json response
+    async fn banner_needs_update_returns_unknown_on_bad_json_response() {
+        let srv = MockServer::start();
+
+        let mock = Mock::new()
+            .expect_method(GET)
+            .expect_path("/latest")
+            .return_status(200)
+            .return_body("not json")
+            .create_on(&srv);
+
+        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1").await;
+
+        assert_eq!(mock.times_called(), 1);
+        assert!(matches!(result, UpdateStatus::Unknown));
+    }
+
+    #[tokio::test(core_threads = 1)]
+    /// test return value of good url with json response that lacks the tag_name field
+    async fn banner_needs_update_returns_unknown_on_json_without_correct_tag() {
+        let srv = MockServer::start();
+
+        let mock = Mock::new()
+            .expect_method(GET)
+            .expect_path("/latest")
+            .return_status(200)
+            .return_body("{\"no tag_name\": \"doesn't exist\"}")
+            .create_on(&srv);
+
+        let result = needs_update(&CONFIGURATION.client, &srv.url("/latest"), "1.0.1").await;
+
+        assert_eq!(mock.times_called(), 1);
+        assert!(matches!(result, UpdateStatus::Unknown));
     }
 }

src/config.rs

@@ -123,6 +123,10 @@ pub struct Configuration {
     #[serde(default = "depth")]
     pub depth: usize,
 
+    /// Number of concurrent scans permitted; a limit of 0 means no limit is imposed
+    #[serde(default)]
+    pub scan_limit: usize,
+
     /// Filter out messages of a particular size
     #[serde(default)]
     pub sizefilters: Vec<u64>,
@@ -184,6 +188,7 @@ impl Default for Configuration {
             quiet: false,
             stdin: false,
             verbosity: 0,
+            scan_limit: 0,
             addslash: false,
             insecure: false,
             redirects: false,
@@ -232,6 +237,7 @@ impl Configuration {
     /// - **stdin**: `false`
     /// - **dontfilter**: `false` (auto filter wildcard responses)
     /// - **depth**: `4` (maximum recursion depth)
+    /// - **scan_limit**: `0` (no limit on concurrent scans imposed)
     ///
     /// After which, any values defined in a
     /// [ferox-config.toml](constant.DEFAULT_CONFIG_NAME.html) config file will override the
@@ -316,6 +322,12 @@ impl Configuration {
             config.depth = depth;
         }
 
+        if args.value_of("scan_limit").is_some() {
+            let scan_limit =
+                value_t!(args.value_of("scan_limit"), usize).unwrap_or_else(|e| e.exit());
+            config.scan_limit = scan_limit;
+        }
+
         if args.value_of("wordlist").is_some() {
             config.wordlist = String::from(args.value_of("wordlist").unwrap());
         }
@@ -534,6 +546,7 @@ impl Configuration {
         settings.depth = settings_to_merge.depth;
         settings.sizefilters = settings_to_merge.sizefilters;
         settings.dontfilter = settings_to_merge.dontfilter;
+        settings.scan_limit = settings_to_merge.scan_limit;
     }
 
     /// If present, read in `DEFAULT_CONFIG_NAME` and deserialize the specified values
@@ -575,6 +588,7 @@ mod tests {
             proxy = "http://127.0.0.1:8080"
             quiet = true
             verbosity = 1
+            scan_limit = 6
             output = "/some/otherpath"
             redirects = true
             insecure = true
@@ -608,6 +622,7 @@ mod tests {
         assert_eq!(config.depth, depth());
         assert_eq!(config.timeout, timeout());
         assert_eq!(config.verbosity, 0);
+        assert_eq!(config.scan_limit, 0);
         assert_eq!(config.quiet, false);
         assert_eq!(config.dontfilter, false);
         assert_eq!(config.norecursion, false);
@@ -650,6 +665,13 @@ mod tests {
         assert_eq!(config.depth, 1);
     }
 
+    #[test]
+    /// parse the test config and see that the value parsed is correct
+    fn config_reads_scan_limit() {
+        let config = setup_config_test();
+        assert_eq!(config.scan_limit, 6);
+    }
+
     #[test]
     /// parse the test config and see that the value parsed is correct
     fn config_reads_timeout() {

src/main.rs

@@ -1,11 +1,11 @@
 use feroxbuster::config::{CONFIGURATION, PROGRESS_PRINTER};
 use feroxbuster::scanner::scan_url;
 use feroxbuster::utils::{ferox_print, get_current_depth, module_colorizer, status_colorizer};
-use feroxbuster::{banner, heuristics, logger, reporter, FeroxResponse, FeroxResult};
+use feroxbuster::{banner, heuristics, logger, reporter, FeroxResponse, FeroxResult, VERSION};
 use futures::StreamExt;
 use std::collections::HashSet;
 use std::fs::File;
-use std::io::{BufRead, BufReader};
+use std::io::{stderr, BufRead, BufReader};
 use std::process;
 use std::sync::Arc;
 use tokio::io;
@@ -158,7 +158,8 @@ async fn main() {
 
     if !CONFIGURATION.quiet {
         // only print banner if -q isn't used
-        banner::initialize(&targets, &CONFIGURATION);
+        let std_stderr = stderr(); // std::io::stderr
+        banner::initialize(&targets, &CONFIGURATION, &VERSION, std_stderr).await;
     }
 
     // discard non-responsive targets

src/parser.rs

@@ -202,7 +202,14 @@ pub fn initialize() -> App<'static, 'static> {
                 .takes_value(false)
                 .help("Extract links from response body (html, javascript, etc...); make new requests based on findings (default: false)")
         )
-
+        .arg(
+            Arg::with_name("scan_limit")
+                .short("L")
+                .long("scan-limit")
+                .value_name("SCAN_LIMIT")
+                .takes_value(true)
+                .help("Limit total number of concurrent scans (default: 0, i.e. no limit)")
+        )
         .after_help(r#"NOTE:
     Options that take multiple values are very flexible.  Consider the following ways of specifying
     extensions:

src/scanner.rs

@@ -13,6 +13,7 @@ use std::ops::Deref;
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::{Arc, RwLock};
 use tokio::sync::mpsc::{self, UnboundedReceiver, UnboundedSender};
+use tokio::sync::Semaphore;
 use tokio::task::JoinHandle;
 
 /// Single atomic number that gets incremented once, used to track first scan vs. all others
@@ -24,6 +25,9 @@ lazy_static! {
 
     /// Vector of WildcardFilters that have been ID'd through heuristics
     static ref WILDCARD_FILTERS: Arc<RwLock<Vec<Arc<WildcardFilter>>>> = Arc::new(RwLock::new(Vec::<Arc<WildcardFilter>>::new()));
+
+    /// Bounded semaphore used as a barrier to limit concurrent scans
+    static ref SCAN_LIMITER: Semaphore = Semaphore::new(CONFIGURATION.scan_limit);
 }
 
 /// Adds the given url to `SCANNED_URLS`
@@ -120,6 +124,7 @@ fn spawn_recursion_handler(
 
     let boxed_future = async move {
         let mut scans = vec![];
+
         while let Some(resp) = recursion_channel.recv().await {
             let unknown = add_url_to_list_of_scanned_urls(&resp, &SCANNED_URLS);
 
@@ -555,8 +560,21 @@ pub async fn scan_url(
         // this protection around join also allows us to add the first scanned url to SCANNED_URLS
         // from within the scan_url function instead of the recursion handler
         add_url_to_list_of_scanned_urls(&target_url, &SCANNED_URLS);
+
+        if CONFIGURATION.scan_limit == 0 {
+            // scan_limit == 0 means no limit should be imposed... however, scoping the Semaphore
+            // permit is tricky, so as a workaround, we'll add a ridiculous number of permits to
+            // the semaphore (1,152,921,504,606,846,975 to be exact) and call that 'unlimited'
+            SCAN_LIMITER.add_permits(usize::MAX >> 4);
+        }
     }
 
+    // When acquire is called and the semaphore has remaining permits, the function immediately
+    // returns a permit. However, if no remaining permits are available, acquire (asynchronously)
+    // waits until an outstanding permit is dropped. At this point, the freed permit is assigned
+    // to the caller.
+    let permit = SCAN_LIMITER.acquire().await;
+
     // Arc clones to be passed around to the various scans
     let wildcard_bar = progress_bar.clone();
     let heuristics_file_clone = tx_file.clone();
@@ -599,7 +617,7 @@ pub async fn scan_url(
         .for_each_concurrent(CONFIGURATION.threads, |(resp, bar)| async move {
             match resp.await {
                 Ok(_) => {
-                    bar.inc(1);
+                    bar.inc((CONFIGURATION.extensions.len() + 1) as u64);
                 }
                 Err(e) => {
                     log::error!("error awaiting a response: {}", e);
@@ -612,6 +630,9 @@ pub async fn scan_url(
     producers.await;
     log::trace!("done awaiting scan producers");
 
+    // drop the current permit so the semaphore will allow another scan to proceed
+    drop(permit);
+
     progress_bar.finish();
 
     // manually drop tx in order for the rx task's while loops to eval to false

tests/test_banner.rs

@@ -563,3 +563,31 @@ fn banner_prints_extract_links() -> Result<(), Box<dyn std::error::Error>> {
         );
     Ok(())
 }
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + scan-limit
+fn banner_prints_scan_limit() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-L")
+        .arg("4")
+        .assert()
+        .failure()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Concurrent Scan Limit"))
+                .and(predicate::str::contains("│ 4"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}