Merge pull request #96 from epi052/FEATURE-limit-number-of-scans

Added ability to limit number of scans

.github/workflows/build.yml

@@ -41,6 +41,9 @@ jobs:
           use-cross: true
           command: build
           args: --release --target=${{ matrix.target }}
+      - name: Strip symbols from binary
+        run: |
+          strip -s ${{ matrix.path }}
       - name: Build tar.gz for homebrew installs
         if: matrix.type == 'ubuntu-x64'
         run: |
@@ -83,6 +86,9 @@ jobs:
           use-cross: true
           command: build
           args: --release --target=x86_64-apple-darwin
+      - name: Strip symbols from binary
+        run: |
+          strip -u -r target/x86_64-apple-darwin/release/feroxbuster
       - name: Build tar.gz for homebrew installs
         run: |
           tar czf x86_64-macos-feroxbuster.tar.gz -C target/x86_64-apple-darwin/release feroxbuster

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "feroxbuster"
-version = "1.1.1"
+version = "1.2.0"
 authors = ["Ben 'epi' Risher <epibar052@gmail.com>"]
 license = "MIT"
 edition = "2018"

README.md

@@ -79,7 +79,10 @@ This attack is also known as Predictable Resource Location, File Enumeration, Di
     - [Proxy traffic through Burp](#proxy-traffic-through-burp)
     - [Proxy traffic through a SOCKS proxy](#proxy-traffic-through-a-socks-proxy)
     - [Pass auth token via query parameter](#pass-auth-token-via-query-parameter)
+    - [Limit Total Number of Concurrent Scans (new in `v1.2.0`)](#limit-total-number-of-concurrent-scans-new-in-v120)
 - [Comparison w/ Similar Tools](#-comparison-w-similar-tools)
+- [Common Problems/Issues (FAQ)](#-common-problemsissues-faq)
+    - [No file descriptors available](#no-file-descriptors-available)
 
 ## 💿 Installation
 
@@ -235,6 +238,7 @@ Configuration begins with with the following built-in default values baked into
 - wordlist: `/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt`
 - threads: `50`
 - verbosity: `0` (no logging enabled)
+- scan_limit: `0` (no limit imposed on concurrent scans)
 - statuscodes: `200 204 301 302 307 308 401 403 405`
 - useragent: `feroxbuster/VERSION`
 - recursion depth: `4`
@@ -291,6 +295,7 @@ A pre-made configuration file with examples of all available settings can be fou
 # timeout = 5
 # proxy = "http://127.0.0.1:8080"
 # verbosity = 1
+# scan_limit = 6
 # quiet = true
 # output = "/targets/ellingson_mineral_company/gibson.txt"
 # useragent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0"
@@ -348,6 +353,7 @@ OPTIONS:
     -o, --output <FILE>                     Output file to write results to (default: stdout)
     -p, --proxy <PROXY>                     Proxy to use for requests (ex: http(s)://host:port, socks5://host:port)
     -Q, --query <QUERY>...                  Specify URL query parameters (ex: -Q token=stuff -Q secret=key)
+    -L, --scan-limit <SCAN_LIMIT>           Limit total number of concurrent scans (default: 7)
     -S, --sizefilter <SIZE>...              Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)
     -s, --statuscodes <STATUS_CODE>...      Status Codes of interest (default: 200 204 301 302 307 308 401 403 405)
     -t, --threads <THREADS>                 Number of concurrent threads (default: 50)
@@ -421,12 +427,23 @@ cat targets | ./feroxbuster --stdin --quiet -s 200 301 302 --redirects -x js | f
 ./feroxbuster -u http://127.1 --proxy socks5://127.0.0.1:9050
 ```
 
-### Pass auth token via query parameter
+### Pass auth token via query parameter 
 
 ```
 ./feroxbuster -u http://127.1 --query token=0123456789ABCDEF
 ```
 
+### Limit Total Number of Concurrent Scans (new in `v1.2.0`)
+
+Limit the number of scans permitted to run at any given time.  Recursion will still identify new directories, but newly
+discovered directories can only begin scanning when the total number of active scans drops below the value passed to 
+`--scan-limit`.
+
+```
+./feroxbuster -u http://127.1 --scan-limit 2
+```
+
+![limit-demo](img/limit-demo.gif)
 
 ## 🧐 Comparison w/ Similar Tools
 
@@ -470,3 +487,74 @@ came across rustbuster when I was naming my tool (😢). I don't have any experi
 be able to do POST requests with an HTTP body, has SOCKS support, and has an 8.3 shortname scanner (in addition to vhost
 dns, directory, etc...).  In short, it definitely looks interesting and may be what you're looking for as it has some 
 capability I haven't seen in similar tools.  
+
+## 🤯 Common Problems/Issues (FAQ)
+
+### No file descriptors available
+
+Why do I get a bunch of `No file descriptors available (os error 24)` errors?
+
+---
+
+There are a few potential causes of this error.  The simplest is that your operating system sets an open file limit that is aggressively low.  Through personal testing, I've found that `4096` is a reasonable open file limit (this will vary based on your exact setup).
+
+There are quite a few options to solve this particular problem, of which a handful are shown below.  
+
+#### Increase the Number of Open Files
+
+We'll start by increasing the number of open files the OS allows. On my Kali install, the default was `1024`, and I know some MacOS installs use `256` 😕.
+
+##### Edit `/etc/security/limits.conf`
+
+One option to up the limit is to edit `/etc/security/limits.conf` so that it includes the two lines below.  
+
+- `*` represents all users
+- `hard` and `soft` indicate the hard and soft limits for the OS 
+- `nofile` is the number of open files option. 
+
+```
+/etc/security/limits.conf
+-------------------------
+...
+*        soft nofile 4096
+*        hard nofile 8192
+...
+```
+
+##### Use `ulimit` directly
+
+A faster option, that is **not** persistent, is to simply use the `ulimit` command to change the setting.
+
+```
+ulimit -n 4096
+```
+
+#### Additional Tweaks (may not be needed)
+
+If you still find yourself hitting the file limit with the above changes, there are a few additional tweaks that may help.  
+
+> This section was shamelessly stolen from this [stackoverflow answer](https://stackoverflow.com/a/3923785).  More information is included in that post and is recommended reading if you end up needing to use this section.
+
+✨ Special thanks to HTB user [@sparkla](https://www.hackthebox.eu/home/users/profile/221599) for their help with identifying these additional tweaks ✨
+
+##### Increase the ephemeral port range, and decrease the tcp_fin_timeout.
+
+The ephermal port range defines the maximum number of outbound sockets a host can create from a particular I.P. address. The fin_timeout defines the minimum time these sockets will stay in TIME_WAIT state (unusable after being used once). Usual system defaults are
+
+- `net.ipv4.ip_local_port_range = 32768   61000`
+- `net.ipv4.tcp_fin_timeout = 60`
+
+This basically means your system cannot consistently guarantee more than `(61000 - 32768) / 60 = 470` sockets per second.
+
+```
+sudo sysctl net.ipv4.ip_local_port_range="15000 61000"
+sudo sysctl net.ipv4.tcp_fin_timeout=30
+```
+
+##### Allow socket reuse while in a `TIME_WAIT` status
+
+This allows fast cycling of sockets in time_wait state and re-using them. Make sure to read post [Coping with the TCP TIME-WAIT](https://vincent.bernat.ch/en/blog/2014-tcp-time-wait-state-linux) from Vincent Bernat to understand the implications.
+
+```
+sudo sysctl net.ipv4.tcp_tw_reuse=1 
+```

ferox-config.toml.example

@@ -13,6 +13,7 @@
 # timeout = 5
 # proxy = "http://127.0.0.1:8080"
 # verbosity = 1
+# scan_limit = 6
 # quiet = true
 # output = "/targets/ellingson_mineral_company/gibson.txt"
 # useragent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0"

src/banner.rs

@@ -389,6 +389,15 @@ by Ben "epi" Risher {}                  ver: {}"#,
         .unwrap_or_default(); // 🚫
     }
 
+    if CONFIGURATION.scan_limit > 0 {
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!("\u{1f9a5}", "Concurrent Scan Limit", config.scan_limit)
+        )
+        .unwrap_or_default(); // 🦥
+    }
+
     if matches!(status, UpdateStatus::OutOfDate) {
         writeln!(
             &mut writer,

src/config.rs

@@ -123,6 +123,10 @@ pub struct Configuration {
     #[serde(default = "depth")]
     pub depth: usize,
 
+    /// Number of concurrent scans permitted; a limit of 0 means no limit is imposed
+    #[serde(default)]
+    pub scan_limit: usize,
+
     /// Filter out messages of a particular size
     #[serde(default)]
     pub sizefilters: Vec<u64>,
@@ -184,6 +188,7 @@ impl Default for Configuration {
             quiet: false,
             stdin: false,
             verbosity: 0,
+            scan_limit: 0,
             addslash: false,
             insecure: false,
             redirects: false,
@@ -232,6 +237,7 @@ impl Configuration {
     /// - **stdin**: `false`
     /// - **dontfilter**: `false` (auto filter wildcard responses)
     /// - **depth**: `4` (maximum recursion depth)
+    /// - **scan_limit**: `0` (no limit on concurrent scans imposed)
     ///
     /// After which, any values defined in a
     /// [ferox-config.toml](constant.DEFAULT_CONFIG_NAME.html) config file will override the
@@ -316,6 +322,12 @@ impl Configuration {
             config.depth = depth;
         }
 
+        if args.value_of("scan_limit").is_some() {
+            let scan_limit =
+                value_t!(args.value_of("scan_limit"), usize).unwrap_or_else(|e| e.exit());
+            config.scan_limit = scan_limit;
+        }
+
         if args.value_of("wordlist").is_some() {
             config.wordlist = String::from(args.value_of("wordlist").unwrap());
         }
@@ -534,6 +546,7 @@ impl Configuration {
         settings.depth = settings_to_merge.depth;
         settings.sizefilters = settings_to_merge.sizefilters;
         settings.dontfilter = settings_to_merge.dontfilter;
+        settings.scan_limit = settings_to_merge.scan_limit;
     }
 
     /// If present, read in `DEFAULT_CONFIG_NAME` and deserialize the specified values
@@ -575,6 +588,7 @@ mod tests {
             proxy = "http://127.0.0.1:8080"
             quiet = true
             verbosity = 1
+            scan_limit = 6
             output = "/some/otherpath"
             redirects = true
             insecure = true
@@ -608,6 +622,7 @@ mod tests {
         assert_eq!(config.depth, depth());
         assert_eq!(config.timeout, timeout());
         assert_eq!(config.verbosity, 0);
+        assert_eq!(config.scan_limit, 0);
         assert_eq!(config.quiet, false);
         assert_eq!(config.dontfilter, false);
         assert_eq!(config.norecursion, false);
@@ -650,6 +665,13 @@ mod tests {
         assert_eq!(config.depth, 1);
     }
 
+    #[test]
+    /// parse the test config and see that the value parsed is correct
+    fn config_reads_scan_limit() {
+        let config = setup_config_test();
+        assert_eq!(config.scan_limit, 6);
+    }
+
     #[test]
     /// parse the test config and see that the value parsed is correct
     fn config_reads_timeout() {

src/parser.rs

@@ -202,7 +202,14 @@ pub fn initialize() -> App<'static, 'static> {
                 .takes_value(false)
                 .help("Extract links from response body (html, javascript, etc...); make new requests based on findings (default: false)")
         )
-
+        .arg(
+            Arg::with_name("scan_limit")
+                .short("L")
+                .long("scan-limit")
+                .value_name("SCAN_LIMIT")
+                .takes_value(true)
+                .help("Limit total number of concurrent scans (default: 0, i.e. no limit)")
+        )
         .after_help(r#"NOTE:
     Options that take multiple values are very flexible.  Consider the following ways of specifying
     extensions:

src/scanner.rs

@@ -13,6 +13,7 @@ use std::ops::Deref;
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::{Arc, RwLock};
 use tokio::sync::mpsc::{self, UnboundedReceiver, UnboundedSender};
+use tokio::sync::Semaphore;
 use tokio::task::JoinHandle;
 
 /// Single atomic number that gets incremented once, used to track first scan vs. all others
@@ -24,6 +25,9 @@ lazy_static! {
 
     /// Vector of WildcardFilters that have been ID'd through heuristics
     static ref WILDCARD_FILTERS: Arc<RwLock<Vec<Arc<WildcardFilter>>>> = Arc::new(RwLock::new(Vec::<Arc<WildcardFilter>>::new()));
+
+    /// Bounded semaphore used as a barrier to limit concurrent scans
+    static ref SCAN_LIMITER: Semaphore = Semaphore::new(CONFIGURATION.scan_limit);
 }
 
 /// Adds the given url to `SCANNED_URLS`
@@ -120,6 +124,7 @@ fn spawn_recursion_handler(
 
     let boxed_future = async move {
         let mut scans = vec![];
+
         while let Some(resp) = recursion_channel.recv().await {
             let unknown = add_url_to_list_of_scanned_urls(&resp, &SCANNED_URLS);
 
@@ -555,8 +560,21 @@ pub async fn scan_url(
         // this protection around join also allows us to add the first scanned url to SCANNED_URLS
         // from within the scan_url function instead of the recursion handler
         add_url_to_list_of_scanned_urls(&target_url, &SCANNED_URLS);
+
+        if CONFIGURATION.scan_limit == 0 {
+            // scan_limit == 0 means no limit should be imposed... however, scoping the Semaphore
+            // permit is tricky, so as a workaround, we'll add a ridiculous number of permits to
+            // the semaphore (1,152,921,504,606,846,975 to be exact) and call that 'unlimited'
+            SCAN_LIMITER.add_permits(usize::MAX >> 4);
+        }
     }
 
+    // When acquire is called and the semaphore has remaining permits, the function immediately
+    // returns a permit. However, if no remaining permits are available, acquire (asynchronously)
+    // waits until an outstanding permit is dropped. At this point, the freed permit is assigned
+    // to the caller.
+    let permit = SCAN_LIMITER.acquire().await;
+
     // Arc clones to be passed around to the various scans
     let wildcard_bar = progress_bar.clone();
     let heuristics_file_clone = tx_file.clone();
@@ -599,7 +617,7 @@ pub async fn scan_url(
         .for_each_concurrent(CONFIGURATION.threads, |(resp, bar)| async move {
             match resp.await {
                 Ok(_) => {
-                    bar.inc(1);
+                    bar.inc((CONFIGURATION.extensions.len() + 1) as u64);
                 }
                 Err(e) => {
                     log::error!("error awaiting a response: {}", e);
@@ -612,6 +630,9 @@ pub async fn scan_url(
     producers.await;
     log::trace!("done awaiting scan producers");
 
+    // drop the current permit so the semaphore will allow another scan to proceed
+    drop(permit);
+
     progress_bar.finish();
 
     // manually drop tx in order for the rx task's while loops to eval to false

tests/test_banner.rs

@@ -563,3 +563,31 @@ fn banner_prints_extract_links() -> Result<(), Box<dyn std::error::Error>> {
         );
     Ok(())
 }
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + scan-limit
+fn banner_prints_scan_limit() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-L")
+        .arg("4")
+        .assert()
+        .failure()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Concurrent Scan Limit"))
+                .and(predicate::str::contains("│ 4"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}