Merge pull request #96 from epi052/FEATURE-limit-number-of-scans

Added ability to limit number of scans

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "feroxbuster"
-version = "1.1.1"
+version = "1.2.0"
 authors = ["Ben 'epi' Risher <epibar052@gmail.com>"]
 license = "MIT"
 edition = "2018"

README.md

@@ -79,6 +79,7 @@ This attack is also known as Predictable Resource Location, File Enumeration, Di
     - [Proxy traffic through Burp](#proxy-traffic-through-burp)
     - [Proxy traffic through a SOCKS proxy](#proxy-traffic-through-a-socks-proxy)
     - [Pass auth token via query parameter](#pass-auth-token-via-query-parameter)
+    - [Limit Total Number of Concurrent Scans (new in `v1.2.0`)](#limit-total-number-of-concurrent-scans-new-in-v120)
 - [Comparison w/ Similar Tools](#-comparison-w-similar-tools)
 - [Common Problems/Issues (FAQ)](#-common-problemsissues-faq)
     - [No file descriptors available](#no-file-descriptors-available)
@@ -237,6 +238,7 @@ Configuration begins with with the following built-in default values baked into
 - wordlist: `/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt`
 - threads: `50`
 - verbosity: `0` (no logging enabled)
+- scan_limit: `0` (no limit imposed on concurrent scans)
 - statuscodes: `200 204 301 302 307 308 401 403 405`
 - useragent: `feroxbuster/VERSION`
 - recursion depth: `4`
@@ -293,6 +295,7 @@ A pre-made configuration file with examples of all available settings can be fou
 # timeout = 5
 # proxy = "http://127.0.0.1:8080"
 # verbosity = 1
+# scan_limit = 6
 # quiet = true
 # output = "/targets/ellingson_mineral_company/gibson.txt"
 # useragent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0"
@@ -350,6 +353,7 @@ OPTIONS:
     -o, --output <FILE>                     Output file to write results to (default: stdout)
     -p, --proxy <PROXY>                     Proxy to use for requests (ex: http(s)://host:port, socks5://host:port)
     -Q, --query <QUERY>...                  Specify URL query parameters (ex: -Q token=stuff -Q secret=key)
+    -L, --scan-limit <SCAN_LIMIT>           Limit total number of concurrent scans (default: 7)
     -S, --sizefilter <SIZE>...              Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)
     -s, --statuscodes <STATUS_CODE>...      Status Codes of interest (default: 200 204 301 302 307 308 401 403 405)
     -t, --threads <THREADS>                 Number of concurrent threads (default: 50)
@@ -423,12 +427,23 @@ cat targets | ./feroxbuster --stdin --quiet -s 200 301 302 --redirects -x js | f
 ./feroxbuster -u http://127.1 --proxy socks5://127.0.0.1:9050
 ```
 
-### Pass auth token via query parameter
+### Pass auth token via query parameter 
 
 ```
 ./feroxbuster -u http://127.1 --query token=0123456789ABCDEF
 ```
 
+### Limit Total Number of Concurrent Scans (new in `v1.2.0`)
+
+Limit the number of scans permitted to run at any given time.  Recursion will still identify new directories, but newly
+discovered directories can only begin scanning when the total number of active scans drops below the value passed to 
+`--scan-limit`.
+
+```
+./feroxbuster -u http://127.1 --scan-limit 2
+```
+
+![limit-demo](img/limit-demo.gif)
 
 ## 🧐 Comparison w/ Similar Tools
 

ferox-config.toml.example

@@ -13,6 +13,7 @@
 # timeout = 5
 # proxy = "http://127.0.0.1:8080"
 # verbosity = 1
+# scan_limit = 6
 # quiet = true
 # output = "/targets/ellingson_mineral_company/gibson.txt"
 # useragent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0"

src/banner.rs

@@ -389,6 +389,15 @@ by Ben "epi" Risher {}                  ver: {}"#,
         .unwrap_or_default(); // 🚫
     }
 
+    if CONFIGURATION.scan_limit > 0 {
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!("\u{1f9a5}", "Concurrent Scan Limit", config.scan_limit)
+        )
+        .unwrap_or_default(); // 🦥
+    }
+
     if matches!(status, UpdateStatus::OutOfDate) {
         writeln!(
             &mut writer,

src/config.rs

@@ -123,6 +123,10 @@ pub struct Configuration {
     #[serde(default = "depth")]
     pub depth: usize,
 
+    /// Number of concurrent scans permitted; a limit of 0 means no limit is imposed
+    #[serde(default)]
+    pub scan_limit: usize,
+
     /// Filter out messages of a particular size
     #[serde(default)]
     pub sizefilters: Vec<u64>,
@@ -184,6 +188,7 @@ impl Default for Configuration {
             quiet: false,
             stdin: false,
             verbosity: 0,
+            scan_limit: 0,
             addslash: false,
             insecure: false,
             redirects: false,
@@ -232,6 +237,7 @@ impl Configuration {
     /// - **stdin**: `false`
     /// - **dontfilter**: `false` (auto filter wildcard responses)
     /// - **depth**: `4` (maximum recursion depth)
+    /// - **scan_limit**: `0` (no limit on concurrent scans imposed)
     ///
     /// After which, any values defined in a
     /// [ferox-config.toml](constant.DEFAULT_CONFIG_NAME.html) config file will override the
@@ -316,6 +322,12 @@ impl Configuration {
             config.depth = depth;
         }
 
+        if args.value_of("scan_limit").is_some() {
+            let scan_limit =
+                value_t!(args.value_of("scan_limit"), usize).unwrap_or_else(|e| e.exit());
+            config.scan_limit = scan_limit;
+        }
+
         if args.value_of("wordlist").is_some() {
             config.wordlist = String::from(args.value_of("wordlist").unwrap());
         }
@@ -534,6 +546,7 @@ impl Configuration {
         settings.depth = settings_to_merge.depth;
         settings.sizefilters = settings_to_merge.sizefilters;
         settings.dontfilter = settings_to_merge.dontfilter;
+        settings.scan_limit = settings_to_merge.scan_limit;
     }
 
     /// If present, read in `DEFAULT_CONFIG_NAME` and deserialize the specified values
@@ -575,6 +588,7 @@ mod tests {
             proxy = "http://127.0.0.1:8080"
             quiet = true
             verbosity = 1
+            scan_limit = 6
             output = "/some/otherpath"
             redirects = true
             insecure = true
@@ -608,6 +622,7 @@ mod tests {
         assert_eq!(config.depth, depth());
         assert_eq!(config.timeout, timeout());
         assert_eq!(config.verbosity, 0);
+        assert_eq!(config.scan_limit, 0);
         assert_eq!(config.quiet, false);
         assert_eq!(config.dontfilter, false);
         assert_eq!(config.norecursion, false);
@@ -650,6 +665,13 @@ mod tests {
         assert_eq!(config.depth, 1);
     }
 
+    #[test]
+    /// parse the test config and see that the value parsed is correct
+    fn config_reads_scan_limit() {
+        let config = setup_config_test();
+        assert_eq!(config.scan_limit, 6);
+    }
+
     #[test]
     /// parse the test config and see that the value parsed is correct
     fn config_reads_timeout() {

src/parser.rs

@@ -202,7 +202,14 @@ pub fn initialize() -> App<'static, 'static> {
                 .takes_value(false)
                 .help("Extract links from response body (html, javascript, etc...); make new requests based on findings (default: false)")
         )
-
+        .arg(
+            Arg::with_name("scan_limit")
+                .short("L")
+                .long("scan-limit")
+                .value_name("SCAN_LIMIT")
+                .takes_value(true)
+                .help("Limit total number of concurrent scans (default: 0, i.e. no limit)")
+        )
         .after_help(r#"NOTE:
     Options that take multiple values are very flexible.  Consider the following ways of specifying
     extensions:

src/scanner.rs

@@ -13,6 +13,7 @@ use std::ops::Deref;
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::{Arc, RwLock};
 use tokio::sync::mpsc::{self, UnboundedReceiver, UnboundedSender};
+use tokio::sync::Semaphore;
 use tokio::task::JoinHandle;
 
 /// Single atomic number that gets incremented once, used to track first scan vs. all others
@@ -24,6 +25,9 @@ lazy_static! {
 
     /// Vector of WildcardFilters that have been ID'd through heuristics
     static ref WILDCARD_FILTERS: Arc<RwLock<Vec<Arc<WildcardFilter>>>> = Arc::new(RwLock::new(Vec::<Arc<WildcardFilter>>::new()));
+
+    /// Bounded semaphore used as a barrier to limit concurrent scans
+    static ref SCAN_LIMITER: Semaphore = Semaphore::new(CONFIGURATION.scan_limit);
 }
 
 /// Adds the given url to `SCANNED_URLS`
@@ -120,6 +124,7 @@ fn spawn_recursion_handler(
 
     let boxed_future = async move {
         let mut scans = vec![];
+
         while let Some(resp) = recursion_channel.recv().await {
             let unknown = add_url_to_list_of_scanned_urls(&resp, &SCANNED_URLS);
 
@@ -555,8 +560,21 @@ pub async fn scan_url(
         // this protection around join also allows us to add the first scanned url to SCANNED_URLS
         // from within the scan_url function instead of the recursion handler
         add_url_to_list_of_scanned_urls(&target_url, &SCANNED_URLS);
+
+        if CONFIGURATION.scan_limit == 0 {
+            // scan_limit == 0 means no limit should be imposed... however, scoping the Semaphore
+            // permit is tricky, so as a workaround, we'll add a ridiculous number of permits to
+            // the semaphore (1,152,921,504,606,846,975 to be exact) and call that 'unlimited'
+            SCAN_LIMITER.add_permits(usize::MAX >> 4);
+        }
     }
 
+    // When acquire is called and the semaphore has remaining permits, the function immediately
+    // returns a permit. However, if no remaining permits are available, acquire (asynchronously)
+    // waits until an outstanding permit is dropped. At this point, the freed permit is assigned
+    // to the caller.
+    let permit = SCAN_LIMITER.acquire().await;
+
     // Arc clones to be passed around to the various scans
     let wildcard_bar = progress_bar.clone();
     let heuristics_file_clone = tx_file.clone();
@@ -612,6 +630,9 @@ pub async fn scan_url(
     producers.await;
     log::trace!("done awaiting scan producers");
 
+    // drop the current permit so the semaphore will allow another scan to proceed
+    drop(permit);
+
     progress_bar.finish();
 
     // manually drop tx in order for the rx task's while loops to eval to false

tests/test_banner.rs

@@ -563,3 +563,31 @@ fn banner_prints_extract_links() -> Result<(), Box<dyn std::error::Error>> {
         );
     Ok(())
 }
+
+#[test]
+/// test allows non-existent wordlist to trigger the banner printing to stderr
+/// expect to see all mandatory prints + scan-limit
+fn banner_prints_scan_limit() -> Result<(), Box<dyn std::error::Error>> {
+    Command::cargo_bin("feroxbuster")
+        .unwrap()
+        .arg("--url")
+        .arg("http://localhost")
+        .arg("-L")
+        .arg("4")
+        .assert()
+        .failure()
+        .stderr(
+            predicate::str::contains("─┬─")
+                .and(predicate::str::contains("Target Url"))
+                .and(predicate::str::contains("http://localhost"))
+                .and(predicate::str::contains("Threads"))
+                .and(predicate::str::contains("Wordlist"))
+                .and(predicate::str::contains("Status Codes"))
+                .and(predicate::str::contains("Timeout (secs)"))
+                .and(predicate::str::contains("User-Agent"))
+                .and(predicate::str::contains("Concurrent Scan Limit"))
+                .and(predicate::str::contains("│ 4"))
+                .and(predicate::str::contains("─┴─")),
+        );
+    Ok(())
+}