Merge pull request #120 from epi052/fix-directory-extraction-bug

Fix directory extraction bug

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "feroxbuster"
-version = "1.5.1"
+version = "1.5.3"
 authors = ["Ben 'epi' Risher <epibar052@gmail.com>"]
 license = "MIT"
 edition = "2018"
@@ -33,6 +33,7 @@ openssl = { version = "0.10", features = ["vendored"] }
 dirs = "3.0"
 regex = "1"
 crossterm = "0.18"
+rlimit = "0.5"
 
 [dev-dependencies]
 tempfile = "3.1"

src/lib.rs

@@ -42,6 +42,9 @@ pub type FeroxChannel<T> = (UnboundedSender<T>, UnboundedReceiver<T>);
 /// Version pulled from Cargo.toml at compile time
 pub const VERSION: &str = env!("CARGO_PKG_VERSION");
 
+/// Maximum number of file descriptors that can be opened during a scan
+pub const DEFAULT_OPEN_FILE_LIMIT: usize = 8192;
+
 /// Default wordlist to use when `-w|--wordlist` isn't specified and not `wordlist` isn't set
 /// in a [ferox-config.toml](constant.DEFAULT_CONFIG_NAME.html) config file.
 ///

src/main.rs

@@ -7,6 +7,8 @@ use feroxbuster::{
     utils::{ferox_print, get_current_depth, module_colorizer, status_colorizer},
     FeroxError, FeroxResponse, FeroxResult, SLEEP_DURATION, VERSION,
 };
+#[cfg(not(target_os = "windows"))]
+use feroxbuster::{utils::set_open_file_limit, DEFAULT_OPEN_FILE_LIMIT};
 use futures::StreamExt;
 use std::{
     collections::HashSet,
@@ -294,6 +296,10 @@ fn main() {
     // setup logging based on the number of -v's used
     logger::initialize(CONFIGURATION.verbosity);
 
+    // this function uses rlimit, which is not supported on windows
+    #[cfg(not(target_os = "windows"))]
+    set_open_file_limit(DEFAULT_OPEN_FILE_LIMIT);
+
     if let Ok(mut runtime) = tokio::runtime::Runtime::new() {
         let future = wrapped_main();
         runtime.block_on(future);

src/scanner.rs

@@ -130,17 +130,9 @@ fn add_url_to_list_of_scanned_urls(resp: &str, scanned_urls: &RwLock<HashSet<Str
     match scanned_urls.write() {
         // check new url against what's already been scanned
         Ok(mut urls) => {
-            let normalized_url = if resp.ends_with('/') {
-                // append a / to the list of 'seen' urls, this is to prevent the case where
-                // 3xx and 2xx duplicate eachother
-                resp.to_string()
-            } else {
-                format!("{}/", resp)
-            };
-
             // If the set did not contain resp, true is returned.
             // If the set did contain resp, false is returned.
-            let response = urls.insert(normalized_url);
+            let response = urls.insert(resp.to_string());
 
             log::trace!("exit: add_url_to_list_of_scanned_urls -> {}", response);
             response
@@ -855,7 +847,7 @@ mod tests {
         assert_eq!(
             urls.write()
                 .unwrap()
-                .insert("http://unknown_url/".to_string()),
+                .insert("http://unknown_url".to_string()),
             true
         );
 

src/utils.rs

@@ -3,6 +3,8 @@ use console::{strip_ansi_codes, style, user_attended};
 use indicatif::ProgressBar;
 use reqwest::Url;
 use reqwest::{Client, Response};
+#[cfg(not(target_os = "windows"))]
+use rlimit::{getrlimit, setrlimit, Resource, Rlim};
 use std::convert::TryInto;
 
 /// Helper function that determines the current depth of a given url
@@ -259,10 +261,89 @@ pub async fn make_request(client: &Client, url: &Url) -> FeroxResult<Response> {
     }
 }
 
+/// Attempts to set the soft limit for the RLIMIT_NOFILE resource
+///
+/// RLIMIT_NOFILE is the maximum number of file descriptors that can be opened by this process
+///
+/// The soft limit is the value that the kernel enforces for the corresponding resource.
+/// The hard limit acts as a ceiling for the soft limit: an unprivileged process may set only its
+/// soft limit to a value in the range from 0 up to the hard limit, and (irreversibly) lower its
+/// hard limit.
+///
+/// A child process created via fork(2) inherits its parent's resource limits. Resource limits are
+/// per-process attributes that are shared by all of the threads in a process.
+///
+/// Based on the above information, no attempt is made to restore the limit to its pre-scan value
+/// as the adjustment made here is only valid for the scan itself (and any child processes, of which
+/// there are none).
+#[cfg(not(target_os = "windows"))]
+pub fn set_open_file_limit(limit: usize) -> bool {
+    log::trace!("enter: set_open_file_limit");
+
+    if let Ok((soft, hard)) = getrlimit(Resource::NOFILE) {
+        if hard.as_usize() > limit {
+            // our default open file limit is less than the current hard limit, this means we can
+            // set the soft limit to our default
+            let new_soft_limit = Rlim::from_usize(limit);
+
+            if setrlimit(Resource::NOFILE, new_soft_limit, hard).is_ok() {
+                log::debug!("set open file descriptor limit to {}", limit);
+
+                log::trace!("exit: set_open_file_limit -> {}", true);
+                return true;
+            }
+        } else if soft != hard {
+            // hard limit is lower than our default, the next best option is to set the soft limit as
+            // high as the hard limit will allow
+            if setrlimit(Resource::NOFILE, hard, hard).is_ok() {
+                log::debug!("set open file descriptor limit to {}", limit);
+
+                log::trace!("exit: set_open_file_limit -> {}", true);
+                return true;
+            }
+        }
+    }
+
+    // failed to set a new limit, as limit adjustments are a 'nice to have', we'll just log
+    // and move along
+    log::warn!("could not set open file descriptor limit to {}", limit);
+
+    log::trace!("exit: set_open_file_limit -> {}", false);
+    false
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
 
+    #[test]
+    /// set_open_file_limit with a low requested limit succeeds
+    fn utils_set_open_file_limit_with_low_requested_limit() {
+        let (_, hard) = getrlimit(Resource::NOFILE).unwrap();
+        let lower_limit = hard.as_usize() - 1;
+        assert!(set_open_file_limit(lower_limit));
+    }
+
+    #[test]
+    /// set_open_file_limit with a high requested limit succeeds
+    fn utils_set_open_file_limit_with_high_requested_limit() {
+        let (_, hard) = getrlimit(Resource::NOFILE).unwrap();
+        let higher_limit = hard.as_usize() + 1;
+        // calculate a new soft to ensure soft != hard and hit that logic branch
+        let new_soft = Rlim::from_usize(hard.as_usize() - 1);
+        setrlimit(Resource::NOFILE, new_soft, hard).unwrap();
+        assert!(set_open_file_limit(higher_limit));
+    }
+
+    #[test]
+    /// set_open_file_limit should fail when hard == soft
+    fn utils_set_open_file_limit_with_fails_when_both_limits_are_equal() {
+        let (_, hard) = getrlimit(Resource::NOFILE).unwrap();
+        // calculate a new soft to ensure soft == hard and hit the failure logic branch
+        setrlimit(Resource::NOFILE, hard, hard).unwrap();
+        assert!(!set_open_file_limit(hard.as_usize())); // returns false
+    }
+
     #[test]
     /// base url returns 1
     fn get_current_depth_base_url_returns_1() {