From 0dbc3bee23644b7bca00ebcd0237a2d895819e4a Mon Sep 17 00:00:00 2001 From: epi Date: Sat, 7 Nov 2020 15:05:07 -0600 Subject: [PATCH] added auto-adjustment of open file limit --- Cargo.toml | 3 +- src/lib.rs | 3 ++ src/main.rs | 10 +++++-- src/utils.rs | 79 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 92 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 8d802f1..d542068 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "feroxbuster" -version = "1.5.1" +version = "1.5.2" authors = ["Ben 'epi' Risher "] license = "MIT" edition = "2018" @@ -33,6 +33,7 @@ openssl = { version = "0.10", features = ["vendored"] } dirs = "3.0" regex = "1" crossterm = "0.18" +rlimit = "0.5" [dev-dependencies] tempfile = "3.1" diff --git a/src/lib.rs b/src/lib.rs index ccad808..8051d86 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -42,6 +42,9 @@ pub type FeroxChannel = (UnboundedSender, UnboundedReceiver); /// Version pulled from Cargo.toml at compile time pub const VERSION: &str = env!("CARGO_PKG_VERSION"); +/// Maximum number of file descriptors that can be opened during a scan +pub const DEFAULT_OPEN_FILE_LIMIT: usize = 8192; + /// Default wordlist to use when `-w|--wordlist` isn't specified and not `wordlist` isn't set /// in a [ferox-config.toml](constant.DEFAULT_CONFIG_NAME.html) config file. /// diff --git a/src/main.rs b/src/main.rs index de77116..9c1c529 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,8 +4,10 @@ use feroxbuster::{ config::{CONFIGURATION, PROGRESS_BAR, PROGRESS_PRINTER}, heuristics, logger, reporter, scanner::{scan_url, PAUSE_SCAN}, - utils::{ferox_print, get_current_depth, module_colorizer, status_colorizer}, - FeroxError, FeroxResponse, FeroxResult, SLEEP_DURATION, VERSION, + utils::{ + ferox_print, get_current_depth, module_colorizer, set_open_file_limit, status_colorizer, + }, + FeroxError, FeroxResponse, FeroxResult, DEFAULT_OPEN_FILE_LIMIT, SLEEP_DURATION, VERSION, }; use futures::StreamExt; use std::{ @@ -294,6 +296,10 @@ fn main() { // setup logging based on the number of -v's used logger::initialize(CONFIGURATION.verbosity); + // this function uses rlimit, which is not supported on windows + #[cfg(not(target_os = "windows"))] + set_open_file_limit(DEFAULT_OPEN_FILE_LIMIT); + if let Ok(mut runtime) = tokio::runtime::Runtime::new() { let future = wrapped_main(); runtime.block_on(future); diff --git a/src/utils.rs b/src/utils.rs index 9d09436..8ef8438 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -3,6 +3,7 @@ use console::{strip_ansi_codes, style, user_attended}; use indicatif::ProgressBar; use reqwest::Url; use reqwest::{Client, Response}; +use rlimit::{getrlimit, setrlimit, Resource, Rlim}; use std::convert::TryInto; /// Helper function that determines the current depth of a given url @@ -259,10 +260,88 @@ pub async fn make_request(client: &Client, url: &Url) -> FeroxResult { } } +/// Attempts to set the soft limit for the RLIMIT_NOFILE resource +/// +/// RLIMIT_NOFILE is the maximum number of file descriptors that can be opened by this process +/// +/// The soft limit is the value that the kernel enforces for the corresponding resource. +/// The hard limit acts as a ceiling for the soft limit: an unprivileged process may set only its +/// soft limit to a value in the range from 0 up to the hard limit, and (irreversibly) lower its +/// hard limit. +/// +/// A child process created via fork(2) inherits its parent's resource limits. Resource limits are +/// per-process attributes that are shared by all of the threads in a process. +/// +/// Based on the above information, no attempt is made to restore the limit to its pre-scan value +/// as the adjustment made here is only valid for the scan itself (and any child processes, of which +/// there are none). +pub fn set_open_file_limit(limit: usize) -> bool { + log::trace!("enter: set_open_file_limit"); + + if let Ok((soft, hard)) = getrlimit(Resource::NOFILE) { + if hard.as_usize() > limit { + // our default open file limit is less than the current hard limit, this means we can + // set the soft limit to our default + let new_soft_limit = Rlim::from_usize(limit); + + if setrlimit(Resource::NOFILE, new_soft_limit, hard).is_ok() { + log::debug!("set open file descriptor limit to {}", limit); + + log::trace!("exit: set_open_file_limit -> {}", true); + return true; + } + } else if soft != hard { + // hard limit is lower than our default, the next best option is to set the soft limit as + // high as the hard limit will allow + if setrlimit(Resource::NOFILE, hard, hard).is_ok() { + log::debug!("set open file descriptor limit to {}", limit); + + log::trace!("exit: set_open_file_limit -> {}", true); + return true; + } + } + } + + // failed to set a new limit, as limit adjustments are a 'nice to have', we'll just log + // and move along + log::warn!("could not set open file descriptor limit to {}", limit); + + log::trace!("exit: set_open_file_limit -> {}", false); + false +} + #[cfg(test)] mod tests { use super::*; + #[test] + /// set_open_file_limit with a low requested limit succeeds + fn utils_set_open_file_limit_with_low_requested_limit() { + let (_, hard) = getrlimit(Resource::NOFILE).unwrap(); + let lower_limit = hard.as_usize() - 1; + assert!(set_open_file_limit(lower_limit)); + } + + #[test] + /// set_open_file_limit with a high requested limit succeeds + fn utils_set_open_file_limit_with_high_requested_limit() { + let (_, hard) = getrlimit(Resource::NOFILE).unwrap(); + let higher_limit = hard.as_usize() + 1; + // calculate a new soft to ensure soft != hard and hit that logic branch + let new_soft = Rlim::from_usize(hard.as_usize() - 1); + setrlimit(Resource::NOFILE, new_soft, hard).unwrap(); + assert!(set_open_file_limit(higher_limit)); + } + + #[test] + /// set_open_file_limit should fail when hard == soft + fn utils_set_open_file_limit_with_fails_when_both_limits_are_equal() { + let (_, hard) = getrlimit(Resource::NOFILE).unwrap(); + // calculate a new soft to ensure soft == hard and hit the failure logic branch + setrlimit(Resource::NOFILE, hard, hard).unwrap(); + assert!(!set_open_file_limit(hard.as_usize())); // returns false + } + #[test] /// base url returns 1 fn get_current_depth_base_url_returns_1() {