ran cargo fmt

Add wordcount and line count filtering to address #89

.github/workflows/check.yml

@@ -1,6 +1,6 @@
 name: CI Pipeline
 
-on: [push]
+on: [push, pull_request]
 
 jobs:
   check:
@@ -61,4 +61,4 @@ jobs:
       - uses: actions-rs/cargo@v1
         with:
           command: clippy
-          args: --all-targets --all-features -- -D warnings -A clippy::unnecessary_unwrap
+          args: --all-targets --all-features -- -D warnings -A clippy::unnecessary_unwrap -A clippy::deref_addrof

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "feroxbuster"
-version = "1.5.1"
+version = "1.6.1"
 authors = ["Ben 'epi' Risher <epibar052@gmail.com>"]
 license = "MIT"
 edition = "2018"
@@ -33,6 +33,7 @@ openssl = { version = "0.10", features = ["vendored"] }
 dirs = "3.0"
 regex = "1"
 crossterm = "0.18"
+rlimit = "0.5"
 
 [dev-dependencies]
 tempfile = "3.1"

README.md

@@ -61,6 +61,7 @@ This attack is also known as Predictable Resource Location, File Enumeration, Di
 -----------------
 - [Installation](#-installation)
     - [Download a Release](#download-a-release)
+    - [Snap Install](#snap-install)
     - [Homebrew on MacOS and Linux](#homebrew-on-macos-and-linux)
     - [Cargo Install](#cargo-install)
     - [apt Install](#apt-install)
@@ -68,6 +69,7 @@ This attack is also known as Predictable Resource Location, File Enumeration, Di
     - [Docker Install](#docker-install)
 - [Configuration](#%EF%B8%8F-configuration)
     - [Default Values](#default-values)
+    - [Threads and Connection Limits At A High-Level](#threads-and-connection-limits-at-a-high-level)
     - [ferox-config.toml](#ferox-configtoml)
     - [Command Line Parsing](#command-line-parsing)
 - [Example Usage](#-example-usage)
@@ -115,9 +117,34 @@ Expand-Archive .\feroxbuster.zip
 .\feroxbuster\feroxbuster.exe -V
 ```
 
+### Snap Install
+
+Install using `snap`
+
+```
+sudo snap install feroxbuster
+```
+
+The only gotcha here is that the snap package can only read wordlists from a few specific locations. There are a few 
+possible solutions, of which two are shown below.
+
+If the wordlist is on the same partition as your home directory, it can be hard-linked into `~/snap/feroxbuster/common`
+
+```
+ln /path/to/the/wordlist ~/snap/feroxbuster/common
+./feroxbuster -u http://localhost -w ~/snap/feroxbuster/common/wordlist
+``` 
+
+If the wordlist is on a separate partition, hard-linking won't work.  You'll need to copy it into the snap directory.
+
+```
+cp /path/to/the/wordlist ~/snap/feroxbuster/common
+./feroxbuster -u http://localhost -w ~/snap/feroxbuster/common/wordlist
+``` 
+
 ### Homebrew on MacOS and Linux
 
-Installable by Homebrew throughout own formulas:
+Install using Homebrew via tap
 
 🍏 [MacOS](https://github.com/TGotwig/homebrew-feroxbuster/blob/main/feroxbuster.rb)
 
@@ -230,6 +257,23 @@ Configuration begins with with the following built-in default values baked into
 - auto-filter wildcards - `true`
 - output: `stdout`
 
+### Threads and Connection Limits At A High-Level
+
+This section explains how the `-t` and `-L` options work together to determine the overall aggressiveness of a scan. The combination of the two values set by these options determines how hard your target will get hit and to some extent also determines how many resources will be consumed on your local machine.
+
+#### A Note on Green Threads
+
+`feroxbuster` uses so-called [green threads](https://en.wikipedia.org/wiki/Green_threads) as opposed to traditional kernel/OS threads. This means (at a high-level) that the threads are implemented entirely in userspace, within a single running process. As a result, a scan with 30 green threads will appear to the OS to be a single process with no additional light-weight processes associated with it as far as the kernel is concerned. As such, there will not be any impact to process (`nproc`) limits when specifying larger values for `-t`. However, these threads will still consume file descriptors, so you will need to ensure that you have a suitable `nlimit` set when scaling up the amount of threads. More detailed documentation on setting appropriate `nlimit` values can be found in the [No File Descriptors Available](#no-file-descriptors-available) section of the FAQ
+
+#### Threads and Connection Limits: The Implementation
+
+* Threads: The `-t` option specifies the maximum amount of active threads *per-directory* during a scan
+* Connection Limits: The `-L` option specifies the maximum amount of active connections per thread
+
+#### Threads and Connection Limits: Examples
+
+To truly have only 30 active requests to a site at any given time, `-t 30 -L 1` is necessary. Using `-t 30 -L 2` will result in a maximum of 60 total requests being processed at any given time for that site. And so on. For a conversation on this, please see [Issue #126](https://github.com/epi052/feroxbuster/issues/126) which may provide more (or less) clarity :wink:
+
 ### ferox-config.toml
 After setting built-in default values, any values defined in a `ferox-config.toml` config file will override the
 built-in defaults.  
@@ -297,6 +341,8 @@ A pre-made configuration file with examples of all available settings can be fou
 # extract_links = true
 # depth = 1
 # filter_size = [5174]
+# filter_word_count = [993]
+# filter_line_count = [35, 36]
 # queries = [["name","value"], ["rick", "astley"]]
 
 # headers can be specified on multiple lines or as an inline table
@@ -337,8 +383,10 @@ FLAGS:
 OPTIONS:
     -d, --depth <RECURSION_DEPTH>           Maximum recursion depth, a depth of 0 is infinite recursion (default: 4)
     -x, --extensions <FILE_EXTENSION>...    File extension(s) to search for (ex: -x php -x pdf js)
+    -N, --filter-lines <LINES>...           Filter out messages of a particular line count (ex: -N 20 -N 31,30)
     -S, --filter-size <SIZE>...             Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)
     -C, --filter-status <STATUS_CODE>...    Filter out status codes (deny list) (ex: -C 200 -C 401)
+    -W, --filter-words <WORDS>...           Filter out messages of a particular word count (ex: -W 312 -W 91,82)
     -H, --headers <HEADER>...               Specify HTTP headers (ex: -H Header:val 'stuff: things')
     -o, --output <FILE>                     Output file to write results to (default: stdout)
     -p, --proxy <PROXY>                     Proxy to use for requests (ex: http(s)://host:port, socks5://host:port)
@@ -511,7 +559,7 @@ a few of the use-cases in which feroxbuster may be a better fit:
 | configuration file for default value override                    | ✔ |   | ✔ |
 | can accept urls via STDIN as part of a pipeline                  | ✔ |   | ✔ |
 | can accept wordlists via STDIN                                   |   | ✔ | ✔ |
-| filter by response size                                          | ✔ |   | ✔ |
+| filter based on response size, wordcount, and linecount          | ✔ |   | ✔ |
 | auto-filter wildcard responses                                   | ✔ |   | ✔ |
 | performs other scans (vhost, dns, etc)                           |   | ✔ | ✔ |
 | time delay / rate limiting                                       |   | ✔ | ✔ |

ferox-config.toml.example

@@ -30,6 +30,8 @@
 # extract_links = true
 # depth = 1
 # filter_size = [5174]
+# filter_word_count = [993]
+# filter_line_count = [35, 36]
 # queries = [["name","value"], ["rick", "astley"]]
 
 # headers can be specified on multiple lines or as an inline table

snapcraft.yaml

@@ -0,0 +1,41 @@
+name: feroxbuster
+version: git
+summary: A simple, fast, recursive content discovery tool written in Rust
+description: |
+  feroxbuster is a tool designed to perform Forced Browsing.
+
+  Forced browsing is an attack where the aim is to enumerate and access resources that are not referenced by the web application, but are still accessible by an attacker.
+
+  feroxbuster uses brute force combined with a wordlist to search for unlinked content in target directories. These resources may store sensitive information about web applications and operational systems, such as source code, credentials, internal network addressing, etc...
+
+  This attack is also known as Predictable Resource Location, File Enumeration, Directory Enumeration, and Resource Enumeration.
+
+
+base: core18
+
+plugs:
+  etc-feroxbuster:
+    interface: system-files
+    read:
+    - /etc/feroxbuster
+  dot-config-feroxbuster:
+    interface: personal-files
+    read:
+    - $HOME/.config/feroxbuster
+
+architectures:
+  - build-on: amd64
+  - build-on: i386
+
+parts:
+  feroxbuster:
+    plugin: rust
+    source: .
+
+apps:
+  feroxbuster:
+    command: bin/feroxbuster
+    plugs:
+      - etc-feroxbuster
+      - dot-config-feroxbuster
+      - network

src/banner.rs

@@ -297,6 +297,24 @@ by Ben "epi" Risher {}                  ver: {}"#,
         }
     }
 
+    for filter in &config.filter_word_count {
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!("\u{1f4a2}", "Word Count Filter", filter)
+        )
+        .unwrap_or_default(); // 💢
+    }
+
+    for filter in &config.filter_line_count {
+        writeln!(
+            &mut writer,
+            "{}",
+            format_banner_entry!("\u{1f4a2}", "Line Count Filter", filter)
+        )
+        .unwrap_or_default(); // 💢
+    }
+
     if config.extract_links {
         writeln!(
             &mut writer,

src/config.rs

@@ -163,6 +163,14 @@ pub struct Configuration {
     #[serde(default)]
     pub filter_size: Vec<u64>,
 
+    /// Filter out messages of a particular line count
+    #[serde(default)]
+    pub filter_line_count: Vec<usize>,
+
+    /// Filter out messages of a particular word count
+    #[serde(default)]
+    pub filter_word_count: Vec<usize>,
+
     /// Don't auto-filter wildcard responses
     #[serde(default)]
     pub dont_filter: bool,
@@ -240,6 +248,8 @@ impl Default for Configuration {
             queries: Vec::new(),
             extensions: Vec::new(),
             filter_size: Vec::new(),
+            filter_line_count: Vec::new(),
+            filter_word_count: Vec::new(),
             filter_status: Vec::new(),
             headers: HashMap::new(),
             depth: depth(),
@@ -270,6 +280,8 @@ impl Configuration {
     /// - **insecure**: `false` (don't be insecure, i.e. don't allow invalid certs)
     /// - **extensions**: `None`
     /// - **filter_size**: `None`
+    /// - **filter_word_count**: `None`
+    /// - **filter_line_count**: `None`
     /// - **headers**: `None`
     /// - **queries**: `None`
     /// - **no_recursion**: `false` (recursively scan enumerated sub-directories)
@@ -352,36 +364,30 @@ impl Configuration {
 
         let args = parser::initialize().get_matches();
 
-        // the .is_some appears clunky, but it allows default values to be incrementally
-        // overwritten from Struct defaults, to file config, to command line args, soooo ¯\_(ツ)_/¯
-        if args.value_of("threads").is_some() {
-            let threads = value_t!(args.value_of("threads"), usize).unwrap_or_else(|e| e.exit());
-            config.threads = threads;
+        macro_rules! update_config_if_present {
+            ($c:expr, $m:ident, $v:expr, $t:ty) => {
+                match value_t!($m, $v, $t) {
+                    Ok(value) => *$c = value, // Update value
+                    Err(clap::Error {
+                        kind: clap::ErrorKind::ArgumentNotFound,
+                        message: _,
+                        info: _,
+                    }) => {
+                        // Do nothing if argument not found
+                    }
+                    Err(e) => e.exit(), // Exit with error on parse error
+                }
+            };
         }
 
-        if args.value_of("depth").is_some() {
-            let depth = value_t!(args.value_of("depth"), usize).unwrap_or_else(|e| e.exit());
-            config.depth = depth;
-        }
+        update_config_if_present!(&mut config.threads, args, "threads", usize);
+        update_config_if_present!(&mut config.depth, args, "depth", usize);
+        update_config_if_present!(&mut config.scan_limit, args, "scan_limit", usize);
+        update_config_if_present!(&mut config.wordlist, args, "wordlist", String);
+        update_config_if_present!(&mut config.output, args, "output", String);
 
-        if args.value_of("scan_limit").is_some() {
-            let scan_limit =
-                value_t!(args.value_of("scan_limit"), usize).unwrap_or_else(|e| e.exit());
-            config.scan_limit = scan_limit;
-        }
-
-        if args.value_of("wordlist").is_some() {
-            config.wordlist = String::from(args.value_of("wordlist").unwrap());
-        }
-
-        if args.value_of("output").is_some() {
-            config.output = String::from(args.value_of("output").unwrap());
-        }
-
-        if args.values_of("status_codes").is_some() {
-            config.status_codes = args
-                .values_of("status_codes")
-                .unwrap() // already known good
+        if let Some(arg) = args.values_of("status_codes") {
+            config.status_codes = arg
                 .map(|code| {
                     StatusCode::from_bytes(code.as_bytes())
                         .unwrap_or_else(|e| report_and_exit(&e.to_string()))
@@ -390,11 +396,9 @@ impl Configuration {
                 .collect();
         }
 
-        if args.values_of("replay_codes").is_some() {
+        if let Some(arg) = args.values_of("replay_codes") {
             // replay codes passed in by the user
-            config.replay_codes = args
-                .values_of("replay_codes")
-                .unwrap() // already known good
+            config.replay_codes = arg
                 .map(|code| {
                     StatusCode::from_bytes(code.as_bytes())
                         .unwrap_or_else(|e| report_and_exit(&e.to_string()))
@@ -406,10 +410,8 @@ impl Configuration {
             config.replay_codes = config.status_codes.clone();
         }
 
-        if args.values_of("filter_status").is_some() {
-            config.filter_status = args
-                .values_of("filter_status")
-                .unwrap() // already known good
+        if let Some(arg) = args.values_of("filter_status") {
+            config.filter_status = arg
                 .map(|code| {
                     StatusCode::from_bytes(code.as_bytes())
                         .unwrap_or_else(|e| report_and_exit(&e.to_string()))
@@ -418,20 +420,32 @@ impl Configuration {
                 .collect();
         }
 
-        if args.values_of("extensions").is_some() {
-            config.extensions = args
-                .values_of("extensions")
-                .unwrap()
-                .map(|val| val.to_string())
+        if let Some(arg) = args.values_of("extensions") {
+            config.extensions = arg.map(|val| val.to_string()).collect();
+        }
+
+        if let Some(arg) = args.values_of("filter_size") {
+            config.filter_size = arg
+                .map(|size| {
+                    size.parse::<u64>()
+                        .unwrap_or_else(|e| report_and_exit(&e.to_string()))
+                })
                 .collect();
         }
 
-        if args.values_of("filter_size").is_some() {
-            config.filter_size = args
-                .values_of("filter_size")
-                .unwrap() // already known good
+        if let Some(arg) = args.values_of("filter_words") {
+            config.filter_word_count = arg
                 .map(|size| {
-                    size.parse::<u64>()
+                    size.parse::<usize>()
+                        .unwrap_or_else(|e| report_and_exit(&e.to_string()))
+                })
+                .collect();
+        }
+
+        if let Some(arg) = args.values_of("filter_lines") {
+            config.filter_line_count = arg
+                .map(|size| {
+                    size.parse::<usize>()
                         .unwrap_or_else(|e| report_and_exit(&e.to_string()))
                 })
                 .collect();
@@ -442,11 +456,11 @@ impl Configuration {
             // consider a user specifying quiet = true in ferox-config.toml
             // if the line below is outside of the if, we'd overwrite true with
             // false if no -q is used on the command line
-            config.quiet = args.is_present("quiet");
+            config.quiet = true;
         }
 
         if args.is_present("dont_filter") {
-            config.dont_filter = args.is_present("dont_filter");
+            config.dont_filter = true;
         }
 
         if args.occurrences_of("verbosity") > 0 {
@@ -456,19 +470,19 @@ impl Configuration {
         }
 
         if args.is_present("no_recursion") {
-            config.no_recursion = args.is_present("no_recursion");
+            config.no_recursion = true;
         }
 
         if args.is_present("add_slash") {
-            config.add_slash = args.is_present("add_slash");
+            config.add_slash = true;
         }
 
         if args.is_present("extract_links") {
-            config.extract_links = args.is_present("extract_links");
+            config.extract_links = true;
         }
 
         if args.is_present("stdin") {
-            config.stdin = args.is_present("stdin");
+            config.stdin = true;
         } else {
             config.target_url = String::from(args.value_of("url").unwrap());
         }
@@ -476,33 +490,21 @@ impl Configuration {
         ////
         // organizational breakpoint; all options below alter the Client configuration
         ////
-        if args.value_of("proxy").is_some() {
-            config.proxy = String::from(args.value_of("proxy").unwrap());
-        }
-
-        if args.value_of("replay_proxy").is_some() {
-            config.replay_proxy = String::from(args.value_of("replay_proxy").unwrap());
-        }
-
-        if args.value_of("user_agent").is_some() {
-            config.user_agent = String::from(args.value_of("user_agent").unwrap());
-        }
-
-        if args.value_of("timeout").is_some() {
-            let timeout = value_t!(args.value_of("timeout"), u64).unwrap_or_else(|e| e.exit());
-            config.timeout = timeout;
-        }
+        update_config_if_present!(&mut config.proxy, args, "proxy", String);
+        update_config_if_present!(&mut config.replay_proxy, args, "replay_proxy", String);
+        update_config_if_present!(&mut config.user_agent, args, "user_agent", String);
+        update_config_if_present!(&mut config.timeout, args, "timeout", u64);
 
         if args.is_present("redirects") {
-            config.redirects = args.is_present("redirects");
+            config.redirects = true;
         }
 
         if args.is_present("insecure") {
-            config.insecure = args.is_present("insecure");
+            config.insecure = true;
         }
 
-        if args.values_of("headers").is_some() {
-            for val in args.values_of("headers").unwrap() {
+        if let Some(headers) = args.values_of("headers") {
+            for val in headers {
                 let mut split_val = val.split(':');
 
                 // explicitly take first split value as header's name
@@ -515,8 +517,8 @@ impl Configuration {
             }
         }
 
-        if args.values_of("queries").is_some() {
-            for val in args.values_of("queries").unwrap() {
+        if let Some(queries) = args.values_of("queries") {
+            for val in queries {
                 // same basic logic used as reading in the headers HashMap above
                 let mut split_val = val.split('=');
 
@@ -616,6 +618,8 @@ impl Configuration {
         settings.stdin = settings_to_merge.stdin;
         settings.depth = settings_to_merge.depth;
         settings.filter_size = settings_to_merge.filter_size;
+        settings.filter_word_count = settings_to_merge.filter_word_count;
+        settings.filter_line_count = settings_to_merge.filter_line_count;
         settings.filter_status = settings_to_merge.filter_status;
         settings.dont_filter = settings_to_merge.dont_filter;
         settings.scan_limit = settings_to_merge.scan_limit;
@@ -678,6 +682,8 @@ mod tests {
             extract_links = true
             depth = 1
             filter_size = [4120]
+            filter_word_count = [994, 992]
+            filter_line_count = [34]
             filter_status = [201]
         "#;
         let tmp_dir = TempDir::new().unwrap();
@@ -714,6 +720,8 @@ mod tests {
         assert_eq!(config.queries, Vec::new());
         assert_eq!(config.extensions, Vec::<String>::new());
         assert_eq!(config.filter_size, Vec::<u64>::new());
+        assert_eq!(config.filter_word_count, Vec::<usize>::new());
+        assert_eq!(config.filter_line_count, Vec::<usize>::new());
         assert_eq!(config.filter_status, Vec::<u16>::new());
         assert_eq!(config.headers, HashMap::new());
     }
@@ -865,6 +873,20 @@ mod tests {
         assert_eq!(config.filter_size, vec![4120]);
     }
 
+    #[test]
+    /// parse the test config and see that the value parsed is correct
+    fn config_reads_filter_word_count() {
+        let config = setup_config_test();
+        assert_eq!(config.filter_word_count, vec![994, 992]);
+    }
+
+    #[test]
+    /// parse the test config and see that the value parsed is correct
+    fn config_reads_filter_line_count() {
+        let config = setup_config_test();
+        assert_eq!(config.filter_line_count, vec![34]);
+    }
+
     #[test]
     /// parse the test config and see that the value parsed is correct
     fn config_reads_filter_status() {

src/heuristics.rs

@@ -183,14 +183,18 @@ async fn make_wildcard_request(
                 .contains(&response.status().as_u16())
             {
                 // found a wildcard response
-                let ferox_response = FeroxResponse::from(response, false).await;
+                let ferox_response = FeroxResponse::from(response, true).await;
                 let url_len = get_url_path_length(&ferox_response.url());
                 let content_len = ferox_response.content_length();
+                let content_words = ferox_response.word_count();
+                let content_lines = ferox_response.line_count();
 
                 if !CONFIGURATION.quiet && !should_filter_response(&ferox_response) {
                     let msg = format!(
-                        "{} {:>10} Got {} for {} (url length: {})\n",
+                        "{} {:>8}l {:>8}w {:>8}c Got {} for {} (url length: {})\n",
                         wildcard,
+                        content_lines,
+                        content_words,
                         content_len,
                         status_colorizer(&ferox_response.status().as_str()),
                         ferox_response.url(),
@@ -212,8 +216,10 @@ async fn make_wildcard_request(
                         let next_loc_str = next_loc.to_str().unwrap_or("Unknown");
                         if !CONFIGURATION.quiet && !should_filter_response(&ferox_response) {
                             let msg = format!(
-                                "{} {:>10} {} redirects to => {}\n",
+                                "{} {:>8}l {:>8}w {:>8}c {} redirects to => {}\n",
                                 wildcard,
+                                content_lines,
+                                content_words,
                                 content_len,
                                 ferox_response.url(),
                                 next_loc_str

src/lib.rs

@@ -42,6 +42,9 @@ pub type FeroxChannel<T> = (UnboundedSender<T>, UnboundedReceiver<T>);
 /// Version pulled from Cargo.toml at compile time
 pub const VERSION: &str = env!("CARGO_PKG_VERSION");
 
+/// Maximum number of file descriptors that can be opened during a scan
+pub const DEFAULT_OPEN_FILE_LIMIT: usize = 8192;
+
 /// Default wordlist to use when `-w|--wordlist` isn't specified and not `wordlist` isn't set
 /// in a [ferox-config.toml](constant.DEFAULT_CONFIG_NAME.html) config file.
 ///
@@ -160,6 +163,19 @@ impl FeroxResponse {
         self.url.query_pairs().count() > 0 || has_extension
     }
 
+    /// Returns line count of the response text.
+    pub fn line_count(&self) -> usize {
+        self.text().lines().count()
+    }
+
+    /// Returns word count of the response text.
+    pub fn word_count(&self) -> usize {
+        self.text()
+            .lines()
+            .map(|s| s.split_whitespace().count())
+            .sum()
+    }
+
     /// Create a new `FeroxResponse` from the given `Response`
     pub async fn from(response: Response, read_body: bool) -> Self {
         let url = response.url().clone();

src/main.rs

@@ -7,6 +7,8 @@ use feroxbuster::{
     utils::{ferox_print, get_current_depth, module_colorizer, status_colorizer},
     FeroxError, FeroxResponse, FeroxResult, SLEEP_DURATION, VERSION,
 };
+#[cfg(not(target_os = "windows"))]
+use feroxbuster::{utils::set_open_file_limit, DEFAULT_OPEN_FILE_LIMIT};
 use futures::StreamExt;
 use std::{
     collections::HashSet,
@@ -294,6 +296,10 @@ fn main() {
     // setup logging based on the number of -v's used
     logger::initialize(CONFIGURATION.verbosity);
 
+    // this function uses rlimit, which is not supported on windows
+    #[cfg(not(target_os = "windows"))]
+    set_open_file_limit(DEFAULT_OPEN_FILE_LIMIT);
+
     if let Ok(mut runtime) = tokio::runtime::Runtime::new() {
         let future = wrapped_main();
         runtime.block_on(future);

src/parser.rs

@@ -218,6 +218,30 @@ pub fn initialize() -> App<'static, 'static> {
                     "Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)",
                 ),
         )
+        .arg(
+            Arg::with_name("filter_words")
+                .short("W")
+                .long("filter-words")
+                .value_name("WORDS")
+                .takes_value(true)
+                .multiple(true)
+                .use_delimiter(true)
+                .help(
+                    "Filter out messages of a particular word count (ex: -W 312 -W 91,82)",
+                ),
+        )
+        .arg(
+            Arg::with_name("filter_lines")
+                .short("N")
+                .long("filter-lines")
+                .value_name("LINES")
+                .takes_value(true)
+                .multiple(true)
+                .use_delimiter(true)
+                .help(
+                    "Filter out messages of a particular line count (ex: -N 20 -N 31,30)",
+                ),
+        )
         .arg(
             Arg::with_name("filter_status")
                 .short("C")

src/reporter.rs

@@ -104,8 +104,10 @@ async fn spawn_terminal_reporter(
                 format!(
                     // example output
                     // 200       3280 https://localhost.com/FAQ
-                    "{} {:>10} {}\n",
+                    "{} {:>8}l {:>8}w {:>8}c {}\n",
                     status,
+                    resp.line_count(),
+                    resp.word_count(),
                     resp.content_length(),
                     resp.url()
                 )

src/scanner.rs

@@ -130,17 +130,9 @@ fn add_url_to_list_of_scanned_urls(resp: &str, scanned_urls: &RwLock<HashSet<Str
     match scanned_urls.write() {
         // check new url against what's already been scanned
         Ok(mut urls) => {
-            let normalized_url = if resp.ends_with('/') {
-                // append a / to the list of 'seen' urls, this is to prevent the case where
-                // 3xx and 2xx duplicate eachother
-                resp.to_string()
-            } else {
-                format!("{}/", resp)
-            };
-
             // If the set did not contain resp, true is returned.
             // If the set did contain resp, false is returned.
-            let response = urls.insert(normalized_url);
+            let response = urls.insert(resp.to_string());
 
             log::trace!("exit: add_url_to_list_of_scanned_urls -> {}", response);
             response
@@ -429,6 +421,12 @@ pub fn should_filter_response(response: &FeroxResponse) -> bool {
     if CONFIGURATION
         .filter_size
         .contains(&response.content_length())
+        || CONFIGURATION
+            .filter_line_count
+            .contains(&response.line_count())
+        || CONFIGURATION
+            .filter_word_count
+            .contains(&response.word_count())
     {
         // filtered value from --filter-size, size filters and wildcards are two separate filters
         // and are applied independently
@@ -478,7 +476,7 @@ async fn make_requests(
     for url in urls {
         if let Ok(response) = make_request(&CONFIGURATION.client, &url).await {
             // response came back without error, convert it to FeroxResponse
-            let ferox_response = FeroxResponse::from(response, CONFIGURATION.extract_links).await;
+            let ferox_response = FeroxResponse::from(response, true).await;
 
             // do recursion if appropriate
             if !CONFIGURATION.no_recursion {
@@ -521,8 +519,7 @@ async fn make_requests(
                         Err(_) => continue,
                     };
 
-                    let mut new_ferox_response =
-                        FeroxResponse::from(new_response, CONFIGURATION.extract_links).await;
+                    let mut new_ferox_response = FeroxResponse::from(new_response, true).await;
 
                     // filter if necessary
                     if should_filter_response(&new_ferox_response) {
@@ -855,7 +852,7 @@ mod tests {
         assert_eq!(
             urls.write()
                 .unwrap()
-                .insert("http://unknown_url/".to_string()),
+                .insert("http://unknown_url".to_string()),
             true
         );
 

src/utils.rs

@@ -3,6 +3,8 @@ use console::{strip_ansi_codes, style, user_attended};
 use indicatif::ProgressBar;
 use reqwest::Url;
 use reqwest::{Client, Response};
+#[cfg(not(target_os = "windows"))]
+use rlimit::{getrlimit, setrlimit, Resource, Rlim};
 use std::convert::TryInto;
 
 /// Helper function that determines the current depth of a given url
@@ -259,10 +261,89 @@ pub async fn make_request(client: &Client, url: &Url) -> FeroxResult<Response> {
     }
 }
 
+/// Attempts to set the soft limit for the RLIMIT_NOFILE resource
+///
+/// RLIMIT_NOFILE is the maximum number of file descriptors that can be opened by this process
+///
+/// The soft limit is the value that the kernel enforces for the corresponding resource.
+/// The hard limit acts as a ceiling for the soft limit: an unprivileged process may set only its
+/// soft limit to a value in the range from 0 up to the hard limit, and (irreversibly) lower its
+/// hard limit.
+///
+/// A child process created via fork(2) inherits its parent's resource limits. Resource limits are
+/// per-process attributes that are shared by all of the threads in a process.
+///
+/// Based on the above information, no attempt is made to restore the limit to its pre-scan value
+/// as the adjustment made here is only valid for the scan itself (and any child processes, of which
+/// there are none).
+#[cfg(not(target_os = "windows"))]
+pub fn set_open_file_limit(limit: usize) -> bool {
+    log::trace!("enter: set_open_file_limit");
+
+    if let Ok((soft, hard)) = getrlimit(Resource::NOFILE) {
+        if hard.as_usize() > limit {
+            // our default open file limit is less than the current hard limit, this means we can
+            // set the soft limit to our default
+            let new_soft_limit = Rlim::from_usize(limit);
+
+            if setrlimit(Resource::NOFILE, new_soft_limit, hard).is_ok() {
+                log::debug!("set open file descriptor limit to {}", limit);
+
+                log::trace!("exit: set_open_file_limit -> {}", true);
+                return true;
+            }
+        } else if soft != hard {
+            // hard limit is lower than our default, the next best option is to set the soft limit as
+            // high as the hard limit will allow
+            if setrlimit(Resource::NOFILE, hard, hard).is_ok() {
+                log::debug!("set open file descriptor limit to {}", limit);
+
+                log::trace!("exit: set_open_file_limit -> {}", true);
+                return true;
+            }
+        }
+    }
+
+    // failed to set a new limit, as limit adjustments are a 'nice to have', we'll just log
+    // and move along
+    log::warn!("could not set open file descriptor limit to {}", limit);
+
+    log::trace!("exit: set_open_file_limit -> {}", false);
+    false
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
 
+    #[test]
+    /// set_open_file_limit with a low requested limit succeeds
+    fn utils_set_open_file_limit_with_low_requested_limit() {
+        let (_, hard) = getrlimit(Resource::NOFILE).unwrap();
+        let lower_limit = hard.as_usize() - 1;
+        assert!(set_open_file_limit(lower_limit));
+    }
+
+    #[test]
+    /// set_open_file_limit with a high requested limit succeeds
+    fn utils_set_open_file_limit_with_high_requested_limit() {
+        let (_, hard) = getrlimit(Resource::NOFILE).unwrap();
+        let higher_limit = hard.as_usize() + 1;
+        // calculate a new soft to ensure soft != hard and hit that logic branch
+        let new_soft = Rlim::from_usize(hard.as_usize() - 1);
+        setrlimit(Resource::NOFILE, new_soft, hard).unwrap();
+        assert!(set_open_file_limit(higher_limit));
+    }
+
+    #[test]
+    /// set_open_file_limit should fail when hard == soft
+    fn utils_set_open_file_limit_with_fails_when_both_limits_are_equal() {
+        let (_, hard) = getrlimit(Resource::NOFILE).unwrap();
+        // calculate a new soft to ensure soft == hard and hit the failure logic branch
+        setrlimit(Resource::NOFILE, hard, hard).unwrap();
+        assert!(!set_open_file_limit(hard.as_usize())); // returns false
+    }
+
     #[test]
     /// base url returns 1
     fn get_current_depth_base_url_returns_1() {

tests/test_banner.rs

@@ -117,7 +117,7 @@ fn banner_prints_headers() -> Result<(), Box<dyn std::error::Error>> {
 #[test]
 /// test allows non-existent wordlist to trigger the banner printing to stderr
 /// expect to see all mandatory prints + multiple size filters
-fn banner_prints_filter_sizes() -> Result<(), Box<dyn std::error::Error>> {
+fn banner_prints_filter_sizes() {
     Command::cargo_bin("feroxbuster")
         .unwrap()
         .arg("--url")
@@ -126,6 +126,14 @@ fn banner_prints_filter_sizes() -> Result<(), Box<dyn std::error::Error>> {
         .arg("789456123")
         .arg("--filter-size")
         .arg("44444444")
+        .arg("-N")
+        .arg("678")
+        .arg("--filter-lines")
+        .arg("679")
+        .arg("-W")
+        .arg("93")
+        .arg("--filter-words")
+        .arg("94")
         .assert()
         .success()
         .stderr(
@@ -138,11 +146,16 @@ fn banner_prints_filter_sizes() -> Result<(), Box<dyn std::error::Error>> {
                 .and(predicate::str::contains("Timeout (secs)"))
                 .and(predicate::str::contains("User-Agent"))
                 .and(predicate::str::contains("Size Filter"))
+                .and(predicate::str::contains("Word Count Filter"))
+                .and(predicate::str::contains("Line Count Filter"))
                 .and(predicate::str::contains("789456123"))
                 .and(predicate::str::contains("44444444"))
+                .and(predicate::str::contains("93"))
+                .and(predicate::str::contains("94"))
+                .and(predicate::str::contains("678"))
+                .and(predicate::str::contains("679"))
                 .and(predicate::str::contains("─┴─")),
         );
-    Ok(())
 }
 
 #[test]