updated multiple items from new-feature checklist

Cargo.lock

@@ -596,7 +596,7 @@ dependencies = [
 
 [[package]]
 name = "feroxbuster"
-version = "2.3.3"
+version = "2.3.4"
 dependencies = [
  "anyhow",
  "assert_cmd",

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "feroxbuster"
-version = "2.3.3"
+version = "2.3.4"
 authors = ["Ben 'epi' Risher <epibar052@gmail.com>"]
 license = "MIT"
 edition = "2018"

ferox-config.toml.example

@@ -31,6 +31,7 @@
 # insecure = true
 # extensions = ["php", "html"]
 # url_denylist = ["http://dont-scan.me", "https://also-not.me"]
+# regex_denylist = ["/deny.*"]
 # no_recursion = true
 # add_slash = true
 # stdin = true

shell_completions/_feroxbuster

@@ -41,7 +41,7 @@ _feroxbuster() {
 '--user-agent=[Sets the User-Agent (default: feroxbuster/VERSION)]' \
 '*-x+[File extension(s) to search for (ex: -x php -x pdf js)]' \
 '*--extensions=[File extension(s) to search for (ex: -x php -x pdf js)]' \
-'*--dont-scan=[URL(s) to exclude from recursion/scans]' \
+'*--dont-scan=[URL(s) or Regex Pattern(s) to exclude from recursion/scans]' \
 '*-H+[Specify HTTP headers (ex: -H Header:val '\''stuff: things'\'')]' \
 '*--headers=[Specify HTTP headers (ex: -H Header:val '\''stuff: things'\'')]' \
 '*-Q+[Specify URL query parameters (ex: -Q token=stuff -Q secret=key)]' \

shell_completions/_feroxbuster.ps1

@@ -46,7 +46,7 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock {
             [CompletionResult]::new('--user-agent', 'user-agent', [CompletionResultType]::ParameterName, 'Sets the User-Agent (default: feroxbuster/VERSION)')
             [CompletionResult]::new('-x', 'x', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js)')
             [CompletionResult]::new('--extensions', 'extensions', [CompletionResultType]::ParameterName, 'File extension(s) to search for (ex: -x php -x pdf js)')
-            [CompletionResult]::new('--dont-scan', 'dont-scan', [CompletionResultType]::ParameterName, 'URL(s) to exclude from recursion/scans')
+            [CompletionResult]::new('--dont-scan', 'dont-scan', [CompletionResultType]::ParameterName, 'URL(s) or Regex Pattern(s) to exclude from recursion/scans')
             [CompletionResult]::new('-H', 'H', [CompletionResultType]::ParameterName, 'Specify HTTP headers (ex: -H Header:val ''stuff: things'')')
             [CompletionResult]::new('--headers', 'headers', [CompletionResultType]::ParameterName, 'Specify HTTP headers (ex: -H Header:val ''stuff: things'')')
             [CompletionResult]::new('-Q', 'Q', [CompletionResultType]::ParameterName, 'Specify URL query parameters (ex: -Q token=stuff -Q secret=key)')

shell_completions/feroxbuster.fish

@@ -12,7 +12,7 @@ complete -c feroxbuster -n "__fish_use_subcommand" -l resume-from -d 'State file
 complete -c feroxbuster -n "__fish_use_subcommand" -l debug-log -d 'Output file to write log entries (use w/ --json for JSON entries)'
 complete -c feroxbuster -n "__fish_use_subcommand" -s a -l user-agent -d 'Sets the User-Agent (default: feroxbuster/VERSION)'
 complete -c feroxbuster -n "__fish_use_subcommand" -s x -l extensions -d 'File extension(s) to search for (ex: -x php -x pdf js)'
-complete -c feroxbuster -n "__fish_use_subcommand" -l dont-scan -d 'URL(s) to exclude from recursion/scans'
+complete -c feroxbuster -n "__fish_use_subcommand" -l dont-scan -d 'URL(s) or Regex Pattern(s) to exclude from recursion/scans'
 complete -c feroxbuster -n "__fish_use_subcommand" -s H -l headers -d 'Specify HTTP headers (ex: -H Header:val \'stuff: things\')'
 complete -c feroxbuster -n "__fish_use_subcommand" -s Q -l query -d 'Specify URL query parameters (ex: -Q token=stuff -Q secret=key)'
 complete -c feroxbuster -n "__fish_use_subcommand" -s S -l filter-size -d 'Filter out messages of a particular size (ex: -S 5120 -S 4927,1970)'

src/config/tests.rs

@@ -1,6 +1,7 @@
 use super::utils::*;
 use super::*;
 use crate::{traits::FeroxSerialize, DEFAULT_CONFIG_NAME};
+use regex::Regex;
 use reqwest::Url;
 use std::{collections::HashMap, fs::write};
 use tempfile::TempDir;
@@ -31,6 +32,7 @@ fn setup_config_test() -> Configuration {
             insecure = true
             extensions = ["html", "php", "js"]
             url_denylist = ["http://dont-scan.me", "https://also-not.me"]
+            regex_denylist = ["/deny.*"]
             headers = {stuff = "things", mostuff = "mothings"}
             queries = [["name","value"], ["rick", "astley"]]
             no_recursion = true
@@ -291,6 +293,16 @@ fn config_reads_extensions() {
     assert_eq!(config.extensions, vec!["html", "php", "js"]);
 }
 
+#[test]
+/// parse the test config and see that the value parsed is correct
+fn config_reads_regex_denylist() {
+    let config = setup_config_test();
+    assert_eq!(
+        config.regex_denylist[0].as_str(),
+        Regex::new("/deny.*").unwrap().as_str()
+    );
+}
+
 #[test]
 /// parse the test config and see that the value parsed is correct
 fn config_reads_url_denylist() {

src/parser.rs

@@ -226,7 +226,7 @@ pub fn initialize() -> App<'static, 'static> {
                 .multiple(true)
                 .use_delimiter(true)
                 .help(
-                    "URL(s) to exclude from recursion/scans",
+                    "URL(s) or Regex Pattern(s) to exclude from recursion/scans",
                 ),
         )
         .arg(

tests/test_banner.rs

@@ -115,7 +115,7 @@ fn banner_prints_headers() {
 
 #[test]
 /// test allows non-existent wordlist to trigger the banner printing to stderr
-/// expect to see all mandatory prints + multiple dont scan entries
+/// expect to see all mandatory prints + multiple dont scan url & regex entries
 fn banner_prints_denied_urls() {
     Command::cargo_bin("feroxbuster")
         .unwrap()
@@ -123,8 +123,9 @@ fn banner_prints_denied_urls() {
         .arg("http://localhost")
         .arg("--dont-scan")
         .arg("http://dont-scan.me")
-        .arg("--dont-scan")
         .arg("https://also-not.me")
+        .arg("https:")
+        .arg("/deny.*")
         .assert()
         .success()
         .stderr(
@@ -136,9 +137,12 @@ fn banner_prints_denied_urls() {
                 .and(predicate::str::contains("Status Codes"))
                 .and(predicate::str::contains("Timeout (secs)"))
                 .and(predicate::str::contains("User-Agent"))
-                .and(predicate::str::contains("Don't Scan"))
+                .and(predicate::str::contains("Don't Scan Url"))
+                .and(predicate::str::contains("Don't Scan Regex"))
                 .and(predicate::str::contains("http://dont-scan.me"))
                 .and(predicate::str::contains("https://also-not.me"))
+                .and(predicate::str::contains("https:"))
+                .and(predicate::str::contains("/deny.*"))
                 .and(predicate::str::contains("─┴─")),
         );
 }