From 0b75e1a548350c955c241e87f3ae736a7c55c452 Mon Sep 17 00:00:00 2001 From: Daniel Saxton Date: Sun, 19 Sep 2021 18:18:18 -0500 Subject: [PATCH 01/19] Implement random user agent flag --- ferox-config.toml.example | 1 + src/banner/container.rs | 12 +++++++++++- src/config/container.rs | 11 +++++++++++ src/config/tests.rs | 7 +++++++ 4 files changed, 30 insertions(+), 1 deletion(-) diff --git a/ferox-config.toml.example b/ferox-config.toml.example index cf8396d..c14ecf7 100644 --- a/ferox-config.toml.example +++ b/ferox-config.toml.example @@ -27,6 +27,7 @@ # output = "/targets/ellingson_mineral_company/gibson.txt" # debug_log = "/var/log/find-the-derp.log" # user_agent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0" +# random_agent = false # redirects = true # insecure = true # extensions = ["php", "html"] diff --git a/src/banner/container.rs b/src/banner/container.rs index c01f426..92d35cf 100644 --- a/src/banner/container.rs +++ b/src/banner/container.rs @@ -50,6 +50,9 @@ pub struct Banner { /// represents Configuration.user_agent user_agent: BannerEntry, + /// represents Configuration.random_agent + random_agent: BannerEntry, + /// represents Configuration.config config: BannerEntry, @@ -276,6 +279,7 @@ impl Banner { let wordlist = BannerEntry::new("📖", "Wordlist", &config.wordlist); let timeout = BannerEntry::new("💥", "Timeout (secs)", &config.timeout.to_string()); let user_agent = BannerEntry::new("🦡", "User-Agent", &config.user_agent); + let random_agent = BannerEntry::new("🦡", "User-Agent", "(Random)"); let extract_links = BannerEntry::new("🔎", "Extract Links", &config.extract_links.to_string()); let json = BannerEntry::new("🧔", "JSON Output", &config.json.to_string()); @@ -304,6 +308,7 @@ impl Banner { filter_status, timeout, user_agent, + random_agent, auto_bail, auto_tune, proxy, @@ -437,7 +442,12 @@ by Ben "epi" Risher {} ver: {}"#, } writeln!(&mut writer, "{}", self.timeout)?; - writeln!(&mut writer, "{}", self.user_agent)?; + + if config.random_agent { + writeln!(&mut writer, "{}", self.random_agent)?; + } else { + writeln!(&mut writer, "{}", self.user_agent)?; + } // followed by the maybe printed or variably displayed values if !config.config.is_empty() { diff --git a/src/config/container.rs b/src/config/container.rs index 014772a..df2bb76 100644 --- a/src/config/container.rs +++ b/src/config/container.rs @@ -154,6 +154,10 @@ pub struct Configuration { #[serde(default = "user_agent")] pub user_agent: String, + /// Use random User-Agent + #[serde(default)] + pub random_agent: bool, + /// Follow redirects #[serde(default)] pub redirects: bool, @@ -295,6 +299,7 @@ impl Default for Configuration { redirects: false, no_recursion: false, extract_links: false, + random_agent: false, save_state: true, proxy: String::new(), config: String::new(), @@ -344,6 +349,7 @@ impl Configuration { /// - **auto_bail**: `false` /// - **save_state**: `true` /// - **user_agent**: `feroxbuster/VERSION` + /// - **random_agent**: `false` /// - **insecure**: `false` (don't be insecure, i.e. don't allow invalid certs) /// - **extensions**: `None` /// - **url_denylist**: `None` @@ -647,6 +653,10 @@ impl Configuration { update_config_if_present!(&mut config.user_agent, args, "user_agent", String); update_config_if_present!(&mut config.timeout, args, "timeout", u64); + if args.is_present("random_agent") { + config.random_agent = true; + } + if args.is_present("redirects") { config.redirects = true; } @@ -824,6 +834,7 @@ impl Configuration { update_if_not_default!(&mut conf.timeout, new.timeout, timeout()); update_if_not_default!(&mut conf.user_agent, new.user_agent, user_agent()); + update_if_not_default!(&mut conf.random_agent, new.random_agent, false); update_if_not_default!(&mut conf.threads, new.threads, threads()); update_if_not_default!(&mut conf.depth, new.depth, depth()); update_if_not_default!(&mut conf.wordlist, new.wordlist, wordlist()); diff --git a/src/config/tests.rs b/src/config/tests.rs index daeaddb..7896acc 100644 --- a/src/config/tests.rs +++ b/src/config/tests.rs @@ -81,6 +81,7 @@ fn default_configuration() { assert!(!config.auto_bail); assert_eq!(config.requester_policy, RequesterPolicy::Default); assert!(!config.no_recursion); + assert!(!config.random_agent); assert!(!config.json); assert!(config.save_state); assert!(!config.stdin); @@ -383,6 +384,12 @@ fn config_reads_queries() { assert_eq!(config.queries, queries); } +#[test] +fn config_default_not_random_agent() { + let config = setup_config_test(); + assert!(!config.random_agent); +} + #[test] #[should_panic] /// test that an error message is printed and panic is called when report_and_exit is called From 28f63aae94d330801caf5037c55cb75c415797b0 Mon Sep 17 00:00:00 2001 From: Daniel Saxton Date: Wed, 22 Sep 2021 17:09:11 -0500 Subject: [PATCH 02/19] Test --- tests/test_banner.rs | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/test_banner.rs b/tests/test_banner.rs index e1a3299..1f4b373 100644 --- a/tests/test_banner.rs +++ b/tests/test_banner.rs @@ -143,6 +143,31 @@ fn banner_prints_denied_urls() { ); } +#[test] +/// test allows non-existent wordlist to trigger the banner printing to stderr +/// expect to see all mandatory prints + multiple headers +fn banner_prints_random_agent() { + Command::cargo_bin("feroxbuster") + .unwrap() + .arg("--url") + .arg("http://localhost") + .arg("--random-agent") + .assert() + .success() + .stderr( + predicate::str::contains("─┬─") + .and(predicate::str::contains("Target Url")) + .and(predicate::str::contains("http://localhost")) + .and(predicate::str::contains("Threads")) + .and(predicate::str::contains("Wordlist")) + .and(predicate::str::contains("Status Codes")) + .and(predicate::str::contains("Timeout (secs)")) + .and(predicate::str::contains("User-Agent")) + .and(predicate::str::contains("(Random)")) + .and(predicate::str::contains("─┴─")), + ); +} + #[test] /// test allows non-existent wordlist to trigger the banner printing to stderr /// expect to see all mandatory prints + multiple size filters From 40fccb97611e8226b009fbf1221b8a3c041804cb Mon Sep 17 00:00:00 2001 From: Daniel Saxton Date: Wed, 22 Sep 2021 19:14:07 -0500 Subject: [PATCH 03/19] Bump minor version --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 76383a8..596dc0c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -596,7 +596,7 @@ dependencies = [ [[package]] name = "feroxbuster" -version = "2.3.3" +version = "2.4.0" dependencies = [ "anyhow", "assert_cmd", diff --git a/Cargo.toml b/Cargo.toml index b2ef130..0eddae1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "feroxbuster" -version = "2.3.3" +version = "2.4.0" authors = ["Ben 'epi' Risher "] license = "MIT" edition = "2018" From 8d11bb1800776680a7bc7d4b23fe60571a282dd8 Mon Sep 17 00:00:00 2001 From: Daniel Saxton Date: Wed, 22 Sep 2021 19:21:07 -0500 Subject: [PATCH 04/19] README --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 6499f96..137ef99 100644 --- a/README.md +++ b/README.md @@ -482,6 +482,9 @@ FLAGS: -V, --version Prints version information + -A, --random-agent + Use a random User-Agent + -v, --verbosity Increase verbosity level (use -vv or more for greater effect. [CAUTION] 4 -v's is probably too much) From bce55e77f32452413c7a7a2c1d0bbfe6d558aada Mon Sep 17 00:00:00 2001 From: Daniel Saxton Date: Wed, 22 Sep 2021 19:41:52 -0500 Subject: [PATCH 05/19] Add arg --- shell_completions/_feroxbuster | 2 ++ shell_completions/_feroxbuster.ps1 | 2 ++ shell_completions/feroxbuster.bash | 2 +- shell_completions/feroxbuster.fish | 1 + src/parser.rs | 9 +++++++++ 5 files changed, 15 insertions(+), 1 deletion(-) diff --git a/shell_completions/_feroxbuster b/shell_completions/_feroxbuster index 42b3552..dba5517 100644 --- a/shell_completions/_feroxbuster +++ b/shell_completions/_feroxbuster @@ -72,6 +72,8 @@ _feroxbuster() { '--json[Emit JSON logs to --output and --debug-log instead of normal text]' \ '-D[Don'\''t auto-filter wildcard responses]' \ '--dont-filter[Don'\''t auto-filter wildcard responses]' \ +'-A[Use a random User-Agent]' \ +'--random-agent[Use a random User-Agent]' \ '-r[Follow redirects]' \ '--redirects[Follow redirects]' \ '-k[Disables TLS certificate validation]' \ diff --git a/shell_completions/_feroxbuster.ps1 b/shell_completions/_feroxbuster.ps1 index 81539c5..bad2b9c 100644 --- a/shell_completions/_feroxbuster.ps1 +++ b/shell_completions/_feroxbuster.ps1 @@ -77,6 +77,8 @@ Register-ArgumentCompleter -Native -CommandName 'feroxbuster' -ScriptBlock { [CompletionResult]::new('--json', 'json', [CompletionResultType]::ParameterName, 'Emit JSON logs to --output and --debug-log instead of normal text') [CompletionResult]::new('-D', 'D', [CompletionResultType]::ParameterName, 'Don''t auto-filter wildcard responses') [CompletionResult]::new('--dont-filter', 'dont-filter', [CompletionResultType]::ParameterName, 'Don''t auto-filter wildcard responses') + [CompletionResult]::new('-A', 'A', [CompletionResultType]::ParameterName, 'Use a random User-Agent') + [CompletionResult]::new('--random-agent', 'random-agent', [CompletionResultType]::ParameterName, 'Use a random User-Agent') [CompletionResult]::new('-r', 'r', [CompletionResultType]::ParameterName, 'Follow redirects') [CompletionResult]::new('--redirects', 'redirects', [CompletionResultType]::ParameterName, 'Follow redirects') [CompletionResult]::new('-k', 'k', [CompletionResultType]::ParameterName, 'Disables TLS certificate validation') diff --git a/shell_completions/feroxbuster.bash b/shell_completions/feroxbuster.bash index 1a23ade..5e1016e 100644 --- a/shell_completions/feroxbuster.bash +++ b/shell_completions/feroxbuster.bash @@ -20,7 +20,7 @@ _feroxbuster() { case "${cmd}" in feroxbuster) - opts=" -v -q -D -r -k -n -f -e -h -V -w -u -t -d -T -p -P -R -s -o -a -x -H -Q -S -X -W -N -C -L --verbosity --silent --quiet --auto-tune --auto-bail --json --dont-filter --redirects --insecure --no-recursion --add-slash --stdin --extract-links --help --version --wordlist --url --threads --depth --timeout --proxy --replay-proxy --replay-codes --status-codes --output --resume-from --debug-log --user-agent --extensions --dont-scan --headers --query --filter-size --filter-regex --filter-words --filter-lines --filter-status --filter-similar-to --scan-limit --parallel --rate-limit --time-limit " + opts=" -v -q -D -A -r -k -n -f -e -h -V -w -u -t -d -T -p -P -R -s -o -a -x -H -Q -S -X -W -N -C -L --verbosity --silent --quiet --auto-tune --auto-bail --json --dont-filter --random-agent --redirects --insecure --no-recursion --add-slash --stdin --extract-links --help --version --wordlist --url --threads --depth --timeout --proxy --replay-proxy --replay-codes --status-codes --output --resume-from --debug-log --user-agent --extensions --dont-scan --headers --query --filter-size --filter-regex --filter-words --filter-lines --filter-status --filter-similar-to --scan-limit --parallel --rate-limit --time-limit " if [[ ${cur} == -* || ${COMP_CWORD} -eq 1 ]] ; then COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) return 0 diff --git a/shell_completions/feroxbuster.fish b/shell_completions/feroxbuster.fish index 382bca2..146f9b5 100644 --- a/shell_completions/feroxbuster.fish +++ b/shell_completions/feroxbuster.fish @@ -32,6 +32,7 @@ complete -c feroxbuster -n "__fish_use_subcommand" -l auto-tune -d 'Automaticall complete -c feroxbuster -n "__fish_use_subcommand" -l auto-bail -d 'Automatically stop scanning when an excessive amount of errors are encountered' complete -c feroxbuster -n "__fish_use_subcommand" -l json -d 'Emit JSON logs to --output and --debug-log instead of normal text' complete -c feroxbuster -n "__fish_use_subcommand" -s D -l dont-filter -d 'Don\'t auto-filter wildcard responses' +complete -c feroxbuster -n "__fish_use_subcommand" -s A -l random-agent -d 'Use a random User-Agent' complete -c feroxbuster -n "__fish_use_subcommand" -s r -l redirects -d 'Follow redirects' complete -c feroxbuster -n "__fish_use_subcommand" -s k -l insecure -d 'Disables TLS certificate validation' complete -c feroxbuster -n "__fish_use_subcommand" -s n -l no-recursion -d 'Do not scan recursively' diff --git a/src/parser.rs b/src/parser.rs index d241a73..e6b6ede 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -192,6 +192,15 @@ pub fn initialize() -> App<'static, 'static> { "Sets the User-Agent (default: feroxbuster/VERSION)" ), ) + .arg( + Arg::with_name("random_agent") + .short("A") + .long("random-agent") + .takes_value(false) + .help( + "Use a random User-Agent" + ), + ) .arg( Arg::with_name("redirects") .short("r") From 5235208aa8fd1be53a78d4d8aad3c7b65c953f73 Mon Sep 17 00:00:00 2001 From: epi Date: Thu, 23 Sep 2021 21:22:52 -0500 Subject: [PATCH 06/19] dynamically generate user agents at build time for inclusion into lib.rs --- Cargo.lock | 1 + Cargo.toml | 1 + build.rs | 44 +++++++++++++++++++++++++++++++++++++++++++- src/lib.rs | 3 +++ src/main.rs | 3 +++ 5 files changed, 51 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 596dc0c..dc46f48 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -615,6 +615,7 @@ dependencies = [ "log", "openssl", "predicates", + "rand", "regex", "reqwest", "rlimit", diff --git a/Cargo.toml b/Cargo.toml index 0eddae1..22ccb07 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ clap = "2.33" regex = "1" lazy_static = "1.4" dirs = "4.0" +rand = "0.8" [dependencies] futures = { version = "0.3"} diff --git a/build.rs b/build.rs index 591a124..d012407 100644 --- a/build.rs +++ b/build.rs @@ -7,10 +7,38 @@ use clap::Shell; include!("src/parser.rs"); + +/// this code is taken from +/// https://play.rust-lang.org/?version=stable&mode=debug&edition=2018&gist=4734c208edf7da625588fa16b5e2ed93 +/// which was linked in the random user agent issue discussion +/// +/// ** I'm not advocating for the use of this code, just using it for demonstration purposes ** +use rand::prelude::*; +use std::fmt::Write as fmt_Write; +use std::fs::write; +const WINDOWS_VERSION: [&str; 11] = [ + "3.1", "3.5", "4.0", "5.0", "5.1", "5.2", "6.0", "6.1", "6.2", "6.3", "10.0", +]; +fn generate_randomua() -> String { + let mut rng = thread_rng(); + let gecko_version = format!("{:.1}", rng.gen_range(40.0..60.0)); + let platform = if (rng.gen::() % 2) == 0 { + let version = WINDOWS_VERSION.choose(&mut rng).unwrap(); + format!("(Windows NT {}; rv:{}) Gecko/20100101 Firefox/{}", + version, gecko_version, gecko_version) + } else { + let version = format!("10.{}", rng.gen_range(0..19)); + format!("(Macintosh; Intel Mac OS X {}; rv:{}) Gecko/20100101 Firefox/{}", + version, gecko_version, gecko_version) + }; + format!("Mozilla/5.0 {}", platform) +} + + fn main() { println!("cargo:rerun-if-env-changed=src/parser.rs"); - if std::env::var("DOCS_RS").is_ok() { + if env::var("DOCS_RS").is_ok() { return; // only build when we're not generating docs } @@ -83,4 +111,18 @@ fn main() { eprintln!("Couldn't copy example config into config directory"); } } + + let user_agents = (0..20).map(|_| generate_randomua()).collect::>(); + + let mut definition = String::from("pub static USER_AGENTS:[&'static str; "); + + definition.push_str(user_agents.len().to_string().as_str()); + definition.push_str("] = ["); + for user_agent in user_agents { + definition.push_str(&format!("\"{}\",", user_agent)); + } + definition.push_str("];"); + + let out_file = format!("{}/user-agents.rs", env::var("OUT_DIR").unwrap()); + write(out_file, definition); } diff --git a/src/lib.rs b/src/lib.rs index c187471..02e50e4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -92,6 +92,9 @@ pub const DEFAULT_STATUS_CODES: [StatusCode; 10] = [ /// Expected location is in the same directory as the feroxbuster binary. pub const DEFAULT_CONFIG_NAME: &str = "ferox-config.toml"; +// includes user agents generated at build time +include!(concat!(env!("OUT_DIR"), "/user-agents.rs")); + #[cfg(test)] mod tests { use super::*; diff --git a/src/main.rs b/src/main.rs index 3fb96ea..1f855f5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -17,6 +17,7 @@ use tokio::{ use tokio_util::codec::{FramedRead, LinesCodec}; use feroxbuster::{ + USER_AGENTS, banner::{Banner, UPDATE_URL}, config::{Configuration, OutputLevel}, event_handlers::{ @@ -175,6 +176,8 @@ async fn wrapped_main(config: Arc) -> Result<()> { PROGRESS_BAR.join().unwrap(); }); + PROGRESS_PRINTER.println(USER_AGENTS[0]); + // spawn all event handlers, expect back a JoinHandle and a *Handle to the specific event let (stats_task, stats_handle) = StatsHandler::initialize(config.clone()); let (filters_task, filters_handle) = FiltersHandler::initialize(); From efd706cb9ba34385f5fde276241ee319602c3eae Mon Sep 17 00:00:00 2001 From: Daniel Saxton Date: Sun, 26 Sep 2021 10:09:52 -0500 Subject: [PATCH 07/19] cargo fmt --- build.rs | 18 +++++++++++------- src/main.rs | 2 +- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/build.rs b/build.rs index d012407..7f3786e 100644 --- a/build.rs +++ b/build.rs @@ -7,7 +7,6 @@ use clap::Shell; include!("src/parser.rs"); - /// this code is taken from /// https://play.rust-lang.org/?version=stable&mode=debug&edition=2018&gist=4734c208edf7da625588fa16b5e2ed93 /// which was linked in the random user agent issue discussion @@ -24,17 +23,20 @@ fn generate_randomua() -> String { let gecko_version = format!("{:.1}", rng.gen_range(40.0..60.0)); let platform = if (rng.gen::() % 2) == 0 { let version = WINDOWS_VERSION.choose(&mut rng).unwrap(); - format!("(Windows NT {}; rv:{}) Gecko/20100101 Firefox/{}", - version, gecko_version, gecko_version) + format!( + "(Windows NT {}; rv:{}) Gecko/20100101 Firefox/{}", + version, gecko_version, gecko_version + ) } else { let version = format!("10.{}", rng.gen_range(0..19)); - format!("(Macintosh; Intel Mac OS X {}; rv:{}) Gecko/20100101 Firefox/{}", - version, gecko_version, gecko_version) + format!( + "(Macintosh; Intel Mac OS X {}; rv:{}) Gecko/20100101 Firefox/{}", + version, gecko_version, gecko_version + ) }; format!("Mozilla/5.0 {}", platform) } - fn main() { println!("cargo:rerun-if-env-changed=src/parser.rs"); @@ -112,7 +114,9 @@ fn main() { } } - let user_agents = (0..20).map(|_| generate_randomua()).collect::>(); + let user_agents = (0..20) + .map(|_| generate_randomua()) + .collect::>(); let mut definition = String::from("pub static USER_AGENTS:[&'static str; "); diff --git a/src/main.rs b/src/main.rs index 1f855f5..56fa8a1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -17,7 +17,6 @@ use tokio::{ use tokio_util::codec::{FramedRead, LinesCodec}; use feroxbuster::{ - USER_AGENTS, banner::{Banner, UPDATE_URL}, config::{Configuration, OutputLevel}, event_handlers::{ @@ -30,6 +29,7 @@ use feroxbuster::{ scan_manager::{self}, scanner, utils::{fmt_err, slugify_filename}, + USER_AGENTS, }; #[cfg(not(target_os = "windows"))] use feroxbuster::{utils::set_open_file_limit, DEFAULT_OPEN_FILE_LIMIT}; From c5e59b70f7813659df836f13321dde097c0c6203 Mon Sep 17 00:00:00 2001 From: Daniel Saxton Date: Tue, 28 Sep 2021 15:44:11 -0500 Subject: [PATCH 08/19] Hard code --- build.rs | 48 +----------------------------------------------- src/lib.rs | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 50 deletions(-) diff --git a/build.rs b/build.rs index 7f3786e..591a124 100644 --- a/build.rs +++ b/build.rs @@ -7,40 +7,10 @@ use clap::Shell; include!("src/parser.rs"); -/// this code is taken from -/// https://play.rust-lang.org/?version=stable&mode=debug&edition=2018&gist=4734c208edf7da625588fa16b5e2ed93 -/// which was linked in the random user agent issue discussion -/// -/// ** I'm not advocating for the use of this code, just using it for demonstration purposes ** -use rand::prelude::*; -use std::fmt::Write as fmt_Write; -use std::fs::write; -const WINDOWS_VERSION: [&str; 11] = [ - "3.1", "3.5", "4.0", "5.0", "5.1", "5.2", "6.0", "6.1", "6.2", "6.3", "10.0", -]; -fn generate_randomua() -> String { - let mut rng = thread_rng(); - let gecko_version = format!("{:.1}", rng.gen_range(40.0..60.0)); - let platform = if (rng.gen::() % 2) == 0 { - let version = WINDOWS_VERSION.choose(&mut rng).unwrap(); - format!( - "(Windows NT {}; rv:{}) Gecko/20100101 Firefox/{}", - version, gecko_version, gecko_version - ) - } else { - let version = format!("10.{}", rng.gen_range(0..19)); - format!( - "(Macintosh; Intel Mac OS X {}; rv:{}) Gecko/20100101 Firefox/{}", - version, gecko_version, gecko_version - ) - }; - format!("Mozilla/5.0 {}", platform) -} - fn main() { println!("cargo:rerun-if-env-changed=src/parser.rs"); - if env::var("DOCS_RS").is_ok() { + if std::env::var("DOCS_RS").is_ok() { return; // only build when we're not generating docs } @@ -113,20 +83,4 @@ fn main() { eprintln!("Couldn't copy example config into config directory"); } } - - let user_agents = (0..20) - .map(|_| generate_randomua()) - .collect::>(); - - let mut definition = String::from("pub static USER_AGENTS:[&'static str; "); - - definition.push_str(user_agents.len().to_string().as_str()); - definition.push_str("] = ["); - for user_agent in user_agents { - definition.push_str(&format!("\"{}\",", user_agent)); - } - definition.push_str("];"); - - let out_file = format!("{}/user-agents.rs", env::var("OUT_DIR").unwrap()); - write(out_file, definition); } diff --git a/src/lib.rs b/src/lib.rs index 02e50e4..7d21910 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -91,9 +91,20 @@ pub const DEFAULT_STATUS_CODES: [StatusCode; 10] = [ /// /// Expected location is in the same directory as the feroxbuster binary. pub const DEFAULT_CONFIG_NAME: &str = "ferox-config.toml"; - -// includes user agents generated at build time -include!(concat!(env!("OUT_DIR"), "/user-agents.rs")); +pub const USER_AGENTS: [&str; 12] = [ + "Mozilla/5.0 (Linux; Android 8.0.0; SM-G960F Build/R16NW) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.84 Mobile Safari/537.36", + "Mozilla/5.0 (iPhone; CPU iPhone OS 12_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1", + "Mozilla/5.0 (Windows Phone 10.0; Android 6.0.1; Microsoft; RM-1152) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Mobile Safari/537.36 Edge/15.15254", + "Mozilla/5.0 (Linux; Android 7.0; Pixel C Build/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/52.0.2743.98 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246", + "Mozilla/5.0 (X11; CrOS x86_64 8172.45.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.64 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/601.3.9 (KHTML, like Gecko) Version/9.0.2 Safari/601.3.9", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.111 Safari/537.36", + "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 Firefox/15.0.1", + "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", + "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)", + "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)", +]; #[cfg(test)] mod tests { From 87aaa84f1ef7817582670eac8f53fef84149a7dd Mon Sep 17 00:00:00 2001 From: epi Date: Tue, 12 Oct 2021 18:21:05 -0500 Subject: [PATCH 09/19] added user-agent logic --- src/event_handlers/outputs.rs | 1 + src/extractor/container.rs | 1 + src/utils.rs | 26 ++++++++++++++++++++++++-- 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/src/event_handlers/outputs.rs b/src/event_handlers/outputs.rs index d94cb90..7d70b7e 100644 --- a/src/event_handlers/outputs.rs +++ b/src/event_handlers/outputs.rs @@ -215,6 +215,7 @@ impl TermOutHandler { self.config.replay_client.as_ref().unwrap(), resp.url(), self.config.output_level, + &self.config, tx_stats.clone(), ) .await diff --git a/src/extractor/container.rs b/src/extractor/container.rs index d86ab60..4e84ea4 100644 --- a/src/extractor/container.rs +++ b/src/extractor/container.rs @@ -402,6 +402,7 @@ impl<'a> Extractor<'a> { &client, &url, self.handles.config.output_level, + &self.handles.config, self.handles.stats.tx.clone(), ) .await?; diff --git a/src/utils.rs b/src/utils.rs index a58662a..85ee145 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -5,7 +5,9 @@ use reqwest::{Client, Response, StatusCode, Url}; #[cfg(not(target_os = "windows"))] use rlimit::{getrlimit, setrlimit, Resource}; use std::{ + thread_local, fs, + cell::RefCell, io::{self, BufWriter, Write}, sync::Arc, time::Duration, @@ -21,9 +23,17 @@ use crate::{ }, progress::PROGRESS_PRINTER, send_command, + USER_AGENTS, statistics::StatError::{Connection, Other, Redirection, Request, Timeout}, traits::FeroxSerialize, }; +use crate::config::Configuration; + +thread_local! { + /// simple counter for grabbing 'random' user agents + static USER_AGENT_CTR: RefCell = RefCell::new(0); +} + /// Given the path to a file, open the file in append mode (create it if it doesn't exist) and /// return a reference to the buffered file @@ -94,7 +104,7 @@ pub async fn logged_request(url: &Url, handles: Arc) -> Result, ) -> Result { log::trace!( @@ -130,7 +141,18 @@ pub async fn make_request( tx_stats ); - match client.get(url.to_owned()).send().await { + let mut request = client.get(url.to_owned()); + + if config.random_agent { + let user_agent = USER_AGENT_CTR.with(|ua_ctr| { + let mut inner = ua_ctr.borrow_mut(); + *inner += 1; + USER_AGENTS[*inner % USER_AGENTS.len()] + }); + request = request.header("User-Agent", user_agent); + } + + match request.send().await { Err(e) => { log::trace!("exit: make_request -> {}", e); From 39b2da9735be646cfe5b31e93cb1fcc6398842f6 Mon Sep 17 00:00:00 2001 From: epi Date: Tue, 12 Oct 2021 20:22:01 -0500 Subject: [PATCH 10/19] changed counter type --- src/utils.rs | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/utils.rs b/src/utils.rs index 85ee145..2adf12f 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -5,9 +5,7 @@ use reqwest::{Client, Response, StatusCode, Url}; #[cfg(not(target_os = "windows"))] use rlimit::{getrlimit, setrlimit, Resource}; use std::{ - thread_local, fs, - cell::RefCell, io::{self, BufWriter, Write}, sync::Arc, time::Duration, @@ -29,10 +27,9 @@ use crate::{ }; use crate::config::Configuration; -thread_local! { - /// simple counter for grabbing 'random' user agents - static USER_AGENT_CTR: RefCell = RefCell::new(0); -} +/// simple counter for grabbing 'random' user agents +static mut USER_AGENT_CTR: usize = 0; + /// Given the path to a file, open the file in append mode (create it if it doesn't exist) and @@ -144,11 +141,13 @@ pub async fn make_request( let mut request = client.get(url.to_owned()); if config.random_agent { - let user_agent = USER_AGENT_CTR.with(|ua_ctr| { - let mut inner = ua_ctr.borrow_mut(); - *inner += 1; - USER_AGENTS[*inner % USER_AGENTS.len()] - }); + let index = unsafe { + USER_AGENT_CTR += 1; + USER_AGENT_CTR % USER_AGENTS.len() + }; + + let user_agent = USER_AGENTS[index]; + request = request.header("User-Agent", user_agent); } From 2a4a150598d4dd5f46821886c216b7c7edf37423 Mon Sep 17 00:00:00 2001 From: epi Date: Tue, 12 Oct 2021 20:24:05 -0500 Subject: [PATCH 11/19] changed counter type --- src/utils.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/utils.rs b/src/utils.rs index 2adf12f..6ab38e9 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -13,6 +13,7 @@ use std::{ }; use tokio::sync::mpsc::UnboundedSender; +use crate::config::Configuration; use crate::{ config::OutputLevel, event_handlers::{ @@ -21,17 +22,14 @@ use crate::{ }, progress::PROGRESS_PRINTER, send_command, - USER_AGENTS, statistics::StatError::{Connection, Other, Redirection, Request, Timeout}, traits::FeroxSerialize, + USER_AGENTS, }; -use crate::config::Configuration; /// simple counter for grabbing 'random' user agents static mut USER_AGENT_CTR: usize = 0; - - /// Given the path to a file, open the file in append mode (create it if it doesn't exist) and /// return a reference to the buffered file pub fn open_file(filename: &str) -> Result> { From 521d341e3633c74b57e7e510136ce14be445c423 Mon Sep 17 00:00:00 2001 From: Daniel Saxton Date: Wed, 13 Oct 2021 16:03:32 -0500 Subject: [PATCH 12/19] Remove unused and update make_request call --- Cargo.lock | 1 - Cargo.toml | 1 - src/extractor/tests.rs | 15 +++++++++++---- src/main.rs | 3 --- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dc46f48..596dc0c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -615,7 +615,6 @@ dependencies = [ "log", "openssl", "predicates", - "rand", "regex", "reqwest", "rlimit", diff --git a/Cargo.toml b/Cargo.toml index 22ccb07..0eddae1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,6 @@ clap = "2.33" regex = "1" lazy_static = "1.4" dirs = "4.0" -rand = "0.8" [dependencies] futures = { version = "0.3"} diff --git a/src/extractor/tests.rs b/src/extractor/tests.rs index 477eb01..94b4d2f 100644 --- a/src/extractor/tests.rs +++ b/src/extractor/tests.rs @@ -211,16 +211,23 @@ async fn extractor_get_links_with_absolute_url_that_differs_from_target_domain() let mock = srv.mock(|when, then| { when.method(GET).path("/some-path"); then.status(200).body( - "\"http://defintely.not.a.thing.probably.com/homepage/assets/img/icons/handshake.svg\"", + "\"http://definitely.not.a.thing.probably.com/homepage/assets/img/icons/handshake.svg\"", ); }); let client = Client::new(); let url = Url::parse(&srv.url("/some-path")).unwrap(); + let config = Configuration::new().unwrap(); - let response = make_request(&client, &url, OutputLevel::Default, tx_stats.clone()) - .await - .unwrap(); + let response = make_request( + &client, + &url, + OutputLevel::Default, + &config, + tx_stats.clone(), + ) + .await + .unwrap(); let (handles, _rx) = Handles::for_testing(None, None); let handles = Arc::new(handles); diff --git a/src/main.rs b/src/main.rs index 56fa8a1..3fb96ea 100644 --- a/src/main.rs +++ b/src/main.rs @@ -29,7 +29,6 @@ use feroxbuster::{ scan_manager::{self}, scanner, utils::{fmt_err, slugify_filename}, - USER_AGENTS, }; #[cfg(not(target_os = "windows"))] use feroxbuster::{utils::set_open_file_limit, DEFAULT_OPEN_FILE_LIMIT}; @@ -176,8 +175,6 @@ async fn wrapped_main(config: Arc) -> Result<()> { PROGRESS_BAR.join().unwrap(); }); - PROGRESS_PRINTER.println(USER_AGENTS[0]); - // spawn all event handlers, expect back a JoinHandle and a *Handle to the specific event let (stats_task, stats_handle) = StatsHandler::initialize(config.clone()); let (filters_task, filters_handle) = FiltersHandler::initialize(); From 7b3201f2f8826cce19d74f071dad18901b90c70d Mon Sep 17 00:00:00 2001 From: Daniel Saxton Date: Wed, 13 Oct 2021 16:14:42 -0500 Subject: [PATCH 13/19] Fix test --- ferox-stdin-1634159598.state | 1 + src/scan_manager/tests.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 ferox-stdin-1634159598.state diff --git a/ferox-stdin-1634159598.state b/ferox-stdin-1634159598.state new file mode 100644 index 0000000..b4e824a --- /dev/null +++ b/ferox-stdin-1634159598.state @@ -0,0 +1 @@ +{"scans":[{"id":"812b1971bcde441f8b5b7c21c72e7067","url":"http://127.0.0.1:46075/","scan_type":"Directory","status":"Running","num_requests":102774},{"id":"c2e3579de1cb4f67bae2c959ea3dfc61","url":"http://127.0.0.1:32907/","scan_type":"Directory","status":"Running","num_requests":102774}],"config":{"type":"configuration","wordlist":"/tmp/.tmpusrADA/wordlist","config":"/home/dsaxton/.config/feroxbuster/ferox-config.toml","proxy":"","replay_proxy":"","target_url":"","status_codes":[200,204,301,302,307,308,401,403,405,500],"replay_codes":[200,204,301,302,307,308,401,403,405,500],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"silent":false,"quiet":false,"auto_bail":false,"auto_tune":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/2.4.0","random_agent":false,"redirects":false,"insecure":false,"extensions":[],"headers":{},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":true,"depth":4,"scan_limit":0,"parallel":0,"rate_limit":0,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false,"resumed":false,"resume_from":"","save_state":true,"time_limit":"5s","filter_similar":[],"url_denylist":[]},"responses":[],"statistics":{"type":"statistics","timeouts":0,"requests":7249,"expected_per_scan":102774,"total_expected":205548,"errors":0,"successes":0,"redirects":0,"client_errors":7249,"server_errors":0,"total_scans":2,"initial_targets":0,"links_extracted":0,"status_200s":0,"status_301s":0,"status_302s":0,"status_401s":0,"status_403s":1,"status_429s":0,"status_500s":0,"status_503s":0,"status_504s":0,"status_508s":0,"wildcards_filtered":0,"responses_filtered":0,"resources_discovered":0,"url_format_errors":0,"redirection_errors":0,"connection_errors":0,"request_errors":0,"directory_scan_times":[],"total_runtime":[0.0]}} \ No newline at end of file diff --git a/src/scan_manager/tests.rs b/src/scan_manager/tests.rs index 40bfd8e..bbdab7b 100644 --- a/src/scan_manager/tests.rs +++ b/src/scan_manager/tests.rs @@ -379,7 +379,7 @@ fn feroxstates_feroxserialize_implementation() { let json_state = ferox_state.as_json().unwrap(); let expected = format!( - r#"{{"scans":[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","status":"NotStarted","num_requests":0}}],"config":{{"type":"configuration","wordlist":"/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt","config":"","proxy":"","replay_proxy":"","target_url":"","status_codes":[200,204,301,302,307,308,401,403,405,500],"replay_codes":[200,204,301,302,307,308,401,403,405,500],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"silent":false,"quiet":false,"auto_bail":false,"auto_tune":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/{}","redirects":false,"insecure":false,"extensions":[],"headers":{{}},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":false,"depth":4,"scan_limit":0,"parallel":0,"rate_limit":0,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false,"resumed":false,"resume_from":"","save_state":false,"time_limit":"","filter_similar":[],"url_denylist":[]}},"responses":[{{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{{"server":"nginx/1.16.1"}}}}]"#, + r#"{{"scans":[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","status":"NotStarted","num_requests":0}}],"config":{{"type":"configuration","wordlist":"/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt","config":"","proxy":"","replay_proxy":"","target_url":"","status_codes":[200,204,301,302,307,308,401,403,405,500],"replay_codes":[200,204,301,302,307,308,401,403,405,500],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"silent":false,"quiet":false,"auto_bail":false,"auto_tune":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/{}","random_agent":false,"redirects":false,"insecure":false,"extensions":[],"headers":{{}},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":false,"depth":4,"scan_limit":0,"parallel":0,"rate_limit":0,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false,"resumed":false,"resume_from":"","save_state":false,"time_limit":"","filter_similar":[],"url_denylist":[]}},"responses":[{{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{{"server":"nginx/1.16.1"}}}}]"#, saved_id, VERSION ); println!("{}\n{}", expected, json_state); From 0a3130934c4cf2035e3bd71940525ac7ad170b53 Mon Sep 17 00:00:00 2001 From: Daniel Saxton Date: Wed, 13 Oct 2021 16:30:20 -0500 Subject: [PATCH 14/19] Ignore --- .gitignore | 1 + ferox-stdin-1634159598.state | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 ferox-stdin-1634159598.state diff --git a/.gitignore b/.gitignore index a6b9e1e..31567f9 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ lcov_cobertura.py # state file created during tests ferox-http* +ferox-stdin* # python stuff cuz reasons Pipfile* diff --git a/ferox-stdin-1634159598.state b/ferox-stdin-1634159598.state deleted file mode 100644 index b4e824a..0000000 --- a/ferox-stdin-1634159598.state +++ /dev/null @@ -1 +0,0 @@ -{"scans":[{"id":"812b1971bcde441f8b5b7c21c72e7067","url":"http://127.0.0.1:46075/","scan_type":"Directory","status":"Running","num_requests":102774},{"id":"c2e3579de1cb4f67bae2c959ea3dfc61","url":"http://127.0.0.1:32907/","scan_type":"Directory","status":"Running","num_requests":102774}],"config":{"type":"configuration","wordlist":"/tmp/.tmpusrADA/wordlist","config":"/home/dsaxton/.config/feroxbuster/ferox-config.toml","proxy":"","replay_proxy":"","target_url":"","status_codes":[200,204,301,302,307,308,401,403,405,500],"replay_codes":[200,204,301,302,307,308,401,403,405,500],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"silent":false,"quiet":false,"auto_bail":false,"auto_tune":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/2.4.0","random_agent":false,"redirects":false,"insecure":false,"extensions":[],"headers":{},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":true,"depth":4,"scan_limit":0,"parallel":0,"rate_limit":0,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false,"resumed":false,"resume_from":"","save_state":true,"time_limit":"5s","filter_similar":[],"url_denylist":[]},"responses":[],"statistics":{"type":"statistics","timeouts":0,"requests":7249,"expected_per_scan":102774,"total_expected":205548,"errors":0,"successes":0,"redirects":0,"client_errors":7249,"server_errors":0,"total_scans":2,"initial_targets":0,"links_extracted":0,"status_200s":0,"status_301s":0,"status_302s":0,"status_401s":0,"status_403s":1,"status_429s":0,"status_500s":0,"status_503s":0,"status_504s":0,"status_508s":0,"wildcards_filtered":0,"responses_filtered":0,"resources_discovered":0,"url_format_errors":0,"redirection_errors":0,"connection_errors":0,"request_errors":0,"directory_scan_times":[],"total_runtime":[0.0]}} \ No newline at end of file From b8bfbb09f3ba5ec3913214c52c585e5d6f0f24e2 Mon Sep 17 00:00:00 2001 From: Daniel Saxton Date: Wed, 13 Oct 2021 16:32:39 -0500 Subject: [PATCH 15/19] Oops --- .gitignore | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 31567f9..cfcc7f9 100644 --- a/.gitignore +++ b/.gitignore @@ -23,8 +23,7 @@ lcov_cobertura.py .dockerignore # state file created during tests -ferox-http* -ferox-stdin* +ferox-*.state # python stuff cuz reasons Pipfile* From 19cd5c910a1acc446b7d7fa5bf65690ea3da3e0c Mon Sep 17 00:00:00 2001 From: Daniel Saxton Date: Wed, 13 Oct 2021 20:08:28 -0500 Subject: [PATCH 16/19] Try a test refactor --- src/scan_manager/tests.rs | 79 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 73 insertions(+), 6 deletions(-) diff --git a/src/scan_manager/tests.rs b/src/scan_manager/tests.rs index bbdab7b..93fa9c0 100644 --- a/src/scan_manager/tests.rs +++ b/src/scan_manager/tests.rs @@ -378,12 +378,79 @@ fn feroxstates_feroxserialize_implementation() { assert!(expected_strs.eval(&ferox_state.as_str())); let json_state = ferox_state.as_json().unwrap(); - let expected = format!( - r#"{{"scans":[{{"id":"{}","url":"https://spiritanimal.com","scan_type":"Directory","status":"NotStarted","num_requests":0}}],"config":{{"type":"configuration","wordlist":"/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt","config":"","proxy":"","replay_proxy":"","target_url":"","status_codes":[200,204,301,302,307,308,401,403,405,500],"replay_codes":[200,204,301,302,307,308,401,403,405,500],"filter_status":[],"threads":50,"timeout":7,"verbosity":0,"silent":false,"quiet":false,"auto_bail":false,"auto_tune":false,"json":false,"output":"","debug_log":"","user_agent":"feroxbuster/{}","random_agent":false,"redirects":false,"insecure":false,"extensions":[],"headers":{{}},"queries":[],"no_recursion":false,"extract_links":false,"add_slash":false,"stdin":false,"depth":4,"scan_limit":0,"parallel":0,"rate_limit":0,"filter_size":[],"filter_line_count":[],"filter_word_count":[],"filter_regex":[],"dont_filter":false,"resumed":false,"resume_from":"","save_state":false,"time_limit":"","filter_similar":[],"url_denylist":[]}},"responses":[{{"type":"response","url":"https://nerdcore.com/css","path":"/css","wildcard":true,"status":301,"content_length":173,"line_count":10,"word_count":16,"headers":{{"server":"nginx/1.16.1"}}}}]"#, - saved_id, VERSION - ); - println!("{}\n{}", expected, json_state); - assert!(predicates::str::contains(expected).eval(&json_state)); + for expected in [ + r#""scans""#, + &format!(r#""id":"{}""#, saved_id), + r#""url":"https://spiritanimal.com""#, + r#""scan_type":"Directory""#, + r#""status":"NotStarted""#, + r#""num_requests":0"#, + r#""config""#, + r#""type":"configuration""#, + r#""wordlist":"/usr/share/seclists/Discovery/Web-Content/raft-medium-directories.txt""#, + r#""config""#, + r#""proxy":"""#, + r#""replay_proxy":"""#, + r#""target_url":"""#, + r#""status_codes":[200,204,301,302,307,308,401,403,405,500]"#, + r#""replay_codes":[200,204,301,302,307,308,401,403,405,500]"#, + r#""filter_status":[]"#, + r#""threads":50"#, + r#""timeout":7"#, + r#""verbosity":0"#, + r#""silent":false"#, + r#""quiet":false"#, + r#""auto_bail":false"#, + r#""auto_tune":false"#, + r#""json":false"#, + r#""output":"""#, + r#""debug_log":"""#, + &format!(r#""user_agent":"feroxbuster/{}""#, VERSION), + r#""random_agent":false"#, + r#""redirects":false"#, + r#""insecure":false"#, + r#""extensions":[]"#, + r#""headers""#, + r#""queries":[]"#, + r#""no_recursion":false"#, + r#""extract_links":false"#, + r#""add_slash":false"#, + r#""stdin":false"#, + r#""depth":4"#, + r#""scan_limit":0"#, + r#""parallel":0"#, + r#""rate_limit":0"#, + r#""filter_size":[]"#, + r#""filter_line_count":[]"#, + r#""filter_word_count":[]"#, + r#""filter_regex":[]"#, + r#""dont_filter":false"#, + r#""resumed":false"#, + r#""resume_from":"""#, + r#""save_state":false"#, + r#""time_limit":"""#, + r#""filter_similar":[]"#, + r#""url_denylist":["#, + r#""responses""#, + r#""type":"response""#, + r#""url":"https://nerdcore.com/css""#, + r#""path":"/css""#, + r#""wildcard":true"#, + r#""status":301"#, + r#""content_length":173"#, + r#""line_count":10"#, + r#""word_count":16"#, + r#""headers""#, + r#""server":"nginx/1.16.1"#, + ] + .iter() + { + assert!( + predicates::str::contains(*expected).eval(&json_state), + "{}", + expected + ) + } } #[should_panic] From 3536587260358d6cabf8f1a68edaaa283434f321 Mon Sep 17 00:00:00 2001 From: Daniel Saxton Date: Wed, 13 Oct 2021 20:11:02 -0500 Subject: [PATCH 17/19] Update banner --- src/banner/container.rs | 2 +- tests/test_banner.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/banner/container.rs b/src/banner/container.rs index 92d35cf..9d8ec12 100644 --- a/src/banner/container.rs +++ b/src/banner/container.rs @@ -279,7 +279,7 @@ impl Banner { let wordlist = BannerEntry::new("📖", "Wordlist", &config.wordlist); let timeout = BannerEntry::new("💥", "Timeout (secs)", &config.timeout.to_string()); let user_agent = BannerEntry::new("🦡", "User-Agent", &config.user_agent); - let random_agent = BannerEntry::new("🦡", "User-Agent", "(Random)"); + let random_agent = BannerEntry::new("🦡", "User-Agent", "Random"); let extract_links = BannerEntry::new("🔎", "Extract Links", &config.extract_links.to_string()); let json = BannerEntry::new("🧔", "JSON Output", &config.json.to_string()); diff --git a/tests/test_banner.rs b/tests/test_banner.rs index 1f4b373..bcfbc56 100644 --- a/tests/test_banner.rs +++ b/tests/test_banner.rs @@ -163,7 +163,7 @@ fn banner_prints_random_agent() { .and(predicate::str::contains("Status Codes")) .and(predicate::str::contains("Timeout (secs)")) .and(predicate::str::contains("User-Agent")) - .and(predicate::str::contains("(Random)")) + .and(predicate::str::contains("Random")) .and(predicate::str::contains("─┴─")), ); } From 1e3cd3a2091917565a77fe3b114ed575b9aba9df Mon Sep 17 00:00:00 2001 From: Daniel Saxton Date: Wed, 13 Oct 2021 20:12:43 -0500 Subject: [PATCH 18/19] Doc --- src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib.rs b/src/lib.rs index 7d21910..fc8ead6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -91,6 +91,7 @@ pub const DEFAULT_STATUS_CODES: [StatusCode; 10] = [ /// /// Expected location is in the same directory as the feroxbuster binary. pub const DEFAULT_CONFIG_NAME: &str = "ferox-config.toml"; +/// User agents to select from when random agent is being used pub const USER_AGENTS: [&str; 12] = [ "Mozilla/5.0 (Linux; Android 8.0.0; SM-G960F Build/R16NW) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.84 Mobile Safari/537.36", "Mozilla/5.0 (iPhone; CPU iPhone OS 12_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1", From 9c50038a25245236792e71375f65dc14b20977f9 Mon Sep 17 00:00:00 2001 From: Daniel Saxton Date: Thu, 14 Oct 2021 15:15:32 -0500 Subject: [PATCH 19/19] Nit --- src/scan_manager/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scan_manager/tests.rs b/src/scan_manager/tests.rs index 93fa9c0..613e94e 100644 --- a/src/scan_manager/tests.rs +++ b/src/scan_manager/tests.rs @@ -430,7 +430,7 @@ fn feroxstates_feroxserialize_implementation() { r#""save_state":false"#, r#""time_limit":"""#, r#""filter_similar":[]"#, - r#""url_denylist":["#, + r#""url_denylist":[]"#, r#""responses""#, r#""type":"response""#, r#""url":"https://nerdcore.com/css""#,